sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Geobuf.input (#3715)

Browse source 
* geobuf.input - initial implementation (WIP)

* geobuf.input - fix reading MultiPolygon's

* geobuf.input - don't store invalid bounding boxes in r-tree.

* Use `protozero` lib to read `geobuf` (Initial implementation, not complete!)

* geobuf.input - poring to `protozero`. (TODO: add support for `GeometryCollection`)

* add initial support for `GeometryCollection` + cleanup namespaces

* remove stderr + cleanup + update copyright

* fix GeometryCollection and MultiLineString decoding + add support for standalone Feature and Geometry

* don't use `reserve` as undelying data format doesn't allow for efficient `distance` implementation.

* add geobuf unit test + update test data

* update data
This commit is contained in:
Artem Pavlenko 2017-08-17 10:52:07 +01:00 committed by GitHub
parent add410e619
commit 0253653676
11 changed files with 1354 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitmodules vendored
View file

@ -14,3 +14,7 @@
path = deps/mapbox/geometry path = deps/mapbox/geometry
url = https://github.com/mapbox/geometry.hpp.git url = https://github.com/mapbox/geometry.hpp.git
branch = master branch = master
[submodule "deps/mapbox/protozero"]
path = deps/mapbox/protozero
url = https://github.com/mapbox/protozero.git
branch = master

4
SConstruct
View file

@ -1,6 +1,6 @@
# This file is part of Mapnik (c++ mapping toolkit) # This file is part of Mapnik (c++ mapping toolkit)
# #
# Copyright (C) 2015 Artem Pavlenko # Copyright (C) 2017 Artem Pavlenko
# #
# Mapnik is free software; you can redistribute it and/or # Mapnik is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
@ -116,6 +116,7 @@ PLUGINS = { # plugins with external dependencies
'csv': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'}, 'csv': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
'raster': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'}, 'raster': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
'geojson': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'}, 'geojson': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
'geobuf': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
'topojson':{'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'} 'topojson':{'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'}
} }
@ -1603,6 +1604,7 @@ if not preconfigured:
env.Prepend(LIBPATH = '#deps/agg') env.Prepend(LIBPATH = '#deps/agg')
env.Prepend(CPPPATH = '#deps/mapbox/variant/include') env.Prepend(CPPPATH = '#deps/mapbox/variant/include')
env.Prepend(CPPPATH = '#deps/mapbox/geometry/include') env.Prepend(CPPPATH = '#deps/mapbox/geometry/include')
env.Prepend(CPPPATH = '#deps/mapbox/protozero/include')
# prepend deps dir for auxillary headers # prepend deps dir for auxillary headers
env.Prepend(CPPPATH = '#deps') env.Prepend(CPPPATH = '#deps')

1
deps/mapbox/protozero vendored Submodule

@ -0,0 +1 @@
Subproject commit a1eb7327e853161e939a409a8d74a8302dd1ecc5

64
plugins/input/geobuf/build.py Normal file
View file

@ -0,0 +1,64 @@
#
# This file is part of Mapnik (c++ mapping toolkit)
#
# Copyright (C) 2017 Artem Pavlenko
#
# Mapnik is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
#
Import ('env')
Import ('plugin_base')
PLUGIN_NAME = 'geobuf'
plugin_env = plugin_base.Clone()
plugin_sources = Split(
"""
%(PLUGIN_NAME)s_datasource.cpp
%(PLUGIN_NAME)s_featureset.cpp
""" % locals()
)
# Link Library to Dependencies
libraries = []
libraries.append(env['ICU_LIB_NAME'])
libraries.append('boost_system%s' % env['BOOST_APPEND'])
libraries.append('mapnik-json')
if env['PLUGIN_LINKING'] == 'shared':
libraries.append(env['MAPNIK_NAME'])
TARGET = plugin_env.SharedLibrary('../%s' % PLUGIN_NAME,
SHLIBPREFIX='',
SHLIBSUFFIX='.input',
source=plugin_sources,
LIBS=libraries)
# if the plugin links to libmapnik ensure it is built first
Depends(TARGET, env.subst('../../../src/%s' % env['MAPNIK_LIB_NAME']))
if 'uninstall' not in COMMAND_LINE_TARGETS:
env.Install(env['MAPNIK_INPUT_PLUGINS_DEST'], TARGET)
env.Alias('install', env['MAPNIK_INPUT_PLUGINS_DEST'])
plugin_obj = {
'LIBS': libraries,
'SOURCES': plugin_sources,
}
Return('plugin_obj')

564
plugins/input/geobuf/geobuf.hpp Normal file
View file

@ -0,0 +1,564 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2017 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#ifndef MAPNIK_UTIL_GEOBUF_HPP
#define MAPNIK_UTIL_GEOBUF_HPP
#include <mapnik/debug.hpp>
#include <mapnik/value.hpp>
#include <mapnik/unicode.hpp>
#include <mapnik/feature.hpp>
#include <mapnik/feature_factory.hpp>
#include <mapnik/util/noncopyable.hpp>
#include <cmath>
#include <cassert>
#include <vector>
#include <boost/optional.hpp>
#include <protozero/pbf_reader.hpp>
namespace mapnik { namespace util {
enum geometry_type_e
{
Unknown = -1,
Point = 0,
MultiPoint = 1,
LineString = 2,
MultiLineString = 3,
Polygon = 4,
MultiPolygon = 5,
GeometryCollection = 6
};
namespace detail {
struct value_visitor
{
value_visitor(feature_impl & feature, transcoder const& tr, std::string const& name)
: feature_(feature),
tr_(tr),
name_(name) {}
void operator() (std::string const& val) // unicode
{
feature_.put_new(name_, tr_.transcode(val.c_str()));
}
template <typename T>
void operator() (T val)
{
feature_.put_new(name_, val);
}
feature_impl & feature_;
transcoder const& tr_;
std::string const& name_;
};
}
template <typename FeatureCallback>
struct geobuf : util::noncopyable
{
using value_type = util::variant<bool, int, double, std::string>;
unsigned dim = 2;
double precision = std::pow(10,6);
bool is_topo = false;
bool transformed = false;
std::size_t lengths = 0;
std::vector<std::string> keys_;
std::vector<value_type> values_;
protozero::pbf_reader reader_;
FeatureCallback & callback_;
context_ptr ctx_;
const std::unique_ptr<transcoder> tr_;
public:
//ctor
geobuf (char const* buf, std::size_t size, FeatureCallback & callback)
: reader_(buf, size),
callback_(callback),
ctx_(std::make_shared<context_type>()),
tr_(new transcoder("utf8")) {}
void read()
{
while (reader_.next())
{
switch (reader_.tag())
{
case 1: // keys
{
keys_.push_back(reader_.get_string());
break;
}
case 2:
{
dim = reader_.get_uint32();
break;
}
case 3:
{
precision = std::pow(10,reader_.get_uint32());
break;
}
case 4:
{
auto feature_collection = reader_.get_message();
read_feature_collection(feature_collection);
break;
}
case 5:
{
// standalone Feature
auto message = reader_.get_message();
read_feature(message);
break;
}
case 6:
{
// standalone Geometry
auto feature = feature_factory::create(ctx_,1);
auto message = reader_.get_message();
feature->set_geometry(std::move(read_geometry(message)));
callback_(feature);
break;
}
default:
MAPNIK_LOG_DEBUG(geobuf) << "Unsupported tag=" << reader_.tag();
reader_.skip();
break;
}
}
}
private:
double transform(std::int64_t input)
{
return (transformed) ? (static_cast<double>(input)) : (input/precision);
}
template <typename T>
void read_value(T & reader)
{
while (reader.next())
{
switch (reader.tag())
{
case 1:
{
values_.emplace_back(reader.get_string());
break;
}
case 2:
{
values_.emplace_back(reader.get_double());
break;
}
case 3:
{
values_.emplace_back(static_cast<int>(reader.get_uint32()));
break;
}
case 4:
{
values_.emplace_back(-static_cast<int>(reader.get_uint32()));
break;
}
case 5:
{
values_.emplace_back(reader.get_bool());
break;
}
case 6:
{
values_.emplace_back(reader.get_string()); // JSON value
break;
}
default:
break;
}
}
}
template <typename T, typename Feature>
void read_props(T & reader, Feature & feature)
{
auto pi = reader.get_packed_uint32();
for (auto it = pi.first; it != pi.second; ++it)
{
auto key_index = *it++;
auto value_index = *it;
assert(key_index < keys_.size());
assert(value_index < values_.size());
std::string const& name = keys_[key_index];
util::apply_visitor(detail::value_visitor(feature, *tr_, name), values_[value_index]);
}
values_.clear();
}
template <typename T>
void read_feature (T & reader)
{
auto feature = feature_factory::create(ctx_,1);
while (reader.next())
{
switch (reader.tag())
{
case 1:
{
auto message = reader.get_message();
auto geom = read_geometry(message);
feature->set_geometry(std::move(geom));
break;
}
case 11:
{
auto feature_id = reader.get_string();
break;
}
case 12:
{
feature->set_id(reader.get_sint64());
break;
}
case 13:
{
auto message = reader.get_message();
read_value(message);
break;
}
case 14:
{
// feature props
read_props(reader, *feature);
break;
}
case 15:
{
// generic props
read_props(reader, *feature);
break;
}
default:
MAPNIK_LOG_DEBUG(geobuf) << "Unsupported tag=" << reader.tag();
break;
}
}
callback_(feature);
}
template <typename T>
void read_feature_collection(T & reader)
{
while (reader.next())
{
switch (reader.tag())
{
case 1:
{
auto message = reader.get_message();
read_feature(message);
break;
}
case 13:
{
auto message = reader.get_message();
read_value(message);
break;
}
default:
{
reader.skip();
break;
}
}
}
}
template <typename T>
geometry::point<double> read_point(T & reader)
{
double x = 0.0;
double y = 0.0;
auto pi = reader.get_packed_sint64();
std::size_t count = 0;
for (auto it = pi.first ; it != pi.second; ++it, ++count)
{
if (count == 0) x = transform(*it);
else if (count == 1) y = transform(*it);
}
return geometry::point<double>(x, y);
}
template <typename T>
geometry::geometry<double> read_coords(T & reader, geometry_type_e type,
boost::optional<std::vector<std::uint32_t>> const& lengths)
{
geometry::geometry<double> geom = geometry::geometry_empty();
switch (type)
{
case Point:
{
geom = read_point(reader);
break;
}
case Polygon:
{
geom = read_polygon(reader, lengths);
break;
}
case LineString:
{
geom = read_line_string(reader);
break;
}
case MultiPoint:
{
geom = read_multi_point(reader);
break;
}
case MultiLineString:
{
geom = read_multi_linestring(reader, lengths);
break;
}
case MultiPolygon:
{
geom = read_multi_polygon(reader, lengths);
break;
}
default:
{
reader.skip();
break;
}
}
return geom;
}
template <typename T>
std::vector<std::uint32_t> read_lengths(T & reader)
{
std::vector<std::uint32_t> lengths;
auto pi = reader.get_packed_uint32();
for (auto it = pi.first; it != pi.second; ++it)
{
lengths.push_back(*it);
}
return lengths;
}
template <typename T, typename Iterator, typename Ring>
void read_linear_ring(T & reader, Iterator begin, Iterator end, Ring & ring, bool close = false)
{
double x = 0.0;
double y = 0.0;
std::size_t count = 0;
for (auto it = begin; it != end; ++it)
{
std::int64_t delta= *it;
auto d = count % dim;
if (d == 0) x += delta;
else if (d == 1)
{
y += delta;
ring.emplace_back(transform(x), transform(y));
}
//we're only interested in X and Y, ignoring any extra coordinates
++count;
}
if (close && !ring.empty())
{
ring.emplace_back(ring.front());
}
}
template <typename T>
geometry::multi_point<double> read_multi_point(T & reader)
{
geometry::multi_point<double> multi_point;
double x = 0.0;
double y = 0.0;
auto pi = reader.get_packed_sint64();
std::size_t count = 0;
for (auto it = pi.first; it != pi.second; ++it)
{
auto delta = *it;
auto d = count % dim;
if (d == 0) x += delta;
else if (d == 1)
{
y += delta;
geometry::point<double> pt(transform(x), transform(y));
multi_point.push_back(std::move(pt));
}
++count;
}
return multi_point;
}
template <typename T>
geometry::line_string<double> read_line_string(T & reader)
{
geometry::line_string<double> line;
auto pi = reader.get_packed_sint64();
read_linear_ring(reader, pi.first, pi.second, line);
return line;
}
template <typename T>
geometry::multi_line_string<double> read_multi_linestring(T & reader, boost::optional<std::vector<std::uint32_t>> const& lengths)
{
geometry::multi_line_string<double> multi_line;
multi_line.reserve(!lengths ? 1 : lengths->size());
auto pi = reader.get_packed_sint64();
if (!lengths)
{
geometry::line_string<double> line;
read_linear_ring(reader, pi.first, pi.second, line);
multi_line.push_back(std::move(line));
}
else
{
for (auto len : *lengths)
{
geometry::line_string<double> line;
read_linear_ring(reader, pi.first, std::next(pi.first, dim * len), line);
multi_line.push_back(std::move(line));
std::advance(pi.first, dim * len);
}
}
return multi_line;
}
template <typename T>
geometry::polygon<double> read_polygon(T & reader, boost::optional<std::vector<std::uint32_t>> const& lengths)
{
geometry::polygon<double> poly;
poly.reserve(!lengths ? 1 : lengths->size());
auto pi = reader.get_packed_sint64();
if (!lengths)
{
geometry::linear_ring<double> ring;
read_linear_ring(reader, pi.first, pi.second, ring, true);
poly.push_back(std::move(ring));
}
else
{
for (auto len : *lengths)
{
geometry::linear_ring<double> ring;
read_linear_ring(reader, pi.first, std::next(pi.first, dim * len), ring, true);
poly.push_back(std::move(ring));
std::advance(pi.first, dim * len);
}
}
return poly;
}
template <typename T>
geometry::multi_polygon<double> read_multi_polygon(T & reader, boost::optional<std::vector<std::uint32_t>> const& lengths)
{
geometry::multi_polygon<double> multi_poly;
if (!lengths)
{
auto poly = read_polygon(reader, lengths);
multi_poly.push_back(std::move(poly));
}
else if ((*lengths).size() > 0)
{
int j = 1;
auto pi = reader.get_packed_sint64();
for (std::size_t i = 0; i < (*lengths)[0]; ++i)
{
geometry::polygon<double> poly;
for (std::size_t k = 0; k < (*lengths)[j]; ++k)
{
geometry::linear_ring<double> ring;
std::size_t len = dim * (*lengths)[j + k + 1];
read_linear_ring(reader, pi.first, std::next(pi.first, len), ring, true);
poly.push_back(std::move(ring));
std::advance(pi.first, len);
}
multi_poly.push_back(std::move(poly));
j += (*lengths)[j] + 1;
}
}
return multi_poly;
}
template <typename T>
geometry::geometry<double> read_geometry(T & reader)
{
geometry::geometry<double> geom = geometry::geometry_empty();
geometry_type_e type = Unknown;
boost::optional<std::vector<std::uint32_t>> lengths;
while (reader.next())
{
switch (reader.tag())
{
case 1: // type
{
type = static_cast<geometry_type_e>(reader.get_uint32());
break;
}
case 2:
{
auto val = read_lengths(reader);
if (!val.empty()) lengths = std::move(val);
break;
}
case 3:
{
geom = std::move(read_coords(reader, type, lengths));
break;
}
case 4:
{
if (geom.is<geometry::geometry_empty>())
geom = geometry::geometry_collection<double>();
auto & collection = geom.get<geometry::geometry_collection<double>>();
auto message = reader.get_message();
collection.push_back(std::move(read_geometry(message)));
break;
}
case 13:
{
auto value = reader.get_message();
read_value(value);
break;
}
default:
{
reader.skip();
break;
}
}
}
return geom;
}
};
}}
#endif // MAPNIK_UTIL_GEOBUF_HPP

251
plugins/input/geobuf/geobuf_datasource.cpp Normal file
View file

@ -0,0 +1,251 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2017 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#include "geobuf_datasource.hpp"
#include "geobuf_featureset.hpp"
#include "geobuf.hpp"
#include <fstream>
#include <algorithm>
#include <functional>
// boost
#include <boost/algorithm/string.hpp>
// mapnik
#include <mapnik/unicode.hpp>
#include <mapnik/feature.hpp>
#include <mapnik/feature_kv_iterator.hpp>
#include <mapnik/value/types.hpp>
#include <mapnik/debug.hpp>
#include <mapnik/proj_transform.hpp>
#include <mapnik/projection.hpp>
#include <mapnik/util/geometry_to_ds_type.hpp>
#include <mapnik/util/variant.hpp>
#include <mapnik/util/file_io.hpp>
#include <mapnik/make_unique.hpp>
#include <mapnik/geometry/boost_adapters.hpp>
using mapnik::datasource;
using mapnik::parameters;
DATASOURCE_PLUGIN(geobuf_datasource)
struct attr_value_converter
{
mapnik::eAttributeType operator() (mapnik::value_integer) const
{
return mapnik::Integer;
}
mapnik::eAttributeType operator() (double) const
{
return mapnik::Double;
}
mapnik::eAttributeType operator() (float) const
{
return mapnik::Double;
}
mapnik::eAttributeType operator() (bool) const
{
return mapnik::Boolean;
}
mapnik::eAttributeType operator() (std::string const& ) const
{
return mapnik::String;
}
mapnik::eAttributeType operator() (mapnik::value_unicode_string const&) const
{
return mapnik::String;
}
mapnik::eAttributeType operator() (mapnik::value_null const& ) const
{
return mapnik::String;
}
};
geobuf_datasource::geobuf_datasource(parameters const& params)
: datasource(params),
type_(datasource::Vector),
desc_(geobuf_datasource::name(),
*params.get<std::string>("encoding","utf-8")),
filename_(),
extent_(),
features_(),
tree_(nullptr)
{
boost::optional<std::string> file = params.get<std::string>("file");
if (!file) throw mapnik::datasource_exception("Geobuf Plugin: missing <file> parameter");
boost::optional<std::string> base = params.get<std::string>("base");
if (base)
filename_ = *base + "/" + *file;
else
filename_ = *file;
mapnik::util::file in(filename_);
if (!in.is_open())
{
throw mapnik::datasource_exception("Geobuf Plugin: could not open: '" + filename_ + "'");
}
std::vector<char> geobuf;
geobuf.resize(in.size());
std::fread(geobuf.data(), in.size(), 1, in.get());
parse_geobuf(geobuf.data(), geobuf.size());
}
namespace {
template <typename T>
struct push_feature
{
using features_container = T;
push_feature(features_container & features)
: features_(features) {}
void operator() (mapnik::feature_ptr const& feature)
{
features_.push_back(feature);
}
features_container & features_;
};
}
void geobuf_datasource::parse_geobuf(char const* data, std::size_t size)
{
using push_feature_callback = push_feature<std::vector<mapnik::feature_ptr>>;
push_feature_callback callback(features_);
mapnik::util::geobuf<push_feature_callback> buf(data, size, callback);
buf.read();
using values_container = std::vector< std::pair<box_type, std::pair<std::size_t, std::size_t>>>;
values_container values;
values.reserve(features_.size());
tree_ = std::make_unique<spatial_index_type>(values);
std::size_t geometry_index = 0;
for (mapnik::feature_ptr const& f : features_)
{
mapnik::box2d<double> box = f->envelope();
if (box.valid())
{
if (geometry_index == 0)
{
extent_ = box;
for ( auto const& kv : *f)
{
desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
mapnik::util::apply_visitor(attr_value_converter(),
std::get<1>(kv))));
}
}
else
{
extent_.expand_to_include(box);
}
values.emplace_back(box, std::make_pair(geometry_index, 0));
}
++geometry_index;
}
// packing algorithm
tree_ = std::make_unique<spatial_index_type>(values);
}
geobuf_datasource::~geobuf_datasource() {}
const char * geobuf_datasource::name()
{
return "geobuf";
}
boost::optional<mapnik::datasource_geometry_t> geobuf_datasource::get_geometry_type() const
{
boost::optional<mapnik::datasource_geometry_t> result;
int multi_type = 0;
unsigned num_features = features_.size();
for (unsigned i = 0; i < num_features && i < 5; ++i)
{
result = mapnik::util::to_ds_type(features_[i]->get_geometry());
if (result)
{
int type = static_cast<int>(*result);
if (multi_type > 0 && multi_type != type)
{
result.reset(mapnik::datasource_geometry_t::Collection);
return result;
}
multi_type = type;
}
}
return result;
}
mapnik::datasource::datasource_t geobuf_datasource::type() const
{
return type_;
}
mapnik::box2d<double> geobuf_datasource::envelope() const
{
return extent_;
}
mapnik::layer_descriptor geobuf_datasource::get_descriptor() const
{
return desc_;
}
mapnik::featureset_ptr geobuf_datasource::features(mapnik::query const& q) const
{
// if the query box intersects our world extent then query for features
mapnik::box2d<double> const& box = q.get_bbox();
if (extent_.intersects(box))
{
geobuf_featureset::array_type index_array;
if (tree_)
{
tree_->query(boost::geometry::index::intersects(box), std::back_inserter(index_array));
return std::make_shared<geobuf_featureset>(features_, std::move(index_array));
}
}
return mapnik::featureset_ptr();
}
mapnik::featureset_ptr geobuf_datasource::features_at_point(mapnik::coord2d const& pt, double tol) const
{
mapnik::box2d<double> query_bbox(pt, pt);
query_bbox.pad(tol);
mapnik::query q(query_bbox);
std::vector<mapnik::attribute_descriptor> const& desc = desc_.get_descriptors();
std::vector<mapnik::attribute_descriptor>::const_iterator itr = desc.begin();
std::vector<mapnik::attribute_descriptor>::const_iterator end = desc.end();
for ( ;itr!=end;++itr)
{
q.add_property_name(itr->get_name());
}
return features(q);
}

104
plugins/input/geobuf/geobuf_datasource.hpp Normal file
View file

@ -0,0 +1,104 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2017 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#ifndef GEOBUF_DATASOURCE_HPP
#define GEOBUF_DATASOURCE_HPP
// mapnik
#include <mapnik/datasource.hpp>
#include <mapnik/params.hpp>
#include <mapnik/query.hpp>
#include <mapnik/feature.hpp>
#include <mapnik/geometry/box2d.hpp>
#include <mapnik/coord.hpp>
#include <mapnik/feature_layer_desc.hpp>
#include <mapnik/unicode.hpp>
// boost
#include <boost/optional.hpp>
#pragma GCC diagnostic push
#include <mapnik/warning_ignore.hpp>
#include <boost/optional.hpp>
#include <boost/version.hpp>
#include <boost/geometry/index/rtree.hpp>
#pragma GCC diagnostic pop
// stl
#include <memory>
#include <vector>
#include <string>
#include <map>
#include <deque>
using mapnik::datasource;
template <std::size_t Max, std::size_t Min>
struct geobuf_linear : boost::geometry::index::linear<Max,Min> {};
namespace boost { namespace geometry { namespace index { namespace detail { namespace rtree {
template <std::size_t Max, std::size_t Min>
struct options_type<geobuf_linear<Max,Min> >
{
using type = options<geobuf_linear<Max, Min>,
insert_default_tag,
choose_by_content_diff_tag,
split_default_tag,
linear_tag,
#if BOOST_VERSION >= 105700
node_variant_static_tag>;
#else
node_s_mem_static_tag>;
#endif
};
}}}}}
class geobuf_datasource : public mapnik::datasource
{
public:
using box_type = mapnik::box2d<double>;
using item_type = std::pair<box_type, std::pair<std::size_t, std::size_t>>;
using spatial_index_type = boost::geometry::index::rtree<item_type,geobuf_linear<16,4> >;
// constructor
geobuf_datasource(mapnik::parameters const& params);
virtual ~geobuf_datasource ();
mapnik::datasource::datasource_t type() const;
static const char * name();
mapnik::featureset_ptr features(mapnik::query const& q) const;
mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt, double tol = 0) const;
mapnik::box2d<double> envelope() const;
mapnik::layer_descriptor get_descriptor() const;
boost::optional<mapnik::datasource_geometry_t> get_geometry_type() const;
void parse_geobuf(char const* buffer, std::size_t size);
private:
mapnik::datasource::datasource_t type_;
mapnik::layer_descriptor desc_;
std::string filename_;
mapnik::box2d<double> extent_;
std::vector<mapnik::feature_ptr> features_;
std::unique_ptr<spatial_index_type> tree_;
};
#endif // GEOBUF_DATASOURCE_HPP

58
plugins/input/geobuf/geobuf_featureset.cpp Normal file
View file

@ -0,0 +1,58 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2017 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
// mapnik
#include <mapnik/feature.hpp>
#include <mapnik/feature_factory.hpp>
// stl
#include <string>
#include <vector>
#include <deque>
#include "geobuf_featureset.hpp"
geobuf_featureset::geobuf_featureset(std::vector<mapnik::feature_ptr> const& features,array_type && index_array)
: features_(features),
index_array_(std::move(index_array)),
index_itr_(index_array_.begin()),
index_end_(index_array_.end()),
ctx_(std::make_shared<mapnik::context_type>()) {}
geobuf_featureset::~geobuf_featureset() {}
mapnik::feature_ptr geobuf_featureset::next()
{
if (index_itr_ != index_end_)
{
#if BOOST_VERSION >= 105600
geobuf_datasource::item_type const& item = *index_itr_++;
std::size_t index = item.second.first;
#else
std::size_t index = *index_itr_++;
#endif
if ( index < features_.size())
{
return features_.at(index);
}
}
return mapnik::feature_ptr();
}

50
plugins/input/geobuf/geobuf_featureset.hpp Normal file
View file

@ -0,0 +1,50 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2017 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#ifndef GEOBUF_FEATURESET_HPP
#define GEOBUF_FEATURESET_HPP
#include <mapnik/feature.hpp>
#include "geobuf_datasource.hpp"
#include <vector>
#include <deque>
#include <cstdio>
class geobuf_featureset : public mapnik::Featureset
{
public:
typedef std::deque<geobuf_datasource::item_type> array_type;
geobuf_featureset(std::vector<mapnik::feature_ptr> const& features,
array_type && index_array);
virtual ~geobuf_featureset();
mapnik::feature_ptr next();
private:
std::vector<mapnik::feature_ptr> const& features_;
const array_type index_array_;
array_type::const_iterator index_itr_;
array_type::const_iterator index_end_;
mapnik::context_ptr ctx_;
};
#endif // GEOBUF_FEATURESET_HPP

2
test/data

@ -1 +1 @@
Subproject commit f95fe1c7b56a5eeb4fa2c2bcdc403d9254ce7448 Subproject commit 190950ace6650e91511b02b4982be6fdf73d0781

254
test/unit/datasource/geobuf.cpp Normal file
View file

@ -0,0 +1,254 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2017 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#include "catch.hpp"
#include "ds_test_util.hpp"
#include <mapnik/unicode.hpp>
#include <mapnik/datasource.hpp>
#include <mapnik/datasource_cache.hpp>
#include <mapnik/geometry.hpp>
#include <mapnik/geometry/geometry_type.hpp>
#include <mapnik/util/fs.hpp>
#include <cstdlib>
#include <algorithm>
#include <cctype>
#include <locale>
#include <boost/optional/optional_io.hpp>
TEST_CASE("Geobuf") {
std::string geobuf_plugin("./plugins/input/geobuf.input");
if (mapnik::util::exists(geobuf_plugin))
{
SECTION("Point")
{
//{"type":"Feature","id":1,"geometry":{"type":"Point","coordinates":[102,0.5]},"properties":{"prop0":"value0"}}
mapnik::parameters params;
params["type"] = "geobuf";
params["file"] = "./test/data/geobuf/point.geobuf";
auto ds = mapnik::datasource_cache::instance().create(params);
auto fs = all_features(ds);
auto f = fs->next();
auto const& geometry = f->get_geometry();
auto const& pt = mapnik::util::get<mapnik::geometry::point<double> >(geometry);
REQUIRE(pt.x == 102.0);
REQUIRE(pt.y == 0.5);
CHECK(fs->next() == nullptr);
}
SECTION("MultiPoint")
{
//{"type":"Feature","id":1,"geometry":{"type":"MultiPoint","coordinates":[[100,0],[101,1]]},"properties":{"prop0":"value0"}}
mapnik::parameters params;
params["type"] = "geobuf";
params["file"] = "./test/data/geobuf/multipoint.geobuf";
auto ds = mapnik::datasource_cache::instance().create(params);
auto fs = all_features(ds);
auto f = fs->next();
auto const& geometry = f->get_geometry();
auto const& mpt = mapnik::util::get<mapnik::geometry::multi_point<double> >(geometry);
CHECK(mpt.size() == 2);
REQUIRE(mpt[0].x == 100.0);
REQUIRE(mpt[0].y == 0.0);
REQUIRE(mpt[1].x == 101.0);
REQUIRE(mpt[1].y == 1.0);
CHECK(fs->next() == nullptr);
}
SECTION("LineString")
{
//{"type":"Feature","id":1,"geometry":{"type":"LineString","coordinates":[[102,0],[103,1],[104,0],[105,1]]},"properties":{"prop0":"value0","prop1":0}}
mapnik::parameters params;
params["type"] = "geobuf";
params["file"] = "./test/data/geobuf/linestring.geobuf";
auto ds = mapnik::datasource_cache::instance().create(params);
auto fs = all_features(ds);
auto f = fs->next();
auto const& geometry = f->get_geometry();
auto const& line = mapnik::util::get<mapnik::geometry::line_string<double> >(geometry);
CHECK(line.size() == 4);
REQUIRE(line[0].x == 102.0);
REQUIRE(line[0].y == 0);
REQUIRE(line[1].x == 103.0);
REQUIRE(line[1].y == 1);
REQUIRE(line[2].x == 104.0);
REQUIRE(line[2].y == 0);
REQUIRE(line[3].x == 105.0);
REQUIRE(line[3].y == 1);
CHECK(fs->next() == nullptr);
}
SECTION("MultiLineString")
{
//{"type":"Feature","id":1,"geometry":{"type":"MultiLineString","coordinates":[[[100,0],[101,1]],[[102,2],[103,3]]]},"properties":{"prop0":"value0","prop1":0}}
mapnik::parameters params;
params["type"] = "geobuf";
params["file"] = "./test/data/geobuf/multilinestring.geobuf";
auto ds = mapnik::datasource_cache::instance().create(params);
auto fs = all_features(ds);
auto f = fs->next();
auto const& geometry = f->get_geometry();
auto const& mline = mapnik::util::get<mapnik::geometry::multi_line_string<double> >(geometry);
CHECK(mline.size() == 2);
auto const& line1 = mline[0];
REQUIRE(line1[0].x == 100.0);
REQUIRE(line1[0].y == 0.0);
REQUIRE(line1[1].x == 101.0);
REQUIRE(line1[1].y == 1.0);
auto const& line2 = mline[1];
REQUIRE(line2[0].x == 102.0);
REQUIRE(line2[0].y == 2.0);
REQUIRE(line2[1].x == 103.0);
REQUIRE(line2[1].y == 3.0);
CHECK(fs->next() == nullptr);
}
SECTION("Polygon")
{
//{"type":"Feature","id":1,"geometry":{"type":"Polygon","coordinates":[[[100,0],[101,0],[101,1],[100,1],[100,0]],[[100.8,0.8],[100.8,0.2],[100.2,0.2],[100.2,0.8],[100.8,0.8]]]},"properties":{"prop0":"value0","prop1":"{\"this\":\"that\"}"}}
auto files =
{
"./test/data/geobuf/polygon.geobuf",
"./test/data/geobuf/standalone-feature.geobuf",
"./test/data/geobuf/standalone-geometry.geobuf"
};
mapnik::parameters params;
params["type"] = "geobuf";
for (auto const& filename : files)
{
params["file"] = filename;
auto ds = mapnik::datasource_cache::instance().create(params);
auto fs = all_features(ds);
auto f = fs->next();
auto const& geometry = f->get_geometry();
auto const& poly = mapnik::util::get<mapnik::geometry::polygon<double> >(geometry);
CHECK(poly.size() == 2);
auto const& exterior = poly[0];
REQUIRE(exterior[0].x == 100);
REQUIRE(exterior[0].y == 0);
REQUIRE(exterior[1].x == 101);
REQUIRE(exterior[1].y == 0);
REQUIRE(exterior[2].x == 101);
REQUIRE(exterior[2].y == 1);
REQUIRE(exterior[3].x == 100);
REQUIRE(exterior[3].y == 1);
REQUIRE(exterior[4].x == 100);
REQUIRE(exterior[4].y == 0);
auto const& interior = poly[1];
REQUIRE(interior[0].x == 100.8);
REQUIRE(interior[0].y == 0.8);
REQUIRE(interior[1].x == 100.8);
REQUIRE(interior[1].y == 0.2);
REQUIRE(interior[2].x == 100.2);
REQUIRE(interior[2].y == 0.2);
REQUIRE(interior[3].x == 100.2);
REQUIRE(interior[3].y == 0.8);
REQUIRE(interior[4].x == 100.8);
REQUIRE(interior[4].y == 0.8);
CHECK(fs->next() == nullptr);
}
}
SECTION("MultiPolygon")
{
//{"type":"Feature","id":1,"geometry":{"type":"MultiPolygon","coordinates":[[[[102,2],[103,2],[103,3],[102,3],[102,2]]],[[[100,0],[101,0],[101,1],[100,1],[100,0]],[[100.2,0.2],[100.2,0.8],[100.8,0.8],[100.8,0.2],[100.2,0.2]]]]},"properties":{"prop0":"value0","prop1":"{\"this\":\"that\"}"}}
mapnik::parameters params;
params["type"] = "geobuf";
params["file"] = "./test/data/geobuf/multipolygon.geobuf";
auto ds = mapnik::datasource_cache::instance().create(params);
auto fs = all_features(ds);
auto f = fs->next();
auto const& geometry = f->get_geometry();
auto const& mpoly = mapnik::util::get<mapnik::geometry::multi_polygon<double> >(geometry);
CHECK(mpoly.size() == 2);
{
auto const& poly = mpoly[0];
auto const& exterior = poly[0];
REQUIRE(exterior[0].x == 102);
REQUIRE(exterior[0].y == 2);
REQUIRE(exterior[1].x == 103);
REQUIRE(exterior[1].y == 2);
REQUIRE(exterior[2].x == 103);
REQUIRE(exterior[2].y == 3);
REQUIRE(exterior[3].x == 102);
REQUIRE(exterior[3].y == 3);
REQUIRE(exterior[4].x == 102);
REQUIRE(exterior[4].y == 2);
}
{
auto const& poly = mpoly[1];
auto const& exterior = poly[0];
REQUIRE(exterior[0].x == 100);
REQUIRE(exterior[0].y == 0);
REQUIRE(exterior[1].x == 101);
REQUIRE(exterior[1].y == 0);
REQUIRE(exterior[2].x == 101);
REQUIRE(exterior[2].y == 1);
REQUIRE(exterior[3].x == 100);
REQUIRE(exterior[3].y == 1);
REQUIRE(exterior[4].x == 100);
REQUIRE(exterior[4].y == 0);
auto const& interior = poly[1];
REQUIRE(interior[0].x == 100.2);
REQUIRE(interior[0].y == 0.2);
REQUIRE(interior[1].x == 100.2);
REQUIRE(interior[1].y == 0.8);
REQUIRE(interior[2].x == 100.8);
REQUIRE(interior[2].y == 0.8);
REQUIRE(interior[3].x == 100.8);
REQUIRE(interior[3].y == 0.2);
REQUIRE(interior[4].x == 100.2);
REQUIRE(interior[4].y == 0.2);
}
CHECK(fs->next() == nullptr);
}
SECTION("GeometryCollection")
{
//{"type":"Feature","id":1,"geometry":{"type":"GeometryCollection","geometries":[{"type":"Point","coordinates":[100,0]},{"type":"LineString","coordinates":[[101,0],[102,1]]}]},"properties":{"prop0":"value0","prop1":"{\"this\":\"that\"}"}}
mapnik::parameters params;
params["type"] = "geobuf";
params["file"] = "./test/data/geobuf/geometrycollection.geobuf";
auto ds = mapnik::datasource_cache::instance().create(params);
auto fs = all_features(ds);
auto f = fs->next();
auto const& geometry = f->get_geometry();
auto const& collection = mapnik::util::get<mapnik::geometry::geometry_collection<double> >(geometry);
CHECK(collection.size() == 2);
auto const& pt = mapnik::util::get<mapnik::geometry::point<double> >(collection[0]);
REQUIRE(pt.x == 100.0);
REQUIRE(pt.y == 0.0);
auto const& line = mapnik::util::get<mapnik::geometry::line_string<double> >(collection[1]);
REQUIRE(line[0].x == 101.0);
REQUIRE(line[0].y == 0.0);
REQUIRE(line[1].x == 102.0);
REQUIRE(line[1].y == 1);
CHECK(fs->next() == nullptr);
}
}
}