From 02536536769a154257f88edf81dccdce26a0a516 Mon Sep 17 00:00:00 2001 From: Artem Pavlenko Date: Thu, 17 Aug 2017 10:52:07 +0100 Subject: [PATCH] Geobuf.input (#3715) * geobuf.input - initial implementation (WIP) * geobuf.input - fix reading MultiPolygon's * geobuf.input - don't store invalid bounding boxes in r-tree. * Use `protozero` lib to read `geobuf` (Initial implementation, not complete!) * geobuf.input - poring to `protozero`. (TODO: add support for `GeometryCollection`) * add initial support for `GeometryCollection` + cleanup namespaces * remove stderr + cleanup + update copyright * fix GeometryCollection and MultiLineString decoding + add support for standalone Feature and Geometry * don't use `reserve` as undelying data format doesn't allow for efficient `distance` implementation. * add geobuf unit test + update test data * update data --- .gitmodules | 4 + SConstruct | 4 +- deps/mapbox/protozero | 1 + plugins/input/geobuf/build.py | 64 +++ plugins/input/geobuf/geobuf.hpp | 564 +++++++++++++++++++++ plugins/input/geobuf/geobuf_datasource.cpp | 251 +++++++++ plugins/input/geobuf/geobuf_datasource.hpp | 104 ++++ plugins/input/geobuf/geobuf_featureset.cpp | 58 +++ plugins/input/geobuf/geobuf_featureset.hpp | 50 ++ test/data | 2 +- test/unit/datasource/geobuf.cpp | 254 ++++++++++ 11 files changed, 1354 insertions(+), 2 deletions(-) create mode 160000 deps/mapbox/protozero create mode 100644 plugins/input/geobuf/build.py create mode 100644 plugins/input/geobuf/geobuf.hpp create mode 100644 plugins/input/geobuf/geobuf_datasource.cpp create mode 100644 plugins/input/geobuf/geobuf_datasource.hpp create mode 100644 plugins/input/geobuf/geobuf_featureset.cpp create mode 100644 plugins/input/geobuf/geobuf_featureset.hpp create mode 100644 test/unit/datasource/geobuf.cpp diff --git a/.gitmodules b/.gitmodules index 4d4b9dec8..aaa9d544d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,7 @@ path = deps/mapbox/geometry url = https://github.com/mapbox/geometry.hpp.git branch = master +[submodule "deps/mapbox/protozero"] + path = deps/mapbox/protozero + url = https://github.com/mapbox/protozero.git + branch = master diff --git a/SConstruct b/SConstruct index d640afe15..db090ba97 100644 --- a/SConstruct +++ b/SConstruct @@ -1,6 +1,6 @@ # This file is part of Mapnik (c++ mapping toolkit) # -# Copyright (C) 2015 Artem Pavlenko +# Copyright (C) 2017 Artem Pavlenko # # Mapnik is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -116,6 +116,7 @@ PLUGINS = { # plugins with external dependencies 'csv': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'}, 'raster': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'}, 'geojson': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'}, + 'geobuf': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'}, 'topojson':{'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'} } @@ -1603,6 +1604,7 @@ if not preconfigured: env.Prepend(LIBPATH = '#deps/agg') env.Prepend(CPPPATH = '#deps/mapbox/variant/include') env.Prepend(CPPPATH = '#deps/mapbox/geometry/include') + env.Prepend(CPPPATH = '#deps/mapbox/protozero/include') # prepend deps dir for auxillary headers env.Prepend(CPPPATH = '#deps') diff --git a/deps/mapbox/protozero b/deps/mapbox/protozero new file mode 160000 index 000000000..a1eb7327e --- /dev/null +++ b/deps/mapbox/protozero @@ -0,0 +1 @@ +Subproject commit a1eb7327e853161e939a409a8d74a8302dd1ecc5 diff --git a/plugins/input/geobuf/build.py b/plugins/input/geobuf/build.py new file mode 100644 index 000000000..45b092423 --- /dev/null +++ b/plugins/input/geobuf/build.py @@ -0,0 +1,64 @@ +# +# This file is part of Mapnik (c++ mapping toolkit) +# +# Copyright (C) 2017 Artem Pavlenko +# +# Mapnik is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# + +Import ('env') + +Import ('plugin_base') + +PLUGIN_NAME = 'geobuf' + +plugin_env = plugin_base.Clone() + +plugin_sources = Split( + """ + %(PLUGIN_NAME)s_datasource.cpp + %(PLUGIN_NAME)s_featureset.cpp + """ % locals() +) + +# Link Library to Dependencies +libraries = [] +libraries.append(env['ICU_LIB_NAME']) +libraries.append('boost_system%s' % env['BOOST_APPEND']) +libraries.append('mapnik-json') + +if env['PLUGIN_LINKING'] == 'shared': + libraries.append(env['MAPNIK_NAME']) + + TARGET = plugin_env.SharedLibrary('../%s' % PLUGIN_NAME, + SHLIBPREFIX='', + SHLIBSUFFIX='.input', + source=plugin_sources, + LIBS=libraries) + + # if the plugin links to libmapnik ensure it is built first + Depends(TARGET, env.subst('../../../src/%s' % env['MAPNIK_LIB_NAME'])) + + if 'uninstall' not in COMMAND_LINE_TARGETS: + env.Install(env['MAPNIK_INPUT_PLUGINS_DEST'], TARGET) + env.Alias('install', env['MAPNIK_INPUT_PLUGINS_DEST']) + +plugin_obj = { + 'LIBS': libraries, + 'SOURCES': plugin_sources, +} + +Return('plugin_obj') diff --git a/plugins/input/geobuf/geobuf.hpp b/plugins/input/geobuf/geobuf.hpp new file mode 100644 index 000000000..55fa25532 --- /dev/null +++ b/plugins/input/geobuf/geobuf.hpp @@ -0,0 +1,564 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2017 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef MAPNIK_UTIL_GEOBUF_HPP +#define MAPNIK_UTIL_GEOBUF_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace mapnik { namespace util { + +enum geometry_type_e +{ + Unknown = -1, + Point = 0, + MultiPoint = 1, + LineString = 2, + MultiLineString = 3, + Polygon = 4, + MultiPolygon = 5, + GeometryCollection = 6 +}; + +namespace detail { +struct value_visitor +{ + value_visitor(feature_impl & feature, transcoder const& tr, std::string const& name) + : feature_(feature), + tr_(tr), + name_(name) {} + + void operator() (std::string const& val) // unicode + { + feature_.put_new(name_, tr_.transcode(val.c_str())); + } + + template + void operator() (T val) + { + feature_.put_new(name_, val); + } + + feature_impl & feature_; + transcoder const& tr_; + std::string const& name_; +}; +} + +template +struct geobuf : util::noncopyable +{ + using value_type = util::variant; + unsigned dim = 2; + double precision = std::pow(10,6); + bool is_topo = false; + bool transformed = false; + std::size_t lengths = 0; + std::vector keys_; + std::vector values_; + protozero::pbf_reader reader_; + FeatureCallback & callback_; + context_ptr ctx_; + const std::unique_ptr tr_; +public: + //ctor + geobuf (char const* buf, std::size_t size, FeatureCallback & callback) + : reader_(buf, size), + callback_(callback), + ctx_(std::make_shared()), + tr_(new transcoder("utf8")) {} + + void read() + { + while (reader_.next()) + { + switch (reader_.tag()) + { + case 1: // keys + { + keys_.push_back(reader_.get_string()); + break; + } + case 2: + { + dim = reader_.get_uint32(); + break; + } + case 3: + { + precision = std::pow(10,reader_.get_uint32()); + break; + } + case 4: + { + auto feature_collection = reader_.get_message(); + read_feature_collection(feature_collection); + break; + } + case 5: + { + // standalone Feature + auto message = reader_.get_message(); + read_feature(message); + break; + } + case 6: + { + // standalone Geometry + auto feature = feature_factory::create(ctx_,1); + auto message = reader_.get_message(); + feature->set_geometry(std::move(read_geometry(message))); + callback_(feature); + break; + } + default: + MAPNIK_LOG_DEBUG(geobuf) << "Unsupported tag=" << reader_.tag(); + reader_.skip(); + break; + } + } + } + +private: + + double transform(std::int64_t input) + { + return (transformed) ? (static_cast(input)) : (input/precision); + } + + template + void read_value(T & reader) + { + while (reader.next()) + { + switch (reader.tag()) + { + case 1: + { + values_.emplace_back(reader.get_string()); + break; + } + case 2: + { + values_.emplace_back(reader.get_double()); + break; + } + case 3: + { + values_.emplace_back(static_cast(reader.get_uint32())); + break; + } + case 4: + { + values_.emplace_back(-static_cast(reader.get_uint32())); + break; + } + case 5: + { + values_.emplace_back(reader.get_bool()); + break; + } + case 6: + { + values_.emplace_back(reader.get_string()); // JSON value + break; + } + default: + break; + } + } + } + + template + void read_props(T & reader, Feature & feature) + { + auto pi = reader.get_packed_uint32(); + for (auto it = pi.first; it != pi.second; ++it) + { + auto key_index = *it++; + auto value_index = *it; + assert(key_index < keys_.size()); + assert(value_index < values_.size()); + std::string const& name = keys_[key_index]; + util::apply_visitor(detail::value_visitor(feature, *tr_, name), values_[value_index]); + } + values_.clear(); + } + + template + void read_feature (T & reader) + { + auto feature = feature_factory::create(ctx_,1); + while (reader.next()) + { + switch (reader.tag()) + { + case 1: + { + auto message = reader.get_message(); + auto geom = read_geometry(message); + feature->set_geometry(std::move(geom)); + break; + } + case 11: + { + auto feature_id = reader.get_string(); + break; + } + case 12: + { + feature->set_id(reader.get_sint64()); + break; + } + case 13: + { + auto message = reader.get_message(); + read_value(message); + break; + } + case 14: + { + // feature props + read_props(reader, *feature); + break; + } + case 15: + { + // generic props + read_props(reader, *feature); + break; + } + default: + MAPNIK_LOG_DEBUG(geobuf) << "Unsupported tag=" << reader.tag(); + break; + } + } + callback_(feature); + } + + template + void read_feature_collection(T & reader) + { + while (reader.next()) + { + switch (reader.tag()) + { + case 1: + { + auto message = reader.get_message(); + read_feature(message); + break; + } + case 13: + { + auto message = reader.get_message(); + read_value(message); + break; + } + default: + { + reader.skip(); + break; + } + } + } + } + + template + geometry::point read_point(T & reader) + { + double x = 0.0; + double y = 0.0; + auto pi = reader.get_packed_sint64(); + std::size_t count = 0; + for (auto it = pi.first ; it != pi.second; ++it, ++count) + { + if (count == 0) x = transform(*it); + else if (count == 1) y = transform(*it); + } + return geometry::point(x, y); + } + + template + geometry::geometry read_coords(T & reader, geometry_type_e type, + boost::optional> const& lengths) + { + geometry::geometry geom = geometry::geometry_empty(); + switch (type) + { + case Point: + { + geom = read_point(reader); + break; + } + case Polygon: + { + geom = read_polygon(reader, lengths); + break; + } + case LineString: + { + geom = read_line_string(reader); + break; + } + case MultiPoint: + { + geom = read_multi_point(reader); + break; + } + case MultiLineString: + { + geom = read_multi_linestring(reader, lengths); + break; + } + case MultiPolygon: + { + geom = read_multi_polygon(reader, lengths); + break; + } + default: + { + reader.skip(); + break; + } + } + return geom; + } + + template + std::vector read_lengths(T & reader) + { + std::vector lengths; + auto pi = reader.get_packed_uint32(); + for (auto it = pi.first; it != pi.second; ++it) + { + lengths.push_back(*it); + } + return lengths; + } + + template + void read_linear_ring(T & reader, Iterator begin, Iterator end, Ring & ring, bool close = false) + { + double x = 0.0; + double y = 0.0; + std::size_t count = 0; + for (auto it = begin; it != end; ++it) + { + std::int64_t delta= *it; + auto d = count % dim; + if (d == 0) x += delta; + else if (d == 1) + { + y += delta; + ring.emplace_back(transform(x), transform(y)); + } + //we're only interested in X and Y, ignoring any extra coordinates + ++count; + } + if (close && !ring.empty()) + { + ring.emplace_back(ring.front()); + } + } + + template + geometry::multi_point read_multi_point(T & reader) + { + geometry::multi_point multi_point; + double x = 0.0; + double y = 0.0; + auto pi = reader.get_packed_sint64(); + std::size_t count = 0; + for (auto it = pi.first; it != pi.second; ++it) + { + auto delta = *it; + auto d = count % dim; + if (d == 0) x += delta; + else if (d == 1) + { + y += delta; + geometry::point pt(transform(x), transform(y)); + multi_point.push_back(std::move(pt)); + } + ++count; + } + return multi_point; + } + + template + geometry::line_string read_line_string(T & reader) + { + geometry::line_string line; + auto pi = reader.get_packed_sint64(); + read_linear_ring(reader, pi.first, pi.second, line); + return line; + + } + + template + geometry::multi_line_string read_multi_linestring(T & reader, boost::optional> const& lengths) + { + geometry::multi_line_string multi_line; + multi_line.reserve(!lengths ? 1 : lengths->size()); + auto pi = reader.get_packed_sint64(); + if (!lengths) + { + geometry::line_string line; + read_linear_ring(reader, pi.first, pi.second, line); + multi_line.push_back(std::move(line)); + } + else + { + for (auto len : *lengths) + { + geometry::line_string line; + read_linear_ring(reader, pi.first, std::next(pi.first, dim * len), line); + multi_line.push_back(std::move(line)); + std::advance(pi.first, dim * len); + } + } + return multi_line; + } + + template + geometry::polygon read_polygon(T & reader, boost::optional> const& lengths) + { + geometry::polygon poly; + poly.reserve(!lengths ? 1 : lengths->size()); + auto pi = reader.get_packed_sint64(); + if (!lengths) + { + geometry::linear_ring ring; + read_linear_ring(reader, pi.first, pi.second, ring, true); + poly.push_back(std::move(ring)); + } + else + { + for (auto len : *lengths) + { + geometry::linear_ring ring; + read_linear_ring(reader, pi.first, std::next(pi.first, dim * len), ring, true); + poly.push_back(std::move(ring)); + std::advance(pi.first, dim * len); + } + } + return poly; + } + + template + geometry::multi_polygon read_multi_polygon(T & reader, boost::optional> const& lengths) + { + geometry::multi_polygon multi_poly; + if (!lengths) + { + auto poly = read_polygon(reader, lengths); + multi_poly.push_back(std::move(poly)); + } + else if ((*lengths).size() > 0) + { + int j = 1; + auto pi = reader.get_packed_sint64(); + for (std::size_t i = 0; i < (*lengths)[0]; ++i) + { + geometry::polygon poly; + for (std::size_t k = 0; k < (*lengths)[j]; ++k) + { + geometry::linear_ring ring; + std::size_t len = dim * (*lengths)[j + k + 1]; + read_linear_ring(reader, pi.first, std::next(pi.first, len), ring, true); + poly.push_back(std::move(ring)); + std::advance(pi.first, len); + } + multi_poly.push_back(std::move(poly)); + j += (*lengths)[j] + 1; + } + } + return multi_poly; + } + + template + geometry::geometry read_geometry(T & reader) + { + geometry::geometry geom = geometry::geometry_empty(); + geometry_type_e type = Unknown; + boost::optional> lengths; + while (reader.next()) + { + switch (reader.tag()) + { + case 1: // type + { + type = static_cast(reader.get_uint32()); + break; + } + case 2: + { + auto val = read_lengths(reader); + if (!val.empty()) lengths = std::move(val); + break; + } + case 3: + { + geom = std::move(read_coords(reader, type, lengths)); + break; + } + case 4: + { + if (geom.is()) + geom = geometry::geometry_collection(); + auto & collection = geom.get>(); + auto message = reader.get_message(); + collection.push_back(std::move(read_geometry(message))); + break; + } + case 13: + { + auto value = reader.get_message(); + read_value(value); + break; + } + default: + { + reader.skip(); + break; + } + } + } + return geom; + } +}; + +}} + +#endif // MAPNIK_UTIL_GEOBUF_HPP diff --git a/plugins/input/geobuf/geobuf_datasource.cpp b/plugins/input/geobuf/geobuf_datasource.cpp new file mode 100644 index 000000000..3b4094316 --- /dev/null +++ b/plugins/input/geobuf/geobuf_datasource.cpp @@ -0,0 +1,251 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2017 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#include "geobuf_datasource.hpp" +#include "geobuf_featureset.hpp" +#include "geobuf.hpp" + +#include +#include +#include + +// boost + +#include + +// mapnik +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using mapnik::datasource; +using mapnik::parameters; + +DATASOURCE_PLUGIN(geobuf_datasource) + +struct attr_value_converter +{ + mapnik::eAttributeType operator() (mapnik::value_integer) const + { + return mapnik::Integer; + } + + mapnik::eAttributeType operator() (double) const + { + return mapnik::Double; + } + + mapnik::eAttributeType operator() (float) const + { + return mapnik::Double; + } + + mapnik::eAttributeType operator() (bool) const + { + return mapnik::Boolean; + } + + mapnik::eAttributeType operator() (std::string const& ) const + { + return mapnik::String; + } + + mapnik::eAttributeType operator() (mapnik::value_unicode_string const&) const + { + return mapnik::String; + } + + mapnik::eAttributeType operator() (mapnik::value_null const& ) const + { + return mapnik::String; + } +}; + +geobuf_datasource::geobuf_datasource(parameters const& params) + : datasource(params), + type_(datasource::Vector), + desc_(geobuf_datasource::name(), + *params.get("encoding","utf-8")), + filename_(), + extent_(), + features_(), + tree_(nullptr) +{ + boost::optional file = params.get("file"); + if (!file) throw mapnik::datasource_exception("Geobuf Plugin: missing parameter"); + + boost::optional base = params.get("base"); + if (base) + filename_ = *base + "/" + *file; + else + filename_ = *file; + + + mapnik::util::file in(filename_); + if (!in.is_open()) + { + throw mapnik::datasource_exception("Geobuf Plugin: could not open: '" + filename_ + "'"); + } + std::vector geobuf; + geobuf.resize(in.size()); + std::fread(geobuf.data(), in.size(), 1, in.get()); + parse_geobuf(geobuf.data(), geobuf.size()); +} + +namespace { +template +struct push_feature +{ + using features_container = T; + push_feature(features_container & features) + : features_(features) {} + + void operator() (mapnik::feature_ptr const& feature) + { + features_.push_back(feature); + } + features_container & features_; +}; +} + + +void geobuf_datasource::parse_geobuf(char const* data, std::size_t size) +{ + using push_feature_callback = push_feature>; + push_feature_callback callback(features_); + mapnik::util::geobuf buf(data, size, callback); + buf.read(); + using values_container = std::vector< std::pair>>; + values_container values; + values.reserve(features_.size()); + tree_ = std::make_unique(values); + std::size_t geometry_index = 0; + for (mapnik::feature_ptr const& f : features_) + { + mapnik::box2d box = f->envelope(); + if (box.valid()) + { + if (geometry_index == 0) + { + extent_ = box; + for ( auto const& kv : *f) + { + desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv), + mapnik::util::apply_visitor(attr_value_converter(), + std::get<1>(kv)))); + } + } + else + { + extent_.expand_to_include(box); + } + values.emplace_back(box, std::make_pair(geometry_index, 0)); + } + ++geometry_index; + } + // packing algorithm + tree_ = std::make_unique(values); +} + +geobuf_datasource::~geobuf_datasource() {} + +const char * geobuf_datasource::name() +{ + return "geobuf"; +} + +boost::optional geobuf_datasource::get_geometry_type() const +{ + boost::optional result; + int multi_type = 0; + unsigned num_features = features_.size(); + for (unsigned i = 0; i < num_features && i < 5; ++i) + { + result = mapnik::util::to_ds_type(features_[i]->get_geometry()); + if (result) + { + int type = static_cast(*result); + if (multi_type > 0 && multi_type != type) + { + result.reset(mapnik::datasource_geometry_t::Collection); + return result; + } + multi_type = type; + } + } + return result; +} + +mapnik::datasource::datasource_t geobuf_datasource::type() const +{ + return type_; +} + +mapnik::box2d geobuf_datasource::envelope() const +{ + return extent_; +} + +mapnik::layer_descriptor geobuf_datasource::get_descriptor() const +{ + return desc_; +} + +mapnik::featureset_ptr geobuf_datasource::features(mapnik::query const& q) const +{ + // if the query box intersects our world extent then query for features + mapnik::box2d const& box = q.get_bbox(); + if (extent_.intersects(box)) + { + geobuf_featureset::array_type index_array; + if (tree_) + { + tree_->query(boost::geometry::index::intersects(box), std::back_inserter(index_array)); + return std::make_shared(features_, std::move(index_array)); + } + } + return mapnik::featureset_ptr(); +} + +mapnik::featureset_ptr geobuf_datasource::features_at_point(mapnik::coord2d const& pt, double tol) const +{ + mapnik::box2d query_bbox(pt, pt); + query_bbox.pad(tol); + mapnik::query q(query_bbox); + std::vector const& desc = desc_.get_descriptors(); + std::vector::const_iterator itr = desc.begin(); + std::vector::const_iterator end = desc.end(); + for ( ;itr!=end;++itr) + { + q.add_property_name(itr->get_name()); + } + return features(q); +} diff --git a/plugins/input/geobuf/geobuf_datasource.hpp b/plugins/input/geobuf/geobuf_datasource.hpp new file mode 100644 index 000000000..d7a3c5d9a --- /dev/null +++ b/plugins/input/geobuf/geobuf_datasource.hpp @@ -0,0 +1,104 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2017 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef GEOBUF_DATASOURCE_HPP +#define GEOBUF_DATASOURCE_HPP + +// mapnik +#include +#include +#include +#include +#include +#include +#include +#include +// boost +#include +#pragma GCC diagnostic push +#include +#include +#include +#include +#pragma GCC diagnostic pop + +// stl +#include +#include +#include +#include +#include + +using mapnik::datasource; + +template +struct geobuf_linear : boost::geometry::index::linear {}; + +namespace boost { namespace geometry { namespace index { namespace detail { namespace rtree { + +template +struct options_type > +{ + using type = options, + insert_default_tag, + choose_by_content_diff_tag, + split_default_tag, + linear_tag, +#if BOOST_VERSION >= 105700 + node_variant_static_tag>; +#else + node_s_mem_static_tag>; + +#endif +}; + +}}}}} + +class geobuf_datasource : public mapnik::datasource +{ +public: + using box_type = mapnik::box2d; + using item_type = std::pair>; + using spatial_index_type = boost::geometry::index::rtree >; + + // constructor + geobuf_datasource(mapnik::parameters const& params); + virtual ~geobuf_datasource (); + mapnik::datasource::datasource_t type() const; + static const char * name(); + mapnik::featureset_ptr features(mapnik::query const& q) const; + mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt, double tol = 0) const; + mapnik::box2d envelope() const; + mapnik::layer_descriptor get_descriptor() const; + boost::optional get_geometry_type() const; + void parse_geobuf(char const* buffer, std::size_t size); +private: + mapnik::datasource::datasource_t type_; + mapnik::layer_descriptor desc_; + std::string filename_; + mapnik::box2d extent_; + std::vector features_; + std::unique_ptr tree_; +}; + + +#endif // GEOBUF_DATASOURCE_HPP diff --git a/plugins/input/geobuf/geobuf_featureset.cpp b/plugins/input/geobuf/geobuf_featureset.cpp new file mode 100644 index 000000000..67a49f4c9 --- /dev/null +++ b/plugins/input/geobuf/geobuf_featureset.cpp @@ -0,0 +1,58 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2017 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +// mapnik +#include +#include +// stl +#include +#include +#include + +#include "geobuf_featureset.hpp" + +geobuf_featureset::geobuf_featureset(std::vector const& features,array_type && index_array) + : features_(features), + index_array_(std::move(index_array)), + index_itr_(index_array_.begin()), + index_end_(index_array_.end()), + ctx_(std::make_shared()) {} + +geobuf_featureset::~geobuf_featureset() {} + +mapnik::feature_ptr geobuf_featureset::next() +{ + if (index_itr_ != index_end_) + { +#if BOOST_VERSION >= 105600 + geobuf_datasource::item_type const& item = *index_itr_++; + std::size_t index = item.second.first; +#else + std::size_t index = *index_itr_++; +#endif + if ( index < features_.size()) + { + return features_.at(index); + } + } + return mapnik::feature_ptr(); +} diff --git a/plugins/input/geobuf/geobuf_featureset.hpp b/plugins/input/geobuf/geobuf_featureset.hpp new file mode 100644 index 000000000..5e16f8e8d --- /dev/null +++ b/plugins/input/geobuf/geobuf_featureset.hpp @@ -0,0 +1,50 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2017 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef GEOBUF_FEATURESET_HPP +#define GEOBUF_FEATURESET_HPP + +#include +#include "geobuf_datasource.hpp" + +#include +#include +#include + +class geobuf_featureset : public mapnik::Featureset +{ +public: + typedef std::deque array_type; + geobuf_featureset(std::vector const& features, + array_type && index_array); + virtual ~geobuf_featureset(); + mapnik::feature_ptr next(); + +private: + std::vector const& features_; + const array_type index_array_; + array_type::const_iterator index_itr_; + array_type::const_iterator index_end_; + mapnik::context_ptr ctx_; +}; + +#endif // GEOBUF_FEATURESET_HPP diff --git a/test/data b/test/data index f95fe1c7b..190950ace 160000 --- a/test/data +++ b/test/data @@ -1 +1 @@ -Subproject commit f95fe1c7b56a5eeb4fa2c2bcdc403d9254ce7448 +Subproject commit 190950ace6650e91511b02b4982be6fdf73d0781 diff --git a/test/unit/datasource/geobuf.cpp b/test/unit/datasource/geobuf.cpp new file mode 100644 index 000000000..1cb694e90 --- /dev/null +++ b/test/unit/datasource/geobuf.cpp @@ -0,0 +1,254 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2017 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#include "catch.hpp" +#include "ds_test_util.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +TEST_CASE("Geobuf") { + + std::string geobuf_plugin("./plugins/input/geobuf.input"); + if (mapnik::util::exists(geobuf_plugin)) + { + SECTION("Point") + { + //{"type":"Feature","id":1,"geometry":{"type":"Point","coordinates":[102,0.5]},"properties":{"prop0":"value0"}} + mapnik::parameters params; + params["type"] = "geobuf"; + params["file"] = "./test/data/geobuf/point.geobuf"; + auto ds = mapnik::datasource_cache::instance().create(params); + auto fs = all_features(ds); + auto f = fs->next(); + auto const& geometry = f->get_geometry(); + auto const& pt = mapnik::util::get >(geometry); + REQUIRE(pt.x == 102.0); + REQUIRE(pt.y == 0.5); + CHECK(fs->next() == nullptr); + } + + SECTION("MultiPoint") + { + //{"type":"Feature","id":1,"geometry":{"type":"MultiPoint","coordinates":[[100,0],[101,1]]},"properties":{"prop0":"value0"}} + mapnik::parameters params; + params["type"] = "geobuf"; + params["file"] = "./test/data/geobuf/multipoint.geobuf"; + auto ds = mapnik::datasource_cache::instance().create(params); + auto fs = all_features(ds); + auto f = fs->next(); + auto const& geometry = f->get_geometry(); + auto const& mpt = mapnik::util::get >(geometry); + CHECK(mpt.size() == 2); + REQUIRE(mpt[0].x == 100.0); + REQUIRE(mpt[0].y == 0.0); + REQUIRE(mpt[1].x == 101.0); + REQUIRE(mpt[1].y == 1.0); + CHECK(fs->next() == nullptr); + } + + SECTION("LineString") + { + //{"type":"Feature","id":1,"geometry":{"type":"LineString","coordinates":[[102,0],[103,1],[104,0],[105,1]]},"properties":{"prop0":"value0","prop1":0}} + mapnik::parameters params; + params["type"] = "geobuf"; + params["file"] = "./test/data/geobuf/linestring.geobuf"; + auto ds = mapnik::datasource_cache::instance().create(params); + auto fs = all_features(ds); + auto f = fs->next(); + auto const& geometry = f->get_geometry(); + auto const& line = mapnik::util::get >(geometry); + CHECK(line.size() == 4); + REQUIRE(line[0].x == 102.0); + REQUIRE(line[0].y == 0); + REQUIRE(line[1].x == 103.0); + REQUIRE(line[1].y == 1); + REQUIRE(line[2].x == 104.0); + REQUIRE(line[2].y == 0); + REQUIRE(line[3].x == 105.0); + REQUIRE(line[3].y == 1); + CHECK(fs->next() == nullptr); + } + + SECTION("MultiLineString") + { + //{"type":"Feature","id":1,"geometry":{"type":"MultiLineString","coordinates":[[[100,0],[101,1]],[[102,2],[103,3]]]},"properties":{"prop0":"value0","prop1":0}} + mapnik::parameters params; + params["type"] = "geobuf"; + params["file"] = "./test/data/geobuf/multilinestring.geobuf"; + auto ds = mapnik::datasource_cache::instance().create(params); + auto fs = all_features(ds); + auto f = fs->next(); + auto const& geometry = f->get_geometry(); + auto const& mline = mapnik::util::get >(geometry); + CHECK(mline.size() == 2); + auto const& line1 = mline[0]; + REQUIRE(line1[0].x == 100.0); + REQUIRE(line1[0].y == 0.0); + REQUIRE(line1[1].x == 101.0); + REQUIRE(line1[1].y == 1.0); + auto const& line2 = mline[1]; + REQUIRE(line2[0].x == 102.0); + REQUIRE(line2[0].y == 2.0); + REQUIRE(line2[1].x == 103.0); + REQUIRE(line2[1].y == 3.0); + CHECK(fs->next() == nullptr); + } + + SECTION("Polygon") + { + //{"type":"Feature","id":1,"geometry":{"type":"Polygon","coordinates":[[[100,0],[101,0],[101,1],[100,1],[100,0]],[[100.8,0.8],[100.8,0.2],[100.2,0.2],[100.2,0.8],[100.8,0.8]]]},"properties":{"prop0":"value0","prop1":"{\"this\":\"that\"}"}} + auto files = + { + "./test/data/geobuf/polygon.geobuf", + "./test/data/geobuf/standalone-feature.geobuf", + "./test/data/geobuf/standalone-geometry.geobuf" + }; + + mapnik::parameters params; + params["type"] = "geobuf"; + + for (auto const& filename : files) + { + params["file"] = filename; + auto ds = mapnik::datasource_cache::instance().create(params); + auto fs = all_features(ds); + auto f = fs->next(); + auto const& geometry = f->get_geometry(); + auto const& poly = mapnik::util::get >(geometry); + CHECK(poly.size() == 2); + auto const& exterior = poly[0]; + REQUIRE(exterior[0].x == 100); + REQUIRE(exterior[0].y == 0); + REQUIRE(exterior[1].x == 101); + REQUIRE(exterior[1].y == 0); + REQUIRE(exterior[2].x == 101); + REQUIRE(exterior[2].y == 1); + REQUIRE(exterior[3].x == 100); + REQUIRE(exterior[3].y == 1); + REQUIRE(exterior[4].x == 100); + REQUIRE(exterior[4].y == 0); + auto const& interior = poly[1]; + REQUIRE(interior[0].x == 100.8); + REQUIRE(interior[0].y == 0.8); + REQUIRE(interior[1].x == 100.8); + REQUIRE(interior[1].y == 0.2); + REQUIRE(interior[2].x == 100.2); + REQUIRE(interior[2].y == 0.2); + REQUIRE(interior[3].x == 100.2); + REQUIRE(interior[3].y == 0.8); + REQUIRE(interior[4].x == 100.8); + REQUIRE(interior[4].y == 0.8); + + CHECK(fs->next() == nullptr); + } + } + + SECTION("MultiPolygon") + { + //{"type":"Feature","id":1,"geometry":{"type":"MultiPolygon","coordinates":[[[[102,2],[103,2],[103,3],[102,3],[102,2]]],[[[100,0],[101,0],[101,1],[100,1],[100,0]],[[100.2,0.2],[100.2,0.8],[100.8,0.8],[100.8,0.2],[100.2,0.2]]]]},"properties":{"prop0":"value0","prop1":"{\"this\":\"that\"}"}} + + mapnik::parameters params; + params["type"] = "geobuf"; + params["file"] = "./test/data/geobuf/multipolygon.geobuf"; + auto ds = mapnik::datasource_cache::instance().create(params); + auto fs = all_features(ds); + auto f = fs->next(); + auto const& geometry = f->get_geometry(); + auto const& mpoly = mapnik::util::get >(geometry); + CHECK(mpoly.size() == 2); + { + auto const& poly = mpoly[0]; + auto const& exterior = poly[0]; + REQUIRE(exterior[0].x == 102); + REQUIRE(exterior[0].y == 2); + REQUIRE(exterior[1].x == 103); + REQUIRE(exterior[1].y == 2); + REQUIRE(exterior[2].x == 103); + REQUIRE(exterior[2].y == 3); + REQUIRE(exterior[3].x == 102); + REQUIRE(exterior[3].y == 3); + REQUIRE(exterior[4].x == 102); + REQUIRE(exterior[4].y == 2); + } + + { + auto const& poly = mpoly[1]; + auto const& exterior = poly[0]; + REQUIRE(exterior[0].x == 100); + REQUIRE(exterior[0].y == 0); + REQUIRE(exterior[1].x == 101); + REQUIRE(exterior[1].y == 0); + REQUIRE(exterior[2].x == 101); + REQUIRE(exterior[2].y == 1); + REQUIRE(exterior[3].x == 100); + REQUIRE(exterior[3].y == 1); + REQUIRE(exterior[4].x == 100); + REQUIRE(exterior[4].y == 0); + auto const& interior = poly[1]; + REQUIRE(interior[0].x == 100.2); + REQUIRE(interior[0].y == 0.2); + REQUIRE(interior[1].x == 100.2); + REQUIRE(interior[1].y == 0.8); + REQUIRE(interior[2].x == 100.8); + REQUIRE(interior[2].y == 0.8); + REQUIRE(interior[3].x == 100.8); + REQUIRE(interior[3].y == 0.2); + REQUIRE(interior[4].x == 100.2); + REQUIRE(interior[4].y == 0.2); + } + CHECK(fs->next() == nullptr); + } + SECTION("GeometryCollection") + { + //{"type":"Feature","id":1,"geometry":{"type":"GeometryCollection","geometries":[{"type":"Point","coordinates":[100,0]},{"type":"LineString","coordinates":[[101,0],[102,1]]}]},"properties":{"prop0":"value0","prop1":"{\"this\":\"that\"}"}} + mapnik::parameters params; + params["type"] = "geobuf"; + params["file"] = "./test/data/geobuf/geometrycollection.geobuf"; + auto ds = mapnik::datasource_cache::instance().create(params); + auto fs = all_features(ds); + auto f = fs->next(); + auto const& geometry = f->get_geometry(); + auto const& collection = mapnik::util::get >(geometry); + CHECK(collection.size() == 2); + auto const& pt = mapnik::util::get >(collection[0]); + REQUIRE(pt.x == 100.0); + REQUIRE(pt.y == 0.0); + auto const& line = mapnik::util::get >(collection[1]); + REQUIRE(line[0].x == 101.0); + REQUIRE(line[0].y == 0.0); + REQUIRE(line[1].x == 102.0); + REQUIRE(line[1].y == 1); + CHECK(fs->next() == nullptr); + } + } +}