From 3bbb5f413a2a4074dafac14c6c8edc6b4aaf4bf1 Mon Sep 17 00:00:00 2001 From: artemp Date: Fri, 4 Aug 2017 18:03:08 +0100 Subject: [PATCH] Use `protozero` lib to read `geobuf` (Initial implementation, not complete!) --- .gitmodules | 4 + SConstruct | 3 +- deps/mapbox/protozero | 1 + plugins/input/geobuf/geobuf.hpp | 222 +++++++++++---------- plugins/input/geobuf/geobuf_datasource.cpp | 4 +- plugins/input/geobuf/geobuf_datasource.hpp | 2 +- plugins/input/geobuf/pbf.hpp | 184 ----------------- 7 files changed, 124 insertions(+), 296 deletions(-) create mode 160000 deps/mapbox/protozero delete mode 100644 plugins/input/geobuf/pbf.hpp diff --git a/.gitmodules b/.gitmodules index 4d4b9dec8..aaa9d544d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,7 @@ path = deps/mapbox/geometry url = https://github.com/mapbox/geometry.hpp.git branch = master +[submodule "deps/mapbox/protozero"] + path = deps/mapbox/protozero + url = https://github.com/mapbox/protozero.git + branch = master diff --git a/SConstruct b/SConstruct index 0618f5792..db090ba97 100644 --- a/SConstruct +++ b/SConstruct @@ -1,6 +1,6 @@ # This file is part of Mapnik (c++ mapping toolkit) # -# Copyright (C) 2015 Artem Pavlenko +# Copyright (C) 2017 Artem Pavlenko # # Mapnik is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -1604,6 +1604,7 @@ if not preconfigured: env.Prepend(LIBPATH = '#deps/agg') env.Prepend(CPPPATH = '#deps/mapbox/variant/include') env.Prepend(CPPPATH = '#deps/mapbox/geometry/include') + env.Prepend(CPPPATH = '#deps/mapbox/protozero/include') # prepend deps dir for auxillary headers env.Prepend(CPPPATH = '#deps') diff --git a/deps/mapbox/protozero b/deps/mapbox/protozero new file mode 160000 index 000000000..a1eb7327e --- /dev/null +++ b/deps/mapbox/protozero @@ -0,0 +1 @@ +Subproject commit a1eb7327e853161e939a409a8d74a8302dd1ecc5 diff --git a/plugins/input/geobuf/geobuf.hpp b/plugins/input/geobuf/geobuf.hpp index 5f6b90c5e..7b090b0bb 100644 --- a/plugins/input/geobuf/geobuf.hpp +++ b/plugins/input/geobuf/geobuf.hpp @@ -34,7 +34,7 @@ #include #include -#include "pbf.hpp" +#include namespace mapnik { namespace util { @@ -86,59 +86,59 @@ struct geobuf : mapnik::util::noncopyable std::size_t lengths = 0; std::vector keys_; std::vector values_; - mbgl::pbf pbf_; + protozero::pbf_reader reader_; FeatureCallback & callback_; mapnik::context_ptr ctx_; const std::unique_ptr tr_; public: //ctor - geobuf (unsigned char const* buf, std::size_t size, FeatureCallback & callback) - : pbf_(buf, size), + geobuf (char const* buf, std::size_t size, FeatureCallback & callback) + : reader_(buf, size), callback_(callback), ctx_(std::make_shared()), tr_(new mapnik::transcoder("utf8")) {} void read() { - while (pbf_.next()) + while (reader_.next()) { - switch (pbf_.tag) + switch (reader_.tag()) { case 1: // keys { - keys_.push_back(pbf_.string()); + keys_.push_back(reader_.get_string()); break; } case 2: { - dim = pbf_.varint(); + dim = reader_.get_uint32(); break; } case 3: { - precision = std::pow(10,pbf_.varint()); + precision = std::pow(10,reader_.get_uint32()); break; } case 4: { - auto feature_collection = pbf_.message(); + auto feature_collection = reader_.get_message(); read_feature_collection(feature_collection); break; } case 6: { - pbf_.message(); + reader_.get_message(); MAPNIK_LOG_DEBUG(geobuf) << "standalone Geometry is not supported"; break; } case 7: { - pbf_.message(); + reader_.get_message(); MAPNIK_LOG_DEBUG(geobuf) << "Topology is not supported"; break; } default: - MAPNIK_LOG_DEBUG(geobuf) << "FIXME tag=" << pbf_.tag; + MAPNIK_LOG_DEBUG(geobuf) << "Unsupported tag=" << reader_.tag(); break; } } @@ -152,40 +152,40 @@ private: } template - void read_value(T & pbf) + void read_value(T & reader) { - while (pbf.next()) + while (reader.next()) { - switch (pbf.tag) + switch (reader.tag()) { case 1: { - values_.emplace_back(pbf.string()); + values_.emplace_back(reader.get_string()); break; } case 2: { - values_.emplace_back(pbf.float64()); + values_.emplace_back(reader.get_double()); break; } case 3: { - values_.emplace_back(static_cast(pbf.varint())); + values_.emplace_back(static_cast(reader.get_uint32())); break; } case 4: { - values_.emplace_back(-static_cast(pbf.varint())); + values_.emplace_back(-static_cast(reader.get_uint32())); break; } case 5: { - values_.emplace_back(pbf.boolean()); + values_.emplace_back(reader.get_bool()); break; } case 6: { - values_.emplace_back(pbf.string()); // JSON value + values_.emplace_back(reader.get_string()); // JSON value break; } default: @@ -195,66 +195,69 @@ private: } template - void read_props(T & pbf, Feature & feature) + void read_props(T & reader, Feature & feature) { - std::uint8_t const* end = pbf.data + pbf.varint(); - while (pbf.data < end) + //char const* end = reader.get_data().first + reader.get_uint32(); + auto pi = reader.get_packed_uint32(); + for (auto it = pi.first; it != pi.second; ++it) + //while (reader.get_data().first < end) { - auto key_index = pbf.varint(); - auto value_index = pbf.varint(); + auto key_index = *it++; + auto value_index = *it; assert(key_index < keys_.size()); - assert(value_index < values_.size()); + assert(value_index< values_.size()); std::string const& name = keys_[key_index]; + //std::cerr << "name:" << name << std::endl; mapnik::util::apply_visitor(detail::value_visitor(feature, *tr_, name), values_[value_index]); } values_.clear(); } template - void read_feature (T & pbf) + void read_feature (T & reader) { auto feature = feature_factory::create(ctx_,1); - while (pbf.next()) + while (reader.next()) { - switch (pbf.tag) + switch (reader.tag()) { case 1: { - auto message = pbf.message(); + auto message = reader.get_message(); auto geom = read_geometry(message); feature->set_geometry(std::move(geom)); break; } case 11: { - auto feature_id = pbf.string(); + auto feature_id = reader.get_string(); break; } case 12: { - feature->set_id(pbf.template svarint()); + feature->set_id(reader.get_sint64()); break; } case 13: { - auto message = pbf.message(); + auto message = reader.get_message(); read_value(message); break; } case 14: { // feature props - read_props(pbf, *feature); + read_props(reader, *feature); break; } case 15: { // generic props - read_props(pbf, *feature); + read_props(reader, *feature); break; } default: - MAPNIK_LOG_DEBUG(geobuf) << "FAIL tag=" << pbf.tag; + MAPNIK_LOG_DEBUG(geobuf) << "Unsupported tag=" << reader.tag(); break; } } @@ -262,27 +265,27 @@ private: } template - void read_feature_collection(T & pbf) + void read_feature_collection(T & reader) { - while (pbf.next()) + while (reader.next()) { - switch (pbf.tag) + switch (reader.tag()) { case 1: { - auto message = pbf.message(); + auto message = reader.get_message(); read_feature(message); break; } case 13: { - auto message = pbf.message(); + auto message = reader.get_message(); read_value(message); break; } case 15: { - pbf.skipBytes(pbf.varint()); + reader.skip(); break; } default: @@ -292,15 +295,15 @@ private: } template - mapnik::geometry::point read_point(T & pbf) + mapnik::geometry::point read_point(T & reader) { - auto size = pbf.varint(); - std::uint8_t const* end = pbf.data + size; + auto size = reader.get_uint32(); + char const* end = reader.get_data().first + size; std::size_t count = 0; double x, y; - while (pbf.data < end) + while (reader.get_data().first < end) { - auto p = pbf.svarint(); + auto p = reader.get_sint32(); auto index = count % dim; if ( index == 0) { @@ -316,39 +319,39 @@ private: } template - mapnik::geometry::geometry read_coords(T & pbf, geometry_type_e type, - boost::optional> const& lengths) + mapnik::geometry::geometry read_coords(T & reader, geometry_type_e type, + boost::optional> const& lengths) { mapnik::geometry::geometry geom = mapnik::geometry::geometry_empty(); switch (type) { case Point: { - geom = read_point(pbf); + geom = read_point(reader); break; } case Polygon: { - geom = read_polygon(pbf, lengths); + geom = read_polygon(reader, lengths); break; } case LineString: { - geom = read_line_string(pbf); + geom = read_line_string(reader); break; } case MultiPoint: { - geom = read_multi_point(pbf); + geom = read_multi_point(reader); break; } case MultiLineString: { - geom = read_multi_linestring(pbf, lengths); + geom = read_multi_linestring(reader, lengths); } case MultiPolygon: { - geom = read_multi_polygon(pbf, lengths); + geom = read_multi_polygon(reader, lengths); break; } case GeometryCollection: @@ -364,58 +367,59 @@ private: } template - std::vector read_lengths(T & pbf) + std::vector read_lengths(T & reader) { - std::vector lengths; - auto size = pbf.varint(); - std::uint8_t const* end = pbf.data + size; - while (pbf.data < end) + std::vector lengths; + auto pi = reader.get_packed_uint32(); + for (auto it = pi.first; it != pi.second; ++it) { - lengths.push_back(pbf.varint()); + lengths.push_back(*it); } return lengths; } - template - void read_linear_ring(T & pbf, int len, std::size_t size, Ring & ring, bool close = false) + template + void read_linear_ring(T & reader, Iterator begin, Iterator end, Ring & ring, bool close = false) { - int i = 0; double x = 0.0; double y = 0.0; + auto size = std::distance(begin, end); ring.reserve(close ? size + 1 : size); - std::uint8_t const* end = pbf.data + size; - while ( (len > 0) ? i < len : pbf.data < end) + std::size_t count = 0; + + for (auto it = begin; it != end; ++it) { - for (unsigned d = 0; d < dim; ++d) + std::int64_t delta= *it; + auto d = count % dim; + if (d == 0) x += delta; + else if (d == 1) { - std::int64_t delta = pbf.template svarint(); - if (d == 0) x += delta; - else if (d == 1) y += delta; + y += delta; + ring.emplace_back(transform(x), transform(y)); } - ring.emplace_back(transform(x), transform(y)); - ++i; + //we're only interested in X and Y, ignoring any extra coordinates + ++count; } if (close && !ring.empty()) { ring.emplace_back(ring.front()); } - } template - mapnik::geometry::multi_point read_multi_point(T & pbf) + mapnik::geometry::multi_point read_multi_point(T & reader) { mapnik::geometry::multi_point multi_point; double x = 0.0; double y = 0.0; - auto size = pbf.varint(); - std::uint8_t const* end = pbf.data + size; - while (pbf.data < end) + auto size = reader.get_uint32(); + char const* end = reader.get_data().first + size; + while (reader.get_data().first < end) { for (unsigned d = 0; d < dim; ++d) { - if (d == 0) x += pbf.svarint(); - else if (d == 1) y += pbf.svarint(); + if (d == 0) x += reader.get_int32(); + else if (d == 1) y += reader.get_int32(); } mapnik::geometry::point pt(transform(x), transform(y)); multi_point.push_back(std::move(pt)); @@ -424,25 +428,25 @@ private: } template - mapnik::geometry::line_string read_line_string(T & pbf) + mapnik::geometry::line_string read_line_string(T & reader) { mapnik::geometry::line_string line; - auto size = pbf.varint(); - read_linear_ring(pbf, 0, size, line); + auto pi = reader.get_packed_sint64(); + read_linear_ring(reader, pi.first, pi.second, line); return line; } template - mapnik::geometry::multi_line_string read_multi_linestring(T & pbf, boost::optional> const& lengths) + mapnik::geometry::multi_line_string read_multi_linestring(T & reader, boost::optional> const& lengths) { mapnik::geometry::multi_line_string multi_line; multi_line.reserve(!lengths ? 1 : lengths->size()); - auto size = pbf.varint(); + //auto size = reader.get_uint32(); if (!lengths) { mapnik::geometry::line_string line; - read_linear_ring(pbf, 0, size, line); + //read_linear_ring(reader, 0, size, line); multi_line.push_back(std::move(line)); } else @@ -450,7 +454,7 @@ private: for (auto len : *lengths) { mapnik::geometry::line_string line; - read_linear_ring(pbf, len, size, line); + //read_linear_ring(reader, len, size, line); multi_line.push_back(std::move(line)); } } @@ -458,16 +462,15 @@ private: } template - mapnik::geometry::polygon read_polygon(T & pbf, boost::optional> const& lengths) + mapnik::geometry::polygon read_polygon(T & reader, boost::optional> const& lengths) { mapnik::geometry::polygon poly; poly.reserve(!lengths ? 1 : lengths->size()); - auto size = pbf.varint(); - + auto pi = reader.get_packed_sint64(); if (!lengths) { mapnik::geometry::linear_ring ring; - read_linear_ring(pbf, 0, size, ring, true); + read_linear_ring(reader, pi.first, pi.second, ring, true); poly.push_back(std::move(ring)); } else @@ -475,34 +478,37 @@ private: for (auto len : *lengths) { mapnik::geometry::linear_ring ring; - read_linear_ring(pbf, len, size, ring, true); + read_linear_ring(reader, pi.first, std::next(pi.first, dim * len), ring, true); poly.push_back(std::move(ring)); + std::advance(pi.first, dim * len); } } return poly; } template - mapnik::geometry::multi_polygon read_multi_polygon(T & pbf, boost::optional> const& lengths) + mapnik::geometry::multi_polygon read_multi_polygon(T & reader, boost::optional> const& lengths) { mapnik::geometry::multi_polygon multi_poly; - auto size = pbf.varint(); if (!lengths) { - auto poly = read_polygon(pbf, lengths); + auto poly = read_polygon(reader, lengths); multi_poly.push_back(std::move(poly)); } else if ((*lengths).size() > 0) { int j = 1; + auto pi = reader.get_packed_sint64(); for (int i = 0; i < (*lengths)[0]; ++i) { mapnik::geometry::polygon poly; for (int k = 0; k < (*lengths)[j]; ++k) { mapnik::geometry::linear_ring ring; - read_linear_ring(pbf, (*lengths)[j + 1 + k], size, ring, true); + std::size_t len = dim * (*lengths)[j + k + 1]; + read_linear_ring(reader, pi.first, std::next(pi.first, len), ring, true); poly.push_back(std::move(ring)); + std::advance(pi.first, len); } multi_poly.push_back(std::move(poly)); j += (*lengths)[j] + 1; @@ -512,28 +518,28 @@ private: } template - mapnik::geometry::geometry read_geometry(T & pbf) + mapnik::geometry::geometry read_geometry(T & reader) { mapnik::geometry::geometry geom = mapnik::geometry::geometry_empty(); geometry_type_e type = Unknown; - boost::optional> lengths; - while (pbf.next()) + boost::optional> lengths; + while (reader.next()) { - switch (pbf.tag) + switch (reader.tag()) { case 1: // type { - type = static_cast(pbf.varint()); + type = static_cast(reader.get_uint32()); break; } case 2: { - lengths = std::move(read_lengths(pbf)); + lengths = std::move(read_lengths(reader)); break; } case 3: { - geom = std::move(read_coords(pbf, type, lengths)); + geom = std::move(read_coords(reader, type, lengths)); break; } case 4: @@ -543,26 +549,26 @@ private: } case 11: { - auto geom_id = pbf.string(); + auto geom_id = reader.get_string(); MAPNIK_LOG_DEBUG(geobuf) << "geometry id's are not supported geom_id=" << geom_id ; break; } case 12: { - auto geom_id = pbf.template varint(); + auto geom_id = reader.get_int64(); MAPNIK_LOG_DEBUG(geobuf) << "geometry id's are not supported geom_id=" << geom_id ; break; } case 13: { - auto value = pbf.message(); + auto value = reader.get_message(); read_value(value); break; } case 14: case 15: { - pbf.skipBytes(pbf.varint()); + reader.skip(); break; } default: diff --git a/plugins/input/geobuf/geobuf_datasource.cpp b/plugins/input/geobuf/geobuf_datasource.cpp index 96bfb0a33..da6f8225c 100644 --- a/plugins/input/geobuf/geobuf_datasource.cpp +++ b/plugins/input/geobuf/geobuf_datasource.cpp @@ -114,7 +114,7 @@ geobuf_datasource::geobuf_datasource(parameters const& params) { throw mapnik::datasource_exception("Geobuf Plugin: could not open: '" + filename_ + "'"); } - std::vector geobuf; + std::vector geobuf; geobuf.resize(in.size()); std::fread(geobuf.data(), in.size(), 1, in.get()); parse_geobuf(geobuf.data(), geobuf.size()); @@ -137,7 +137,7 @@ struct push_feature } -void geobuf_datasource::parse_geobuf(std::uint8_t const* data, std::size_t size) +void geobuf_datasource::parse_geobuf(char const* data, std::size_t size) { using push_feature_callback = push_feature>; push_feature_callback callback(features_); diff --git a/plugins/input/geobuf/geobuf_datasource.hpp b/plugins/input/geobuf/geobuf_datasource.hpp index 4f8f83bbf..d7a3c5d9a 100644 --- a/plugins/input/geobuf/geobuf_datasource.hpp +++ b/plugins/input/geobuf/geobuf_datasource.hpp @@ -90,7 +90,7 @@ public: mapnik::box2d envelope() const; mapnik::layer_descriptor get_descriptor() const; boost::optional get_geometry_type() const; - void parse_geobuf(std::uint8_t const* buffer, std::size_t size); + void parse_geobuf(char const* buffer, std::size_t size); private: mapnik::datasource::datasource_t type_; mapnik::layer_descriptor desc_; diff --git a/plugins/input/geobuf/pbf.hpp b/plugins/input/geobuf/pbf.hpp deleted file mode 100644 index d017219a5..000000000 --- a/plugins/input/geobuf/pbf.hpp +++ /dev/null @@ -1,184 +0,0 @@ -#ifndef MBGL_UTIL_PBF -#define MBGL_UTIL_PBF - -/* - * Some parts are from upb - a minimalist implementation of protocol buffers. - * - * Copyright (c) 2008-2011 Google Inc. See LICENSE for details. - * Author: Josh Haberman - */ - -#include -#include - -namespace mbgl { - -struct pbf { - struct exception : std::exception { const char *what() const noexcept { return "pbf exception"; } }; - struct unterminated_varint_exception : exception { const char *what() const noexcept { return "pbf unterminated varint exception"; } }; - struct varint_too_long_exception : exception { const char *what() const noexcept { return "pbf varint too long exception"; } }; - struct unknown_field_type_exception : exception { const char *what() const noexcept { return "pbf unknown field type exception"; } }; - struct end_of_buffer_exception : exception { const char *what() const noexcept { return "pbf end of buffer exception"; } }; - - inline pbf(const unsigned char *data, size_t length); - inline pbf(); - - inline operator bool() const; - - inline bool next(); - inline bool next(uint32_t tag); - template inline T varint(); - template inline T svarint(); - - template inline T fixed(); - inline float float32(); - inline double float64(); - - inline std::string string(); - inline bool boolean(); - - inline pbf message(); - - inline void skip(); - inline void skipValue(uint32_t val); - inline void skipBytes(uint32_t bytes); - - const uint8_t *data = nullptr; - const uint8_t *end = nullptr; - uint32_t value = 0; - uint32_t tag = 0; -}; - -pbf::pbf(const unsigned char *data_, size_t length) - : data(data_), - end(data_ + length), - value(0), - tag(0) { -} - -pbf::pbf() - : data(nullptr), - end(nullptr), - value(0), - tag(0) { -} - - -pbf::operator bool() const { - return data < end; -} - -bool pbf::next() { - if (data < end) { - value = static_cast(varint()); - tag = value >> 3; - return true; - } - return false; -} - -bool pbf::next(uint32_t requested_tag) { - while (next()) { - if (tag == requested_tag) { - return true; - } else { - skip(); - } - } - return false; -} - -template -T pbf::varint() { - uint8_t byte = 0x80; - T result = 0; - int bitpos; - for (bitpos = 0; bitpos < 70 && (byte & 0x80); bitpos += 7) { - if (data >= end) { - throw unterminated_varint_exception(); - } - result |= ((T)(byte = *data) & 0x7F) << bitpos; - - data++; - } - if (bitpos == 70 && (byte & 0x80)) { - throw varint_too_long_exception(); - } - - return result; -} - -template -T pbf::svarint() { - T n = varint(); - return (n >> 1) ^ -(T)(n & 1); -} - -template -T pbf::fixed() { - skipBytes(bytes); - T result; - memcpy(&result, data - bytes, bytes); - return result; -} - -float pbf::float32() { - return fixed(); -} - -double pbf::float64() { - return fixed(); -} - -std::string pbf::string() { - uint32_t bytes = static_cast(varint()); - const char *string_data = reinterpret_cast(data); - skipBytes(bytes); - return std::string(string_data, bytes); -} - -bool pbf::boolean() { - skipBytes(1); - return *(bool *)(data - 1); -} - -pbf pbf::message() { - uint32_t bytes = static_cast(varint()); - const uint8_t *pos = data; - skipBytes(bytes); - return pbf(pos, bytes); -} - -void pbf::skip() { - skipValue(value); -} - -void pbf::skipValue(uint32_t val) { - switch (val & 0x7) { - case 0: // varint - varint(); - break; - case 1: // 64 bit - skipBytes(8); - break; - case 2: // string/message - skipBytes(static_cast(varint())); - break; - case 5: // 32 bit - skipBytes(4); - break; - default: - throw unknown_field_type_exception(); - } -} - -void pbf::skipBytes(uint32_t bytes) { - if (data + bytes > end) { - throw end_of_buffer_exception(); - } - data += bytes; -} - -} // end namespace mbgl - -#endif