Merge branch 'master' into spirit-x3

include/mapnik/quad_tree.hpp

@@ -181,12 +181,7 @@ public:
                       "Values stored in quad-tree must be standard layout types to allow serialisation");
         char header[16];
         std::memset(header,0,16);
-        header[0]='m';
+        std::strcpy(header,"mapnik-index");
-        header[1]='a';
-        header[2]='p';
-        header[3]='n';
-        header[4]='i';
-        header[5]='k';
         out.write(header,16);
         write_node(out,root_);
     }

include/mapnik/util/spatial_index.hpp

@@ -30,6 +30,7 @@
 #include <mapnik/geom_util.hpp>
 // stl
 #include <type_traits>
+#include <cstring>
 
 using mapnik::box2d;
 using mapnik::query;
@@ -44,7 +45,6 @@ public:
     static box2d<double> bounding_box( InputStream& in );
     static void query_first_n(Filter const& filter, InputStream & in, std::vector<Value>& pos, std::size_t count);
 private:
-
     spatial_index();
     ~spatial_index();
     spatial_index(spatial_index const&);
@@ -53,12 +53,23 @@ private:
     static void read_envelope(InputStream& in, box2d<double>& envelope);
     static void query_node(Filter const& filter, InputStream& in, std::vector<Value> & results);
     static void query_first_n_impl(Filter const& filter, InputStream& in, std::vector<Value> & results, std::size_t count);
+    static bool check_header(InputStream& in);
 };
 
+template <typename Value, typename Filter, typename InputStream>
+bool spatial_index<Value, Filter, InputStream>::check_header(InputStream& in)
+{
+    static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout type");
+    char header[17]; // mapnik-index
+    std::memset(header, 0, 17);
+    in.read(header,16);
+    return (std::strncmp(header, "mapnik-index",12) == 0);
+}
+
 template <typename Value, typename Filter, typename InputStream>
 box2d<double> spatial_index<Value, Filter, InputStream>::bounding_box(InputStream& in)
 {
-    static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout type");
+    if (!check_header(in)) throw std::runtime_error("Invalid index file");
     in.seekg(16 + 4, std::ios::beg);
     box2d<double> box;
     read_envelope(in, box);
@@ -69,7 +80,7 @@ box2d<double> spatial_index<Value, Filter, InputStream>::bounding_box(InputStrea
 template <typename Value, typename Filter, typename InputStream>
 void spatial_index<Value, Filter, InputStream>::query(Filter const& filter, InputStream& in, std::vector<Value>& results)
 {
-    static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout types");
+    if (!check_header(in)) throw std::runtime_error("Invalid index file");
     in.seekg(16, std::ios::beg);
     query_node(filter, in, results);
 }
@@ -104,7 +115,7 @@ void spatial_index<Value, Filter, InputStream>::query_node(Filter const& filter,
 template <typename Value, typename Filter, typename InputStream>
 void spatial_index<Value, Filter, InputStream>::query_first_n(Filter const& filter, InputStream& in, std::vector<Value>& results, std::size_t count)
 {
-    static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout types");
+    if (!check_header(in)) throw std::runtime_error("Invalid index file");
     in.seekg(16, std::ios::beg);
     query_first_n_impl(filter, in, results, count);
 }

plugins/input/shape/shape_index_featureset.cpp

@@ -63,12 +63,13 @@ shape_index_featureset<filterT>::shape_index_featureset(filterT const& filter,
     if (index)
     {
 #if defined(MAPNIK_MEMORY_MAPPED_FILE)
-        mapnik::util::spatial_index<int, filterT,boost::interprocess::ibufferstream>::query(filter, index->file(), offsets_);
+        mapnik::util::spatial_index<mapnik::detail::node, filterT,boost::interprocess::ibufferstream>::query(filter, index->file(), offsets_);
 #else
-        mapnik::util::spatial_index<int, filterT, std::ifstream>::query(filter, index->file(), offsets_);
+        mapnik::util::spatial_index<mapnik::detail::node, filterT, std::ifstream>::query(filter, index->file(), offsets_);
 #endif
     }
-    std::sort(offsets_.begin(), offsets_.end());
+    std::sort(offsets_.begin(), offsets_.end(), [](mapnik::detail::node const& n0, mapnik::detail::node const& n1)
+              {return n0.offset != n1.offset ? n0.offset < n1.offset : n0.start < n1.start;});
     MAPNIK_LOG_DEBUG(shape) << "shape_index_featureset: Query size=" << offsets_.size();
     itr_ = offsets_.begin();
 }
@@ -83,7 +84,15 @@ feature_ptr shape_index_featureset<filterT>::next()
 
     while ( itr_ != offsets_.end())
     {
-        shape_ptr_->move_to(*itr_++);
+        int offset = itr_->offset;
+        shape_ptr_->move_to(offset);
+        std::vector<std::pair<int,int>> parts;
+        while (itr_->offset == offset && itr_ != offsets_.end())
+        {
+            if (itr_->start!= -1) parts.emplace_back(itr_->start, itr_->end);
+            ++itr_;
+        }
+        //std::cerr << "PARTS SIZE=" << parts.size() <<" offset=" << offset << std::endl;
         mapnik::value_integer feature_id = shape_ptr_->id();
         shape_file::record_type record(shape_ptr_->reclength_ * 2);
         shape_ptr_->shp().read_record(record);
@@ -124,7 +133,8 @@ feature_ptr shape_index_featureset<filterT>::next()
         {
             shape_io::read_bbox(record, feature_bbox_);
             if (!filter_.pass(feature_bbox_)) continue;
-            feature->set_geometry(shape_io::read_polyline(record));
+            if (parts.size() < 2) feature->set_geometry(shape_io::read_polyline(record));
+            else feature->set_geometry(shape_io::read_polyline_parts(record, parts));
             break;
         }
         case shape_io::shape_polygon:
@@ -133,7 +143,8 @@ feature_ptr shape_index_featureset<filterT>::next()
         {
             shape_io::read_bbox(record, feature_bbox_);
             if (!filter_.pass(feature_bbox_)) continue;
-            feature->set_geometry(shape_io::read_polygon(record));
+            if (parts.size() < 2) feature->set_geometry(shape_io::read_polygon(record));
+            else feature->set_geometry(shape_io::read_polygon_parts(record, parts));
             break;
         }
         default :

plugins/input/shape/shape_index_featureset.hpp

@@ -45,6 +45,21 @@ using mapnik::box2d;
 using mapnik::feature_ptr;
 using mapnik::context_ptr;
 
+namespace mapnik { namespace detail
+{
+struct node
+{
+    node() = default;
+    node(int offset_, int start_, int end_)
+        : offset(offset_),
+          start(start_),
+          end(end_) {}
+    int offset;
+    int start;
+    int end;
+};
+}} // ns
+
 template <typename filterT>
 class shape_index_featureset : public Featureset
 {
@@ -63,8 +78,8 @@ private:
     context_ptr ctx_;
     std::unique_ptr<shape_io> shape_ptr_;
     const std::unique_ptr<mapnik::transcoder> tr_;
-    std::vector<int> offsets_;
+    std::vector<mapnik::detail::node> offsets_;
-    std::vector<int>::iterator itr_;
+    std::vector<mapnik::detail::node>::iterator itr_;
     std::vector<int> attr_ids_;
     mapnik::value_integer row_limit_;
     mutable int count_;

plugins/input/shape/shape_io.cpp

@@ -125,6 +125,7 @@ mapnik::geometry::geometry<double> shape_io::read_polyline(shape_file::record_ty
         std::for_each(parts.begin(), parts.end(), [&](int & part) { part = record.read_ndr_integer();});
         int start, end;
         mapnik::geometry::multi_line_string<double> multi_line;
+        multi_line.reserve(num_parts);
         for (int k = 0; k < num_parts; ++k)
         {
             start = parts[k];
@@ -152,6 +153,34 @@ mapnik::geometry::geometry<double> shape_io::read_polyline(shape_file::record_ty
     return geom;
 }
 
+mapnik::geometry::geometry<double> shape_io::read_polyline_parts(shape_file::record_type & record, std::vector<std::pair<int, int>> const& parts)
+{
+    mapnik::geometry::geometry<double> geom; // default empty
+    int total_num_parts = record.read_ndr_integer();
+    int num_parts = parts.size();
+    mapnik::geometry::multi_line_string<double> multi_line;
+    multi_line.reserve(num_parts);
+    for (int k = 0; k < num_parts; ++k)
+    {
+        int start = parts[k].first;
+        int end = parts[k].second;
+        unsigned pos = 4 + 32 + 8 + 4 * total_num_parts + start * 16;
+        record.set_pos(pos);
+
+        mapnik::geometry::line_string<double> line;
+        line.reserve(end - start);
+        for (int j = start; j < end; ++j)
+        {
+            double x = record.read_double();
+            double y = record.read_double();
+            line.emplace_back(x, y);
+        }
+        multi_line.push_back(std::move(line));
+    }
+    geom = std::move(multi_line);
+    return geom;
+}
+
 
 mapnik::geometry::geometry<double> shape_io::read_polygon(shape_file::record_type & record)
 {
@@ -207,3 +236,53 @@ mapnik::geometry::geometry<double> shape_io::read_polygon(shape_file::record_typ
     mapnik::geometry::correct(geom);
     return geom;
 }
+
+mapnik::geometry::geometry<double> shape_io::read_polygon_parts(shape_file::record_type & record, std::vector<std::pair<int,int>> const& parts)
+{
+    mapnik::geometry::geometry<double> geom; // default empty
+    int total_num_parts = record.read_ndr_integer();
+    mapnik::geometry::polygon<double> poly;
+    mapnik::geometry::multi_polygon<double> multi_poly;
+    int num_parts = parts.size();
+    for (int k = 0; k < num_parts; ++k)
+    {
+        int start = parts[k].first;
+        int end = parts[k].second;
+        unsigned pos = 4 + 32 + 8 + 4 * total_num_parts + start * 16;
+        record.set_pos(pos);
+        mapnik::geometry::linear_ring<double> ring;
+        ring.reserve(end - start);
+        for (int j = start; j < end; ++j)
+        {
+            double x = record.read_double();
+            double y = record.read_double();
+            ring.emplace_back(x, y);
+        }
+        if (k == 0)
+        {
+            poly.set_exterior_ring(std::move(ring));
+        }
+        else if (mapnik::util::is_clockwise(ring))
+        {
+            multi_poly.emplace_back(std::move(poly));
+            poly.interior_rings.clear();
+            poly.set_exterior_ring(std::move(ring));
+        }
+        else
+        {
+            poly.add_hole(std::move(ring));
+        }
+    }
+
+    if (multi_poly.size() > 0) // multi
+    {
+        multi_poly.emplace_back(std::move(poly));
+        geom = std::move(multi_poly);
+    }
+    else
+    {
+        geom = std::move(poly);
+    }
+    mapnik::geometry::correct(geom);
+    return geom;
+}

plugins/input/shape/shape_io.hpp

@@ -80,6 +80,8 @@ public:
     static void read_bbox(shape_file::record_type & record, mapnik::box2d<double> & bbox);
     static mapnik::geometry::geometry<double> read_polyline(shape_file::record_type & record);
     static mapnik::geometry::geometry<double> read_polygon(shape_file::record_type & record);
+    static mapnik::geometry::geometry<double> read_polyline_parts(shape_file::record_type & record,std::vector<std::pair<int,int>> const& parts);
+    static mapnik::geometry::geometry<double> read_polygon_parts(shape_file::record_type & record, std::vector<std::pair<int,int>> const& parts);
 
     shapeType type_;
     shape_file shp_;

plugins/input/shape/shapefile.hpp

@@ -105,6 +105,11 @@ struct shape_record
         pos += n;
     }
 
+    void set_pos(unsigned pos_)
+    {
+        pos = pos_;
+    }
+
     int read_ndr_integer()
     {
         std::int32_t val;

test/unit/datasource/csv.cpp

@@ -80,7 +80,7 @@ int create_disk_index(std::string const& filename, bool silent = true)
     std::string cmd;
     if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
     {
-        cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
+        cmd += std::string("DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " ";
     }
     cmd += "mapnik-index " + filename;
     if (silent)

test/unit/datasource/geojson.cpp

@@ -66,7 +66,7 @@ int create_disk_index(std::string const& filename, bool silent = true)
     std::string cmd;
     if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
     {
-        cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
+        cmd += std::string("DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " ";
     }
     cmd += "mapnik-index " + filename;
     if (silent)

test/unit/datasource/postgis.cpp

@@ -43,7 +43,7 @@ int run(std::string const& command, bool silent = false)
     std::string cmd;
     if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
     {
-        cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
+        cmd += std::string("DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " ";
     }
     cmd += command;
     if (silent)
@@ -91,4 +91,4 @@ TEST_CASE("postgis") {
 }
 
 
-}
+}

test/unit/datasource/shapeindex.cpp

@@ -58,14 +58,17 @@ std::size_t count_shapefile_features(std::string const& filename)
     return feature_count;
 }
 
-int create_shapefile_index(std::string const& filename, bool silent = true)
+int create_shapefile_index(std::string const& filename, bool index_parts, bool silent = true)
 {
     std::string cmd;
     if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
     {
-        cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
+        cmd += std::string("DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " ";
     }
-    cmd += "shapeindex " + filename;
+
+    cmd += "shapeindex";
+    if (index_parts) cmd+= " --index-parts ";
+    cmd += filename;
     if (silent)
     {
 #ifndef _WINDOWS
@@ -90,28 +93,31 @@ TEST_CASE("shapeindex")
             {
                 if (boost::iends_with(path,".shp"))
                 {
-                    std::string index_path = path.substr(0, path.rfind(".")) + ".index";
+                    for (bool index_parts : {false, true} )
-                    // remove *.index if present
-                    if (mapnik::util::exists(index_path))
                     {
-                        mapnik::util::remove(index_path);
+                        std::string index_path = path.substr(0, path.rfind(".")) + ".index";
-                    }
+                        // remove *.index if present
-                    // count features
+                        if (mapnik::util::exists(index_path))
-                    std::size_t feature_count = count_shapefile_features(path);
+                        {
-                    // create *.index
+                            mapnik::util::remove(index_path);
-                    create_shapefile_index(path);
+                        }
-                    if (feature_count == 0)
+                        // count features
-                    {
+                        std::size_t feature_count = count_shapefile_features(path);
-                        REQUIRE(!mapnik::util::exists(index_path)); // index won't be created if there's no features
+                        // create *.index
-                    }
+                        create_shapefile_index(path, index_parts);
-                    // count features
+                        if (feature_count == 0)
-                    std::size_t feature_count_indexed = count_shapefile_features(path);
+                        {
-                    // ensure number of features are the same
+                            REQUIRE(!mapnik::util::exists(index_path)); // index won't be created if there's no features
-                    REQUIRE(feature_count == feature_count_indexed);
+                        }
-                    // remove *.index if present
+                        // count features
-                    if (mapnik::util::exists(index_path))
+                        std::size_t feature_count_indexed = count_shapefile_features(path);
-                    {
+                        // ensure number of features are the same
-                        mapnik::util::remove(index_path);
+                        REQUIRE(feature_count == feature_count_indexed);
+                        // remove *.index if present
+                        if (mapnik::util::exists(index_path))
+                        {
+                            mapnik::util::remove(index_path);
+                        }
                     }
                 }
             }

utils/shapeindex/shapeindex.cpp

@@ -26,9 +26,10 @@
 #include <string>
 #include <mapnik/util/fs.hpp>
 #include <mapnik/quad_tree.hpp>
+#include <mapnik/geometry_envelope.hpp>
 #include "shapefile.hpp"
 #include "shape_io.hpp"
-
+#include "shape_index_featureset.hpp"
 #pragma GCC diagnostic push
 #include <mapnik/warning_ignore.hpp>
 #include <boost/algorithm/string.hpp>
@@ -44,8 +45,9 @@ int main (int argc,char** argv)
     namespace po = boost::program_options;
 
     bool verbose=false;
-    unsigned int depth=DEFAULT_DEPTH;
+    bool index_parts = false;
-    double ratio=DEFAULT_RATIO;
+    unsigned int depth = DEFAULT_DEPTH;
+    double ratio = DEFAULT_RATIO;
     std::vector<std::string> shape_files;
 
     try
@@ -54,6 +56,7 @@ int main (int argc,char** argv)
         desc.add_options()
             ("help,h", "produce usage message")
             ("version,V","print version string")
+            ("index-parts","index individual shape parts (default: no)")
             ("verbose,v","verbose output")
             ("depth,d", po::value<unsigned int>(), "max tree depth\n(default 8)")
             ("ratio,r",po::value<double>(),"split ratio (default 0.55)")
@@ -81,6 +84,10 @@ int main (int argc,char** argv)
         {
             verbose = true;
         }
+        if (vm.count("index-parts"))
+        {
+            index_parts = true;
+        }
         if (vm.count("depth"))
         {
             depth = vm["depth"].as<unsigned int>();
@@ -159,7 +166,7 @@ int main (int argc,char** argv)
 
         int pos = 50;
         shx.seek(pos * 2);
-        mapnik::quad_tree<int> tree(extent, depth, ratio);
+        mapnik::quad_tree<mapnik::detail::node> tree(extent, depth, ratio);
         int count = 0;
 
         if (shape_type != shape_io::shape_null)
@@ -188,17 +195,56 @@ int main (int argc,char** argv)
                     double y=shp.read_double();
                     item_ext=box2d<double>(x,y,x,y);
                 }
+                else if (index_parts &&
+                         (shape_type == shape_io::shape_polygon || shape_type == shape_io::shape_polygonm || shape_type == shape_io::shape_polygonz
+                          || shape_type == shape_io::shape_polyline || shape_type == shape_io::shape_polylinem || shape_type == shape_io::shape_polylinez))
+                {
+                    shp.read_envelope(item_ext);
+                    int num_parts = shp.read_ndr_integer();
+                    int num_points = shp.read_ndr_integer();
+                    std::vector<int> parts;
+                    parts.resize(num_parts);
+                    std::for_each(parts.begin(), parts.end(), [&](int & part) { part = shp.read_ndr_integer();});
+                    for (int k = 0; k < num_parts; ++k)
+                    {
+                        int start = parts[k];
+                        int end;
+                        if (k == num_parts - 1) end = num_points;
+                        else end = parts[k + 1];
+
+                        mapnik::geometry::linear_ring<double> ring;
+                        ring.reserve(end - start);
+                        for (int j = start; j < end; ++j)
+                        {
+                            double x = shp.read_double();
+                            double y = shp.read_double();
+                            ring.emplace_back(x, y);
+                        }
+                        item_ext = mapnik::geometry::envelope(ring);
+                        if (item_ext.valid())
+                        {
+                            if (verbose)
+                            {
+                                std::clog << "record number " << record_number << " box=" << item_ext << std::endl;
+                            }
+                            tree.insert(mapnik::detail::node(offset * 2, start, end),item_ext);
+                            ++count;
+                        }
+                    }
+                    item_ext = mapnik::box2d<double>(); //invalid
+                }
                 else
                 {
                     shp.read_envelope(item_ext);
                 }
-                if (verbose)
+
-                {
-                    std::clog << "record number " << record_number << " box=" << item_ext << std::endl;
-                }
                 if (item_ext.valid())
                 {
-                    tree.insert(offset * 2,item_ext);
+                    if (verbose)
+                    {
+                        std::clog << "record number " << record_number << " box=" << item_ext << std::endl;
+                    }
+                    tree.insert(mapnik::detail::node(offset * 2,-1,0),item_ext);
                     ++count;
                 }
             }