Merge branch 'master' into release/image_data_Any

Conflicts:
	benchmark/test_polygon_clipping.cpp

.gitignore

@@ -45,3 +45,4 @@ demo/viewer/ui_about.h
 demo/viewer/ui_info.h
 demo/viewer/ui_layer_info.h
 tests/cpp_tests/*-bin
+tests/cxx/run

benchmark/build.py

@@ -8,6 +8,7 @@ test_env = env.Clone()
 
 test_env['LIBS'] = [env['MAPNIK_NAME']]
 test_env.AppendUnique(LIBS=copy(env['LIBMAPNIK_LIBS']))
+test_env.AppendUnique(LIBS='mapnik-wkt')
 if env['PLATFORM'] == 'Linux':
     test_env.AppendUnique(LIBS='dl')
     test_env.AppendUnique(LIBS='rt')
@@ -34,7 +35,7 @@ benchmarks = [
     #"test_to_double.cpp",
     #"test_to_int.cpp",
     #"test_utf_encoding.cpp"
-    #"test_polygon_clipping.cpp",
+    "test_polygon_clipping.cpp",
     #"test_polygon_clipping_rendering.cpp",
     "test_proj_transform1.cpp",
     "test_expression_parse.cpp",

benchmark/test_polygon_clipping.cpp

@@ -1,13 +1,10 @@
 #include "bench_framework.hpp"
 #include "compare_images.hpp"
-#include "agg_conv_clip_polygon.h"
 #include <mapnik/geometry.hpp>
 #include <mapnik/vertex.hpp>
 #include <mapnik/transform_path_adapter.hpp>
 #include <mapnik/view_transform.hpp>
 #include <mapnik/wkt/wkt_factory.hpp>
-#include <mapnik/wkt/wkt_grammar_impl.hpp>
-#include <mapnik/geometry.hpp>
 #include <mapnik/projection.hpp>
 #include <mapnik/proj_transform.hpp>
 #include <mapnik/util/fs.hpp>
@@ -28,14 +25,17 @@
 
 // stl
 #include <fstream>
+#include <iostream>
+#include <cstdlib>
 
-void render(mapnik::geometry_type & geom,
+void render(mapnik::geometry_type const& geom,
             mapnik::box2d<double> const& extent,
             std::string const& name)
 {
-    using path_type = mapnik::transform_path_adapter<mapnik::view_transform,mapnik::geometry_type>;
+    using path_type = mapnik::transform_path_adapter<mapnik::view_transform,mapnik::vertex_adapter>;
     using ren_base = agg::renderer_base<agg::pixfmt_rgba32_plain>;
     using renderer = agg::renderer_scanline_aa_solid<ren_base>;
+    mapnik::vertex_adapter va(geom);
     mapnik::image_rgba8 im(256,256);
     mapnik::fill(im, mapnik::color("white"));
     mapnik::box2d<double> padded_extent = extent;
@@ -48,13 +48,11 @@ void render(mapnik::geometry_type & geom,
     ren.color(agg::rgba8(127,127,127,255));
     agg::rasterizer_scanline_aa<> ras;
     mapnik::proj_transform prj_trans(mapnik::projection("+init=epsg:4326"),mapnik::projection("+init=epsg:4326"));
-    geom.rewind(0);
+    path_type path(tr,va,prj_trans);
-    path_type path(tr,geom,prj_trans);
     ras.add_path(path);
     agg::scanline_u8 sl;
     agg::render_scanlines(ras, sl, ren);
     mapnik::save_to_file(im,name);
-    geom.rewind(0);
 }
 
 class test1 : public benchmark::test_case
@@ -63,7 +61,7 @@ class test1 : public benchmark::test_case
     mapnik::box2d<double> extent_;
     std::string expected_;
 public:
-    using conv_clip = agg::conv_clip_polygon<mapnik::geometry_type>;
+    using conv_clip = agg::conv_clip_polygon<mapnik::vertex_adapter>;
     test1(mapnik::parameters const& params,
           std::string const& wkt_in,
           mapnik::box2d<double> const& extent)
@@ -84,8 +82,9 @@ public:
             std::clog << "paths.size() != 1\n";
             return false;
         }
-        mapnik::geometry_type & geom = paths[0];
+        mapnik::geometry_type const& geom = paths[0];
-        conv_clip clipped(geom);
+        mapnik::vertex_adapter va(geom);
+        conv_clip clipped(va);
         clipped.clip_box(
                     extent_.minx(),
                     extent_.miny(),
@@ -99,12 +98,13 @@ public:
         }
         std::string expect = expected_+".png";
         std::string actual = expected_+"_actual.png";
-        if (!mapnik::util::exists(expect))
+        auto env = mapnik::envelope(geom);
+        if (!mapnik::util::exists(expect) || (std::getenv("UPDATE") != nullptr))
         {
             std::clog << "generating expected image: " << expect << "\n";
-            render(geom2,geom.envelope(),expect);
+            render(geom2,env,expect);
         }
-        render(geom2,geom.envelope(),actual);
+        render(geom2,env,actual);
         return benchmark::compare_images(actual,expect);
     }
     bool operator()() const
@@ -114,11 +114,13 @@ public:
         {
             throw std::runtime_error("Failed to parse WKT");
         }
+        unsigned count = 0;
         for (unsigned i=0;i<iterations_;++i)
         {
-            for (mapnik::geometry_type & geom : paths)
+            for (mapnik::geometry_type const& geom : paths)
             {
-                conv_clip clipped(geom);
+                mapnik::vertex_adapter va(geom);
+                conv_clip clipped(va);
                 clipped.clip_box(
                             extent_.minx(),
                             extent_.miny(),
@@ -126,10 +128,18 @@ public:
                             extent_.maxy());
                 unsigned cmd;
                 double x,y;
-                while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {}
+                while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {
+                    count++;
                 }
             }
-        return true;
+        }
+        // TODO - sometimes this is 310001: what is causing that?
+        unsigned expected_count = 310002;
+        bool valid = (count == expected_count);
+        if (!valid) {
+            std::clog << "test1: clipping failed: processed " << count << " verticies but expected " << expected_count << "\n";
+        }
+        return valid;
     }
 };
 
@@ -139,7 +149,7 @@ class test2 : public benchmark::test_case
     mapnik::box2d<double> extent_;
     std::string expected_;
 public:
-    using poly_clipper = agg::conv_clipper<mapnik::geometry_type, agg::path_storage>;
+    using poly_clipper = agg::conv_clipper<mapnik::vertex_adapter, agg::path_storage>;
     test2(mapnik::parameters const& params,
           std::string const& wkt_in,
           mapnik::box2d<double> const& extent)
@@ -166,8 +176,9 @@ public:
             std::clog << "paths.size() != 1\n";
             return false;
         }
-        mapnik::geometry_type & geom = paths[0];
+        mapnik::geometry_type const& geom = paths[0];
-        poly_clipper clipped(geom,ps,
+        mapnik::vertex_adapter va(geom);
+        poly_clipper clipped(va,ps,
                              agg::clipper_and,
                              agg::clipper_non_zero,
                              agg::clipper_non_zero,
@@ -181,12 +192,13 @@ public:
         }
         std::string expect = expected_+".png";
         std::string actual = expected_+"_actual.png";
-        if (!mapnik::util::exists(expect))
+        auto env = mapnik::envelope(geom);
+        if (!mapnik::util::exists(expect) || (std::getenv("UPDATE") != nullptr))
         {
             std::clog << "generating expected image: " << expect << "\n";
-            render(geom2,geom.envelope(),expect);
+            render(geom2,env,expect);
         }
-        render(geom2,geom.envelope(),actual);
+        render(geom2,env,actual);
         return benchmark::compare_images(actual,expect);
     }
     bool operator()() const
@@ -202,11 +214,13 @@ public:
         ps.line_to(extent_.maxx(), extent_.maxy());
         ps.line_to(extent_.maxx(), extent_.miny());
         ps.close_polygon();
+        unsigned count = 0;
         for (unsigned i=0;i<iterations_;++i)
         {
-            for (mapnik::geometry_type & geom : paths)
+            for (mapnik::geometry_type const& geom : paths)
             {
-                poly_clipper clipped(geom,ps,
+                mapnik::vertex_adapter va(geom);
+                poly_clipper clipped(va,ps,
                                      agg::clipper_and,
                                      agg::clipper_non_zero,
                                      agg::clipper_non_zero,
@@ -214,10 +228,17 @@ public:
                 clipped.rewind(0);
                 unsigned cmd;
                 double x,y;
-                while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {}
+                while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {
+                    count++;
                 }
             }
-        return true;
+        }
+        unsigned expected_count = 290000;
+        bool valid = (count == expected_count);
+        if (!valid) {
+            std::clog << "test2: clipping failed: processed " << count << " verticies but expected " << expected_count << "\n";
+        }
+        return valid;
     }
 };
 
@@ -227,7 +248,7 @@ class test3 : public benchmark::test_case
     mapnik::box2d<double> extent_;
     std::string expected_;
 public:
-    using poly_clipper = mapnik::polygon_clipper<mapnik::geometry_type>;
+    using poly_clipper = mapnik::polygon_clipper<mapnik::vertex_adapter>;
     test3(mapnik::parameters const& params,
           std::string const& wkt_in,
           mapnik::box2d<double> const& extent)
@@ -248,8 +269,9 @@ public:
             std::clog << "paths.size() != 1\n";
             return false;
         }
-        mapnik::geometry_type & geom = paths[0];
+        mapnik::geometry_type const& geom = paths[0];
-        poly_clipper clipped(extent_, geom);
+        mapnik::vertex_adapter va(geom);
+        poly_clipper clipped(extent_, va);
         unsigned cmd;
         double x,y;
         mapnik::geometry_type geom2(mapnik::geometry_type::types::Polygon);
@@ -258,12 +280,13 @@ public:
         }
         std::string expect = expected_+".png";
         std::string actual = expected_+"_actual.png";
-        if (!mapnik::util::exists(expect))
+        auto env = mapnik::envelope(geom);
+        if (!mapnik::util::exists(expect) || (std::getenv("UPDATE") != nullptr))
         {
             std::clog << "generating expected image: " << expect << "\n";
-            render(geom2,geom.envelope(),expect);
+            render(geom2,env,expect);
         }
-        render(geom2,geom.envelope(),actual);
+        render(geom2,env,actual);
         return benchmark::compare_images(actual,expect);
     }
     bool operator()() const
@@ -273,17 +296,26 @@ public:
         {
             throw std::runtime_error("Failed to parse WKT");
         }
+        unsigned count = 0;
         for (unsigned i=0;i<iterations_;++i)
         {
-            for ( mapnik::geometry_type & geom : paths)
+            for ( mapnik::geometry_type const& geom : paths)
             {
-                poly_clipper clipped(extent_, geom);
+                mapnik::vertex_adapter va(geom);
+                poly_clipper clipped(extent_, va);
                 unsigned cmd;
                 double x,y;
-                while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {}
+                while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {
+                    count++;
                 }
             }
-        return true;
+        }
+        unsigned expected_count = 310000;
+        bool valid = (count == expected_count);
+        if (!valid) {
+            std::clog << "test3: clipping failed: processed " << count << " verticies but expected " << expected_count << "\n";
+        }
+        return valid;
     }
 };
 

include/mapnik/json/error_handler.hpp

@@ -34,7 +34,7 @@ struct error_handler
 {
     using result_type = void;
     void operator() (
-        Iterator, Iterator last,
+        Iterator, Iterator,
         Iterator err_pos, boost::spirit::info const& what) const
     {
         std::stringstream s;

include/mapnik/polygon_clipper.hpp

@@ -67,7 +67,9 @@ struct polygon_clipper
     polygon_clipper(box2d<double> const& clip_box, Geometry & geom)
         :state_(clip),
          clip_box_(clip_box),
-         geom_(geom)
+         geom_(geom),
+         output_(),
+         output_adapter_(output_)
     {
         init();
     }
@@ -86,7 +88,7 @@ struct polygon_clipper
 
     void rewind(unsigned path_id)
     {
-        if (state_ == clip) output_.rewind(path_id);
+        if (state_ == clip) output_adapter_.rewind(path_id);
         else geom_.rewind(path_id);
     }
 
@@ -95,7 +97,7 @@ struct polygon_clipper
         switch (state_)
         {
         case clip:
-            return output_.vertex(x,y);
+            return output_adapter_.vertex(x,y);
         case no_clip:
             return geom_.vertex(x,y);
         case ignore:
@@ -230,6 +232,7 @@ private:
     box2d<double> clip_box_;
     Geometry & geom_;
     mapnik::geometry_type output_;
+    mapnik::vertex_adapter output_adapter_;
 
 };
 

include/mapnik/sql_utils.hpp

@@ -73,7 +73,7 @@ namespace mapnik { namespace sql_utils {
             {
                 table_name=table_name.substr(idx);
             }
-            idx = table_name.find_first_of(" )");
+            idx = table_name.find_first_of(", )");
             if (idx != std::string::npos)
             {
                 table_name = table_name.substr(0,idx);

include/mapnik/util/path_iterator.hpp

@@ -80,7 +80,7 @@ private:
     const path_type *vertices_;
 };
 
-// specialization for mapnik::geometry_type - vertex interfacce has been removed
+// specialization for mapnik::geometry_type - vertex interface has been removed
 template <>
 class path_iterator<geometry_type>
     : public boost::iterator_facade< path_iterator<geometry_type>,
@@ -95,7 +95,8 @@ public:
     using size_type = path_type::size_type;
     path_iterator()
         : v_(mapnik::SEG_END,0,0),
-          vertices_()
+          vertices_(),
+          pos_(0)
     {}
 
     explicit path_iterator(path_type const& vertices)

plugins/input/geojson/geojson_datasource.cpp

@@ -292,6 +292,8 @@ boost::optional<mapnik::datasource::geometry_t> geojson_datasource::get_geometry
 {
     boost::optional<mapnik::datasource::geometry_t> result;
     int multi_type = 0;
+    if (cache_features_)
+    {
         unsigned num_features = features_.size();
         for (unsigned i = 0; i < num_features && i < 5; ++i)
         {
@@ -307,6 +309,54 @@ boost::optional<mapnik::datasource::geometry_t> geojson_datasource::get_geometry
                 multi_type = type;
             }
         }
+    }
+    else
+    {
+        mapnik::util::file file(filename_);
+        if (!file.open())
+        {
+            throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
+        }
+        auto itr = tree_->qbegin(boost::geometry::index::intersects(extent_));
+        auto end = tree_->qend();
+        mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
+        for (std::size_t count = 0; itr !=end &&  count < 5; ++itr,++count)
+        {
+            geojson_datasource::item_type const& item = *itr;
+            std::size_t file_offset = item.second.first;
+            std::size_t size = item.second.second;
+
+            std::fseek(file.get(), file_offset, SEEK_SET);
+            std::vector<char> json;
+            json.resize(size);
+            std::fread(json.data(), size, 1, file.get());
+
+            using chr_iterator_type = char const*;
+            chr_iterator_type start = json.data();
+            chr_iterator_type end = start + json.size();
+
+            static const mapnik::transcoder tr("utf8");
+            static const mapnik::json::feature_grammar<chr_iterator_type,mapnik::feature_impl> grammar(tr);
+            using namespace boost::spirit;
+            ascii::space_type space;
+            mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
+            if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space))
+            {
+                throw std::runtime_error("Failed to parse geojson feature");
+            }
+            mapnik::util::to_ds_type(feature->paths(),result);
+            if (result)
+            {
+                int type = static_cast<int>(*result);
+                if (multi_type > 0 && multi_type != type)
+                {
+                    result.reset(mapnik::datasource::Collection);
+                    return result;
+                }
+                multi_type = type;
+            }
+        }
+    }
     return result;
 }
 

plugins/input/geojson/large_geojson_featureset.cpp

@@ -23,8 +23,8 @@
 // mapnik
 #include <mapnik/feature.hpp>
 #include <mapnik/feature_factory.hpp>
-#include <mapnik/json/geometry_grammar_impl.hpp>
+#include <mapnik/json/geometry_grammar.hpp>
-#include <mapnik/json/feature_grammar_impl.hpp>
+#include <mapnik/json/feature_grammar.hpp>
 #include <mapnik/utils.hpp>
 // stl
 #include <string>
@@ -62,9 +62,10 @@ mapnik::feature_ptr large_geojson_featureset::next()
         std::vector<char> json;
         json.resize(size);
         std::fread(json.data(), size, 1, file_.get());
-        using chr_iterator_type = std::vector<char>::const_iterator;
+
-        chr_iterator_type start = json.begin();
+        using chr_iterator_type = char const*;
-        chr_iterator_type end = json.end();
+        chr_iterator_type start = json.data();
+        chr_iterator_type end = start + json.size();
 
         static const mapnik::transcoder tr("utf8");
         static const mapnik::json::feature_grammar<chr_iterator_type,mapnik::feature_impl> grammar(tr);

tests/cxx/sql_parse.cpp

@@ -0,0 +1,33 @@
+
+#include "catch.hpp"
+
+#include <mapnik/sql_utils.hpp>
+
+TEST_CASE("sql parse") {
+
+SECTION("table") {
+    std::string subquery("table");
+    REQUIRE( subquery == mapnik::sql_utils::table_from_sql(subquery) );
+}
+
+SECTION("complex sql 1") {
+    std::string subquery("(select * FROM table1, table2) AS data");
+    REQUIRE( "table1" == mapnik::sql_utils::table_from_sql(subquery) );
+}
+
+SECTION("complex sql 2") {
+    std::string subquery("(select * FROM table1 , table2) AS data");
+    REQUIRE( "table1" == mapnik::sql_utils::table_from_sql(subquery) );
+}
+
+SECTION("complex sql 3") {
+    std::string subquery("(select * FROM table1,table2) AS data");
+    REQUIRE( "table1" == mapnik::sql_utils::table_from_sql(subquery) );
+}
+
+SECTION("complex sql 4") {
+    std::string subquery("(select * FROM table1) AS data");
+    REQUIRE( "table1" == mapnik::sql_utils::table_from_sql(subquery) );
+}
+
+}

tests/data/shp/3d_point_empty.prj

@@ -0,0 +1 @@
+PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.017453292519943295]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]]

tests/python_tests/geojson_plugin_test.py

@@ -4,6 +4,10 @@
 from nose.tools import eq_,assert_almost_equal
 from utilities import execution_path, run_all
 import os, mapnik
+try:
+    import json
+except ImportError:
+    import simplejson as json
 
 def setup():
     # All of the paths used are relative, if we run the tests
@@ -35,6 +39,12 @@ if 'geojson' in mapnik.DatasourceCache.plugin_names():
         eq_(f['boolean'], True)
         eq_(f['NOM_FR'], u'Qu\xe9bec')
         eq_(f['NOM_FR'], u'Québec')
+        eq_(f['array'], u'[[[1],["deux"]],[["\\u0442\\u0440\\u0438","four","\\u4e94"]]]')
+        array = json.loads(f['array'])
+        eq_(array,[[[1], [u'deux']], [[u'\u0442\u0440\u0438', u'four', u'\u4e94']]])
+        eq_(f['object'], u'{"value":{"type":"\\u041c\\u0430pni\\u043a","array":[3,0,"x"]}}')
+        object = json.loads(f['object'])
+        eq_(object,{u'value': {u'array': [3, 0, u'x'], u'type': u'\u041c\u0430pni\u043a'}})
 
         ds = mapnik.Datasource(type='geojson',file='../data/json/escaped.geojson')
         f = ds.all_features()[0]
@@ -51,6 +61,56 @@ if 'geojson' in mapnik.DatasourceCache.plugin_names():
         eq_(f['boolean'], True)
         eq_(f['NOM_FR'], u'Qu\xe9bec')
         eq_(f['NOM_FR'], u'Québec')
+        eq_(f['array'], u'[[[1],["deux"]],[["\\u0442\\u0440\\u0438","four","\\u4e94"]]]')
+        array = json.loads(f['array'])
+        eq_(array,[[[1], [u'deux']], [[u'\u0442\u0440\u0438', u'four', u'\u4e94']]])
+        eq_(f['object'], u'{"value":{"type":"\\u041c\\u0430pni\\u043a","array":[3,0,"x"]}}')
+        object = json.loads(f['object'])
+        eq_(object,{u'value': {u'array': [3, 0, u'x'], u'type': u'\u041c\u0430pni\u043a'}})
+
+    def test_large_geojson_properties():
+        ds = mapnik.Datasource(type='geojson',file='../data/json/escaped.geojson',cache_features = False)
+        f = ds.features_at_point(ds.envelope().center()).features[0]
+        eq_(len(ds.fields()),9)
+        desc = ds.describe()
+        eq_(desc['geometry_type'],mapnik.DataGeometryType.Point)
+
+        eq_(f['name'], u'Test')
+        eq_(f['int'], 1)
+        eq_(f['description'], u'Test: \u005C')
+        eq_(f['spaces'], u'this has spaces')
+        eq_(f['double'], 1.1)
+        eq_(f['boolean'], True)
+        eq_(f['NOM_FR'], u'Qu\xe9bec')
+        eq_(f['NOM_FR'], u'Québec')
+        eq_(f['array'], u'[[[1],["deux"]],[["\\u0442\\u0440\\u0438","four","\\u4e94"]]]')
+        array = json.loads(f['array'])
+        eq_(array,[[[1], [u'deux']], [[u'\u0442\u0440\u0438', u'four', u'\u4e94']]])
+        eq_(f['object'], u'{"value":{"type":"\\u041c\\u0430pni\\u043a","array":[3,0,"x"]}}')
+        object = json.loads(f['object'])
+        eq_(object,{u'value': {u'array': [3, 0, u'x'], u'type': u'\u041c\u0430pni\u043a'}})
+
+        ds = mapnik.Datasource(type='geojson',file='../data/json/escaped.geojson')
+        f = ds.all_features()[0]
+        eq_(len(ds.fields()),9)
+
+        desc = ds.describe()
+        eq_(desc['geometry_type'],mapnik.DataGeometryType.Point)
+
+        eq_(f['name'], u'Test')
+        eq_(f['int'], 1)
+        eq_(f['description'], u'Test: \u005C')
+        eq_(f['spaces'], u'this has spaces')
+        eq_(f['double'], 1.1)
+        eq_(f['boolean'], True)
+        eq_(f['NOM_FR'], u'Qu\xe9bec')
+        eq_(f['NOM_FR'], u'Québec')
+        eq_(f['array'], u'[[[1],["deux"]],[["\\u0442\\u0440\\u0438","four","\\u4e94"]]]')
+        array = json.loads(f['array'])
+        eq_(array,[[[1], [u'deux']], [[u'\u0442\u0440\u0438', u'four', u'\u4e94']]])
+        eq_(f['object'], u'{"value":{"type":"\\u041c\\u0430pni\\u043a","array":[3,0,"x"]}}')
+        object = json.loads(f['object'])
+        eq_(object,{u'value': {u'array': [3, 0, u'x'], u'type': u'\u041c\u0430pni\u043a'}})
 
     def test_geojson_from_in_memory_string():
         # will silently fail since it is a geometry and needs to be a featurecollection.

tests/python_tests/shapeindex_test.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+from nose.tools import eq_
+from utilities import execution_path, run_all
+from subprocess import Popen, PIPE
+import shutil
+import os
+import fnmatch
+import mapnik
+
+def setup():
+    # All of the paths used are relative, if we run the tests
+    # from another directory we need to chdir()
+    os.chdir(execution_path('.'))
+
+def test_shapeindex():
+    # first copy shapefiles to tmp directory
+    source_dir = '../data/shp/'
+    working_dir = '/tmp/mapnik-shp-tmp/'
+    if os.path.exists(working_dir):
+      shutil.rmtree(working_dir)
+    shutil.copytree(source_dir,working_dir)
+    matches = []
+    for root, dirnames, filenames in os.walk('%s' % source_dir):
+      for filename in fnmatch.filter(filenames, '*.shp'):
+          matches.append(os.path.join(root, filename))
+    for shp in matches:
+      source_file = os.path.join(source_dir,os.path.relpath(shp,source_dir))
+      dest_file = os.path.join(working_dir,os.path.relpath(shp,source_dir))
+      ds = mapnik.Shapefile(file=source_file)
+      count = 0;
+      fs = ds.featureset()
+      try:
+        while (fs.next()):
+          count = count+1
+      except StopIteration:
+        pass
+      stdin, stderr = Popen('shapeindex %s' % dest_file, shell=True, stdout=PIPE, stderr=PIPE).communicate()
+      ds2 = mapnik.Shapefile(file=dest_file)
+      count2 = 0;
+      fs = ds.featureset()
+      try:
+        while (fs.next()):
+          count2 = count2+1
+      except StopIteration:
+        pass
+      eq_(count,count2)
+
+if __name__ == "__main__":
+    setup()
+    exit(run_all(eval(x) for x in dir() if x.startswith("test_")))

utils/shapeindex/shapeindex.cpp

@@ -164,18 +164,24 @@ int main (int argc,char** argv)
             shape_type = shp.read_ndr_integer();
             box2d<double> item_ext;
             if (shape_type==shape_io::shape_null)
+            {
+                if (pos >= file_length)
+                {
+                    break;
+                }
+                else
                 {
                     // still need to increment pos, or the pos counter
                     // won't indicate EOF until too late.
                     pos+=4+content_length;
                     continue;
                 }
+            }
             else if (shape_type==shape_io::shape_point)
             {
                 double x=shp.read_double();
                 double y=shp.read_double();
                 item_ext=box2d<double>(x,y,x,y);
-
             }
             else if (shape_type==shape_io::shape_pointm)
             {
@@ -184,7 +190,6 @@ int main (int argc,char** argv)
                 // skip m
                 shp.read_double();
                 item_ext=box2d<double>(x,y,x,y);
-
             }
             else if (shape_type==shape_io::shape_pointz)
             {
@@ -192,31 +197,39 @@ int main (int argc,char** argv)
                 double y=shp.read_double();
                 // skip z
                 shp.read_double();
-                //skip m if exists
+                // According to ESRI shapefile doc
-                if ( content_length == 8 + 36)
+                // A PointZ consists of a triplet of double-precision coordinates in the order X, Y, Z plus a
+                // measure.
+                // PointZ
+                // {
+                //     Double X // X coordinate
+                //     Double Y // Y coordinate
+                //     Double Z // Z coordinate
+                //     Double M // Measure
+                // }
+                // But OGR creates shapefiles with M missing so we need skip M only if present
+                // NOTE: content_length is in 16-bit words
+                if ( content_length == 18)
                 {
                     shp.read_double();
                 }
                 item_ext=box2d<double>(x,y,x,y);
             }
-
             else
             {
                 shp.read_envelope(item_ext);
                 shp.skip(2*content_length-4*8-4);
             }
-
             tree.insert(offset,item_ext);
-            if (verbose) {
+            if (verbose)
+            {
                 clog << "record number " << record_number << " box=" << item_ext << endl;
             }
 
             pos+=4+content_length;
             ++count;
 
-            if (pos>=file_length) {
+            if (pos >= file_length) break;
-                break;
-            }
         }
 
         clog << " number shapes=" << count << endl;