Merge pull request #3130 from mapnik/smart-geojson

geojson.input - split parsing FeatureCollection and single Feature/Ge…

include/mapnik/attribute_descriptor.hpp

@@ -48,7 +48,7 @@ public:
           type_(type),
           size_(size),
           precision_(precision),
-          primary_key_(primary_key) {}
+            primary_key_(primary_key) {}
 
     attribute_descriptor(attribute_descriptor const& other)
         : name_(other.name_),
@@ -57,21 +57,15 @@ public:
           precision_(other.precision_),
           primary_key_(other.primary_key_) {}
 
-    attribute_descriptor& operator=(attribute_descriptor const& other)
+    attribute_descriptor& operator=(attribute_descriptor rhs)
     {
-        if (this == &other)
+        using std::swap;
-        {
+        std::swap(name_, rhs.name_);
-            return *this;
+        std::swap(type_, rhs.type_);
-        }
+        std::swap(size_, rhs.size_);
-        else
+        std::swap(precision_, rhs.precision_);
-        {
+        std::swap(primary_key_, rhs.primary_key_);
-            name_=other.name_;
+        return *this;
-            type_=other.type_;
-            size_=other.size_;
-            precision_=other.precision_;
-            primary_key_=other.primary_key_;
-            return *this;
-        }
     }
 
     std::string const& get_name() const

include/mapnik/feature_layer_desc.hpp

@@ -30,6 +30,7 @@
 // stl
 #include <iosfwd>
 #include <vector>
+#include <algorithm>
 
 namespace mapnik
 {
@@ -40,13 +41,13 @@ public:
     layer_descriptor(std::string const& name, std::string const& encoding)
         : name_(name),
           encoding_(encoding),
-          desc_ar_(),
+          descriptors_(),
           extra_params_() {}
 
     layer_descriptor(layer_descriptor const& other)
         : name_(other.name_),
           encoding_(other.encoding_),
-          desc_ar_(other.desc_ar_),
+          descriptors_(other.descriptors_),
           extra_params_(other.extra_params_) {}
 
     void set_name(std::string const& name)
@@ -71,17 +72,17 @@ public:
 
     void add_descriptor(attribute_descriptor const& desc)
     {
-        desc_ar_.push_back(desc);
+        descriptors_.push_back(desc);
     }
 
     std::vector<attribute_descriptor> const& get_descriptors() const
     {
-        return desc_ar_;
+        return descriptors_;
     }
 
     std::vector<attribute_descriptor>& get_descriptors()
     {
-        return desc_ar_;
+        return descriptors_;
     }
 
     parameters const& get_extra_parameters() const
@@ -93,11 +94,16 @@ public:
     {
         return extra_params_;
     }
-
+    bool has_name(std::string const& name) const
+    {
+        auto result = std::find_if(std::begin(descriptors_), std::end(descriptors_),
+                                [&name](attribute_descriptor const& desc) { return name == desc.get_name();});
+        return result != std::end(descriptors_);
+    }
 private:
     std::string name_;
     std::string encoding_;
-    std::vector<attribute_descriptor> desc_ar_;
+    std::vector<attribute_descriptor> descriptors_;
     parameters extra_params_;
 };
 

include/mapnik/json/feature_collection_grammar.hpp

@@ -66,19 +66,40 @@ struct feature_collection_grammar :
     feature_collection_grammar(mapnik::transcoder const& tr);
     // grammars
     feature_grammar<Iterator,FeatureType> feature_g;
-    geometry_grammar<Iterator> geometry_g;
+    //geometry_grammar<Iterator> geometry_g;
     // rules
     qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> start; // START
     qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> feature_collection;
     qi::rule<Iterator, space_type> type;
     qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> features;
     qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature;
+    //qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature_from_geometry;
+    // phoenix functions
+    //phoenix::function<json::set_geometry_impl> set_geometry;
+    phoenix::function<apply_feature_callback> on_feature;
+};
+
+template <typename Iterator, typename FeatureType, typename FeatureCallback = default_feature_callback>
+struct feature_grammar_callback :
+        qi::grammar<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback &), space_type>
+{
+    feature_grammar_callback(mapnik::transcoder const& tr);
+    // grammars
+    feature_grammar<Iterator, FeatureType> feature_g;
+    geometry_grammar<Iterator> geometry_g;
+    // rules
+    qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> start; // START
+    //qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> feature_collection;
+    //qi::rule<Iterator, space_type> type;
+    //qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> features;
+    qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature;
     qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature_from_geometry;
     // phoenix functions
     phoenix::function<json::set_geometry_impl> set_geometry;
     phoenix::function<apply_feature_callback> on_feature;
 };
 
+
 }}
 
 #endif // MAPNIK_FEATURE_COLLECTION_GRAMMAR_HPP

include/mapnik/json/feature_collection_grammar_impl.hpp

@@ -38,7 +38,7 @@ feature_collection_grammar<Iterator,FeatureType, FeatureCallback>::feature_colle
 {
         qi::lit_type lit;
         qi::eps_type eps;
-        qi::_1_type _1;
+        //qi::_1_type _1;
         qi::_2_type _2;
         qi::_3_type _3;
         qi::_4_type _4;
@@ -50,7 +50,7 @@ feature_collection_grammar<Iterator,FeatureType, FeatureCallback>::feature_colle
         using phoenix::new_;
         using phoenix::val;
 
-        start = feature_collection(_r1, _r2, _r3) | feature_from_geometry(_r1, _r2, _r3) | feature(_r1, _r2, _r3)
+        start = /*feature_from_geometry(_r1, _r2, _r3) | feature(_r1, _r2, _r3) | */feature_collection(_r1, _r2, _r3)
             ;
 
         feature_collection = lit('{') >> (type | features(_r1, _r2, _r3) | feature_g.json_.key_value) % lit(',') >> lit('}')
@@ -70,14 +70,67 @@ feature_collection_grammar<Iterator,FeatureType, FeatureCallback>::feature_colle
             >> feature_g(*_a)[on_feature(_r3,_a)]
             ;
 
+        //feature_from_geometry =
+        //    eps[_a = phoenix::construct<mapnik::feature_ptr>(new_<mapnik::feature_impl>(_r1, _r2))]
+        //    >> geometry_g[set_geometry(*_a, _1)] [on_feature(_r3, _a)]
+        //    ;
+
+        start.name("start");
+        type.name("type");
+        features.name("features");
+        feature.name("feature");
+        //feature_from_geometry.name("feature-from-geometry");
+        feature_g.name("feature-grammar");
+        //geometry_g.name("geometry-grammar");
+
+        qi::on_error<qi::fail>
+            (
+                start
+                , std::clog
+                << phoenix::val("Error parsing GeoJSON ")
+                << _4
+                << phoenix::val(" here: \"")
+                << construct<std::string>(_3, _2)
+                << phoenix::val('\"')
+                << std::endl
+                );
+}
+
+//
+
+
+template <typename Iterator, typename FeatureType, typename FeatureCallback>
+feature_grammar_callback<Iterator,FeatureType, FeatureCallback>::feature_grammar_callback(mapnik::transcoder const& tr)
+    : feature_grammar_callback::base_type(start,"start"),
+      feature_g(tr)
+{
+        qi::lit_type lit;
+        qi::eps_type eps;
+        qi::_1_type _1;
+        qi::_2_type _2;
+        qi::_3_type _3;
+        qi::_4_type _4;
+        qi::_a_type _a;
+        qi::_r1_type _r1;
+        qi::_r2_type _r2;
+        qi::_r3_type _r3;
+        using phoenix::construct;
+        using phoenix::new_;
+        using phoenix::val;
+
+        start = feature_from_geometry(_r1, _r2, _r3) | feature(_r1, _r2, _r3)
+            ;
+
+        feature = eps[_a = phoenix::construct<mapnik::feature_ptr>(new_<mapnik::feature_impl>(_r1, _r2))]
+            >> feature_g(*_a)[on_feature(_r3,_a)]
+            ;
+
         feature_from_geometry =
             eps[_a = phoenix::construct<mapnik::feature_ptr>(new_<mapnik::feature_impl>(_r1, _r2))]
             >> geometry_g[set_geometry(*_a, _1)] [on_feature(_r3, _a)]
             ;
 
         start.name("start");
-        type.name("type");
-        features.name("features");
         feature.name("feature");
         feature_from_geometry.name("feature-from-geometry");
         feature_g.name("feature-grammar");
@@ -85,7 +138,7 @@ feature_collection_grammar<Iterator,FeatureType, FeatureCallback>::feature_colle
 
         qi::on_error<qi::fail>
             (
-                feature_collection
+                start
                 , std::clog
                 << phoenix::val("Error parsing GeoJSON ")
                 << _4

plugins/input/geojson/build.py

@@ -42,7 +42,9 @@ else:
       """
       %(PLUGIN_NAME)s_datasource.cpp
       %(PLUGIN_NAME)s_featureset.cpp
+      %(PLUGIN_NAME)s_index_featureset.cpp
       large_%(PLUGIN_NAME)s_featureset.cpp
+
       """ % locals()
     )
 

plugins/input/geojson/geojson_datasource.cpp

@@ -22,6 +22,7 @@
 
 #include "geojson_datasource.hpp"
 #include "geojson_featureset.hpp"
+#include "geojson_index_featureset.hpp"
 #include "large_geojson_featureset.hpp"
 #include <fstream>
 #include <algorithm>
@@ -57,6 +58,9 @@
 #include <mapnik/geometry_adapters.hpp>
 #include <mapnik/json/feature_collection_grammar.hpp>
 #include <mapnik/json/extract_bounding_box_grammar_impl.hpp>
+#include <mapnik/util/fs.hpp>
+#include <mapnik/util/spatial_index.hpp>
+#include <mapnik/geom_util.hpp>
 
 #if defined(SHAPE_MEMORY_MAPPED_FILE)
 #pragma GCC diagnostic push
@@ -136,13 +140,19 @@ geojson_datasource::geojson_datasource(parameters const& params)
             filename_ = *base + "/" + *file;
         else
             filename_ = *file;
+        has_disk_index_ = mapnik::util::exists(filename_ + ".index");
     }
+
     if (!inline_string_.empty())
     {
         char const* start = inline_string_.c_str();
         char const* end = start + inline_string_.size();
         parse_geojson(start, end);
     }
+    else if (has_disk_index_)
+    {
+        initialise_disk_index(filename_);
+    }
     else
     {
         cache_features_ = *params.get<mapnik::boolean_type>("cache_features", true);
@@ -192,53 +202,149 @@ namespace {
 using base_iterator_type = char const*;
 const mapnik::transcoder geojson_datasource_static_tr("utf8");
 const mapnik::json::feature_collection_grammar<base_iterator_type,mapnik::feature_impl> geojson_datasource_static_fc_grammar(geojson_datasource_static_tr);
+const mapnik::json::feature_grammar_callback<base_iterator_type,mapnik::feature_impl> geojson_datasource_static_feature_callback_grammar(geojson_datasource_static_tr);
 const mapnik::json::feature_grammar<base_iterator_type, mapnik::feature_impl> geojson_datasource_static_feature_grammar(geojson_datasource_static_tr);
 const mapnik::json::extract_bounding_box_grammar<base_iterator_type> geojson_datasource_static_bbox_grammar;
 }
 
+void geojson_datasource::initialise_disk_index(std::string const& filename)
+{
+    // read extent
+    using value_type = std::pair<std::size_t, std::size_t>;
+    std::ifstream index(filename_ + ".index", std::ios::binary);
+    if (!index) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + ".index'");
+    extent_ = mapnik::util::spatial_index<value_type,
+                                          mapnik::filter_in_box,
+                                          std::ifstream>::bounding_box(index);
+    mapnik::filter_in_box filter(extent_);
+    std::vector<value_type> positions;
+    mapnik::util::spatial_index<value_type,
+                                mapnik::filter_in_box,
+                                std::ifstream>::query_first_n(filter, index, positions, 5);
+
+    mapnik::util::file file(filename_);
+    if (!file.open()) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
+
+    for (auto const& pos : positions)
+    {
+        std::fseek(file.get(), pos.first, SEEK_SET);
+        std::vector<char> record;
+        record.resize(pos.second);
+        std::fread(record.data(), pos.second, 1, file.get());
+        auto const* start = record.data();
+        auto const*  end = start + record.size();
+        mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
+        mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
+        using namespace boost::spirit;
+        standard::space_type space;
+        if (!boost::spirit::qi::phrase_parse(start, end,
+                                             (geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space)
+            || start != end)
+        {
+            throw std::runtime_error("Failed to parse geojson feature");
+        }
+        for ( auto const& kv : *feature)
+        {
+            auto const& name = std::get<0>(kv);
+            if (!desc_.has_name(name))
+            {
+                desc_.add_descriptor(mapnik::attribute_descriptor(name,
+                                                                  mapnik::util::apply_visitor(attr_value_converter(),
+                                                                                              std::get<1>(kv))));
+            }
+        }
+    }
+}
+
 template <typename Iterator>
 void geojson_datasource::initialise_index(Iterator start, Iterator end)
 {
     mapnik::json::boxes_type boxes;
     boost::spirit::standard::space_type space;
     Iterator itr = start;
-    if (!boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_bbox_grammar)(boost::phoenix::ref(boxes)) , space)
+    if (!boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_bbox_grammar)(boost::phoenix::ref(boxes)) , space))
-        || itr != end)
     {
-        throw mapnik::datasource_exception("GeoJSON Plugin: could not parse: '" + filename_ + "'");
+        cache_features_ = true; // force caching single feature
-    }
+        itr = start; // reset iteraror
-    // bulk insert initialise r-tree
+        // try parsing as single Feature or single Geometry JSON
-    tree_ = std::make_unique<spatial_index_type>(boxes);
+        mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
-    // calculate total extent
+        std::size_t start_id = 1;
-    for (auto const& item : boxes)
+        mapnik::json::default_feature_callback callback(features_);
-    {
+        bool result = boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_feature_callback_grammar)
-        auto const& box = std::get<0>(item);
+                                                 (boost::phoenix::ref(ctx),boost::phoenix::ref(start_id), boost::phoenix::ref(callback)),
-        auto const& geometry_index = std::get<1>(item);
+                                                 space);
-        if (!extent_.valid())
+        if (!result || itr != end)
         {
-            extent_ = box;
+            if (!inline_string_.empty()) throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file from in-memory string");
-            // parse first feature to extract attributes schema.
+            else throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file '" + filename_ + "'");
-            // NOTE: this doesn't yield correct answer for geoJSON in general, just an indication
-            Iterator itr2 = start + geometry_index.first;
-            Iterator end2 = itr2 + geometry_index.second;
-            mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
-            mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
-            if (!boost::spirit::qi::phrase_parse(itr2, end2,
-                                                 (geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space)
-                || itr2 != end2)
-            {
-                throw std::runtime_error("Failed to parse geojson feature");
-            }
-            for ( auto const& kv : *feature)
-            {
-                desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
-                                                                  mapnik::util::apply_visitor(attr_value_converter(),
-                                                                                              std::get<1>(kv))));
-            }
         }
-        else
+
+        using values_container = std::vector< std::pair<box_type, std::pair<std::size_t, std::size_t>>>;
+        values_container values;
+        values.reserve(features_.size());
+
+        std::size_t geometry_index = 0;
+        for (mapnik::feature_ptr const& f : features_)
         {
-            extent_.expand_to_include(box);
+            mapnik::box2d<double> box = f->envelope();
+            if (box.valid())
+            {
+                if (geometry_index == 0)
+                {
+                    extent_ = box;
+                    for ( auto const& kv : *f)
+                    {
+                        desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
+                                                                          mapnik::util::apply_visitor(attr_value_converter(),
+                                                                                                      std::get<1>(kv))));
+                    }
+                }
+                else
+                {
+                    extent_.expand_to_include(box);
+                }
+                values.emplace_back(box, std::make_pair(geometry_index,0));
+            }
+            ++geometry_index;
+        }
+        // packing algorithm
+        tree_ = std::make_unique<spatial_index_type>(values);
+    }
+    else
+    {
+        // bulk insert initialise r-tree
+        tree_ = std::make_unique<spatial_index_type>(boxes);
+        // calculate total extent
+        for (auto const& item : boxes)
+        {
+            auto const& box = std::get<0>(item);
+            auto const& geometry_index = std::get<1>(item);
+            if (!extent_.valid())
+            {
+                extent_ = box;
+                // parse first feature to extract attributes schema.
+                // NOTE: this doesn't yield correct answer for geoJSON in general, just an indication
+                Iterator itr2 = start + geometry_index.first;
+                Iterator end2 = itr2 + geometry_index.second;
+                mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
+                mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
+                if (!boost::spirit::qi::phrase_parse(itr2, end2,
+                                                     (geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space)
+                    || itr2 != end2)
+                {
+                    throw std::runtime_error("Failed to parse geojson feature");
+                }
+                for ( auto const& kv : *feature)
+                {
+                    desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
+                                                                      mapnik::util::apply_visitor(attr_value_converter(),
+                                                                                                  std::get<1>(kv))));
+                }
+            }
+            else
+            {
+                extent_.expand_to_include(box);
+            }
         }
     }
 }
@@ -251,16 +357,30 @@ void geojson_datasource::parse_geojson(Iterator start, Iterator end)
     std::size_t start_id = 1;
 
     mapnik::json::default_feature_callback callback(features_);
-
+    Iterator itr = start;
-    bool result = boost::spirit::qi::phrase_parse(start, end, (geojson_datasource_static_fc_grammar)
+    bool result = boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_fc_grammar)
                                                   (boost::phoenix::ref(ctx),boost::phoenix::ref(start_id), boost::phoenix::ref(callback)),
                                                   space);
-    if (!result || start != end)
+    if (!result || itr != end)
     {
         if (!inline_string_.empty()) throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file from in-memory string");
         else throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file '" + filename_ + "'");
     }
 
+    if (features_.size() == 0)
+    {
+        itr = start;
+        // try parsing as single Feature or single Geometry JSON
+        result = boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_feature_callback_grammar)
+                                                 (boost::phoenix::ref(ctx),boost::phoenix::ref(start_id), boost::phoenix::ref(callback)),
+                                                 space);
+        if (!result || itr != end)
+        {
+            if (!inline_string_.empty()) throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file from in-memory string");
+            else throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file '" + filename_ + "'");
+        }
+    }
+
     using values_container = std::vector< std::pair<box_type, std::pair<std::size_t, std::size_t>>>;
     values_container values;
     values.reserve(features_.size());
@@ -320,7 +440,11 @@ boost::optional<mapnik::datasource_geometry_t> geojson_datasource::get_geometry_
 {
     boost::optional<mapnik::datasource_geometry_t> result;
     int multi_type = 0;
-    if (cache_features_)
+    if (has_disk_index_)
+    {
+
+    }
+    else if (cache_features_)
     {
         unsigned num_features = features_.size();
         for (unsigned i = 0; i < num_features && i < 5; ++i)
@@ -411,6 +535,12 @@ mapnik::featureset_ptr geojson_datasource::features(mapnik::query const& q) cons
                 return std::make_shared<large_geojson_featureset>(filename_, std::move(index_array));
             }
         }
+        else if (has_disk_index_)
+        {
+            mapnik::filter_in_box filter(q.get_bbox());
+            return std::make_shared<geojson_index_featureset>(filename_, filter);
+        }
+
     }
     // otherwise return an empty featureset pointer
     return mapnik::featureset_ptr();

plugins/input/geojson/geojson_datasource.hpp

@@ -98,6 +98,7 @@ public:
     void parse_geojson(Iterator start, Iterator end);
     template <typename Iterator>
     void initialise_index(Iterator start, Iterator end);
+    void initialise_disk_index(std::string const& filename);
 private:
     mapnik::datasource::datasource_t type_;
     mapnik::layer_descriptor desc_;
@@ -107,6 +108,7 @@ private:
     std::vector<mapnik::feature_ptr> features_;
     std::unique_ptr<spatial_index_type> tree_;
     bool cache_features_ = true;
+    bool has_disk_index_ = false;
 };
 
 

plugins/input/geojson/geojson_index_featureset.cpp

@@ -0,0 +1,104 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+// mapnik
+#include "geojson_index_featureset.hpp"
+#include <mapnik/feature.hpp>
+#include <mapnik/feature_factory.hpp>
+#include <mapnik/json/geometry_grammar.hpp>
+#include <mapnik/json/feature_grammar.hpp>
+#include <mapnik/util/utf_conv_win.hpp>
+#include <mapnik/util/spatial_index.hpp>
+// stl
+#include <string>
+#include <vector>
+#include <fstream>
+
+geojson_index_featureset::geojson_index_featureset(std::string const& filename, mapnik::filter_in_box const& filter)
+    :
+#if defined(GEOJSON_MEMORY_MAPPED_FILE)
+    //
+#elif defined _WINDOWS
+    file_(_wfopen(mapnik::utf8_to_utf16(filename).c_str(), L"rb"), std::fclose),
+#else
+    file_(std::fopen(filename.c_str(),"rb"), std::fclose),
+#endif
+    ctx_(std::make_shared<mapnik::context_type>())
+{
+
+#if defined (GEOJSON_MEMORY_MAPPED_FILE)
+    boost::optional<mapnik::mapped_region_ptr> memory =
+        mapnik::mapped_memory_cache::instance().find(filename, true);
+    if (memory)
+    {
+        mapped_region_ = *memory;
+    }
+    else
+    {
+        throw std::runtime_error("could not create file mapping for " + filename);
+    }
+#else
+    if (!file_) throw std::runtime_error("Can't open " + filename);
+#endif
+    std::string indexname = filename + ".index";
+    std::ifstream index(indexname.c_str(), std::ios::binary);
+    if (!index) throw mapnik::datasource_exception("GeoJSON Plugin: can't open index file " + indexname);
+    mapnik::util::spatial_index<value_type,
+                                mapnik::filter_in_box,
+                                std::ifstream>::query(filter, index, positions_);
+
+    std::sort(positions_.begin(), positions_.end(),
+              [](value_type const& lhs, value_type const& rhs) { return lhs.first < rhs.first;});
+    itr_ = positions_.begin();
+}
+
+geojson_index_featureset::~geojson_index_featureset() {}
+
+mapnik::feature_ptr geojson_index_featureset::next()
+{
+    while( itr_ != positions_.end())
+    {
+        auto pos = *itr_++;
+#if defined(GEOJSON_MEMORY_MAPPED_FILE)
+        char const* start = (char const*)mapped_region_->get_address() + pos.first;
+        char const*  end = start + pos.second;
+#else
+        std::fseek(file_.get(), pos.first, SEEK_SET);
+        std::vector<char> record;
+        record.resize(pos.second);
+        std::fread(record.data(), pos.second, 1, file_.get());
+        auto const* start = record.data();
+        auto const*  end = start + record.size();
+#endif
+        static const mapnik::transcoder tr("utf8");
+        static const mapnik::json::feature_grammar<char const*, mapnik::feature_impl> grammar(tr);
+        using namespace boost::spirit;
+        standard::space_type space;
+        mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx_,1));
+        if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space) || start != end)
+        {
+            throw std::runtime_error("Failed to parse geojson feature");
+        }
+        return feature;
+    }
+    return mapnik::feature_ptr();
+}

plugins/input/geojson/geojson_index_featureset.hpp

@@ -0,0 +1,66 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef GEOJSON_INDEX_FEATURESET_HPP
+#define GEOJSON_INDEX_FEATURESET_HPP
+
+#define GEOJSON_MEMORY_MAPPED_FILE
+
+#include "geojson_datasource.hpp"
+#include <mapnik/feature.hpp>
+#include <mapnik/geom_util.hpp>
+
+#ifdef GEOJSON_MEMORY_MAPPED_FILE
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wshadow"
+#pragma GCC diagnostic ignored "-Wsign-conversion"
+#include <boost/interprocess/mapped_region.hpp>
+#include <boost/interprocess/streams/bufferstream.hpp>
+#pragma GCC diagnostic pop
+#include <mapnik/mapped_memory_cache.hpp>
+#endif
+
+#include <deque>
+#include <cstdio>
+
+class geojson_index_featureset : public mapnik::Featureset
+{
+    using value_type = std::pair<std::size_t, std::size_t>;
+public:
+    geojson_index_featureset(std::string const& filename, mapnik::filter_in_box const& filter);
+    virtual ~geojson_index_featureset();
+    mapnik::feature_ptr next();
+
+private:
+#if defined (GEOJSON_MEMORY_MAPPED_FILE)
+    using file_source_type = boost::interprocess::ibufferstream;
+    mapnik::mapped_region_ptr mapped_region_;
+#else
+    using file_ptr = std::unique_ptr<std::FILE, int (*)(std::FILE *)>;
+    file_ptr file_;
+#endif
+    mapnik::context_ptr ctx_;
+    std::vector<value_type> positions_;
+    std::vector<value_type>::iterator itr_;
+};
+
+#endif // GEOJSON_INDEX_FEATURESE_HPP

src/json/mapnik_json_feature_collection_grammar.cpp

@@ -26,3 +26,4 @@
 
 using iterator_type = char const*;
 template struct mapnik::json::feature_collection_grammar<iterator_type,mapnik::feature_impl, mapnik::json::default_feature_callback> ;
+template struct mapnik::json::feature_grammar_callback<iterator_type,mapnik::feature_impl, mapnik::json::default_feature_callback> ;

utils/mapnik-index/process_geojson_file.cpp

@@ -52,7 +52,7 @@ std::pair<bool,box2d<double>> process_geojson_file(T & boxes, std::string const&
         mapnik::mapped_memory_cache::instance().find(filename, true);
     if (!memory)
     {
-        std::clog << "Error : cannot mmap " << filename << std::endl;
+        std::clog << "Error : cannot memory map " << filename << std::endl;
         return std::make_pair(false, extent);
     }
     else
@@ -66,7 +66,8 @@ std::pair<bool,box2d<double>> process_geojson_file(T & boxes, std::string const&
     {
         if (!boost::spirit::qi::phrase_parse(start, end, (geojson_datasource_static_bbox_grammar)(boost::phoenix::ref(boxes)) , space))
         {
-            std::clog << "mapnik-index (GeoJSON) : could not parse: '" <<  filename <<  "'";
+            std::clog << "mapnik-index (GeoJSON) : could extract bounding boxes from : '" <<  filename <<  "'";
+            std::clog << " expected FeatureCollection" << std::endl;
             return std::make_pair(false, extent);
         }
     }