steps toward a generic spatial indexing api

bindings/python/mapnik_datasource.cpp

@@ -174,6 +174,8 @@ void export_datasource()
         .def("encoding",&encoding) //todo expose as property
         .def("name",&name)
         .def("features_at_point",&datasource::features_at_point)
+        .def("index_on",&datasource::index_on)
+        .def("create_index",&datasource::create_index)
         .def("params",&datasource::params,return_value_policy<copy_const_reference>(), 
              "The configuration parameters of the data source. "  
              "These vary depending on the type of data source.")

include/mapnik/datasource.hpp

@@ -91,6 +91,11 @@ public:
         return params_;
     }
 
+    void set_param(parameter const& p)
+    {
+        params_[p.first] = p.second;
+    }
+        
     /*!
      * @brief Get the type of the datasource
      * @return The type of the datasource (Vector or Raster)
@@ -106,6 +111,9 @@ public:
     virtual featureset_ptr features_at_point(coord2d const& pt) const=0;
     virtual box2d<double> envelope() const=0;
     virtual layer_descriptor get_descriptor() const=0;
+    virtual bool index_on(std::string const& field) const=0;
+    virtual bool create_index(std::string const& field, mapnik::parameters const& params) const=0;
+    virtual std::string geometry_field() const=0;
     virtual ~datasource() {};
 protected:
     parameters params_;

include/mapnik/memory_datasource.hpp

@@ -44,6 +44,9 @@ public:
     featureset_ptr features_at_point(coord2d const& pt) const;
     box2d<double> envelope() const;
     layer_descriptor get_descriptor() const;
+    bool index_on(std::string const& field) const { return false; }
+    bool create_index(std::string const& field, mapnik::parameters const& params) const { return false; }
+    std::string geometry_field() const { return ""; }
     size_t size() const;
 private:
     std::vector<feature_ptr> features_;

plugins/input/csv/csv_datasource.hpp

@@ -9,7 +9,7 @@
 
 class csv_datasource : public mapnik::datasource 
 {
-   public:
+public:
     csv_datasource(mapnik::parameters const& params, bool bind=true);
     virtual ~csv_datasource ();
     int type() const;
@@ -18,13 +18,16 @@ class csv_datasource : public mapnik::datasource
     mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const;
     mapnik::box2d<double> envelope() const;
     mapnik::layer_descriptor get_descriptor() const;
+    bool index_on(std::string field) const { return false };
+    bool create_index(std::string const& field, mapnik::parameters const& params) const { return false; }
+    std::string geometry_field() const { return ""; }
     void bind() const;
     template <typename T>
     void parse_csv(T& stream,
        std::string const& escape,
        std::string const& separator,
        std::string const& quote) const;
-   private:
+private:
     mutable mapnik::layer_descriptor desc_;
     mutable mapnik::box2d<double> extent_;
     mutable std::string filename_;

plugins/input/postgis/postgis_datasource.cpp

@@ -357,6 +357,7 @@ std::string postgis_datasource::sql_bbox(box2d<double> const& env) const
     std::ostringstream b;
     if (srid_ > 0)
         b << "SetSRID(";
+    //ST_MakeBox2D(ST_Point(),ST_Point())
     b << "'BOX3D(";
     b << std::setprecision(16);
     b << env.minx() << " " << env.miny() << ",";

plugins/input/shape/shape_datasource.hpp

@@ -51,6 +51,9 @@ public:
     featureset_ptr features_at_point(coord2d const& pt) const;
     box2d<double> envelope() const;
     layer_descriptor get_descriptor() const;
+    bool index_on(std::string field) const { return false };
+    bool create_index(std::string const& field, mapnik::parameters const& params) const { return false; }
+    std::string geometry_field() const { return ""; }
     void bind() const;   
 private:
     shape_datasource(const shape_datasource&);

plugins/input/sqlite/sqlite_datasource.cpp

@@ -67,7 +67,6 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind)
 {
     /* TODO
       - throw if no primary key but spatial index is present?
-      - remove auto-indexing
     */
     
     boost::optional<std::string> file = params_.get<std::string>("file");
@@ -84,6 +83,52 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind)
     }
 }
 
+bool sqlite_datasource::index_on(std::string const& field) const
+{
+    if (! is_bound_) bind();
+
+    if (field == geometry_field_)
+    {
+        if (use_spatial_index_ && !index_table_.empty())
+        {
+            std::string index_db = sqlite_utils::index_for_db(dataset_name_);
+            if (boost::filesystem::exists(index_db))
+            {
+                dataset_->execute("attach database '" + index_db + "' as " + index_table_);
+            }
+            
+            return sqlite_utils::has_rtree(index_table_,dataset_);
+        }
+    }
+    // else check or attribute index?
+    return false;
+}
+
+bool sqlite_datasource::create_index(std::string const& field, mapnik::parameters const& params) const
+{
+    if (! is_bound_) bind();
+    if (field == geometry_field_)
+    {
+        if (!key_field_.empty() && !index_table_.empty())
+        {
+            std::string index_db = sqlite_utils::index_for_db(dataset_name_);
+            std::ostringstream query;
+            query << "SELECT " 
+                  << geometry_field_
+                  << "," << key_field_
+                  << " FROM ("
+                  << geometry_table_ << ")";
+            boost::shared_ptr<sqlite_resultset> rs = dataset_->execute_query(query.str());
+            sqlite_utils::create_rtree(index_db,index_table_,rs);
+            use_spatial_index_ = sqlite_utils::has_rtree(index_table_,dataset_);
+            return use_spatial_index_;
+        }
+    }
+    // else create attribute index
+    return false;
+
+}
+
 void sqlite_datasource::bind() const
 {
     if (is_bound_) return;
@@ -105,9 +150,6 @@ void sqlite_datasource::bind() const
     multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries", false);
     use_spatial_index_ = *params_.get<mapnik::boolean>("use_spatial_index", true);
 
-    // TODO - remove this option once all datasources have an indexing api
-    bool auto_index = *params_.get<mapnik::boolean>("auto_index", true);
-
     boost::optional<std::string> ext  = params_.get<std::string>("extent");
     if (ext) extent_initialized_ = extent_.from_string(*ext);
         
@@ -232,34 +274,6 @@ void sqlite_datasource::bind() const
         has_spatial_index_ = sqlite_utils::has_rtree(index_table_,dataset_);
     }
 
-
-    if (! extent_initialized_ 
-        && !has_spatial_index_
-        && auto_index)
-    {
-            if (! key_field_.empty())
-            {
-                std::ostringstream query;
-                query << "SELECT " 
-                      << geometry_field_
-                      << "," << key_field_
-                      << " FROM ("
-                      << geometry_table_ << ")";
-                boost::shared_ptr<sqlite_resultset> rs = dataset_->execute_query(query.str());
-                sqlite_utils::create_spatial_index(index_db,index_table_,rs,extent_);
-                extent_initialized_ = true;
-            }
-            else
-            {
-                std::ostringstream s;
-                s << "Sqlite Plugin: key_field is empty for " 
-                  << geometry_field_
-                  << " and "
-                  << geometry_table_;
-                throw datasource_exception(s.str());
-            }
-    }
-
     if (! extent_initialized_)
     {
 

plugins/input/sqlite/sqlite_datasource.hpp

@@ -48,6 +48,9 @@ public:
     mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const;
     mapnik::box2d<double> envelope() const;
     mapnik::layer_descriptor get_descriptor() const;
+    bool index_on(std::string const& field) const;
+    bool create_index(std::string const& field, mapnik::parameters const& params) const;
+    std::string geometry_field() const { return geometry_field_; }
     void bind() const;
 
 private:

plugins/input/sqlite/sqlite_utils.hpp

@@ -110,10 +110,9 @@ public:
         }
     }
     
-    static void create_spatial_index(std::string const& index_db,
+    static void create_rtree(std::string const& index_db,
                                      std::string const& index_table,
-                                     boost::shared_ptr<sqlite_resultset> rs,
+                                     boost::shared_ptr<sqlite_resultset> rs)
-                                     mapnik::box2d<double>& extent)
     {
         /* TODO
           - speedups
@@ -153,10 +152,8 @@ public:
                    << " values (?,?,?,?,?)";
 
         ds->execute(create_idx.str());
-        
         prepared_index_statement ps(ds,insert_idx.str());
 
-        bool first = true;
         while (rs->is_valid() && rs->step_next())
         {
             int size;
@@ -172,16 +169,6 @@ public:
                     mapnik::box2d<double> const& bbox = paths[i].envelope();
                     if (bbox.valid())
                     {
-                        if (first)
-                        {
-                            first = false;
-                            extent = bbox;
-                        }
-                        else
-                        {
-                            extent.expand_to_include(bbox);
-                        }
-
                         ps.bind(bbox);
     
                         const int type_oid = rs->column_type(1);