From d33873d3b707b1bde1aae232070b083dec908d7b Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Fri, 4 Nov 2011 20:36:45 -0700 Subject: [PATCH] steps toward a generic spatial indexing api --- bindings/python/mapnik_datasource.cpp | 2 + include/mapnik/datasource.hpp | 8 ++ include/mapnik/memory_datasource.hpp | 3 + plugins/input/csv/csv_datasource.hpp | 65 ++++++++-------- plugins/input/postgis/postgis_datasource.cpp | 1 + plugins/input/shape/shape_datasource.hpp | 3 + plugins/input/sqlite/sqlite_datasource.cpp | 78 ++++++++++++-------- plugins/input/sqlite/sqlite_datasource.hpp | 3 + plugins/input/sqlite/sqlite_utils.hpp | 17 +---- 9 files changed, 102 insertions(+), 78 deletions(-) diff --git a/bindings/python/mapnik_datasource.cpp b/bindings/python/mapnik_datasource.cpp index cc9cf23dc..f8b0f3b4d 100644 --- a/bindings/python/mapnik_datasource.cpp +++ b/bindings/python/mapnik_datasource.cpp @@ -174,6 +174,8 @@ void export_datasource() .def("encoding",&encoding) //todo expose as property .def("name",&name) .def("features_at_point",&datasource::features_at_point) + .def("index_on",&datasource::index_on) + .def("create_index",&datasource::create_index) .def("params",&datasource::params,return_value_policy(), "The configuration parameters of the data source. " "These vary depending on the type of data source.") diff --git a/include/mapnik/datasource.hpp b/include/mapnik/datasource.hpp index 0495ab450..cb9be6581 100644 --- a/include/mapnik/datasource.hpp +++ b/include/mapnik/datasource.hpp @@ -90,6 +90,11 @@ public: { return params_; } + + void set_param(parameter const& p) + { + params_[p.first] = p.second; + } /*! * @brief Get the type of the datasource @@ -106,6 +111,9 @@ public: virtual featureset_ptr features_at_point(coord2d const& pt) const=0; virtual box2d envelope() const=0; virtual layer_descriptor get_descriptor() const=0; + virtual bool index_on(std::string const& field) const=0; + virtual bool create_index(std::string const& field, mapnik::parameters const& params) const=0; + virtual std::string geometry_field() const=0; virtual ~datasource() {}; protected: parameters params_; diff --git a/include/mapnik/memory_datasource.hpp b/include/mapnik/memory_datasource.hpp index 4a51b46a2..ac84fe24c 100644 --- a/include/mapnik/memory_datasource.hpp +++ b/include/mapnik/memory_datasource.hpp @@ -44,6 +44,9 @@ public: featureset_ptr features_at_point(coord2d const& pt) const; box2d envelope() const; layer_descriptor get_descriptor() const; + bool index_on(std::string const& field) const { return false; } + bool create_index(std::string const& field, mapnik::parameters const& params) const { return false; } + std::string geometry_field() const { return ""; } size_t size() const; private: std::vector features_; diff --git a/plugins/input/csv/csv_datasource.hpp b/plugins/input/csv/csv_datasource.hpp index d2e51bf8c..2451246a6 100644 --- a/plugins/input/csv/csv_datasource.hpp +++ b/plugins/input/csv/csv_datasource.hpp @@ -9,37 +9,40 @@ class csv_datasource : public mapnik::datasource { - public: - csv_datasource(mapnik::parameters const& params, bool bind=true); - virtual ~csv_datasource (); - int type() const; - static std::string name(); - mapnik::featureset_ptr features(mapnik::query const& q) const; - mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const; - mapnik::box2d envelope() const; - mapnik::layer_descriptor get_descriptor() const; - void bind() const; - template - void parse_csv(T& stream, - std::string const& escape, - std::string const& separator, - std::string const& quote) const; - private: - mutable mapnik::layer_descriptor desc_; - mutable mapnik::box2d extent_; - mutable std::string filename_; - mutable std::string inline_string_; - mutable unsigned file_length_; - mutable int row_limit_; - mutable std::vector features_; - mutable std::string escape_; - mutable std::string separator_; - mutable std::string quote_; - mutable std::vector headers_; - mutable std::string manual_headers_; - mutable bool strict_; - mutable bool quiet_; - mutable double filesize_max_; +public: + csv_datasource(mapnik::parameters const& params, bool bind=true); + virtual ~csv_datasource (); + int type() const; + static std::string name(); + mapnik::featureset_ptr features(mapnik::query const& q) const; + mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const; + mapnik::box2d envelope() const; + mapnik::layer_descriptor get_descriptor() const; + bool index_on(std::string field) const { return false }; + bool create_index(std::string const& field, mapnik::parameters const& params) const { return false; } + std::string geometry_field() const { return ""; } + void bind() const; + template + void parse_csv(T& stream, + std::string const& escape, + std::string const& separator, + std::string const& quote) const; +private: + mutable mapnik::layer_descriptor desc_; + mutable mapnik::box2d extent_; + mutable std::string filename_; + mutable std::string inline_string_; + mutable unsigned file_length_; + mutable int row_limit_; + mutable std::vector features_; + mutable std::string escape_; + mutable std::string separator_; + mutable std::string quote_; + mutable std::vector headers_; + mutable std::string manual_headers_; + mutable bool strict_; + mutable bool quiet_; + mutable double filesize_max_; }; diff --git a/plugins/input/postgis/postgis_datasource.cpp b/plugins/input/postgis/postgis_datasource.cpp index 3af5a8eda..eb84e4a15 100644 --- a/plugins/input/postgis/postgis_datasource.cpp +++ b/plugins/input/postgis/postgis_datasource.cpp @@ -357,6 +357,7 @@ std::string postgis_datasource::sql_bbox(box2d const& env) const std::ostringstream b; if (srid_ > 0) b << "SetSRID("; + //ST_MakeBox2D(ST_Point(),ST_Point()) b << "'BOX3D("; b << std::setprecision(16); b << env.minx() << " " << env.miny() << ","; diff --git a/plugins/input/shape/shape_datasource.hpp b/plugins/input/shape/shape_datasource.hpp index 477b8035b..57b609a4d 100644 --- a/plugins/input/shape/shape_datasource.hpp +++ b/plugins/input/shape/shape_datasource.hpp @@ -51,6 +51,9 @@ public: featureset_ptr features_at_point(coord2d const& pt) const; box2d envelope() const; layer_descriptor get_descriptor() const; + bool index_on(std::string field) const { return false }; + bool create_index(std::string const& field, mapnik::parameters const& params) const { return false; } + std::string geometry_field() const { return ""; } void bind() const; private: shape_datasource(const shape_datasource&); diff --git a/plugins/input/sqlite/sqlite_datasource.cpp b/plugins/input/sqlite/sqlite_datasource.cpp index 990d583ca..df48071c2 100644 --- a/plugins/input/sqlite/sqlite_datasource.cpp +++ b/plugins/input/sqlite/sqlite_datasource.cpp @@ -67,7 +67,6 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind) { /* TODO - throw if no primary key but spatial index is present? - - remove auto-indexing */ boost::optional file = params_.get("file"); @@ -84,6 +83,52 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind) } } +bool sqlite_datasource::index_on(std::string const& field) const +{ + if (! is_bound_) bind(); + + if (field == geometry_field_) + { + if (use_spatial_index_ && !index_table_.empty()) + { + std::string index_db = sqlite_utils::index_for_db(dataset_name_); + if (boost::filesystem::exists(index_db)) + { + dataset_->execute("attach database '" + index_db + "' as " + index_table_); + } + + return sqlite_utils::has_rtree(index_table_,dataset_); + } + } + // else check or attribute index? + return false; +} + +bool sqlite_datasource::create_index(std::string const& field, mapnik::parameters const& params) const +{ + if (! is_bound_) bind(); + if (field == geometry_field_) + { + if (!key_field_.empty() && !index_table_.empty()) + { + std::string index_db = sqlite_utils::index_for_db(dataset_name_); + std::ostringstream query; + query << "SELECT " + << geometry_field_ + << "," << key_field_ + << " FROM (" + << geometry_table_ << ")"; + boost::shared_ptr rs = dataset_->execute_query(query.str()); + sqlite_utils::create_rtree(index_db,index_table_,rs); + use_spatial_index_ = sqlite_utils::has_rtree(index_table_,dataset_); + return use_spatial_index_; + } + } + // else create attribute index + return false; + +} + void sqlite_datasource::bind() const { if (is_bound_) return; @@ -104,9 +149,6 @@ void sqlite_datasource::bind() const multiple_geometries_ = *params_.get("multiple_geometries", false); use_spatial_index_ = *params_.get("use_spatial_index", true); - - // TODO - remove this option once all datasources have an indexing api - bool auto_index = *params_.get("auto_index", true); boost::optional ext = params_.get("extent"); if (ext) extent_initialized_ = extent_.from_string(*ext); @@ -232,34 +274,6 @@ void sqlite_datasource::bind() const has_spatial_index_ = sqlite_utils::has_rtree(index_table_,dataset_); } - - if (! extent_initialized_ - && !has_spatial_index_ - && auto_index) - { - if (! key_field_.empty()) - { - std::ostringstream query; - query << "SELECT " - << geometry_field_ - << "," << key_field_ - << " FROM (" - << geometry_table_ << ")"; - boost::shared_ptr rs = dataset_->execute_query(query.str()); - sqlite_utils::create_spatial_index(index_db,index_table_,rs,extent_); - extent_initialized_ = true; - } - else - { - std::ostringstream s; - s << "Sqlite Plugin: key_field is empty for " - << geometry_field_ - << " and " - << geometry_table_; - throw datasource_exception(s.str()); - } - } - if (! extent_initialized_) { diff --git a/plugins/input/sqlite/sqlite_datasource.hpp b/plugins/input/sqlite/sqlite_datasource.hpp index 0a964e61d..6ba0307b6 100644 --- a/plugins/input/sqlite/sqlite_datasource.hpp +++ b/plugins/input/sqlite/sqlite_datasource.hpp @@ -48,6 +48,9 @@ public: mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const; mapnik::box2d envelope() const; mapnik::layer_descriptor get_descriptor() const; + bool index_on(std::string const& field) const; + bool create_index(std::string const& field, mapnik::parameters const& params) const; + std::string geometry_field() const { return geometry_field_; } void bind() const; private: diff --git a/plugins/input/sqlite/sqlite_utils.hpp b/plugins/input/sqlite/sqlite_utils.hpp index 87c5a593c..ab5cb1815 100644 --- a/plugins/input/sqlite/sqlite_utils.hpp +++ b/plugins/input/sqlite/sqlite_utils.hpp @@ -110,10 +110,9 @@ public: } } - static void create_spatial_index(std::string const& index_db, + static void create_rtree(std::string const& index_db, std::string const& index_table, - boost::shared_ptr rs, - mapnik::box2d& extent) + boost::shared_ptr rs) { /* TODO - speedups @@ -153,10 +152,8 @@ public: << " values (?,?,?,?,?)"; ds->execute(create_idx.str()); - prepared_index_statement ps(ds,insert_idx.str()); - bool first = true; while (rs->is_valid() && rs->step_next()) { int size; @@ -172,16 +169,6 @@ public: mapnik::box2d const& bbox = paths[i].envelope(); if (bbox.valid()) { - if (first) - { - first = false; - extent = bbox; - } - else - { - extent.expand_to_include(bbox); - } - ps.bind(bbox); const int type_oid = rs->column_type(1);