From 55b37160e4ca924d2462ccf7085914d00c8b1032 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Mon, 11 Jul 2011 17:46:53 +0000 Subject: [PATCH] support attachdb, initdb, and index_table parameters in sqlite plugin - patch and tests from stella - closes #793 --- plugins/input/sqlite/sqlite_datasource.cpp | 100 ++++++++++++++++++--- plugins/input/sqlite/sqlite_datasource.hpp | 6 ++ plugins/input/sqlite/sqlite_types.hpp | 40 ++++++--- tests/python_tests/sqlite_test.py | 58 ++++++++++++ 4 files changed, 180 insertions(+), 24 deletions(-) create mode 100644 tests/python_tests/sqlite_test.py diff --git a/plugins/input/sqlite/sqlite_datasource.cpp b/plugins/input/sqlite/sqlite_datasource.cpp index fddc2e5fd..eed1ae619 100644 --- a/plugins/input/sqlite/sqlite_datasource.cpp +++ b/plugins/input/sqlite/sqlite_datasource.cpp @@ -62,6 +62,8 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind) metadata_(*params_.get("metadata","")), geometry_table_(*params_.get("geometry_table","")), geometry_field_(*params_.get("geometry_field","")), + // index_table_ defaults to "idx_{geometry_table_}_{geometry_field_}" + index_table_(*params_.get("index_table","")), // http://www.sqlite.org/lang_createtable.html#rowid key_field_(*params_.get("key_field","rowid")), row_offset_(*params_.get("row_offset",0)), @@ -101,12 +103,68 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind) else dataset_name_ = *file; + // Populate init_statements_ + // 1. Build attach database statements from the "attachdb" parameter + // 2. Add explicit init statements from "initdb" parameter + // Note that we do some extra work to make sure that any attached + // databases are relative to directory containing dataset_name_. Sqlite + // will default to attaching from cwd. Typicaly usage means that the + // map loader will produce full paths here. + boost::optional attachdb = params_.get("attachdb"); + if (attachdb) { + parse_attachdb(*attachdb); + } + + boost::optional initdb = params_.get("initdb"); + if (initdb) { + init_statements_.push_back(*initdb); + } + if (bind) { this->bind(); } } +void sqlite_datasource::parse_attachdb(std::string const& attachdb) { + boost::char_separator sep(","); + boost::tokenizer > tok(attachdb, sep); + + // The attachdb line is a comma sparated list of + // [dbname@]filename + for (boost::tokenizer >::iterator beg = tok.begin(); + beg != tok.end(); ++beg) + { + std::string const& spec(*beg); + std::string dbname; + std::string filename; + size_t atpos=spec.find('@'); + + // See if it contains an @ sign + if (atpos==spec.npos) { + throw datasource_exception("attachdb parameter has syntax dbname@filename[,...]"); + } + + // Break out the dbname and the filename + dbname=boost::trim_copy(spec.substr(0, atpos)); + filename=boost::trim_copy(spec.substr(atpos+1)); + + // Normalize the filename and make it relative to dataset_name_ + if (filename.compare(":memory:")!=0) { + boost::filesystem::path child_path(filename); + if (!child_path.has_root_directory() && !child_path.has_root_name()) { + // It is a relative path. Fix it. + boost::filesystem::path absolute_path(dataset_name_); + absolute_path.remove_filename(); + filename=absolute_path.string() + filename; + } + } + + // And add an init_statement_ + init_statements_.push_back("attach database '" + filename + "' as " + dbname); + } +} + void sqlite_datasource::bind() const { if (is_bound_) return; @@ -116,6 +174,16 @@ void sqlite_datasource::bind() const dataset_ = new sqlite_connection (dataset_name_); + // Execute init_statements_ + for (std::vector::const_iterator iter=init_statements_.begin(); iter!=init_statements_.end(); ++iter) + { +#ifdef MAPNIK_DEBUG + std::clog << "Sqlite Plugin: Execute init sql: " << *iter << std::endl; +#endif + dataset_->execute(*iter); + } + + if(geometry_table_.empty()) { geometry_table_ = mapnik::table_from_sql(table_); @@ -262,18 +330,22 @@ void sqlite_datasource::bind() const if (use_spatial_index_) { - std::ostringstream s; - s << "SELECT COUNT (*) FROM sqlite_master"; - s << " WHERE LOWER(name) = LOWER('idx_" << geometry_table_ << "_" << geometry_field_ << "')"; - boost::scoped_ptr rs (dataset_->execute_query (s.str())); - if (rs->is_valid () && rs->step_next()) - { - use_spatial_index_ = rs->column_integer (0) == 1; + if (index_table_.size() == 0) { + // Generate implicit index_table name - need to do this after + // we have discovered meta-data or else we don't know the column + // name + index_table_ = "idx_" + geometry_table_ + "_" + geometry_field_; + } + + std::ostringstream s; + s << "SELECT pkid,xmin,xmax,ymin,ymax FROM " << index_table_; + s << " LIMIT 0"; + if (dataset_->execute_with_code(s.str()) != SQLITE_OK) + { + use_spatial_index_ = false; + std::clog << "Sqlite Plugin: No suitable spatial index found for " + << geometry_table_ << " (checked " << s.str() << ")" << std::endl; } - - if (!use_spatial_index_) - std::clog << "Sqlite Plugin: spatial index not found for table '" - << geometry_table_ << "'" << std::endl; } if (metadata_ != "" && !extent_initialized_) @@ -298,7 +370,7 @@ void sqlite_datasource::bind() const { std::ostringstream s; s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM " - << "idx_" << geometry_table_ << "_" << geometry_field_; + << index_table_; boost::scoped_ptr rs (dataset_->execute_query (s.str())); if (rs->is_valid () && rs->step_next()) { @@ -381,7 +453,7 @@ featureset_ptr sqlite_datasource::features(query const& q) const { std::ostringstream spatial_sql; spatial_sql << std::setprecision(16); - spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM idx_" << geometry_table_ << "_" << geometry_field_; + spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM " << index_table_; spatial_sql << " WHERE xmax>=" << e.minx() << " AND xmin<=" << e.maxx() ; spatial_sql << " AND ymax>=" << e.miny() << " AND ymin<=" << e.maxy() << ")"; if (boost::algorithm::ifind_first(query, "WHERE")) @@ -445,7 +517,7 @@ featureset_ptr sqlite_datasource::features_at_point(coord2d const& pt) const { std::ostringstream spatial_sql; spatial_sql << std::setprecision(16); - spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM idx_" << geometry_table_ << "_" << geometry_field_; + spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM " << index_table_; spatial_sql << " WHERE xmax>=" << e.minx() << " AND xmin<=" << e.maxx() ; spatial_sql << " AND ymax>=" << e.miny() << " AND ymin<=" << e.maxy() << ")"; if (boost::algorithm::ifind_first(query, "WHERE")) diff --git a/plugins/input/sqlite/sqlite_datasource.hpp b/plugins/input/sqlite/sqlite_datasource.hpp index d3785bc40..eea681373 100644 --- a/plugins/input/sqlite/sqlite_datasource.hpp +++ b/plugins/input/sqlite/sqlite_datasource.hpp @@ -60,6 +60,7 @@ class sqlite_datasource : public mapnik::datasource std::string metadata_; mutable std::string geometry_table_; mutable std::string geometry_field_; + mutable std::string index_table_; std::string key_field_; const int row_offset_; const int row_limit_; @@ -67,6 +68,11 @@ class sqlite_datasource : public mapnik::datasource mapnik::wkbFormat format_; bool multiple_geometries_; mutable bool use_spatial_index_; + std::vector init_statements_; + + // Fill init_statements with any statements + // needed to attach auxillary databases + void parse_attachdb(std::string const& attachdb); }; diff --git a/plugins/input/sqlite/sqlite_types.hpp b/plugins/input/sqlite/sqlite_types.hpp index 757dbb6ac..9e8c25cdb 100644 --- a/plugins/input/sqlite/sqlite_types.hpp +++ b/plugins/input/sqlite/sqlite_types.hpp @@ -137,6 +137,19 @@ public: sqlite3_close (db_); } + void throw_sqlite_error(const std::string& sql) + { + std::ostringstream s; + s << "Sqlite Plugin: "; + if (db_) + s << "'" << sqlite3_errmsg(db_) << "'"; + else + s << "unknown error, lost connection"; + + s << "\nFull sql was: '" << sql << "'\n"; + throw mapnik::datasource_exception( s.str() ); + } + sqlite_resultset* execute_query(const std::string& sql) { sqlite3_stmt* stmt = 0; @@ -144,20 +157,27 @@ public: int rc = sqlite3_prepare_v2 (db_, sql.c_str(), -1, &stmt, 0); if (rc != SQLITE_OK) { - std::ostringstream s; - s << "Sqlite Plugin: "; - if (db_) - s << "'" << sqlite3_errmsg(db_) << "'"; - else - s << "unknown error, lost connection"; - - s << "\nFull sql was: '" << sql << "'\n"; - throw mapnik::datasource_exception( s.str() ); + throw_sqlite_error(sql); } return new sqlite_resultset (stmt); - } + } + + void execute(const std::string& sql) + { + int rc=sqlite3_exec(db_, sql.c_str(), 0, 0, 0); + if (rc != SQLITE_OK) + { + throw_sqlite_error(sql); + } + } + int execute_with_code(const std::string& sql) + { + int rc=sqlite3_exec(db_, sql.c_str(), 0, 0, 0); + return rc; + } + sqlite3* operator*() { return db_; diff --git a/tests/python_tests/sqlite_test.py b/tests/python_tests/sqlite_test.py new file mode 100644 index 000000000..ecadbd95d --- /dev/null +++ b/tests/python_tests/sqlite_test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +from nose.tools import * +from utilities import execution_path + +import os, mapnik2 + +def setup(): + # All of the paths used are relative, if we run the tests + # from another directory we need to chdir() + os.chdir(execution_path('.')) + +# Note that without an extent or a spatial index, a sqlite +# datasource will error on creation. We use this fact to test +# database attachdb and initdb options + +def test_attachdb_with_relative_file(): + # The point table and index is in the qgis_spatiallite.sqlite + # database. If either is not found, then this fails + ds = mapnik2.SQLite(file='../data/sqlite/world.sqlite', + table='point', + attachdb='scratch@qgis_spatiallite.sqlite' + ) + +def test_attachdb_with_multiple_files(): + ds = mapnik2.SQLite(file='../data/sqlite/world.sqlite', + table='attachedtest', + attachdb='scratch1@:memory:,scratch2@:memory:', + initdb='create table scratch1.attachedtest (the_geom);\n' + + 'create virtual table scratch2.idx_attachedtest_the_geom using rtree(pkid,xmin,xmax,ymin,ymax);\n' + ) + +def test_attachdb_with_absolute_file(): + # The point table and index is in the qgis_spatiallite.sqlite + # database. If either is not found, then this fails + ds = mapnik2.SQLite(file=os.getcwd() + '/../data/sqlite/world.sqlite', + table='point', + attachdb='scratch@qgis_spatiallite.sqlite' + ) + +def test_attachdb_with_index(): + ds = mapnik2.SQLite(file='../data/sqlite/world.sqlite', + table='attachedtest', + attachdb='scratch@:memory:', + initdb='create table scratch.attachedtest (the_geom);\n' + + 'create virtual table scratch.idx_attachedtest_the_geom using rtree(pkid,xmin,xmax,ymin,ymax);\n' + ) + +def test_attachdb_with_explicit_index(): + ds = mapnik2.SQLite(file='../data/sqlite/world.sqlite', + table='attachedtest', + index_table='myindex', + attachdb='scratch@:memory:', + initdb='create table scratch.attachedtest (the_geom);\n' + + 'create virtual table scratch.myindex using rtree(pkid,xmin,xmax,ymin,ymax);\n' + ) + +