sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

support attachdb, initdb, and index_table parameters in sqlite plugin - patch and tests from stella - closes #793

Browse source
This commit is contained in:
Dane Springmeyer 2011-07-11 17:46:53 +00:00
parent aab601da64
commit 55b37160e4
4 changed files with 180 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

100
plugins/input/sqlite/sqlite_datasource.cpp
View file

@ -62,6 +62,8 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind)
metadata_(*params_.get<std::string>("metadata","")),
geometry_table_(*params_.get<std::string>("geometry_table","")),
geometry_field_(*params_.get<std::string>("geometry_field","")),
// index_table_ defaults to "idx_{geometry_table_}_{geometry_field_}"
index_table_(*params_.get<std::string>("index_table","")),
// http://www.sqlite.org/lang_createtable.html#rowid
key_field_(*params_.get<std::string>("key_field","rowid")),
row_offset_(*params_.get<int>("row_offset",0)),
@ -101,12 +103,68 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind)
else
dataset_name_ = *file;
// Populate init_statements_
// 1. Build attach database statements from the "attachdb" parameter
// 2. Add explicit init statements from "initdb" parameter
// Note that we do some extra work to make sure that any attached
// databases are relative to directory containing dataset_name_. Sqlite
// will default to attaching from cwd. Typicaly usage means that the
// map loader will produce full paths here.
boost::optional<std::string> attachdb = params_.get<std::string>("attachdb");
if (attachdb) {
parse_attachdb(*attachdb);
}
boost::optional<std::string> initdb = params_.get<std::string>("initdb");
if (initdb) {
init_statements_.push_back(*initdb);
}
if (bind)
{
this->bind();
}
}
void sqlite_datasource::parse_attachdb(std::string const& attachdb) {
boost::char_separator<char> sep(",");
boost::tokenizer<boost::char_separator<char> > tok(attachdb, sep);
// The attachdb line is a comma sparated list of
// [dbname@]filename
for (boost::tokenizer<boost::char_separator<char> >::iterator beg = tok.begin();
beg != tok.end(); ++beg)
{
std::string const& spec(*beg);
std::string dbname;
std::string filename;
size_t atpos=spec.find('@');
// See if it contains an @ sign
if (atpos==spec.npos) {
throw datasource_exception("attachdb parameter has syntax dbname@filename[,...]");
}
// Break out the dbname and the filename
dbname=boost::trim_copy(spec.substr(0, atpos));
filename=boost::trim_copy(spec.substr(atpos+1));
// Normalize the filename and make it relative to dataset_name_
if (filename.compare(":memory:")!=0) {
boost::filesystem::path child_path(filename);
if (!child_path.has_root_directory() && !child_path.has_root_name()) {
// It is a relative path. Fix it.
boost::filesystem::path absolute_path(dataset_name_);
absolute_path.remove_filename();
filename=absolute_path.string() + filename;
}
}
// And add an init_statement_
init_statements_.push_back("attach database '" + filename + "' as " + dbname);
}
}
void sqlite_datasource::bind() const
{
if (is_bound_) return;
@ -116,6 +174,16 @@ void sqlite_datasource::bind() const
dataset_ = new sqlite_connection (dataset_name_);
// Execute init_statements_
for (std::vector<std::string>::const_iterator iter=init_statements_.begin(); iter!=init_statements_.end(); ++iter)
{
#ifdef MAPNIK_DEBUG
std::clog << "Sqlite Plugin: Execute init sql: " << *iter << std::endl;
#endif
dataset_->execute(*iter);
}
if(geometry_table_.empty())
{
geometry_table_ = mapnik::table_from_sql(table_);
@ -262,18 +330,22 @@ void sqlite_datasource::bind() const
if (use_spatial_index_)
{
std::ostringstream s;
s << "SELECT COUNT (*) FROM sqlite_master";
s << " WHERE LOWER(name) = LOWER('idx_" << geometry_table_ << "_" << geometry_field_ << "')";
boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str()));
if (rs->is_valid () && rs->step_next())
{
use_spatial_index_ = rs->column_integer (0) == 1;
if (index_table_.size() == 0) {
// Generate implicit index_table name - need to do this after
// we have discovered meta-data or else we don't know the column
// name
index_table_ = "idx_" + geometry_table_ + "_" + geometry_field_;
}
std::ostringstream s;
s << "SELECT pkid,xmin,xmax,ymin,ymax FROM " << index_table_;
s << " LIMIT 0";
if (dataset_->execute_with_code(s.str()) != SQLITE_OK)
{
use_spatial_index_ = false;
std::clog << "Sqlite Plugin: No suitable spatial index found for "
<< geometry_table_ << " (checked " << s.str() << ")" << std::endl;
}
if (!use_spatial_index_)
std::clog << "Sqlite Plugin: spatial index not found for table '"
<< geometry_table_ << "'" << std::endl;
}
if (metadata_ != "" && !extent_initialized_)
@ -298,7 +370,7 @@ void sqlite_datasource::bind() const
{
std::ostringstream s;
s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM "
<< "idx_" << geometry_table_ << "_" << geometry_field_;
<< index_table_;
boost::scoped_ptr<sqlite_resultset> rs (dataset_->execute_query (s.str()));
if (rs->is_valid () && rs->step_next())
{
@ -381,7 +453,7 @@ featureset_ptr sqlite_datasource::features(query const& q) const
{
std::ostringstream spatial_sql;
spatial_sql << std::setprecision(16);
spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM idx_" << geometry_table_ << "_" << geometry_field_;
spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM " << index_table_;
spatial_sql << " WHERE xmax>=" << e.minx() << " AND xmin<=" << e.maxx() ;
spatial_sql << " AND ymax>=" << e.miny() << " AND ymin<=" << e.maxy() << ")";
if (boost::algorithm::ifind_first(query, "WHERE"))
@ -445,7 +517,7 @@ featureset_ptr sqlite_datasource::features_at_point(coord2d const& pt) const
{
std::ostringstream spatial_sql;
spatial_sql << std::setprecision(16);
spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM idx_" << geometry_table_ << "_" << geometry_field_;
spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM " << index_table_;
spatial_sql << " WHERE xmax>=" << e.minx() << " AND xmin<=" << e.maxx() ;
spatial_sql << " AND ymax>=" << e.miny() << " AND ymin<=" << e.maxy() << ")";
if (boost::algorithm::ifind_first(query, "WHERE"))

6
plugins/input/sqlite/sqlite_datasource.hpp
View file

@ -60,6 +60,7 @@ class sqlite_datasource : public mapnik::datasource
std::string metadata_;
mutable std::string geometry_table_;
mutable std::string geometry_field_;
mutable std::string index_table_;
std::string key_field_;
const int row_offset_;
const int row_limit_;
@ -67,6 +68,11 @@ class sqlite_datasource : public mapnik::datasource
mapnik::wkbFormat format_;
bool multiple_geometries_;
mutable bool use_spatial_index_;
std::vector<std::string> init_statements_;
// Fill init_statements with any statements
// needed to attach auxillary databases
void parse_attachdb(std::string const& attachdb);
};

40
plugins/input/sqlite/sqlite_types.hpp
View file

@ -137,6 +137,19 @@ public:
sqlite3_close (db_);
}
void throw_sqlite_error(const std::string& sql)
{
std::ostringstream s;
s << "Sqlite Plugin: ";
if (db_)
s << "'" << sqlite3_errmsg(db_) << "'";
else
s << "unknown error, lost connection";
s << "\nFull sql was: '" << sql << "'\n";
throw mapnik::datasource_exception( s.str() );
}
sqlite_resultset* execute_query(const std::string& sql)
{
sqlite3_stmt* stmt = 0;
@ -144,20 +157,27 @@ public:
int rc = sqlite3_prepare_v2 (db_, sql.c_str(), -1, &stmt, 0);
if (rc != SQLITE_OK)
{
std::ostringstream s;
s << "Sqlite Plugin: ";
if (db_)
s << "'" << sqlite3_errmsg(db_) << "'";
else
s << "unknown error, lost connection";
s << "\nFull sql was: '" << sql << "'\n";
throw mapnik::datasource_exception( s.str() );
throw_sqlite_error(sql);
}
return new sqlite_resultset (stmt);
}
}
void execute(const std::string& sql)
{
int rc=sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
if (rc != SQLITE_OK)
{
throw_sqlite_error(sql);
}
}
int execute_with_code(const std::string& sql)
{
int rc=sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
return rc;
}
sqlite3* operator*()
{
return db_;

58
tests/python_tests/sqlite_test.py Normal file
View file

@ -0,0 +1,58 @@
#!/usr/bin/env python
from nose.tools import *
from utilities import execution_path
import os, mapnik2
def setup():
# All of the paths used are relative, if we run the tests
# from another directory we need to chdir()
os.chdir(execution_path('.'))
# Note that without an extent or a spatial index, a sqlite
# datasource will error on creation. We use this fact to test
# database attachdb and initdb options
def test_attachdb_with_relative_file():
# The point table and index is in the qgis_spatiallite.sqlite
# database. If either is not found, then this fails
ds = mapnik2.SQLite(file='../data/sqlite/world.sqlite',
table='point',
attachdb='scratch@qgis_spatiallite.sqlite'
)
def test_attachdb_with_multiple_files():
ds = mapnik2.SQLite(file='../data/sqlite/world.sqlite',
table='attachedtest',
attachdb='scratch1@:memory:,scratch2@:memory:',
initdb='create table scratch1.attachedtest (the_geom);\n' +
'create virtual table scratch2.idx_attachedtest_the_geom using rtree(pkid,xmin,xmax,ymin,ymax);\n'
)
def test_attachdb_with_absolute_file():
# The point table and index is in the qgis_spatiallite.sqlite
# database. If either is not found, then this fails
ds = mapnik2.SQLite(file=os.getcwd() + '/../data/sqlite/world.sqlite',
table='point',
attachdb='scratch@qgis_spatiallite.sqlite'
)
def test_attachdb_with_index():
ds = mapnik2.SQLite(file='../data/sqlite/world.sqlite',
table='attachedtest',
attachdb='scratch@:memory:',
initdb='create table scratch.attachedtest (the_geom);\n' +
'create virtual table scratch.idx_attachedtest_the_geom using rtree(pkid,xmin,xmax,ymin,ymax);\n'
)
def test_attachdb_with_explicit_index():
ds = mapnik2.SQLite(file='../data/sqlite/world.sqlite',
table='attachedtest',
index_table='myindex',
attachdb='scratch@:memory:',
initdb='create table scratch.attachedtest (the_geom);\n' +
'create virtual table scratch.myindex using rtree(pkid,xmin,xmax,ymin,ymax);\n'
)