sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

- cosmetics changes to sqlite datasource plugin

Browse source 
- added initial wkbAuto to auto determine WKB type in sqlite
This commit is contained in:
kunitoki 2011-10-18 22:19:03 +02:00
parent a6522fde77
commit b23697e1b3
7 changed files with 395 additions and 277 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
include/mapnik/wkb.hpp
View file

@ -45,8 +45,9 @@ namespace mapnik
*/ */
enum wkbFormat enum wkbFormat
{ {
wkbGeneric=1, wkbAuto=1,
wkbSpatiaLite=2 wkbGeneric=2,
wkbSpatiaLite=3
}; };
class MAPNIK_DECL geometry_utils class MAPNIK_DECL geometry_utils

399
plugins/input/sqlite/sqlite_datasource.cpp
View file

@ -74,7 +74,7 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind)
row_offset_(*params_.get<int>("row_offset",0)), row_offset_(*params_.get<int>("row_offset",0)),
row_limit_(*params_.get<int>("row_limit",0)), row_limit_(*params_.get<int>("row_limit",0)),
desc_(*params_.get<std::string>("type"), *params_.get<std::string>("encoding","utf-8")), desc_(*params_.get<std::string>("type"), *params_.get<std::string>("encoding","utf-8")),
format_(mapnik::wkbGeneric) format_(mapnik::wkbAuto)
{ {
// TODO // TODO
// - change param from 'file' to 'dbname' // - change param from 'file' to 'dbname'
@ -83,7 +83,8 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind)
boost::optional<std::string> file = params_.get<std::string>("file"); boost::optional<std::string> file = params_.get<std::string>("file");
if (!file) throw datasource_exception("Sqlite Plugin: missing <file> parameter"); if (!file) throw datasource_exception("Sqlite Plugin: missing <file> parameter");
if (table_.empty()) { if (table_.empty())
{
throw mapnik::datasource_exception("Sqlite Plugin: missing <table> parameter"); throw mapnik::datasource_exception("Sqlite Plugin: missing <table> parameter");
} }
@ -98,16 +99,16 @@ void sqlite_datasource::parse_attachdb(std::string const& attachdb) const
boost::char_separator<char> sep(","); boost::char_separator<char> sep(",");
boost::tokenizer<boost::char_separator<char> > tok(attachdb, sep); boost::tokenizer<boost::char_separator<char> > tok(attachdb, sep);
// The attachdb line is a comma sparated list of // The attachdb line is a comma sparated list of [dbname@]filename
// [dbname@]filename
for (boost::tokenizer<boost::char_separator<char> >::iterator beg = tok.begin(); for (boost::tokenizer<boost::char_separator<char> >::iterator beg = tok.begin();
beg != tok.end(); ++beg) beg != tok.end(); ++beg)
{ {
std::string const& spec(*beg); std::string const& spec(*beg);
size_t atpos=spec.find('@'); size_t atpos = spec.find('@');
// See if it contains an @ sign // See if it contains an @ sign
if (atpos==spec.npos) { if (atpos == spec.npos)
{
throw datasource_exception("attachdb parameter has syntax dbname@filename[,...]"); throw datasource_exception("attachdb parameter has syntax dbname@filename[,...]");
} }
@ -116,33 +117,34 @@ void sqlite_datasource::parse_attachdb(std::string const& attachdb) const
std::string filename = boost::trim_copy(spec.substr(atpos+1)); std::string filename = boost::trim_copy(spec.substr(atpos+1));
// Normalize the filename and make it relative to dataset_name_ // Normalize the filename and make it relative to dataset_name_
if (filename.compare(":memory:") != 0) { if (filename.compare(":memory:") != 0)
{
boost::filesystem::path child_path(filename); boost::filesystem::path child_path(filename);
// It is a relative path. Fix it. // It is a relative path. Fix it.
if (!child_path.has_root_directory() && !child_path.has_root_name()) { if (! child_path.has_root_directory() && ! child_path.has_root_name())
{
boost::filesystem::path absolute_path(dataset_name_); boost::filesystem::path absolute_path(dataset_name_);
// support symlinks // support symlinks
#if (BOOST_FILESYSTEM_VERSION == 3) #if (BOOST_FILESYSTEM_VERSION == 3)
if (boost::filesystem::is_symlink(absolute_path)) if (boost::filesystem::is_symlink(absolute_path))
{ {
absolute_path = boost::filesystem::read_symlink(absolute_path); absolute_path = boost::filesystem::read_symlink(absolute_path);
} }
filename = boost::filesystem::absolute(absolute_path.parent_path()/filename).string(); filename = boost::filesystem::absolute(absolute_path.parent_path() / filename).string();
#else #else
if (boost::filesystem::is_symlink(absolute_path)) if (boost::filesystem::is_symlink(absolute_path))
{ {
//cannot figure out how to resolve on in v2 so just print a warning //cannot figure out how to resolve on in v2 so just print a warning
std::clog << "###Warning: '" << absolute_path.string() << "' is a symlink which is not supported in attachdb\n"; std::clog << "###Warning: '" << absolute_path.string() << "' is a symlink which is not supported in attachdb\n";
} }
filename = boost::filesystem::complete(absolute_path.branch_path()/filename).normalize().string(); filename = boost::filesystem::complete(absolute_path.branch_path() / filename).normalize().string();
#endif #endif
} }
} }
@ -156,14 +158,18 @@ void sqlite_datasource::bind() const
if (is_bound_) return; if (is_bound_) return;
boost::optional<std::string> file = params_.get<std::string>("file"); boost::optional<std::string> file = params_.get<std::string>("file");
if (!file) throw datasource_exception("Sqlite Plugin: missing <file> parameter"); if (! file) throw datasource_exception("Sqlite Plugin: missing <file> parameter");
boost::optional<std::string> key_field_name = params_.get<std::string>("key_field"); boost::optional<std::string> key_field_name = params_.get<std::string>("key_field");
if (key_field_name) { if (key_field_name)
{
std::string const& key_field_string = *key_field_name; std::string const& key_field_string = *key_field_name;
if (key_field_string.empty()) { if (key_field_string.empty())
{
key_field_ = "rowid"; key_field_ = "rowid";
} else { }
else
{
key_field_ = key_field_string; key_field_ = key_field_string;
} }
} }
@ -177,6 +183,10 @@ void sqlite_datasource::bind() const
{ {
if (*wkb == "spatialite") if (*wkb == "spatialite")
format_ = mapnik::wkbSpatiaLite; format_ = mapnik::wkbSpatiaLite;
else if (*wkb == "generic")
format_ = mapnik::wkbGeneric;
else
format_ = mapnik::wkbAuto;
} }
multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries",false); multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries",false);
@ -201,12 +211,14 @@ void sqlite_datasource::bind() const
// will default to attaching from cwd. Typicaly usage means that the // will default to attaching from cwd. Typicaly usage means that the
// map loader will produce full paths here. // map loader will produce full paths here.
boost::optional<std::string> attachdb = params_.get<std::string>("attachdb"); boost::optional<std::string> attachdb = params_.get<std::string>("attachdb");
if (attachdb) { if (attachdb)
{
parse_attachdb(*attachdb); parse_attachdb(*attachdb);
} }
boost::optional<std::string> initdb = params_.get<std::string>("initdb"); boost::optional<std::string> initdb = params_.get<std::string>("initdb");
if (initdb) { if (initdb)
{
init_statements_.push_back(*initdb); init_statements_.push_back(*initdb);
} }
@ -216,7 +228,8 @@ void sqlite_datasource::bind() const
dataset_ = new sqlite_connection (dataset_name_); dataset_ = new sqlite_connection (dataset_name_);
// Execute init_statements_ // Execute init_statements_
for (std::vector<std::string>::const_iterator iter=init_statements_.begin(); iter!=init_statements_.end(); ++iter) for (std::vector<std::string>::const_iterator iter = init_statements_.begin();
iter!=init_statements_.end(); ++iter)
{ {
#ifdef MAPNIK_DEBUG #ifdef MAPNIK_DEBUG
std::clog << "Sqlite Plugin: Execute init sql: " << *iter << std::endl; std::clog << "Sqlite Plugin: Execute init sql: " << *iter << std::endl;
@ -224,7 +237,7 @@ void sqlite_datasource::bind() const
dataset_->execute(*iter); dataset_->execute(*iter);
} }
if(geometry_table_.empty()) if (geometry_table_.empty())
{ {
geometry_table_ = mapnik::table_from_sql(table_); geometry_table_ = mapnik::table_from_sql(table_);
} }
@ -240,7 +253,7 @@ void sqlite_datasource::bind() const
use_pragma_table_info = false; use_pragma_table_info = false;
} }
if (!use_pragma_table_info) if (! use_pragma_table_info)
{ {
std::ostringstream s; std::ostringstream s;
s << "SELECT " << fields_ << " FROM (" << table_ << ") LIMIT 1"; s << "SELECT " << fields_ << " FROM (" << table_ << ") LIMIT 1";
@ -250,43 +263,44 @@ void sqlite_datasource::bind() const
{ {
for (int i = 0; i < rs->column_count (); ++i) for (int i = 0; i < rs->column_count (); ++i)
{ {
const int type_oid = rs->column_type (i); const int type_oid = rs->column_type (i);
const char* fld_name = rs->column_name (i); const char* fld_name = rs->column_name (i);
switch (type_oid) switch (type_oid)
{ {
case SQLITE_INTEGER: case SQLITE_INTEGER:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
break; break;
case SQLITE_FLOAT: case SQLITE_FLOAT:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double));
break; break;
case SQLITE_TEXT: case SQLITE_TEXT:
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
break; break;
case SQLITE_NULL: case SQLITE_NULL:
// sqlite reports based on value, not actual column type unless // sqlite reports based on value, not actual column type unless
// PRAGMA table_info is used so here we assume the column is a string // PRAGMA table_info is used so here we assume the column is a string
// which is a lesser evil than altogether dropping the column // which is a lesser evil than altogether dropping the column
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
case SQLITE_BLOB: case SQLITE_BLOB:
if (geometry_field_.empty() && if (geometry_field_.empty()
(boost::algorithm::icontains(fld_name,"geom") || && (boost::algorithm::icontains(fld_name, "geom") ||
boost::algorithm::icontains(fld_name,"point") || boost::algorithm::icontains(fld_name, "point") ||
boost::algorithm::icontains(fld_name,"linestring") || boost::algorithm::icontains(fld_name, "linestring") ||
boost::algorithm::icontains(fld_name,"polygon")) boost::algorithm::icontains(fld_name, "polygon")))
) {
geometry_field_ = std::string(fld_name); geometry_field_ = std::string(fld_name);
break; }
break;
default: default:
#ifdef MAPNIK_DEBUG #ifdef MAPNIK_DEBUG
std::clog << "Sqlite Plugin: unknown type_oid=" << type_oid << std::endl; std::clog << "Sqlite Plugin: unknown type_oid=" << type_oid << std::endl;
#endif #endif
break; break;
} }
} }
} }
@ -301,85 +315,95 @@ void sqlite_datasource::bind() const
} }
if (key_field_ == "rowid") if (key_field_ == "rowid")
desc_.add_descriptor(attribute_descriptor("rowid",mapnik::Integer)); desc_.add_descriptor(attribute_descriptor("rowid", mapnik::Integer));
if (use_pragma_table_info) if (use_pragma_table_info)
{ {
std::ostringstream s; std::ostringstream s;
s << "PRAGMA table_info(" << geometry_table_ << ")"; s << "PRAGMA table_info(" << geometry_table_ << ")";
boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str())); boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
bool found_table = false; bool found_table = false;
while (rs->is_valid() && rs->step_next()) while (rs->is_valid() && rs->step_next())
{ {
found_table = true; found_table = true;
// TODO - support unicode strings?
// TODO - support unicode strings
const char* fld_name = rs->column_text(1); const char* fld_name = rs->column_text(1);
std::string fld_type(rs->column_text(2)); std::string fld_type(rs->column_text(2));
boost::algorithm::to_lower(fld_type); boost::algorithm::to_lower(fld_type);
// see 2.1 "Column Affinity" at http://www.sqlite.org/datatype3.html // see 2.1 "Column Affinity" at http://www.sqlite.org/datatype3.html
if (geometry_field_.empty() && if (geometry_field_.empty()
( && (boost::algorithm::contains(fld_type, "geom") ||
(boost::algorithm::icontains(fld_name,"geom") || boost::algorithm::contains(fld_type, "point") ||
boost::algorithm::icontains(fld_name,"point") || boost::algorithm::contains(fld_type, "linestring") ||
boost::algorithm::icontains(fld_name,"linestring") || boost::algorithm::contains(fld_type, "polygon")))
boost::algorithm::icontains(fld_name,"polygon"))
||
(boost::algorithm::contains(fld_type,"geom") ||
boost::algorithm::contains(fld_type,"point") ||
boost::algorithm::contains(fld_type,"linestring") ||
boost::algorithm::contains(fld_type,"polygon"))
)
)
geometry_field_ = std::string(fld_name);
else if (boost::algorithm::contains(fld_type,"int"))
{ {
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Integer)); geometry_field_ = std::string(fld_name);
} }
else if (boost::algorithm::contains(fld_type,"text") || else if (boost::algorithm::contains(fld_type, "int"))
boost::algorithm::contains(fld_type,"char") ||
boost::algorithm::contains(fld_type,"clob"))
{ {
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
} }
else if (boost::algorithm::contains(fld_type,"real") || else if (boost::algorithm::contains(fld_type, "text") ||
boost::algorithm::contains(fld_type,"float") || boost::algorithm::contains(fld_type, "char") ||
boost::algorithm::contains(fld_type,"double")) boost::algorithm::contains(fld_type, "clob"))
{ {
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
} }
else if (boost::algorithm::contains(fld_type,"blob") && !geometry_field_.empty()) else if (boost::algorithm::contains(fld_type, "real") ||
boost::algorithm::contains(fld_type, "float") ||
boost::algorithm::contains(fld_type, "double"))
{ {
desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::String)); desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double));
}
else if (boost::algorithm::contains(fld_type, "blob"))
{
if (! geometry_field_.empty())
{
desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
}
} }
#ifdef MAPNIK_DEBUG #ifdef MAPNIK_DEBUG
else else
{ {
// "Column Affinity" says default to "Numeric" but for now we pass.. // "Column Affinity" says default to "Numeric" but for now we pass..
//desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); //desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
std::clog << "Sqlite Plugin: column '" << std::string(fld_name) << "' unhandled due to unknown type: " << fld_type << std::endl;
// TODO - this should not fail when we specify geometry_field in XML file
std::clog << "Sqlite Plugin: column '"
<< std::string(fld_name)
<< "' unhandled due to unknown type: "
<< fld_type << std::endl;
} }
#endif #endif
} }
if (!found_table)
if (! found_table)
{ {
std::ostringstream s; std::ostringstream s;
s << "Sqlite Plugin: could not query table '" << geometry_table_ << "' "; s << "Sqlite Plugin: could not query table '" << geometry_table_ << "' ";
if (using_subquery_) if (using_subquery_)
s << " from subquery '" << table_ << "' "; s << " from subquery '" << table_ << "' ";
s << "using 'PRAGMA table_info(" << geometry_table_ << ")' "; s << "using 'PRAGMA table_info(" << geometry_table_ << ")' ";
std::string sq_err = std::string(sqlite3_errmsg(*(*dataset_))); std::string sq_err = std::string(sqlite3_errmsg(*(*dataset_)));
if (sq_err != "unknown error") if (sq_err != "unknown error")
s << ": " << sq_err; s << ": " << sq_err;
throw datasource_exception(s.str()); throw datasource_exception(s.str());
} }
} }
if (geometry_field_.empty()) { if (geometry_field_.empty())
{
throw datasource_exception("Sqlite Plugin: cannot detect geometry_field, please supply the name of the geometry_field to use."); throw datasource_exception("Sqlite Plugin: cannot detect geometry_field, please supply the name of the geometry_field to use.");
} }
if (index_table_.size() == 0) { if (index_table_.size() == 0)
{
// Generate implicit index_table name - need to do this after // Generate implicit index_table name - need to do this after
// we have discovered meta-data or else we don't know the column name // we have discovered meta-data or else we don't know the column name
index_table_ = "\"idx_" + mapnik::unquote_sql2(geometry_table_) + "_" + geometry_field_ + "\""; index_table_ = "\"idx_" + mapnik::unquote_sql2(geometry_table_) + "_" + geometry_field_ + "\"";
@ -420,7 +444,7 @@ void sqlite_datasource::bind() const
} }
} }
if (!extent_initialized_ && has_spatial_index_) if (! extent_initialized_ && has_spatial_index_)
{ {
std::ostringstream s; std::ostringstream s;
s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM " s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM "
@ -428,7 +452,8 @@ void sqlite_datasource::bind() const
boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str())); boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
if (rs->is_valid() && rs->step_next()) if (rs->is_valid() && rs->step_next())
{ {
if (!rs->column_isnull(0)) { if (!rs->column_isnull(0))
{
try try
{ {
double xmin = lexical_cast<double>(rs->column_double(0)); double xmin = lexical_cast<double>(rs->column_double(0));
@ -448,27 +473,37 @@ void sqlite_datasource::bind() const
} }
#ifdef MAPNIK_DEBUG #ifdef MAPNIK_DEBUG
if (!has_spatial_index_ if (! has_spatial_index_
&& use_spatial_index_ && use_spatial_index_
&& using_subquery_ && using_subquery_
&& key_field_ == "rowid" && key_field_ == "rowid"
&& !boost::algorithm::icontains(table_,"rowid")) { && ! boost::algorithm::icontains(table_,"rowid"))
{
// this is an impossible situation because rowid will be null via a subquery // this is an impossible situation because rowid will be null via a subquery
std::clog << "Sqlite Plugin: WARNING: spatial index usage will fail because rowid is not present in your subquery. You have 4 options: 1) Add rowid into your select statement, 2) alias your primary key as rowid, 3) supply a 'key_field' value that references the primary key of your spatial table, or 4) avoid using a spatial index by setting 'use_spatial_index'=false"; std::clog << "Sqlite Plugin: WARNING: spatial index usage will fail because rowid "
<< "is not present in your subquery. You have 4 options: "
<< "1) Add rowid into your select statement, "
<< "2) alias your primary key as rowid, "
<< "3) supply a 'key_field' value that references the primary key of your spatial table, or "
<< "4) avoid using a spatial index by setting 'use_spatial_index'=false" << std::endl;
} }
#endif #endif
// final fallback to gather extent // final fallback to gather extent
if (!extent_initialized_ || !has_spatial_index_) { if (! extent_initialized_ || ! has_spatial_index_)
{
std::ostringstream s; std::ostringstream s;
s << "SELECT " << geometry_field_ << "," << key_field_ s << "SELECT " << geometry_field_ << "," << key_field_
<< " FROM " << geometry_table_; << " FROM " << geometry_table_;
if (row_limit_ > 0) {
if (row_limit_ > 0)
{
s << " LIMIT " << row_limit_; s << " LIMIT " << row_limit_;
} }
if (row_offset_ > 0) {
if (row_offset_ > 0)
{
s << " OFFSET " << row_offset_; s << " OFFSET " << row_offset_;
} }
@ -485,15 +520,17 @@ void sqlite_datasource::bind() const
+ " values (?,?,?,?,?)" ; + " values (?,?,?,?,?)" ;
sqlite3_stmt* stmt = 0; sqlite3_stmt* stmt = 0;
if (use_spatial_index_) { if (use_spatial_index_)
{
dataset_->execute(spatial_index_sql); dataset_->execute(spatial_index_sql);
int rc = sqlite3_prepare_v2 (*(*dataset_), spatial_index_insert_sql.c_str(), -1, &stmt, 0); int rc = sqlite3_prepare_v2 (*(*dataset_), spatial_index_insert_sql.c_str(), -1, &stmt, 0);
if (rc != SQLITE_OK) if (rc != SQLITE_OK)
{ {
std::ostringstream index_error; std::ostringstream index_error;
index_error << "Sqlite Plugin: auto-index table creation failed: '" index_error << "Sqlite Plugin: auto-index table creation failed: '"
<< sqlite3_errmsg(*(*dataset_)) << "' query was: " << spatial_index_insert_sql; << sqlite3_errmsg(*(*dataset_)) << "' query was: "
throw datasource_exception(index_error.str()); << spatial_index_insert_sql;
throw datasource_exception(index_error.str());
} }
} }
@ -502,26 +539,33 @@ void sqlite_datasource::bind() const
{ {
int size; int size;
const char* data = (const char *) rs->column_blob (0, size); const char* data = (const char *) rs->column_blob (0, size);
if (data) { if (data)
{
// create a tmp feature to be able to parse geometry // create a tmp feature to be able to parse geometry
// ideally we would not have to do this. // ideally we would not have to do this.
// see: http://trac.mapnik.org/ticket/745 // see: http://trac.mapnik.org/ticket/745
mapnik::feature_ptr feature(mapnik::feature_factory::create(0)); mapnik::feature_ptr feature(mapnik::feature_factory::create(0));
mapnik::geometry_utils::from_wkb(feature->paths(),data,size,multiple_geometries_,format_); mapnik::geometry_utils::from_wkb(feature->paths(), data, size, multiple_geometries_, format_);
mapnik::box2d<double> const& bbox = feature->envelope(); mapnik::box2d<double> const& bbox = feature->envelope();
if (bbox.valid()) { if (bbox.valid())
{
extent_initialized_ = true; extent_initialized_ = true;
if (first) { if (first)
{
first = false; first = false;
extent_ = bbox; extent_ = bbox;
} else { }
else
{
extent_.expand_to_include(bbox); extent_.expand_to_include(bbox);
} }
// index creation // index creation
if (use_spatial_index_) { if (use_spatial_index_)
{
const int type_oid = rs->column_type(1); const int type_oid = rs->column_type(1);
if (type_oid != SQLITE_INTEGER) { if (type_oid != SQLITE_INTEGER)
{
std::ostringstream type_error; std::ostringstream type_error;
type_error << "Sqlite Plugin: invalid type for key field '" type_error << "Sqlite Plugin: invalid type for key field '"
<< key_field_ << "' when creating index '" << index_table_ << key_field_ << "' when creating index '" << index_table_
@ -534,28 +578,36 @@ void sqlite_datasource::bind() const
{ {
throw datasource_exception("invalid value for for key field while generating index"); throw datasource_exception("invalid value for for key field while generating index");
} }
if ((sqlite3_bind_double(stmt, 2 , bbox.minx() ) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 3 , bbox.maxx() ) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 4 , bbox.miny() ) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 5 , bbox.maxy() ) != SQLITE_OK)) {
throw datasource_exception("invalid value for for extent while generating index");
}
int res = sqlite3_step(stmt); if ((sqlite3_bind_double(stmt, 2 , bbox.minx() ) != SQLITE_OK) ||
if (res != SQLITE_DONE) { (sqlite3_bind_double(stmt, 3 , bbox.maxx() ) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 4 , bbox.miny() ) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 5 , bbox.maxy() ) != SQLITE_OK))
{
throw datasource_exception("invalid value for for extent while generating index");
}
const int res = sqlite3_step(stmt);
if (res != SQLITE_DONE)
{
std::ostringstream s; std::ostringstream s;
s << "SQLite Plugin: inserting bbox into rtree index failed: " s << "SQLite Plugin: inserting bbox into rtree index failed: "
<< "error code " << sqlite3_errcode(*(*dataset_)) << ": '" << "error code " << sqlite3_errcode(*(*dataset_)) << ": '"
<< sqlite3_errmsg(*(*dataset_)) << "' query was: " << spatial_index_insert_sql; << sqlite3_errmsg(*(*dataset_)) << "' query was: "
<< spatial_index_insert_sql;
throw datasource_exception(s.str()); throw datasource_exception(s.str());
} }
sqlite3_reset(stmt); sqlite3_reset(stmt);
} }
} }
else { else
{
std::ostringstream index_error; std::ostringstream index_error;
index_error << "SQLite Plugin: encountered invalid bbox at '" index_error << "SQLite Plugin: encountered invalid bbox at '"
<< key_field_ << "' == " << rs->column_integer64(1); << key_field_ << "' == " << rs->column_integer64(1);
throw datasource_exception(index_error.str()); throw datasource_exception(index_error.str());
} }
} }
@ -566,10 +618,10 @@ void sqlite_datasource::bind() const
{ {
throw datasource_exception("auto-indexing failed: set use_spatial_index=false to disable auto-indexing and avoid this error"); throw datasource_exception("auto-indexing failed: set use_spatial_index=false to disable auto-indexing and avoid this error");
} }
} }
if (!extent_initialized_) { if (!extent_initialized_)
{
std::ostringstream s; std::ostringstream s;
s << "Sqlite Plugin: extent could not be determined for table '" s << "Sqlite Plugin: extent could not be determined for table '"
<< geometry_table_ << "' and geometry field '" << geometry_field_ << "'" << geometry_table_ << "' and geometry field '" << geometry_field_ << "'"
@ -591,29 +643,32 @@ sqlite_datasource::~sqlite_datasource()
std::string sqlite_datasource::name() std::string sqlite_datasource::name()
{ {
return "sqlite"; return "sqlite";
} }
int sqlite_datasource::type() const int sqlite_datasource::type() const
{ {
return type_; return type_;
} }
box2d<double> sqlite_datasource::envelope() const box2d<double> sqlite_datasource::envelope() const
{ {
if (!is_bound_) bind(); if (!is_bound_) bind();
return extent_;
return extent_;
} }
layer_descriptor sqlite_datasource::get_descriptor() const layer_descriptor sqlite_datasource::get_descriptor() const
{ {
if (!is_bound_) bind(); if (!is_bound_) bind();
return desc_;
return desc_;
} }
featureset_ptr sqlite_datasource::features(query const& q) const featureset_ptr sqlite_datasource::features(query const& q) const
{ {
if (!is_bound_) bind(); if (!is_bound_) bind();
if (dataset_) if (dataset_)
{ {
mapnik::box2d<double> const& e = q.get_bbox(); mapnik::box2d<double> const& e = q.get_bbox();
@ -634,39 +689,39 @@ featureset_ptr sqlite_datasource::features(query const& q) const
std::string query (table_); std::string query (table_);
/* todo // TODO
throw if select * and key_field == rowid? // throw if select * and key_field == rowid?
or add schema support so sqlite throws // or add schema support so sqlite throws
*/
if (has_spatial_index_) if (has_spatial_index_)
{ {
/* // Use rtree to limit record id's to a given bbox
Use rtree to limit record id's to a given bbox // then a btree to pull the records for those ids.
then a btree to pull the records for those ids.
*/ std::ostringstream spatial_sql;
std::ostringstream spatial_sql; spatial_sql << std::setprecision(16);
spatial_sql << std::setprecision(16); spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM " << index_table_;
spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM " << index_table_; spatial_sql << " WHERE xmax>=" << e.minx() << " AND xmin<=" << e.maxx() ;
spatial_sql << " WHERE xmax>=" << e.minx() << " AND xmin<=" << e.maxx() ; spatial_sql << " AND ymax>=" << e.miny() << " AND ymin<=" << e.maxy() << ")";
spatial_sql << " AND ymax>=" << e.miny() << " AND ymin<=" << e.maxy() << ")"; if (boost::algorithm::ifind_first(query, "WHERE"))
if (boost::algorithm::ifind_first(query, "WHERE")) {
{ boost::algorithm::ireplace_first(query, "WHERE", spatial_sql.str() + " AND ");
boost::algorithm::ireplace_first(query, "WHERE", spatial_sql.str() + " AND "); }
} else if (boost::algorithm::ifind_first(query, geometry_table_))
else if (boost::algorithm::ifind_first(query, geometry_table_)) {
{ boost::algorithm::ireplace_first(query, table_, table_ + " " + spatial_sql.str());
boost::algorithm::ireplace_first(query, table_, table_ + " " + spatial_sql.str()); }
}
} }
s << query ; s << query ;
if (row_limit_ > 0) { if (row_limit_ > 0)
{
s << " LIMIT " << row_limit_; s << " LIMIT " << row_limit_;
} }
if (row_offset_ > 0) { if (row_offset_ > 0)
{
s << " OFFSET " << row_offset_; s << " OFFSET " << row_offset_;
} }
@ -677,7 +732,11 @@ featureset_ptr sqlite_datasource::features(query const& q) const
boost::shared_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str())); boost::shared_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
return boost::make_shared<sqlite_featureset>(rs, desc_.get_encoding(), format_, multiple_geometries_, using_subquery_); return boost::make_shared<sqlite_featureset>(rs,
desc_.get_encoding(),
format_,
multiple_geometries_,
using_subquery_);
} }
return featureset_ptr(); return featureset_ptr();
@ -700,7 +759,10 @@ featureset_ptr sqlite_datasource::features_at_point(coord2d const& pt) const
{ {
std::string fld_name = itr->get_name(); std::string fld_name = itr->get_name();
if (fld_name != key_field_) if (fld_name != key_field_)
{
s << ",\"" << itr->get_name() << "\""; s << ",\"" << itr->get_name() << "\"";
}
++itr; ++itr;
} }
@ -710,28 +772,30 @@ featureset_ptr sqlite_datasource::features_at_point(coord2d const& pt) const
if (has_spatial_index_) if (has_spatial_index_)
{ {
std::ostringstream spatial_sql; std::ostringstream spatial_sql;
spatial_sql << std::setprecision(16); spatial_sql << std::setprecision(16);
spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM " << index_table_; spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM " << index_table_;
spatial_sql << " WHERE xmax>=" << e.minx() << " AND xmin<=" << e.maxx() ; spatial_sql << " WHERE xmax>=" << e.minx() << " AND xmin<=" << e.maxx() ;
spatial_sql << " AND ymax>=" << e.miny() << " AND ymin<=" << e.maxy() << ")"; spatial_sql << " AND ymax>=" << e.miny() << " AND ymin<=" << e.maxy() << ")";
if (boost::algorithm::ifind_first(query, "WHERE")) if (boost::algorithm::ifind_first(query, "WHERE"))
{ {
boost::algorithm::ireplace_first(query, "WHERE", spatial_sql.str() + " AND "); boost::algorithm::ireplace_first(query, "WHERE", spatial_sql.str() + " AND ");
} }
else if (boost::algorithm::ifind_first(query, geometry_table_)) else if (boost::algorithm::ifind_first(query, geometry_table_))
{ {
boost::algorithm::ireplace_first(query, table_, table_ + " " + spatial_sql.str()); boost::algorithm::ireplace_first(query, table_, table_ + " " + spatial_sql.str());
} }
} }
s << query ; s << query ;
if (row_limit_ > 0) { if (row_limit_ > 0)
{
s << " LIMIT " << row_limit_; s << " LIMIT " << row_limit_;
} }
if (row_offset_ > 0) { if (row_offset_ > 0)
{
s << " OFFSET " << row_offset_; s << " OFFSET " << row_offset_;
} }
@ -741,9 +805,12 @@ featureset_ptr sqlite_datasource::features_at_point(coord2d const& pt) const
boost::shared_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str())); boost::shared_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
return boost::make_shared<sqlite_featureset>(rs, desc_.get_encoding(), format_, multiple_geometries_, using_subquery_); return boost::make_shared<sqlite_featureset>(rs,
desc_.get_encoding(),
format_,
multiple_geometries_,
using_subquery_);
} }
return featureset_ptr(); return featureset_ptr();
} }

3
plugins/input/sqlite/sqlite_datasource.hpp
View file

@ -37,6 +37,9 @@
// sqlite // sqlite
#include "sqlite_types.hpp" #include "sqlite_types.hpp"
//==============================================================================
class sqlite_datasource : public mapnik::datasource class sqlite_datasource : public mapnik::datasource
{ {
public: public:

177
plugins/input/sqlite/sqlite_featureset.cpp
View file

@ -31,7 +31,6 @@
#include <mapnik/wkb.hpp> #include <mapnik/wkb.hpp>
#include <mapnik/unicode.hpp> #include <mapnik/unicode.hpp>
#include <mapnik/feature_factory.hpp> #include <mapnik/feature_factory.hpp>
#include <mapnik/sql_utils.hpp>
#include <string.h> #include <string.h>
// ogr // ogr
@ -61,137 +60,152 @@ sqlite_featureset::sqlite_featureset(boost::shared_ptr<sqlite_resultset> rs,
sqlite_featureset::~sqlite_featureset() {} sqlite_featureset::~sqlite_featureset() {}
void sqlite_dequote(char *z){ // TODO - refactor, make a static member using std::string or better UnicodeString
char quote; /* Quote character (if any ) */
quote = z[0]; void sqlite_dequote(char *z)
if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ {
int iIn = 1; /* Index of next byte to read from input */ char quote = z[0];
int iOut = 0; /* Index of next byte to write to output */
/* If the first byte was a '[', then the close-quote character is a ']' */ if (quote=='[' || quote=='\'' || quote=='"' || quote=='`')
if( quote=='[' ) quote = ']'; {
int iIn = 1; // Index of next byte to read from input
int iOut = 0; // Index of next byte to write to output
while( z[iIn] ){ // If the first byte was a '[', then the close-quote character is a ']'
if( z[iIn]==quote ){ if (quote == '[')
if( z[iIn+1]!=quote ) break; {
z[iOut++] = quote; quote = ']';
iIn += 2; }
}else{
z[iOut++] = z[iIn++]; while (z[iIn])
} {
if (z[iIn] == quote)
{
if (z[iIn+1] != quote) break;
z[iOut++] = quote;
iIn += 2;
}
else
{
z[iOut++] = z[iIn++];
}
}
z[iOut] = '\0';
} }
z[iOut] = '\0';
}
} }
feature_ptr sqlite_featureset::next() feature_ptr sqlite_featureset::next()
{ {
if (rs_->is_valid () && rs_->step_next ()) if (rs_->is_valid () && rs_->step_next ())
{ {
int size; int size;
const char* data = (const char *) rs_->column_blob (0, size); const char* data = (const char *) rs_->column_blob (0, size);
if (!data) if (! data)
{
return feature_ptr(); return feature_ptr();
}
int feature_id = rs_->column_integer (1); int feature_id = rs_->column_integer (1);
feature_ptr feature(feature_factory::create(feature_id)); feature_ptr feature(feature_factory::create(feature_id));
geometry_utils::from_wkb(feature->paths(),data,size,multiple_geometries_,format_); geometry_utils::from_wkb(feature->paths(), data, size, multiple_geometries_, format_);
for (int i = 2; i < rs_->column_count (); ++i) for (int i = 2; i < rs_->column_count (); ++i)
{ {
const int type_oid = rs_->column_type (i); const int type_oid = rs_->column_type (i);
const char* fld_name = rs_->column_name(i); const char* fld_name = rs_->column_name(i);
if (!fld_name) if (! fld_name)
continue; continue;
if (!using_subquery_) if (! using_subquery_)
{ {
switch (type_oid) switch (type_oid)
{ {
case SQLITE_INTEGER: case SQLITE_INTEGER:
{ {
boost::put(*feature,fld_name,rs_->column_integer (i)); boost::put(*feature, fld_name, rs_->column_integer (i));
break; break;
} }
case SQLITE_FLOAT: case SQLITE_FLOAT:
{ {
boost::put(*feature,fld_name,rs_->column_double (i)); boost::put(*feature, fld_name, rs_->column_double (i));
break; break;
} }
case SQLITE_TEXT: case SQLITE_TEXT:
{ {
int text_size; int text_size;
const char * data = rs_->column_text(i,text_size); const char * data = rs_->column_text(i, text_size);
UnicodeString ustr = tr_->transcode(data,text_size); UnicodeString ustr = tr_->transcode(data, text_size);
boost::put(*feature,fld_name,ustr); boost::put(*feature, fld_name, ustr);
break; break;
} }
case SQLITE_NULL: case SQLITE_NULL:
{ {
boost::put(*feature,fld_name,mapnik::value_null()); boost::put(*feature,fld_name,mapnik::value_null());
break; break;
} }
case SQLITE_BLOB: case SQLITE_BLOB:
break; break;
default: default:
#ifdef MAPNIK_DEBUG #ifdef MAPNIK_DEBUG
std::clog << "Sqlite Plugin: unhandled type_oid=" << type_oid << std::endl; std::clog << "Sqlite Plugin: unhandled type_oid=" << type_oid << std::endl;
#endif #endif
break; break;
} }
} }
else else
{ {
// TODO - refactor this code, it is C99 but not valid in C++ (even if GCC allows this)
// subqueries in sqlite lead to field double quoting which we need to strip // subqueries in sqlite lead to field double quoting which we need to strip
char fld_name2[strlen(fld_name)]; char fld_name2[strlen(fld_name)];
strcpy(fld_name2,fld_name); strcpy(fld_name2,fld_name);
sqlite_dequote(fld_name2); sqlite_dequote(fld_name2);
switch (type_oid) switch (type_oid)
{ {
case SQLITE_INTEGER: case SQLITE_INTEGER:
{ {
boost::put(*feature,fld_name2,rs_->column_integer (i)); boost::put(*feature,fld_name2,rs_->column_integer (i));
break; break;
} }
case SQLITE_FLOAT: case SQLITE_FLOAT:
{ {
boost::put(*feature,fld_name2,rs_->column_double (i)); boost::put(*feature,fld_name2,rs_->column_double (i));
break; break;
} }
case SQLITE_TEXT: case SQLITE_TEXT:
{ {
int text_size; int text_size;
const char * data = rs_->column_text(i,text_size); const char * data = rs_->column_text(i,text_size);
UnicodeString ustr = tr_->transcode(data,text_size); UnicodeString ustr = tr_->transcode(data,text_size);
boost::put(*feature,fld_name2,ustr); boost::put(*feature,fld_name2,ustr);
break; break;
} }
case SQLITE_NULL: case SQLITE_NULL:
{ {
boost::put(*feature,fld_name2,mapnik::value_null()); boost::put(*feature,fld_name2,mapnik::value_null());
break; break;
} }
case SQLITE_BLOB: case SQLITE_BLOB:
break; break;
default: default:
#ifdef MAPNIK_DEBUG #ifdef MAPNIK_DEBUG
std::clog << "Sqlite Plugin: unhandled type_oid=" << type_oid << std::endl; std::clog << "Sqlite Plugin: unhandled type_oid=" << type_oid << std::endl;
#endif #endif
break; break;
} }
} }
} }
@ -201,4 +215,3 @@ feature_ptr sqlite_featureset::next()
return feature_ptr(); return feature_ptr();
} }

3
plugins/input/sqlite/sqlite_featureset.hpp
View file

@ -37,6 +37,8 @@
#include "sqlite_types.hpp" #include "sqlite_types.hpp"
//==============================================================================
class sqlite_featureset : public mapnik::Featureset class sqlite_featureset : public mapnik::Featureset
{ {
public: public:
@ -55,4 +57,5 @@ class sqlite_featureset : public mapnik::Featureset
bool using_subquery_; bool using_subquery_;
}; };
#endif // SQLITE_FEATURESET_HPP #endif // SQLITE_FEATURESET_HPP

63
plugins/input/sqlite/sqlite_types.hpp
View file

@ -36,6 +36,8 @@ extern "C" {
} }
//==============================================================================
class sqlite_resultset class sqlite_resultset
{ {
public: public:
@ -48,7 +50,9 @@ public:
~sqlite_resultset () ~sqlite_resultset ()
{ {
if (stmt_) if (stmt_)
{
sqlite3_finalize (stmt_); sqlite3_finalize (stmt_);
}
} }
bool is_valid () bool is_valid ()
@ -58,12 +62,18 @@ public:
bool step_next () bool step_next ()
{ {
int status = sqlite3_step (stmt_); const int status = sqlite3_step (stmt_);
if (status != SQLITE_ROW && status != SQLITE_DONE) { if (status != SQLITE_ROW && status != SQLITE_DONE)
{
std::ostringstream s; std::ostringstream s;
s << "SQLite Plugin: retrieving next row failed"; s << "SQLite Plugin: retrieving next row failed";
std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_))); std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_)));
if (msg != "unknown error") s << ": " << msg; if (msg != "unknown error")
{
s << ": " << msg;
}
throw mapnik::datasource_exception(s.str()); throw mapnik::datasource_exception(s.str());
} }
return status == SQLITE_ROW; return status == SQLITE_ROW;
@ -134,6 +144,7 @@ private:
}; };
//==============================================================================
class sqlite_connection class sqlite_connection
{ {
@ -144,54 +155,56 @@ public:
{ {
// sqlite3_open_v2 is available earlier but // sqlite3_open_v2 is available earlier but
// shared cache not available until >= 3.6.18 // shared cache not available until >= 3.6.18
#if SQLITE_VERSION_NUMBER >= 3006018 #if SQLITE_VERSION_NUMBER >= 3006018
int rc = sqlite3_enable_shared_cache(1); const int rc = sqlite3_enable_shared_cache(1);
if (rc != SQLITE_OK) if (rc != SQLITE_OK)
{ {
throw mapnik::datasource_exception (sqlite3_errmsg (db_)); throw mapnik::datasource_exception (sqlite3_errmsg (db_));
} }
int mode = SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE; int mode = SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE;
if (sqlite3_open_v2 (file.c_str(), &db_, mode, NULL)) if (sqlite3_open_v2 (file.c_str(), &db_, mode, NULL))
#else #else
#warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.6.18 which may make rendering slow..." #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.6.18 which may make rendering slow..."
if (sqlite3_open (file.c_str(), &db_)) if (sqlite3_open (file.c_str(), &db_))
#endif #endif
{ {
std::ostringstream s; std::ostringstream s;
s << "Sqlite Plugin: "; s << "Sqlite Plugin: " << sqlite3_errmsg (db_);
throw mapnik::datasource_exception (sqlite3_errmsg (db_));
throw mapnik::datasource_exception (s.str());
} }
//sqlite3_enable_load_extension(db_, 1);
} }
~sqlite_connection () virtual ~sqlite_connection ()
{ {
if (db_) if (db_)
{
sqlite3_close (db_); sqlite3_close (db_);
}
} }
void throw_sqlite_error(const std::string& sql) void throw_sqlite_error(const std::string& sql)
{ {
std::ostringstream s; std::ostringstream s;
s << "Sqlite Plugin: "; s << "Sqlite Plugin: ";
if (db_) if (db_)
s << "'" << sqlite3_errmsg(db_) << "'"; s << "'" << sqlite3_errmsg(db_) << "'";
else else
s << "unknown error, lost connection"; s << "unknown error, lost connection";
s << "\nFull sql was: '" << sql << "'\n";
s << "\nFull sql was: '" << sql << "'\n"; throw mapnik::datasource_exception (s.str());
throw mapnik::datasource_exception( s.str() );
} }
sqlite_resultset* execute_query(const std::string& sql) sqlite_resultset* execute_query(const std::string& sql)
{ {
sqlite3_stmt* stmt = 0; sqlite3_stmt* stmt = 0;
int rc = sqlite3_prepare_v2 (db_, sql.c_str(), -1, &stmt, 0); const int rc = sqlite3_prepare_v2 (db_, sql.c_str(), -1, &stmt, 0);
if (rc != SQLITE_OK) if (rc != SQLITE_OK)
{ {
throw_sqlite_error(sql); throw_sqlite_error(sql);
} }
return new sqlite_resultset (stmt); return new sqlite_resultset (stmt);
@ -199,7 +212,7 @@ public:
void execute(const std::string& sql) void execute(const std::string& sql)
{ {
int rc=sqlite3_exec(db_, sql.c_str(), 0, 0, 0); const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
if (rc != SQLITE_OK) if (rc != SQLITE_OK)
{ {
throw_sqlite_error(sql); throw_sqlite_error(sql);
@ -208,7 +221,7 @@ public:
int execute_with_code(const std::string& sql) int execute_with_code(const std::string& sql)
{ {
int rc=sqlite3_exec(db_, sql.c_str(), 0, 0, 0); const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
return rc; return rc;
} }
@ -222,5 +235,5 @@ private:
sqlite3* db_; sqlite3* db_;
}; };
#endif //SQLITE_TYPES_HPP
#endif //SQLITE_TYPES_HPP

18
src/wkb.cpp
View file

@ -39,6 +39,7 @@ private:
wkbXDR=0, wkbXDR=0,
wkbNDR=1 wkbNDR=1
}; };
const char* wkb_; const char* wkb_;
unsigned size_; unsigned size_;
unsigned pos_; unsigned pos_;
@ -64,6 +65,23 @@ public:
pos_(0), pos_(0),
format_(format) format_(format)
{ {
// try to determine WKB format automatically
if (format_ == wkbAuto)
{
if (size > 38
&& wkb_[0] == 0x00
&& (wkb_[1] == 0x00 || wkb_[1] == 0x01)
&& wkb_[38] == 0x7C
&& wkb_[size - 1] == 0xFE)
{
format_ = wkbSpatiaLite;
}
else
{
format_ = wkbGeneric;
}
}
switch (format_) switch (format_)
{ {
case wkbSpatiaLite: case wkbSpatiaLite: