From 03137cf523356585aed5b8b8a5169e176dcda6ec Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Fri, 28 Oct 2011 21:24:47 -0700 Subject: [PATCH 1/5] first pass at refactoring bind() to be more manageable in sqlite plugin - refs #928 --- plugins/input/sqlite/sqlite_datasource.cpp | 566 +++++--------------- plugins/input/sqlite/sqlite_datasource.hpp | 2 +- plugins/input/sqlite/sqlite_types.hpp | 578 ++++++++++++++++++--- 3 files changed, 634 insertions(+), 512 deletions(-) diff --git a/plugins/input/sqlite/sqlite_datasource.cpp b/plugins/input/sqlite/sqlite_datasource.cpp index 4e6f3b8aa..caf98c974 100644 --- a/plugins/input/sqlite/sqlite_datasource.cpp +++ b/plugins/input/sqlite/sqlite_datasource.cpp @@ -26,18 +26,13 @@ // mapnik #include #include -#include // to enable extent fallback hack // boost #include -#include -#include #include #include #include - -using boost::lexical_cast; -using boost::bad_lexical_cast; +#include using mapnik::box2d; using mapnik::coord2d; @@ -68,9 +63,10 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind) desc_(*params_.get("type"), *params_.get("encoding", "utf-8")), format_(mapnik::wkbAuto) { - // TODO - // - change param from 'file' to 'dbname' - // - ensure that the supplied key_field is a valid "integer primary key" + /* TODO + - auto detect primary key - https://github.com/mapnik/mapnik/issues/82 + - throw if no primary key but spatial index is present? + */ boost::optional file = params_.get("file"); if (! file) throw datasource_exception("Sqlite Plugin: missing parameter"); @@ -92,11 +88,18 @@ void sqlite_datasource::bind() const boost::optional file = params_.get("file"); if (! file) throw datasource_exception("Sqlite Plugin: missing parameter"); - - // Be careful, automatically using rowid as key field of the table can lead - // to misleading results http://www.gaia-gis.it/spatialite-2.4.0-5/SpatialIndex-Update.pdf - // TODO - We should try to avoid this and eventually force to specify a valid table primary key + boost::optional base = params_.get("base"); + if (base) + dataset_name_ = *base + "/" + *file; + else + dataset_name_ = *file; + + if (!boost::filesystem::exists(dataset_name_)) + { + throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist"); + } + boost::optional key_field_name = params_.get("key_field"); if (key_field_name) { @@ -116,18 +119,10 @@ void sqlite_datasource::bind() const multiple_geometries_ = *params_.get("multiple_geometries", false); use_spatial_index_ = *params_.get("use_spatial_index", true); - has_spatial_index_ = false; - using_subquery_ = false; boost::optional ext = params_.get("extent"); if (ext) extent_initialized_ = extent_.from_string(*ext); - boost::optional base = params_.get("base"); - if (base) - dataset_name_ = *base + "/" + *file; - else - dataset_name_ = *file; - // Populate init_statements_ // 1. Build attach database statements from the "attachdb" parameter // 2. Add explicit init statements from "initdb" parameter @@ -147,10 +142,23 @@ void sqlite_datasource::bind() const init_statements_.push_back(*initdb); } - if (!boost::filesystem::exists(dataset_name_)) - throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist"); - - dataset_ = new sqlite_connection(dataset_name_); + if (geometry_table_.empty()) + { + geometry_table_ = mapnik::table_from_sql(table_); + } + + // if 'table_' is a subquery then we try to deduce names + // and types from the first row returned from that query + using_subquery_ = false; + if (table_ != geometry_table_) + { + using_subquery_ = true; + } + + std::string index_db = sqlite_utils::get_index_db_name(dataset_name_); + + // now actually create the connection and start executing setup sql + dataset_ = boost::make_shared(dataset_name_); // Execute init_statements_ for (std::vector::const_iterator iter = init_statements_.begin(); @@ -162,193 +170,37 @@ void sqlite_datasource::bind() const dataset_->execute(*iter); } - if (geometry_table_.empty()) - { - geometry_table_ = mapnik::table_from_sql(table_); - } - - // should we deduce column names and types using PRAGMA? - bool use_pragma_table_info = true; - - if (table_ != geometry_table_) - { - // if 'table_' is a subquery then we try to deduce names - // and types from the first row returned from that query - using_subquery_ = true; - use_pragma_table_info = false; - - if (key_field_.empty()) - { - key_field_ = "rowid"; - } - } - - if (! use_pragma_table_info) + bool found_types_via_subquery = false; + if (using_subquery_) { std::ostringstream s; s << "SELECT " << fields_ << " FROM (" << table_ << ") LIMIT 1"; - - boost::scoped_ptr rs(dataset_->execute_query(s.str())); - if (rs->is_valid() && rs->step_next()) - { - for (int i = 0; i < rs->column_count(); ++i) - { - const int type_oid = rs->column_type(i); - const char* fld_name = rs->column_name(i); - switch (type_oid) - { - case SQLITE_INTEGER: - desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); - break; - - case SQLITE_FLOAT: - desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double)); - break; - - case SQLITE_TEXT: - desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); - break; - - case SQLITE_NULL: - // sqlite reports based on value, not actual column type unless - // PRAGMA table_info is used so here we assume the column is a string - // which is a lesser evil than altogether dropping the column - desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); - - case SQLITE_BLOB: - if (geometry_field_.empty() - && (boost::algorithm::icontains(fld_name, "geom") || - boost::algorithm::icontains(fld_name, "point") || - boost::algorithm::icontains(fld_name, "linestring") || - boost::algorithm::icontains(fld_name, "polygon"))) - { - geometry_field_ = std::string(fld_name); - } - break; - - default: -#ifdef MAPNIK_DEBUG - std::clog << "Sqlite Plugin: unknown type_oid=" << type_oid << std::endl; -#endif - break; - } - } - } - else - { - // if we do not have at least a row and - // we cannot determine the right columns types and names - // as all column_type are SQLITE_NULL - // so we fallback to using PRAGMA table_info - use_pragma_table_info = true; - } + found_types_via_subquery = dataset_->detect_types_from_subquery(s.str(),geometry_field_,desc_); } - if (key_field_.empty()) + // TODO - consider removing this + if (key_field_ == "rowid") { - use_pragma_table_info = true; - } - else - { - // TODO - we can't trust so much the rowid here - if (key_field_ == "rowid") - { - desc_.add_descriptor(attribute_descriptor("rowid", mapnik::Integer)); - } + desc_.add_descriptor(attribute_descriptor("rowid", mapnik::Integer)); } - if (use_pragma_table_info) + bool found_table = dataset_->table_info(key_field_, + found_types_via_subquery, + geometry_field_, + geometry_table_, + desc_); + + if (! found_table) { std::ostringstream s; - s << "PRAGMA table_info(" << geometry_table_ << ")"; + s << "Sqlite Plugin: could not query table '" << geometry_table_ << "' "; + if (using_subquery_) s << " from subquery '" << table_ << "' "; + s << "using 'PRAGMA table_info(" << geometry_table_ << ")' "; - boost::scoped_ptr rs(dataset_->execute_query(s.str())); - bool found_table = false; - bool found_pk = false; - while (rs->is_valid() && rs->step_next()) - { - found_table = true; + std::string sq_err = std::string(sqlite3_errmsg(*(*dataset_))); + if (sq_err != "unknown error") s << ": " << sq_err; - // TODO - support unicode strings - const char* fld_name = rs->column_text(1); - std::string fld_type(rs->column_text(2)); - int fld_pk = rs->column_integer(5); - boost::algorithm::to_lower(fld_type); - - // TODO - how to handle primary keys on multiple columns ? - if (key_field_.empty() && ! found_pk && fld_pk != 0) - { - key_field_ = fld_name; - found_pk = true; - } - - // see 2.1 "Column Affinity" at http://www.sqlite.org/datatype3.html - // TODO - refactor this somehow ? - if (geometry_field_.empty() - && ((boost::algorithm::contains(fld_type, "geom") || - boost::algorithm::contains(fld_type, "point") || - boost::algorithm::contains(fld_type, "linestring") || - boost::algorithm::contains(fld_type, "polygon")) - || - (boost::algorithm::icontains(fld_name, "geom") || - boost::algorithm::icontains(fld_name, "point") || - boost::algorithm::icontains(fld_name, "linestring") || - boost::algorithm::icontains(fld_name, "polygon"))) - ) - { - geometry_field_ = std::string(fld_name); - } - else if (boost::algorithm::contains(fld_type, "int")) - { - desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); - } - else if (boost::algorithm::contains(fld_type, "text") || - boost::algorithm::contains(fld_type, "char") || - boost::algorithm::contains(fld_type, "clob")) - { - desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); - } - else if (boost::algorithm::contains(fld_type, "real") || - boost::algorithm::contains(fld_type, "float") || - boost::algorithm::contains(fld_type, "double")) - { - desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double)); - } - else if (boost::algorithm::contains(fld_type, "blob")) - { - if (! geometry_field_.empty()) - { - desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); - } - } -#ifdef MAPNIK_DEBUG - else - { - // "Column Affinity" says default to "Numeric" but for now we pass.. - //desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); - - // TODO - this should not fail when we specify geometry_field in XML file - - std::clog << "Sqlite Plugin: column '" - << std::string(fld_name) - << "' unhandled due to unknown type: " - << fld_type << std::endl; - } -#endif - } - - if (! found_table) - { - std::ostringstream s; - s << "Sqlite Plugin: could not query table '" << geometry_table_ << "' "; - if (using_subquery_) s << " from subquery '" << table_ << "' "; - s << "using 'PRAGMA table_info(" << geometry_table_ << ")' "; - - std::string sq_err = std::string(sqlite3_errmsg(*(*dataset_))); - if (sq_err != "unknown error") s << ": " << sq_err; - - throw datasource_exception(s.str()); - } + throw datasource_exception(s.str()); } if (geometry_field_.empty()) @@ -360,251 +212,80 @@ void sqlite_datasource::bind() const { // Generate implicit index_table name - need to do this after // we have discovered meta-data or else we don't know the column name - index_table_ = "\"idx_" + mapnik::unquote_sql2(geometry_table_) + "_" + geometry_field_ + "\""; + index_table_ = sqlite_utils::index_for_table(geometry_table_,geometry_field_); } - + + has_spatial_index_ = false; if (use_spatial_index_) { - std::ostringstream s; - s << "SELECT pkid,xmin,xmax,ymin,ymax FROM " << index_table_; - s << " LIMIT 0"; - - if (dataset_->execute_with_code(s.str()) == SQLITE_OK) + /*if (boost::filesystem::exists(index_db)) { - has_spatial_index_ = true; - } -#ifdef MAPNIK_DEBUG - else - { - std::clog << "SQLite Plugin: rtree index lookup did not succeed: '" << sqlite3_errmsg(*(*dataset_)) << "'\n"; - } -#endif - } - - if (! metadata_.empty() && ! extent_initialized_) - { - std::ostringstream s; - s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata_; - s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table_ << "')"; - - boost::scoped_ptr rs(dataset_->execute_query(s.str())); - if (rs->is_valid() && rs->step_next()) - { - double xmin = rs->column_double (0); - double ymin = rs->column_double (1); - double xmax = rs->column_double (2); - double ymax = rs->column_double (3); - - extent_.init (xmin, ymin, xmax, ymax); - extent_initialized_ = true; - } - } - - if (! extent_initialized_ && has_spatial_index_) - { - std::ostringstream s; - s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM " - << index_table_; - - boost::scoped_ptr rs(dataset_->execute_query(s.str())); - if (rs->is_valid() && rs->step_next()) - { - if (! rs->column_isnull(0)) - { - try - { - double xmin = lexical_cast(rs->column_double(0)); - double ymin = lexical_cast(rs->column_double(1)); - double xmax = lexical_cast(rs->column_double(2)); - double ymax = lexical_cast(rs->column_double(3)); - extent_.init(xmin, ymin, xmax, ymax); - extent_initialized_ = true; - } - catch (bad_lexical_cast& ex) - { - std::clog << boost::format("SQLite Plugin: warning: could not determine extent from query: %s\nError was: '%s'\n") % s.str() % ex.what() << std::endl; - } - } - } - } - -#ifdef MAPNIK_DEBUG - if (! has_spatial_index_ && use_spatial_index_ && using_subquery_ - && key_field_ == "rowid" && ! boost::algorithm::icontains(table_, "rowid")) - { - // this is an impossible situation because rowid will be null via a subquery - std::clog << "Sqlite Plugin: WARNING: spatial index usage will fail because rowid " - << "is not present in your subquery. You have 4 options: " - << "1) Add rowid into your select statement, " - << "2) alias your primary key as rowid, " - << "3) supply a 'key_field' value that references the primary key of your spatial table, or " - << "4) avoid using a spatial index by setting 'use_spatial_index'=false" << std::endl; - } -#endif - - // final fallback to gather extent - if (! extent_initialized_ || ! has_spatial_index_) - { - std::ostringstream s; - s << "SELECT " << geometry_field_ << "," << key_field_ - << " FROM " << geometry_table_; - - if (row_limit_ > 0) - { - s << " LIMIT " << row_limit_; - } - - if (row_offset_ > 0) - { - s << " OFFSET " << row_offset_; - } - -#ifdef MAPNIK_DEBUG - std::clog << "Sqlite Plugin: " << s.str() << std::endl; -#endif - - boost::shared_ptr rs(dataset_->execute_query(s.str())); - - // spatial index sql - std::ostringstream spatial_index_sql; - spatial_index_sql << "create virtual table " << index_table_ - << " using rtree(pkid, xmin, xmax, ymin, ymax)"; - - std::ostringstream spatial_index_insert_sql; - spatial_index_insert_sql << "insert into " << index_table_ - << " values (?,?,?,?,?)"; - - sqlite3_stmt* stmt = 0; - if (use_spatial_index_) - { - dataset_->execute(spatial_index_sql.str()); - - const int rc = sqlite3_prepare_v2 (*(*dataset_), - spatial_index_insert_sql.str().c_str(), - -1, - &stmt, - 0); - if (rc != SQLITE_OK) - { - std::ostringstream index_error; - index_error << "Sqlite Plugin: auto-index table creation failed: '" - << sqlite3_errmsg(*(*dataset_)) << "' query was: " - << spatial_index_insert_sql; - - throw datasource_exception(index_error.str()); - } - } - - bool first = true; - while (rs->is_valid() && rs->step_next()) - { - int size; - const char* data = (const char*) rs->column_blob(0, size); - if (data) - { - // create a tmp feature to be able to parse geometry - // ideally we would not have to do this. - // see: http://trac.mapnik.org/ticket/745 - mapnik::feature_ptr feature(mapnik::feature_factory::create(0)); - mapnik::geometry_utils::from_wkb(feature->paths(), data, size, multiple_geometries_, format_); - mapnik::box2d const& bbox = feature->envelope(); - - if (bbox.valid()) - { - extent_initialized_ = true; - if (first) - { - first = false; - extent_ = bbox; - } - else - { - extent_.expand_to_include(bbox); - } - - // index creation - if (use_spatial_index_) - { - const int type_oid = rs->column_type(1); - if (type_oid != SQLITE_INTEGER) - { - std::ostringstream type_error; - type_error << "Sqlite Plugin: invalid type for key field '" - << key_field_ << "' when creating index '" << index_table_ - << "' type was: " << type_oid << ""; - - throw datasource_exception(type_error.str()); - } - - const sqlite_int64 pkid = rs->column_integer64(1); - if (sqlite3_bind_int64(stmt, 1, pkid) != SQLITE_OK) - { - throw datasource_exception("invalid value for for key field while generating index"); - } - - if ((sqlite3_bind_double(stmt, 2, bbox.minx()) != SQLITE_OK) || - (sqlite3_bind_double(stmt, 3, bbox.maxx()) != SQLITE_OK) || - (sqlite3_bind_double(stmt, 4, bbox.miny()) != SQLITE_OK) || - (sqlite3_bind_double(stmt, 5, bbox.maxy()) != SQLITE_OK)) - { - throw datasource_exception("invalid value for for extent while generating index"); - } - - const int res = sqlite3_step(stmt); - if (res != SQLITE_DONE) - { - std::ostringstream s; - s << "SQLite Plugin: inserting bbox into rtree index failed: " - << "error code " << sqlite3_errcode(*(*dataset_)) << ": '" - << sqlite3_errmsg(*(*dataset_)) << "' query was: " - << spatial_index_insert_sql; - - throw datasource_exception(s.str()); - } - - sqlite3_reset(stmt); - } - } - else - { - std::ostringstream index_error; - index_error << "SQLite Plugin: encountered invalid bbox at '" - << key_field_ << "' == " << rs->column_integer64(1); - - throw datasource_exception(index_error.str()); - } - } - } - - const int res = sqlite3_finalize(stmt); - if (res != SQLITE_OK) - { - throw datasource_exception("auto-indexing failed: set use_spatial_index=false to disable auto-indexing and avoid this error"); + dataset_->execute("attach database '" + index_db + "' as " + index_table_); } + */ + has_spatial_index_ = dataset_->has_rtree(index_table_); } if (! extent_initialized_) { - std::ostringstream s; - s << "Sqlite Plugin: extent could not be determined for table '" - << geometry_table_ << "' and geometry field '" << geometry_field_ << "'" - << " because an rtree spatial index is missing or empty." - << " - either set the table 'extent' or create an rtree spatial index"; - - throw datasource_exception(s.str()); - } + if (!sqlite_utils::detect_extent(dataset_, + has_spatial_index_, + extent_, + index_table_, + metadata_, + geometry_field_, + geometry_table_, + key_field_, + table_)) + { + std::ostringstream s; + s << "Sqlite Plugin: extent could not be determined for table '" + << geometry_table_ << "' and geometry field '" << geometry_field_ << "'" + << " because an rtree spatial index is missing or empty." + << " - either set the table 'extent' or create an rtree spatial index"; + throw datasource_exception(s.str()); + } + } is_bound_ = true; } sqlite_datasource::~sqlite_datasource() { - if (is_bound_) - { - delete dataset_; - } } +#if (BOOST_FILESYSTEM_VERSION <= 2) +namespace boost { +namespace filesystem { + path read_symlink(const path& p) + { + path symlink_path; + +#ifdef BOOST_POSIX_API + for (std::size_t path_max = 64;; path_max *= 2)// loop 'til buffer large enough + { + boost::scoped_array buf(new char[path_max]); + ssize_t result; + if ((result=::readlink(p.string().c_str(), buf.get(), path_max))== -1) + { + throw std::runtime_error("could not read symlink"); + } + else + { + if(result != static_cast(path_max)) + { + symlink_path.assign(buf.get(), buf.get() + result); + break; + } + } + } +#endif + return symlink_path; + } +} +} +#endif + void sqlite_datasource::parse_attachdb(std::string const& attachdb) const { boost::char_separator sep(","); @@ -638,23 +319,15 @@ void sqlite_datasource::parse_attachdb(std::string const& attachdb) const boost::filesystem::path absolute_path(dataset_name_); // support symlinks - #if (BOOST_FILESYSTEM_VERSION == 3) if (boost::filesystem::is_symlink(absolute_path)) { absolute_path = boost::filesystem::read_symlink(absolute_path); } + #if (BOOST_FILESYSTEM_VERSION == 3) filename = boost::filesystem::absolute(absolute_path.parent_path() / filename).string(); - #else - if (boost::filesystem::is_symlink(absolute_path)) - { - //cannot figure out how to resolve on in v2 so just print a warning - std::clog << "###Warning: '" << absolute_path.string() << "' is a symlink which is not supported in attachdb\n"; - } - filename = boost::filesystem::complete(absolute_path.branch_path() / filename).normalize().string(); - #endif } } @@ -698,7 +371,9 @@ featureset_ptr sqlite_datasource::features(query const& q) const std::ostringstream s; - s << "SELECT " << geometry_field_ << "," << key_field_; + s << "SELECT " << geometry_field_; + if (!key_field_.empty()) + s << "," << key_field_; std::set const& props = q.property_names(); std::set::const_iterator pos = props.begin(); std::set::const_iterator end = props.end(); @@ -712,15 +387,8 @@ featureset_ptr sqlite_datasource::features(query const& q) const std::string query (table_); - // TODO - // throw if select * and key_field == rowid? - // or add schema support so sqlite throws - - if (has_spatial_index_) + if (! key_field_.empty() && has_spatial_index_) { - // Use rtree to limit record id's to a given bbox - // then a btree to pull the records for those ids. - std::ostringstream spatial_sql; spatial_sql << std::setprecision(16); spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM " << index_table_; @@ -775,7 +443,9 @@ featureset_ptr sqlite_datasource::features_at_point(coord2d const& pt) const mapnik::box2d const e(pt.x, pt.y, pt.x, pt.y); std::ostringstream s; - s << "SELECT " << geometry_field_ << "," << key_field_; + s << "SELECT " << geometry_field_; + if (!key_field_.empty()) + s << "," << key_field_; std::vector::const_iterator itr = desc_.get_descriptors().begin(); std::vector::const_iterator end = desc_.get_descriptors().end(); while (itr != end) @@ -793,7 +463,7 @@ featureset_ptr sqlite_datasource::features_at_point(coord2d const& pt) const std::string query(table_); - if (has_spatial_index_) + if (! key_field_.empty() && has_spatial_index_) { std::ostringstream spatial_sql; spatial_sql << std::setprecision(16); diff --git a/plugins/input/sqlite/sqlite_datasource.hpp b/plugins/input/sqlite/sqlite_datasource.hpp index 9d83b2855..c62b32208 100644 --- a/plugins/input/sqlite/sqlite_datasource.hpp +++ b/plugins/input/sqlite/sqlite_datasource.hpp @@ -55,7 +55,7 @@ private: mutable bool extent_initialized_; int type_; mutable std::string dataset_name_; - mutable sqlite_connection* dataset_; + mutable boost::shared_ptr dataset_; std::string table_; std::string fields_; std::string metadata_; diff --git a/plugins/input/sqlite/sqlite_types.hpp b/plugins/input/sqlite/sqlite_types.hpp index 3da0e55b3..55a86686e 100644 --- a/plugins/input/sqlite/sqlite_types.hpp +++ b/plugins/input/sqlite/sqlite_types.hpp @@ -24,14 +24,19 @@ #define SQLITE_TYPES_HPP // stl -#include +#include // mapnik #include +#include +#include // to enable extent fallback hack + // boost #include +#include #include +#include // sqlite extern "C" { @@ -39,25 +44,19 @@ extern "C" { } -class sqlite_utils -{ -public: - static void dequote(std::string& z) - { - boost::algorithm::trim_if(z, boost::algorithm::is_any_of("[]'\"`")); - } -}; +//============================================================================== class sqlite_resultset { public: - sqlite_resultset(sqlite3_stmt* stmt) + + sqlite_resultset (sqlite3_stmt* stmt) : stmt_(stmt) { } - virtual ~sqlite_resultset() + ~sqlite_resultset () { if (stmt_) { @@ -65,19 +64,18 @@ public: } } - bool is_valid() + bool is_valid () { return stmt_ != 0; } - bool step_next() + bool step_next () { - const int status = sqlite3_step(stmt_); + const int status = sqlite3_step (stmt_); if (status != SQLITE_ROW && status != SQLITE_DONE) { std::ostringstream s; s << "SQLite Plugin: retrieving next row failed"; - std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_))); if (msg != "unknown error") { @@ -89,56 +87,56 @@ public: return status == SQLITE_ROW; } - int column_count() + int column_count () { - return sqlite3_column_count(stmt_); + return sqlite3_column_count (stmt_); } - int column_type(int col) + int column_type (int col) { - return sqlite3_column_type(stmt_, col); + return sqlite3_column_type (stmt_, col); } - const char* column_name(int col) + const char* column_name (int col) { - return sqlite3_column_name(stmt_, col); + return sqlite3_column_name (stmt_, col); } - bool column_isnull(int col) + bool column_isnull (int col) { - return sqlite3_column_type(stmt_, col) == SQLITE_NULL; + return sqlite3_column_type (stmt_, col) == SQLITE_NULL; } - int column_integer(int col) + int column_integer (int col) { - return sqlite3_column_int(stmt_, col); + return sqlite3_column_int (stmt_, col); } - int column_integer64(int col) + int column_integer64 (int col) { - return sqlite3_column_int64(stmt_, col); + return sqlite3_column_int64 (stmt_, col); } - double column_double(int col) + double column_double (int col) { - return sqlite3_column_double(stmt_, col); + return sqlite3_column_double (stmt_, col); } - const char* column_text(int col, int& len) + const char* column_text (int col, int& len) { - len = sqlite3_column_bytes(stmt_, col); - return (const char*) sqlite3_column_text(stmt_, col); + len = sqlite3_column_bytes (stmt_, col); + return (const char*) sqlite3_column_text (stmt_, col); } - const char* column_text(int col) + const char* column_text (int col) { - return (const char*) sqlite3_column_text(stmt_, col); + return (const char*) sqlite3_column_text (stmt_, col); } - const void* column_blob(int col, int& bytes) + const void* column_blob (int col, int& bytes) { - bytes = sqlite3_column_bytes(stmt_, col); - return (const char*) sqlite3_column_blob(stmt_, col); + bytes = sqlite3_column_bytes (stmt_, col); + return (const char*) sqlite3_column_blob (stmt_, col); } sqlite3_stmt* get_statement() @@ -152,41 +150,62 @@ private: }; +//============================================================================== + class sqlite_connection { public: - sqlite_connection(const std::string& file) - : db_(0) + sqlite_connection (const std::string& file) + : db_(0), + file_(file) { - // sqlite3_open_v2 is available earlier but - // shared cache not available until >= 3.6.18 - #if SQLITE_VERSION_NUMBER >= 3006018 - const int rc = sqlite3_enable_shared_cache(1); +#if SQLITE_VERSION_NUMBER >= 3005000 + int mode = SQLITE_OPEN_READWRITE; + #if SQLITE_VERSION_NUMBER >= 3006018 + // shared cache flag not available until >= 3.6.18 + mode |= SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE; + #endif + const int rc = sqlite3_open_v2 (file_.c_str(), &db_, mode, 0); +#else + #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..." + const int rc = sqlite3_open (file_.c_str(), &db_); +#endif if (rc != SQLITE_OK) { - throw mapnik::datasource_exception(sqlite3_errmsg (db_)); + std::ostringstream s; + s << "Sqlite Plugin: " << sqlite3_errmsg (db_); + + throw mapnik::datasource_exception (s.str()); } - int mode = SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE; - if (sqlite3_open_v2(file.c_str(), &db_, mode, NULL)) - #else - #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.6.18 which may make rendering slow..." - if (sqlite3_open(file.c_str(), &db_)) - #endif + sqlite3_busy_timeout(db_,5000); + } + + sqlite_connection (const std::string& file, int flags) + : db_(0), + file_(file) + { +#if SQLITE_VERSION_NUMBER >= 3005000 + const int rc = sqlite3_open_v2 (file_.c_str(), &db_, flags, 0); +#else + #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..." + const int rc = sqlite3_open (file_.c_str(), &db_); +#endif + if (rc != SQLITE_OK) { std::ostringstream s; - s << "Sqlite Plugin: " << sqlite3_errmsg(db_); + s << "Sqlite Plugin: " << sqlite3_errmsg (db_); - throw mapnik::datasource_exception(s.str()); + throw mapnik::datasource_exception (s.str()); } } - virtual ~sqlite_connection() + virtual ~sqlite_connection () { if (db_) { - sqlite3_close(db_); + sqlite3_close (db_); } } @@ -194,17 +213,13 @@ public: { std::ostringstream s; s << "Sqlite Plugin: "; - if (db_) - { s << "'" << sqlite3_errmsg(db_) << "'"; - } else - { s << "unknown error, lost connection"; - } - - s << "\nFull sql was: '" << sql << "'\n"; + s << " (" << file_ << ")" + << "\nFull sql was: '" + << sql << "'"; throw mapnik::datasource_exception (s.str()); } @@ -213,7 +228,7 @@ public: { sqlite3_stmt* stmt = 0; - const int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, 0); + const int rc = sqlite3_prepare_v2 (db_, sql.c_str(), -1, &stmt, 0); if (rc != SQLITE_OK) { throw_sqlite_error(sql); @@ -242,9 +257,446 @@ public: return db_; } + bool has_rtree(std::string const& index_table) + { + try + { + std::ostringstream s; + s << "SELECT pkid,xmin,xmax,ymin,ymax FROM " << index_table << " LIMIT 1"; + boost::scoped_ptr rs(execute_query(s.str())); + if (rs->is_valid() && rs->step_next()) + { + return true; + } + } + catch (...) + { + return false; + } + return false; + } + + bool detect_types_from_subquery(std::string const& query, + std::string & geometry_field, + mapnik::layer_descriptor & desc) + { + bool found = false; + boost::scoped_ptr rs(execute_query(query)); + if (rs->is_valid() && rs->step_next()) + { + for (int i = 0; i < rs->column_count(); ++i) + { + found = true; + const int type_oid = rs->column_type(i); + const char* fld_name = rs->column_name(i); + switch (type_oid) + { + case SQLITE_INTEGER: + desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::Integer)); + break; + + case SQLITE_FLOAT: + desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::Double)); + break; + + case SQLITE_TEXT: + desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::String)); + break; + + case SQLITE_NULL: + // sqlite reports based on value, not actual column type unless + // PRAGMA table_info is used so here we assume the column is a string + // which is a lesser evil than altogether dropping the column + desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::String)); + + case SQLITE_BLOB: + if (geometry_field.empty() + && (boost::algorithm::icontains(fld_name, "geom") || + boost::algorithm::icontains(fld_name, "point") || + boost::algorithm::icontains(fld_name, "linestring") || + boost::algorithm::icontains(fld_name, "polygon"))) + { + geometry_field = std::string(fld_name); + } + break; + + default: +#ifdef MAPNIK_DEBUG + std::clog << "Sqlite Plugin: unknown type_oid=" << type_oid << std::endl; +#endif + break; + } + } + } + + return found; + } + + bool table_info(std::string & key_field, + bool detected_types, + std::string & field, + std::string & table, + mapnik::layer_descriptor & desc) + { + + // http://www.sqlite.org/pragma.html#pragma_table_info + // use pragma table_info to detect primary key + // and to detect types if no subquery is used or + // if the subquery-based type detection failed + std::ostringstream s; + s << "PRAGMA table_info(" << table << ")"; + boost::scoped_ptr rs(execute_query(s.str())); + bool found_table = false; + bool found_pk = false; + while (rs->is_valid() && rs->step_next()) + { + found_table = true; + const char* fld_name = rs->column_text(1); + std::string fld_type(rs->column_text(2)); + int fld_pk = rs->column_integer(5); + boost::algorithm::to_lower(fld_type); + + // TODO - how to handle primary keys on multiple columns ? + if (key_field.empty() && ! found_pk && fld_pk != 0) + { + key_field = fld_name; + found_pk = true; + } + if (! detected_types) + { + // see 2.1 "Column Affinity" at http://www.sqlite.org/datatype3.html + // TODO - refactor this somehow ? + if (field.empty() + && ((boost::algorithm::contains(fld_type, "geom") || + boost::algorithm::contains(fld_type, "point") || + boost::algorithm::contains(fld_type, "linestring") || + boost::algorithm::contains(fld_type, "polygon")) + || + (boost::algorithm::icontains(fld_name, "geom") || + boost::algorithm::icontains(fld_name, "point") || + boost::algorithm::icontains(fld_name, "linestring") || + boost::algorithm::icontains(fld_name, "polygon"))) + ) + { + field = std::string(fld_name); + } + else if (boost::algorithm::contains(fld_type, "int")) + { + desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::Integer)); + } + else if (boost::algorithm::contains(fld_type, "text") || + boost::algorithm::contains(fld_type, "char") || + boost::algorithm::contains(fld_type, "clob")) + { + desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::String)); + } + else if (boost::algorithm::contains(fld_type, "real") || + boost::algorithm::contains(fld_type, "float") || + boost::algorithm::contains(fld_type, "double")) + { + desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::Double)); + } + else if (boost::algorithm::contains(fld_type, "blob")) + { + if (! field.empty()) + { + desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::String)); + } + } + #ifdef MAPNIK_DEBUG + else + { + // "Column Affinity" says default to "Numeric" but for now we pass.. + //desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double)); + + // TODO - this should not fail when we specify geometry_field in XML file + + std::clog << "Sqlite Plugin: column '" + << std::string(fld_name) + << "' unhandled due to unknown type: " + << fld_type << std::endl; + } + #endif + } + } + + return found_table; + } + private: sqlite3* db_; + std::string file_; }; -#endif // SQLITE_TYPES_HPP + +//============================================================================== + +class sqlite_utils +{ +public: + + static void dequote(std::string & z) + { + boost::algorithm::trim_if(z,boost::algorithm::is_any_of("[]'\"`")); + } + + static std::string index_for_table(std::string const& table,std::string const& field) + { + return "\"idx_" + mapnik::unquote_sql2(table) + "_" + field + "\""; + } + + static std::string get_index_db_name(std::string const& file) + { + //std::size_t idx = file.find_last_of("."); + //if(idx != std::string::npos) { + // return file.substr(0,idx) + ".index" + file.substr(idx); + //} + //else + //{ + return file + ".index"; + //} + } + + static void query_extent(boost::shared_ptr rs, + mapnik::box2d& extent) + { + + bool first = true; + while (rs->is_valid() && rs->step_next()) + { + int size; + const char* data = (const char*) rs->column_blob(0, size); + if (data) + { + // create a feature to parse geometry into + // see: http://trac.mapnik.org/ticket/745 + mapnik::feature_ptr feature(mapnik::feature_factory::create(0)); + mapnik::geometry_utils::from_wkb(feature->paths(), data, size, false, mapnik::wkbAuto); + mapnik::box2d const& bbox = feature->envelope(); + + if (bbox.valid()) + { + if (first) + { + first = false; + extent = bbox; + } + else + { + extent.expand_to_include(bbox); + } + } + } + } + } + + static void create_spatial_index(std::string const& index_db, + std::string const& table, + std::string const& key_field, + boost::shared_ptr rs, + mapnik::box2d& extent) + { + // TODO - return early/recover from failure? + // TODO - allow either in db or in separate + + //std::clog << "creating rtree: " << index_db << " : " << table << "\n"; +#if SQLITE_VERSION_NUMBER >= 3005000 + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX; +#else + //int flags; +#endif + + boost::shared_ptr dataset_ = boost::make_shared(index_db,flags); + + dataset_->execute("BEGIN TRANSACTION"); + dataset_->execute("PRAGMA synchronous=OFF"); + // first drop the index if it already exists + std::ostringstream spatial_index_drop_sql; + spatial_index_drop_sql << "DROP TABLE IF EXISTS " << table; + dataset_->execute(spatial_index_drop_sql.str()); + + // create the spatial index + std::ostringstream spatial_index_sql; + spatial_index_sql << "create virtual table " << table + << " using rtree(pkid, xmin, xmax, ymin, ymax)"; + + std::ostringstream spatial_index_insert_sql; + spatial_index_insert_sql << "insert into " << table + << " values (?,?,?,?,?)"; + + sqlite3_stmt* stmt = 0; + dataset_->execute(spatial_index_sql.str()); + + const int rc = sqlite3_prepare_v2 (*(*dataset_), + spatial_index_insert_sql.str().c_str(), + -1, + &stmt, + 0); + if (rc != SQLITE_OK) + { + std::ostringstream index_error; + index_error << "Sqlite Plugin: auto-index table creation failed: '" + << sqlite3_errmsg(*(*dataset_)) << "' query was: " + << spatial_index_insert_sql; + + throw mapnik::datasource_exception(index_error.str()); + } + + bool first = true; + while (rs->is_valid() && rs->step_next()) + { + int size; + const char* data = (const char*) rs->column_blob(0, size); + if (data) + { + // create a tmp feature to be able to parse geometry + // ideally we would not have to do this. + // see: http://trac.mapnik.org/ticket/745 + mapnik::feature_ptr feature(mapnik::feature_factory::create(0)); + mapnik::geometry_utils::from_wkb(feature->paths(), data, size, false, mapnik::wkbAuto); + mapnik::box2d bbox = feature->envelope(); + + if (bbox.valid()) + { + if (first) + { + first = false; + extent = bbox; + } + else + { + extent.expand_to_include(bbox); + } + + const int type_oid = rs->column_type(1); + if (type_oid != SQLITE_INTEGER) + { + std::ostringstream type_error; + type_error << "Sqlite Plugin: invalid type for key field '" + << key_field << "' when creating index '" << table + << "' type was: " << type_oid << ""; + + throw mapnik::datasource_exception(type_error.str()); + } + + const sqlite_int64 pkid = rs->column_integer64(1); + if (sqlite3_bind_int64(stmt, 1, pkid) != SQLITE_OK) + { + throw mapnik::datasource_exception("invalid value for for key field while generating index"); + } + + if ((sqlite3_bind_double(stmt, 2, bbox.minx()) != SQLITE_OK) || + (sqlite3_bind_double(stmt, 3, bbox.maxx()) != SQLITE_OK) || + (sqlite3_bind_double(stmt, 4, bbox.miny()) != SQLITE_OK) || + (sqlite3_bind_double(stmt, 5, bbox.maxy()) != SQLITE_OK)) + { + throw mapnik::datasource_exception("invalid value for for extent while generating index"); + } + + const int res = sqlite3_step(stmt); + if (res != SQLITE_ROW && res != SQLITE_DONE) + { + std::ostringstream s; + s << "SQLite Plugin: inserting bbox into rtree index failed: " + << "error code " << sqlite3_errcode(*(*dataset_)) << ": '" + << sqlite3_errmsg(*(*dataset_)) << "' query was: " + << spatial_index_insert_sql; + + throw mapnik::datasource_exception(s.str()); + } + + sqlite3_reset(stmt); + } + else + { + std::ostringstream index_error; + index_error << "SQLite Plugin: encountered invalid bbox at '" + << key_field << "' == " << rs->column_integer64(1); + + throw mapnik::datasource_exception(index_error.str()); + } + } + } + + const int res = sqlite3_finalize(stmt); + if (res != SQLITE_OK) + { + throw mapnik::datasource_exception("auto-indexing failed: set use_spatial_index=false to disable auto-indexing and avoid this error"); + } + + dataset_->execute("COMMIT"); + } + + static bool detect_extent(boost::shared_ptr dataset_, + bool has_spatial_index, + mapnik::box2d & extent, + std::string const& index_table, + std::string const& metadata, + std::string const& geometry_field, + std::string const& geometry_table, + std::string const& key_field, + std::string const& table + ) + { + if (has_spatial_index) + { + std::ostringstream s; + s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM " + << index_table; + + boost::scoped_ptr rs(dataset_->execute_query(s.str())); + if (rs->is_valid() && rs->step_next()) + { + if (! rs->column_isnull(0)) + { + try + { + double xmin = boost::lexical_cast(rs->column_double(0)); + double ymin = boost::lexical_cast(rs->column_double(1)); + double xmax = boost::lexical_cast(rs->column_double(2)); + double ymax = boost::lexical_cast(rs->column_double(3)); + extent.init(xmin, ymin, xmax, ymax); + return true; + } + catch (boost::bad_lexical_cast& ex) + { + std::ostringstream ss; + ss << "SQLite Plugin: warning: could not determine extent from query: " + << "'" << s.str() << "' \n problem was: " << ex.what() << std::endl; + std::clog << ss.str(); + } + } + } + } + else if (! metadata.empty()) + { + std::ostringstream s; + s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata; + s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table << "')"; + boost::scoped_ptr rs(dataset_->execute_query(s.str())); + if (rs->is_valid() && rs->step_next()) + { + double xmin = rs->column_double (0); + double ymin = rs->column_double (1); + double xmax = rs->column_double (2); + double ymax = rs->column_double (3); + extent.init (xmin, ymin, xmax, ymax); + return true; + } + } + else if (! key_field.empty()) + { + std::ostringstream s; + s << "SELECT " << geometry_field << "," << key_field + << " FROM (" << table << ")"; + boost::shared_ptr rs(dataset_->execute_query(s.str())); + sqlite_utils::query_extent(rs,extent); + return true; + } + return false; + } + +}; + +#endif //SQLITE_TYPES_HPP From 35e026eb8c81e790c46508b524869d20e084ac62 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Thu, 3 Nov 2011 19:50:08 -0400 Subject: [PATCH 2/5] remove indexes --- tests/data/sqlite/world.sqlite | Bin 531456 -> 507904 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/data/sqlite/world.sqlite b/tests/data/sqlite/world.sqlite index bb189518cafee85130ae8ee648e2d8d9be9a6684..802f2dfa7a224f9c1da4c6a9e3741d7a7c5b362b 100644 GIT binary patch delta 110 zcmZqJq0rDEKS5ehgn@za0|Nr6PSi0L6=BeQq`=F}!N9`E$-vCRq{4KPk#n*j&(!UK z$Jk{Ur{^7If6M~Z!!ntN`4yK!m}8K$hhva}W)n*r3u7A#QyU9&8w-%l+Q!1RjD>v< E0KD)Q6951J delta 12946 zcmc(Fdt6P~`~TT{uR5n9m2?qGs3b}e5}oxN-DoOJNK9@?LM2IP9LHeXl^WAvTyh&R zG`Y_hoxQ_g+y;X&7)Fds%*bUjGxA+~owY}U=JWaE`}_XR>$RU}o%edyWnZ4P*ZbMe z%$cq;_qpY}*xIO6yju0@*Sf$bKyRb!__M1S0eRt>EY-rFq%f5f4Yp=#$DTboLyznc zBgc;(o18mrN=DZBUyx($qsiywfa+~P4rg}S|I z>`Oawn>=Z3`WUyY?5xqFy)ws*P4;oioiH|CmLqa~+@{jCtb zg^h%JmMRtZ*n)@}vsq{^;^rcLE)wPM7o%J8~;gD)+BVyx84*ADPQx?sZawl|Pysr zp%d&MxC?K3;EBKA@H@ksNag9e&J4p#B27X4kO_ zTWMkNbt$FUT5+F~rP#u6E5q$z-<$<-yx(oy^3yx`*5@Y~-ZUia+$L?#zBujC_~AHw zLwAPT>s)3X(s_2(ioLyO8cRP=u3R&z$ShTSxGzr9tGAVCKeN5h8aUwcY6o0Zng{I; z^nlK#&l&CrResyy_Ku@k!;~A^PoL~%xKm|g)gdWvvOjo|`@bnBnc>cGLUe7L{{#p-Q^zxo@-I8c+M$ZpfgBDnqehU9F z-cd{_HPe1RM7hT;@g09_{H)@vacMtmqy34`nSCoLjM<|zKa`4>^-;mVi$fUh3R#+Y z@X`4zxVj)7ny;VCaJ?IRajX>f1$pBOPaol%H*c_t*3#u$e@L@0#!JITHW3#+n8k2+ zspRBvY0VTryt2<_yt;Ng!`onutqnGOmK^fE*BKnQwF$%9;z2X>vC(X;wkCTQKJb+j z!`p$+_3_}ZZ~U7WKKx6uTWDui;UTThoGHyYm};CymL-n(Ij@ra{H2D7IBLN4#inu-csstoFkblUUl+89|BV&RFp*P}HR%aio2tk{ z)q>OiUo4^jvVwZQZ3&gH4dRxnxTU-|e?rLNe-zSGDTTJ0clF5NWUh7$4do4;|8315 zmY$K^(EEVaagW&YZ(8=<-(2_Uq^YgDm)u)=hFfHGVH{_tcS|0TJ;Dv5VxYajRjqam z59bX%|8+K58B@k4H#7vT30;Ivf|uYfv=AHw8^KaQd>vn_=WF%Aw z(x@CuWh%wT&~gfuqp3`$auktfXe5;*s2on^Fd})BM9c3{Ih05}kA~1ci9`x$FfAug zIfzIe4I~nw0aV6Q{9P*Jh!jwNipNqJLy>4I-=Q*!NFMd0<-SBB6iJZ?S`L?SJqnW* z3Z)f&h%`ewS_UdHkqBvt6i^67ded?*S`H=>p`H{8qU9b`b|;cY-6$SNq#5c;@h%h( zpk;q5^___lkRO%4RCXfL40R+DAs;F`P~4l!_EdTi$stnjt451>{H(2O<$_MiG08G^NsxNHb(haT_8L zYC_A_6mLw+ji|Ju(vsp9MDj>O5w)O`hcTy+8IfiP5lI9slYPlwh72PA)XQ&ROW}}8 zcq05EJQVKKUk7&D+FJ;PZkjb=NFIno@<1Gt2jb{J98%M=JPwECapWOn$x|thM)yCQ zs)tI|gS^*$|0Xw_ifp6kPpI_W;UMnb)QW8R$KjwdR+pjag>ZvEZ)3Su%JMA^c_~naub5!q&@Qm&$8&QzYTpapM zD;(FqtlpVNiJ76i@k!Zu?Ju|I;V7>(Hc+bA?fP=#gN0ADi+U`>n!P%P4-zeQ8jY^q zM2h=Vym+oEjBm%=_gVZByb( zre8xGq#<6~5I6C{DZB3AA2+0Gn>e~)-#c?z|Ae#X-%47x-LW;pci}b9eek^(oy1FCrQ*{5Rt(=O#(#NMY}==c_NR7f zxHWQN_~C=jLjps-i@&dpIW*^9y<2cZd+KqX_QCwE!5{l~3HdQr@rFw{*eV4dZ@EOg z%h`(arte`L-4Rn-WEpq+J(S$zzB0ae(r2fwCXajl2@}{8=D}JuM5P+y@4cnTx-==J zx<-m?`HA$-X)o5#E7tl?(yFa=y5e1F{zxTW|9H6GSm&ZpskPFy!tT=GxEbP=BiF>0 z;(XRoL-e{sWxF4?Rkn=r5X+WTi8dW-nEm^e?H|snZ2w(^xT4iPvACQtG5T94l@ujN z^9Ngsg}WbqeLYL|Mf!v_6tU4UxbD42A^XFYv5x9-zW3+iL0Jl$eMg$S z)Lt5K<6%hg5guFQe!}cOl~VnNN(o%skjIZkXhq*o8QxP$)Tf0>(FgLhji(v4`{m-WM&?T!r>w$0Q##;@ z>ER3ylXCvLE=}IE8sl!gapaK*hU=w8ZhfVNmAkM${HJelzv0U25+jwD50p0PX5s#A zV)3G$%B5>4v2YYkP=Yv7LRX8lNWlNKBo}RKir*l^p%v|KT(>rZxtSTxC)O7RD8!MP1)uo zWx2%S)&~#Zz%P|6aY%k)RCc`ET-=tEBTg&$g7s5b>7x&->=0NhI$C!UEe^*iSK>Qq z?5Zp&epp|r(R;JRnodeE>6|qAJ3A@+i$L60a>aw@=d%XqrDftWDdU~tA)b@$aG#V| zhF`Aq`Q}1phjb*4IFuv~?xegJZb%E>Jt9pP+hV+2hZi?eu25Aa?=o9@?!IuIc*%rM)cUgcUF|CQl_w<}{*k;IeO~uj#C(>-O6jkmu*Df;S;;D`^OP0XX-^K$ zy55u7PZV|cE@(geAqKZePsYc5mCL$Wobz>^I3%G0`(ORDk-P5_gUHS)9IS0PE-|J{eOY zZ_&Ueb0v6PQYQ2*zMtF*T31Ag!BS%fkCNBnHy zVtY`{TOs~?MGH|Ps+c4N?H<{inkg1lpI^sBgbk&j`gx> z8Z{bAq&a%Uj?GaDMMhJZOe8|1h_s;W*b>R?*b>R?*aFGySVP&d1(MmZC6d{(nv!En zB$HzeCC3&>CdZcOZ*r_BvwOvkEs@NQEm0hi=9C>I-yp|&t9tBMjb5>1HImt}c|CSaxCEVw%#PLd*s&#L$L8o2J2prD#0H^P zNo69c$1lc5Fe}u_cn(u?1r6n3U_2V>Ob= zu>~c^=9C<(kzQuUR`uAi8ogr27WLS%6=la7B(q}+B(q};lG(9_l2(C|R!d4+)qk_& zf0AQMN?NTbIku$K6(LS0@EVyNXR3Zv3D1QZ;l5BszS3O~P76l`QTS5WNg`w$h1J5x zLcTDc#K)!!*}?=NMHoh+WBrA`f=&n)x{}zKr{E?y3wDAPiHyDApYgx)claA5EOw4R z!5`)i@_R{8Yztq)ujGsP#Uv!A=X3dqygrQ|MIvGY_-HB@J&cG zjN|^`9&^>)EfNd6$erPiaT50xiG+R5ZQ|B&#asc2gMGl|ag(`pZVU;64d&vweq0}} z7YTv+a$a0(t~u9~xVV|BPW3xEL-s=k*(1%6AzeOyb7Yu6loc|JH>q(%A!Lxcd2*++>Q zD*Hb%HN~(DvY!gbFqkM}DtjuA46>^d(}6^hkz`*b|Kw1XfDE#?5+%DU>7~EvpX{(? z_Oi!nkRh5V(m?iEGVLhSzkVjwR}SEjhLPki8Bq3IVk)~XX&*)u89vmc`Vb``gU+Pn z;1+3zi6V2-n!pgE$aRuknAG(miVP?_G3nD3<1z%9{>gq!#*-acPjok;$w+}jk@j7Q zA}vhbYzQ#b$^J}0*`Y}TKcdK~$u3Psl6{&q?`Q&jh$3^6y_$69ZBp%tBJE_yCJj7^ zA}8n}gX9BDJEBS7vUdx}(1s{7Z#nu!Oid1Ma5L4p5=FX^eOy2W*~y7&K@*2I`EGemO0z{ zIWxbB(0;Y`1kU`(8hTyLhoQeJ@r$we!-AuDS56g#oRA>gOJR{g80*jA<);%t=dcc< zeve@N%+v0_?5RC-Gz7wme}Ks8z6>{L-AlJ=PaJfBu-|g@5M6H1DpK(NxUYeKRt4{f zFJMGW8N&la&tXMIW7-plo@WI~!OB)~4Bot;8edp)6cT1XfsrYHF#8XU#WxF#3kM8@ zgvaM0G3heHJ7R7$;LLMtA+h#%7_)u?!Rgj-H2!StWqc#S0fwdd!C3QOSc9&3`{pzF z#?V`ka^eDH_$tx4V7y^t1^#PKYsk160E?PynEhWticLxk#i$lyXHqot<8cZIm#`F z6AulXCJuV>CCupf1I+1npV@~?%UrTVnD#Sd{CTxrRIs7%=j-vvk$+)hRVzs8bGQnI z+A5KmuW{!yfmj@JS6ux`W0>pm7n?sU@#Z$xv{uSIjF;YYgw+KfySFPS-sUg9v; z<8@)R*Gec4zY=%e$S@f;wRUxum&INJm%!;ynD_P*PFEw zr3~MsYpg2Kxz0W> z_O$<~V)48k4BrAnuh&7+MN4s;#YJ%o>QApfsq6%vUfZDK=Cio7e;|(fY&B~Tsk7O9 zP-k=91`lw*g@^ofp5b3W_wQ_Agj=21vbol{fF^p$Rr}y{gcF?p(<@}l+<77U0t%S@ zerQt@2JdWIP;v3x*I0XBS=sc*bbdBu)~_EN(9;-<79F|vs}v;U#e-*0B6pQ}nt`8ZmPKYoqurId4UcQ;|F&q3hWV`G?Y`3F)ArJ2 zAq{7L<%y^7S<6OmC^0ER%B@M1a(B$abG!TF58qRQ3k{`)`APZQGHKqXGQ7BJkshy+ zlvhlmwCeL3sd!=_URpK_7ujd9fj^f%yizR9%8SPN?}gx^T3d!kOGQ=X($e%a}S>ExC-C2P`LlFN}t|6gLQsCc+1~& zw8cF75%R2Je z3s?HwhQM`lFo)jIB&e;3(DCI9z%jiG_#Dq=4Ti(>o`R0QKOD?TSAdIF3D13?v$@?# zXM4>Kj~aI#Z=4du>_w2iNPu%;3$(i@e2dqgSAsJQC5~Ny6TKsB;PS&`*mCqb{MCtX zSVy;XsB65=yd*(eI`emJ?F;37_XMibg5e40j>~4m;Iq$9z4eA#vK#8edfP!i>m`*s z;pw@TfQwCM_z`;0`M?#O6ZDD(P1hnzE zRPRIP2nh1C16;Zs-&*k)KbqslI(l37rd;(I^ozR<(OC)ji2Dxw-KuYy{m&3RU^R@G zF$3=@560hptNTvJ@KkEr$*%J`6t#!(ODr zvV>x2mgf%L8Y`aLP)cD5Oq!YnD>q)o6`|9?{<0P8M}oP=U9fyu9mF)9=n8G0PGJ@A z!o1O8e}B@o{z^{Rm4ENSf>rxq?Ub|7a?S_Pv1GRiR;`4!EsCHlvIRch&J{nIFqS-F z^eY^)F&8fUBYvQKRo~wQ879l*0vWPRlw%T60y1O~MQF@KlgcE@tT7qpAN<3LJ*x`; z=!;jXpWz#)OrM%PS6*cEnv6%@I*(Jy?%GCh2Ak)~6`w1Anl_g1>I=4=)`0CO2WCG| zd+vLZ4(n_VO$QV~@DZi|etzmS;`NWZ;C-z;psjBjcsDu8I%+6u6&p9iY{pKP zv%#bLJ@!ppZn6j$ZukS29G?qb?E|3G*7tdQq^P(c%Qrv9ek9~GfDV; zYrbzOE^#Qr4`LUCZ&*Iyftox(s|j{|0=2Q1&->aoFY-IKt}*_$6k5 z-`*THp*?t@TOFQ#r5XZre}mu}T^ zIxxIKd+^~y?75&5jOpHLH-yx6V--EH)BKrue!K)hqaQ+_3(9hlh8>GrV&@_rf?swa zymTb9e`NH&b;0Q75({CITp;ZDAcof%o1D!vs-9Lt==9qVdRTt#CBMNvl+<+P?afI? z7f+TCaPhMU1D zXA63C>nL^sB3OmFRPeMFJ%YB3l*qwwwdC3)Ts~m=_Fv?KI$S}ll^oT#Q_J4?kEp)l zk9++Z%w~}z14>Sd92THCEh?n6e6-xin_2L^RD3Uz#5$5(HeX(EXk@8&?A41mv~MWa z@9)&r*V6bX&Z9R|58q~| Date: Thu, 3 Nov 2011 19:50:36 -0400 Subject: [PATCH 3/5] sqlite: do not alias geometry field name --- tests/python_tests/sqlite_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_tests/sqlite_test.py b/tests/python_tests/sqlite_test.py index 11d2bfdcc..464c9690f 100644 --- a/tests/python_tests/sqlite_test.py +++ b/tests/python_tests/sqlite_test.py @@ -164,7 +164,7 @@ if 'sqlite' in mapnik2.DatasourceCache.instance().plugin_names(): eq_(feature['lat'],17.078) ds = mapnik2.SQLite(file='../data/sqlite/world.sqlite', - table='(select OGC_FID,GEOMETRY as geom from world_merc)', + table='(select OGC_FID,GEOMETRY from world_merc)', ) fs = ds.featureset() feature = fs.next() From d965bcbeaee7f77a19548dd279e6c0717a2ecdb0 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Thu, 3 Nov 2011 19:51:37 -0400 Subject: [PATCH 4/5] sqlite: futher refactoring of sqlite plugin, allowing all tests to pass --- include/mapnik/sql_utils.hpp | 9 +- plugins/input/postgis/postgis_datasource.cpp | 22 +- plugins/input/postgis/postgis_featureset.cpp | 2 +- plugins/input/sqlite/sqlite_connection.hpp | 161 ++++ plugins/input/sqlite/sqlite_datasource.cpp | 84 ++- plugins/input/sqlite/sqlite_datasource.hpp | 8 +- plugins/input/sqlite/sqlite_featureset.cpp | 1 + plugins/input/sqlite/sqlite_featureset.hpp | 8 +- plugins/input/sqlite/sqlite_prepared.hpp | 132 ++++ plugins/input/sqlite/sqlite_resultset.hpp | 143 ++++ .../{sqlite_types.hpp => sqlite_utils.hpp} | 693 ++++++------------ src/agg/process_text_symbolizer.cpp | 1 + 12 files changed, 744 insertions(+), 520 deletions(-) create mode 100644 plugins/input/sqlite/sqlite_connection.hpp create mode 100644 plugins/input/sqlite/sqlite_prepared.hpp create mode 100644 plugins/input/sqlite/sqlite_resultset.hpp rename plugins/input/sqlite/{sqlite_types.hpp => sqlite_utils.hpp} (51%) diff --git a/include/mapnik/sql_utils.hpp b/include/mapnik/sql_utils.hpp index 666cf1dbd..ee6805cb7 100644 --- a/include/mapnik/sql_utils.hpp +++ b/include/mapnik/sql_utils.hpp @@ -27,17 +27,16 @@ #include #include -namespace mapnik -{ +namespace mapnik { namespace sql_utils { -inline std::string unquote_sql(const std::string& sql) +inline std::string unquote_double(const std::string& sql) { std::string table_name = boost::algorithm::to_lower_copy(sql); boost::algorithm::trim_if(table_name,boost::algorithm::is_any_of("\"")); return table_name; } -inline std::string unquote_sql2(const std::string& sql) +inline std::string unquote(const std::string& sql) { std::string table_name = boost::algorithm::to_lower_copy(sql); boost::algorithm::trim_if(table_name,boost::algorithm::is_any_of("\"\'")); @@ -65,7 +64,6 @@ inline std::string table_from_sql(const std::string& sql) std::string::size_type idx = table_name.rfind(" from "); if (idx!=std::string::npos) { - idx = table_name.find_first_not_of(" ",idx+5); if (idx != std::string::npos) { @@ -191,6 +189,7 @@ inline std::string numeric2string(const char* buf) } return ss.str(); } +} } diff --git a/plugins/input/postgis/postgis_datasource.cpp b/plugins/input/postgis/postgis_datasource.cpp index 343cdd9e1..3af5a8eda 100644 --- a/plugins/input/postgis/postgis_datasource.cpp +++ b/plugins/input/postgis/postgis_datasource.cpp @@ -126,7 +126,7 @@ void postgis_datasource::bind() const if(geometry_table_.empty()) { - geometry_table_ = mapnik::table_from_sql(table_); + geometry_table_ = mapnik::sql_utils::table_from_sql(table_); } std::string::size_type idx = geometry_table_.find_last_of('.'); if (idx!=std::string::npos) @@ -150,13 +150,13 @@ void postgis_datasource::bind() const { std::ostringstream s; s << "SELECT f_geometry_column, srid FROM "; - s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << mapnik::unquote_sql(geometry_table_) <<"'"; + s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << mapnik::sql_utils::unquote_double(geometry_table_) <<"'"; if (schema_.length() > 0) - s << " AND f_table_schema='" << mapnik::unquote_sql(schema_) << "'"; + s << " AND f_table_schema='" << mapnik::sql_utils::unquote_double(schema_) << "'"; if (geometry_field_.length() > 0) - s << " AND f_geometry_column='" << mapnik::unquote_sql(geometry_field_) << "'"; + s << " AND f_geometry_column='" << mapnik::sql_utils::unquote_double(geometry_field_) << "'"; /* if (show_queries_) @@ -502,14 +502,14 @@ featureset_ptr postgis_datasource::features(const query& q) const s << "AsBinary(\"" << geometryColumn_ << "\") AS geom"; if (!key_field_.empty()) - mapnik::quote_attr(s,key_field_); + mapnik::sql_utils::quote_attr(s,key_field_); std::set const& props=q.property_names(); std::set::const_iterator pos=props.begin(); std::set::const_iterator end=props.end(); while (pos != end) { - mapnik::quote_attr(s,*pos); + mapnik::sql_utils::quote_attr(s,*pos); ++pos; } @@ -578,14 +578,14 @@ featureset_ptr postgis_datasource::features_at_point(coord2d const& pt) const s << "AsBinary(\"" << geometryColumn_ << "\") AS geom"; if (!key_field_.empty()) - mapnik::quote_attr(s,key_field_); + mapnik::sql_utils::quote_attr(s,key_field_); std::vector::const_iterator itr = desc_.get_descriptors().begin(); std::vector::const_iterator end = desc_.get_descriptors().end(); unsigned size=0; while (itr != end) { - mapnik::quote_attr(s,itr->get_name()); + mapnik::sql_utils::quote_attr(s,itr->get_name()); ++itr; ++size; } @@ -644,11 +644,11 @@ box2d postgis_datasource::envelope() const if (schema_.length() > 0) { - s << mapnik::unquote_sql(schema_) << "','"; + s << mapnik::sql_utils::unquote_double(schema_) << "','"; } - s << mapnik::unquote_sql(geometry_table_) << "','" - << mapnik::unquote_sql(geometryColumn_) << "') as ext) as tmp"; + s << mapnik::sql_utils::unquote_double(geometry_table_) << "','" + << mapnik::sql_utils::unquote_double(geometryColumn_) << "') as ext) as tmp"; } else { diff --git a/plugins/input/postgis/postgis_featureset.cpp b/plugins/input/postgis/postgis_featureset.cpp index 801815173..4129ef147 100644 --- a/plugins/input/postgis/postgis_featureset.cpp +++ b/plugins/input/postgis/postgis_featureset.cpp @@ -162,7 +162,7 @@ feature_ptr postgis_featureset::next() } else if (oid == 1700) // numeric { - std::string str = mapnik::numeric2string(buf); + std::string str = mapnik::sql_utils::numeric2string(buf); try { double val = boost::lexical_cast(str); diff --git a/plugins/input/sqlite/sqlite_connection.hpp b/plugins/input/sqlite/sqlite_connection.hpp new file mode 100644 index 000000000..0eb0bfdba --- /dev/null +++ b/plugins/input/sqlite/sqlite_connection.hpp @@ -0,0 +1,161 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2011 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef MAPNIK_SQLITE_CONNECTION_HPP +#define MAPNIK_SQLITE_CONNECTION_HPP + +// stl +#include + +// mapnik +#include +#include + + +// boost +#include +#include +#include + +// sqlite +extern "C" { + #include +} + +#include "sqlite_resultset.hpp" + + +//============================================================================== + +class sqlite_connection +{ +public: + + sqlite_connection (const std::string& file) + : db_(0), + file_(file) + { +#if SQLITE_VERSION_NUMBER >= 3005000 + int mode = SQLITE_OPEN_READWRITE; + #if SQLITE_VERSION_NUMBER >= 3006018 + // shared cache flag not available until >= 3.6.18 + mode |= SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE; + #endif + const int rc = sqlite3_open_v2 (file_.c_str(), &db_, mode, 0); +#else + #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..." + const int rc = sqlite3_open (file_.c_str(), &db_); +#endif + if (rc != SQLITE_OK) + { + std::ostringstream s; + s << "Sqlite Plugin: " << sqlite3_errmsg (db_); + + throw mapnik::datasource_exception (s.str()); + } + + sqlite3_busy_timeout(db_,5000); + } + + sqlite_connection (const std::string& file, int flags) + : db_(0), + file_(file) + { +#if SQLITE_VERSION_NUMBER >= 3005000 + const int rc = sqlite3_open_v2 (file_.c_str(), &db_, flags, 0); +#else + #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..." + const int rc = sqlite3_open (file_.c_str(), &db_); +#endif + if (rc != SQLITE_OK) + { + std::ostringstream s; + s << "Sqlite Plugin: " << sqlite3_errmsg (db_); + + throw mapnik::datasource_exception (s.str()); + } + } + + virtual ~sqlite_connection () + { + if (db_) + { + sqlite3_close (db_); + } + } + + void throw_sqlite_error(const std::string& sql) + { + std::ostringstream s; + s << "Sqlite Plugin: "; + if (db_) + s << "'" << sqlite3_errmsg(db_) << "'"; + else + s << "unknown error, lost connection"; + s << " (" << file_ << ")" + << "\nFull sql was: '" + << sql << "'"; + + throw mapnik::datasource_exception (s.str()); + } + + boost::shared_ptr execute_query(const std::string& sql) + { + sqlite3_stmt* stmt = 0; + + const int rc = sqlite3_prepare_v2 (db_, sql.c_str(), -1, &stmt, 0); + if (rc != SQLITE_OK) + { + throw_sqlite_error(sql); + } + + return boost::make_shared(stmt); + } + + void execute(const std::string& sql) + { + const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0); + if (rc != SQLITE_OK) + { + throw_sqlite_error(sql); + } + } + + int execute_with_code(const std::string& sql) + { + const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0); + return rc; + } + + sqlite3* operator*() + { + return db_; + } + + +private: + + sqlite3* db_; + std::string file_; +}; + +#endif // MAPNIK_SQLITE_CONNECTION_HPP diff --git a/plugins/input/sqlite/sqlite_datasource.cpp b/plugins/input/sqlite/sqlite_datasource.cpp index caf98c974..990d583ca 100644 --- a/plugins/input/sqlite/sqlite_datasource.cpp +++ b/plugins/input/sqlite/sqlite_datasource.cpp @@ -22,6 +22,8 @@ #include "sqlite_datasource.hpp" #include "sqlite_featureset.hpp" +#include "sqlite_resultset.hpp" +#include "sqlite_utils.hpp" // mapnik #include @@ -64,8 +66,8 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind) format_(mapnik::wkbAuto) { /* TODO - - auto detect primary key - https://github.com/mapnik/mapnik/issues/82 - throw if no primary key but spatial index is present? + - remove auto-indexing */ boost::optional file = params_.get("file"); @@ -95,34 +97,37 @@ void sqlite_datasource::bind() const else dataset_name_ = *file; - if (!boost::filesystem::exists(dataset_name_)) + if ((dataset_name_.compare(":memory:") != 0) && (!boost::filesystem::exists(dataset_name_))) { throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist"); } + + multiple_geometries_ = *params_.get("multiple_geometries", false); + use_spatial_index_ = *params_.get("use_spatial_index", true); - boost::optional key_field_name = params_.get("key_field"); - if (key_field_name) - { - key_field_ = *key_field_name; - } - + // TODO - remove this option once all datasources have an indexing api + bool auto_index = *params_.get("auto_index", true); + + boost::optional ext = params_.get("extent"); + if (ext) extent_initialized_ = extent_.from_string(*ext); + boost::optional wkb = params_.get("wkb_format"); if (wkb) { if (*wkb == "spatialite") + { format_ = mapnik::wkbSpatiaLite; + } else if (*wkb == "generic") + { format_ = mapnik::wkbGeneric; + } else + { format_ = mapnik::wkbAuto; + } } - multiple_geometries_ = *params_.get("multiple_geometries", false); - use_spatial_index_ = *params_.get("use_spatial_index", true); - - boost::optional ext = params_.get("extent"); - if (ext) extent_initialized_ = extent_.from_string(*ext); - // Populate init_statements_ // 1. Build attach database statements from the "attachdb" parameter // 2. Add explicit init statements from "initdb" parameter @@ -144,7 +149,7 @@ void sqlite_datasource::bind() const if (geometry_table_.empty()) { - geometry_table_ = mapnik::table_from_sql(table_); + geometry_table_ = mapnik::sql_utils::table_from_sql(table_); } // if 'table_' is a subquery then we try to deduce names @@ -155,8 +160,6 @@ void sqlite_datasource::bind() const using_subquery_ = true; } - std::string index_db = sqlite_utils::get_index_db_name(dataset_name_); - // now actually create the connection and start executing setup sql dataset_ = boost::make_shared(dataset_name_); @@ -175,7 +178,7 @@ void sqlite_datasource::bind() const { std::ostringstream s; s << "SELECT " << fields_ << " FROM (" << table_ << ") LIMIT 1"; - found_types_via_subquery = dataset_->detect_types_from_subquery(s.str(),geometry_field_,desc_); + found_types_via_subquery = sqlite_utils::detect_types_from_subquery(s.str(),geometry_field_,desc_,dataset_); } // TODO - consider removing this @@ -184,11 +187,12 @@ void sqlite_datasource::bind() const desc_.add_descriptor(attribute_descriptor("rowid", mapnik::Integer)); } - bool found_table = dataset_->table_info(key_field_, + bool found_table = sqlite_utils::table_info(key_field_, found_types_via_subquery, geometry_field_, geometry_table_, - desc_); + desc_, + dataset_); if (! found_table) { @@ -214,20 +218,52 @@ void sqlite_datasource::bind() const // we have discovered meta-data or else we don't know the column name index_table_ = sqlite_utils::index_for_table(geometry_table_,geometry_field_); } + + std::string index_db = sqlite_utils::index_for_db(dataset_name_); has_spatial_index_ = false; if (use_spatial_index_) { - /*if (boost::filesystem::exists(index_db)) + if (boost::filesystem::exists(index_db)) { dataset_->execute("attach database '" + index_db + "' as " + index_table_); } - */ - has_spatial_index_ = dataset_->has_rtree(index_table_); + + has_spatial_index_ = sqlite_utils::has_rtree(index_table_,dataset_); + } + + + if (! extent_initialized_ + && !has_spatial_index_ + && auto_index) + { + if (! key_field_.empty()) + { + std::ostringstream query; + query << "SELECT " + << geometry_field_ + << "," << key_field_ + << " FROM (" + << geometry_table_ << ")"; + boost::shared_ptr rs = dataset_->execute_query(query.str()); + sqlite_utils::create_spatial_index(index_db,index_table_,rs,extent_); + extent_initialized_ = true; + } + else + { + std::ostringstream s; + s << "Sqlite Plugin: key_field is empty for " + << geometry_field_ + << " and " + << geometry_table_; + throw datasource_exception(s.str()); + } } if (! extent_initialized_) { + + // TODO - clean this up - reducing arguments if (!sqlite_utils::detect_extent(dataset_, has_spatial_index_, extent_, @@ -262,7 +298,7 @@ namespace filesystem { path symlink_path; #ifdef BOOST_POSIX_API - for (std::size_t path_max = 64;; path_max *= 2)// loop 'til buffer large enough + for (std::size_t path_max = 64;; path_max *= 2)// loop 'til buffer is large enough { boost::scoped_array buf(new char[path_max]); ssize_t result; diff --git a/plugins/input/sqlite/sqlite_datasource.hpp b/plugins/input/sqlite/sqlite_datasource.hpp index c62b32208..0a964e61d 100644 --- a/plugins/input/sqlite/sqlite_datasource.hpp +++ b/plugins/input/sqlite/sqlite_datasource.hpp @@ -20,8 +20,8 @@ * *****************************************************************************/ -#ifndef SQLITE_DATASOURCE_HPP -#define SQLITE_DATASOURCE_HPP +#ifndef MAPNIK_SQLITE_DATASOURCE_HPP +#define MAPNIK_SQLITE_DATASOURCE_HPP // mapnik #include @@ -34,7 +34,7 @@ #include // sqlite -#include "sqlite_types.hpp" +#include "sqlite_connection.hpp" class sqlite_datasource : public mapnik::datasource @@ -78,4 +78,4 @@ private: void parse_attachdb(std::string const& attachdb) const; }; -#endif // SQLITE_DATASOURCE_HPP +#endif // MAPNIK_SQLITE_DATASOURCE_HPP diff --git a/plugins/input/sqlite/sqlite_featureset.cpp b/plugins/input/sqlite/sqlite_featureset.cpp index 7e7dfd4b3..9d53f0fe3 100644 --- a/plugins/input/sqlite/sqlite_featureset.cpp +++ b/plugins/input/sqlite/sqlite_featureset.cpp @@ -33,6 +33,7 @@ // ogr #include "sqlite_featureset.hpp" +#include "sqlite_utils.hpp" using mapnik::query; using mapnik::box2d; diff --git a/plugins/input/sqlite/sqlite_featureset.hpp b/plugins/input/sqlite/sqlite_featureset.hpp index b24274bd9..819050c60 100644 --- a/plugins/input/sqlite/sqlite_featureset.hpp +++ b/plugins/input/sqlite/sqlite_featureset.hpp @@ -20,8 +20,8 @@ * *****************************************************************************/ -#ifndef SQLITE_FEATURESET_HPP -#define SQLITE_FEATURESET_HPP +#ifndef MAPNIK_SQLITE_FEATURESET_HPP +#define MAPNIK_SQLITE_FEATURESET_HPP // mapnik #include @@ -33,7 +33,7 @@ #include // sqlite -#include "sqlite_types.hpp" +#include "sqlite_resultset.hpp" class sqlite_featureset : public mapnik::Featureset @@ -55,4 +55,4 @@ private: bool using_subquery_; }; -#endif // SQLITE_FEATURESET_HPP +#endif // MAPNIK_SQLITE_FEATURESET_HPP diff --git a/plugins/input/sqlite/sqlite_prepared.hpp b/plugins/input/sqlite/sqlite_prepared.hpp new file mode 100644 index 000000000..4ad5271f2 --- /dev/null +++ b/plugins/input/sqlite/sqlite_prepared.hpp @@ -0,0 +1,132 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2011 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef MAPNIK_SQLITE_PREPARED_HPP +#define MAPNIK_SQLITE_PREPARED_HPP + +// mapnik +#include +#include + +// boost +#include +#include + +// stl +#include + +#include "sqlite_connection.hpp" + +// sqlite +extern "C" { + #include +} + +class prepared_index_statement : boost::noncopyable +{ + +public: + prepared_index_statement(boost::shared_ptr ds, std::string const& sql) + : ds_(ds), + stmt_(0) + { + + const int rc = sqlite3_prepare_v2(*(*ds_), + sql.c_str(), + -1, + &stmt_, + 0); + + if (rc != SQLITE_OK) + { + std::ostringstream index_error; + index_error << "Sqlite Plugin: auto-index table creation failed: '" + << sqlite3_errmsg(*(*ds_)) << "' query was: " + << sql; + + throw mapnik::datasource_exception(index_error.str()); + } + } + + ~prepared_index_statement() + { + if (stmt_) + { + int res = sqlite3_finalize(stmt_); + if (res != SQLITE_OK) + { + if (*(*ds_)) + { + std::cerr << "ERR:" << sqlite3_errmsg(*(*ds_)) << "\n"; + } + else + { + std::cerr << "SQLite Plugin: " << res << "\n"; + } + } + } + } + + void bind(sqlite_int64 const pkid) + { + if (sqlite3_bind_int64(stmt_, 1, pkid) != SQLITE_OK) + { + throw mapnik::datasource_exception("SQLite Plugin: invalid value for for key field while generating index"); + } + + } + + void bind(mapnik::box2d const& bbox) + { + if ((sqlite3_bind_double(stmt_, 2, bbox.minx()) != SQLITE_OK) || + (sqlite3_bind_double(stmt_, 3, bbox.maxx()) != SQLITE_OK) || + (sqlite3_bind_double(stmt_, 4, bbox.miny()) != SQLITE_OK) || + (sqlite3_bind_double(stmt_, 5, bbox.maxy()) != SQLITE_OK)) + { + throw mapnik::datasource_exception("SQLite Plugin: invalid value for for extent while generating index"); + } + } + + bool step_next () + { + const int status = sqlite3_step(stmt_); + if (status != SQLITE_ROW && status != SQLITE_DONE) + { + std::ostringstream s; + s << "SQLite Plugin: inserting bbox into rtree index failed"; + std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_))); + if (msg != "unknown error") + { + s << ": " << msg; + } + throw mapnik::datasource_exception(s.str()); + } + sqlite3_reset(stmt_); + return status == SQLITE_ROW; + } + +private: + boost::shared_ptr ds_; + sqlite3_stmt * stmt_; +}; + +#endif // MAPNIK_SQLITE_PREPARED_HPP diff --git a/plugins/input/sqlite/sqlite_resultset.hpp b/plugins/input/sqlite/sqlite_resultset.hpp new file mode 100644 index 000000000..3123af9f8 --- /dev/null +++ b/plugins/input/sqlite/sqlite_resultset.hpp @@ -0,0 +1,143 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2011 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef MAPNIK_SQLITE_RESULTSET_HPP +#define MAPNIK_SQLITE_RESULTSET_HPP + +// mapnik +#include + +// stl +#include + +// sqlite +extern "C" { + #include +} + + + +//============================================================================== + +class sqlite_resultset +{ +public: + + sqlite_resultset (sqlite3_stmt* stmt) + : stmt_(stmt) + { + } + + ~sqlite_resultset () + { + if (stmt_) + { + sqlite3_finalize (stmt_); + } + } + + bool is_valid () + { + return stmt_ != 0; + } + + bool step_next () + { + const int status = sqlite3_step (stmt_); + if (status != SQLITE_ROW && status != SQLITE_DONE) + { + std::ostringstream s; + s << "SQLite Plugin: retrieving next row failed"; + std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_))); + if (msg != "unknown error") + { + s << ": " << msg; + } + + throw mapnik::datasource_exception(s.str()); + } + return status == SQLITE_ROW; + } + + int column_count () + { + return sqlite3_column_count (stmt_); + } + + int column_type (int col) + { + return sqlite3_column_type (stmt_, col); + } + + const char* column_name (int col) + { + return sqlite3_column_name (stmt_, col); + } + + bool column_isnull (int col) + { + return sqlite3_column_type (stmt_, col) == SQLITE_NULL; + } + + int column_integer (int col) + { + return sqlite3_column_int (stmt_, col); + } + + int column_integer64 (int col) + { + return sqlite3_column_int64 (stmt_, col); + } + + double column_double (int col) + { + return sqlite3_column_double (stmt_, col); + } + + const char* column_text (int col, int& len) + { + len = sqlite3_column_bytes (stmt_, col); + return (const char*) sqlite3_column_text (stmt_, col); + } + + const char* column_text (int col) + { + return (const char*) sqlite3_column_text (stmt_, col); + } + + const void* column_blob (int col, int& bytes) + { + bytes = sqlite3_column_bytes (stmt_, col); + return (const char*) sqlite3_column_blob (stmt_, col); + } + + sqlite3_stmt* get_statement() + { + return stmt_; + } + +private: + + sqlite3_stmt* stmt_; +}; + +#endif // MAPNIK_SQLITE_RESULTSET_HPP diff --git a/plugins/input/sqlite/sqlite_types.hpp b/plugins/input/sqlite/sqlite_utils.hpp similarity index 51% rename from plugins/input/sqlite/sqlite_types.hpp rename to plugins/input/sqlite/sqlite_utils.hpp index 55a86686e..87c5a593c 100644 --- a/plugins/input/sqlite/sqlite_types.hpp +++ b/plugins/input/sqlite/sqlite_utils.hpp @@ -20,16 +20,16 @@ * *****************************************************************************/ -#ifndef SQLITE_TYPES_HPP -#define SQLITE_TYPES_HPP +#ifndef MAPNIK_SQLITE_UTILS_HPP +#define MAPNIK_SQLITE_UTILS_HPP // stl #include // mapnik #include +#include #include -#include // to enable extent fallback hack // boost @@ -43,227 +43,251 @@ extern "C" { #include } +#include "sqlite_resultset.hpp" +#include "sqlite_prepared.hpp" +#include "sqlite_connection.hpp" //============================================================================== -class sqlite_resultset +class sqlite_utils { public: - sqlite_resultset (sqlite3_stmt* stmt) - : stmt_(stmt) + static void dequote(std::string & z) { + boost::algorithm::trim_if(z,boost::algorithm::is_any_of("[]'\"`")); + } + + static std::string index_for_table(std::string const& table, std::string const& field) + { + return "\"idx_" + mapnik::sql_utils::unquote(table) + "_" + field + "\""; } - ~sqlite_resultset () + static std::string index_for_db(std::string const& file) { - if (stmt_) + //std::size_t idx = file.find_last_of("."); + //if(idx != std::string::npos) { + // return file.substr(0,idx) + ".index" + file.substr(idx); + //} + //else + //{ + return file + ".index"; + //} + } + + static void query_extent(boost::shared_ptr rs, + mapnik::box2d& extent) + { + + bool first = true; + while (rs->is_valid() && rs->step_next()) { - sqlite3_finalize (stmt_); - } - } - - bool is_valid () - { - return stmt_ != 0; - } - - bool step_next () - { - const int status = sqlite3_step (stmt_); - if (status != SQLITE_ROW && status != SQLITE_DONE) - { - std::ostringstream s; - s << "SQLite Plugin: retrieving next row failed"; - std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_))); - if (msg != "unknown error") + int size; + const char* data = (const char*) rs->column_blob(0, size); + if (data) { - s << ": " << msg; + boost::ptr_vector paths; + mapnik::geometry_utils::from_wkb(paths, data, size, false, mapnik::wkbAuto); + for (unsigned i=0; i const& bbox = paths[i].envelope(); + + if (bbox.valid()) + { + if (first) + { + first = false; + extent = bbox; + } + else + { + extent.expand_to_include(bbox); + } + } + } } - - throw mapnik::datasource_exception(s.str()); } - return status == SQLITE_ROW; - } - - int column_count () - { - return sqlite3_column_count (stmt_); - } - - int column_type (int col) - { - return sqlite3_column_type (stmt_, col); } - const char* column_name (int col) + static void create_spatial_index(std::string const& index_db, + std::string const& index_table, + boost::shared_ptr rs, + mapnik::box2d& extent) { - return sqlite3_column_name (stmt_, col); - } - - bool column_isnull (int col) - { - return sqlite3_column_type (stmt_, col) == SQLITE_NULL; - } - - int column_integer (int col) - { - return sqlite3_column_int (stmt_, col); - } - - int column_integer64 (int col) - { - return sqlite3_column_int64 (stmt_, col); - } - - double column_double (int col) - { - return sqlite3_column_double (stmt_, col); - } - - const char* column_text (int col, int& len) - { - len = sqlite3_column_bytes (stmt_, col); - return (const char*) sqlite3_column_text (stmt_, col); - } - - const char* column_text (int col) - { - return (const char*) sqlite3_column_text (stmt_, col); - } - - const void* column_blob (int col, int& bytes) - { - bytes = sqlite3_column_bytes (stmt_, col); - return (const char*) sqlite3_column_blob (stmt_, col); - } - - sqlite3_stmt* get_statement() - { - return stmt_; - } - -private: - - sqlite3_stmt* stmt_; -}; - - -//============================================================================== - -class sqlite_connection -{ -public: - - sqlite_connection (const std::string& file) - : db_(0), - file_(file) - { -#if SQLITE_VERSION_NUMBER >= 3005000 - int mode = SQLITE_OPEN_READWRITE; - #if SQLITE_VERSION_NUMBER >= 3006018 - // shared cache flag not available until >= 3.6.18 - mode |= SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE; - #endif - const int rc = sqlite3_open_v2 (file_.c_str(), &db_, mode, 0); -#else - #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..." - const int rc = sqlite3_open (file_.c_str(), &db_); -#endif - if (rc != SQLITE_OK) - { - std::ostringstream s; - s << "Sqlite Plugin: " << sqlite3_errmsg (db_); - - throw mapnik::datasource_exception (s.str()); - } + /* TODO + - speedups + - return early/recover from failure + - allow either in db or in separate + */ - sqlite3_busy_timeout(db_,5000); + if (!rs->is_valid()) + return; + +#if SQLITE_VERSION_NUMBER >= 3005000 + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX; +#else + int flags; +#endif + + boost::shared_ptr ds = boost::make_shared(index_db,flags); + + // first drop the index if it already exists + std::ostringstream spatial_index_drop_sql; + spatial_index_drop_sql << "DROP TABLE IF EXISTS " << index_table; + ds->execute(spatial_index_drop_sql.str()); + + ds->execute("PRAGMA synchronous=OFF"); + ds->execute("BEGIN TRANSACTION"); + + // create the spatial index + std::ostringstream create_idx; + create_idx << "create virtual table " + << index_table + << " using rtree(pkid, xmin, xmax, ymin, ymax)"; + + // insert for prepared statement + std::ostringstream insert_idx; + insert_idx << "insert into " + << index_table + << " values (?,?,?,?,?)"; + + ds->execute(create_idx.str()); + + prepared_index_statement ps(ds,insert_idx.str()); + + bool first = true; + while (rs->is_valid() && rs->step_next()) + { + int size; + const char* data = (const char*) rs->column_blob(0, size); + if (data) + { + boost::ptr_vector paths; + // TODO - contraint fails if multiple_geometries = true + bool multiple_geometries = false; + mapnik::geometry_utils::from_wkb(paths, data, size, multiple_geometries, mapnik::wkbAuto); + for (unsigned i=0; i const& bbox = paths[i].envelope(); + if (bbox.valid()) + { + if (first) + { + first = false; + extent = bbox; + } + else + { + extent.expand_to_include(bbox); + } + + ps.bind(bbox); + + const int type_oid = rs->column_type(1); + if (type_oid != SQLITE_INTEGER) + { + std::ostringstream error_msg; + error_msg << "Sqlite Plugin: invalid type for key field '" + << rs->column_name(1) << "' when creating index '" << index_table + << "' type was: " << type_oid << ""; + throw mapnik::datasource_exception(error_msg.str()); + } + + const sqlite_int64 pkid = rs->column_integer64(1); + ps.bind(pkid); + } + else + { + std::ostringstream error_msg; + error_msg << "SQLite Plugin: encountered invalid bbox at '" + << rs->column_name(1) << "' == " << rs->column_integer64(1); + throw mapnik::datasource_exception(error_msg.str()); + } + + ps.step_next(); + } + } + } + ds->execute("COMMIT"); } - sqlite_connection (const std::string& file, int flags) - : db_(0), - file_(file) + static bool detect_extent(boost::shared_ptr ds, + bool has_spatial_index, + mapnik::box2d & extent, + std::string const& index_table, + std::string const& metadata, + std::string const& geometry_field, + std::string const& geometry_table, + std::string const& key_field, + std::string const& table + ) { -#if SQLITE_VERSION_NUMBER >= 3005000 - const int rc = sqlite3_open_v2 (file_.c_str(), &db_, flags, 0); -#else - #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..." - const int rc = sqlite3_open (file_.c_str(), &db_); -#endif - if (rc != SQLITE_OK) + if (has_spatial_index) { std::ostringstream s; - s << "Sqlite Plugin: " << sqlite3_errmsg (db_); - - throw mapnik::datasource_exception (s.str()); - } - } - - virtual ~sqlite_connection () - { - if (db_) - { - sqlite3_close (db_); - } - } - - void throw_sqlite_error(const std::string& sql) - { - std::ostringstream s; - s << "Sqlite Plugin: "; - if (db_) - s << "'" << sqlite3_errmsg(db_) << "'"; - else - s << "unknown error, lost connection"; - s << " (" << file_ << ")" - << "\nFull sql was: '" - << sql << "'"; - - throw mapnik::datasource_exception (s.str()); - } + s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM " + << index_table; - sqlite_resultset* execute_query(const std::string& sql) - { - sqlite3_stmt* stmt = 0; - - const int rc = sqlite3_prepare_v2 (db_, sql.c_str(), -1, &stmt, 0); - if (rc != SQLITE_OK) - { - throw_sqlite_error(sql); + boost::shared_ptr rs(ds->execute_query(s.str())); + if (rs->is_valid() && rs->step_next()) + { + if (! rs->column_isnull(0)) + { + try + { + double xmin = boost::lexical_cast(rs->column_double(0)); + double ymin = boost::lexical_cast(rs->column_double(1)); + double xmax = boost::lexical_cast(rs->column_double(2)); + double ymax = boost::lexical_cast(rs->column_double(3)); + extent.init(xmin, ymin, xmax, ymax); + return true; + } + catch (boost::bad_lexical_cast& ex) + { + std::ostringstream ss; + ss << "SQLite Plugin: warning: could not determine extent from query: " + << "'" << s.str() << "' \n problem was: " << ex.what() << std::endl; + std::clog << ss.str(); + } + } + } } - - return new sqlite_resultset (stmt); - } - - void execute(const std::string& sql) - { - const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0); - if (rc != SQLITE_OK) + else if (! metadata.empty()) { - throw_sqlite_error(sql); + std::ostringstream s; + s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata; + s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table << "')"; + boost::shared_ptr rs(ds->execute_query(s.str())); + if (rs->is_valid() && rs->step_next()) + { + double xmin = rs->column_double (0); + double ymin = rs->column_double (1); + double xmax = rs->column_double (2); + double ymax = rs->column_double (3); + extent.init (xmin, ymin, xmax, ymax); + return true; + } } + else if (! key_field.empty()) + { + std::ostringstream s; + s << "SELECT " << geometry_field << "," << key_field + << " FROM (" << table << ")"; + boost::shared_ptr rs(ds->execute_query(s.str())); + sqlite_utils::query_extent(rs,extent); + return true; + } + return false; } - int execute_with_code(const std::string& sql) - { - const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0); - return rc; - } - - sqlite3* operator*() - { - return db_; - } - - bool has_rtree(std::string const& index_table) + static bool has_rtree(std::string const& index_table,boost::shared_ptr ds) { try { std::ostringstream s; s << "SELECT pkid,xmin,xmax,ymin,ymax FROM " << index_table << " LIMIT 1"; - boost::scoped_ptr rs(execute_query(s.str())); + boost::shared_ptr rs = ds->execute_query(s.str()); if (rs->is_valid() && rs->step_next()) { return true; @@ -276,12 +300,13 @@ public: return false; } - bool detect_types_from_subquery(std::string const& query, + static bool detect_types_from_subquery(std::string const& query, std::string & geometry_field, - mapnik::layer_descriptor & desc) + mapnik::layer_descriptor & desc, + boost::shared_ptr ds) { bool found = false; - boost::scoped_ptr rs(execute_query(query)); + boost::shared_ptr rs(ds->execute_query(query)); if (rs->is_valid() && rs->step_next()) { for (int i = 0; i < rs->column_count(); ++i) @@ -332,11 +357,12 @@ public: return found; } - bool table_info(std::string & key_field, + static bool table_info(std::string & key_field, bool detected_types, std::string & field, std::string & table, - mapnik::layer_descriptor & desc) + mapnik::layer_descriptor & desc, + boost::shared_ptr ds) { // http://www.sqlite.org/pragma.html#pragma_table_info @@ -345,7 +371,7 @@ public: // if the subquery-based type detection failed std::ostringstream s; s << "PRAGMA table_info(" << table << ")"; - boost::scoped_ptr rs(execute_query(s.str())); + boost::shared_ptr rs(ds->execute_query(s.str())); bool found_table = false; bool found_pk = false; while (rs->is_valid() && rs->step_next()) @@ -422,281 +448,6 @@ public: return found_table; } - -private: - - sqlite3* db_; - std::string file_; }; - -//============================================================================== - -class sqlite_utils -{ -public: - - static void dequote(std::string & z) - { - boost::algorithm::trim_if(z,boost::algorithm::is_any_of("[]'\"`")); - } - - static std::string index_for_table(std::string const& table,std::string const& field) - { - return "\"idx_" + mapnik::unquote_sql2(table) + "_" + field + "\""; - } - - static std::string get_index_db_name(std::string const& file) - { - //std::size_t idx = file.find_last_of("."); - //if(idx != std::string::npos) { - // return file.substr(0,idx) + ".index" + file.substr(idx); - //} - //else - //{ - return file + ".index"; - //} - } - - static void query_extent(boost::shared_ptr rs, - mapnik::box2d& extent) - { - - bool first = true; - while (rs->is_valid() && rs->step_next()) - { - int size; - const char* data = (const char*) rs->column_blob(0, size); - if (data) - { - // create a feature to parse geometry into - // see: http://trac.mapnik.org/ticket/745 - mapnik::feature_ptr feature(mapnik::feature_factory::create(0)); - mapnik::geometry_utils::from_wkb(feature->paths(), data, size, false, mapnik::wkbAuto); - mapnik::box2d const& bbox = feature->envelope(); - - if (bbox.valid()) - { - if (first) - { - first = false; - extent = bbox; - } - else - { - extent.expand_to_include(bbox); - } - } - } - } - } - - static void create_spatial_index(std::string const& index_db, - std::string const& table, - std::string const& key_field, - boost::shared_ptr rs, - mapnik::box2d& extent) - { - // TODO - return early/recover from failure? - // TODO - allow either in db or in separate - - //std::clog << "creating rtree: " << index_db << " : " << table << "\n"; -#if SQLITE_VERSION_NUMBER >= 3005000 - int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX; -#else - //int flags; -#endif - - boost::shared_ptr dataset_ = boost::make_shared(index_db,flags); - - dataset_->execute("BEGIN TRANSACTION"); - dataset_->execute("PRAGMA synchronous=OFF"); - // first drop the index if it already exists - std::ostringstream spatial_index_drop_sql; - spatial_index_drop_sql << "DROP TABLE IF EXISTS " << table; - dataset_->execute(spatial_index_drop_sql.str()); - - // create the spatial index - std::ostringstream spatial_index_sql; - spatial_index_sql << "create virtual table " << table - << " using rtree(pkid, xmin, xmax, ymin, ymax)"; - - std::ostringstream spatial_index_insert_sql; - spatial_index_insert_sql << "insert into " << table - << " values (?,?,?,?,?)"; - - sqlite3_stmt* stmt = 0; - dataset_->execute(spatial_index_sql.str()); - - const int rc = sqlite3_prepare_v2 (*(*dataset_), - spatial_index_insert_sql.str().c_str(), - -1, - &stmt, - 0); - if (rc != SQLITE_OK) - { - std::ostringstream index_error; - index_error << "Sqlite Plugin: auto-index table creation failed: '" - << sqlite3_errmsg(*(*dataset_)) << "' query was: " - << spatial_index_insert_sql; - - throw mapnik::datasource_exception(index_error.str()); - } - - bool first = true; - while (rs->is_valid() && rs->step_next()) - { - int size; - const char* data = (const char*) rs->column_blob(0, size); - if (data) - { - // create a tmp feature to be able to parse geometry - // ideally we would not have to do this. - // see: http://trac.mapnik.org/ticket/745 - mapnik::feature_ptr feature(mapnik::feature_factory::create(0)); - mapnik::geometry_utils::from_wkb(feature->paths(), data, size, false, mapnik::wkbAuto); - mapnik::box2d bbox = feature->envelope(); - - if (bbox.valid()) - { - if (first) - { - first = false; - extent = bbox; - } - else - { - extent.expand_to_include(bbox); - } - - const int type_oid = rs->column_type(1); - if (type_oid != SQLITE_INTEGER) - { - std::ostringstream type_error; - type_error << "Sqlite Plugin: invalid type for key field '" - << key_field << "' when creating index '" << table - << "' type was: " << type_oid << ""; - - throw mapnik::datasource_exception(type_error.str()); - } - - const sqlite_int64 pkid = rs->column_integer64(1); - if (sqlite3_bind_int64(stmt, 1, pkid) != SQLITE_OK) - { - throw mapnik::datasource_exception("invalid value for for key field while generating index"); - } - - if ((sqlite3_bind_double(stmt, 2, bbox.minx()) != SQLITE_OK) || - (sqlite3_bind_double(stmt, 3, bbox.maxx()) != SQLITE_OK) || - (sqlite3_bind_double(stmt, 4, bbox.miny()) != SQLITE_OK) || - (sqlite3_bind_double(stmt, 5, bbox.maxy()) != SQLITE_OK)) - { - throw mapnik::datasource_exception("invalid value for for extent while generating index"); - } - - const int res = sqlite3_step(stmt); - if (res != SQLITE_ROW && res != SQLITE_DONE) - { - std::ostringstream s; - s << "SQLite Plugin: inserting bbox into rtree index failed: " - << "error code " << sqlite3_errcode(*(*dataset_)) << ": '" - << sqlite3_errmsg(*(*dataset_)) << "' query was: " - << spatial_index_insert_sql; - - throw mapnik::datasource_exception(s.str()); - } - - sqlite3_reset(stmt); - } - else - { - std::ostringstream index_error; - index_error << "SQLite Plugin: encountered invalid bbox at '" - << key_field << "' == " << rs->column_integer64(1); - - throw mapnik::datasource_exception(index_error.str()); - } - } - } - - const int res = sqlite3_finalize(stmt); - if (res != SQLITE_OK) - { - throw mapnik::datasource_exception("auto-indexing failed: set use_spatial_index=false to disable auto-indexing and avoid this error"); - } - - dataset_->execute("COMMIT"); - } - - static bool detect_extent(boost::shared_ptr dataset_, - bool has_spatial_index, - mapnik::box2d & extent, - std::string const& index_table, - std::string const& metadata, - std::string const& geometry_field, - std::string const& geometry_table, - std::string const& key_field, - std::string const& table - ) - { - if (has_spatial_index) - { - std::ostringstream s; - s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM " - << index_table; - - boost::scoped_ptr rs(dataset_->execute_query(s.str())); - if (rs->is_valid() && rs->step_next()) - { - if (! rs->column_isnull(0)) - { - try - { - double xmin = boost::lexical_cast(rs->column_double(0)); - double ymin = boost::lexical_cast(rs->column_double(1)); - double xmax = boost::lexical_cast(rs->column_double(2)); - double ymax = boost::lexical_cast(rs->column_double(3)); - extent.init(xmin, ymin, xmax, ymax); - return true; - } - catch (boost::bad_lexical_cast& ex) - { - std::ostringstream ss; - ss << "SQLite Plugin: warning: could not determine extent from query: " - << "'" << s.str() << "' \n problem was: " << ex.what() << std::endl; - std::clog << ss.str(); - } - } - } - } - else if (! metadata.empty()) - { - std::ostringstream s; - s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata; - s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table << "')"; - boost::scoped_ptr rs(dataset_->execute_query(s.str())); - if (rs->is_valid() && rs->step_next()) - { - double xmin = rs->column_double (0); - double ymin = rs->column_double (1); - double xmax = rs->column_double (2); - double ymax = rs->column_double (3); - extent.init (xmin, ymin, xmax, ymax); - return true; - } - } - else if (! key_field.empty()) - { - std::ostringstream s; - s << "SELECT " << geometry_field << "," << key_field - << " FROM (" << table << ")"; - boost::shared_ptr rs(dataset_->execute_query(s.str())); - sqlite_utils::query_extent(rs,extent); - return true; - } - return false; - } - -}; - -#endif //SQLITE_TYPES_HPP +#endif // MAPNIK_SQLITE_UTILS_HPP diff --git a/src/agg/process_text_symbolizer.cpp b/src/agg/process_text_symbolizer.cpp index f58d73289..a127a19ee 100644 --- a/src/agg/process_text_symbolizer.cpp +++ b/src/agg/process_text_symbolizer.cpp @@ -35,6 +35,7 @@ void agg_renderer::process(text_symbolizer const& sym, { + // Use a boost::ptr_vector here instread of std::vector? std::vector geometries_to_process; unsigned num_geom = feature.num_geometries(); for (unsigned i=0; i Date: Tue, 8 Nov 2011 11:50:56 +0000 Subject: [PATCH 5/5] use rint from global.hpp in windows --- plugins/input/gdal/gdal_featureset.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/input/gdal/gdal_featureset.cpp b/plugins/input/gdal/gdal_featureset.cpp index fd8969b4f..1ab5c895f 100644 --- a/plugins/input/gdal/gdal_featureset.cpp +++ b/plugins/input/gdal/gdal_featureset.cpp @@ -21,6 +21,7 @@ *****************************************************************************/ // mapnik +#include #include #include "gdal_featureset.hpp" @@ -39,6 +40,9 @@ using mapnik::geometry_type; using mapnik::datasource_exception; using mapnik::feature_factory; +#ifdef _WINDOWS +using mapnik::rint; +#endif gdal_featureset::gdal_featureset(GDALDataset& dataset, int band,