sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sqlite: futher refactoring of sqlite plugin, allowing all tests to pass

Browse source
This commit is contained in:
Dane Springmeyer 2011-11-03 19:51:37 -04:00
parent b9b50899a6
commit 6257ac57c9
12 changed files with 744 additions and 520 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
include/mapnik/sql_utils.hpp
View file

@ -27,17 +27,16 @@
#include <boost/algorithm/string.hpp>
#include <boost/scoped_array.hpp>
namespace mapnik
{
namespace mapnik { namespace sql_utils {
inline std::string unquote_sql(const std::string& sql)
inline std::string unquote_double(const std::string& sql)
{
std::string table_name = boost::algorithm::to_lower_copy(sql);
boost::algorithm::trim_if(table_name,boost::algorithm::is_any_of("\""));
return table_name;
}
inline std::string unquote_sql2(const std::string& sql)
inline std::string unquote(const std::string& sql)
{
std::string table_name = boost::algorithm::to_lower_copy(sql);
boost::algorithm::trim_if(table_name,boost::algorithm::is_any_of("\"\'"));
@ -65,7 +64,6 @@ inline std::string table_from_sql(const std::string& sql)
std::string::size_type idx = table_name.rfind(" from ");
if (idx!=std::string::npos)
{
idx = table_name.find_first_not_of(" ",idx+5);
if (idx != std::string::npos)
{
@ -191,6 +189,7 @@ inline std::string numeric2string(const char* buf)
}
return ss.str();
}
}
}

22
plugins/input/postgis/postgis_datasource.cpp
View file

@ -126,7 +126,7 @@ void postgis_datasource::bind() const
if(geometry_table_.empty())
{
geometry_table_ = mapnik::table_from_sql(table_);
geometry_table_ = mapnik::sql_utils::table_from_sql(table_);
}
std::string::size_type idx = geometry_table_.find_last_of('.');
if (idx!=std::string::npos)
@ -150,13 +150,13 @@ void postgis_datasource::bind() const
{
std::ostringstream s;
s << "SELECT f_geometry_column, srid FROM ";
s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << mapnik::unquote_sql(geometry_table_) <<"'";
s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << mapnik::sql_utils::unquote_double(geometry_table_) <<"'";
if (schema_.length() > 0)
s << " AND f_table_schema='" << mapnik::unquote_sql(schema_) << "'";
s << " AND f_table_schema='" << mapnik::sql_utils::unquote_double(schema_) << "'";
if (geometry_field_.length() > 0)
s << " AND f_geometry_column='" << mapnik::unquote_sql(geometry_field_) << "'";
s << " AND f_geometry_column='" << mapnik::sql_utils::unquote_double(geometry_field_) << "'";
/*
if (show_queries_)
@ -502,14 +502,14 @@ featureset_ptr postgis_datasource::features(const query& q) const
s << "AsBinary(\"" << geometryColumn_ << "\") AS geom";
if (!key_field_.empty())
mapnik::quote_attr(s,key_field_);
mapnik::sql_utils::quote_attr(s,key_field_);
std::set<std::string> const& props=q.property_names();
std::set<std::string>::const_iterator pos=props.begin();
std::set<std::string>::const_iterator end=props.end();
while (pos != end)
{
mapnik::quote_attr(s,*pos);
mapnik::sql_utils::quote_attr(s,*pos);
++pos;
}
@ -578,14 +578,14 @@ featureset_ptr postgis_datasource::features_at_point(coord2d const& pt) const
s << "AsBinary(\"" << geometryColumn_ << "\") AS geom";
if (!key_field_.empty())
mapnik::quote_attr(s,key_field_);
mapnik::sql_utils::quote_attr(s,key_field_);
std::vector<attribute_descriptor>::const_iterator itr = desc_.get_descriptors().begin();
std::vector<attribute_descriptor>::const_iterator end = desc_.get_descriptors().end();
unsigned size=0;
while (itr != end)
{
mapnik::quote_attr(s,itr->get_name());
mapnik::sql_utils::quote_attr(s,itr->get_name());
++itr;
++size;
}
@ -644,11 +644,11 @@ box2d<double> postgis_datasource::envelope() const
if (schema_.length() > 0)
{
s << mapnik::unquote_sql(schema_) << "','";
s << mapnik::sql_utils::unquote_double(schema_) << "','";
}
s << mapnik::unquote_sql(geometry_table_) << "','"
<< mapnik::unquote_sql(geometryColumn_) << "') as ext) as tmp";
s << mapnik::sql_utils::unquote_double(geometry_table_) << "','"
<< mapnik::sql_utils::unquote_double(geometryColumn_) << "') as ext) as tmp";
}
else
{

2
plugins/input/postgis/postgis_featureset.cpp
View file

@ -162,7 +162,7 @@ feature_ptr postgis_featureset::next()
}
else if (oid == 1700) // numeric
{
std::string str = mapnik::numeric2string(buf);
std::string str = mapnik::sql_utils::numeric2string(buf);
try
{
double val = boost::lexical_cast<double>(str);

161
plugins/input/sqlite/sqlite_connection.hpp Normal file
View file

@ -0,0 +1,161 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2011 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#ifndef MAPNIK_SQLITE_CONNECTION_HPP
#define MAPNIK_SQLITE_CONNECTION_HPP
// stl
#include <string.h>
// mapnik
#include <mapnik/datasource.hpp>
#include <mapnik/sql_utils.hpp>
// boost
#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>
#include <boost/algorithm/string.hpp>
// sqlite
extern "C" {
#include <sqlite3.h>
}
#include "sqlite_resultset.hpp"
//==============================================================================
class sqlite_connection
{
public:
sqlite_connection (const std::string& file)
: db_(0),
file_(file)
{
#if SQLITE_VERSION_NUMBER >= 3005000
int mode = SQLITE_OPEN_READWRITE;
#if SQLITE_VERSION_NUMBER >= 3006018
// shared cache flag not available until >= 3.6.18
mode |= SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE;
#endif
const int rc = sqlite3_open_v2 (file_.c_str(), &db_, mode, 0);
#else
#warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..."
const int rc = sqlite3_open (file_.c_str(), &db_);
#endif
if (rc != SQLITE_OK)
{
std::ostringstream s;
s << "Sqlite Plugin: " << sqlite3_errmsg (db_);
throw mapnik::datasource_exception (s.str());
}
sqlite3_busy_timeout(db_,5000);
}
sqlite_connection (const std::string& file, int flags)
: db_(0),
file_(file)
{
#if SQLITE_VERSION_NUMBER >= 3005000
const int rc = sqlite3_open_v2 (file_.c_str(), &db_, flags, 0);
#else
#warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..."
const int rc = sqlite3_open (file_.c_str(), &db_);
#endif
if (rc != SQLITE_OK)
{
std::ostringstream s;
s << "Sqlite Plugin: " << sqlite3_errmsg (db_);
throw mapnik::datasource_exception (s.str());
}
}
virtual ~sqlite_connection ()
{
if (db_)
{
sqlite3_close (db_);
}
}
void throw_sqlite_error(const std::string& sql)
{
std::ostringstream s;
s << "Sqlite Plugin: ";
if (db_)
s << "'" << sqlite3_errmsg(db_) << "'";
else
s << "unknown error, lost connection";
s << " (" << file_ << ")"
<< "\nFull sql was: '"
<< sql << "'";
throw mapnik::datasource_exception (s.str());
}
boost::shared_ptr<sqlite_resultset> execute_query(const std::string& sql)
{
sqlite3_stmt* stmt = 0;
const int rc = sqlite3_prepare_v2 (db_, sql.c_str(), -1, &stmt, 0);
if (rc != SQLITE_OK)
{
throw_sqlite_error(sql);
}
return boost::make_shared<sqlite_resultset>(stmt);
}
void execute(const std::string& sql)
{
const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
if (rc != SQLITE_OK)
{
throw_sqlite_error(sql);
}
}
int execute_with_code(const std::string& sql)
{
const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
return rc;
}
sqlite3* operator*()
{
return db_;
}
private:
sqlite3* db_;
std::string file_;
};
#endif // MAPNIK_SQLITE_CONNECTION_HPP

84
plugins/input/sqlite/sqlite_datasource.cpp
View file

@ -22,6 +22,8 @@
#include "sqlite_datasource.hpp"
#include "sqlite_featureset.hpp"
#include "sqlite_resultset.hpp"
#include "sqlite_utils.hpp"
// mapnik
#include <mapnik/ptree_helpers.hpp>
@ -64,8 +66,8 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind)
format_(mapnik::wkbAuto)
{
/* TODO
- auto detect primary key - https://github.com/mapnik/mapnik/issues/82
- throw if no primary key but spatial index is present?
- remove auto-indexing
*/
boost::optional<std::string> file = params_.get<std::string>("file");
@ -95,34 +97,37 @@ void sqlite_datasource::bind() const
else
dataset_name_ = *file;
if (!boost::filesystem::exists(dataset_name_))
if ((dataset_name_.compare(":memory:") != 0) && (!boost::filesystem::exists(dataset_name_)))
{
throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist");
}
multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries", false);
use_spatial_index_ = *params_.get<mapnik::boolean>("use_spatial_index", true);
boost::optional<std::string> key_field_name = params_.get<std::string>("key_field");
if (key_field_name)
{
key_field_ = *key_field_name;
}
// TODO - remove this option once all datasources have an indexing api
bool auto_index = *params_.get<mapnik::boolean>("auto_index", true);
boost::optional<std::string> ext = params_.get<std::string>("extent");
if (ext) extent_initialized_ = extent_.from_string(*ext);
boost::optional<std::string> wkb = params_.get<std::string>("wkb_format");
if (wkb)
{
if (*wkb == "spatialite")
{
format_ = mapnik::wkbSpatiaLite;
}
else if (*wkb == "generic")
{
format_ = mapnik::wkbGeneric;
}
else
{
format_ = mapnik::wkbAuto;
}
}
multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries", false);
use_spatial_index_ = *params_.get<mapnik::boolean>("use_spatial_index", true);
boost::optional<std::string> ext = params_.get<std::string>("extent");
if (ext) extent_initialized_ = extent_.from_string(*ext);
// Populate init_statements_
// 1. Build attach database statements from the "attachdb" parameter
// 2. Add explicit init statements from "initdb" parameter
@ -144,7 +149,7 @@ void sqlite_datasource::bind() const
if (geometry_table_.empty())
{
geometry_table_ = mapnik::table_from_sql(table_);
geometry_table_ = mapnik::sql_utils::table_from_sql(table_);
}
// if 'table_' is a subquery then we try to deduce names
@ -155,8 +160,6 @@ void sqlite_datasource::bind() const
using_subquery_ = true;
}
std::string index_db = sqlite_utils::get_index_db_name(dataset_name_);
// now actually create the connection and start executing setup sql
dataset_ = boost::make_shared<sqlite_connection>(dataset_name_);
@ -175,7 +178,7 @@ void sqlite_datasource::bind() const
{
std::ostringstream s;
s << "SELECT " << fields_ << " FROM (" << table_ << ") LIMIT 1";
found_types_via_subquery = dataset_->detect_types_from_subquery(s.str(),geometry_field_,desc_);
found_types_via_subquery = sqlite_utils::detect_types_from_subquery(s.str(),geometry_field_,desc_,dataset_);
}
// TODO - consider removing this
@ -184,11 +187,12 @@ void sqlite_datasource::bind() const
desc_.add_descriptor(attribute_descriptor("rowid", mapnik::Integer));
}
bool found_table = dataset_->table_info(key_field_,
bool found_table = sqlite_utils::table_info(key_field_,
found_types_via_subquery,
geometry_field_,
geometry_table_,
desc_);
desc_,
dataset_);
if (! found_table)
{
@ -214,20 +218,52 @@ void sqlite_datasource::bind() const
// we have discovered meta-data or else we don't know the column name
index_table_ = sqlite_utils::index_for_table(geometry_table_,geometry_field_);
}
std::string index_db = sqlite_utils::index_for_db(dataset_name_);
has_spatial_index_ = false;
if (use_spatial_index_)
{
/*if (boost::filesystem::exists(index_db))
if (boost::filesystem::exists(index_db))
{
dataset_->execute("attach database '" + index_db + "' as " + index_table_);
}
*/
has_spatial_index_ = dataset_->has_rtree(index_table_);
has_spatial_index_ = sqlite_utils::has_rtree(index_table_,dataset_);
}
if (! extent_initialized_
&& !has_spatial_index_
&& auto_index)
{
if (! key_field_.empty())
{
std::ostringstream query;
query << "SELECT "
<< geometry_field_
<< "," << key_field_
<< " FROM ("
<< geometry_table_ << ")";
boost::shared_ptr<sqlite_resultset> rs = dataset_->execute_query(query.str());
sqlite_utils::create_spatial_index(index_db,index_table_,rs,extent_);
extent_initialized_ = true;
}
else
{
std::ostringstream s;
s << "Sqlite Plugin: key_field is empty for "
<< geometry_field_
<< " and "
<< geometry_table_;
throw datasource_exception(s.str());
}
}
if (! extent_initialized_)
{
// TODO - clean this up - reducing arguments
if (!sqlite_utils::detect_extent(dataset_,
has_spatial_index_,
extent_,
@ -262,7 +298,7 @@ namespace filesystem {
path symlink_path;
#ifdef BOOST_POSIX_API
for (std::size_t path_max = 64;; path_max *= 2)// loop 'til buffer large enough
for (std::size_t path_max = 64;; path_max *= 2)// loop 'til buffer is large enough
{
boost::scoped_array<char> buf(new char[path_max]);
ssize_t result;

8
plugins/input/sqlite/sqlite_datasource.hpp
View file

@ -20,8 +20,8 @@
*
*****************************************************************************/
#ifndef SQLITE_DATASOURCE_HPP
#define SQLITE_DATASOURCE_HPP
#ifndef MAPNIK_SQLITE_DATASOURCE_HPP
#define MAPNIK_SQLITE_DATASOURCE_HPP
// mapnik
#include <mapnik/datasource.hpp>
@ -34,7 +34,7 @@
#include <boost/scoped_ptr.hpp>
// sqlite
#include "sqlite_types.hpp"
#include "sqlite_connection.hpp"
class sqlite_datasource : public mapnik::datasource
@ -78,4 +78,4 @@ private:
void parse_attachdb(std::string const& attachdb) const;
};
#endif // SQLITE_DATASOURCE_HPP
#endif // MAPNIK_SQLITE_DATASOURCE_HPP

1
plugins/input/sqlite/sqlite_featureset.cpp
View file

@ -33,6 +33,7 @@
// ogr
#include "sqlite_featureset.hpp"
#include "sqlite_utils.hpp"
using mapnik::query;
using mapnik::box2d;

8
plugins/input/sqlite/sqlite_featureset.hpp
View file

@ -20,8 +20,8 @@
*
*****************************************************************************/
#ifndef SQLITE_FEATURESET_HPP
#define SQLITE_FEATURESET_HPP
#ifndef MAPNIK_SQLITE_FEATURESET_HPP
#define MAPNIK_SQLITE_FEATURESET_HPP
// mapnik
#include <mapnik/datasource.hpp>
@ -33,7 +33,7 @@
#include <boost/shared_ptr.hpp>
// sqlite
#include "sqlite_types.hpp"
#include "sqlite_resultset.hpp"
class sqlite_featureset : public mapnik::Featureset
@ -55,4 +55,4 @@ private:
bool using_subquery_;
};
#endif // SQLITE_FEATURESET_HPP
#endif // MAPNIK_SQLITE_FEATURESET_HPP

132
plugins/input/sqlite/sqlite_prepared.hpp Normal file
View file

@ -0,0 +1,132 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2011 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#ifndef MAPNIK_SQLITE_PREPARED_HPP
#define MAPNIK_SQLITE_PREPARED_HPP
// mapnik
#include <mapnik/datasource.hpp>
#include <mapnik/box2d.hpp>
// boost
#include <boost/shared_ptr.hpp>
#include <boost/utility.hpp>
// stl
#include <string.h>
#include "sqlite_connection.hpp"
// sqlite
extern "C" {
#include <sqlite3.h>
}
class prepared_index_statement : boost::noncopyable
{
public:
prepared_index_statement(boost::shared_ptr<sqlite_connection> ds, std::string const& sql)
: ds_(ds),
stmt_(0)
{
const int rc = sqlite3_prepare_v2(*(*ds_),
sql.c_str(),
-1,
&stmt_,
0);
if (rc != SQLITE_OK)
{
std::ostringstream index_error;
index_error << "Sqlite Plugin: auto-index table creation failed: '"
<< sqlite3_errmsg(*(*ds_)) << "' query was: "
<< sql;
throw mapnik::datasource_exception(index_error.str());
}
}
~prepared_index_statement()
{
if (stmt_)
{
int res = sqlite3_finalize(stmt_);
if (res != SQLITE_OK)
{
if (*(*ds_))
{
std::cerr << "ERR:" << sqlite3_errmsg(*(*ds_)) << "\n";
}
else
{
std::cerr << "SQLite Plugin: " << res << "\n";
}
}
}
}
void bind(sqlite_int64 const pkid)
{
if (sqlite3_bind_int64(stmt_, 1, pkid) != SQLITE_OK)
{
throw mapnik::datasource_exception("SQLite Plugin: invalid value for for key field while generating index");
}
}
void bind(mapnik::box2d<double> const& bbox)
{
if ((sqlite3_bind_double(stmt_, 2, bbox.minx()) != SQLITE_OK) ||
(sqlite3_bind_double(stmt_, 3, bbox.maxx()) != SQLITE_OK) ||
(sqlite3_bind_double(stmt_, 4, bbox.miny()) != SQLITE_OK) ||
(sqlite3_bind_double(stmt_, 5, bbox.maxy()) != SQLITE_OK))
{
throw mapnik::datasource_exception("SQLite Plugin: invalid value for for extent while generating index");
}
}
bool step_next ()
{
const int status = sqlite3_step(stmt_);
if (status != SQLITE_ROW && status != SQLITE_DONE)
{
std::ostringstream s;
s << "SQLite Plugin: inserting bbox into rtree index failed";
std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_)));
if (msg != "unknown error")
{
s << ": " << msg;
}
throw mapnik::datasource_exception(s.str());
}
sqlite3_reset(stmt_);
return status == SQLITE_ROW;
}
private:
boost::shared_ptr<sqlite_connection> ds_;
sqlite3_stmt * stmt_;
};
#endif // MAPNIK_SQLITE_PREPARED_HPP

143
plugins/input/sqlite/sqlite_resultset.hpp Normal file
View file

@ -0,0 +1,143 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2011 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#ifndef MAPNIK_SQLITE_RESULTSET_HPP
#define MAPNIK_SQLITE_RESULTSET_HPP
// mapnik
#include <mapnik/datasource.hpp>
// stl
#include <string.h>
// sqlite
extern "C" {
#include <sqlite3.h>
}
//==============================================================================
class sqlite_resultset
{
public:
sqlite_resultset (sqlite3_stmt* stmt)
: stmt_(stmt)
{
}
~sqlite_resultset ()
{
if (stmt_)
{
sqlite3_finalize (stmt_);
}
}
bool is_valid ()
{
return stmt_ != 0;
}
bool step_next ()
{
const int status = sqlite3_step (stmt_);
if (status != SQLITE_ROW && status != SQLITE_DONE)
{
std::ostringstream s;
s << "SQLite Plugin: retrieving next row failed";
std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_)));
if (msg != "unknown error")
{
s << ": " << msg;
}
throw mapnik::datasource_exception(s.str());
}
return status == SQLITE_ROW;
}
int column_count ()
{
return sqlite3_column_count (stmt_);
}
int column_type (int col)
{
return sqlite3_column_type (stmt_, col);
}
const char* column_name (int col)
{
return sqlite3_column_name (stmt_, col);
}
bool column_isnull (int col)
{
return sqlite3_column_type (stmt_, col) == SQLITE_NULL;
}
int column_integer (int col)
{
return sqlite3_column_int (stmt_, col);
}
int column_integer64 (int col)
{
return sqlite3_column_int64 (stmt_, col);
}
double column_double (int col)
{
return sqlite3_column_double (stmt_, col);
}
const char* column_text (int col, int& len)
{
len = sqlite3_column_bytes (stmt_, col);
return (const char*) sqlite3_column_text (stmt_, col);
}
const char* column_text (int col)
{
return (const char*) sqlite3_column_text (stmt_, col);
}
const void* column_blob (int col, int& bytes)
{
bytes = sqlite3_column_bytes (stmt_, col);
return (const char*) sqlite3_column_blob (stmt_, col);
}
sqlite3_stmt* get_statement()
{
return stmt_;
}
private:
sqlite3_stmt* stmt_;
};
#endif // MAPNIK_SQLITE_RESULTSET_HPP

693
plugins/input/sqlite/sqlite_types.hpp → plugins/input/sqlite/sqlite_utils.hpp
View file

@ -20,16 +20,16 @@
*
*****************************************************************************/
#ifndef SQLITE_TYPES_HPP
#define SQLITE_TYPES_HPP
#ifndef MAPNIK_SQLITE_UTILS_HPP
#define MAPNIK_SQLITE_UTILS_HPP
// stl
#include <string.h>
// mapnik
#include <mapnik/datasource.hpp>
#include <mapnik/geometry.hpp>
#include <mapnik/sql_utils.hpp>
#include <mapnik/feature_factory.hpp> // to enable extent fallback hack
// boost
@ -43,227 +43,251 @@ extern "C" {
#include <sqlite3.h>
}
#include "sqlite_resultset.hpp"
#include "sqlite_prepared.hpp"
#include "sqlite_connection.hpp"
//==============================================================================
class sqlite_resultset
class sqlite_utils
{
public:
sqlite_resultset (sqlite3_stmt* stmt)
: stmt_(stmt)
static void dequote(std::string & z)
{
boost::algorithm::trim_if(z,boost::algorithm::is_any_of("[]'\"`"));
}
static std::string index_for_table(std::string const& table, std::string const& field)
{
return "\"idx_" + mapnik::sql_utils::unquote(table) + "_" + field + "\"";
}
~sqlite_resultset ()
static std::string index_for_db(std::string const& file)
{
if (stmt_)
//std::size_t idx = file.find_last_of(".");
//if(idx != std::string::npos) {
// return file.substr(0,idx) + ".index" + file.substr(idx);
//}
//else
//{
return file + ".index";
//}
}
static void query_extent(boost::shared_ptr<sqlite_resultset> rs,
mapnik::box2d<double>& extent)
{
bool first = true;
while (rs->is_valid() && rs->step_next())
{
sqlite3_finalize (stmt_);
}
}
bool is_valid ()
{
return stmt_ != 0;
}
bool step_next ()
{
const int status = sqlite3_step (stmt_);
if (status != SQLITE_ROW && status != SQLITE_DONE)
{
std::ostringstream s;
s << "SQLite Plugin: retrieving next row failed";
std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_)));
if (msg != "unknown error")
int size;
const char* data = (const char*) rs->column_blob(0, size);
if (data)
{
s << ": " << msg;
boost::ptr_vector<mapnik::geometry_type> paths;
mapnik::geometry_utils::from_wkb(paths, data, size, false, mapnik::wkbAuto);
for (unsigned i=0; i<paths.size(); ++i)
{
mapnik::box2d<double> const& bbox = paths[i].envelope();
if (bbox.valid())
{
if (first)
{
first = false;
extent = bbox;
}
else
{
extent.expand_to_include(bbox);
}
}
}
}
throw mapnik::datasource_exception(s.str());
}
return status == SQLITE_ROW;
}
int column_count ()
{
return sqlite3_column_count (stmt_);
}
int column_type (int col)
{
return sqlite3_column_type (stmt_, col);
}
const char* column_name (int col)
static void create_spatial_index(std::string const& index_db,
std::string const& index_table,
boost::shared_ptr<sqlite_resultset> rs,
mapnik::box2d<double>& extent)
{
return sqlite3_column_name (stmt_, col);
}
bool column_isnull (int col)
{
return sqlite3_column_type (stmt_, col) == SQLITE_NULL;
}
int column_integer (int col)
{
return sqlite3_column_int (stmt_, col);
}
int column_integer64 (int col)
{
return sqlite3_column_int64 (stmt_, col);
}
double column_double (int col)
{
return sqlite3_column_double (stmt_, col);
}
const char* column_text (int col, int& len)
{
len = sqlite3_column_bytes (stmt_, col);
return (const char*) sqlite3_column_text (stmt_, col);
}
const char* column_text (int col)
{
return (const char*) sqlite3_column_text (stmt_, col);
}
const void* column_blob (int col, int& bytes)
{
bytes = sqlite3_column_bytes (stmt_, col);
return (const char*) sqlite3_column_blob (stmt_, col);
}
sqlite3_stmt* get_statement()
{
return stmt_;
}
private:
sqlite3_stmt* stmt_;
};
//==============================================================================
class sqlite_connection
{
public:
sqlite_connection (const std::string& file)
: db_(0),
file_(file)
{
#if SQLITE_VERSION_NUMBER >= 3005000
int mode = SQLITE_OPEN_READWRITE;
#if SQLITE_VERSION_NUMBER >= 3006018
// shared cache flag not available until >= 3.6.18
mode |= SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE;
#endif
const int rc = sqlite3_open_v2 (file_.c_str(), &db_, mode, 0);
#else
#warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..."
const int rc = sqlite3_open (file_.c_str(), &db_);
#endif
if (rc != SQLITE_OK)
{
std::ostringstream s;
s << "Sqlite Plugin: " << sqlite3_errmsg (db_);
throw mapnik::datasource_exception (s.str());
}
/* TODO
- speedups
- return early/recover from failure
- allow either in db or in separate
*/
sqlite3_busy_timeout(db_,5000);
if (!rs->is_valid())
return;
#if SQLITE_VERSION_NUMBER >= 3005000
int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX;
#else
int flags;
#endif
boost::shared_ptr<sqlite_connection> ds = boost::make_shared<sqlite_connection>(index_db,flags);
// first drop the index if it already exists
std::ostringstream spatial_index_drop_sql;
spatial_index_drop_sql << "DROP TABLE IF EXISTS " << index_table;
ds->execute(spatial_index_drop_sql.str());
ds->execute("PRAGMA synchronous=OFF");
ds->execute("BEGIN TRANSACTION");
// create the spatial index
std::ostringstream create_idx;
create_idx << "create virtual table "
<< index_table
<< " using rtree(pkid, xmin, xmax, ymin, ymax)";
// insert for prepared statement
std::ostringstream insert_idx;
insert_idx << "insert into "
<< index_table
<< " values (?,?,?,?,?)";
ds->execute(create_idx.str());
prepared_index_statement ps(ds,insert_idx.str());
bool first = true;
while (rs->is_valid() && rs->step_next())
{
int size;
const char* data = (const char*) rs->column_blob(0, size);
if (data)
{
boost::ptr_vector<mapnik::geometry_type> paths;
// TODO - contraint fails if multiple_geometries = true
bool multiple_geometries = false;
mapnik::geometry_utils::from_wkb(paths, data, size, multiple_geometries, mapnik::wkbAuto);
for (unsigned i=0; i<paths.size(); ++i)
{
mapnik::box2d<double> const& bbox = paths[i].envelope();
if (bbox.valid())
{
if (first)
{
first = false;
extent = bbox;
}
else
{
extent.expand_to_include(bbox);
}
ps.bind(bbox);
const int type_oid = rs->column_type(1);
if (type_oid != SQLITE_INTEGER)
{
std::ostringstream error_msg;
error_msg << "Sqlite Plugin: invalid type for key field '"
<< rs->column_name(1) << "' when creating index '" << index_table
<< "' type was: " << type_oid << "";
throw mapnik::datasource_exception(error_msg.str());
}
const sqlite_int64 pkid = rs->column_integer64(1);
ps.bind(pkid);
}
else
{
std::ostringstream error_msg;
error_msg << "SQLite Plugin: encountered invalid bbox at '"
<< rs->column_name(1) << "' == " << rs->column_integer64(1);
throw mapnik::datasource_exception(error_msg.str());
}
ps.step_next();
}
}
}
ds->execute("COMMIT");
}
sqlite_connection (const std::string& file, int flags)
: db_(0),
file_(file)
static bool detect_extent(boost::shared_ptr<sqlite_connection> ds,
bool has_spatial_index,
mapnik::box2d<double> & extent,
std::string const& index_table,
std::string const& metadata,
std::string const& geometry_field,
std::string const& geometry_table,
std::string const& key_field,
std::string const& table
)
{
#if SQLITE_VERSION_NUMBER >= 3005000
const int rc = sqlite3_open_v2 (file_.c_str(), &db_, flags, 0);
#else
#warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..."
const int rc = sqlite3_open (file_.c_str(), &db_);
#endif
if (rc != SQLITE_OK)
if (has_spatial_index)
{
std::ostringstream s;
s << "Sqlite Plugin: " << sqlite3_errmsg (db_);
throw mapnik::datasource_exception (s.str());
}
}
virtual ~sqlite_connection ()
{
if (db_)
{
sqlite3_close (db_);
}
}
void throw_sqlite_error(const std::string& sql)
{
std::ostringstream s;
s << "Sqlite Plugin: ";
if (db_)
s << "'" << sqlite3_errmsg(db_) << "'";
else
s << "unknown error, lost connection";
s << " (" << file_ << ")"
<< "\nFull sql was: '"
<< sql << "'";
throw mapnik::datasource_exception (s.str());
}
s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM "
<< index_table;
sqlite_resultset* execute_query(const std::string& sql)
{
sqlite3_stmt* stmt = 0;
const int rc = sqlite3_prepare_v2 (db_, sql.c_str(), -1, &stmt, 0);
if (rc != SQLITE_OK)
{
throw_sqlite_error(sql);
boost::shared_ptr<sqlite_resultset> rs(ds->execute_query(s.str()));
if (rs->is_valid() && rs->step_next())
{
if (! rs->column_isnull(0))
{
try
{
double xmin = boost::lexical_cast<double>(rs->column_double(0));
double ymin = boost::lexical_cast<double>(rs->column_double(1));
double xmax = boost::lexical_cast<double>(rs->column_double(2));
double ymax = boost::lexical_cast<double>(rs->column_double(3));
extent.init(xmin, ymin, xmax, ymax);
return true;
}
catch (boost::bad_lexical_cast& ex)
{
std::ostringstream ss;
ss << "SQLite Plugin: warning: could not determine extent from query: "
<< "'" << s.str() << "' \n problem was: " << ex.what() << std::endl;
std::clog << ss.str();
}
}
}
}
return new sqlite_resultset (stmt);
}
void execute(const std::string& sql)
{
const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
if (rc != SQLITE_OK)
else if (! metadata.empty())
{
throw_sqlite_error(sql);
std::ostringstream s;
s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata;
s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table << "')";
boost::shared_ptr<sqlite_resultset> rs(ds->execute_query(s.str()));
if (rs->is_valid() && rs->step_next())
{
double xmin = rs->column_double (0);
double ymin = rs->column_double (1);
double xmax = rs->column_double (2);
double ymax = rs->column_double (3);
extent.init (xmin, ymin, xmax, ymax);
return true;
}
}
else if (! key_field.empty())
{
std::ostringstream s;
s << "SELECT " << geometry_field << "," << key_field
<< " FROM (" << table << ")";
boost::shared_ptr<sqlite_resultset> rs(ds->execute_query(s.str()));
sqlite_utils::query_extent(rs,extent);
return true;
}
return false;
}
int execute_with_code(const std::string& sql)
{
const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
return rc;
}
sqlite3* operator*()
{
return db_;
}
bool has_rtree(std::string const& index_table)
static bool has_rtree(std::string const& index_table,boost::shared_ptr<sqlite_connection> ds)
{
try
{
std::ostringstream s;
s << "SELECT pkid,xmin,xmax,ymin,ymax FROM " << index_table << " LIMIT 1";
boost::scoped_ptr<sqlite_resultset> rs(execute_query(s.str()));
boost::shared_ptr<sqlite_resultset> rs = ds->execute_query(s.str());
if (rs->is_valid() && rs->step_next())
{
return true;
@ -276,12 +300,13 @@ public:
return false;
}
bool detect_types_from_subquery(std::string const& query,
static bool detect_types_from_subquery(std::string const& query,
std::string & geometry_field,
mapnik::layer_descriptor & desc)
mapnik::layer_descriptor & desc,
boost::shared_ptr<sqlite_connection> ds)
{
bool found = false;
boost::scoped_ptr<sqlite_resultset> rs(execute_query(query));
boost::shared_ptr<sqlite_resultset> rs(ds->execute_query(query));
if (rs->is_valid() && rs->step_next())
{
for (int i = 0; i < rs->column_count(); ++i)
@ -332,11 +357,12 @@ public:
return found;
}
bool table_info(std::string & key_field,
static bool table_info(std::string & key_field,
bool detected_types,
std::string & field,
std::string & table,
mapnik::layer_descriptor & desc)
mapnik::layer_descriptor & desc,
boost::shared_ptr<sqlite_connection> ds)
{
// http://www.sqlite.org/pragma.html#pragma_table_info
@ -345,7 +371,7 @@ public:
// if the subquery-based type detection failed
std::ostringstream s;
s << "PRAGMA table_info(" << table << ")";
boost::scoped_ptr<sqlite_resultset> rs(execute_query(s.str()));
boost::shared_ptr<sqlite_resultset> rs(ds->execute_query(s.str()));
bool found_table = false;
bool found_pk = false;
while (rs->is_valid() && rs->step_next())
@ -422,281 +448,6 @@ public:
return found_table;
}
private:
sqlite3* db_;
std::string file_;
};
//==============================================================================
class sqlite_utils
{
public:
static void dequote(std::string & z)
{
boost::algorithm::trim_if(z,boost::algorithm::is_any_of("[]'\"`"));
}
static std::string index_for_table(std::string const& table,std::string const& field)
{
return "\"idx_" + mapnik::unquote_sql2(table) + "_" + field + "\"";
}
static std::string get_index_db_name(std::string const& file)
{
//std::size_t idx = file.find_last_of(".");
//if(idx != std::string::npos) {
// return file.substr(0,idx) + ".index" + file.substr(idx);
//}
//else
//{
return file + ".index";
//}
}
static void query_extent(boost::shared_ptr<sqlite_resultset> rs,
mapnik::box2d<double>& extent)
{
bool first = true;
while (rs->is_valid() && rs->step_next())
{
int size;
const char* data = (const char*) rs->column_blob(0, size);
if (data)
{
// create a feature to parse geometry into
// see: http://trac.mapnik.org/ticket/745
mapnik::feature_ptr feature(mapnik::feature_factory::create(0));
mapnik::geometry_utils::from_wkb(feature->paths(), data, size, false, mapnik::wkbAuto);
mapnik::box2d<double> const& bbox = feature->envelope();
if (bbox.valid())
{
if (first)
{
first = false;
extent = bbox;
}
else
{
extent.expand_to_include(bbox);
}
}
}
}
}
static void create_spatial_index(std::string const& index_db,
std::string const& table,
std::string const& key_field,
boost::shared_ptr<sqlite_resultset> rs,
mapnik::box2d<double>& extent)
{
// TODO - return early/recover from failure?
// TODO - allow either in db or in separate
//std::clog << "creating rtree: " << index_db << " : " << table << "\n";
#if SQLITE_VERSION_NUMBER >= 3005000
int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX;
#else
//int flags;
#endif
boost::shared_ptr<sqlite_connection> dataset_ = boost::make_shared<sqlite_connection>(index_db,flags);
dataset_->execute("BEGIN TRANSACTION");
dataset_->execute("PRAGMA synchronous=OFF");
// first drop the index if it already exists
std::ostringstream spatial_index_drop_sql;
spatial_index_drop_sql << "DROP TABLE IF EXISTS " << table;
dataset_->execute(spatial_index_drop_sql.str());
// create the spatial index
std::ostringstream spatial_index_sql;
spatial_index_sql << "create virtual table " << table
<< " using rtree(pkid, xmin, xmax, ymin, ymax)";
std::ostringstream spatial_index_insert_sql;
spatial_index_insert_sql << "insert into " << table
<< " values (?,?,?,?,?)";
sqlite3_stmt* stmt = 0;
dataset_->execute(spatial_index_sql.str());
const int rc = sqlite3_prepare_v2 (*(*dataset_),
spatial_index_insert_sql.str().c_str(),
-1,
&stmt,
0);
if (rc != SQLITE_OK)
{
std::ostringstream index_error;
index_error << "Sqlite Plugin: auto-index table creation failed: '"
<< sqlite3_errmsg(*(*dataset_)) << "' query was: "
<< spatial_index_insert_sql;
throw mapnik::datasource_exception(index_error.str());
}
bool first = true;
while (rs->is_valid() && rs->step_next())
{
int size;
const char* data = (const char*) rs->column_blob(0, size);
if (data)
{
// create a tmp feature to be able to parse geometry
// ideally we would not have to do this.
// see: http://trac.mapnik.org/ticket/745
mapnik::feature_ptr feature(mapnik::feature_factory::create(0));
mapnik::geometry_utils::from_wkb(feature->paths(), data, size, false, mapnik::wkbAuto);
mapnik::box2d<double> bbox = feature->envelope();
if (bbox.valid())
{
if (first)
{
first = false;
extent = bbox;
}
else
{
extent.expand_to_include(bbox);
}
const int type_oid = rs->column_type(1);
if (type_oid != SQLITE_INTEGER)
{
std::ostringstream type_error;
type_error << "Sqlite Plugin: invalid type for key field '"
<< key_field << "' when creating index '" << table
<< "' type was: " << type_oid << "";
throw mapnik::datasource_exception(type_error.str());
}
const sqlite_int64 pkid = rs->column_integer64(1);
if (sqlite3_bind_int64(stmt, 1, pkid) != SQLITE_OK)
{
throw mapnik::datasource_exception("invalid value for for key field while generating index");
}
if ((sqlite3_bind_double(stmt, 2, bbox.minx()) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 3, bbox.maxx()) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 4, bbox.miny()) != SQLITE_OK) ||
(sqlite3_bind_double(stmt, 5, bbox.maxy()) != SQLITE_OK))
{
throw mapnik::datasource_exception("invalid value for for extent while generating index");
}
const int res = sqlite3_step(stmt);
if (res != SQLITE_ROW && res != SQLITE_DONE)
{
std::ostringstream s;
s << "SQLite Plugin: inserting bbox into rtree index failed: "
<< "error code " << sqlite3_errcode(*(*dataset_)) << ": '"
<< sqlite3_errmsg(*(*dataset_)) << "' query was: "
<< spatial_index_insert_sql;
throw mapnik::datasource_exception(s.str());
}
sqlite3_reset(stmt);
}
else
{
std::ostringstream index_error;
index_error << "SQLite Plugin: encountered invalid bbox at '"
<< key_field << "' == " << rs->column_integer64(1);
throw mapnik::datasource_exception(index_error.str());
}
}
}
const int res = sqlite3_finalize(stmt);
if (res != SQLITE_OK)
{
throw mapnik::datasource_exception("auto-indexing failed: set use_spatial_index=false to disable auto-indexing and avoid this error");
}
dataset_->execute("COMMIT");
}
static bool detect_extent(boost::shared_ptr<sqlite_connection> dataset_,
bool has_spatial_index,
mapnik::box2d<double> & extent,
std::string const& index_table,
std::string const& metadata,
std::string const& geometry_field,
std::string const& geometry_table,
std::string const& key_field,
std::string const& table
)
{
if (has_spatial_index)
{
std::ostringstream s;
s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM "
<< index_table;
boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
if (rs->is_valid() && rs->step_next())
{
if (! rs->column_isnull(0))
{
try
{
double xmin = boost::lexical_cast<double>(rs->column_double(0));
double ymin = boost::lexical_cast<double>(rs->column_double(1));
double xmax = boost::lexical_cast<double>(rs->column_double(2));
double ymax = boost::lexical_cast<double>(rs->column_double(3));
extent.init(xmin, ymin, xmax, ymax);
return true;
}
catch (boost::bad_lexical_cast& ex)
{
std::ostringstream ss;
ss << "SQLite Plugin: warning: could not determine extent from query: "
<< "'" << s.str() << "' \n problem was: " << ex.what() << std::endl;
std::clog << ss.str();
}
}
}
}
else if (! metadata.empty())
{
std::ostringstream s;
s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata;
s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table << "')";
boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
if (rs->is_valid() && rs->step_next())
{
double xmin = rs->column_double (0);
double ymin = rs->column_double (1);
double xmax = rs->column_double (2);
double ymax = rs->column_double (3);
extent.init (xmin, ymin, xmax, ymax);
return true;
}
}
else if (! key_field.empty())
{
std::ostringstream s;
s << "SELECT " << geometry_field << "," << key_field
<< " FROM (" << table << ")";
boost::shared_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
sqlite_utils::query_extent(rs,extent);
return true;
}
return false;
}
};
#endif //SQLITE_TYPES_HPP
#endif // MAPNIK_SQLITE_UTILS_HPP

1
src/agg/process_text_symbolizer.cpp
View file

@ -35,6 +35,7 @@ void agg_renderer<T>::process(text_symbolizer const& sym,
{
// Use a boost::ptr_vector here instread of std::vector?
std::vector<geometry_type*> geometries_to_process;
unsigned num_geom = feature.num_geometries();
for (unsigned i=0; i<num_geom; ++i)