sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add optional 'geometry_table' and 'extent_from_subquery' parameter and 'scale_denominator' substitution ability to PostGIS driver while enhancing error reporting - closes #260,#426,#456, updates CHANGELOG with other recent PostGIS enhancements and fixes

Browse source
This commit is contained in:
Dane Springmeyer 2009-12-11 01:50:55 +00:00
parent 1b91db1db3
commit 4dd6259903
3 changed files with 222 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
CHANGELOG
View file

@ -12,29 +12,57 @@ For a complete change history, see the SVN log.
Mapnik 0.6.2 Release
Mapnik 0.7.0 Release
--------------------
- XML: added support for using CDATA with libxml2 parser (r1364)
- Gdal Plugin: Add support for Gdal overviews, enabling fast loading of > 1GB rasters (#54)
- XML: added missing serialization of PointSymbolizer 'opacity' and 'allow_overlap' attributes (r1358)
- PostGIS: Added an optional 'geometry_table' parameter. This is used by Mapnik to look up metadata in the
geometry_columns and calculate extents (when the 'geometry_field' and 'srid' parameters are not supplied).
If 'geometry_table' is not specified Mapnik will attempt to determine the name of the table to query based
on parsing the 'table' parameter, which may fail for complex queries with more than one 'from' keyword.
Using this parameter should allow for existing metadata and table indexes to be used while opening the door
to much more complicated subqueries being passed within the 'table' parameter without failing (#260, #426).
- PointDatasource: fixed problem with missing geometries (#402)
- PostGIS Plugin: Added optional 'geometry_field' and 'srid' parameters. If specified these will allow
Mapnik to skip several queries to try to determine these values dynamically, and can be helpful to avoid
possible query failures during metadata lookup with complex subqueries as discussed in #260 and #436, but
also solvable by specifying the 'geometry_table' parameter.
- PostGIS: Added an optional 'extent_from_subquery' parameter that when true (and the 'extent' parameter is
not provided and 'estimate_extent' is false), directly Mapnik to calculate the extent upon the exact table
or sql provided in the 'table' parameter. If a sub-select is used for the table parameter then this will
in many cases provide a faster and more accurate layer extent, but will have no effect if the 'table'
parameter is simply an existing table. This parameter is false by default.
- PostGIS Plugin: Added 'bbox' substitution ability in sql query string. This opens the door for various
complex queries that may aggregate geometries to be kept fast by allowing proper placement of the bbox
query to be used by indexes. (r1292) (#415)
- PostGIS Plugin: Added 'scale_denominator' substitution ability in sql query string (#415/#465)
- PostGIS Plugin: Added support for quoted table names to allow for tables with characters that postgres
requires quoting for like dashes (r1454) (#393)
- PostGIS: Add a 'persist_connection' option (default true), that when false will release
the idle psql connection after datasource goes out of scope (r1337) (#433,#434)
- PostGIS: Added support for BigInt (int8) postgres type (384)
- PostGIS Plugin: Throw and report errors if SQL execution fails (r1291) (#363, #242)
- PostGIS Plugin: Added missing support for BigInt(int8) postgres datatypes (r1250) (#384)
- XML: Added support for using CDATA with libxml2 parser (r1364)
- XML: Added missing serialization of PointSymbolizer 'opacity' and 'allow_overlap' attributes (r1358)
- PointDatasource: Fixed problem with missing geometries (#402)
- Filters: Add support for '!=' as an alias to '<>' for not-equals filters (avoids &lt;&gt;) (r1326) (#427)
- Gdal Plugin: Add support for Gdal overviews, enabling fast loading of > 1GB rasters (r1321) (#54)
- PostGIS Plugin: Add bbox substitution ability in sql query string (r1292) (#415)
- PostGIS Plugin: Throw and report errors if SQL execution fails (r1291) (#363)
- Python: Added 'mapnik.has_pycairo()' function to test for pycairo support (r1278) (#284)
- PostGIS Plugin: Added missing support for BigInt(int8) postgres datatypes (r1250) (#384)

257
plugins/input/postgis/postgis.cpp
View file

@ -37,6 +37,7 @@
#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/tokenizer.hpp>
#include <boost/format.hpp>
// stl
#include <string>
@ -51,6 +52,8 @@
#define WKB_ENCODING "XDR"
#endif
#define FMAX std::numeric_limits<double>::max()
DATASOURCE_PLUGIN(postgis_datasource)
const std::string postgis_datasource::GEOMETRY_COLUMNS="geometry_columns";
@ -67,37 +70,31 @@ using mapnik::PoolGuard;
using mapnik::attribute_descriptor;
postgis_datasource::postgis_datasource(parameters const& params)
: datasource (params),
table_(*params.get<std::string>("table","")),
geometry_field_(*params.get<std::string>("geometry_field","")),
: datasource(params),
table_(*params_.get<std::string>("table","")),
geometry_table_(*params_.get<std::string>("geometry_table","")),
geometry_field_(*params_.get<std::string>("geometry_field","")),
cursor_fetch_size_(*params_.get<int>("cursor_size",0)),
row_limit_(*params_.get<int>("row_limit",0)),
type_(datasource::Vector),
srid_(*params_.get<int>("srid",0)),
extent_initialized_(false),
desc_(*params.get<std::string>("type"),"utf-8"),
desc_(*params_.get<std::string>("type"),"utf-8"),
creator_(params.get<std::string>("host"),
params.get<std::string>("port"),
params.get<std::string>("dbname"),
params.get<std::string>("user"),
params.get<std::string>("password")),
bbox_token_("!bbox!"),
persist_connection_(*params_.get<mapnik::boolean>("persist_connection",true))
scale_denom_token_("!scale_denominator!"),
persist_connection_(*params_.get<mapnik::boolean>("persist_connection",true)),
extent_from_subquery_(*params_.get<mapnik::boolean>("extent_from_subquery",false))
//show_queries_(*params_.get<mapnik::boolean>("show_queries",false))
{
if (table_.empty()) throw mapnik::datasource_exception("PostGIS: missing <table> parameter");
#ifdef MAPNIK_DEBUG
if (persist_connection_)
{
clog << "PostGIS: persisting connection pool..." << endl;
}
else
{
clog << "PostGIS: not persisting connection..." << endl;
}
#endif
boost::optional<int> initial_size = params_.get<int>("inital_size",1);
boost::optional<int> max_size = params_.get<int>("max_size",10);
@ -150,26 +147,35 @@ postgis_datasource::postgis_datasource(parameters const& params)
shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
desc_.set_encoding(conn->client_encoding());
std::string table_name=table_from_sql(table_);
if(geometry_table_.empty())
{
geometry_table_ = table_from_sql(table_);
}
std::string schema_name="";
std::string::size_type idx=table_name.find_last_of('.');
std::string::size_type idx = geometry_table_.find_last_of('.');
if (idx!=std::string::npos)
{
schema_name=table_name.substr(0,idx);
table_name=table_name.substr(idx+1);
schema_name = geometry_table_.substr(0,idx);
geometry_table_ = geometry_table_.substr(idx+1);
}
else
{
table_name=table_name.substr(0);
geometry_table_ = geometry_table_.substr(0);
}
// If we do not know both the geometry_field and the srid
// then first attempt to fetch the geometry name from a geometry_columns entry.
// This will return no records if we are querying a bogus table returned
// from the simplistic table parsing in table_from_sql() or if
// the table parameter references a table, view, or subselect not
// registered in the geometry columns.
geometryColumn_ = geometry_field_;
if (!geometryColumn_.length() > 0 || srid_ == 0)
{
std::ostringstream s;
s << "SELECT f_geometry_column, srid FROM ";
s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << unquote(table_name) <<"'";
s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << unquote(geometry_table_) <<"'";
if (schema_name.length() > 0)
s << " AND f_table_schema='" << unquote(schema_name) << "'";
@ -177,24 +183,21 @@ postgis_datasource::postgis_datasource(parameters const& params)
if (geometry_field_.length() > 0)
s << " AND f_geometry_column='" << unquote(geometry_field_) << "'";
#ifdef MAPNIK_DEBUG
clog << s.str() << endl;
#endif
/*if (show_queries_)
{
clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}*/
shared_ptr<ResultSet> rs=conn->executeQuery(s.str());
if (rs->next())
{
geometryColumn_ = rs->getValue("f_geometry_column");
#ifdef MAPNIK_DEBUG
clog << "setting geometry field to=" << geometryColumn_ << "\n";
#endif
if (srid_ == 0)
{
try
{
srid_ = lexical_cast<int>(rs->getValue("srid"));
#ifdef MAPNIK_DEBUG
clog << "setting SRID to=" << srid_ << "\n";
#endif
}
catch (bad_lexical_cast &ex)
{
@ -204,29 +207,26 @@ postgis_datasource::postgis_datasource(parameters const& params)
}
rs->close();
if (geometryColumn_.length() == 0)
throw mapnik::datasource_exception( "PostGIS Driver Error: Geometry column not specified or found in " + GEOMETRY_COLUMNS + " table: '" + table_name + "'. Try setting the 'geometry_field' parameter or adding a proper " + GEOMETRY_COLUMNS + " record");
if (srid_ <= 0)
// If we still do not know the srid then we can try to fetch
// it from the 'table_' parameter, which should work even if it is
// a subselect as long as we know the geometry_field to query
if (geometryColumn_.length() && srid_ <= 0)
{
s.str("");
s << "SELECT SRID(\"" << geometryColumn_ << "\") AS srid FROM ";
if (schema_name.length() > 0)
s << schema_name << ".";
s << table_name << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;";
s << populate_tokens(table_) << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;";
/*if (show_queries_)
{
clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}*/
#ifdef MAPNIK_DEBUG
clog << s.str() << endl;
#endif
shared_ptr<ResultSet> rs=conn->executeQuery(s.str());
if (rs->next())
{
try
{
srid_ = lexical_cast<int>(rs->getValue("srid"));
#ifdef MAPNIK_DEBUG
clog << "setting SRID to=" << srid_ << endl;
#endif
}
catch (bad_lexical_cast &ex)
{
@ -240,18 +240,24 @@ postgis_datasource::postgis_datasource(parameters const& params)
if (srid_ == 0)
{
srid_ = -1;
clog << "SRID: warning, using srid=-1" << endl;
clog << "PostGIS: SRID warning, using srid=-1" << endl;
}
// At this point the geometry_field may still not be known
// but we'll catch that where more useful...
#ifdef MAPNIK_DEBUG
clog << "using srid=" << srid_ << endl;
clog << "using geometry_column=" << geometryColumn_ << endl;
clog << "PostGIS: using SRID=" << srid_ << endl;
clog << "PostGIS: using geometry_column=" << geometryColumn_ << endl;
#endif
// collect attribute desc
std::ostringstream s;
std::string table_with_bbox = populate_sql_bbox(table_,extent_);
s << "select * from " << table_with_bbox << " limit 0";
s << "select * from " << populate_tokens(table_) << " limit 0";
/*if (show_queries_)
{
clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}*/
shared_ptr<ResultSet> rs=conn->executeQuery(s.str());
int count = rs->getNumFields();
@ -277,9 +283,15 @@ postgis_datasource::postgis_datasource(parameters const& params)
break;
default: // should not get here
#ifdef MAPNIK_DEBUG
std::ostringstream s_oid;
s_oid << "select oid, typname from pg_type where oid = " << type_oid;
shared_ptr<ResultSet> rs_oid=conn->executeQuery(s_oid.str());
s.str("");
s << "select oid, typname from pg_type where oid = " << type_oid;
/*if (show_queries_)
{
clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}*/
shared_ptr<ResultSet> rs_oid = conn->executeQuery(s.str());
if (rs_oid->next())
{
clog << "PostGIS: unknown type = " << rs_oid->getValue("typname") << " (oid:" << rs_oid->getValue("oid") << ")\n";
@ -315,31 +327,64 @@ layer_descriptor postgis_datasource::get_descriptor() const
return desc_;
}
std::string postgis_datasource::populate_sql_bbox(const std::string& sql, Envelope<double> const& box) const
std::string postgis_datasource::sql_bbox(Envelope<double> const& env) const
{
std::string sql_with_bbox = sql;
std::ostringstream b;
if (srid_ > 0)
b << "SetSRID(";
b << "'BOX3D(";
b << std::setprecision(16);
b << box.minx() << " " << box.miny() << ",";
b << box.maxx() << " " << box.maxy() << ")'::box3d";
b << env.minx() << " " << env.miny() << ",";
b << env.maxx() << " " << env.maxy() << ")'::box3d";
if (srid_ > 0)
b << ", " << srid_ << ")";
return b.str();
}
std::string postgis_datasource::populate_tokens(const std::string& sql) const
{
std::string populated_sql = sql;
if ( boost::algorithm::icontains(sql,bbox_token_) )
{
boost::algorithm::replace_all(sql_with_bbox,bbox_token_,b.str());
return sql_with_bbox;
Envelope<double> max_env(-1 * FMAX,-1 * FMAX,FMAX,FMAX);
std::string max_box = sql_bbox(max_env);
boost::algorithm::replace_all(populated_sql,bbox_token_,max_box);
}
if ( boost::algorithm::icontains(sql,scale_denom_token_) )
{
std::string max_denom = lexical_cast<std::string>(FMAX);
boost::algorithm::replace_all(populated_sql,scale_denom_token_,max_denom);
}
return populated_sql;
}
std::string postgis_datasource::populate_tokens(const std::string& sql, double const& scale_denom, Envelope<double> const& env) const
{
std::string populated_sql = sql;
std::string box = sql_bbox(env);
if ( boost::algorithm::icontains(populated_sql,scale_denom_token_) )
{
std::string max_denom = lexical_cast<std::string>(scale_denom);
boost::algorithm::replace_all(populated_sql,scale_denom_token_,max_denom);
}
if ( boost::algorithm::icontains(populated_sql,bbox_token_) )
{
boost::algorithm::replace_all(populated_sql,bbox_token_,box);
return populated_sql;
}
else
{
std::ostringstream s;
s << " WHERE \"" << geometryColumn_ << "\" && " << b.str();
return sql_with_bbox + s.str();
s << " WHERE \"" << geometryColumn_ << "\" && " << box;
return populated_sql + s.str();
}
}
std::string postgis_datasource::unquote(const std::string& sql)
{
std::string table_name = boost::algorithm::to_lower_copy(sql);
@ -347,6 +392,8 @@ std::string postgis_datasource::unquote(const std::string& sql)
return table_name;
}
// TODO - make smarter and potentially move to reusable utilities
// available to other SQL-based plugins
std::string postgis_datasource::table_from_sql(const std::string& sql)
{
std::string table_name = boost::algorithm::to_lower_copy(sql);
@ -379,9 +426,11 @@ boost::shared_ptr<IResultSet> postgis_datasource::get_resultset(boost::shared_pt
csql << "DECLARE " << cursor_name << " BINARY INSENSITIVE NO SCROLL CURSOR WITH HOLD FOR " << sql << " FOR READ ONLY";
#ifdef MAPNIK_DEBUG
clog << csql.str() << "\n";
#endif
/*if (show_queries_)
{
clog << boost::format("PostGIS: sending query: %s\n") % csql.str();
}*/
if (!conn->execute(csql.str())) {
throw mapnik::datasource_exception( "PSQL Error: Creating cursor for data select." );
}
@ -389,9 +438,12 @@ boost::shared_ptr<IResultSet> postgis_datasource::get_resultset(boost::shared_pt
} else {
// no cursor
#ifdef MAPNIK_DEBUG
clog << sql << "\n";
#endif
/*if (show_queries_)
{
clog << boost::format("PostGIS: sending query: %s\n") % sql;
}*/
return conn->executeQuery(sql,1);
}
}
@ -402,7 +454,8 @@ featureset_ptr postgis_datasource::features(const query& q) const
mapnik::wall_clock_progress_timer timer(clog, "end feature query: ");
#endif
Envelope<double> const& box=q.get_bbox();
Envelope<double> const& box = q.get_bbox();
double scale_denom = q.scale_denominator();
ConnectionManager *mgr=ConnectionManager::instance();
shared_ptr<Pool<Connection,ConnectionCreator> > pool=mgr->getPool(creator_.id());
if (pool)
@ -412,8 +465,17 @@ featureset_ptr postgis_datasource::features(const query& q) const
{
PoolGuard<shared_ptr<Connection>,shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
if (!geometryColumn_.length() > 0)
{
std::ostringstream s_error;
s_error << "PostGIS: geometry name lookup failed for table '" << geometry_table_
<< "'. Please manually provide the 'geometry_field' parameter or add an entry "
<< "in the geometry_columns for '" << geometry_table_ << "'.";
throw mapnik::datasource_exception(s_error.str());
}
std::ostringstream s;
s << "SELECT AsBinary(\""<<geometryColumn_<<"\",'"<< WKB_ENCODING << "') AS geom";
s << "SELECT AsBinary(\"" << geometryColumn_ << "\",'" << WKB_ENCODING << "') AS geom";
std::set<std::string> const& props=q.property_names();
std::set<std::string>::const_iterator pos=props.begin();
std::set<std::string>::const_iterator end=props.end();
@ -423,7 +485,7 @@ featureset_ptr postgis_datasource::features(const query& q) const
++pos;
}
std::string table_with_bbox = populate_sql_bbox(table_,box);
std::string table_with_bbox = populate_tokens(table_,scale_denom,box);
s << " from " << table_with_bbox;
@ -449,21 +511,30 @@ featureset_ptr postgis_datasource::features_at_point(coord2d const& pt) const
{
PoolGuard<shared_ptr<Connection>,shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
std::ostringstream s;
s << "SELECT AsBinary(\"" << geometryColumn_ << "\",'"<< WKB_ENCODING << "') AS geom";
if (!geometryColumn_.length() > 0)
{
std::ostringstream s_error;
s_error << "PostGIS: geometry name lookup failed for table '" << geometry_table_
<< "'. Please manually provide the 'geometry_field' parameter or add an entry "
<< "in the geometry_columns for '" << geometry_table_ << "'.";
throw mapnik::datasource_exception(s_error.str());
}
s << "SELECT AsBinary(\"" << geometryColumn_ << "\",'" << WKB_ENCODING << "') AS geom";
std::vector<attribute_descriptor>::const_iterator itr = desc_.get_descriptors().begin();
std::vector<attribute_descriptor>::const_iterator end = desc_.get_descriptors().end();
unsigned size=0;
while (itr != end)
{
s <<",\""<< itr->get_name() << "\"";
s << ",\"" << itr->get_name() << "\"";
++itr;
++size;
}
Envelope<double> box(pt.x,pt.y,pt.x,pt.y);
std::string table_with_bbox = populate_sql_bbox(table_,box);
std::string table_with_bbox = populate_tokens(table_,FMAX,box);
s << " from " << table_with_bbox;
@ -491,23 +562,45 @@ Envelope<double> postgis_datasource::envelope() const
{
PoolGuard<shared_ptr<Connection>,shared_ptr<Pool<Connection,ConnectionCreator> > > guard(conn,pool);
std::ostringstream s;
std::string table_name = table_from_sql(table_);
boost::optional<std::string> estimate_extent = params_.get<std::string>("estimate_extent");
if (!geometryColumn_.length() > 0)
{
std::ostringstream s_error;
s_error << "PostGIS: unable to query the layer extent of table '"
<< geometry_table_ << "' because we cannot determine the geometry field name."
<< "\nPlease provide either 1) an 'extent' parameter to skip this query, "
<< "2) a 'geometry_field' and/or 'geometry_table' parameter, or 3) add a "
<< "record to the 'geometry_columns' for your table.";
throw mapnik::datasource_exception(s_error.str());
}
// TODO - do we need to respect schema here?
if (estimate_extent && *estimate_extent == "true")
{
s << "select xmin(ext),ymin(ext),xmax(ext),ymax(ext)"
<< " from (select estimated_extent('"
<< table_name <<"','"
<< geometry_table_ << "','"
<< geometryColumn_ << "') as ext) as tmp";
}
else
{
s << "select xmin(ext),ymin(ext),xmax(ext),ymax(ext)"
<< " from (select extent(" <<geometryColumn_<< ") as ext from "
<< table_name << ") as tmp";
<< " from (select extent(" <<geometryColumn_<< ") as ext from ";
if (extent_from_subquery_)
// if a subselect limits records then calculating the extent upon the
// subquery will be faster and the bounds will be more accurate
s << populate_tokens(table_) << ") as tmp";
else
// but if the subquery does not limit records then querying the
// actual table will be faster as indexes can be used
s << geometry_table_ << ") as tmp";
}
/*if (show_queries_)
{
clog << boost::format("PostGIS: sending query: %s\n") % s.str();
}*/
shared_ptr<ResultSet> rs=conn->executeQuery(s.str());
if (rs->next())
{
@ -522,7 +615,7 @@ Envelope<double> postgis_datasource::envelope() const
}
catch (bad_lexical_cast &ex)
{
clog << ex.what() << endl;
clog << boost::format("PostGIS: warning: could not determine extent from query: %s\nError was: '%s'\n") % s.str() % ex.what();
}
}
rs->close();

10
plugins/input/postgis/postgis.hpp
View file

@ -57,10 +57,11 @@ class postgis_datasource : public datasource
const std::string username_;
const std::string password_;
const std::string table_;
mutable std::string geometry_table_;
const std::string geometry_field_;
const int cursor_fetch_size_;
const int row_limit_;
std::string geometryColumn_;
mutable std::string geometryColumn_;
int type_;
int srid_;
mutable bool extent_initialized_;
@ -70,7 +71,10 @@ class postgis_datasource : public datasource
bool multiple_geometries_;
static const std::string name_;
const std::string bbox_token_;
const std::string scale_denom_token_;
bool persist_connection_;
bool extent_from_subquery_;
//bool show_queries_;
public:
static std::string name();
int type() const;
@ -81,7 +85,9 @@ class postgis_datasource : public datasource
postgis_datasource(const parameters &params);
~postgis_datasource();
private:
std::string populate_sql_bbox(const std::string& sql, Envelope<double> const& box) const;
std::string sql_bbox(Envelope<double> const& env) const;
std::string populate_tokens(const std::string& sql, double const& scale_denom, Envelope<double> const& env) const;
std::string populate_tokens(const std::string& sql) const;
static std::string unquote(const std::string& sql);
static std::string table_from_sql(const std::string& sql);
boost::shared_ptr<IResultSet> get_resultset(boost::shared_ptr<Connection> const &conn, const std::string &sql) const;