From 4dd625990339320e40d2f9b48c646f40561790f2 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Fri, 11 Dec 2009 01:50:55 +0000 Subject: [PATCH] add optional 'geometry_table' and 'extent_from_subquery' parameter and 'scale_denominator' substitution ability to PostGIS driver while enhancing error reporting - closes #260,#426,#456, updates CHANGELOG with other recent PostGIS enhancements and fixes --- CHANGELOG | 50 ++++-- plugins/input/postgis/postgis.cpp | 257 ++++++++++++++++++++---------- plugins/input/postgis/postgis.hpp | 10 +- 3 files changed, 222 insertions(+), 95 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 3ff22644d..a60dc34af 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -12,29 +12,57 @@ For a complete change history, see the SVN log. -Mapnik 0.6.2 Release +Mapnik 0.7.0 Release -------------------- -- XML: added support for using CDATA with libxml2 parser (r1364) +- Gdal Plugin: Add support for Gdal overviews, enabling fast loading of > 1GB rasters (#54) -- XML: added missing serialization of PointSymbolizer 'opacity' and 'allow_overlap' attributes (r1358) +- PostGIS: Added an optional 'geometry_table' parameter. This is used by Mapnik to look up metadata in the + geometry_columns and calculate extents (when the 'geometry_field' and 'srid' parameters are not supplied). + If 'geometry_table' is not specified Mapnik will attempt to determine the name of the table to query based + on parsing the 'table' parameter, which may fail for complex queries with more than one 'from' keyword. + Using this parameter should allow for existing metadata and table indexes to be used while opening the door + to much more complicated subqueries being passed within the 'table' parameter without failing (#260, #426). -- PointDatasource: fixed problem with missing geometries (#402) +- PostGIS Plugin: Added optional 'geometry_field' and 'srid' parameters. If specified these will allow + Mapnik to skip several queries to try to determine these values dynamically, and can be helpful to avoid + possible query failures during metadata lookup with complex subqueries as discussed in #260 and #436, but + also solvable by specifying the 'geometry_table' parameter. + +- PostGIS: Added an optional 'extent_from_subquery' parameter that when true (and the 'extent' parameter is + not provided and 'estimate_extent' is false), directly Mapnik to calculate the extent upon the exact table + or sql provided in the 'table' parameter. If a sub-select is used for the table parameter then this will + in many cases provide a faster and more accurate layer extent, but will have no effect if the 'table' + parameter is simply an existing table. This parameter is false by default. + +- PostGIS Plugin: Added 'bbox' substitution ability in sql query string. This opens the door for various + complex queries that may aggregate geometries to be kept fast by allowing proper placement of the bbox + query to be used by indexes. (r1292) (#415) + +- PostGIS Plugin: Added 'scale_denominator' substitution ability in sql query string (#415/#465) + +- PostGIS Plugin: Added support for quoted table names to allow for tables with characters that postgres + requires quoting for like dashes (r1454) (#393) - PostGIS: Add a 'persist_connection' option (default true), that when false will release the idle psql connection after datasource goes out of scope (r1337) (#433,#434) +- PostGIS: Added support for BigInt (int8) postgres type (384) + +- PostGIS Plugin: Throw and report errors if SQL execution fails (r1291) (#363, #242) + +- PostGIS Plugin: Added missing support for BigInt(int8) postgres datatypes (r1250) (#384) + +- XML: Added support for using CDATA with libxml2 parser (r1364) + +- XML: Added missing serialization of PointSymbolizer 'opacity' and 'allow_overlap' attributes (r1358) + +- PointDatasource: Fixed problem with missing geometries (#402) + - Filters: Add support for '!=' as an alias to '<>' for not-equals filters (avoids <>) (r1326) (#427) -- Gdal Plugin: Add support for Gdal overviews, enabling fast loading of > 1GB rasters (r1321) (#54) - -- PostGIS Plugin: Add bbox substitution ability in sql query string (r1292) (#415) - -- PostGIS Plugin: Throw and report errors if SQL execution fails (r1291) (#363) - - Python: Added 'mapnik.has_pycairo()' function to test for pycairo support (r1278) (#284) -- PostGIS Plugin: Added missing support for BigInt(int8) postgres datatypes (r1250) (#384) diff --git a/plugins/input/postgis/postgis.cpp b/plugins/input/postgis/postgis.cpp index fc2484250..2218672ac 100644 --- a/plugins/input/postgis/postgis.cpp +++ b/plugins/input/postgis/postgis.cpp @@ -37,6 +37,7 @@ #include #include #include +#include // stl #include @@ -51,6 +52,8 @@ #define WKB_ENCODING "XDR" #endif +#define FMAX std::numeric_limits::max() + DATASOURCE_PLUGIN(postgis_datasource) const std::string postgis_datasource::GEOMETRY_COLUMNS="geometry_columns"; @@ -67,37 +70,31 @@ using mapnik::PoolGuard; using mapnik::attribute_descriptor; postgis_datasource::postgis_datasource(parameters const& params) - : datasource (params), - table_(*params.get("table","")), - geometry_field_(*params.get("geometry_field","")), + : datasource(params), + table_(*params_.get("table","")), + geometry_table_(*params_.get("geometry_table","")), + geometry_field_(*params_.get("geometry_field","")), cursor_fetch_size_(*params_.get("cursor_size",0)), row_limit_(*params_.get("row_limit",0)), type_(datasource::Vector), srid_(*params_.get("srid",0)), extent_initialized_(false), - desc_(*params.get("type"),"utf-8"), + desc_(*params_.get("type"),"utf-8"), creator_(params.get("host"), params.get("port"), params.get("dbname"), params.get("user"), params.get("password")), bbox_token_("!bbox!"), - persist_connection_(*params_.get("persist_connection",true)) + scale_denom_token_("!scale_denominator!"), + persist_connection_(*params_.get("persist_connection",true)), + extent_from_subquery_(*params_.get("extent_from_subquery",false)) + //show_queries_(*params_.get("show_queries",false)) + { if (table_.empty()) throw mapnik::datasource_exception("PostGIS: missing parameter"); -#ifdef MAPNIK_DEBUG - if (persist_connection_) - { - clog << "PostGIS: persisting connection pool..." << endl; - } - else - { - clog << "PostGIS: not persisting connection..." << endl; - } -#endif - boost::optional initial_size = params_.get("inital_size",1); boost::optional max_size = params_.get("max_size",10); @@ -150,26 +147,35 @@ postgis_datasource::postgis_datasource(parameters const& params) shared_ptr > > guard(conn,pool); desc_.set_encoding(conn->client_encoding()); - - std::string table_name=table_from_sql(table_); + + if(geometry_table_.empty()) + { + geometry_table_ = table_from_sql(table_); + } std::string schema_name=""; - std::string::size_type idx=table_name.find_last_of('.'); + std::string::size_type idx = geometry_table_.find_last_of('.'); if (idx!=std::string::npos) { - schema_name=table_name.substr(0,idx); - table_name=table_name.substr(idx+1); + schema_name = geometry_table_.substr(0,idx); + geometry_table_ = geometry_table_.substr(idx+1); } else { - table_name=table_name.substr(0); + geometry_table_ = geometry_table_.substr(0); } + // If we do not know both the geometry_field and the srid + // then first attempt to fetch the geometry name from a geometry_columns entry. + // This will return no records if we are querying a bogus table returned + // from the simplistic table parsing in table_from_sql() or if + // the table parameter references a table, view, or subselect not + // registered in the geometry columns. geometryColumn_ = geometry_field_; if (!geometryColumn_.length() > 0 || srid_ == 0) { std::ostringstream s; s << "SELECT f_geometry_column, srid FROM "; - s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << unquote(table_name) <<"'"; + s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << unquote(geometry_table_) <<"'"; if (schema_name.length() > 0) s << " AND f_table_schema='" << unquote(schema_name) << "'"; @@ -177,24 +183,21 @@ postgis_datasource::postgis_datasource(parameters const& params) if (geometry_field_.length() > 0) s << " AND f_geometry_column='" << unquote(geometry_field_) << "'"; -#ifdef MAPNIK_DEBUG - clog << s.str() << endl; -#endif + /*if (show_queries_) + { + clog << boost::format("PostGIS: sending query: %s\n") % s.str(); + }*/ + shared_ptr rs=conn->executeQuery(s.str()); if (rs->next()) { geometryColumn_ = rs->getValue("f_geometry_column"); -#ifdef MAPNIK_DEBUG - clog << "setting geometry field to=" << geometryColumn_ << "\n"; -#endif + if (srid_ == 0) { try { srid_ = lexical_cast(rs->getValue("srid")); -#ifdef MAPNIK_DEBUG - clog << "setting SRID to=" << srid_ << "\n"; -#endif } catch (bad_lexical_cast &ex) { @@ -204,29 +207,26 @@ postgis_datasource::postgis_datasource(parameters const& params) } rs->close(); - if (geometryColumn_.length() == 0) - throw mapnik::datasource_exception( "PostGIS Driver Error: Geometry column not specified or found in " + GEOMETRY_COLUMNS + " table: '" + table_name + "'. Try setting the 'geometry_field' parameter or adding a proper " + GEOMETRY_COLUMNS + " record"); - - if (srid_ <= 0) + // If we still do not know the srid then we can try to fetch + // it from the 'table_' parameter, which should work even if it is + // a subselect as long as we know the geometry_field to query + if (geometryColumn_.length() && srid_ <= 0) { s.str(""); s << "SELECT SRID(\"" << geometryColumn_ << "\") AS srid FROM "; - if (schema_name.length() > 0) - s << schema_name << "."; - s << table_name << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;"; + s << populate_tokens(table_) << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;"; + + /*if (show_queries_) + { + clog << boost::format("PostGIS: sending query: %s\n") % s.str(); + }*/ -#ifdef MAPNIK_DEBUG - clog << s.str() << endl; -#endif shared_ptr rs=conn->executeQuery(s.str()); if (rs->next()) { try { srid_ = lexical_cast(rs->getValue("srid")); -#ifdef MAPNIK_DEBUG - clog << "setting SRID to=" << srid_ << endl; -#endif } catch (bad_lexical_cast &ex) { @@ -240,18 +240,24 @@ postgis_datasource::postgis_datasource(parameters const& params) if (srid_ == 0) { srid_ = -1; - clog << "SRID: warning, using srid=-1" << endl; + clog << "PostGIS: SRID warning, using srid=-1" << endl; } - + + // At this point the geometry_field may still not be known + // but we'll catch that where more useful... #ifdef MAPNIK_DEBUG - clog << "using srid=" << srid_ << endl; - clog << "using geometry_column=" << geometryColumn_ << endl; + clog << "PostGIS: using SRID=" << srid_ << endl; + clog << "PostGIS: using geometry_column=" << geometryColumn_ << endl; #endif // collect attribute desc std::ostringstream s; - std::string table_with_bbox = populate_sql_bbox(table_,extent_); - s << "select * from " << table_with_bbox << " limit 0"; + s << "select * from " << populate_tokens(table_) << " limit 0"; + + /*if (show_queries_) + { + clog << boost::format("PostGIS: sending query: %s\n") % s.str(); + }*/ shared_ptr rs=conn->executeQuery(s.str()); int count = rs->getNumFields(); @@ -277,9 +283,15 @@ postgis_datasource::postgis_datasource(parameters const& params) break; default: // should not get here #ifdef MAPNIK_DEBUG - std::ostringstream s_oid; - s_oid << "select oid, typname from pg_type where oid = " << type_oid; - shared_ptr rs_oid=conn->executeQuery(s_oid.str()); + s.str(""); + s << "select oid, typname from pg_type where oid = " << type_oid; + + /*if (show_queries_) + { + clog << boost::format("PostGIS: sending query: %s\n") % s.str(); + }*/ + + shared_ptr rs_oid = conn->executeQuery(s.str()); if (rs_oid->next()) { clog << "PostGIS: unknown type = " << rs_oid->getValue("typname") << " (oid:" << rs_oid->getValue("oid") << ")\n"; @@ -315,31 +327,64 @@ layer_descriptor postgis_datasource::get_descriptor() const return desc_; } -std::string postgis_datasource::populate_sql_bbox(const std::string& sql, Envelope const& box) const + +std::string postgis_datasource::sql_bbox(Envelope const& env) const { - std::string sql_with_bbox = sql; std::ostringstream b; if (srid_ > 0) b << "SetSRID("; b << "'BOX3D("; b << std::setprecision(16); - b << box.minx() << " " << box.miny() << ","; - b << box.maxx() << " " << box.maxy() << ")'::box3d"; + b << env.minx() << " " << env.miny() << ","; + b << env.maxx() << " " << env.maxy() << ")'::box3d"; if (srid_ > 0) b << ", " << srid_ << ")"; + return b.str(); +} + +std::string postgis_datasource::populate_tokens(const std::string& sql) const +{ + std::string populated_sql = sql; + if ( boost::algorithm::icontains(sql,bbox_token_) ) { - boost::algorithm::replace_all(sql_with_bbox,bbox_token_,b.str()); - return sql_with_bbox; + Envelope max_env(-1 * FMAX,-1 * FMAX,FMAX,FMAX); + std::string max_box = sql_bbox(max_env); + boost::algorithm::replace_all(populated_sql,bbox_token_,max_box); + } + if ( boost::algorithm::icontains(sql,scale_denom_token_) ) + { + std::string max_denom = lexical_cast(FMAX); + boost::algorithm::replace_all(populated_sql,scale_denom_token_,max_denom); + } + return populated_sql; +} + +std::string postgis_datasource::populate_tokens(const std::string& sql, double const& scale_denom, Envelope const& env) const +{ + std::string populated_sql = sql; + std::string box = sql_bbox(env); + + if ( boost::algorithm::icontains(populated_sql,scale_denom_token_) ) + { + std::string max_denom = lexical_cast(scale_denom); + boost::algorithm::replace_all(populated_sql,scale_denom_token_,max_denom); + } + + if ( boost::algorithm::icontains(populated_sql,bbox_token_) ) + { + boost::algorithm::replace_all(populated_sql,bbox_token_,box); + return populated_sql; } else { std::ostringstream s; - s << " WHERE \"" << geometryColumn_ << "\" && " << b.str(); - return sql_with_bbox + s.str(); + s << " WHERE \"" << geometryColumn_ << "\" && " << box; + return populated_sql + s.str(); } } + std::string postgis_datasource::unquote(const std::string& sql) { std::string table_name = boost::algorithm::to_lower_copy(sql); @@ -347,6 +392,8 @@ std::string postgis_datasource::unquote(const std::string& sql) return table_name; } +// TODO - make smarter and potentially move to reusable utilities +// available to other SQL-based plugins std::string postgis_datasource::table_from_sql(const std::string& sql) { std::string table_name = boost::algorithm::to_lower_copy(sql); @@ -379,9 +426,11 @@ boost::shared_ptr postgis_datasource::get_resultset(boost::shared_pt csql << "DECLARE " << cursor_name << " BINARY INSENSITIVE NO SCROLL CURSOR WITH HOLD FOR " << sql << " FOR READ ONLY"; -#ifdef MAPNIK_DEBUG - clog << csql.str() << "\n"; -#endif + /*if (show_queries_) + { + clog << boost::format("PostGIS: sending query: %s\n") % csql.str(); + }*/ + if (!conn->execute(csql.str())) { throw mapnik::datasource_exception( "PSQL Error: Creating cursor for data select." ); } @@ -389,9 +438,12 @@ boost::shared_ptr postgis_datasource::get_resultset(boost::shared_pt } else { // no cursor -#ifdef MAPNIK_DEBUG - clog << sql << "\n"; -#endif + + /*if (show_queries_) + { + clog << boost::format("PostGIS: sending query: %s\n") % sql; + }*/ + return conn->executeQuery(sql,1); } } @@ -402,7 +454,8 @@ featureset_ptr postgis_datasource::features(const query& q) const mapnik::wall_clock_progress_timer timer(clog, "end feature query: "); #endif - Envelope const& box=q.get_bbox(); + Envelope const& box = q.get_bbox(); + double scale_denom = q.scale_denominator(); ConnectionManager *mgr=ConnectionManager::instance(); shared_ptr > pool=mgr->getPool(creator_.id()); if (pool) @@ -412,8 +465,17 @@ featureset_ptr postgis_datasource::features(const query& q) const { PoolGuard,shared_ptr > > guard(conn,pool); + if (!geometryColumn_.length() > 0) + { + std::ostringstream s_error; + s_error << "PostGIS: geometry name lookup failed for table '" << geometry_table_ + << "'. Please manually provide the 'geometry_field' parameter or add an entry " + << "in the geometry_columns for '" << geometry_table_ << "'."; + throw mapnik::datasource_exception(s_error.str()); + } + std::ostringstream s; - s << "SELECT AsBinary(\""< const& props=q.property_names(); std::set::const_iterator pos=props.begin(); std::set::const_iterator end=props.end(); @@ -423,7 +485,7 @@ featureset_ptr postgis_datasource::features(const query& q) const ++pos; } - std::string table_with_bbox = populate_sql_bbox(table_,box); + std::string table_with_bbox = populate_tokens(table_,scale_denom,box); s << " from " << table_with_bbox; @@ -449,21 +511,30 @@ featureset_ptr postgis_datasource::features_at_point(coord2d const& pt) const { PoolGuard,shared_ptr > > guard(conn,pool); std::ostringstream s; - - s << "SELECT AsBinary(\"" << geometryColumn_ << "\",'"<< WKB_ENCODING << "') AS geom"; + + if (!geometryColumn_.length() > 0) + { + std::ostringstream s_error; + s_error << "PostGIS: geometry name lookup failed for table '" << geometry_table_ + << "'. Please manually provide the 'geometry_field' parameter or add an entry " + << "in the geometry_columns for '" << geometry_table_ << "'."; + throw mapnik::datasource_exception(s_error.str()); + } + + s << "SELECT AsBinary(\"" << geometryColumn_ << "\",'" << WKB_ENCODING << "') AS geom"; std::vector::const_iterator itr = desc_.get_descriptors().begin(); std::vector::const_iterator end = desc_.get_descriptors().end(); unsigned size=0; while (itr != end) { - s <<",\""<< itr->get_name() << "\""; + s << ",\"" << itr->get_name() << "\""; ++itr; ++size; } Envelope box(pt.x,pt.y,pt.x,pt.y); - std::string table_with_bbox = populate_sql_bbox(table_,box); + std::string table_with_bbox = populate_tokens(table_,FMAX,box); s << " from " << table_with_bbox; @@ -491,23 +562,45 @@ Envelope postgis_datasource::envelope() const { PoolGuard,shared_ptr > > guard(conn,pool); std::ostringstream s; - std::string table_name = table_from_sql(table_); boost::optional estimate_extent = params_.get("estimate_extent"); - + + if (!geometryColumn_.length() > 0) + { + std::ostringstream s_error; + s_error << "PostGIS: unable to query the layer extent of table '" + << geometry_table_ << "' because we cannot determine the geometry field name." + << "\nPlease provide either 1) an 'extent' parameter to skip this query, " + << "2) a 'geometry_field' and/or 'geometry_table' parameter, or 3) add a " + << "record to the 'geometry_columns' for your table."; + throw mapnik::datasource_exception(s_error.str()); + } + // TODO - do we need to respect schema here? if (estimate_extent && *estimate_extent == "true") { s << "select xmin(ext),ymin(ext),xmax(ext),ymax(ext)" << " from (select estimated_extent('" - << table_name <<"','" + << geometry_table_ << "','" << geometryColumn_ << "') as ext) as tmp"; } else { s << "select xmin(ext),ymin(ext),xmax(ext),ymax(ext)" - << " from (select extent(" < rs=conn->executeQuery(s.str()); if (rs->next()) { @@ -522,7 +615,7 @@ Envelope postgis_datasource::envelope() const } catch (bad_lexical_cast &ex) { - clog << ex.what() << endl; + clog << boost::format("PostGIS: warning: could not determine extent from query: %s\nError was: '%s'\n") % s.str() % ex.what(); } } rs->close(); diff --git a/plugins/input/postgis/postgis.hpp b/plugins/input/postgis/postgis.hpp index b4ee4513d..c7300745a 100644 --- a/plugins/input/postgis/postgis.hpp +++ b/plugins/input/postgis/postgis.hpp @@ -57,10 +57,11 @@ class postgis_datasource : public datasource const std::string username_; const std::string password_; const std::string table_; + mutable std::string geometry_table_; const std::string geometry_field_; const int cursor_fetch_size_; const int row_limit_; - std::string geometryColumn_; + mutable std::string geometryColumn_; int type_; int srid_; mutable bool extent_initialized_; @@ -70,7 +71,10 @@ class postgis_datasource : public datasource bool multiple_geometries_; static const std::string name_; const std::string bbox_token_; + const std::string scale_denom_token_; bool persist_connection_; + bool extent_from_subquery_; + //bool show_queries_; public: static std::string name(); int type() const; @@ -81,7 +85,9 @@ class postgis_datasource : public datasource postgis_datasource(const parameters ¶ms); ~postgis_datasource(); private: - std::string populate_sql_bbox(const std::string& sql, Envelope const& box) const; + std::string sql_bbox(Envelope const& env) const; + std::string populate_tokens(const std::string& sql, double const& scale_denom, Envelope const& env) const; + std::string populate_tokens(const std::string& sql) const; static std::string unquote(const std::string& sql); static std::string table_from_sql(const std::string& sql); boost::shared_ptr get_resultset(boost::shared_ptr const &conn, const std::string &sql) const;