diff --git a/SConstruct b/SConstruct index 6e594967b..b70739f21 100644 --- a/SConstruct +++ b/SConstruct @@ -103,6 +103,7 @@ pretty_dep_names = { PLUGINS = { # plugins with external dependencies # configured by calling project, hence 'path':None 'postgis': {'default':True,'path':None,'inc':'libpq-fe.h','lib':'pq','lang':'C'}, + 'pgraster': {'default':True,'path':None,'inc':'libpq-fe.h','lib':'pq','lang':'C'}, 'gdal': {'default':True,'path':None,'inc':'gdal_priv.h','lib':'gdal','lang':'C++'}, 'ogr': {'default':True,'path':None,'inc':'ogrsf_frmts.h','lib':'gdal','lang':'C++'}, # configured with custom paths, hence 'path': PREFIX/INCLUDES/LIBS @@ -1429,7 +1430,7 @@ if not preconfigured: env['LIBS'].remove(libname) else: details['lib'] = libname - elif plugin == 'postgis': + elif plugin == 'postgis' or plugin == 'pgraster': conf.parse_pg_config('PG_CONFIG') elif plugin == 'ogr': if conf.ogr_enabled(): diff --git a/bindings/python/mapnik/__init__.py b/bindings/python/mapnik/__init__.py index 3f94a62bb..c03ea46d6 100644 --- a/bindings/python/mapnik/__init__.py +++ b/bindings/python/mapnik/__init__.py @@ -440,6 +440,51 @@ def PostGIS(**keywords): keywords['type'] = 'postgis' return CreateDatasource(keywords) +def PgRaster(**keywords): + """Create a PgRaster Datasource. + + Required keyword arguments: + dbname -- database name to connect to + table -- table name or subselect query + + *Note: if using subselects for the 'table' value consider also + passing the 'raster_field' and 'srid' and 'extent_from_subquery' + options and/or specifying the 'raster_table' option. + + Optional db connection keyword arguments: + user -- database user to connect as (default: see postgres docs) + password -- password for database user (default: see postgres docs) + host -- portgres hostname (default: see postgres docs) + port -- postgres port (default: see postgres docs) + initial_size -- integer size of connection pool (default: 1) + max_size -- integer max of connection pool (default: 10) + persist_connection -- keep connection open (default: True) + + Optional table-level keyword arguments: + extent -- manually specified data extent (comma delimited string, default: None) + estimate_extent -- boolean, direct PostGIS to use the faster, less accurate `estimate_extent` over `extent` (default: False) + extent_from_subquery -- boolean, direct Mapnik to query Postgis for the extent of the raw 'table' value (default: uses 'geometry_table') + raster_table -- specify geometry table to use to look up metadata (default: automatically parsed from 'table' value) + raster_field -- specify geometry field to use (default: first entry in raster_columns) + srid -- specify srid to use (default: auto-detected from geometry_field) + row_limit -- integer limit of rows to return (default: 0) + cursor_size -- integer size of binary cursor to use (default: 0, no binary cursor is used) + use_overviews -- boolean, use overviews when available (default: false) + prescale_rasters -- boolean, scale rasters on the db side (default: false) + clip_rasters -- boolean, clip rasters on the db side (default: false) + band -- integer, if non-zero interprets the given band (1-based offset) as a data raster (default: 0) + + >>> from mapnik import PgRaster, Layer + >>> params = dict(dbname='mapnik',table='osm',user='postgres',password='gis') + >>> params['estimate_extent'] = False + >>> params['extent'] = '-20037508,-19929239,20037508,19929239' + >>> pgraster = PgRaster(**params) + >>> lyr = Layer('PgRaster Layer') + >>> lyr.datasource = pgraster + + """ + keywords['type'] = 'pgraster' + return CreateDatasource(keywords) def Raster(**keywords): """Create a Raster (Tiff) Datasource. diff --git a/plugins/input/pgraster/README b/plugins/input/pgraster/README new file mode 100644 index 000000000..baf2814a9 --- /dev/null +++ b/plugins/input/pgraster/README @@ -0,0 +1,14 @@ +This plugin shares directives with the "postgis" one, +with the following differences: + + - "raster_field" replaces "geometry_field" + + - "raster_table" replaces "geometry_table" + + - "prescale_rasters" replaces "simplify_geometries" + + - "use_overviews" introduced + + - "clip_rasters" boolean introduced, defaults to false + + - "band" introduced, with same semantic of the GDAL driver diff --git a/plugins/input/pgraster/TODO b/plugins/input/pgraster/TODO new file mode 100644 index 000000000..730c846fa --- /dev/null +++ b/plugins/input/pgraster/TODO @@ -0,0 +1,23 @@ +- Automated tests + - Test subqueries [x] + - Test Data input [x] + - Test RGB input [x] + - Test RGBA input [x] + - Test Grayscale input [x] +- Support for all band types: + - PT_1BB data[x] rgb[ ] grayscale[x] + - PT_2BUI data[x] rgb[ ] grayscale[x] + - PT_4BUI data[x] rgb[ ] grayscale[x] + - PT_8BSI data[x] rgb[x] grayscale[x] + - PT_8BUI data[x] rgb[x] grayscale[x] + - PT_16BSI data[x] rgb[ ] grayscale[x] + - PT_16BUI data[x] rgb[ ] grayscale[x] + - PT_32BSI data[x] rgb[ ] grayscale[x] + - PT_32BUI data[x] rgb[ ] grayscale[x] + - PT_32BF data[x] rgb[ ] grayscale[ ] + - PT_64BF data[x] rgb[ ] grayscale[ ] +- Have pgraster and postgis plugins share the same connection pool +- Make overviews enabled by default if table is not a subquery ? +- Make clipping enabled automatically when needed ? +- Allow more flexible band layout specification, see + http://github.com/mapnik/mapnik/wiki/RFC:-Raster-color-interpretation diff --git a/plugins/input/pgraster/build.py b/plugins/input/pgraster/build.py new file mode 100644 index 000000000..e4b083806 --- /dev/null +++ b/plugins/input/pgraster/build.py @@ -0,0 +1,84 @@ +# +# This file is part of Mapnik (c++ mapping toolkit) +# +# Copyright (C) 2013 Artem Pavlenko +# +# Mapnik is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# + +Import ('plugin_base') +Import ('env') +from copy import copy + +PLUGIN_NAME = 'pgraster' + +plugin_env = plugin_base.Clone() + +plugin_sources = Split( + """ + %(PLUGIN_NAME)s_datasource.cpp + %(PLUGIN_NAME)s_featureset.cpp + %(PLUGIN_NAME)s_wkb_reader.cpp + """ % locals() +) + +cxxflags = [] +plugin_env['LIBS'] = [] + +# TODO: use variable for top-level dir +plugin_env['CXXFLAGS'].append('-Iplugins/input/postgis') + +if env['RUNTIME_LINK'] == 'static': + # pkg-config is more reliable than pg_config across platforms + cmd = 'pkg-config libpq --libs --static' + try: + plugin_env.ParseConfig(cmd) + except OSError, e: + plugin_env.Append(LIBS='pq') +else: + plugin_env.Append(LIBS='pq') + +# Link Library to Dependencies +libraries = copy(plugin_env['LIBS']) + +if env['THREADING'] == 'multi': + libraries.append('boost_thread%s' % env['BOOST_APPEND']) + +if env['PLUGIN_LINKING'] == 'shared': + libraries.insert(0,env['MAPNIK_NAME']) + libraries.append(env['ICU_LIB_NAME']) + libraries.append('boost_system%s' % env['BOOST_APPEND']) + + TARGET = plugin_env.SharedLibrary('../%s' % PLUGIN_NAME, + SHLIBPREFIX='', + SHLIBSUFFIX='.input', + source=plugin_sources, + LIBS=libraries, + LINKFLAGS=env['CUSTOM_LDFLAGS']) + + # if the plugin links to libmapnik ensure it is built first + Depends(TARGET, env.subst('../../../src/%s' % env['MAPNIK_LIB_NAME'])) + + if 'uninstall' not in COMMAND_LINE_TARGETS: + env.Install(env['MAPNIK_INPUT_PLUGINS_DEST'], TARGET) + env.Alias('install', env['MAPNIK_INPUT_PLUGINS_DEST']) + +plugin_obj = { + 'LIBS': libraries, + 'SOURCES': plugin_sources, +} + +Return('plugin_obj') diff --git a/plugins/input/pgraster/pgraster_datasource.cpp b/plugins/input/pgraster/pgraster_datasource.cpp new file mode 100644 index 000000000..9d15b7ef1 --- /dev/null +++ b/plugins/input/pgraster/pgraster_datasource.cpp @@ -0,0 +1,1200 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#include "connection_manager.hpp" +#include "pgraster_datasource.hpp" +#include "pgraster_featureset.hpp" +#include "asyncresultset.hpp" + + +// mapnik +#include +#include // for byte +#include +#include +#include +#include +#include + +// boost +#include +#include +#include + +// stl +#include +#include +#include +#include +#include + +DATASOURCE_PLUGIN(pgraster_datasource) + +const double pgraster_datasource::FMAX = std::numeric_limits::max(); +const std::string pgraster_datasource::RASTER_COLUMNS = "raster_columns"; +const std::string pgraster_datasource::RASTER_OVERVIEWS = "raster_overviews"; +const std::string pgraster_datasource::SPATIAL_REF_SYS = "spatial_ref_system"; + +using boost::shared_ptr; +using mapnik::attribute_descriptor; + +namespace { + // TODO: move to sql_utils + std::string quote_literal(std::string& s) { + return "'" + s + "'"; // TODO: escape internal quotes + } + + // TODO: move to sql_utils + std::string quote_ident(std::string& s) { + return "\"" + s + "\""; // TODO: escape internal quotes + } + +}; + +pgraster_datasource::pgraster_datasource(parameters const& params) + : datasource(params), + table_(*params.get("table", "")), + schema_(""), + raster_table_(*params.get("raster_table", "")), + raster_field_(*params.get("raster_field", "")), + key_field_(*params.get("key_field", "")), + cursor_fetch_size_(*params.get("cursor_size", 0)), + row_limit_(*params.get("row_limit", 0)), + type_(datasource::Raster), + srid_(*params.get("srid", 0)), + band_(*params.get("band", 0)), + extent_initialized_(false), + prescale_rasters_(*params.get("prescale_rasters", false)), + use_overviews_(*params.get("use_overviews", false)), + clip_rasters_(*params.get("clip_rasters", false)), + desc_(*params.get("type"), "utf-8"), + creator_(params.get("host"), + params.get("port"), + params.get("dbname"), + params.get("user"), + params.get("password"), + params.get("connect_timeout", "4")), + bbox_token_("!bbox!"), + scale_denom_token_("!scale_denominator!"), + pixel_width_token_("!pixel_width!"), + pixel_height_token_("!pixel_height!"), + pool_max_size_(*params_.get("max_size", 10)), + persist_connection_(*params.get("persist_connection", true)), + extent_from_subquery_(*params.get("extent_from_subquery", false)), + estimate_extent_(*params.get("estimate_extent", false)), + max_async_connections_(*params_.get("max_async_connection", 1)), + asynchronous_request_(false), + // params below are for testing purposes only and may be removed at any time + intersect_min_scale_(*params.get("intersect_min_scale", 0)), + intersect_max_scale_(*params.get("intersect_max_scale", 0)) +{ +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, "pgraster_datasource::init"); +#endif + if (table_.empty()) + { + throw mapnik::datasource_exception("Pgraster Plugin: missing parameter"); + } + + boost::optional ext = params.get("extent"); + if (ext && !ext->empty()) + { + extent_initialized_ = extent_.from_string(*ext); + } + + // NOTE: In multithread environment, pool_max_size_ should be + // max_async_connections_ * num_threads + if(max_async_connections_ > 1) + { + if(max_async_connections_ > pool_max_size_) + { + std::ostringstream err; + err << "PostGIS Plugin: Error: 'max_async_connections (" + << max_async_connections_ << ") must be <= max_size(" << pool_max_size_ << ")"; + throw mapnik::datasource_exception(err.str()); + } + asynchronous_request_ = true; + } + + boost::optional initial_size = params.get("initial_size", 1); + boost::optional autodetect_key_field = params.get("autodetect_key_field", false); + + ConnectionManager::instance().registerPool(creator_, *initial_size, pool_max_size_); + CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); + if (pool) + { + shared_ptr conn = pool->borrowObject(); + if (!conn) return; + + if (conn->isOK()) + { + + desc_.set_encoding(conn->client_encoding()); + + if (raster_table_.empty()) + { + raster_table_ = mapnik::sql_utils::table_from_sql(table_); + // non-trivial subqueries (having no FROM) make it + // impossible to use overviews + // TODO: improve "table_from_sql" ? + if ( raster_table_[raster_table_.find_first_not_of(" \t\r\n")] == '(' ) + { + raster_table_.clear(); + if ( use_overviews_ ) + { + std::ostringstream err; + err << "Pgraster Plugin: overviews cannot be used " + "with non-trivial subqueries"; + MAPNIK_LOG_WARN(pgraster) << err.str(); + use_overviews_ = false; + } + if ( ! extent_from_subquery_ ) { + std::ostringstream err; + err << "Pgraster Plugin: extent can only be computed " + "from subquery as we could not found table source"; + MAPNIK_LOG_WARN(pgraster) << err.str(); + extent_from_subquery_ = true; + } + + } + } + + std::string::size_type idx = raster_table_.find_last_of('.'); + if (idx != std::string::npos) + { + schema_ = raster_table_.substr(0, idx); + raster_table_ = raster_table_.substr(idx + 1); + } + + // If we do not know either the geometry_field or the srid or we + // want to use overviews but do not know about schema, or + // no extent was specified, then attempt to fetch the missing + // information from a raster_columns entry. + // + // This will return no records if we are querying a bogus table returned + // from the simplistic table parsing in table_from_sql() or if + // the table parameter references a table, view, or subselect not + // registered in the geometry columns. + // + geometryColumn_ = mapnik::sql_utils::unquote_double(raster_field_); + if (!raster_table_.empty() && ( + geometryColumn_.empty() || srid_ == 0 || + (schema_.empty() && use_overviews_) || + ! extent_initialized_ + )) + { +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::init(get_srid_and_geometry_column)"); +#endif + std::ostringstream s; + + try + { + s << "SELECT r_raster_column col, srid, r_table_schema"; + if ( ! extent_initialized_ ) { + s << ", st_xmin(extent) xmin, st_ymin(extent) ymin" + << ", st_xmax(extent) xmax, st_ymax(extent) ymax"; + } + s << " FROM " + << RASTER_COLUMNS << " WHERE r_table_name='" + << mapnik::sql_utils::unquote_double(raster_table_) + << "'"; + if (! schema_.empty()) + { + s << " AND r_table_schema='" + << mapnik::sql_utils::unquote_double(schema_) + << "'"; + } + if (! raster_field_.empty()) + { + s << " AND r_raster_column='" + << mapnik::sql_utils::unquote_double(raster_field_) + << "'"; + } + MAPNIK_LOG_DEBUG(pgraster) << + "pgraster_datasource: running query " << s.str(); + shared_ptr rs = conn->executeQuery(s.str()); + if (rs->next()) + { + geometryColumn_ = rs->getValue("col"); + if ( ! extent_initialized_ ) + { + double lox, loy, hix, hiy; + if (mapnik::util::string2double(rs->getValue("xmin"), lox) && + mapnik::util::string2double(rs->getValue("ymin"), loy) && + mapnik::util::string2double(rs->getValue("xmax"), hix) && + mapnik::util::string2double(rs->getValue("ymax"), hiy)) + { + extent_.init(lox, loy, hix, hiy); + extent_initialized_ = true; + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Layer extent=" << extent_; + } + else + { + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Could not determine extent from query: " << s.str(); + } + } + if (srid_ == 0) + { + const char* srid_c = rs->getValue("srid"); + if (srid_c != NULL) + { + int result = 0; + const char * end = srid_c + std::strlen(srid_c); + if (mapnik::util::string2int(srid_c, end, result)) + { + srid_ = result; + } + } + } + if ( schema_.empty() ) + { + schema_ = rs->getValue("r_table_schema"); + } + } + else + { + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: no response from metadata query " << s.str(); + } + rs->close(); + } + catch (mapnik::datasource_exception const& ex) { + // let this pass on query error and use the fallback below + MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: metadata query failed: " << ex.what(); + } + + // If we still do not know the srid then we can try to fetch + // it from the 'table_' parameter, which should work even if it is + // a subselect as long as we know the geometry_field to query + if (! geometryColumn_.empty() && srid_ <= 0) + { + s.str(""); + + s << "SELECT ST_SRID(\"" << geometryColumn_ << "\") AS srid FROM " + << populate_tokens(table_) << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;"; + + shared_ptr rs = conn->executeQuery(s.str()); + if (rs->next()) + { + const char* srid_c = rs->getValue("srid"); + if (srid_c != NULL) + { + int result = 0; + const char * end = srid_c + std::strlen(srid_c); + if (mapnik::util::string2int(srid_c, end, result)) + { + srid_ = result; + } + } + } + rs->close(); + } + } + + // If overviews were requested, take note of the max scale + // of each available overview, sorted by scale descending + if ( use_overviews_ ) + { + std::ostringstream err; + if ( schema_.empty() ) + { + err << "Pgraster Plugin: unable to lookup available table" + << " overviews due to unknown schema"; + throw mapnik::datasource_exception(err.str()); + } + if ( geometryColumn_.empty() ) + { + err << "Pgraster Plugin: unable to lookup available table" + << " overviews due to unknown column name"; + throw mapnik::datasource_exception(err.str()); + } + + std::ostringstream s; + s << "select " + "r.r_table_schema sch, " + "r.r_table_name tab, " + "r.r_raster_column col, " + "greatest(abs(r.scale_x), abs(r.scale_y)) scl " + "from" + " raster_overviews o," + " raster_columns r " + "where" + " o.r_table_schema = '" + << mapnik::sql_utils::unquote_double(schema_) + << "' and o.r_table_name = '" + << mapnik::sql_utils::unquote_double(raster_table_) + << "' and o.r_raster_column = '" + << mapnik::sql_utils::unquote_double(geometryColumn_) + << "' and r.r_table_schema = o.o_table_schema" + " and r.r_table_name = o.o_table_name" + " and r.r_raster_column = o.o_raster_column" + " ORDER BY scl ASC"; + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: running query " << s.str(); + shared_ptr rs = conn->executeQuery(s.str()); + while (rs->next()) + { + overviews_.resize(overviews_.size()+1); + pgraster_overview& ov = overviews_.back(); + ov.schema = rs->getValue("sch"); + ov.table = rs->getValue("tab"); + ov.column = rs->getValue("col"); + ov.scale = atof(rs->getValue("scl")); + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: found overview " << ov.schema << "." << ov.table << "." << ov.column << " with scale " << ov.scale; + } + rs->close(); + if ( overviews_.empty() ) { + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: no overview found for " << schema_ << "." << raster_table_ << "." << geometryColumn_; + } + } + + // detect primary key + if (*autodetect_key_field && key_field_.empty()) + { +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::bind(get_primary_key)"); +#endif + + std::ostringstream s; + s << "SELECT a.attname, a.attnum, t.typname, t.typname in ('int2','int4','int8') " + "AS is_int FROM pg_class c, pg_attribute a, pg_type t, pg_namespace n, pg_index i " + "WHERE a.attnum > 0 AND a.attrelid = c.oid " + "AND a.atttypid = t.oid AND c.relnamespace = n.oid " + "AND c.oid = i.indrelid AND i.indisprimary = 't' " + "AND t.typname !~ '^geom' AND c.relname =" + << " '" << mapnik::sql_utils::unquote_double(raster_table_) << "' " + //"AND a.attnum = ANY (i.indkey) " // postgres >= 8.1 + << "AND (i.indkey[0]=a.attnum OR i.indkey[1]=a.attnum OR i.indkey[2]=a.attnum " + "OR i.indkey[3]=a.attnum OR i.indkey[4]=a.attnum OR i.indkey[5]=a.attnum " + "OR i.indkey[6]=a.attnum OR i.indkey[7]=a.attnum OR i.indkey[8]=a.attnum " + "OR i.indkey[9]=a.attnum) "; + if (! schema_.empty()) + { + s << "AND n.nspname='" + << mapnik::sql_utils::unquote_double(schema_) + << "' "; + } + s << "ORDER BY a.attnum"; + + shared_ptr rs_key = conn->executeQuery(s.str()); + if (rs_key->next()) + { + unsigned int result_rows = rs_key->size(); + if (result_rows == 1) + { + bool is_int = (std::string(rs_key->getValue(3)) == "t"); + if (is_int) + { + const char* key_field_string = rs_key->getValue(0); + if (key_field_string) + { + key_field_ = std::string(key_field_string); + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: auto-detected key field of '" + << key_field_ << "' on table '" << raster_table_ << "'"; + } + } + else + { + // throw for cases like a numeric primary key, which is invalid + // as it should be floating point (int numerics are useless) + std::ostringstream err; + err << "PostGIS Plugin: Error: '" + << rs_key->getValue(0) + << "' on table '" + << raster_table_ + << "' is not a valid integer primary key field\n"; + throw mapnik::datasource_exception(err.str()); + } + } + else if (result_rows > 1) + { + std::ostringstream err; + err << "PostGIS Plugin: Error: '" + << "multi column primary key detected but is not supported"; + throw mapnik::datasource_exception(err.str()); + } + } + rs_key->close(); + } + + // if a globally unique key field/primary key is required + // but still not known at this point, then throw + if (*autodetect_key_field && key_field_.empty()) + { + throw mapnik::datasource_exception(std::string("PostGIS Plugin: Error: primary key required") + + " but could not be detected for table '" + + raster_table_ + "', please supply 'key_field' option to specify field to use for primary key"); + } + + if (srid_ == 0) + { + srid_ = -1; + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Table " << table_ << " is using SRID=" << srid_; + } + + // At this point the geometry_field may still not be known + // but we'll catch that where more useful... + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Using SRID=" << srid_; + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Using geometry_column=" << geometryColumn_; + + // collect attribute desc +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::bind(get_column_description)"); +#endif + + std::ostringstream s; + s << "SELECT * FROM " << populate_tokens(table_) << " LIMIT 0"; + + shared_ptr rs = conn->executeQuery(s.str()); + int count = rs->getNumFields(); + bool found_key_field = false; + for (int i = 0; i < count; ++i) + { + std::string fld_name = rs->getFieldName(i); + int type_oid = rs->getTypeOID(i); + + // validate type of key_field + if (! found_key_field && ! key_field_.empty() && fld_name == key_field_) + { + if (type_oid == 20 || type_oid == 21 || type_oid == 23) + { + found_key_field = true; + desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); + } + else + { + std::ostringstream error_s; + error_s << "invalid type '"; + + std::ostringstream type_s; + type_s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; + + shared_ptr rs_oid = conn->executeQuery(type_s.str()); + if (rs_oid->next()) + { + error_s << rs_oid->getValue("typname") + << "' (oid:" << rs_oid->getValue("oid") << ")"; + } + else + { + error_s << "oid:" << type_oid << "'"; + } + + rs_oid->close(); + error_s << " for key_field '" << fld_name << "' - " + << "must be an integer primary key"; + + rs->close(); + throw mapnik::datasource_exception(error_s.str()); + } + } + else + { + switch (type_oid) + { + case 16: // bool + desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Boolean)); + break; + case 20: // int8 + case 21: // int2 + case 23: // int4 + desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); + break; + case 700: // float4 + case 701: // float8 + case 1700: // numeric + desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double)); + break; + case 1042: // bpchar + case 1043: // varchar + case 25: // text + case 705: // literal + desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); + break; + default: // should not get here +#ifdef MAPNIK_LOG + s.str(""); + s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; + + shared_ptr rs_oid = conn->executeQuery(s.str()); + if (rs_oid->next()) + { + std::string typname(rs_oid->getValue("typname")); + if (typname != "geometry" && typname != "raster") + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: Unknown type=" << typname + << " (oid:" << rs_oid->getValue("oid") << ")"; + } + } + else + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: Unknown type_oid=" << type_oid; + } + rs_oid->close(); +#endif + break; + } + } + } + + rs->close(); + + } + + // Close explicitly the connection so we can 'fork()' without sharing open connections + conn->close(); + + } +} + +pgraster_datasource::~pgraster_datasource() +{ + if (! persist_connection_) + { + CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); + if (pool) + { + try { + shared_ptr conn = pool->borrowObject(); + if (conn) + { + conn->close(); + } + } catch (mapnik::datasource_exception const& ex) { + // happens when borrowObject tries to + // create a new connection and fails. + // In turn, new connection would be needed + // when our broke and was thus no good to + // be borrowed + // See https://github.com/mapnik/mapnik/issues/2191 + } + } + } +} + +const char * pgraster_datasource::name() +{ + return "pgraster"; +} + +mapnik::datasource::datasource_t pgraster_datasource::type() const +{ + return type_; +} + +layer_descriptor pgraster_datasource::get_descriptor() const +{ + return desc_; +} + +std::string pgraster_datasource::sql_bbox(box2d const& env) const +{ + std::ostringstream b; + + if (srid_ > 0) + { + b << "ST_SetSRID("; + } + + b << "'BOX3D("; + b << std::setprecision(16); + b << env.minx() << " " << env.miny() << ","; + b << env.maxx() << " " << env.maxy() << ")'::box3d"; + + if (srid_ > 0) + { + b << ", " << srid_ << ")"; + } + + return b.str(); +} + +std::string pgraster_datasource::populate_tokens(std::string const& sql) const +{ + std::string populated_sql = sql; + + if (boost::algorithm::icontains(sql, bbox_token_)) + { + box2d max_env(-1.0 * FMAX, -1.0 * FMAX, FMAX, FMAX); + const std::string max_box = sql_bbox(max_env); + boost::algorithm::replace_all(populated_sql, bbox_token_, max_box); + } + + if (boost::algorithm::icontains(sql, scale_denom_token_)) + { + std::ostringstream ss; + ss << FMAX; + boost::algorithm::replace_all(populated_sql, scale_denom_token_, ss.str()); + } + + if (boost::algorithm::icontains(sql, pixel_width_token_)) + { + std::ostringstream ss; + ss << 0; + boost::algorithm::replace_all(populated_sql, pixel_width_token_, ss.str()); + } + + if (boost::algorithm::icontains(sql, pixel_height_token_)) + { + std::ostringstream ss; + ss << 0; + boost::algorithm::replace_all(populated_sql, pixel_height_token_, ss.str()); + } + + return populated_sql; +} + +std::string pgraster_datasource::populate_tokens(std::string const& sql, double scale_denom, box2d const& env, double pixel_width, double pixel_height) const +{ + std::string populated_sql = sql; + std::string box = sql_bbox(env); + + if (boost::algorithm::icontains(populated_sql, scale_denom_token_)) + { + std::ostringstream ss; + ss << scale_denom; + boost::algorithm::replace_all(populated_sql, scale_denom_token_, ss.str()); + } + + if (boost::algorithm::icontains(sql, pixel_width_token_)) + { + std::ostringstream ss; + ss << pixel_width; + boost::algorithm::replace_all(populated_sql, pixel_width_token_, ss.str()); + } + + if (boost::algorithm::icontains(sql, pixel_height_token_)) + { + std::ostringstream ss; + ss << pixel_height; + boost::algorithm::replace_all(populated_sql, pixel_height_token_, ss.str()); + } + + if (boost::algorithm::icontains(populated_sql, bbox_token_)) + { + boost::algorithm::replace_all(populated_sql, bbox_token_, box); + return populated_sql; + } + else + { + std::ostringstream s; + + if (intersect_min_scale_ > 0 && (scale_denom <= intersect_min_scale_)) + { + s << " WHERE ST_Intersects(\"" << geometryColumn_ << "\"," << box << ")"; + } + else if (intersect_max_scale_ > 0 && (scale_denom >= intersect_max_scale_)) + { + // do no bbox restriction + } + else + { + s << " WHERE \"" << geometryColumn_ << "\" && " << box; + } + + return populated_sql + s.str(); + } +} + + +boost::shared_ptr pgraster_datasource::get_resultset(boost::shared_ptr &conn, std::string const& sql, CnxPool_ptr const& pool, processor_context_ptr ctx) const +{ + + if (!ctx) + { + // ! asynchronous_request_ + if (cursor_fetch_size_ > 0) + { + // cursor + std::ostringstream csql; + std::string cursor_name = conn->new_cursor_name(); + + csql << "DECLARE " << cursor_name << " BINARY INSENSITIVE NO SCROLL CURSOR WITH HOLD FOR " << sql << " FOR READ ONLY"; + + if (! conn->execute(csql.str())) + { + // TODO - better error + throw mapnik::datasource_exception("Pgraster Plugin: error creating cursor for data select." ); + } + + return boost::make_shared(conn, cursor_name, cursor_fetch_size_); + + } + else + { + // no cursor + return conn->executeQuery(sql, 1); + } + } + else + { // asynchronous requests + + boost::shared_ptr pgis_ctxt = boost::static_pointer_cast(ctx); + if (conn) + { + // lauch async req & create asyncresult with conn + conn->executeAsyncQuery(sql, 1); + return boost::make_shared(pgis_ctxt, pool, conn, sql); + } + else + { + // create asyncresult with null connection + boost::shared_ptr res = boost::make_shared(pgis_ctxt, pool, conn, sql); + pgis_ctxt->add_request(res); + return res; + } + } +} + +processor_context_ptr pgraster_datasource::get_context(feature_style_context_map & ctx) const +{ + if (!asynchronous_request_) + { + return processor_context_ptr(); + } + + std::string ds_name(name()); + feature_style_context_map::const_iterator itr = ctx.find(ds_name); + if (itr != ctx.end()) + { + return itr->second; + } + else + { + return ctx.insert(std::make_pair(ds_name,boost::make_shared())).first->second; + } +} + +featureset_ptr pgraster_datasource::features(query const& q) const +{ + // if the driver is in asynchronous mode, return the appropriate fetaures + if (asynchronous_request_ ) + { + return features_with_context(q,boost::make_shared()); + } + else + { + return features_with_context(q,processor_context_ptr()); + } +} + +featureset_ptr pgraster_datasource::features_with_context(query const& q,processor_context_ptr proc_ctx) const +{ + +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, "pgraster_datasource::features_with_context"); +#endif + + + box2d const& box = q.get_bbox(); + double scale_denom = q.scale_denominator(); + + CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); + + if (pool) + { + shared_ptr conn; + + if ( asynchronous_request_ ) + { + // limit use to num_async_request_ => if reached don't borrow the last connexion object + boost::shared_ptr pgis_ctxt = boost::static_pointer_cast(proc_ctx); + if ( pgis_ctxt->num_async_requests_ < max_async_connections_ ) + { + conn = pool->borrowObject(); + pgis_ctxt->num_async_requests_++; + } + } + else + { + // Always get a connection in synchronous mode + conn = pool->borrowObject(); + if(!conn ) + { + throw mapnik::datasource_exception("Pgraster Plugin: Null connection"); + } + } + + + if (geometryColumn_.empty()) + { + std::ostringstream s_error; + s_error << "PostGIS: geometry name lookup failed for table '"; + + if (! schema_.empty()) + { + s_error << schema_ << "."; + } + s_error << raster_table_ + << "'. Please manually provide the 'geometry_field' parameter or add an entry " + << "in the geometry_columns for '"; + + if (! schema_.empty()) + { + s_error << schema_ << "."; + } + s_error << raster_table_ << "'."; + + throw mapnik::datasource_exception(s_error.str()); + } + + const double px_gw = 1.0 / boost::get<0>(q.resolution()); + const double px_gh = 1.0 / boost::get<1>(q.resolution()); + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: px_gw=" << px_gw + << " px_gh=" << px_gh; + + std::string table_with_bbox; + std::string col = geometryColumn_; + + if ( use_overviews_ ) { + std::string sch = schema_; + std::string tab = mapnik::sql_utils::unquote_double(raster_table_); + const double scale = std::min(px_gw, px_gh); + std::vector::const_reverse_iterator i; + for (i=overviews_.rbegin(); i!=overviews_.rend(); ++i) { + const pgraster_overview& o = *i; + if ( o.scale < scale ) { + sch = o.schema; + tab = o.table; + col = o.column; + MAPNIK_LOG_DEBUG(pgraster) + << "pgraster_datasource: using overview " + << o.schema << "." << o.table << "." << o.column + << " with scale=" << o.scale + << " for min out scale=" << scale; + break; + } else { + MAPNIK_LOG_DEBUG(pgraster) + << "pgraster_datasource: overview " + << o.schema << "." << o.table << "." << o.column + << " with scale=" << o.scale + << " not good for min out scale " << scale; + } + } + table_with_bbox = table_; // possibly a subquery + boost::algorithm::replace_all(table_with_bbox, + mapnik::sql_utils::unquote_double(raster_table_), tab); + boost::algorithm::replace_all(table_with_bbox, + mapnik::sql_utils::unquote_double(schema_), sch); + boost::algorithm::replace_all(table_with_bbox, + mapnik::sql_utils::unquote_double(geometryColumn_), col); + table_with_bbox = populate_tokens(table_with_bbox, + scale_denom, box, px_gw, px_gh); + } else { + table_with_bbox = populate_tokens(table_, scale_denom, box, px_gw, px_gh); + } + + std::ostringstream s; + + s << "SELECT ST_AsBinary("; + + if (band_) s << "ST_Band("; + + if (prescale_rasters_) s << "ST_Resize("; + + if (clip_rasters_) s << "ST_Clip("; + + s << "\"" << col << "\""; + + if (clip_rasters_) { + s << ", ST_Expand(" << sql_bbox(box) + << ", greatest(abs(ST_ScaleX(\"" + << col << "\")), abs(ST_ScaleY(\"" + << col << "\")))))"; + } + + if (prescale_rasters_) { + const double scale = std::min(px_gw, px_gh); + s << ", least(abs(ST_ScaleX(\"" << col + << "\"))::float8/" << scale + << ", 1.0), least(abs(ST_ScaleY(\"" << col + << "\"))::float8/" << scale << ", 1.0))"; + // TODO: if band_ is given, we'll interpret as indexed so + // the rescaling must NOT ruin it (use algorithm mode!) + } + + if (band_) s << ", " << band_ << ")"; + + s << ") AS geom"; + + mapnik::context_ptr ctx = boost::make_shared(); + std::set const& props = q.property_names(); + std::set::const_iterator pos = props.begin(); + std::set::const_iterator end = props.end(); + + if (! key_field_.empty()) + { + mapnik::sql_utils::quote_attr(s, key_field_); + ctx->push(key_field_); + + for (; pos != end; ++pos) + { + if (*pos != key_field_) + { + mapnik::sql_utils::quote_attr(s, *pos); + ctx->push(*pos); + } + } + } + else + { + for (; pos != end; ++pos) + { + mapnik::sql_utils::quote_attr(s, *pos); + ctx->push(*pos); + } + } + + s << " FROM " << table_with_bbox; + + if (row_limit_ > 0) + { + s << " LIMIT " << row_limit_; + } + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: " + "features query: " << s.str(); + + boost::shared_ptr rs = get_resultset(conn, s.str(), pool, proc_ctx); + return boost::make_shared(rs, ctx, + desc_.get_encoding(), !key_field_.empty(), + band_ ? 1 : 0 // whatever band number is given we'd have + // extracted with ST_Band above so it becomes + // band number 1 + ); + + } + + return featureset_ptr(); +} + + +featureset_ptr pgraster_datasource::features_at_point(coord2d const& pt, double tol) const +{ +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, "pgraster_datasource::features_at_point"); +#endif + CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); + if (pool) + { + shared_ptr conn = pool->borrowObject(); + if (!conn) return featureset_ptr(); + + if (conn->isOK()) + { + if (geometryColumn_.empty()) + { + std::ostringstream s_error; + s_error << "PostGIS: geometry name lookup failed for table '"; + + if (! schema_.empty()) + { + s_error << schema_ << "."; + } + s_error << raster_table_ + << "'. Please manually provide the 'geometry_field' parameter or add an entry " + << "in the geometry_columns for '"; + + if (! schema_.empty()) + { + s_error << schema_ << "."; + } + s_error << raster_table_ << "'."; + + throw mapnik::datasource_exception(s_error.str()); + } + + std::ostringstream s; + s << "SELECT ST_AsBinary(\"" << geometryColumn_ << "\") AS geom"; + + mapnik::context_ptr ctx = boost::make_shared(); + std::vector::const_iterator itr = desc_.get_descriptors().begin(); + std::vector::const_iterator end = desc_.get_descriptors().end(); + + if (! key_field_.empty()) + { + mapnik::sql_utils::quote_attr(s, key_field_); + ctx->push(key_field_); + for (; itr != end; ++itr) + { + if (itr->get_name() != key_field_) + { + mapnik::sql_utils::quote_attr(s, itr->get_name()); + ctx->push(itr->get_name()); + } + } + } + else + { + for (; itr != end; ++itr) + { + mapnik::sql_utils::quote_attr(s, itr->get_name()); + ctx->push(itr->get_name()); + } + } + + box2d box(pt.x - tol, pt.y - tol, pt.x + tol, pt.y + tol); + std::string table_with_bbox = populate_tokens(table_, FMAX, box, 0, 0); + + s << " FROM " << table_with_bbox; + + if (row_limit_ > 0) + { + s << " LIMIT " << row_limit_; + } + + boost::shared_ptr rs = get_resultset(conn, s.str(), pool); + return boost::make_shared(rs, ctx, desc_.get_encoding(), !key_field_.empty()); + } + } + + return featureset_ptr(); +} + +box2d pgraster_datasource::envelope() const +{ + if (extent_initialized_) + { + return extent_; + } + + CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); + if (pool) + { + shared_ptr conn = pool->borrowObject(); + if (!conn) return extent_; + if (conn->isOK()) + { + std::ostringstream s; + + std::string col = mapnik::sql_utils::unquote_double(geometryColumn_); + std::string sch = mapnik::sql_utils::unquote_double(schema_); + std::string tab = mapnik::sql_utils::unquote_double(raster_table_); + + if ( ! overviews_.empty() ) + { + // query from highest-factor overview instead + const pgraster_overview& o = overviews_.back(); + sch = o.schema; + tab = o.table; + col = o.column; + } + + if (col.empty()) + { + std::ostringstream s_error; + s_error << "PostGIS: unable to query the layer extent of table '"; + + if (! sch.empty()) + { + s_error << sch << "."; + } + s_error << raster_table_ << "' because we cannot determine the raster field name." + << "\nPlease provide either an 'extent' parameter to skip this query, " + << "a 'raster_field' and/or 'raster_table' parameter, or add " + << "standard constraints to your raster table."; + + throw mapnik::datasource_exception("Pgraster Plugin: " + s_error.str()); + } + + if (estimate_extent_) + { + if (tab.empty()) + { + std::ostringstream s_error; + s_error << "PostGIS: unable to query the layer extent as " + << "we couldn't determine the raster table name.\n" + << "Please provide either an 'extent' parameter to skip this query, " + << "a 'raster_table' parameter, or do not set 'estimate_extent'"; + throw mapnik::datasource_exception("Pgraster Plugin: " + s_error.str()); + } + s << "SELECT ST_XMin(ext),ST_YMin(ext),ST_XMax(ext),ST_YMax(ext)" + << " FROM (SELECT ST_Estimated_Extent('"; + + if (! sch.empty()) + { + s << mapnik::sql_utils::unquote_double(sch) << "','"; + } + + s << mapnik::sql_utils::unquote_double(tab) << "','" + << mapnik::sql_utils::unquote_double(col) << "') as ext) as tmp"; + } + else + { + s << "SELECT ST_XMin(ext),ST_YMin(ext),ST_XMax(ext),ST_YMax(ext)" + << " FROM (SELECT ST_Extent(" << quote_ident(col) << "::geometry) as ext from "; + + if (extent_from_subquery_) + { + // if a subselect limits records then calculating the extent upon the + // subquery will be faster and the bounds will be more accurate + s << populate_tokens(table_) << ") as tmpx"; + } + else + { + if (! sch.empty()) + { + s << quote_ident(sch) << "."; + } + + // but if the subquery does not limit records then querying the + // actual table will be faster as indexes can be used + s << quote_ident(tab) << ") as tmp"; + } + } + + shared_ptr rs = conn->executeQuery(s.str()); + if (rs->next() && ! rs->isNull(0)) + { + double lox, loy, hix, hiy; + if (mapnik::util::string2double(rs->getValue(0), lox) && + mapnik::util::string2double(rs->getValue(1), loy) && + mapnik::util::string2double(rs->getValue(2), hix) && + mapnik::util::string2double(rs->getValue(3), hiy)) + { + extent_.init(lox, loy, hix, hiy); + extent_initialized_ = true; + } + else + { + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Could not determine extent from query: " << s.str(); + } + } + rs->close(); + } + } + + return extent_; +} + +boost::optional pgraster_datasource::get_geometry_type() const +{ + return boost::optional(); +} + + diff --git a/plugins/input/pgraster/pgraster_datasource.hpp b/plugins/input/pgraster/pgraster_datasource.hpp new file mode 100644 index 000000000..e4f1ae38f --- /dev/null +++ b/plugins/input/pgraster/pgraster_datasource.hpp @@ -0,0 +1,147 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#ifndef PGRASTER_DATASOURCE_HPP +#define PGRASTER_DATASOURCE_HPP + +// mapnik +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// boost +#include +#include +#include + +// stl +#include +#include + +#include "connection_manager.hpp" +#include "resultset.hpp" +#include "cursorresultset.hpp" + +using mapnik::transcoder; +using mapnik::datasource; +using mapnik::feature_style_context_map; +using mapnik::processor_context_ptr; +using mapnik::box2d; +using mapnik::layer_descriptor; +using mapnik::featureset_ptr; +using mapnik::feature_ptr; +using mapnik::query; +using mapnik::parameters; +using mapnik::coord2d; + +typedef boost::shared_ptr< ConnectionManager::PoolType> CnxPool_ptr; + +struct pgraster_overview +{ + std::string schema; + std::string table; + std::string column; + float scale; // max absolute scale between x and y +}; + + +class pgraster_datasource : public datasource +{ +public: + pgraster_datasource(const parameters ¶ms); + ~pgraster_datasource(); + mapnik::datasource::datasource_t type() const; + static const char * name(); + processor_context_ptr get_context(feature_style_context_map &) const; + featureset_ptr features_with_context(query const& q, processor_context_ptr ctx) const; + featureset_ptr features(query const& q) const; + featureset_ptr features_at_point(coord2d const& pt, double tol = 0) const; + mapnik::box2d envelope() const; + boost::optional get_geometry_type() const; + layer_descriptor get_descriptor() const; + +private: + std::string sql_bbox(box2d const& env) const; + std::string populate_tokens(std::string const& sql, double scale_denom, box2d const& env, double pixel_width, double pixel_height) const; + std::string populate_tokens(std::string const& sql) const; + boost::shared_ptr get_resultset(boost::shared_ptr &conn, std::string const& sql, CnxPool_ptr const& pool, processor_context_ptr ctx= processor_context_ptr()) const; + static const std::string RASTER_COLUMNS; + static const std::string RASTER_OVERVIEWS; + static const std::string SPATIAL_REF_SYS; + static const double FMAX; + + const std::string uri_; + const std::string username_; + const std::string password_; + // table name (schema qualified or not) or subquery + const std::string table_; + // schema name (possibly extracted from table_) + std::string schema_; + // table name (possibly extracted from table_) + std::string raster_table_; + const std::string raster_field_; + std::string key_field_; + mapnik::value_integer cursor_fetch_size_; + mapnik::value_integer row_limit_; + std::string geometryColumn_; + mapnik::datasource::datasource_t type_; + int srid_; + // 1-based index of band to extract from the raster + // 0 means fetch all bands + // any index also forces color interpretation off so that values + // arrives untouched into the resulting mapnik raster, for threatment + // by raster colorizer + int band_; + // Available overviews, ordered by max scale, ascending + std::vector overviews_; + mutable bool extent_initialized_; + mutable mapnik::box2d extent_; + bool prescale_rasters_; + bool use_overviews_; + bool clip_rasters_; + layer_descriptor desc_; + ConnectionCreator creator_; + const std::string bbox_token_; + const std::string scale_denom_token_; + const std::string pixel_width_token_; + const std::string pixel_height_token_; + int pool_max_size_; + bool persist_connection_; + bool extent_from_subquery_; + bool estimate_extent_; + int max_async_connections_; + bool asynchronous_request_; + int intersect_min_scale_; + int intersect_max_scale_; +}; + +#endif // PGRASTER_DATASOURCE_HPP diff --git a/plugins/input/pgraster/pgraster_featureset.cpp b/plugins/input/pgraster/pgraster_featureset.cpp new file mode 100644 index 000000000..8e4ded1fd --- /dev/null +++ b/plugins/input/pgraster/pgraster_featureset.cpp @@ -0,0 +1,360 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#include "pgraster_featureset.hpp" +#include "pgraster_wkb_reader.hpp" +#include "resultset.hpp" +#include "cursorresultset.hpp" + +// mapnik +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // for int2net +#include + + +// boost +#include // for boost::int16_t + +// stl +#include +#include + +using mapnik::geometry_type; +using mapnik::byte; +using mapnik::feature_factory; +using mapnik::context_ptr; + +pgraster_featureset::pgraster_featureset(boost::shared_ptr const& rs, + context_ptr const& ctx, + std::string const& encoding, + bool key_field, int bandno) + : rs_(rs), + ctx_(ctx), + tr_(new transcoder(encoding)), + feature_id_(1), + key_field_(key_field), + band_(bandno) +{ +} + +std::string numeric2string(const char* buf); + +feature_ptr pgraster_featureset::next() +{ + while (rs_->next()) + { + // new feature + unsigned pos = 1; + feature_ptr feature; + + if (key_field_) + { + std::string name = rs_->getFieldName(pos); + + // null feature id is not acceptable + if (rs_->isNull(pos)) + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: null value encountered for key_field: " << name; + continue; + } + // create feature with user driven id from attribute + int oid = rs_->getTypeOID(pos); + const char* buf = rs_->getValue(pos); + + // validation happens of this type at initialization + mapnik::value_integer val; + + if (oid == 20) + { + val = int8net(buf); + } + else if (oid == 21) + { + val = int2net(buf); + } + else + { + val = int4net(buf); + } + + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: feature key: " << val; + + feature = feature_factory::create(ctx_, val); + // TODO - extend feature class to know + // that its id is also an attribute to avoid + // this duplication + feature->put(name,val); + ++pos; + } + else + { + // fallback to auto-incrementing id + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: feature id: " << feature_id_; + + feature = feature_factory::create(ctx_, feature_id_); + ++feature_id_; + } + + // null geometry is not acceptable + if (rs_->isNull(0)) + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: null value encountered for raster"; + continue; + } + + // parse geometry + int size = rs_->getFieldLength(0); + const uint8_t *data = (const uint8_t*)rs_->getValue(0); + + mapnik::raster_ptr raster = pgraster_wkb_reader::read(data, size, band_); + if (!raster) + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: could not parse raster wkb"; + // TODO: throw an exception ? + continue; + } + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: raster of " << raster->data_.width() << "x" << raster->data_.height() << " pixels covering extent " << raster->ext_; + feature->set_raster(raster); + + unsigned num_attrs = ctx_->size() + 1; + for (; pos < num_attrs; ++pos) + { + std::string name = rs_->getFieldName(pos); + + // NOTE: we intentionally do not store null here + // since it is equivalent to the attribute not existing + if (!rs_->isNull(pos)) + { + const char* buf = rs_->getValue(pos); + const int oid = rs_->getTypeOID(pos); + switch (oid) + { + case 16: //bool + { + feature->put(name, (buf[0] != 0)); + break; + } + + case 23: //int4 + { + feature->put(name, int4net(buf)); + break; + } + + case 21: //int2 + { + feature->put(name, int2net(buf)); + break; + } + + case 20: //int8/BigInt + { + feature->put(name, int8net(buf)); + break; + } + + case 700: //float4 + { + float val; + float4net(val, buf); + feature->put(name, static_cast(val)); + break; + } + + case 701: //float8 + { + double val; + float8net(val, buf); + feature->put(name, val); + break; + } + + case 25: //text + case 1043: //varchar + case 705: //literal + { + feature->put(name, tr_->transcode(buf)); + break; + } + + case 1042: //bpchar + { + std::string str = mapnik::util::trim_copy(buf); + feature->put(name, tr_->transcode(str.c_str())); + break; + } + + case 1700: //numeric + { + double val; + std::string str = numeric2string(buf); + if (mapnik::util::string2double(str, val)) + { + feature->put(name, val); + } + break; + } + + default: + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: Unknown type_oid=" << oid; + + break; + } + } + } + } + return feature; + } + return feature_ptr(); +} + + +pgraster_featureset::~pgraster_featureset() +{ + rs_->close(); +} + +std::string numeric2string(const char* buf) +{ + boost::int16_t ndigits = int2net(buf); + boost::int16_t weight = int2net(buf+2); + boost::int16_t sign = int2net(buf+4); + boost::int16_t dscale = int2net(buf+6); + + boost::scoped_array digits(new boost::int16_t[ndigits]); + for (int n=0; n < ndigits ;++n) + { + digits[n] = int2net(buf+8+n*2); + } + + std::ostringstream ss; + + if (sign == 0x4000) ss << "-"; + + int i = std::max(weight,boost::int16_t(0)); + int d = 0; + + // Each numeric "digit" is actually a value between 0000 and 9999 stored in a 16 bit field. + // For example, the number 1234567809990001 is stored as four digits: [1234] [5678] [999] [1]. + // Note that the last two digits show that the leading 0's are lost when the number is split. + // We must be careful to re-insert these 0's when building the string. + + while ( i >= 0) + { + if (i <= weight && d < ndigits) + { + // All digits after the first must be padded to make the field 4 characters long + if (d != 0) + { +#ifdef _WINDOWS + int dig = digits[d]; + if (dig < 10) + { + ss << "000"; // 0000 - 0009 + } + else if (dig < 100) + { + ss << "00"; // 0010 - 0099 + } + else + { + ss << "0"; // 0100 - 0999; + } +#else + switch(digits[d]) + { + case 0 ... 9: + ss << "000"; // 0000 - 0009 + break; + case 10 ... 99: + ss << "00"; // 0010 - 0099 + break; + case 100 ... 999: + ss << "0"; // 0100 - 0999 + break; + } +#endif + } + ss << digits[d++]; + } + else + { + if (d == 0) + ss << "0"; + else + ss << "0000"; + } + + i--; + } + if (dscale > 0) + { + ss << '.'; + // dscale counts the number of decimal digits following the point, not the numeric digits + while (dscale > 0) + { + int value; + if (i <= weight && d < ndigits) + value = digits[d++]; + else + value = 0; + + // Output up to 4 decimal digits for this value + if (dscale > 0) { + ss << (value / 1000); + value %= 1000; + dscale--; + } + if (dscale > 0) { + ss << (value / 100); + value %= 100; + dscale--; + } + if (dscale > 0) { + ss << (value / 10); + value %= 10; + dscale--; + } + if (dscale > 0) { + ss << value; + dscale--; + } + + i--; + } + } + return ss.str(); +} diff --git a/plugins/input/pgraster/pgraster_featureset.hpp b/plugins/input/pgraster/pgraster_featureset.hpp new file mode 100644 index 000000000..b4652bf48 --- /dev/null +++ b/plugins/input/pgraster/pgraster_featureset.hpp @@ -0,0 +1,71 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#ifndef PGRASTER_FEATURESET_HPP +#define PGRASTER_FEATURESET_HPP + +// mapnik +#include +#include +#include +#include + +// boost +#include + +using mapnik::Featureset; +using mapnik::box2d; +using mapnik::feature_ptr; +using mapnik::raster_ptr; +using mapnik::transcoder; +using mapnik::context_ptr; + +class IResultSet; + +class pgraster_featureset : public mapnik::Featureset +{ +public: + /// @param bandno band number (1-based). 0 (default) reads all bands. + /// Anything else forces interpretation of colors off + /// (values copied verbatim) + pgraster_featureset(boost::shared_ptr const& rs, + context_ptr const& ctx, + std::string const& encoding, + bool key_field = false, + int bandno = 0); + feature_ptr next(); + ~pgraster_featureset(); + +private: + boost::shared_ptr rs_; + context_ptr ctx_; + boost::scoped_ptr tr_; + mapnik::value_integer feature_id_; + bool key_field_; + int band_; +}; + +#endif // PGRASTER_FEATURESET_HPP diff --git a/plugins/input/pgraster/pgraster_wkb_reader.cpp b/plugins/input/pgraster/pgraster_wkb_reader.cpp new file mode 100644 index 000000000..f61d27ba6 --- /dev/null +++ b/plugins/input/pgraster/pgraster_wkb_reader.cpp @@ -0,0 +1,497 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#include "pgraster_wkb_reader.hpp" + +// mapnik +#include // for datasource_exception +#include +#include +#include +#include +#include +#include +#include +#include // for box2d + +// boost +#include // for boost::int8_t +#include +#include + +namespace { + +uint8_t +read_uint8(const uint8_t** from) { + return *(*from)++; +} + +uint16_t +read_uint16(const boost::uint8_t** from, boost::uint8_t littleEndian) { + uint16_t ret = 0; + + if (littleEndian) { + ret = (*from)[0] | + (*from)[1] << 8; + } else { + /* big endian */ + ret = (*from)[0] << 8 | + (*from)[1]; + } + *from += 2; + return ret; +} + +int16_t +read_int16(const uint8_t** from, uint8_t littleEndian) { + assert(NULL != from); + + return read_uint16(from, littleEndian); +} + +double +read_float64(const boost::uint8_t** from, boost::uint8_t littleEndian) { + + union { + double d; + uint64_t i; + } ret; + + if (littleEndian) { + ret.i = (uint64_t) ((*from)[0] & 0xff) | + (uint64_t) ((*from)[1] & 0xff) << 8 | + (uint64_t) ((*from)[2] & 0xff) << 16 | + (uint64_t) ((*from)[3] & 0xff) << 24 | + (uint64_t) ((*from)[4] & 0xff) << 32 | + (uint64_t) ((*from)[5] & 0xff) << 40 | + (uint64_t) ((*from)[6] & 0xff) << 48 | + (uint64_t) ((*from)[7] & 0xff) << 56; + } else { + /* big endian */ + ret.i = (uint64_t) ((*from)[7] & 0xff) | + (uint64_t) ((*from)[6] & 0xff) << 8 | + (uint64_t) ((*from)[5] & 0xff) << 16 | + (uint64_t) ((*from)[4] & 0xff) << 24 | + (uint64_t) ((*from)[3] & 0xff) << 32 | + (uint64_t) ((*from)[2] & 0xff) << 40 | + (uint64_t) ((*from)[1] & 0xff) << 48 | + (uint64_t) ((*from)[0] & 0xff) << 56; + } + + *from += 8; + return ret.d; +} + +uint32_t +read_uint32(const boost::uint8_t** from, boost::uint8_t littleEndian) { + uint32_t ret = 0; + + if (littleEndian) { + ret = (uint32_t) ((*from)[0] & 0xff) | + (uint32_t) ((*from)[1] & 0xff) << 8 | + (uint32_t) ((*from)[2] & 0xff) << 16 | + (uint32_t) ((*from)[3] & 0xff) << 24; + } else { + /* big endian */ + ret = (uint32_t) ((*from)[3] & 0xff) | + (uint32_t) ((*from)[2] & 0xff) << 8 | + (uint32_t) ((*from)[1] & 0xff) << 16 | + (uint32_t) ((*from)[0] & 0xff) << 24; + } + + *from += 4; + return ret; +} + +int32_t +read_int32(const boost::uint8_t** from, boost::uint8_t littleEndian) { + + return read_uint32(from, littleEndian); +} + +float +read_float32(const uint8_t** from, uint8_t littleEndian) { + + union { + float f; + uint32_t i; + } ret; + + ret.i = read_uint32(from, littleEndian); + + return ret.f; +} + + +typedef enum { + PT_1BB=0, /* 1-bit boolean */ + PT_2BUI=1, /* 2-bit unsigned integer */ + PT_4BUI=2, /* 4-bit unsigned integer */ + PT_8BSI=3, /* 8-bit signed integer */ + PT_8BUI=4, /* 8-bit unsigned integer */ + PT_16BSI=5, /* 16-bit signed integer */ + PT_16BUI=6, /* 16-bit unsigned integer */ + PT_32BSI=7, /* 32-bit signed integer */ + PT_32BUI=8, /* 32-bit unsigned integer */ + PT_32BF=10, /* 32-bit float */ + PT_64BF=11, /* 64-bit float */ + PT_END=13 +} rt_pixtype; + +#define BANDTYPE_FLAGS_MASK 0xF0 +#define BANDTYPE_PIXTYPE_MASK 0x0F +#define BANDTYPE_FLAG_OFFDB (1<<7) +#define BANDTYPE_FLAG_HASNODATA (1<<6) +#define BANDTYPE_FLAG_ISNODATA (1<<5) +#define BANDTYPE_FLAG_RESERVED3 (1<<4) + +#define BANDTYPE_PIXTYPE_MASK 0x0F +#define BANDTYPE_PIXTYPE(x) ((x)&BANDTYPE_PIXTYPE_MASK) +#define BANDTYPE_IS_OFFDB(x) ((x)&BANDTYPE_FLAG_OFFDB) +#define BANDTYPE_HAS_NODATA(x) ((x)&BANDTYPE_FLAG_HASNODATA) +#define BANDTYPE_IS_NODATA(x) ((x)&BANDTYPE_FLAG_ISNODATA) + +} + +using mapnik::box2d; + +template +void read_data_band(mapnik::raster_ptr raster, + uint16_t width, uint16_t height, + bool hasnodata, T reader) +{ + mapnik::image_data_32 & image = raster->data_; + + // Start with plain white (ABGR or RGBA depending on endiannes) + // TODO: set to transparent instead? + image.set(0xffffffff); + + raster->premultiplied_alpha_ = true; + + float* data = (float*)image.getBytes(); + double val; + val = reader(); // nodata value, need to read anyway + if ( hasnodata ) raster->set_nodata(val); + for (int y=0; ydata_; + + // Start with all zeroes + image.set(0); + + uint8_t type = read_uint8(&ptr_); + + int pixtype = BANDTYPE_PIXTYPE(type); + int offline = BANDTYPE_IS_OFFDB(type) ? 1 : 0; + int hasnodata = BANDTYPE_HAS_NODATA(type) ? 1 : 0; + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_wkb_reader: band type:" + << pixtype << " offline:" << offline + << " hasnodata:" << hasnodata; + + if ( offline ) { + MAPNIK_LOG_WARN(pgraster) << "pgraster_wkb_reader: offline band " + " unsupported"; + return; + } + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_wkb_reader: reading " << height_ << "x" << width_ << " pixels"; + + switch (pixtype) { + case PT_1BB: + case PT_2BUI: + case PT_4BUI: + // all <8BPP values are wrote in full bytes anyway + case PT_8BSI: + // mapnik does not support signed anyway + case PT_8BUI: + read_data_band(raster, width_, height_, hasnodata, + boost::bind(read_uint8, &ptr_)); + break; + case PT_16BSI: + // mapnik does not support signed anyway + case PT_16BUI: + read_data_band(raster, width_, height_, hasnodata, + boost::bind(read_uint16, &ptr_, endian_)); + break; + case PT_32BSI: + // mapnik does not support signed anyway + case PT_32BUI: + read_data_band(raster, width_, height_, hasnodata, + boost::bind(read_uint32, &ptr_, endian_)); + break; + case PT_32BF: + read_data_band(raster, width_, height_, hasnodata, + boost::bind(read_float32, &ptr_, endian_)); + break; + case PT_64BF: + read_data_band(raster, width_, height_, hasnodata, + boost::bind(read_float64, &ptr_, endian_)); + break; + default: + std::ostringstream err; + err << "pgraster_wkb_reader: data band type " << pixtype << " unsupported"; + // TODO: accept policy to decide on throw-or-skip ? + //MAPNIK_LOG_WARN(pgraster) << err.str(); + throw mapnik::datasource_exception(err.str()); + } + +} + +template +void read_grayscale_band(mapnik::raster_ptr raster, + uint16_t width, uint16_t height, + bool hasnodata, T reader) +{ + mapnik::image_data_32 & image = raster->data_; + + // Start with plain white (ABGR or RGBA depending on endiannes) + // TODO: set to transparent instead? + image.set(0xffffffff); + + raster->premultiplied_alpha_ = true; + + int val; + uint8_t * data = image.getBytes(); + int ps = 4; // sizeof(image_data::pixel_type) + int off; + val = reader(); // nodata value, need to read anyway + if ( hasnodata ) raster->set_nodata(val); + for (int y=0; ydata_; + + // Start with plain white (ABGR or RGBA depending on endiannes) + image.set(0xffffffff); + //raster->set_nodata(0xffffffff); + + raster->premultiplied_alpha_ = true; + + uint8_t nodataval; + for (int bn=0; bn PT_8BUI || pixtype < PT_8BSI ) { + MAPNIK_LOG_WARN(pgraster) << "pgraster_wkb_reader: band " << bn + << " type " << type << " unsupported"; + continue; + } + + uint8_t tmp = read_uint8(&ptr_); + if ( ! bn ) nodataval = tmp; + else if ( tmp != nodataval ) { + MAPNIK_LOG_WARN(pgraster) << "pgraster_wkb_reader: band " << bn + << " nodataval " << tmp << " != band 0 nodataval " << nodataval; + } + + int ps = 4; // sizeof(image_data::pixel_type) + uint8_t * image_data = image.getBytes(); + for (int y=0; y ext(ipX,ipY,ipX+(width_*scaleX),ipY+(height_*scaleY)); + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_wkb_reader: Raster extent=" << ext; + + mapnik::raster_ptr raster = boost::make_shared(ext, width_, height_); + + if ( bandno_ ) { + if ( bandno_ != 1 ) { + MAPNIK_LOG_WARN(pgraster) << "pgraster_wkb_reader: " + "reading bands other than 1st as indexed is unsupported"; + return mapnik::raster_ptr(); + } + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_wkb_reader: requested band " << bandno_; + read_indexed(raster); + } + else { + switch (numBands_) { + case 1: + read_grayscale(raster); + break; + case 3: + case 4: + read_rgba(raster); + break; + default: + std::ostringstream err; + err << "pgraster_wkb_reader: raster with " + << numBands_ + << " bands is not supported, specify a band number"; + //MAPNIK_LOG_WARN(pgraster) << err.str(); + throw mapnik::datasource_exception(err.str()); + return mapnik::raster_ptr(); + } + } + + return raster; + +} + + diff --git a/plugins/input/pgraster/pgraster_wkb_reader.hpp b/plugins/input/pgraster/pgraster_wkb_reader.hpp new file mode 100644 index 000000000..3a571ffd5 --- /dev/null +++ b/plugins/input/pgraster/pgraster_wkb_reader.hpp @@ -0,0 +1,87 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#ifndef PGRASTER_WKB_READER_HPP +#define PGRASTER_WKB_READER_HPP + +// mapnik +#include // for raster_ptr + +// boost +#include // for boost::uint8_t + +enum pgraster_color_interp { + // Automatic color interpretation: + // uses grayscale for single band, rgb for 3 bands + // rgba for 4 bands + pgr_auto, + // Grayscale interpretation + pgr_grayscale, + pgr_indexed, + pgr_rgb, + pgr_rgba +}; + +class pgraster_wkb_reader +{ +public: + + pgraster_wkb_reader(const uint8_t* wkb, int size, int bnd=0) + : wkbsize_(size), wkb_(wkb), wkbend_(wkb+size), ptr_(wkb), bandno_(bnd) + {} + + mapnik::raster_ptr get_raster(); + + /// @param bnd band number. If 0 (default) it'll try to read all bands + /// with automatic color interpretation (rgb for 3 bands, + /// rgba for 4 bands, grayscale for 1 band). + /// Any other value results in pixel + /// values being copied verbatim into the returned raster + /// for interpretation by the caller. + static mapnik::raster_ptr read(const uint8_t* wkb, int size, int bnd=0) + { + pgraster_wkb_reader reader(wkb,size,bnd); + return reader.get_raster(); + } + +private: + void read_indexed(mapnik::raster_ptr raster); + void read_grayscale(mapnik::raster_ptr raster); + void read_rgba(mapnik::raster_ptr raster); + + int wkbsize_; + const uint8_t* wkb_; + const uint8_t* wkbend_; + const uint8_t* ptr_; + uint8_t endian_; + int bandno_; + uint16_t numBands_; + uint16_t width_; + uint16_t height_; +}; + + +#endif // PGRASTER_WKB_READER_HPP diff --git a/tests/python_tests/pgraster_test.py b/tests/python_tests/pgraster_test.py new file mode 100644 index 000000000..f958b55b4 --- /dev/null +++ b/tests/python_tests/pgraster_test.py @@ -0,0 +1,747 @@ +#!/usr/bin/env python + +from nose.tools import * +import atexit +import cProfile, pstats, io +import time +from utilities import execution_path, run_all +from subprocess import Popen, PIPE +import os, mapnik +from Queue import Queue +import threading +import sys +import re +from binascii import hexlify, unhexlify + + +MAPNIK_TEST_DBNAME = 'mapnik-tmp-pgraster-test-db' +POSTGIS_TEMPLATE_DBNAME = 'template_postgis' + +def setup(): + # All of the paths used are relative, if we run the tests + # from another directory we need to chdir() + os.chdir(execution_path('.')) + +def call(cmd,silent=False): + proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + stdout, stderr = proc.communicate() + if stderr and not silent: + print stderr.strip() + if proc.returncode: + raise RuntimeError(stderr.strip()) + +def psql_can_connect(): + """Test ability to connect to a postgis template db with no options. + + Basically, to run these tests your user must have full read + access over unix sockets without supplying a password. This + keeps these tests simple and focused on postgis not on postgres + auth issues. + """ + try: + call('psql %s -c "select postgis_version()"' % POSTGIS_TEMPLATE_DBNAME) + return True + except RuntimeError, e: + print 'Notice: skipping postgis tests (connection)' + return False + +def psql_run(cmd): + cmd = 'psql --set ON_ERROR_STOP=1 %s -c "%s"' % \ + (MAPNIK_TEST_DBNAME, cmd.replace('"', '\\"')) + print 'DEBUG: running ' + cmd + call(cmd) + +def raster2pgsql_on_path(): + """Test for presence of raster2pgsql on the user path. + + We require this program to load test data into a temporarily database. + """ + try: + call('raster2pgsql') + return True + except RuntimeError, e: + print 'Notice: skipping postgis tests (raster2pgsql)' + return False + +def createdb_and_dropdb_on_path(): + """Test for presence of dropdb/createdb on user path. + + We require these programs to setup and teardown the testing db. + """ + try: + call('createdb --help') + call('dropdb --help') + return True + except RuntimeError, e: + print 'Notice: skipping postgis tests (createdb/dropdb)' + return False + +def postgis_setup(): + call('dropdb %s' % MAPNIK_TEST_DBNAME,silent=True) + call('createdb -T %s %s' % (POSTGIS_TEMPLATE_DBNAME,MAPNIK_TEST_DBNAME),silent=False) + +def postgis_takedown(): + pass + # fails as the db is in use: https://github.com/mapnik/mapnik/issues/960 + #call('dropdb %s' % MAPNIK_TEST_DBNAME) + +def import_raster(filename, tabname, tilesize, constraint, overview): + print 'tile: ' + tilesize + ' constraints: ' + str(constraint) \ + + ' overviews: ' + overview + cmd = 'raster2pgsql -Y -I -q' + if constraint: + cmd += ' -C' + if tilesize: + cmd += ' -t ' + tilesize + if overview: + cmd += ' -l ' + overview + cmd += ' %s %s | psql --set ON_ERROR_STOP=1 -q %s' % (filename,tabname,MAPNIK_TEST_DBNAME) + print 'Import call: ' + cmd + call(cmd) + +def drop_imported(tabname, overview): + psql_run('DROP TABLE IF EXISTS "' + tabname + '";') + if overview: + for of in overview.split(','): + psql_run('DROP TABLE IF EXISTS "o_' + of + '_' + tabname + '";') + +if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ + and createdb_and_dropdb_on_path() \ + and psql_can_connect() \ + and raster2pgsql_on_path(): + + # initialize test database + postgis_setup() + + # dataraster.tif, 2283x1913 int16 single-band + def _test_dataraster_16bsi_rendering(lbl, overview, rescale, clip): + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME,table='"dataRaster"', + band=1,use_overviews=1 if overview else 0, + prescale_rasters=rescale,clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['rid'],1) + lyr = mapnik.Layer('dataraster_16bsi') + lyr.datasource = ds + expenv = mapnik.Box2d(-14637, 3903178, 1126863, 4859678) + env = lyr.envelope() + # As the input size is a prime number both horizontally + # and vertically, we expect the extent of the overview + # tables to be a pixel wider than the original, whereas + # the pixel size in geographical units depends on the + # overview factor. So we start with the original pixel size + # as base scale and multiply by the overview factor. + # NOTE: the overview table extent only grows north and east + pixsize = 500 # see gdalinfo dataraster.tif + tol = pixsize * max(overview.split(',')) if overview else 0 + assert_almost_equal(env.minx, expenv.minx) + assert_almost_equal(env.miny, expenv.miny, delta=tol) + assert_almost_equal(env.maxx, expenv.maxx, delta=tol) + assert_almost_equal(env.maxy, expenv.maxy) + mm = mapnik.Map(256, 256) + style = mapnik.Style() + col = mapnik.RasterColorizer(); + col.default_mode = mapnik.COLORIZER_DISCRETE; + col.add_stop(0, mapnik.Color(0x40,0x40,0x40,255)); + col.add_stop(10, mapnik.Color(0x80,0x80,0x80,255)); + col.add_stop(20, mapnik.Color(0xa0,0xa0,0xa0,255)); + sym = mapnik.RasterSymbolizer() + sym.colorizer = col + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + # no data + eq_(im.view(1,1,1,1).tostring(), '\x00\x00\x00\x00') + eq_(im.view(255,255,1,1).tostring(), '\x00\x00\x00\x00') + eq_(im.view(195,116,1,1).tostring(), '\x00\x00\x00\x00') + # A0A0A0 + eq_(im.view(100,120,1,1).tostring(), '\xa0\xa0\xa0\xff') + eq_(im.view( 75, 80,1,1).tostring(), '\xa0\xa0\xa0\xff') + # 808080 + eq_(im.view( 74,170,1,1).tostring(), '\x80\x80\x80\xff') + eq_(im.view( 30, 50,1,1).tostring(), '\x80\x80\x80\xff') + # 404040 + eq_(im.view(190, 70,1,1).tostring(), '\x40\x40\x40\xff') + eq_(im.view(140,170,1,1).tostring(), '\x40\x40\x40\xff') + + # Now zoom over a portion of the env (1/10) + newenv = mapnik.Box2d(273663,4024478,330738,4072303) + mm.zoom_to_box(newenv) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:1/10' + # nodata + eq_(hexlify(im.view(255,255,1,1).tostring()), '00000000') + eq_(hexlify(im.view(200,254,1,1).tostring()), '00000000') + # A0A0A0 + eq_(hexlify(im.view(90,232,1,1).tostring()), 'a0a0a0ff') + eq_(hexlify(im.view(96,245,1,1).tostring()), 'a0a0a0ff') + # 808080 + eq_(hexlify(im.view(1,1,1,1).tostring()), '808080ff') + eq_(hexlify(im.view(128,128,1,1).tostring()), '808080ff') + # 404040 + eq_(hexlify(im.view(255, 0,1,1).tostring()), '404040ff') + + def _test_dataraster_16bsi(lbl, tilesize, constraint, overview): + rf = os.path.join(execution_path('.'),'../data/raster/dataraster.tif') + import_raster(rf, 'dataRaster', tilesize, constraint, overview) + if constraint: + lbl += ' C' + if tilesize: + lbl += ' T:' + tilesize + if overview: + lbl += ' O:' + overview + for prescale in [0,1]: + for clip in [0,1]: + _test_dataraster_16bsi_rendering(lbl, overview, prescale, clip) + drop_imported('dataRaster', overview) + + def test_dataraster_16bsi(): + for tilesize in ['','256x256']: + for constraint in [0,1]: + for overview in ['','4','2,16']: + _test_dataraster_16bsi('data_16bsi', tilesize, constraint, overview) + + # river.tiff, RGBA 8BUI + def _test_rgba_8bui_rendering(lbl, overview, rescale, clip): + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME,table='(select * from "River") foo', + use_overviews=1 if overview else 0, + prescale_rasters=rescale,clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['rid'],1) + lyr = mapnik.Layer('rgba_8bui') + lyr.datasource = ds + expenv = mapnik.Box2d(0, -210, 256, 0) + env = lyr.envelope() + # As the input size is a prime number both horizontally + # and vertically, we expect the extent of the overview + # tables to be a pixel wider than the original, whereas + # the pixel size in geographical units depends on the + # overview factor. So we start with the original pixel size + # as base scale and multiply by the overview factor. + # NOTE: the overview table extent only grows north and east + pixsize = 1 # see gdalinfo river.tif + tol = pixsize * max(overview.split(',')) if overview else 0 + assert_almost_equal(env.minx, expenv.minx) + assert_almost_equal(env.miny, expenv.miny, delta=tol) + assert_almost_equal(env.maxx, expenv.maxx, delta=tol) + assert_almost_equal(env.maxy, expenv.maxy) + mm = mapnik.Map(256, 256) + style = mapnik.Style() + sym = mapnik.RasterSymbolizer() + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + #im.save('/tmp/xfull.png') # for debugging + # no data + eq_(hexlify(im.view(3,3,1,1).tostring()), '00000000') + eq_(hexlify(im.view(250,250,1,1).tostring()), '00000000') + # full opaque river color + eq_(hexlify(im.view(175,118,1,1).tostring()), 'b9d8f8ff') + # half-transparent pixel + pxstr = hexlify(im.view(122,138,1,1).tostring()) + apat = ".*(..)$" + match = re.match(apat, pxstr) + assert match, 'pixel ' + pxstr + ' does not match pattern ' + apat + alpha = match.group(1) + assert alpha != 'ff' and alpha != '00', \ + 'unexpected full transparent/opaque pixel: ' + alpha + + # Now zoom over a portion of the env (1/10) + newenv = mapnik.Box2d(166,-105,191,-77) + mm.zoom_to_box(newenv) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:1/10' + #im.save('/tmp/xtenth.png') # for debugging + # no data + eq_(hexlify(im.view(255,255,1,1).tostring()), '00000000') + eq_(hexlify(im.view(200,40,1,1).tostring()), '00000000') + # full opaque river color + eq_(hexlify(im.view(100,168,1,1).tostring()), 'b9d8f8ff') + # half-transparent pixel + pxstr = hexlify(im.view(122,138,1,1).tostring()) + apat = ".*(..)$" + match = re.match(apat, pxstr) + assert match, 'pixel ' + pxstr + ' does not match pattern ' + apat + alpha = match.group(1) + assert alpha != 'ff' and alpha != '00', \ + 'unexpected full transparent/opaque pixel: ' + alpha + + def _test_rgba_8bui(lbl, tilesize, constraint, overview): + rf = os.path.join(execution_path('.'),'../data/raster/river.tiff') + import_raster(rf, 'River', tilesize, constraint, overview) + if constraint: + lbl += ' C' + if tilesize: + lbl += ' T:' + tilesize + if overview: + lbl += ' O:' + overview + for prescale in [0,1]: + for clip in [0,1]: + _test_rgba_8bui_rendering(lbl, overview, prescale, clip) + drop_imported('River', overview) + + def test_rgba_8bui(): + for tilesize in ['','16x16']: + for constraint in [0,1]: + for overview in ['2']: + _test_rgba_8bui('rgba_8bui', tilesize, constraint, overview) + + # nodata-edge.tif, RGB 8BUI + def _test_rgb_8bui_rendering(lbl, tnam, overview, rescale, clip): + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME,table=tnam, + use_overviews=1 if overview else 0, + prescale_rasters=rescale,clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['rid'],1) + lyr = mapnik.Layer('rgba_8bui') + lyr.datasource = ds + expenv = mapnik.Box2d(-12329035.7652168,4508650.39854396, \ + -12328653.0279471,4508957.34625536) + env = lyr.envelope() + # As the input size is a prime number both horizontally + # and vertically, we expect the extent of the overview + # tables to be a pixel wider than the original, whereas + # the pixel size in geographical units depends on the + # overview factor. So we start with the original pixel size + # as base scale and multiply by the overview factor. + # NOTE: the overview table extent only grows north and east + pixsize = 2 # see gdalinfo nodata-edge.tif + tol = pixsize * max(overview.split(',')) if overview else 0 + assert_almost_equal(env.minx, expenv.minx, places=0) + assert_almost_equal(env.miny, expenv.miny, delta=tol) + assert_almost_equal(env.maxx, expenv.maxx, delta=tol) + assert_almost_equal(env.maxy, expenv.maxy, places=0) + mm = mapnik.Map(256, 256) + style = mapnik.Style() + sym = mapnik.RasterSymbolizer() + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + #im.save('/tmp/xfull.png') # for debugging + # no data + eq_(hexlify(im.view(3,16,1,1).tostring()), '00000000') + eq_(hexlify(im.view(128,16,1,1).tostring()), '00000000') + eq_(hexlify(im.view(250,16,1,1).tostring()), '00000000') + eq_(hexlify(im.view(3,240,1,1).tostring()), '00000000') + eq_(hexlify(im.view(128,240,1,1).tostring()), '00000000') + eq_(hexlify(im.view(250,240,1,1).tostring()), '00000000') + # dark brown + eq_(hexlify(im.view(174,39,1,1).tostring()), 'c3a698ff') + # dark gray + eq_(hexlify(im.view(195,132,1,1).tostring()), '575f62ff') + # Now zoom over a portion of the env (1/10) + newenv = mapnik.Box2d(-12329035.7652168, 4508926.651484220, \ + -12328997.49148983,4508957.34625536) + mm.zoom_to_box(newenv) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:1/10' + #im.save('/tmp/xtenth.png') # for debugging + # no data + eq_(hexlify(im.view(3,16,1,1).tostring()), '00000000') + eq_(hexlify(im.view(128,16,1,1).tostring()), '00000000') + eq_(hexlify(im.view(250,16,1,1).tostring()), '00000000') + # black + eq_(hexlify(im.view(3,42,1,1).tostring()), '000000ff') + eq_(hexlify(im.view(3,134,1,1).tostring()), '000000ff') + eq_(hexlify(im.view(3,244,1,1).tostring()), '000000ff') + # gray + eq_(hexlify(im.view(135,157,1,1).tostring()), '4e555bff') + # brown + eq_(hexlify(im.view(195,223,1,1).tostring()), 'f2cdbaff') + + def _test_rgb_8bui(lbl, tilesize, constraint, overview): + rf = os.path.join(execution_path('.'),'../data/raster/nodata-edge.tif') + tnam = 'nodataedge' + import_raster(rf, tnam, tilesize, constraint, overview) + if constraint: + lbl += ' C' + if tilesize: + lbl += ' T:' + tilesize + if overview: + lbl += ' O:' + overview + for prescale in [0,1]: + for clip in [0,1]: + _test_rgb_8bui_rendering(lbl, tnam, overview, prescale, clip) + #drop_imported(tnam, overview) + + def test_rgb_8bui(): + for tilesize in ['64x64']: + for constraint in [1]: + for overview in ['']: + _test_rgb_8bui('rgb_8bui', tilesize, constraint, overview) + + def _test_grayscale_subquery(lbl,pixtype,value): + # + # 3 8 13 + # +---+---+---+ + # 3 | v | v | v | NOTE: writes different color + # +---+---+---+ in 13,8 and 8,13 + # 8 | v | v | a | + # +---+---+---+ + # 13 | v | b | v | + # +---+---+---+ + # + val_a = value/3; + val_b = val_a*2; + sql = "(select 3 as i, " \ + " ST_SetValues(" \ + " ST_SetValues(" \ + " ST_AsRaster(" \ + " ST_MakeEnvelope(0,0,14,14), " \ + " 1.0, -1.0, '%s', %s" \ + " ), " \ + " 11, 6, 4, 5, %s::float8" \ + " )," \ + " 6, 11, 5, 4, %s::float8" \ + " ) as \"R\"" \ + ") as foo" % (pixtype,value, val_a, val_b) + rescale = 0 + clip = 0 + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME, table=sql, + raster_field='"R"', use_overviews=1, + prescale_rasters=rescale,clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['i'],3) + lyr = mapnik.Layer('grayscale_subquery') + lyr.datasource = ds + expenv = mapnik.Box2d(0,0,14,14) + env = lyr.envelope() + assert_almost_equal(env.minx, expenv.minx, places=0) + assert_almost_equal(env.miny, expenv.miny, places=0) + assert_almost_equal(env.maxx, expenv.maxx, places=0) + assert_almost_equal(env.maxy, expenv.maxy, places=0) + mm = mapnik.Map(15, 15) + style = mapnik.Style() + sym = mapnik.RasterSymbolizer() + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + #im.save('/tmp/xfull.png') # for debugging + h = format(value, '02x') + hex_v = h+h+h+'ff' + h = format(val_a, '02x') + hex_a = h+h+h+'ff' + h = format(val_b, '02x') + hex_b = h+h+h+'ff' + eq_(hexlify(im.view( 3, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 3, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 8,1,1).tostring()), hex_a); + eq_(hexlify(im.view( 3,13,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8,13,1,1).tostring()), hex_b); + eq_(hexlify(im.view(13,13,1,1).tostring()), hex_v); + + def test_grayscale_2bui_subquery(): + _test_grayscale_subquery('grayscale_2bui_subquery', '2BUI', 3) + + def test_grayscale_4bui_subquery(): + _test_grayscale_subquery('grayscale_4bui_subquery', '4BUI', 15) + + def test_grayscale_8bui_subquery(): + _test_grayscale_subquery('grayscale_8bui_subquery', '8BUI', 63) + + def test_grayscale_8bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_grayscale_subquery('grayscale_8bsi_subquery', '8BSI', 69) + + def test_grayscale_16bui_subquery(): + _test_grayscale_subquery('grayscale_16bui_subquery', '16BUI', 126) + + def test_grayscale_16bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_grayscale_subquery('grayscale_16bsi_subquery', '16BSI', 144) + + def test_grayscale_32bui_subquery(): + _test_grayscale_subquery('grayscale_32bui_subquery', '32BUI', 255) + + def test_grayscale_32bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_grayscale_subquery('grayscale_32bsi_subquery', '32BSI', 129) + + def _test_data_subquery(lbl, pixtype, value): + # + # 3 8 13 + # +---+---+---+ + # 3 | v | v | v | NOTE: writes different values + # +---+---+---+ in 13,8 and 8,13 + # 8 | v | v | a | + # +---+---+---+ + # 13 | v | b | v | + # +---+---+---+ + # + val_a = value/3; + val_b = val_a*2; + sql = "(select 3 as i, " \ + " ST_SetValues(" \ + " ST_SetValues(" \ + " ST_AsRaster(" \ + " ST_MakeEnvelope(0,0,14,14), " \ + " 1.0, -1.0, '%s', %s" \ + " ), " \ + " 11, 6, 5, 5, %s::float8" \ + " )," \ + " 6, 11, 5, 5, %s::float8" \ + " ) as \"R\"" \ + ") as foo" % (pixtype,value, val_a, val_b) + overview = '' + rescale = 0 + clip = 0 + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME, table=sql, + raster_field='R', use_overviews=0 if overview else 0, + band=1, prescale_rasters=rescale, clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['i'],3) + lyr = mapnik.Layer('data_subquery') + lyr.datasource = ds + expenv = mapnik.Box2d(0,0,14,14) + env = lyr.envelope() + assert_almost_equal(env.minx, expenv.minx, places=0) + assert_almost_equal(env.miny, expenv.miny, places=0) + assert_almost_equal(env.maxx, expenv.maxx, places=0) + assert_almost_equal(env.maxy, expenv.maxy, places=0) + mm = mapnik.Map(15, 15) + style = mapnik.Style() + col = mapnik.RasterColorizer(); + col.default_mode = mapnik.COLORIZER_DISCRETE; + col.add_stop(val_a, mapnik.Color(0xff,0x00,0x00,255)); + col.add_stop(val_b, mapnik.Color(0x00,0xff,0x00,255)); + col.add_stop(value, mapnik.Color(0x00,0x00,0xff,255)); + sym = mapnik.RasterSymbolizer() + sym.colorizer = col + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + #im.save('/tmp/xfull.png') # for debugging + h = format(value, '02x') + hex_v = '0000ffff' + hex_a = 'ff0000ff' + hex_b = '00ff00ff' + eq_(hexlify(im.view( 3, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 3, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 8,1,1).tostring()), hex_a); + eq_(hexlify(im.view( 3,13,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8,13,1,1).tostring()), hex_b); + eq_(hexlify(im.view(13,13,1,1).tostring()), hex_v); + + def test_data_2bui_subquery(): + _test_data_subquery('data_2bui_subquery', '2BUI', 3) + + def test_data_4bui_subquery(): + _test_data_subquery('data_4bui_subquery', '4BUI', 15) + + def test_data_8bui_subquery(): + _test_data_subquery('data_8bui_subquery', '8BUI', 63) + + def test_data_8bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_data_subquery('data_8bsi_subquery', '8BSI', 69) + + def test_data_16bui_subquery(): + _test_data_subquery('data_16bui_subquery', '16BUI', 126) + + def test_data_16bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_data_subquery('data_16bsi_subquery', '16BSI', 135) + + def test_data_32bui_subquery(): + _test_data_subquery('data_32bui_subquery', '32BUI', 255) + + def test_data_32bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_data_subquery('data_32bsi_subquery', '32BSI', 264) + + def test_data_32bf_subquery(): + _test_data_subquery('data_32bf_subquery', '32BF', 450) + + def test_data_64bf_subquery(): + _test_data_subquery('data_64bf_subquery', '64BF', 3072) + + def _test_rgba_subquery(lbl, pixtype, r, g, b, a, g1, b1): + # + # 3 8 13 + # +---+---+---+ + # 3 | v | v | h | NOTE: writes different alpha + # +---+---+---+ in 13,8 and 8,13 + # 8 | v | v | a | + # +---+---+---+ + # 13 | v | b | v | + # +---+---+---+ + # + sql = "(select 3 as i, " \ + " ST_SetValues(" \ + " ST_SetValues(" \ + " ST_AddBand(" \ + " ST_AddBand(" \ + " ST_AddBand(" \ + " ST_AsRaster(" \ + " ST_MakeEnvelope(0,0,14,14), " \ + " 1.0, -1.0, '%s', %s" \ + " )," \ + " '%s', %d::float" \ + " ), " \ + " '%s', %d::float" \ + " ), " \ + " '%s', %d::float" \ + " ), " \ + " 2, 11, 6, 4, 5, %s::float8" \ + " )," \ + " 3, 6, 11, 5, 4, %s::float8" \ + " ) as r" \ + ") as foo" % (pixtype, r, pixtype, g, pixtype, b, pixtype, a, g1, b1) + overview = '' + rescale = 0 + clip = 0 + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME, table=sql, + raster_field='r', use_overviews=0 if overview else 0, + prescale_rasters=rescale, clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['i'],3) + lyr = mapnik.Layer('rgba_subquery') + lyr.datasource = ds + expenv = mapnik.Box2d(0,0,14,14) + env = lyr.envelope() + assert_almost_equal(env.minx, expenv.minx, places=0) + assert_almost_equal(env.miny, expenv.miny, places=0) + assert_almost_equal(env.maxx, expenv.maxx, places=0) + assert_almost_equal(env.maxy, expenv.maxy, places=0) + mm = mapnik.Map(15, 15) + style = mapnik.Style() + sym = mapnik.RasterSymbolizer() + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + im.save('/tmp/xfull.png') # for debugging + hex_v = format(r << 24 | g << 16 | b << 8 | a, '08x') + hex_a = format(r << 24 | g1 << 16 | b << 8 | a, '08x') + hex_b = format(r << 24 | g << 16 | b1 << 8 | a, '08x') + eq_(hexlify(im.view( 3, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 3, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 8,1,1).tostring()), hex_a); + eq_(hexlify(im.view( 3,13,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8,13,1,1).tostring()), hex_b); + eq_(hexlify(im.view(13,13,1,1).tostring()), hex_v); + + def test_rgba_8bui_subquery(): + _test_rgba_subquery('rgba_8bui_subquery', '8BUI', 255, 0, 0, 255, 255, 255) + + #def test_rgba_16bui_subquery(): + # _test_rgba_subquery('rgba_16bui_subquery', '16BUI', 65535, 0, 0, 65535, 65535, 65535) + + #def test_rgba_32bui_subquery(): + # _test_rgba_subquery('rgba_32bui_subquery', '32BUI') + + atexit.register(postgis_takedown) + +def enabled(tname): + enabled = len(sys.argv) < 2 or tname in sys.argv + if not enabled: + print "Skipping " + tname + " as not explicitly enabled" + return enabled + +if __name__ == "__main__": + setup() + fail = run_all(eval(x) for x in dir() if x.startswith("test_") and enabled(x)) + exit(fail)