sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add extra metadata to lyr desc + use for srid/keyfield in postgis - closes #2650

Browse source
This commit is contained in:
Dane Springmeyer 2015-01-22 18:44:56 -08:00
parent fcdc826294
commit ec111c8a27
5 changed files with 177 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
bindings/python/mapnik_datasource.cpp
View file

@ -107,6 +107,9 @@ boost::python::dict describe(std::shared_ptr<mapnik::datasource> const& ds)
description["name"] = ld.get_name(); description["name"] = ld.get_name();
description["geometry_type"] = ds->get_geometry_type(); description["geometry_type"] = ds->get_geometry_type();
description["encoding"] = ld.get_encoding(); description["encoding"] = ld.get_encoding();
for (auto const& param : ld.get_extra_parameters()) {
description[param.first] = param.second;
}
return description; return description;
} }

12
include/mapnik/attribute_descriptor.hpp
View file

@ -46,16 +46,16 @@ public:
int precision=-1) int precision=-1)
: name_(name), : name_(name),
type_(type), type_(type),
primary_key_(primary_key),
size_(size), size_(size),
precision_(precision) {} precision_(precision),
primary_key_(primary_key) {}
attribute_descriptor(attribute_descriptor const& other) attribute_descriptor(attribute_descriptor const& other)
: name_(other.name_), : name_(other.name_),
type_(other.type_), type_(other.type_),
primary_key_(other.primary_key_),
size_(other.size_), size_(other.size_),
precision_(other.precision_) {} precision_(other.precision_),
primary_key_(other.primary_key_) {}
attribute_descriptor& operator=(attribute_descriptor const& other) attribute_descriptor& operator=(attribute_descriptor const& other)
{ {
@ -67,9 +67,9 @@ public:
{ {
name_=other.name_; name_=other.name_;
type_=other.type_; type_=other.type_;
primary_key_=other.primary_key_;
size_=other.size_; size_=other.size_;
precision_=other.precision_; precision_=other.precision_;
primary_key_=other.primary_key_;
return *this; return *this;
} }
} }
@ -102,9 +102,9 @@ public:
private: private:
std::string name_; std::string name_;
unsigned int type_; unsigned int type_;
bool primary_key_;
int size_; int size_;
int precision_; int precision_;
bool primary_key_;
}; };
} }

18
include/mapnik/feature_layer_desc.hpp
View file

@ -25,6 +25,7 @@
// mapnik // mapnik
#include <mapnik/attribute_descriptor.hpp> #include <mapnik/attribute_descriptor.hpp>
#include <mapnik/params.hpp>
// stl // stl
#include <iosfwd> #include <iosfwd>
@ -39,12 +40,14 @@ public:
layer_descriptor(std::string const& name, std::string const& encoding) layer_descriptor(std::string const& name, std::string const& encoding)
: name_(name), : name_(name),
encoding_(encoding), encoding_(encoding),
desc_ar_() {} desc_ar_(),
extra_params_() {}
layer_descriptor(layer_descriptor const& other) layer_descriptor(layer_descriptor const& other)
: name_(other.name_), : name_(other.name_),
encoding_(other.encoding_), encoding_(other.encoding_),
desc_ar_(other.desc_ar_) {} desc_ar_(other.desc_ar_),
extra_params_(other.extra_params_) {}
void set_name(std::string const& name) void set_name(std::string const& name)
{ {
@ -81,10 +84,21 @@ public:
return desc_ar_; return desc_ar_;
} }
parameters const& get_extra_parameters() const
{
return extra_params_;
}
parameters& get_extra_parameters()
{
return extra_params_;
}
private: private:
std::string name_; std::string name_;
std::string encoding_; std::string encoding_;
std::vector<attribute_descriptor> desc_ar_; std::vector<attribute_descriptor> desc_ar_;
parameters extra_params_;
}; };
} }

8
plugins/input/postgis/postgis_datasource.cpp
View file

@ -445,6 +445,14 @@ postgis_datasource::postgis_datasource(parameters const& params)
// Close explicitly the connection so we can 'fork()' without sharing open connections // Close explicitly the connection so we can 'fork()' without sharing open connections
conn->close(); conn->close();
// Finally, add unique metadata to layer descriptor
mapnik::parameters & extra_params = desc_.get_extra_parameters();
// explicitly make copies of values due to https://github.com/mapnik/mapnik/issues/2651
extra_params["srid"] = int(srid_);
if (!key_field_.empty())
{
extra_params["key_field"] = std::string(key_field_);
}
} }
} }

152
tests/python_tests/postgis_test.py
View file

@ -245,7 +245,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(feature['subregion'],29) eq_(feature['subregion'],29)
eq_(feature['lon'],-61.783) eq_(feature['lon'],-61.783)
eq_(feature['lat'],17.078) eq_(feature['lat'],17.078)
eq_(ds.describe()['geometry_type'],mapnik.DataGeometryType.Polygon) meta = ds.describe()
eq_(meta['srid'],3857)
eq_(meta.get('key_field'),None)
eq_(meta['encoding'],u'UTF8')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Polygon)
def test_subquery(): def test_subquery():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='(select * from world_merc) as w') ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='(select * from world_merc) as w')
@ -263,7 +267,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(feature['subregion'],29) eq_(feature['subregion'],29)
eq_(feature['lon'],-61.783) eq_(feature['lon'],-61.783)
eq_(feature['lat'],17.078) eq_(feature['lat'],17.078)
eq_(ds.describe()['geometry_type'],mapnik.DataGeometryType.Polygon) meta = ds.describe()
eq_(meta['srid'],3857)
eq_(meta.get('key_field'),None)
eq_(meta['encoding'],u'UTF8')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Polygon)
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='(select gid,geom,fips as _fips from world_merc) as w') ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='(select gid,geom,fips as _fips from world_merc) as w')
fs = ds.featureset() fs = ds.featureset()
@ -271,7 +279,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(feature['gid'],1) eq_(feature['gid'],1)
eq_(feature['_fips'],u'AC') eq_(feature['_fips'],u'AC')
eq_(len(feature),2) eq_(len(feature),2)
eq_(ds.describe()['geometry_type'],mapnik.DataGeometryType.Polygon) meta = ds.describe()
eq_(meta['srid'],3857)
eq_(meta.get('key_field'),None)
eq_(meta['encoding'],u'UTF8')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Polygon)
def test_empty_db(): def test_empty_db():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='empty') ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='empty')
@ -282,12 +294,34 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
except StopIteration: except StopIteration:
pass pass
eq_(feature,None) eq_(feature,None)
eq_(ds.describe()['geometry_type'],None) meta = ds.describe()
eq_(meta['srid'],-1)
eq_(meta.get('key_field'),None)
eq_(meta['encoding'],u'UTF8')
eq_(meta['geometry_type'],None)
def test_manual_srid():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,srid=99, table='empty')
fs = ds.featureset()
feature = None
try:
feature = fs.next()
except StopIteration:
pass
eq_(feature,None)
meta = ds.describe()
eq_(meta['srid'],99)
eq_(meta.get('key_field'),None)
eq_(meta['encoding'],u'UTF8')
eq_(meta['geometry_type'],None)
def test_geometry_detection(): def test_geometry_detection():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test',
geometry_field='geom') geometry_field='geom')
eq_(ds.describe()['geometry_type'],mapnik.DataGeometryType.Collection) meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),None)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Collection)
# will fail with postgis 2.0 because it automatically adds a geometry_columns entry # will fail with postgis 2.0 because it automatically adds a geometry_columns entry
#ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test', #ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test',
@ -327,6 +361,10 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(fs.next().id(),-1000) eq_(fs.next().id(),-1000)
eq_(fs.next().id(),2147483647) eq_(fs.next().id(),2147483647)
eq_(fs.next().id(),-2147483648) eq_(fs.next().id(),-2147483648)
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),u'manual_id')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_auto_detection_will_fail_since_no_primary_key(): def test_auto_detection_will_fail_since_no_primary_key():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test3', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test3',
@ -353,6 +391,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(fs.next().id(),5) eq_(fs.next().id(),5)
eq_(fs.next().id(),6) eq_(fs.next().id(),6)
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),None)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
@raises(RuntimeError) @raises(RuntimeError)
def test_auto_detection_will_fail_and_should_throw(): def test_auto_detection_will_fail_and_should_throw():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test3', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test3',
@ -380,6 +423,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(fs.next().id(),2147483647) eq_(fs.next().id(),2147483647)
eq_(fs.next().id(),-2147483648) eq_(fs.next().id(),-2147483648)
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),u'manual_id')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_disabled_auto_detection_and_subquery(): def test_disabled_auto_detection_and_subquery():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='''(select geom, 'a'::varchar as name from test2) as t''', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='''(select geom, 'a'::varchar as name from test2) as t''',
geometry_field='geom', geometry_field='geom',
@ -404,6 +452,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(feat.id(),6) eq_(feat.id(),6)
eq_(feat['name'],'a') eq_(feat['name'],'a')
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),None)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_auto_detection_and_subquery_including_key(): def test_auto_detection_and_subquery_including_key():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='''(select geom, manual_id from test2) as t''', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='''(select geom, manual_id from test2) as t''',
geometry_field='geom', geometry_field='geom',
@ -424,6 +477,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(fs.next().id(),2147483647) eq_(fs.next().id(),2147483647)
eq_(fs.next().id(),-2147483648) eq_(fs.next().id(),-2147483648)
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),u'manual_id')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
@raises(RuntimeError) @raises(RuntimeError)
def test_auto_detection_of_invalid_numeric_primary_key(): def test_auto_detection_of_invalid_numeric_primary_key():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='''(select geom, manual_id::numeric from test2) as t''', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='''(select geom, manual_id::numeric from test2) as t''',
@ -463,6 +521,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(fs.next().id(),2147483647) eq_(fs.next().id(),2147483647)
eq_(fs.next().id(),-2147483648) eq_(fs.next().id(),-2147483648)
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),u'manual_id')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_numeric_type_feature_id_field(): def test_numeric_type_feature_id_field():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test5', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test5',
geometry_field='geom', geometry_field='geom',
@ -475,6 +538,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(fs.next().id(),1) eq_(fs.next().id(),1)
eq_(fs.next().id(),2) eq_(fs.next().id(),2)
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),None)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_querying_table_with_mixed_case(): def test_querying_table_with_mixed_case():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='"tableWithMixedCase"', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='"tableWithMixedCase"',
geometry_field='geom', geometry_field='geom',
@ -483,6 +551,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
for id in range(1,5): for id in range(1,5):
eq_(fs.next().id(),id) eq_(fs.next().id(),id)
meta = ds.describe()
eq_(meta['srid'],-1)
eq_(meta.get('key_field'),u'gid')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_querying_subquery_with_mixed_case(): def test_querying_subquery_with_mixed_case():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='(SeLeCt * FrOm "tableWithMixedCase") as MixedCaseQuery', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='(SeLeCt * FrOm "tableWithMixedCase") as MixedCaseQuery',
geometry_field='geom', geometry_field='geom',
@ -491,6 +564,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
for id in range(1,5): for id in range(1,5):
eq_(fs.next().id(),id) eq_(fs.next().id(),id)
meta = ds.describe()
eq_(meta['srid'],-1)
eq_(meta.get('key_field'),u'gid')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_bbox_token_in_subquery1(): def test_bbox_token_in_subquery1():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table=''' ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='''
(SeLeCt * FrOm "tableWithMixedCase" where geom && !bbox! ) as MixedCaseQuery''', (SeLeCt * FrOm "tableWithMixedCase" where geom && !bbox! ) as MixedCaseQuery''',
@ -500,6 +578,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
for id in range(1,5): for id in range(1,5):
eq_(fs.next().id(),id) eq_(fs.next().id(),id)
meta = ds.describe()
eq_(meta['srid'],-1)
eq_(meta.get('key_field'),u'gid')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_bbox_token_in_subquery2(): def test_bbox_token_in_subquery2():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table=''' ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='''
(SeLeCt * FrOm "tableWithMixedCase" where ST_Intersects(geom,!bbox!) ) as MixedCaseQuery''', (SeLeCt * FrOm "tableWithMixedCase" where ST_Intersects(geom,!bbox!) ) as MixedCaseQuery''',
@ -509,12 +592,22 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
for id in range(1,5): for id in range(1,5):
eq_(fs.next().id(),id) eq_(fs.next().id(),id)
meta = ds.describe()
eq_(meta['srid'],-1)
eq_(meta.get('key_field'),u'gid')
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_empty_geom(): def test_empty_geom():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test7', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test7',
geometry_field='geom') geometry_field='geom')
fs = ds.featureset() fs = ds.featureset()
eq_(fs.next()['gid'],1) eq_(fs.next()['gid'],1)
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),None)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Collection)
def create_ds(): def create_ds():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME, ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,
table='test', table='test',
@ -523,6 +616,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
fs = ds.all_features() fs = ds.all_features()
eq_(len(fs),8) eq_(len(fs),8)
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),None)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Collection)
def test_threaded_create(NUM_THREADS=100): def test_threaded_create(NUM_THREADS=100):
# run one to start before thread loop # run one to start before thread loop
# to ensure that a throw stops the test # to ensure that a throw stops the test
@ -568,6 +666,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(feat['gid'],2) eq_(feat['gid'],2)
eq_(feat['int_field'],922337203685477580) eq_(feat['int_field'],922337203685477580)
meta = ds.describe()
eq_(meta['srid'],-1)
eq_(meta.get('key_field'),None)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_persist_connection_off(): def test_persist_connection_off():
# NOTE: max_size should be equal or greater than # NOTE: max_size should be equal or greater than
# the pool size. There's currently no API to # the pool size. There's currently no API to
@ -576,19 +679,29 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
# http://github.com/mapnik/mapnik/issues/863 # http://github.com/mapnik/mapnik/issues/863
max_size = 20 max_size = 20
for i in range(0, max_size+1): for i in range(0, max_size+1):
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME, ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,
max_size=1, # unused max_size=1, # unused
persist_connection=False, persist_connection=False,
table='(select ST_MakePoint(0,0) as g, pg_backend_pid() as p, 1 as v) as w', table='(select ST_MakePoint(0,0) as g, pg_backend_pid() as p, 1 as v) as w',
geometry_field='g') geometry_field='g')
fs = ds.featureset() fs = ds.featureset()
eq_(fs.next()['v'], 1) eq_(fs.next()['v'], 1)
meta = ds.describe()
eq_(meta['srid'],-1)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
def test_null_comparision(): def test_null_comparision():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test9', ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test9',
geometry_field='geom') geometry_field='geom')
fs = ds.featureset() fs = ds.featureset()
feat = fs.next() feat = fs.next()
meta = ds.describe()
eq_(meta['srid'],-1)
eq_(meta.get('key_field'),None)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
eq_(feat['gid'],1) eq_(feat['gid'],1)
eq_(feat['name'],'name') eq_(feat['name'],'name')
eq_(mapnik.Expression("[name] = 'name'").evaluate(feat),True) eq_(mapnik.Expression("[name] = 'name'").evaluate(feat),True)
@ -636,6 +749,12 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
geometry_field='geom') geometry_field='geom')
fs = ds.featureset() fs = ds.featureset()
feat = fs.next() feat = fs.next()
meta = ds.describe()
eq_(meta['srid'],-1)
eq_(meta.get('key_field'),None)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
eq_(feat['gid'],1) eq_(feat['gid'],1)
eq_(feat['bool_field'],True) eq_(feat['bool_field'],True)
eq_(mapnik.Expression("[bool_field] = 'name'").evaluate(feat),False) eq_(mapnik.Expression("[bool_field] = 'name'").evaluate(feat),False)
@ -696,6 +815,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(feat.id(),1L) eq_(feat.id(),1L)
eq_(feat['osm_id'],None) eq_(feat['osm_id'],None)
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),None)
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
@raises(StopIteration) @raises(StopIteration)
def test_null_key_field(): def test_null_key_field():
opts = {'type':'postgis', opts = {'type':'postgis',
@ -798,6 +922,13 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
eq_(ds.fields(),['gid', 'dim', 'name']) eq_(ds.fields(),['gid', 'dim', 'name'])
eq_(ds.field_types(),['int', 'int', 'str']) eq_(ds.field_types(),['int', 'int', 'str'])
fs = ds.featureset() fs = ds.featureset()
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),None)
# Note: this is incorrect because we only check first couple geoms
eq_(meta['geometry_type'],mapnik.DataGeometryType.Point)
# Point (2d) # Point (2d)
feat = fs.next() feat = fs.next()
eq_(feat.id(),1) eq_(feat.id(),1)
@ -1024,6 +1155,11 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
for id in range(1,5): for id in range(1,5):
eq_(fs.next().id(),id) eq_(fs.next().id(),id)
meta = ds.describe()
eq_(meta['srid'],4326)
eq_(meta.get('key_field'),"gid")
eq_(meta['geometry_type'],None)
atexit.register(postgis_takedown) atexit.register(postgis_takedown)