sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cleanup python datasource api simplifying access to descriptors

Browse source
This commit is contained in:
Dane Springmeyer 2012-01-11 20:03:47 -08:00
parent 7be62e594a
commit 2a137908c3
6 changed files with 117 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bindings/python/mapnik/__init__.py
View file

@ -233,9 +233,6 @@ class _Projection(Projection,_injector):
class _Datasource(Datasource,_injector):
def describe(self):
return Describe(self)
def all_features(self,fields=None):
query = Query(self.envelope())
attributes = fields or self.fields()
@ -710,7 +707,6 @@ __all__ = [
'SQLite',
'Osm',
'Kismet',
'Describe',
# version and environment
'mapnik_version_string',
'mapnik_version',

41
bindings/python/mapnik_datasource.cpp
View file

@ -82,30 +82,30 @@ boost::shared_ptr<mapnik::datasource> create_datasource(const dict& d)
return mapnik::datasource_cache::create(params, bind);
}
std::string describe(boost::shared_ptr<mapnik::datasource> const& ds)
boost::python::dict describe(boost::shared_ptr<mapnik::datasource> const& ds)
{
std::stringstream ss;
if (ds)
boost::python::dict description;
if (ds->type() == mapnik::datasource::Raster)
{
ss << ds->get_descriptor() << "\n";
description["type"] = "raster";
}
else
{
ss << "Null\n";
description["type"] = "vector";
}
return ss.str();
}
std::string encoding(boost::shared_ptr<mapnik::datasource> const& ds)
{
layer_descriptor ld = ds->get_descriptor();
return ld.get_encoding();
}
std::string name(boost::shared_ptr<mapnik::datasource> const& ds)
{
layer_descriptor ld = ds->get_descriptor();
return ld.get_name();
mapnik::layer_descriptor ld = ds->get_descriptor();
description["name"] = ld.get_name();
description["encoding"] = ld.get_encoding();
std::string geometry_type = ld.get_geometry_type();
if (geometry_type.empty())
{
description["geometry_type"] = NULL;
}
else
{
description["geometry_type"] = geometry_type;
}
return description;
}
boost::python::list fields(boost::shared_ptr<mapnik::datasource> const& ds)
@ -165,21 +165,18 @@ void export_datasource()
class_<datasource,boost::shared_ptr<datasource>,
boost::noncopyable>("Datasource",no_init)
.def("describe",&describe)
.def("envelope",&datasource::envelope)
.def("descriptor",&datasource::get_descriptor) //todo
.def("features",&datasource::features)
.def("bind",&datasource::bind)
.def("fields",&fields)
.def("field_types",&field_types)
.def("encoding",&encoding) //todo expose as property
.def("name",&name)
.def("features_at_point",&datasource::features_at_point)
.def("params",&datasource::params,return_value_policy<copy_const_reference>(),
"The configuration parameters of the data source. "
"These vary depending on the type of data source.")
;
def("Describe",&describe);
def("CreateDatasource",&create_datasource);
class_<point_datasource, bases<datasource>, boost::noncopyable>("PointDatasource", init<>())

25
tests/python_tests/csv_test.py
View file

@ -54,12 +54,14 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
attr = {'City': u'New York, NY', 'geo_accuracy': u'house', 'Phone': u'(212) 334-0711', 'Address': u'19 Elizabeth Street', 'Precinct': u'5th Precinct', 'geo_longitude': -70, 'geo_latitude': 40}
eq_(feat.attributes,attr)
eq_(len(ds.all_features()),2)
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_skipping_blank_rows(**kwargs):
ds = get_csv_ds('blank_rows.csv')
eq_(ds.fields(),['x','y','name'])
eq_(ds.field_types(),['int','int','str'])
eq_(len(ds.all_features()),2)
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_empty_rows(**kwargs):
ds = get_csv_ds('empty_rows.csv')
@ -75,6 +77,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(len(feat),10)
eq_(feat['empty_column'],u'')
feat = fs.next()
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_slashes(**kwargs):
ds = get_csv_ds('has_attributes_with_slashes.csv')
@ -83,6 +86,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(fs[0].attributes,{'x':0,'y':0,'name':u'a/a'})
eq_(fs[1].attributes,{'x':1,'y':4,'name':u'b/b'})
eq_(fs[2].attributes,{'x':10,'y':2.5,'name':u'c/c'})
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_wkt_field(**kwargs):
ds = get_csv_ds('wkt.csv')
@ -110,6 +114,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(fs[6].geometries()[0].type(),mapnik.GeometryType.Polygon)
eq_(len(fs[7].geometries()),2)
eq_(fs[7].geometries()[0].type(),mapnik.GeometryType.Polygon)
eq_(ds.describe(),{'geometry_type': 'collection', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_handling_of_missing_header(**kwargs):
@ -119,6 +124,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
fs = ds.featureset()
feat = fs.next()
eq_(feat['_4'],'missing')
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_handling_of_headers_that_are_numbers(**kwargs):
ds = get_csv_ds('numbers_for_headers.csv')
@ -144,6 +150,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(fs[2]['label'],"0,5")
eq_(fs[3]['label'],"5,0")
eq_(fs[4]['label'],"2.5,2.5")
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_windows_newlines(**kwargs):
ds = get_csv_ds('windows_newlines.csv')
@ -155,6 +162,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(feat['x'],1)
eq_(feat['y'],10)
eq_(feat['z'],9999.9999)
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_mac_newlines(**kwargs):
ds = get_csv_ds('windows_newlines.csv')
@ -166,6 +174,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(feat['x'],1)
eq_(feat['y'],10)
eq_(feat['z'],9999.9999)
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_tabs(**kwargs):
ds = get_csv_ds('tabs_in_csv.csv')
@ -176,6 +185,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(feat['x'],-122)
eq_(feat['y'],48)
eq_(feat['z'],0)
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_separator_pipes(**kwargs):
ds = get_csv_ds('pipe_delimiters.csv')
@ -186,6 +196,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(feat['x'],0)
eq_(feat['y'],0)
eq_(feat['z'],'hello')
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_separator_semicolon(**kwargs):
ds = get_csv_ds('semicolon_delimiters.csv')
@ -196,6 +207,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(feat['x'],0)
eq_(feat['y'],0)
eq_(feat['z'],'hello')
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_that_null_and_bool_keywords_are_empty_strings(**kwargs):
ds = get_csv_ds('nulls_and_booleans_as_strings.csv')
@ -213,6 +225,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(feat['y'],0)
eq_(feat['null'],'')
eq_(feat['boolean'],'false')
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
@raises(RuntimeError)
def test_that_nonexistant_query_field_throws(**kwargs):
@ -226,6 +239,7 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
# also add an invalid one, triggering throw
query.add_property_name('bogus')
fs = ds.features(query)
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_that_leading_zeros_mean_strings(**kwargs):
ds = get_csv_ds('leading_zeros.csv')
@ -245,6 +259,17 @@ if 'csv' in mapnik.DatasourceCache.instance().plugin_names():
eq_(feat['x'],0)
eq_(feat['y'],0)
eq_(feat['fips'],'005')
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'csv', 'encoding': 'utf-8'})
def test_advanced_geometry_detection(**kwargs):
ds = get_csv_ds('point_wkt.csv')
eq_(ds.describe()['geometry_type'],'point')
ds = get_csv_ds('poly_wkt.csv')
eq_(ds.describe()['geometry_type'],'polygon')
ds = get_csv_ds('multi_poly_wkt.csv')
eq_(ds.describe()['geometry_type'],'polygon')
ds = get_csv_ds('line_wkt.csv')
eq_(ds.describe()['geometry_type'],'linestring')
if __name__ == "__main__":
setup()

53
tests/python_tests/datasource_test.py
View file

@ -15,65 +15,68 @@ def test_that_datasources_exist():
print '***NOTICE*** - no datasource plugins have been loaded'
def test_field_listing():
lyr = mapnik.Layer('test')
if 'shape' in mapnik.DatasourceCache.instance().plugin_names():
lyr.datasource = mapnik.Shapefile(file='../data/shp/poly.shp')
fields = lyr.datasource.fields()
ds = mapnik.Shapefile(file='../data/shp/poly.shp')
fields = ds.fields()
eq_(fields, ['AREA', 'EAS_ID', 'PRFEDEA'])
eq_(ds.describe(),{'geometry_type': 'polygon', 'type': 'vector', 'name': 'shape', 'encoding': 'utf-8'})
def test_total_feature_count_shp():
lyr = mapnik.Layer('test')
if 'shape' in mapnik.DatasourceCache.instance().plugin_names():
lyr.datasource = mapnik.Shapefile(file='../data/shp/poly.shp')
features = lyr.datasource.all_features()
ds = mapnik.Shapefile(file='../data/shp/poly.shp')
features = ds.all_features()
num_feats = len(features)
eq_(num_feats, 10)
def test_total_feature_count_json():
lyr = mapnik.Layer('test')
if 'ogr' in mapnik.DatasourceCache.instance().plugin_names():
lyr.datasource = mapnik.Ogr(file='../data/json/points.json',layer_by_index=0)
features = lyr.datasource.all_features()
ds = mapnik.Ogr(file='../data/json/points.json',layer_by_index=0)
eq_(ds.describe(),{'geometry_type': 'point', 'type': 'vector', 'name': 'ogr', 'encoding': 'utf-8'})
features = ds.all_features()
num_feats = len(features)
eq_(num_feats, 5)
def test_sqlite_reading():
if 'sqlite' in mapnik.DatasourceCache.instance().plugin_names():
ds = mapnik.SQLite(file='../data/sqlite/world.sqlite',table_by_index=0)
eq_(ds.describe(),{'geometry_type': 'polygon', 'type': 'vector', 'name': 'sqlite', 'encoding': 'utf-8'})
features = ds.all_features()
num_feats = len(features)
eq_(num_feats, 245)
def test_reading_json_from_string():
json = open('../data/json/points.json','r').read()
lyr = mapnik.Layer('test')
if 'ogr' in mapnik.DatasourceCache.instance().plugin_names():
lyr.datasource = mapnik.Ogr(file=json,layer_by_index=0)
features = lyr.datasource.all_features()
ds = mapnik.Ogr(file=json,layer_by_index=0)
features = ds.all_features()
num_feats = len(features)
eq_(num_feats, 5)
def test_feature_envelope():
lyr = mapnik.Layer('test')
if 'shape' in mapnik.DatasourceCache.instance().plugin_names():
lyr.datasource = mapnik.Shapefile(file='../data/shp/poly.shp')
features = lyr.datasource.all_features()
ds = mapnik.Shapefile(file='../data/shp/poly.shp')
features = ds.all_features()
for feat in features:
env = feat.envelope()
contains = lyr.envelope().contains(env)
contains = ds.envelope().contains(env)
eq_(contains, True)
intersects = lyr.envelope().contains(env)
intersects = ds.envelope().contains(env)
eq_(intersects, True)
def test_feature_attributes():
lyr = mapnik.Layer('test')
if 'shape' in mapnik.DatasourceCache.instance().plugin_names():
lyr.datasource = mapnik.Shapefile(file='../data/shp/poly.shp')
features = lyr.datasource.all_features()
ds = mapnik.Shapefile(file='../data/shp/poly.shp')
features = ds.all_features()
feat = features[0]
attrs = {'PRFEDEA': u'35043411', 'EAS_ID': 168, 'AREA': 215229.266}
eq_(feat.attributes, attrs)
eq_(lyr.datasource.fields(),['AREA', 'EAS_ID', 'PRFEDEA'])
eq_(lyr.datasource.field_types(),['float','int','str'])
eq_(ds.fields(),['AREA', 'EAS_ID', 'PRFEDEA'])
eq_(ds.field_types(),['float','int','str'])
def test_ogr_layer_by_sql():
lyr = mapnik.Layer('test')
if 'ogr' in mapnik.DatasourceCache.instance().plugin_names():
lyr.datasource = mapnik.Ogr(file='../data/shp/poly.shp', layer_by_sql='SELECT * FROM poly WHERE EAS_ID = 168')
features = lyr.datasource.all_features()
ds = mapnik.Ogr(file='../data/shp/poly.shp', layer_by_sql='SELECT * FROM poly WHERE EAS_ID = 168')
features = ds.all_features()
num_feats = len(features)
eq_(num_feats, 1)

4
tests/python_tests/layer_modification_test.py
View file

@ -58,8 +58,8 @@ def test_adding_datasource_to_layer():
m.layers[0].datasource = ds
# now ensure it is attached
eq_(m.layers[0].datasource.name(),"shape")
eq_(lyr.datasource.name(),"shape")
eq_(m.layers[0].datasource.describe()['name'],"shape")
eq_(lyr.datasource.describe()['name'],"shape")
# and since we have now added a shapefile in spherical mercator, adjust the projection
lyr.srs = '+proj=merc +lon_0=0 +lat_ts=0 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m +no_defs'

46
tests/python_tests/postgis_test.py
View file

@ -20,7 +20,7 @@ def call(cmd,silent=False):
stdin, stderr = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE).communicate()
if not stderr:
return stdin.strip()
elif not silent:
elif not silent and not 'NOTICE' in stderr:
raise RuntimeError(stderr.strip())
def psql_can_connect():
@ -63,12 +63,23 @@ def createdb_and_dropdb_on_path():
print 'Notice: skipping postgis tests (createdb/dropdb)'
return False
insert_sql = """
INSERT INTO test(geom) values (GeomFromEWKT('SRID=4326;POINT(0 0)'));
INSERT INTO test(geom) values (GeomFromEWKT('SRID=4326;POINT(-2 2)'));
INSERT INTO test(geom) values (GeomFromEWKT('SRID=4326;MULTIPOINT(2 1,1 2)'));
INSERT INTO test(geom) values (GeomFromEWKT('SRID=4326;LINESTRING(0 0,1 1,1 2)'));
INSERT INTO test(geom) values (GeomFromEWKT('SRID=4326;MULTILINESTRING((1 0,0 1,3 2),(3 2,5 4))'));
INSERT INTO test(geom) values (GeomFromEWKT('SRID=4326;POLYGON((0 0,4 0,4 4,0 4,0 0),(1 1, 2 1, 2 2, 1 2,1 1))'));
INSERT INTO test(geom) values (GeomFromEWKT('SRID=4326;MULTIPOLYGON(((1 1,3 1,3 3,1 3,1 1),(1 1,2 1,2 2,1 2,1 1)), ((-1 -1,-1 -2,-2 -2,-2 -1,-1 -1)))'));
INSERT INTO test(geom) values (GeomFromEWKT('SRID=4326;GEOMETRYCOLLECTION(POLYGON((1 1, 2 1, 2 2, 1 2,1 1)),POINT(2 3),LINESTRING(2 3,3 4))'));
"""
def postgis_setup():
call('dropdb %s' % MAPNIK_TEST_DBNAME,silent=True)
call('createdb -T %s %s' % (POSTGIS_TEMPLATE_DBNAME,MAPNIK_TEST_DBNAME),silent=False)
call('shp2pgsql -s 3857 -g geom -W LATIN1 %s world_merc | psql -q %s' % (SHAPEFILE,MAPNIK_TEST_DBNAME), silent=True)
call('''psql %s -c "CREATE TABLE \"empty\" (key serial);SELECT AddGeometryColumn('','empty','geom','-1','GEOMETRY',4);"''' % MAPNIK_TEST_DBNAME,silent=True)
call('''psql -q %s -c "CREATE TABLE \"empty\" (key serial);SELECT AddGeometryColumn('','empty','geom','-1','GEOMETRY',4);"''' % MAPNIK_TEST_DBNAME,silent=False)
call('''psql -q %s -c "create table test(gid serial PRIMARY KEY, geom geometry);%s"''' % (MAPNIK_TEST_DBNAME,insert_sql),silent=False)
def postgis_takedown():
pass
# fails as the db is in use: https://github.com/mapnik/mapnik/issues/960
@ -98,6 +109,7 @@ if 'postgis' in mapnik.DatasourceCache.instance().plugin_names() \
eq_(feature['subregion'],29)
eq_(feature['lon'],-61.783)
eq_(feature['lat'],17.078)
eq_(ds.describe()['geometry_type'],'polygon')
def test_subquery():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='(select * from world_merc) as w')
@ -115,6 +127,7 @@ if 'postgis' in mapnik.DatasourceCache.instance().plugin_names() \
eq_(feature['subregion'],29)
eq_(feature['lon'],-61.783)
eq_(feature['lat'],17.078)
eq_(ds.describe()['geometry_type'],'polygon')
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='(select gid,geom,fips as _fips from world_merc) as w')
fs = ds.featureset()
@ -122,12 +135,39 @@ if 'postgis' in mapnik.DatasourceCache.instance().plugin_names() \
eq_(feature['gid'],1)
eq_(feature['_fips'],u'AC')
eq_(len(feature),2)
eq_(ds.describe()['geometry_type'],'polygon')
def test_empty_db():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='empty')
fs = ds.featureset()
feature = fs.next()
eq_(feature,None)
eq_(ds.describe()['geometry_type'],'collection')
def test_geometry_detection():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test',
geometry_field='geom')
eq_(ds.describe()['geometry_type'],'collection')
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test',
geometry_field='geom',
row_limit=1)
eq_(ds.describe()['geometry_type'],'point')
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test',
geometry_field='geom',
row_limit=2)
eq_(ds.describe()['geometry_type'],'point')
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test',
geometry_field='geom',
row_limit=3)
eq_(ds.describe()['geometry_type'],'point')
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,table='test',
geometry_field='geom',
row_limit=4)
eq_(ds.describe()['geometry_type'],'collection')
@raises(RuntimeError)
def test_that_nonexistant_query_field_throws(**kwargs):