From ee8ae809f444dc22e9eb421a024983c3090b7960 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Mon, 31 Oct 2011 19:08:07 -0400 Subject: [PATCH] more robust csv tests --- tests/data/csv/empty_rows.csv | 2 +- ...uch_dc8364385fc612b847d66ca7886519749c.csv | 0 tests/python_tests/csv_test.py | 68 +++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) rename tests/data/csv/{ => fails}/datacouch_dc8364385fc612b847d66ca7886519749c.csv (100%) diff --git a/tests/data/csv/empty_rows.csv b/tests/data/csv/empty_rows.csv index d1d0cd26e..f9c860b41 100644 --- a/tests/data/csv/empty_rows.csv +++ b/tests/data/csv/empty_rows.csv @@ -1,4 +1,4 @@ -x,y,text,date,integer,boolean,float,time,datetime,empty_column +"x","y","text","date","integer","boolean","float","time","datetime","empty_column" 0,0,a b,1971-01-01,40,True,1.0,04:14:00,1971-01-01T04:14:00, 0,0,c d,1948-01-01,63,True,1.27,14:57:13,1948-01-01T14:57:13, 0,0,e f,1920-01-01,164,False,41800000.01,00:00:00,1920-01-01T00:00:00, diff --git a/tests/data/csv/datacouch_dc8364385fc612b847d66ca7886519749c.csv b/tests/data/csv/fails/datacouch_dc8364385fc612b847d66ca7886519749c.csv similarity index 100% rename from tests/data/csv/datacouch_dc8364385fc612b847d66ca7886519749c.csv rename to tests/data/csv/fails/datacouch_dc8364385fc612b847d66ca7886519749c.csv diff --git a/tests/python_tests/csv_test.py b/tests/python_tests/csv_test.py index 3befa2350..36d4fb434 100644 --- a/tests/python_tests/csv_test.py +++ b/tests/python_tests/csv_test.py @@ -15,6 +15,9 @@ def setup(): if 'csv' in mapnik2.DatasourceCache.instance().plugin_names(): + def get_csv_ds(filename): + return mapnik2.Datasource(type='csv',file=os.path.join('../data/csv/',filename),quiet=True) + def test_broken_files(visual=False): broken = glob.glob("../data/csv/fails/*.*") broken.extend(glob.glob("../data/csv/warns/*.*")) @@ -42,7 +45,72 @@ if 'csv' in mapnik2.DatasourceCache.instance().plugin_names(): print '\x1b[1;32m✓ \x1b[0m', csv except Exception: print '\x1b[33mfailed\x1b[0m',csv + + def test_type_detection(**kwargs): + ds = get_csv_ds('nypd.csv') + eq_(ds.fields(),['Precinct','Phone','Address','City','geo_longitude','geo_latitude','geo_accuracy']) + eq_(ds.field_types(),['str','str','str','str','float','float','str']) + feat = ds.featureset().next() + attr = {'City': u'New York, NY', 'geo_accuracy': u'house', 'Phone': u'(212) 334-0711', 'Address': u'19 Elizabeth Street', 'Precinct': u'5th Precinct', 'geo_longitude': -70, 'geo_latitude': 40} + eq_(feat.attributes,attr) + eq_(len(ds.all_features()),2) + def test_skipping_blank_rows(**kwargs): + ds = get_csv_ds('blank_rows.csv') + eq_(ds.fields(),['x','y','name']) + eq_(ds.field_types(),['int','int','str']) + eq_(len(ds.all_features()),2) + + def test_empty_rows(**kwargs): + ds = get_csv_ds('empty_rows.csv') + eq_(len(ds.fields()),10) + eq_(len(ds.field_types()),10) + eq_(ds.fields(),['x', 'y', 'text', 'date', 'integer', 'boolean', 'float', 'time', 'datetime', 'empty_column']) + eq_(ds.field_types(),['int', 'int', 'str', 'str', 'int', 'bool', 'float', 'str', 'str', 'str']) + fs = ds.featureset() + feat = fs.next() + attr = {'x': 0, 'empty_column': None, 'text': u'a b', 'float': 1.0, 'datetime': u'1971-01-01T04:14:00', 'y': 0, 'boolean': True, 'time': u'04:14:00', 'date': u'1971-01-01', 'integer': 40} + eq_(feat.attributes,attr) + while feat: + eq_(len(feat),10) + eq_(feat['empty_column'],None) + feat = fs.next() + + def test_slashes(**kwargs): + ds = get_csv_ds('has_attributes_with_slashes.csv') + eq_(len(ds.fields()),3) + fs = ds.all_features() + eq_(fs[0].attributes,{'x':0,'y':0,'name':u'a/a'}) + eq_(fs[1].attributes,{'x':1,'y':4,'name':u'b/b'}) + eq_(fs[2].attributes,{'x':10,'y':2.5,'name':u'c/c'}) + + def test_wkt_field(**kwargs): + ds = get_csv_ds('wkt.csv') + eq_(len(ds.fields()),2) + eq_(ds.fields(),['type','WKT']) + eq_(ds.field_types(),['str','str']) + fs = ds.all_features() + #import pdb;pdb.set_trace() + eq_(len(fs[0].geometries()),1) + eq_(fs[0].geometries()[0].type(),mapnik2.GeometryType.Point) + eq_(len(fs[1].geometries()),1) + eq_(fs[1].geometries()[0].type(),mapnik2.GeometryType.LineString) + eq_(len(fs[2].geometries()),1) + eq_(fs[2].geometries()[0].type(),mapnik2.GeometryType.Polygon) + eq_(len(fs[3].geometries()),1) # one geometry, two parts + eq_(fs[3].geometries()[0].type(),mapnik2.GeometryType.Polygon) + # tests assuming we want to flatten geometries + eq_(len(fs[4].geometries()),4) + eq_(fs[4].geometries()[0].type(),mapnik2.GeometryType.Point) + eq_(len(fs[5].geometries()),2) + eq_(fs[5].geometries()[0].type(),mapnik2.GeometryType.LineString) + eq_(len(fs[6].geometries()),2) + eq_(fs[6].geometries()[0].type(),mapnik2.GeometryType.Polygon) + eq_(len(fs[7].geometries()),2) + eq_(fs[7].geometries()[0].type(),mapnik2.GeometryType.Polygon) + + # ideally we should not have to: + # https://github.com/mapnik/mapnik/issues?labels=multigeom+robustness&sort=created&direction=desc&state=open&page=1 if __name__ == "__main__": setup()