diff --git a/plugins/input/csv/csv_datasource.cpp b/plugins/input/csv/csv_datasource.cpp index 5398930b6..88a2169e5 100644 --- a/plugins/input/csv/csv_datasource.cpp +++ b/plugins/input/csv/csv_datasource.cpp @@ -652,7 +652,7 @@ void csv_datasource::parse_csv(T & stream, } } - // now, add attributes, skipping any WKT or JSON fiels + // now, add attributes, skipping any WKT or JSON fields if ((has_wkt_field) && (i == wkt_idx)) continue; if ((has_json_field) && (i == json_idx)) continue; /* First we detect likely strings, then try parsing likely numbers, @@ -664,27 +664,34 @@ void csv_datasource::parse_csv(T & stream, to assume are numbers) */ + bool matched = false; bool has_dot = value.find(".") != std::string::npos; if (value.empty() || (value_length > 20) || (value_length > 1 && !has_dot && value[0] == '0')) { + matched = true; feature->put(fld_name,tr.transcode(value.c_str())); if (feature_count == 1) { desc_.add_descriptor(mapnik::attribute_descriptor(fld_name,mapnik::String)); } } - else if ((value[0] >= '0' && value[0] <= '9') || value[0] == '-') + else if ((value[0] >= '0' && value[0] <= '9') || + value[0] == '-' || + value[0] == '+' || + value[0] == '.') { - double float_val = 0.0; - std::string::const_iterator str_beg = value.begin(); - std::string::const_iterator str_end = value.end(); - bool r = qi::phrase_parse(str_beg,str_end,qi::double_,ascii::space,float_val); - if (r && (str_beg == str_end)) + bool has_e = value.find("e") != std::string::npos; + if (has_dot || has_e) { - if (has_dot) + double float_val = 0.0; + std::string::const_iterator str_beg = value.begin(); + std::string::const_iterator str_end = value.end(); + if (qi::phrase_parse(str_beg,str_end,qi::double_,ascii::space,float_val) + && (str_beg == str_end)) { + matched = true; feature->put(fld_name,float_val); if (feature_count == 1) { @@ -693,9 +700,17 @@ void csv_datasource::parse_csv(T & stream, fld_name,mapnik::Double)); } } - else + } + else + { + mapnik::value_integer int_val = 0; + std::string::const_iterator str_beg = value.begin(); + std::string::const_iterator str_end = value.end(); + if (qi::phrase_parse(str_beg,str_end,qi::long_long,ascii::space,int_val) + && (str_beg == str_end)) { - feature->put(fld_name,static_cast(float_val)); + matched = true; + feature->put(fld_name,int_val); if (feature_count == 1) { desc_.add_descriptor( @@ -704,19 +719,8 @@ void csv_datasource::parse_csv(T & stream, } } } - else - { - // fallback to normal string - feature->put(fld_name,tr.transcode(value.c_str())); - if (feature_count == 1) - { - desc_.add_descriptor( - mapnik::attribute_descriptor( - fld_name,mapnik::String)); - } - } } - else + if (!matched) { // fallback to normal string feature->put(fld_name,tr.transcode(value.c_str())); diff --git a/tests/data/csv/64bit_int.csv b/tests/data/csv/64bit_int.csv new file mode 100644 index 000000000..a36badb88 --- /dev/null +++ b/tests/data/csv/64bit_int.csv @@ -0,0 +1,3 @@ +x,y,bigint +0,0,2147483648 +0,0,9223372036854775807 diff --git a/tests/data/csv/number_types.csv b/tests/data/csv/number_types.csv new file mode 100644 index 000000000..d767bbbf6 --- /dev/null +++ b/tests/data/csv/number_types.csv @@ -0,0 +1,9 @@ +x,y,floats +0,0,.0 +0,0,+.0 +0,0,1e-06 +0,0,-1e-06 +0,0,0.000001 +0,0,1.234e+16 +0,0,1.234e16 +0,0,"1.234e16" \ No newline at end of file diff --git a/tests/data/sqlite/64bit_int.sqlite b/tests/data/sqlite/64bit_int.sqlite new file mode 100644 index 000000000..2918ddee4 Binary files /dev/null and b/tests/data/sqlite/64bit_int.sqlite differ diff --git a/tests/python_tests/csv_test.py b/tests/python_tests/csv_test.py index ea57f245d..fb8335150 100644 --- a/tests/python_tests/csv_test.py +++ b/tests/python_tests/csv_test.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import glob +import sys from nose.tools import * from utilities import execution_path @@ -529,6 +530,46 @@ if 'csv' in mapnik.DatasourceCache.plugin_names(): eq_(desc['geometry_type'],mapnik.DataGeometryType.Point) eq_(len(ds.all_features()),1) + def test_that_64bit_int_fields_work(**kwargs): + ds = get_csv_ds('64bit_int.csv') + eq_(len(ds.fields()),3) + eq_(ds.fields(),['x','y','bigint']) + eq_(ds.field_types(),['int','int','int']) + fs = ds.featureset() + feat = fs.next() + eq_(feat['bigint'],2147483648) + feat = fs.next() + eq_(feat['bigint'],sys.maxint) + eq_(feat['bigint'],9223372036854775807) + eq_(feat['bigint'],0x7FFFFFFFFFFFFFFF) + desc = ds.describe() + eq_(desc['geometry_type'],mapnik.DataGeometryType.Point) + eq_(len(ds.all_features()),2) + + def test_various_number_types(**kwargs): + ds = get_csv_ds('number_types.csv') + eq_(len(ds.fields()),3) + eq_(ds.fields(),['x','y','floats']) + eq_(ds.field_types(),['int','int','float']) + fs = ds.featureset() + feat = fs.next() + eq_(feat['floats'],.0) + feat = fs.next() + eq_(feat['floats'],+.0) + feat = fs.next() + eq_(feat['floats'],1e-06) + feat = fs.next() + eq_(feat['floats'],-1e-06) + feat = fs.next() + eq_(feat['floats'],0.000001) + feat = fs.next() + eq_(feat['floats'],1.234e+16) + feat = fs.next() + eq_(feat['floats'],1.234e+16) + desc = ds.describe() + eq_(desc['geometry_type'],mapnik.DataGeometryType.Point) + eq_(len(ds.all_features()),8) + if __name__ == "__main__": setup() [eval(run)(visual=True) for run in dir() if 'test_' in run] diff --git a/tests/python_tests/filter_test.py b/tests/python_tests/filter_test.py index 9cfaf2ecd..6a6200f39 100644 --- a/tests/python_tests/filter_test.py +++ b/tests/python_tests/filter_test.py @@ -239,7 +239,9 @@ null_equality = [ [.1,False,float], [False,False,int], # TODO - should become bool [True,False,int], # TODO - should become bool - [None,True,None] + [None,True,None], + [2147483648,False,int], + [922337203685477580,False,int] ] def test_expressions_with_null_equality(): @@ -285,7 +287,9 @@ truthyness = [ [.1,True,float], [False,False,int], # TODO - should become bool [True,True,int], # TODO - should become bool - [None,False,None] + [None,False,None], + [2147483648,True,int], + [922337203685477580,True,int] ] def test_expressions_for_thruthyness(): diff --git a/tests/python_tests/postgis_test.py b/tests/python_tests/postgis_test.py index c0bd56b54..619326588 100644 --- a/tests/python_tests/postgis_test.py +++ b/tests/python_tests/postgis_test.py @@ -132,6 +132,12 @@ CREATE TABLE test7(gid serial PRIMARY KEY, geom geometry); INSERT INTO test7(gid, geom) values (1, GeomFromEWKT('SRID=4326;GEOMETRYCOLLECTION(MULTILINESTRING((10 10,20 20,10 40),(40 40,30 30,40 20,30 10)),LINESTRING EMPTY)')); ''' +insert_table_8 = ''' +CREATE TABLE test8(gid serial PRIMARY KEY,int_field bigint, geom geometry); +INSERT INTO test8(gid, int_field, geom) values (1, 2147483648, ST_MakePoint(1,1)); +INSERT INTO test8(gid, int_field, geom) values (2, 922337203685477580, ST_MakePoint(1,1)); +''' + def postgis_setup(): call('dropdb %s' % MAPNIK_TEST_DBNAME,silent=True) @@ -146,6 +152,7 @@ def postgis_setup(): call("""psql -q %s -c '%s'""" % (MAPNIK_TEST_DBNAME,insert_table_5b),silent=False) call('''psql -q %s -c "%s"''' % (MAPNIK_TEST_DBNAME,insert_table_6),silent=False) call('''psql -q %s -c "%s"''' % (MAPNIK_TEST_DBNAME,insert_table_7),silent=False) + call('''psql -q %s -c "%s"''' % (MAPNIK_TEST_DBNAME,insert_table_8),silent=False) def postgis_takedown(): pass @@ -472,7 +479,21 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \ t.start() t.join() - + def test_that_64bit_int_fields_work(): + ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME, + table='test8') + eq_(len(ds.fields()),2) + eq_(ds.fields(),['gid','int_field']) + eq_(ds.field_types(),['int','int']) + fs = ds.featureset() + feat = fs.next() + eq_(feat.id(),1) + eq_(feat['gid'],1) + eq_(feat['int_field'],2147483648) + feat = fs.next() + eq_(feat.id(),2) + eq_(feat['gid'],2) + eq_(feat['int_field'],922337203685477580) atexit.register(postgis_takedown) diff --git a/tests/python_tests/render_grid_test.py b/tests/python_tests/render_grid_test.py index 1b5147081..aaa28a923 100644 --- a/tests/python_tests/render_grid_test.py +++ b/tests/python_tests/render_grid_test.py @@ -268,16 +268,24 @@ def test_32bit_int_id(): eq_(grid.get_pixel(128,128),int32) utf1 = grid.encode('utf',resolution=4) eq_(utf1['keys'],[str(int32)]) - - # this will fail because it is used internally to mark alpha - #max_neg = -(int32+1) - # so we use max neg-1 max_neg = -(int32) grid = gen_grid_for_id(max_neg) eq_(grid.get_pixel(128,128),max_neg) utf1 = grid.encode('utf',resolution=4) eq_(utf1['keys'],[str(max_neg)]) +def test_64bit_int_id(): + int64 = 0x7FFFFFFFFFFFFFFF + grid = gen_grid_for_id(int64) + eq_(grid.get_pixel(128,128),int64) + utf1 = grid.encode('utf',resolution=4) + eq_(utf1['keys'],[str(int64)]) + max_neg = -(int64) + grid = gen_grid_for_id(max_neg) + eq_(grid.get_pixel(128,128),max_neg) + utf1 = grid.encode('utf',resolution=4) + eq_(utf1['keys'],[str(max_neg)]) + def test_id_zero(): grid = gen_grid_for_id(0) eq_(grid.get_pixel(128,128),0) diff --git a/tests/python_tests/sqlite_test.py b/tests/python_tests/sqlite_test.py index dc94f1f1e..b0c8f864e 100644 --- a/tests/python_tests/sqlite_test.py +++ b/tests/python_tests/sqlite_test.py @@ -360,6 +360,25 @@ if 'sqlite' in mapnik.DatasourceCache.plugin_names(): eq_(len(feat.geometries()),1) eq_(feat.geometries()[0].to_wkt(),'Point(0 0)') + + def test_that_64bit_int_fields_work(): + ds = mapnik.SQLite(file='../data/sqlite/64bit_int.sqlite', + table='int_table', + use_spatial_index=False + ) + eq_(len(ds.fields()),3) + eq_(ds.fields(),['OGC_FID','id','bigint']) + eq_(ds.field_types(),['int','int','int']) + fs = ds.featureset() + feat = fs.next() + eq_(feat.id(),1) + eq_(feat['OGC_FID'],1) + eq_(feat['bigint'],2147483648) + feat = fs.next() + eq_(feat.id(),2) + eq_(feat['OGC_FID'],2) + eq_(feat['bigint'],922337203685477580) + if __name__ == "__main__": setup() [eval(run)() for run in dir() if 'test_' in run]