From aee275a08cecd359869e428d6d57d20f7da7057c Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Fri, 21 Dec 2012 19:53:33 -0800 Subject: [PATCH] fix number handling in csv plugin and add tests for 64bit integer support in sqlite and postgis datasources - refs #1669 --- plugins/input/csv/csv_datasource.cpp | 48 +++++++++++++------------ tests/data/csv/64bit_int.csv | 3 ++ tests/data/csv/number_types.csv | 9 +++++ tests/data/sqlite/64bit_int.sqlite | Bin 0 -> 6144 bytes tests/python_tests/csv_test.py | 41 +++++++++++++++++++++ tests/python_tests/filter_test.py | 8 +++-- tests/python_tests/postgis_test.py | 23 +++++++++++- tests/python_tests/render_grid_test.py | 16 ++++++--- tests/python_tests/sqlite_test.py | 19 ++++++++++ 9 files changed, 138 insertions(+), 29 deletions(-) create mode 100644 tests/data/csv/64bit_int.csv create mode 100644 tests/data/csv/number_types.csv create mode 100644 tests/data/sqlite/64bit_int.sqlite diff --git a/plugins/input/csv/csv_datasource.cpp b/plugins/input/csv/csv_datasource.cpp index 5398930b6..88a2169e5 100644 --- a/plugins/input/csv/csv_datasource.cpp +++ b/plugins/input/csv/csv_datasource.cpp @@ -652,7 +652,7 @@ void csv_datasource::parse_csv(T & stream, } } - // now, add attributes, skipping any WKT or JSON fiels + // now, add attributes, skipping any WKT or JSON fields if ((has_wkt_field) && (i == wkt_idx)) continue; if ((has_json_field) && (i == json_idx)) continue; /* First we detect likely strings, then try parsing likely numbers, @@ -664,27 +664,34 @@ void csv_datasource::parse_csv(T & stream, to assume are numbers) */ + bool matched = false; bool has_dot = value.find(".") != std::string::npos; if (value.empty() || (value_length > 20) || (value_length > 1 && !has_dot && value[0] == '0')) { + matched = true; feature->put(fld_name,tr.transcode(value.c_str())); if (feature_count == 1) { desc_.add_descriptor(mapnik::attribute_descriptor(fld_name,mapnik::String)); } } - else if ((value[0] >= '0' && value[0] <= '9') || value[0] == '-') + else if ((value[0] >= '0' && value[0] <= '9') || + value[0] == '-' || + value[0] == '+' || + value[0] == '.') { - double float_val = 0.0; - std::string::const_iterator str_beg = value.begin(); - std::string::const_iterator str_end = value.end(); - bool r = qi::phrase_parse(str_beg,str_end,qi::double_,ascii::space,float_val); - if (r && (str_beg == str_end)) + bool has_e = value.find("e") != std::string::npos; + if (has_dot || has_e) { - if (has_dot) + double float_val = 0.0; + std::string::const_iterator str_beg = value.begin(); + std::string::const_iterator str_end = value.end(); + if (qi::phrase_parse(str_beg,str_end,qi::double_,ascii::space,float_val) + && (str_beg == str_end)) { + matched = true; feature->put(fld_name,float_val); if (feature_count == 1) { @@ -693,9 +700,17 @@ void csv_datasource::parse_csv(T & stream, fld_name,mapnik::Double)); } } - else + } + else + { + mapnik::value_integer int_val = 0; + std::string::const_iterator str_beg = value.begin(); + std::string::const_iterator str_end = value.end(); + if (qi::phrase_parse(str_beg,str_end,qi::long_long,ascii::space,int_val) + && (str_beg == str_end)) { - feature->put(fld_name,static_cast(float_val)); + matched = true; + feature->put(fld_name,int_val); if (feature_count == 1) { desc_.add_descriptor( @@ -704,19 +719,8 @@ void csv_datasource::parse_csv(T & stream, } } } - else - { - // fallback to normal string - feature->put(fld_name,tr.transcode(value.c_str())); - if (feature_count == 1) - { - desc_.add_descriptor( - mapnik::attribute_descriptor( - fld_name,mapnik::String)); - } - } } - else + if (!matched) { // fallback to normal string feature->put(fld_name,tr.transcode(value.c_str())); diff --git a/tests/data/csv/64bit_int.csv b/tests/data/csv/64bit_int.csv new file mode 100644 index 000000000..a36badb88 --- /dev/null +++ b/tests/data/csv/64bit_int.csv @@ -0,0 +1,3 @@ +x,y,bigint +0,0,2147483648 +0,0,9223372036854775807 diff --git a/tests/data/csv/number_types.csv b/tests/data/csv/number_types.csv new file mode 100644 index 000000000..d767bbbf6 --- /dev/null +++ b/tests/data/csv/number_types.csv @@ -0,0 +1,9 @@ +x,y,floats +0,0,.0 +0,0,+.0 +0,0,1e-06 +0,0,-1e-06 +0,0,0.000001 +0,0,1.234e+16 +0,0,1.234e16 +0,0,"1.234e16" \ No newline at end of file diff --git a/tests/data/sqlite/64bit_int.sqlite b/tests/data/sqlite/64bit_int.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..2918ddee44f2f7c8c6befd6a99ebc23cf0309344 GIT binary patch literal 6144 zcmeI0KX21O6u|H7ByOQq2ZX#7<%zZmAq^5jR4kPmr*4RvHpI9Fd$_`;xizT2}B15J^<%SoH~|**ifT;ik)}w&*{D2-RYt{u2n4Ggr?)R z4Il2I1i~0?03d`G=#!#PC|9YH2nX~VNAyYR0?PN7=EWJ5L_d)@CoBt}@pG#F59!V( zB{`kOUBfpHTBc?DWL!xVHMO9tpcj;i3Yk$r25tdBwNxZq>S{^V;E`6|DQNq! zt?sV^l+@~us?(uTsVcBavm2Q(l#ULpLpt5alI`A3Nb<@Gel4>Syko<+jFx3L%(JMT z9Iq(q3X1wmNkIwCL4U+H+GYlx6tv>Qg2wEJrqeckw?pbq>$Gi0ot|m9^&`@BSyIqV zwtG{MwsuM^!9EXnyiesi7m z8H41-$HG{raWC650MRwLSOV5%lf-gj{*l-39GfAh0Su@+j@uv&t8Lnz<=8QA6zm{j zG}rK;4>tp3=V{|u>?8W&!U=E!Q%7JCOX(Xk7+o9PXW?BXJl$4;^FMa0PhJ1KAx?l3 zm_*=8;E4XJpA^XLoB$^