From 9c09f15de9cbf067543b0273b37dc3ca8affde23 Mon Sep 17 00:00:00 2001 From: artemp Date: Tue, 15 Mar 2016 12:24:17 +0100 Subject: [PATCH] csv_utils - check and skip processing lines consisting only of 0xa (windows) ( fixes #3359 ) --- plugins/input/csv/csv_utils.cpp | 4 +-- plugins/input/sqlite/sqlite_datasource.cpp | 34 ++++++++++---------- plugins/input/sqlite/sqlite_featureset.cpp | 2 +- test/unit/serialization/wkb_formats_test.cpp | 14 ++++++-- utils/pgsql2sqlite/main.cpp | 6 ++-- utils/pgsql2sqlite/pgsql2sqlite.hpp | 28 +++++++++++++--- 6 files changed, 58 insertions(+), 30 deletions(-) diff --git a/plugins/input/csv/csv_utils.cpp b/plugins/input/csv/csv_utils.cpp index e8fe78986..ff75a46ed 100644 --- a/plugins/input/csv/csv_utils.cpp +++ b/plugins/input/csv/csv_utils.cpp @@ -128,11 +128,11 @@ std::tuple autodetect_csv_flavour(std::istream & stream, { if (size < file_length && ss.eof()) { - // we can't be sure last line + // we can't be sure that last line // is not truncated so skip it break; } - if (line.size() == 0) continue; // empty lines are not interesting + if (line.size() == 0 || (line.size() == 1 && line[0] == char(0xa))) continue; // empty lines are not interesting auto num_quotes = std::count(line.begin(), line.end(), quote); if (num_quotes % 2 != 0) { diff --git a/plugins/input/sqlite/sqlite_datasource.cpp b/plugins/input/sqlite/sqlite_datasource.cpp index 939d9d319..4efdb4607 100644 --- a/plugins/input/sqlite/sqlite_datasource.cpp +++ b/plugins/input/sqlite/sqlite_datasource.cpp @@ -53,22 +53,22 @@ using mapnik::parameters; DATASOURCE_PLUGIN(sqlite_datasource) sqlite_datasource::sqlite_datasource(parameters const& params) - : datasource(params), - extent_(), - extent_initialized_(false), - type_(datasource::Vector), - table_(*params.get("table", "")), - fields_(*params.get("fields", "*")), - metadata_(*params.get("metadata", "")), - geometry_table_(*params.get("geometry_table", "")), - geometry_field_(*params.get("geometry_field", "")), - index_table_(*params.get("index_table", "")), - key_field_(*params.get("key_field", "")), - row_offset_(*params.get("row_offset", 0)), - row_limit_(*params.get("row_limit", 0)), - intersects_token_("!intersects!"), - desc_(sqlite_datasource::name(), *params.get("encoding", "utf-8")), - format_(mapnik::wkbAuto) + : datasource(params), + extent_(), + extent_initialized_(false), + type_(datasource::Vector), + table_(*params.get("table", "")), + fields_(*params.get("fields", "*")), + metadata_(*params.get("metadata", "")), + geometry_table_(*params.get("geometry_table", "")), + geometry_field_(*params.get("geometry_field", "")), + index_table_(*params.get("index_table", "")), + key_field_(*params.get("key_field", "")), + row_offset_(*params.get("row_offset", 0)), + row_limit_(*params.get("row_limit", 0)), + intersects_token_("!intersects!"), + desc_(sqlite_datasource::name(), *params.get("encoding", "utf-8")), + format_(mapnik::wkbAuto) { /* TODO - throw if no primary key but spatial index is present? @@ -448,7 +448,7 @@ boost::optional sqlite_datasource::get_geometry_t if (data) { - mapnik::geometry::geometry geom = mapnik::geometry_utils::from_wkb(data, size, format_); + mapnik::geometry::geometry geom = mapnik::geometry_utils::from_twkb(data, size); if (mapnik::geometry::is_empty(geom)) { continue; diff --git a/plugins/input/sqlite/sqlite_featureset.cpp b/plugins/input/sqlite/sqlite_featureset.cpp index 99a44db54..2474ca45d 100644 --- a/plugins/input/sqlite/sqlite_featureset.cpp +++ b/plugins/input/sqlite/sqlite_featureset.cpp @@ -81,7 +81,7 @@ feature_ptr sqlite_featureset::next() } feature_ptr feature = feature_factory::create(ctx_,rs_->column_integer64(1)); - mapnik::geometry::geometry geom = geometry_utils::from_wkb(data, size, format_); + mapnik::geometry::geometry geom = geometry_utils::from_twkb(data, size); if (mapnik::geometry::is_empty(geom)) { continue; diff --git a/test/unit/serialization/wkb_formats_test.cpp b/test/unit/serialization/wkb_formats_test.cpp index 698645266..9ffe8f156 100644 --- a/test/unit/serialization/wkb_formats_test.cpp +++ b/test/unit/serialization/wkb_formats_test.cpp @@ -1,4 +1,3 @@ - #include "catch.hpp" #include @@ -8,6 +7,7 @@ #include #include #include +#include #include TEST_CASE("geometry formats") { @@ -64,8 +64,13 @@ SECTION("wkb") { // spatialite blob mapnik::geometry::geometry geom = mapnik::geometry_utils::from_wkb((const char*)sp_valid_blob, - sizeof(sp_valid_blob) / sizeof(sp_valid_blob[0]), - mapnik::wkbSpatiaLite); + sizeof(sp_valid_blob) / sizeof(sp_valid_blob[0]), + mapnik::wkbSpatiaLite); + std::string wkt; + if (mapnik::util::to_wkt(wkt, geom)) + { + std::cerr << wkt << std::endl; + } // winding order is not correct per OGC so we'll fix it mapnik::geometry::correct(geom); #if BOOST_VERSION >= 105800 @@ -76,7 +81,10 @@ SECTION("wkb") { geom = mapnik::geometry_utils::from_wkb((const char*)sp_valid_blob, sizeof(sp_valid_blob) / sizeof(sp_valid_blob[0]), mapnik::wkbAuto); + + mapnik::geometry::correct(geom); + #if BOOST_VERSION >= 105800 REQUIRE(mapnik::geometry::is_valid(geom)); REQUIRE(mapnik::geometry::is_simple(geom)); diff --git a/utils/pgsql2sqlite/main.cpp b/utils/pgsql2sqlite/main.cpp index 81eff635e..2765a8b38 100644 --- a/utils/pgsql2sqlite/main.cpp +++ b/utils/pgsql2sqlite/main.cpp @@ -58,7 +58,7 @@ int main ( int argc, char** argv) ("query,q",po::value(),"Name of the table/or query to pass to postmaster") ("table,t",po::value(),"Name of the output table to create (default: table in query)") ("file,f",po::value(),"Use this option to specify the name of the file to create.") - + ("twkb", "Output geometries in TWKB format.") ; //po::positional_options_description p; @@ -90,6 +90,8 @@ int main ( int argc, char** argv) boost::optional password; boost::optional connect_timeout("4"); + bool output_twkb = vm.count("twkb") ? true : false; + if (vm.count("host")) host = vm["host"].as(); if (vm.count("port")) port = vm["port"].as(); if (vm.count("dbname")) dbname = vm["dbname"].as(); @@ -107,7 +109,7 @@ int main ( int argc, char** argv) std::cout << "output_table : " << output_table_name << "\n"; - mapnik::pgsql2sqlite(conn,query,output_table_name,output_file); + mapnik::pgsql2sqlite(conn, query, output_table_name, output_file, output_twkb); } catch (mapnik::datasource_exception & ex) { diff --git a/utils/pgsql2sqlite/pgsql2sqlite.hpp b/utils/pgsql2sqlite/pgsql2sqlite.hpp index e339a8f58..8022206c1 100644 --- a/utils/pgsql2sqlite/pgsql2sqlite.hpp +++ b/utils/pgsql2sqlite/pgsql2sqlite.hpp @@ -165,7 +165,8 @@ template void pgsql2sqlite(Connection conn, std::string const& query, std::string const& output_table_name, - std::string const& output_filename) + std::string const& output_filename, + bool output_twkb) { namespace sqlite = mapnik::sqlite; sqlite::database db(output_filename); @@ -223,11 +224,20 @@ void pgsql2sqlite(Connection conn, geom_type = rs->getValue("type"); } - // add AsBinary() modifier std::string select_sql_str = select_sql.str(); - boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","ST_AsBinary(" + geom_col+") as " + geom_col); + if (output_twkb) + { + // add AsTWKB(, 2) modifier + boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","ST_AsTWKB(" + geom_col+", 2) as " + geom_col); + } + else + { + // add AsBinary() modifier + boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","ST_AsBinary(" + geom_col+") as " + geom_col); + } -#ifdef MAPNIK_DEBUG +//#ifdef MAPNIK_DEBUG +#if 1 std::cout << select_sql_str << "\n"; #endif @@ -387,7 +397,15 @@ void pgsql2sqlite(Connection conn, if (oid == geometry_oid) { mapnik::feature_impl feat(ctx,pkid); - mapnik::geometry::geometry geom = geometry_utils::from_wkb(buf, size, wkbGeneric); + mapnik::geometry::geometry geom; + if (output_twkb) + { + geom = geometry_utils::from_twkb(buf, size); + } + else + { + geom = geometry_utils::from_wkb(buf, size, wkbGeneric); + } if (!mapnik::geometry::is_empty(geom)) { box2d bbox = mapnik::geometry::envelope(geom);