sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

csv_utils - check and skip processing lines consisting only of 0xa (windows) ( fixes #3359 )

Browse source
This commit is contained in:
artemp 2016-03-15 12:24:17 +01:00
parent 7044177af0
commit 9c09f15de9
6 changed files with 58 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
plugins/input/csv/csv_utils.cpp
View file

@ -128,11 +128,11 @@ std::tuple<char, bool, char, char> autodetect_csv_flavour(std::istream & stream,
{
if (size < file_length && ss.eof())
{
// we can't be sure last line
// we can't be sure that last line
// is not truncated so skip it
break;
}
if (line.size() == 0) continue; // empty lines are not interesting
if (line.size() == 0 || (line.size() == 1 && line[0] == char(0xa))) continue; // empty lines are not interesting
auto num_quotes = std::count(line.begin(), line.end(), quote);
if (num_quotes % 2 != 0)
{

2
plugins/input/sqlite/sqlite_datasource.cpp
View file

@ -448,7 +448,7 @@ boost::optional<mapnik::datasource_geometry_t> sqlite_datasource::get_geometry_t
if (data)
{
mapnik::geometry::geometry<double> geom = mapnik::geometry_utils::from_wkb(data, size, format_);
mapnik::geometry::geometry<double> geom = mapnik::geometry_utils::from_twkb(data, size);
if (mapnik::geometry::is_empty(geom))
{
continue;

2
plugins/input/sqlite/sqlite_featureset.cpp
View file

@ -81,7 +81,7 @@ feature_ptr sqlite_featureset::next()
}
feature_ptr feature = feature_factory::create(ctx_,rs_->column_integer64(1));
mapnik::geometry::geometry<double> geom = geometry_utils::from_wkb(data, size, format_);
mapnik::geometry::geometry<double> geom = geometry_utils::from_twkb(data, size);
if (mapnik::geometry::is_empty(geom))
{
continue;

10
test/unit/serialization/wkb_formats_test.cpp
View file

@ -1,4 +1,3 @@
#include "catch.hpp"
#include <iostream>
@ -8,6 +7,7 @@
#include <mapnik/geometry_is_simple.hpp>
#include <mapnik/geometry_correct.hpp>
#include <mapnik/feature_factory.hpp>
#include <mapnik/util/geometry_to_wkt.hpp>
#include <boost/version.hpp>
TEST_CASE("geometry formats") {
@ -66,6 +66,11 @@ SECTION("wkb") {
mapnik::geometry::geometry<double> geom = mapnik::geometry_utils::from_wkb((const char*)sp_valid_blob,
sizeof(sp_valid_blob) / sizeof(sp_valid_blob[0]),
mapnik::wkbSpatiaLite);
std::string wkt;
if (mapnik::util::to_wkt(wkt, geom))
{
std::cerr << wkt << std::endl;
}
// winding order is not correct per OGC so we'll fix it
mapnik::geometry::correct(geom);
#if BOOST_VERSION >= 105800
@ -76,7 +81,10 @@ SECTION("wkb") {
geom = mapnik::geometry_utils::from_wkb((const char*)sp_valid_blob,
sizeof(sp_valid_blob) / sizeof(sp_valid_blob[0]),
mapnik::wkbAuto);
mapnik::geometry::correct(geom);
#if BOOST_VERSION >= 105800
REQUIRE(mapnik::geometry::is_valid(geom));
REQUIRE(mapnik::geometry::is_simple(geom));

6
utils/pgsql2sqlite/main.cpp
View file

@ -58,7 +58,7 @@ int main ( int argc, char** argv)
("query,q",po::value<std::string>(),"Name of the table/or query to pass to postmaster")
("table,t",po::value<std::string>(),"Name of the output table to create (default: table in query)")
("file,f",po::value<std::string>(),"Use this option to specify the name of the file to create.")
("twkb", "Output geometries in TWKB format.")
;
//po::positional_options_description p;
@ -90,6 +90,8 @@ int main ( int argc, char** argv)
boost::optional<std::string> password;
boost::optional<std::string> connect_timeout("4");
bool output_twkb = vm.count("twkb") ? true : false;
if (vm.count("host")) host = vm["host"].as<std::string>();
if (vm.count("port")) port = vm["port"].as<std::string>();
if (vm.count("dbname")) dbname = vm["dbname"].as<std::string>();
@ -107,7 +109,7 @@ int main ( int argc, char** argv)
std::cout << "output_table : " << output_table_name << "\n";
mapnik::pgsql2sqlite(conn,query,output_table_name,output_file);
mapnik::pgsql2sqlite(conn, query, output_table_name, output_file, output_twkb);
}
catch (mapnik::datasource_exception & ex)
{

26
utils/pgsql2sqlite/pgsql2sqlite.hpp
View file

@ -165,7 +165,8 @@ template <typename Connection>
void pgsql2sqlite(Connection conn,
std::string const& query,
std::string const& output_table_name,
std::string const& output_filename)
std::string const& output_filename,
bool output_twkb)
{
namespace sqlite = mapnik::sqlite;
sqlite::database db(output_filename);
@ -223,11 +224,20 @@ void pgsql2sqlite(Connection conn,
geom_type = rs->getValue("type");
}
// add AsBinary(<geometry_column>) modifier
std::string select_sql_str = select_sql.str();
if (output_twkb)
{
// add AsTWKB(<geometry_column>, 2) modifier
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","ST_AsTWKB(" + geom_col+", 2) as " + geom_col);
}
else
{
// add AsBinary(<geometry_column>) modifier
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","ST_AsBinary(" + geom_col+") as " + geom_col);
}
#ifdef MAPNIK_DEBUG
//#ifdef MAPNIK_DEBUG
#if 1
std::cout << select_sql_str << "\n";
#endif
@ -387,7 +397,15 @@ void pgsql2sqlite(Connection conn,
if (oid == geometry_oid)
{
mapnik::feature_impl feat(ctx,pkid);
mapnik::geometry::geometry<double> geom = geometry_utils::from_wkb(buf, size, wkbGeneric);
mapnik::geometry::geometry<double> geom;
if (output_twkb)
{
geom = geometry_utils::from_twkb(buf, size);
}
else
{
geom = geometry_utils::from_wkb(buf, size, wkbGeneric);
}
if (!mapnik::geometry::is_empty(geom))
{
box2d<double> bbox = mapnik::geometry::envelope(geom);