sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

csv_utils - check and skip processing lines consisting only of 0xa (windows) ( fixes #3359 )

Browse source
This commit is contained in:
artemp 2016-03-15 12:24:17 +01:00
parent 7044177af0
commit 9c09f15de9
6 changed files with 58 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
plugins/input/csv/csv_utils.cpp
View file

@ -128,11 +128,11 @@ std::tuple<char, bool, char, char> autodetect_csv_flavour(std::istream & stream,
{
if (size < file_length && ss.eof())
{
// we can't be sure last line
// we can't be sure that last line
// is not truncated so skip it
break;
}
if (line.size() == 0) continue; // empty lines are not interesting
if (line.size() == 0 || (line.size() == 1 && line[0] == char(0xa))) continue; // empty lines are not interesting
auto num_quotes = std::count(line.begin(), line.end(), quote);
if (num_quotes % 2 != 0)
{

34
plugins/input/sqlite/sqlite_datasource.cpp
View file

@ -53,22 +53,22 @@ using mapnik::parameters;
DATASOURCE_PLUGIN(sqlite_datasource)
sqlite_datasource::sqlite_datasource(parameters const& params)
: datasource(params),
extent_(),
extent_initialized_(false),
type_(datasource::Vector),
table_(*params.get<std::string>("table", "")),
fields_(*params.get<std::string>("fields", "*")),
metadata_(*params.get<std::string>("metadata", "")),
geometry_table_(*params.get<std::string>("geometry_table", "")),
geometry_field_(*params.get<std::string>("geometry_field", "")),
index_table_(*params.get<std::string>("index_table", "")),
key_field_(*params.get<std::string>("key_field", "")),
row_offset_(*params.get<mapnik::value_integer>("row_offset", 0)),
row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)),
intersects_token_("!intersects!"),
desc_(sqlite_datasource::name(), *params.get<std::string>("encoding", "utf-8")),
format_(mapnik::wkbAuto)
: datasource(params),
extent_(),
extent_initialized_(false),
type_(datasource::Vector),
table_(*params.get<std::string>("table", "")),
fields_(*params.get<std::string>("fields", "*")),
metadata_(*params.get<std::string>("metadata", "")),
geometry_table_(*params.get<std::string>("geometry_table", "")),
geometry_field_(*params.get<std::string>("geometry_field", "")),
index_table_(*params.get<std::string>("index_table", "")),
key_field_(*params.get<std::string>("key_field", "")),
row_offset_(*params.get<mapnik::value_integer>("row_offset", 0)),
row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)),
intersects_token_("!intersects!"),
desc_(sqlite_datasource::name(), *params.get<std::string>("encoding", "utf-8")),
format_(mapnik::wkbAuto)
{
/* TODO
- throw if no primary key but spatial index is present?
@ -448,7 +448,7 @@ boost::optional<mapnik::datasource_geometry_t> sqlite_datasource::get_geometry_t
if (data)
{
mapnik::geometry::geometry<double> geom = mapnik::geometry_utils::from_wkb(data, size, format_);
mapnik::geometry::geometry<double> geom = mapnik::geometry_utils::from_twkb(data, size);
if (mapnik::geometry::is_empty(geom))
{
continue;

2
plugins/input/sqlite/sqlite_featureset.cpp
View file

@ -81,7 +81,7 @@ feature_ptr sqlite_featureset::next()
}
feature_ptr feature = feature_factory::create(ctx_,rs_->column_integer64(1));
mapnik::geometry::geometry<double> geom = geometry_utils::from_wkb(data, size, format_);
mapnik::geometry::geometry<double> geom = geometry_utils::from_twkb(data, size);
if (mapnik::geometry::is_empty(geom))
{
continue;

14
test/unit/serialization/wkb_formats_test.cpp
View file

@ -1,4 +1,3 @@
#include "catch.hpp"
#include <iostream>
@ -8,6 +7,7 @@
#include <mapnik/geometry_is_simple.hpp>
#include <mapnik/geometry_correct.hpp>
#include <mapnik/feature_factory.hpp>
#include <mapnik/util/geometry_to_wkt.hpp>
#include <boost/version.hpp>
TEST_CASE("geometry formats") {
@ -64,8 +64,13 @@ SECTION("wkb") {
// spatialite blob
mapnik::geometry::geometry<double> geom = mapnik::geometry_utils::from_wkb((const char*)sp_valid_blob,
sizeof(sp_valid_blob) / sizeof(sp_valid_blob[0]),
mapnik::wkbSpatiaLite);
sizeof(sp_valid_blob) / sizeof(sp_valid_blob[0]),
mapnik::wkbSpatiaLite);
std::string wkt;
if (mapnik::util::to_wkt(wkt, geom))
{
std::cerr << wkt << std::endl;
}
// winding order is not correct per OGC so we'll fix it
mapnik::geometry::correct(geom);
#if BOOST_VERSION >= 105800
@ -76,7 +81,10 @@ SECTION("wkb") {
geom = mapnik::geometry_utils::from_wkb((const char*)sp_valid_blob,
sizeof(sp_valid_blob) / sizeof(sp_valid_blob[0]),
mapnik::wkbAuto);
mapnik::geometry::correct(geom);
#if BOOST_VERSION >= 105800
REQUIRE(mapnik::geometry::is_valid(geom));
REQUIRE(mapnik::geometry::is_simple(geom));

6
utils/pgsql2sqlite/main.cpp
View file

@ -58,7 +58,7 @@ int main ( int argc, char** argv)
("query,q",po::value<std::string>(),"Name of the table/or query to pass to postmaster")
("table,t",po::value<std::string>(),"Name of the output table to create (default: table in query)")
("file,f",po::value<std::string>(),"Use this option to specify the name of the file to create.")
("twkb", "Output geometries in TWKB format.")
;
//po::positional_options_description p;
@ -90,6 +90,8 @@ int main ( int argc, char** argv)
boost::optional<std::string> password;
boost::optional<std::string> connect_timeout("4");
bool output_twkb = vm.count("twkb") ? true : false;
if (vm.count("host")) host = vm["host"].as<std::string>();
if (vm.count("port")) port = vm["port"].as<std::string>();
if (vm.count("dbname")) dbname = vm["dbname"].as<std::string>();
@ -107,7 +109,7 @@ int main ( int argc, char** argv)
std::cout << "output_table : " << output_table_name << "\n";
mapnik::pgsql2sqlite(conn,query,output_table_name,output_file);
mapnik::pgsql2sqlite(conn, query, output_table_name, output_file, output_twkb);
}
catch (mapnik::datasource_exception & ex)
{

28
utils/pgsql2sqlite/pgsql2sqlite.hpp
View file

@ -165,7 +165,8 @@ template <typename Connection>
void pgsql2sqlite(Connection conn,
std::string const& query,
std::string const& output_table_name,
std::string const& output_filename)
std::string const& output_filename,
bool output_twkb)
{
namespace sqlite = mapnik::sqlite;
sqlite::database db(output_filename);
@ -223,11 +224,20 @@ void pgsql2sqlite(Connection conn,
geom_type = rs->getValue("type");
}
// add AsBinary(<geometry_column>) modifier
std::string select_sql_str = select_sql.str();
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","ST_AsBinary(" + geom_col+") as " + geom_col);
if (output_twkb)
{
// add AsTWKB(<geometry_column>, 2) modifier
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","ST_AsTWKB(" + geom_col+", 2) as " + geom_col);
}
else
{
// add AsBinary(<geometry_column>) modifier
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","ST_AsBinary(" + geom_col+") as " + geom_col);
}
#ifdef MAPNIK_DEBUG
//#ifdef MAPNIK_DEBUG
#if 1
std::cout << select_sql_str << "\n";
#endif
@ -387,7 +397,15 @@ void pgsql2sqlite(Connection conn,
if (oid == geometry_oid)
{
mapnik::feature_impl feat(ctx,pkid);
mapnik::geometry::geometry<double> geom = geometry_utils::from_wkb(buf, size, wkbGeneric);
mapnik::geometry::geometry<double> geom;
if (output_twkb)
{
geom = geometry_utils::from_twkb(buf, size);
}
else
{
geom = geometry_utils::from_wkb(buf, size, wkbGeneric);
}
if (!mapnik::geometry::is_empty(geom))
{
box2d<double> bbox = mapnik::geometry::envelope(geom);