From d59460861e78a219ba1fc6326b565fe05642282e Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Mon, 1 Feb 2016 11:51:23 -0800 Subject: [PATCH 1/2] update to latest variant --- deps/mapbox/variant | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/mapbox/variant b/deps/mapbox/variant index 64596e3aa..24391a9ea 160000 --- a/deps/mapbox/variant +++ b/deps/mapbox/variant @@ -1 +1 @@ -Subproject commit 64596e3aa2c0f47ab3e74a50c5c49acd5c57f5d5 +Subproject commit 24391a9ea3e4b060c7a62debf6eb4e3515b9a5e6 From e76a1d9a8184da0d38c601190385faa70a0cc836 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Mon, 1 Feb 2016 12:23:26 -0800 Subject: [PATCH 2/2] [csv] use faster version of csv_utils::parse_line in more places + fix error output --- plugins/input/csv/csv_datasource.cpp | 19 ++++++++++++++----- plugins/input/csv/csv_inline_featureset.cpp | 4 +++- utils/mapnik-index/process_csv_file.cpp | 20 ++++++++++++++------ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/plugins/input/csv/csv_datasource.cpp b/plugins/input/csv/csv_datasource.cpp index d0dbf06d9..537feeead 100644 --- a/plugins/input/csv/csv_datasource.cpp +++ b/plugins/input/csv/csv_datasource.cpp @@ -321,14 +321,23 @@ void csv_datasource::parse_csv(T & stream) try { - auto values = csv_utils::parse_line(csv_line, separator_, quote_); + auto const* line_start = csv_line.data(); + auto const* line_end = line_start + csv_line.size(); + auto values = csv_utils::parse_line(line_start, line_end, separator_, quote_, num_headers); unsigned num_fields = values.size(); - if (num_fields > num_headers || num_fields < num_headers) + if (num_fields != num_headers) { std::ostringstream s; - s << "CSV Plugin: # of columns(" - << num_fields << ") > # of headers(" - << num_headers << ") parsed for row " << line_number; + s << "CSV Plugin: # of columns(" << num_fields << ")"; + if (num_fields > num_headers) + { + s << " > "; + } + else + { + s << " < "; + } + s << "# of headers(" << num_headers << ") parsed"; throw mapnik::datasource_exception(s.str()); } diff --git a/plugins/input/csv/csv_inline_featureset.cpp b/plugins/input/csv/csv_inline_featureset.cpp index b0fe420cf..195574b6f 100644 --- a/plugins/input/csv/csv_inline_featureset.cpp +++ b/plugins/input/csv/csv_inline_featureset.cpp @@ -54,7 +54,9 @@ csv_inline_featureset::~csv_inline_featureset() {} mapnik::feature_ptr csv_inline_featureset::parse_feature(std::string const& str) { - auto values = csv_utils::parse_line(str, separator_, quote_); + auto const* start = str.data(); + auto const* end = start + str.size(); + auto values = csv_utils::parse_line(start, end, separator_, quote_, headers_.size()); auto geom = detail::extract_geometry(values, locator_); if (!geom.is()) { diff --git a/utils/mapnik-index/process_csv_file.cpp b/utils/mapnik-index/process_csv_file.cpp index 908ec9a79..bfe7d3064 100644 --- a/utils/mapnik-index/process_csv_file.cpp +++ b/utils/mapnik-index/process_csv_file.cpp @@ -186,15 +186,23 @@ std::pair> process_csv_file(T & boxes, std::string const& fil } try { - auto values = csv_utils::parse_line(csv_line, separator, quote); + auto const* start_line = csv_line.data(); + auto const* end_line = start_line + csv_line.size(); + auto values = csv_utils::parse_line(start_line, end_line, separator, quote, num_headers); unsigned num_fields = values.size(); - if (num_fields > num_headers || num_fields < num_headers) + if (num_fields != num_headers) { - // skip this row std::ostringstream s; - s << "CSV Index: # of columns(" - << num_fields << ") > # of headers(" - << num_headers << ") parsed for row " << line_number; + s << "CSV Plugin: # of columns(" << num_fields << ")"; + if (num_fields > num_headers) + { + s << " > "; + } + else + { + s << " < "; + } + s << "# of headers(" << num_headers << ") parsed"; throw mapnik::datasource_exception(s.str()); }