sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

csv.input - restore handling of inline headers and only one line of data without new line

Browse source
This commit is contained in:
artemp 2015-10-05 15:22:09 +01:00
parent 93a237850b
commit 5c6c8ff7a3
3 changed files with 15 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
plugins/input/csv/csv_datasource.cpp
View file

@ -179,17 +179,20 @@ void csv_datasource::parse_csv(T & stream)
stream.seekg(0, std::ios::beg); stream.seekg(0, std::ios::beg);
char newline; char newline;
bool has_newline; bool has_newline;
std::tie(newline, has_newline) = detail::autodect_newline(stream, file_length); std::tie(newline, has_newline) = detail::autodect_newline(stream, file_length);
// set back to start // set back to start
stream.seekg(0, std::ios::beg); stream.seekg(0, std::ios::beg);
std::string csv_line;
csv_utils::getline_csv(stream, csv_line, newline, quote_);
if (separator_ == 0) if (separator_ == 0)
{ {
separator_ = detail::detect_separator(stream, newline, quote_); separator_ = detail::detect_separator(csv_line);
} }
MAPNIK_LOG_DEBUG(csv) << "csv_datasource: separator: '" << separator_ MAPNIK_LOG_DEBUG(csv) << "csv_datasource: separator: '" << separator_
<< "' quote: '" << quote_ << "'"; << "' quote: '" << quote_ << "'";
stream.seekg(0, std::ios::beg); stream.seekg(0, std::ios::beg);
int line_number = 1; int line_number = 1;
@ -206,7 +209,6 @@ void csv_datasource::parse_csv(T & stream)
} }
else // parse first line as headers else // parse first line as headers
{ {
std::string csv_line;
while (csv_utils::getline_csv(stream, csv_line, newline, quote_)) while (csv_utils::getline_csv(stream, csv_line, newline, quote_))
{ {
try try
@ -274,13 +276,12 @@ void csv_datasource::parse_csv(T & stream)
[ & ](std::string const& header){ ctx_->push(header); }); [ & ](std::string const& header){ ctx_->push(header); });
mapnik::transcoder tr(desc_.get_encoding()); mapnik::transcoder tr(desc_.get_encoding());
auto pos = stream.tellg();
auto pos = stream.tellg();
// handle rare case of a single line of data and user-provided headers // handle rare case of a single line of data and user-provided headers
// where a lack of a newline will mean that csv_utils::getline_csv returns false // where a lack of a newline will mean that csv_utils::getline_csv returns false
#if 0 // FIXME
bool is_first_row = false; bool is_first_row = false;
if (!has_newline) if (!has_newline)
{ {
stream.setstate(std::ios::failbit); stream.setstate(std::ios::failbit);
@ -290,14 +291,13 @@ void csv_datasource::parse_csv(T & stream)
is_first_row = true; is_first_row = true;
} }
} }
#endif
if (has_disk_index_) return; if (has_disk_index_) return;
std::vector<item_type> boxes; std::vector<item_type> boxes;
std::string csv_line; while (is_first_row || csv_utils::getline_csv(stream, csv_line, newline, quote_))
while (/*is_first_row || */csv_utils::getline_csv(stream, csv_line, newline, quote_))
{ {
if ((row_limit_ > 0) && (line_number++ > row_limit_)) if ((row_limit_ > 0) && (line_number++ > row_limit_))
{ {
MAPNIK_LOG_DEBUG(csv) << "csv_datasource: row limit hit, exiting at feature: " << feature_count; MAPNIK_LOG_DEBUG(csv) << "csv_datasource: row limit hit, exiting at feature: " << feature_count;
@ -306,10 +306,10 @@ void csv_datasource::parse_csv(T & stream)
auto record_offset = pos; auto record_offset = pos;
auto record_size = csv_line.length(); auto record_size = csv_line.length();
pos = stream.tellg(); pos = stream.tellg();
//is_first_row = false; // FIXME is_first_row = false;
// skip blank lines // skip blank lines
unsigned line_length = csv_line.length(); if (record_size <= 10)
if (line_length <= 10)
{ {
std::string trimmed = csv_line; std::string trimmed = csv_line;
boost::trim_if(trimmed,boost::algorithm::is_any_of("\",'\r\n ")); boost::trim_if(trimmed,boost::algorithm::is_any_of("\",'\r\n "));

5
plugins/input/csv/csv_utils.hpp
View file

@ -141,11 +141,8 @@ std::size_t file_length(T & stream)
return stream.tellg(); return stream.tellg();
} }
template <typename InputStream> static inline char detect_separator(std::string const& str)
static inline char detect_separator(InputStream & stream, char delim, char quote)
{ {
std::string str;
csv_utils::getline_csv(stream, str, delim, quote);
char separator = ','; // default char separator = ','; // default
int num_commas = std::count(str.begin(), str.end(), ','); int num_commas = std::count(str.begin(), str.end(), ',');
// detect tabs // detect tabs

3
utils/csvindex/csvindex.cpp
View file

@ -183,8 +183,7 @@ int main (int argc, char** argv)
// get first line // get first line
std::string csv_line; std::string csv_line;
csv_utils::getline_csv(csv_file, csv_line, newline, quote); csv_utils::getline_csv(csv_file, csv_line, newline, quote);
//mapnik::util::trim(separator); if (separator == 0) separator = detail::detect_separator(csv_line);
if (separator == 0) separator = detail::detect_separator(csv_file, newline, quote);
csv_file.seekg(0, std::ios::beg); csv_file.seekg(0, std::ios::beg);
int line_number = 1; int line_number = 1;
detail::geometry_column_locator locator; detail::geometry_column_locator locator;