sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

csv_utils::getline_csv - add 'quote' argument

Browse source
This commit is contained in:
artemp 2015-10-02 12:17:38 +01:00
parent 2d59fca9f9
commit 98ea1c5cd9
3 changed files with 15 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
plugins/input/csv/csv_datasource.cpp
View file

@ -172,9 +172,13 @@ void csv_datasource::parse_csv(T & stream,
std::tie(newline, has_newline) = detail::autodect_newline(stream, file_length);
// set back to start
stream.seekg(0, std::ios::beg);
std::string quo = mapnik::util::trim_copy(quote);
if (quo.empty()) quo = "\"";
// get first line
std::string csv_line;
csv_utils::getline_csv(stream,csv_line,stream.widen(newline));
csv_utils::getline_csv(stream, csv_line, newline, quo[0]);
// if user has not passed a separator manually
// then attempt to detect by reading first line
@ -188,9 +192,6 @@ void csv_datasource::parse_csv(T & stream,
std::string esc = mapnik::util::trim_copy(escape);
if (esc.empty()) esc = "\\";
std::string quo = mapnik::util::trim_copy(quote);
if (quo.empty()) quo = "\"";
MAPNIK_LOG_DEBUG(csv) << "csv_datasource: csv grammar: sep: '" << sep
<< "' quo: '" << quo << "' esc: '" << esc << "'";
@ -208,7 +209,7 @@ void csv_datasource::parse_csv(T & stream,
}
else // parse first line as headers
{
while (csv_utils::getline_csv(stream,csv_line,stream.widen(newline)))
while (csv_utils::getline_csv(stream,csv_line,newline, quo[0]))
{
try
{
@ -293,7 +294,7 @@ void csv_datasource::parse_csv(T & stream,
if (has_disk_index_) return;
std::vector<item_type> boxes;
while (is_first_row || csv_utils::getline_csv(stream, csv_line, stream.widen(newline)))
while (is_first_row || csv_utils::getline_csv(stream, csv_line, newline, quo[0]))
{
if ((row_limit_ > 0) && (line_number++ > row_limit_))
{

10
plugins/input/csv/csv_utils.hpp
View file

@ -93,7 +93,7 @@ inline bool ignore_case_equal(std::string const& s0, std::string const& s1)
}
template <class CharT, class Traits, class Allocator>
std::basic_istream<CharT, Traits>& getline_csv(std::istream& is, std::basic_string<CharT,Traits,Allocator>& s, CharT delim)
std::basic_istream<CharT, Traits>& getline_csv(std::istream& is, std::basic_string<CharT,Traits,Allocator>& s, CharT delim, CharT quote)
{
typename std::basic_string<CharT,Traits,Allocator>::size_type nread = 0;
typename std::basic_istream<CharT, Traits>::sentry sentry(is, true);
@ -101,7 +101,7 @@ std::basic_istream<CharT, Traits>& getline_csv(std::istream& is, std::basic_stri
{
std::basic_streambuf<CharT, Traits>* buf = is.rdbuf();
s.clear();
bool quote = false;
bool has_quote = false;
while (nread < s.max_size())
{
int c1 = buf->sbumpc();
@ -114,9 +114,9 @@ std::basic_istream<CharT, Traits>& getline_csv(std::istream& is, std::basic_stri
{
++nread;
CharT c = Traits::to_char_type(c1);
if (Traits::eq(c,'"') || Traits::eq(c,'\''))
quote = !quote;
if (!Traits::eq(c, delim) || quote)
if (Traits::eq(c, quote))
has_quote = !has_quote;
if (!Traits::eq(c, delim) || has_quote)
s.push_back(c);
else
break;// Character is extracted but not appended.

6
utils/csvindex/csvindex.cpp
View file

@ -187,7 +187,7 @@ int main (int argc, char** argv)
csv_file.seekg(0, std::ios::beg);
// get first line
std::string csv_line;
csv_utils::getline_csv(csv_file, csv_line, csv_file.widen(newline));
csv_utils::getline_csv(csv_file, csv_line, newline, quote[0] );
mapnik::util::trim(separator);
if (separator.empty()) separator = detail::detect_separator(csv_line);
csv_file.seekg(0, std::ios::beg);
@ -213,7 +213,7 @@ int main (int argc, char** argv)
}
else // parse first line as headers
{
while (csv_utils::getline_csv(csv_file,csv_line,csv_file.widen(newline)))
while (csv_utils::getline_csv(csv_file,csv_line,newline, quote[0]))
{
try
{
@ -281,7 +281,7 @@ int main (int argc, char** argv)
using item_type = std::pair<box_type, std::pair<unsigned, unsigned>>;
std::vector<item_type> boxes;
while (is_first_row || csv_utils::getline_csv(csv_file, csv_line, csv_file.widen(newline)))
while (is_first_row || csv_utils::getline_csv(csv_file, csv_line, csv_file.widen(newline), quote[0]))
{
auto record_offset = pos;
auto record_size = csv_line.length();