sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add initial 'qoute' auto-detection + restore csv_test's

Browse source
This commit is contained in:
artemp 2015-10-05 16:56:33 +01:00
parent 17b6597cd4
commit 63c73b5057
4 changed files with 22 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
plugins/input/csv/csv_datasource.cpp
View file

@ -71,7 +71,7 @@ csv_datasource::csv_datasource(parameters const& params)
row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)), row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)),
inline_string_(), inline_string_(),
separator_(0), separator_(0),
quote_('"'), quote_(0),
headers_(), headers_(),
manual_headers_(mapnik::util::trim_copy(*params.get<std::string>("headers", ""))), manual_headers_(mapnik::util::trim_copy(*params.get<std::string>("headers", ""))),
strict_(*params.get<mapnik::boolean_type>("strict", false)), strict_(*params.get<mapnik::boolean_type>("strict", false)),
@ -179,9 +179,9 @@ void csv_datasource::parse_csv(T & stream)
stream.seekg(0, std::ios::beg); stream.seekg(0, std::ios::beg);
char newline; char newline;
bool has_newline; bool has_newline;
char detected_quote;
std::tie(newline, has_newline) = detail::autodect_newline(stream, file_length); std::tie(newline, has_newline, detected_quote) = detail::autodect_newline_and_quote(stream, file_length);
if (quote_ == 0) quote_ = detected_quote;
// set back to start // set back to start
stream.seekg(0, std::ios::beg); stream.seekg(0, std::ios::beg);
std::string csv_line; std::string csv_line;

15
plugins/input/csv/csv_utils.hpp
View file

@ -177,11 +177,13 @@ static inline char detect_separator(std::string const& str)
} }
template <typename T> template <typename T>
std::tuple<char,bool> autodect_newline(T & stream, std::size_t file_length) std::tuple<char,bool,char> autodect_newline_and_quote(T & stream, std::size_t file_length)
{ {
// autodetect newlines // autodetect newlines
char newline = '\n'; char newline = '\n';
bool has_newline = false; bool has_newline = false;
char quote = '"';
bool has_quote = false;
static std::size_t const max_size = 4000; static std::size_t const max_size = 4000;
std::size_t size = std::min(file_length, max_size); std::size_t size = std::min(file_length, max_size);
for (std::size_t lidx = 0; lidx < size; ++lidx) for (std::size_t lidx = 0; lidx < size; ++lidx)
@ -191,15 +193,20 @@ std::tuple<char,bool> autodect_newline(T & stream, std::size_t file_length)
{ {
newline = '\r'; newline = '\r';
has_newline = true; has_newline = true;
break; //break;
} }
if (c == '\n') if (c == '\n')
{ {
has_newline = true; has_newline = true;
break; //break;
}
else if (!has_quote && c == '\'')
{
quote = '\'';
has_quote = true;
} }
} }
return std::make_tuple(newline,has_newline); return std::make_tuple(newline, has_newline, quote);
} }

4
test/standalone/csv_test.cpp
View file

@ -536,8 +536,8 @@ TEST_CASE("csv") {
for (auto const &file : { for (auto const &file : {
std::string("test/data/csv/geojson_double_quote_escape.csv") std::string("test/data/csv/geojson_double_quote_escape.csv")
//, std::string("test/data/csv/geojson_single_quote.csv") , std::string("test/data/csv/geojson_single_quote.csv")
//, std::string("test/data/csv/geojson_2x_double_quote_filebakery_style.csv") , std::string("test/data/csv/geojson_2x_double_quote_filebakery_style.csv")
}) { }) {
auto ds = get_csv_ds(file); auto ds = get_csv_ds(file);
auto fields = ds->get_descriptor().get_descriptors(); auto fields = ds->get_descriptor().get_descriptors();

9
utils/csvindex/csvindex.cpp
View file

@ -62,8 +62,8 @@ int main (int argc, char** argv)
unsigned int depth = DEFAULT_DEPTH; unsigned int depth = DEFAULT_DEPTH;
double ratio = DEFAULT_RATIO; double ratio = DEFAULT_RATIO;
vector<string> csv_files; vector<string> csv_files;
char separator; char separator = 0;
char quote; char quote = 0;
std::string manual_headers; std::string manual_headers;
try try
{ {
@ -176,8 +176,9 @@ int main (int argc, char** argv)
csv_file.seekg(0, std::ios::beg); csv_file.seekg(0, std::ios::beg);
char newline; char newline;
bool has_newline; bool has_newline;
std::tie(newline, has_newline) = detail::autodect_newline(csv_file, file_length); char detected_quote;
std::tie(newline, has_newline, detected_quote) = detail::autodect_newline_and_quote(csv_file, file_length);
if (quote == 0) quote = detected_quote;
// set back to start // set back to start
csv_file.seekg(0, std::ios::beg); csv_file.seekg(0, std::ios::beg);
// get first line // get first line