sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add initial 'qoute' auto-detection + restore csv_test's

Browse source
This commit is contained in:
artemp 2015-10-05 16:56:33 +01:00
parent 17b6597cd4
commit 63c73b5057
4 changed files with 22 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
plugins/input/csv/csv_datasource.cpp
View file

@ -71,7 +71,7 @@ csv_datasource::csv_datasource(parameters const& params)
row_limit_(*params.get<mapnik::value_integer>("row_limit", 0)),
inline_string_(),
separator_(0),
quote_('"'),
quote_(0),
headers_(),
manual_headers_(mapnik::util::trim_copy(*params.get<std::string>("headers", ""))),
strict_(*params.get<mapnik::boolean_type>("strict", false)),
@ -179,9 +179,9 @@ void csv_datasource::parse_csv(T & stream)
stream.seekg(0, std::ios::beg);
char newline;
bool has_newline;
std::tie(newline, has_newline) = detail::autodect_newline(stream, file_length);
char detected_quote;
std::tie(newline, has_newline, detected_quote) = detail::autodect_newline_and_quote(stream, file_length);
if (quote_ == 0) quote_ = detected_quote;
// set back to start
stream.seekg(0, std::ios::beg);
std::string csv_line;

15
plugins/input/csv/csv_utils.hpp
View file

@ -177,11 +177,13 @@ static inline char detect_separator(std::string const& str)
}
template <typename T>
std::tuple<char,bool> autodect_newline(T & stream, std::size_t file_length)
std::tuple<char,bool,char> autodect_newline_and_quote(T & stream, std::size_t file_length)
{
// autodetect newlines
char newline = '\n';
bool has_newline = false;
char quote = '"';
bool has_quote = false;
static std::size_t const max_size = 4000;
std::size_t size = std::min(file_length, max_size);
for (std::size_t lidx = 0; lidx < size; ++lidx)
@ -191,15 +193,20 @@ std::tuple<char,bool> autodect_newline(T & stream, std::size_t file_length)
{
newline = '\r';
has_newline = true;
break;
//break;
}
if (c == '\n')
{
has_newline = true;
break;
//break;
}
else if (!has_quote && c == '\'')
{
quote = '\'';
has_quote = true;
}
}
return std::make_tuple(newline,has_newline);
return std::make_tuple(newline, has_newline, quote);
}

4
test/standalone/csv_test.cpp
View file

@ -536,8 +536,8 @@ TEST_CASE("csv") {
for (auto const &file : {
std::string("test/data/csv/geojson_double_quote_escape.csv")
//, std::string("test/data/csv/geojson_single_quote.csv")
//, std::string("test/data/csv/geojson_2x_double_quote_filebakery_style.csv")
, std::string("test/data/csv/geojson_single_quote.csv")
, std::string("test/data/csv/geojson_2x_double_quote_filebakery_style.csv")
}) {
auto ds = get_csv_ds(file);
auto fields = ds->get_descriptor().get_descriptors();

9
utils/csvindex/csvindex.cpp
View file

@ -62,8 +62,8 @@ int main (int argc, char** argv)
unsigned int depth = DEFAULT_DEPTH;
double ratio = DEFAULT_RATIO;
vector<string> csv_files;
char separator;
char quote;
char separator = 0;
char quote = 0;
std::string manual_headers;
try
{
@ -176,8 +176,9 @@ int main (int argc, char** argv)
csv_file.seekg(0, std::ios::beg);
char newline;
bool has_newline;
std::tie(newline, has_newline) = detail::autodect_newline(csv_file, file_length);
char detected_quote;
std::tie(newline, has_newline, detected_quote) = detail::autodect_newline_and_quote(csv_file, file_length);
if (quote == 0) quote = detected_quote;
// set back to start
csv_file.seekg(0, std::ios::beg);
// get first line