sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

csv.input - implement get_geometry_type_impl for disk based index

Browse source
This commit is contained in:
artemp 2015-10-06 11:32:21 +01:00
parent 4d08e3f62c
commit d0cf7ad6f4
1 changed files with 74 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
plugins/input/csv/csv_datasource.cpp
View file

@ -323,7 +323,7 @@ void csv_datasource::parse_csv(T & stream)
{ {
auto values = csv_utils::parse_line(csv_line, separator_, quote_); auto values = csv_utils::parse_line(csv_line, separator_, quote_);
unsigned num_fields = values.size(); unsigned num_fields = values.size();
if (num_fields > num_headers) if (num_fields > num_headers || num_fields < num_headers)
{ {
std::ostringstream s; std::ostringstream s;
s << "CSV Plugin: # of columns(" s << "CSV Plugin: # of columns("
@ -331,14 +331,6 @@ void csv_datasource::parse_csv(T & stream)
<< num_headers << ") parsed for row " << line_number << "\n"; << num_headers << ") parsed for row " << line_number << "\n";
throw mapnik::datasource_exception(s.str()); throw mapnik::datasource_exception(s.str());
} }
else if (num_fields < num_headers)
{
std::ostringstream s;
s << "CSV Plugin: # of headers("
<< num_headers << ") > # of columns("
<< num_fields << ") parsed for row " << line_number << "\n";
throw mapnik::datasource_exception(s.str());
}
auto geom = detail::extract_geometry(values, locator_); auto geom = detail::extract_geometry(values, locator_);
if (!geom.is<mapnik::geometry::geometry_empty>()) if (!geom.is<mapnik::geometry::geometry_empty>())
@ -497,40 +489,88 @@ template <typename T>
boost::optional<mapnik::datasource_geometry_t> csv_datasource::get_geometry_type_impl(T & stream) const boost::optional<mapnik::datasource_geometry_t> csv_datasource::get_geometry_type_impl(T & stream) const
{ {
boost::optional<mapnik::datasource_geometry_t> result; boost::optional<mapnik::datasource_geometry_t> result;
int multi_type = 0; if (tree_)
auto itr = tree_->qbegin(boost::geometry::index::intersects(extent_));
auto end = tree_->qend();
for (std::size_t count = 0; itr !=end && count < 5; ++itr, ++count)
{ {
csv_datasource::item_type const& item = *itr; int multi_type = 0;
std::size_t file_offset = item.second.first; auto itr = tree_->qbegin(boost::geometry::index::intersects(extent_));
std::size_t size = item.second.second; auto end = tree_->qend();
stream.seekg(file_offset); for (std::size_t count = 0; itr !=end && count < 5; ++itr, ++count)
std::vector<char> record;
record.resize(size);
stream.read(record.data(), size);
std::string str(record.begin(), record.end());
try
{ {
auto values = csv_utils::parse_line(str, separator_, quote_); csv_datasource::item_type const& item = *itr;
auto geom = detail::extract_geometry(values, locator_); std::size_t file_offset = item.second.first;
result = mapnik::util::to_ds_type(geom); std::size_t size = item.second.second;
if (result) stream.seekg(file_offset);
std::vector<char> record;
record.resize(size);
stream.read(record.data(), size);
std::string str(record.begin(), record.end());
try
{ {
int type = static_cast<int>(*result); auto values = csv_utils::parse_line(str, separator_, quote_);
if (multi_type > 0 && multi_type != type) auto geom = detail::extract_geometry(values, locator_);
result = mapnik::util::to_ds_type(geom);
if (result)
{ {
result.reset(mapnik::datasource_geometry_t::Collection); int type = static_cast<int>(*result);
return result; if (multi_type > 0 && multi_type != type)
{
result.reset(mapnik::datasource_geometry_t::Collection);
return result;
}
multi_type = type;
} }
multi_type = type; }
catch (std::exception const& ex)
{
if (strict_) throw ex;
else MAPNIK_LOG_ERROR(csv) << ex.what();
} }
} }
catch (std::exception const& ex) }
else
{
// try reading *.index
using value_type = std::pair<std::size_t, std::size_t>;
std::ifstream index(filename_ + ".index", std::ios::binary);
if (!index) throw mapnik::datasource_exception("CSV Plugin: could not open: '" + filename_ + ".index'");
mapnik::filter_in_box filter(extent_);
std::vector<value_type> positions;
mapnik::util::spatial_index<value_type,
mapnik::filter_in_box,
std::ifstream>::query_first_n(filter, index, positions, 5);
int multi_type = 0;
for (auto const& val : positions)
{ {
if (strict_) throw ex; std::cerr << val.first << ":" << val.second << std::endl;
else MAPNIK_LOG_ERROR(csv) << ex.what(); stream.seekg(val.first);
std::vector<char> record;
record.resize(val.second);
stream.read(record.data(), val.second);
std::string str(record.begin(), record.end());
try
{
auto values = csv_utils::parse_line(str, separator_, quote_);
auto geom = detail::extract_geometry(values, locator_);
result = mapnik::util::to_ds_type(geom);
if (result)
{
int type = static_cast<int>(*result);
if (multi_type > 0 && multi_type != type)
{
result.reset(mapnik::datasource_geometry_t::Collection);
return result;
}
multi_type = type;
}
}
catch (std::exception const& ex)
{
if (strict_) throw ex;
else MAPNIK_LOG_ERROR(csv) << ex.what();
}
} }
} }
return result; return result;
} }