sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

always query upto num_features_to_query to determine data schema (currently 5) across all access modes (ref #3237 #3238)

Browse source
This commit is contained in:
artemp 2016-02-25 15:27:21 +01:00
parent c044b6d073
commit f44b5ccfd9
2 changed files with 39 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
plugins/input/geojson/geojson_datasource.cpp
View file

@ -200,6 +200,20 @@ const mapnik::json::feature_grammar<base_iterator_type, mapnik::feature_impl> ge
const mapnik::json::extract_bounding_box_grammar<base_iterator_type> geojson_datasource_static_bbox_grammar; const mapnik::json::extract_bounding_box_grammar<base_iterator_type> geojson_datasource_static_bbox_grammar;
} }
void geojson_datasource::initialise_descriptor(mapnik::feature_ptr const& feature)
{
for ( auto const& kv : *feature)
{
auto const& name = std::get<0>(kv);
if (!desc_.has_name(name))
{
desc_.add_descriptor(mapnik::attribute_descriptor(name,
mapnik::util::apply_visitor(attr_value_converter(),
std::get<1>(kv))));
}
}
}
void geojson_datasource::initialise_disk_index(std::string const& filename) void geojson_datasource::initialise_disk_index(std::string const& filename)
{ {
// read extent // read extent
@ -213,7 +227,7 @@ void geojson_datasource::initialise_disk_index(std::string const& filename)
std::vector<value_type> positions; std::vector<value_type> positions;
mapnik::util::spatial_index<value_type, mapnik::util::spatial_index<value_type,
mapnik::filter_in_box, mapnik::filter_in_box,
std::ifstream>::query_first_n(filter, index, positions, 5); std::ifstream>::query_first_n(filter, index, positions, num_features_to_query_);
mapnik::util::file file(filename_); mapnik::util::file file(filename_);
if (!file.open()) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'"); if (!file.open()) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
@ -236,16 +250,7 @@ void geojson_datasource::initialise_disk_index(std::string const& filename)
{ {
throw std::runtime_error("Failed to parse geojson feature"); throw std::runtime_error("Failed to parse geojson feature");
} }
for ( auto const& kv : *feature) initialise_descriptor(feature);
{
auto const& name = std::get<0>(kv);
if (!desc_.has_name(name))
{
desc_.add_descriptor(mapnik::attribute_descriptor(name,
mapnik::util::apply_visitor(attr_value_converter(),
std::get<1>(kv))));
}
}
} }
} }
@ -285,20 +290,18 @@ void geojson_datasource::initialise_index(Iterator start, Iterator end)
if (geometry_index == 0) if (geometry_index == 0)
{ {
extent_ = box; extent_ = box;
for ( auto const& kv : *f)
{
desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
mapnik::util::apply_visitor(attr_value_converter(),
std::get<1>(kv))));
}
} }
else else
{ {
extent_.expand_to_include(box); extent_.expand_to_include(box);
} }
values.emplace_back(box, std::make_pair(geometry_index,0)); values.emplace_back(box, std::make_pair(geometry_index,0));
}
if (geometry_index++ < num_features_to_query_)
{
initialise_descriptor(f);
} }
++geometry_index;
} }
// packing algorithm // packing algorithm
tree_ = std::make_unique<spatial_index_type>(values); tree_ = std::make_unique<spatial_index_type>(values);
@ -308,14 +311,16 @@ void geojson_datasource::initialise_index(Iterator start, Iterator end)
// bulk insert initialise r-tree // bulk insert initialise r-tree
tree_ = std::make_unique<spatial_index_type>(boxes); tree_ = std::make_unique<spatial_index_type>(boxes);
// calculate total extent // calculate total extent
std::size_t feature_count = 0;
for (auto const& item : boxes) for (auto const& item : boxes)
{ {
auto const& box = std::get<0>(item); auto const& box = std::get<0>(item);
auto const& geometry_index = std::get<1>(item); auto const& geometry_index = std::get<1>(item);
if (!extent_.valid()) if (!extent_.valid()) extent_ = box;
else extent_.expand_to_include(box);
if (feature_count++ < num_features_to_query_)
{ {
extent_ = box; // parse first N features to extract attributes schema.
// parse first feature to extract attributes schema.
// NOTE: this doesn't yield correct answer for geoJSON in general, just an indication // NOTE: this doesn't yield correct answer for geoJSON in general, just an indication
Iterator itr2 = start + geometry_index.first; Iterator itr2 = start + geometry_index.first;
Iterator end2 = itr2 + geometry_index.second; Iterator end2 = itr2 + geometry_index.second;
@ -327,16 +332,8 @@ void geojson_datasource::initialise_index(Iterator start, Iterator end)
{ {
throw std::runtime_error("Failed to parse geojson feature"); throw std::runtime_error("Failed to parse geojson feature");
} }
for ( auto const& kv : *feature)
{ initialise_descriptor(feature);
desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
mapnik::util::apply_visitor(attr_value_converter(),
std::get<1>(kv))));
}
}
else
{
extent_.expand_to_include(box);
} }
} }
} }
@ -399,12 +396,7 @@ void geojson_datasource::parse_geojson(Iterator start, Iterator end)
if (geometry_index == 0) if (geometry_index == 0)
{ {
extent_ = box; extent_ = box;
for ( auto const& kv : *f)
{
desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
mapnik::util::apply_visitor(attr_value_converter(),
std::get<1>(kv))));
}
} }
else else
{ {
@ -412,6 +404,10 @@ void geojson_datasource::parse_geojson(Iterator start, Iterator end)
} }
values.emplace_back(box, std::make_pair(geometry_index,0)); values.emplace_back(box, std::make_pair(geometry_index,0));
} }
if (geometry_index < num_features_to_query_)
{
initialise_descriptor(f);
}
++geometry_index; ++geometry_index;
} }
// packing algorithm // packing algorithm
@ -419,7 +415,7 @@ void geojson_datasource::parse_geojson(Iterator start, Iterator end)
} }
geojson_datasource::~geojson_datasource() { } geojson_datasource::~geojson_datasource() {}
const char * geojson_datasource::name() const char * geojson_datasource::name()
{ {
@ -454,7 +450,7 @@ boost::optional<mapnik::datasource_geometry_t> geojson_datasource::get_geometry_
std::vector<value_type> positions; std::vector<value_type> positions;
mapnik::util::spatial_index<value_type, mapnik::util::spatial_index<value_type,
mapnik::filter_in_box, mapnik::filter_in_box,
std::ifstream>::query_first_n(filter, index, positions, 5); std::ifstream>::query_first_n(filter, index, positions, num_features_to_query_);
mapnik::util::file file(filename_); mapnik::util::file file(filename_);
@ -494,7 +490,7 @@ boost::optional<mapnik::datasource_geometry_t> geojson_datasource::get_geometry_
else if (cache_features_) else if (cache_features_)
{ {
unsigned num_features = features_.size(); unsigned num_features = features_.size();
for (unsigned i = 0; i < num_features && i < 5; ++i) for (unsigned i = 0; i < num_features && i < num_features_to_query_; ++i)
{ {
result = mapnik::util::to_ds_type(features_[i]->get_geometry()); result = mapnik::util::to_ds_type(features_[i]->get_geometry());
if (result) if (result)
@ -519,7 +515,7 @@ boost::optional<mapnik::datasource_geometry_t> geojson_datasource::get_geometry_
auto itr = tree_->qbegin(boost::geometry::index::intersects(extent_)); auto itr = tree_->qbegin(boost::geometry::index::intersects(extent_));
auto end = tree_->qend(); auto end = tree_->qend();
mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>(); mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
for (std::size_t count = 0; itr !=end && count < 5; ++itr,++count) for (std::size_t count = 0; itr !=end && count < num_features_to_query_; ++itr,++count)
{ {
geojson_datasource::item_type const& item = *itr; geojson_datasource::item_type const& item = *itr;
std::size_t file_offset = item.second.first; std::size_t file_offset = item.second.first;

2
plugins/input/geojson/geojson_datasource.hpp
View file

@ -94,6 +94,7 @@ public:
void initialise_index(Iterator start, Iterator end); void initialise_index(Iterator start, Iterator end);
void initialise_disk_index(std::string const& filename); void initialise_disk_index(std::string const& filename);
private: private:
void initialise_descriptor(mapnik::feature_ptr const&);
mapnik::datasource::datasource_t type_; mapnik::datasource::datasource_t type_;
mapnik::layer_descriptor desc_; mapnik::layer_descriptor desc_;
std::string filename_; std::string filename_;
@ -103,6 +104,7 @@ private:
std::unique_ptr<spatial_index_type> tree_; std::unique_ptr<spatial_index_type> tree_;
bool cache_features_ = true; bool cache_features_ = true;
bool has_disk_index_ = false; bool has_disk_index_ = false;
const std::size_t num_features_to_query_ = 5;
}; };