sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
mapnik/plugins/input/geojson/geojson_datasource.cpp
Mathis Logemann e7c3d04309 format dir include and src 
format all files

Revert "format all files"

This reverts commit 95d5812e49e7f916b68e786596f5a8eb5bcac414.

Revert "format some files"

This reverts commit ed3c8762d4d828b2b28e7b18809fc33f4f8ccaf5.

format all files

fix formatting in dir include

fix formatting of debug macro
2022-01-27 00:12:08 +01:00

600 lines
21 KiB
C++
Raw Blame History

/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2021 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#include "geojson_datasource.hpp"
#include "geojson_featureset.hpp"
#include "geojson_index_featureset.hpp"
#include "geojson_memory_index_featureset.hpp"
#include <fstream>
#include <algorithm>
#include <mapnik/warning.hpp>
MAPNIK_DISABLE_WARNING_PUSH
#include <mapnik/warning_ignore.hpp>
#include <boost/algorithm/string.hpp>
MAPNIK_DISABLE_WARNING_POP
// mapnik
#include <mapnik/boolean.hpp>
#include <mapnik/unicode.hpp>
#include <mapnik/util/utf_conv_win.hpp>
#include <mapnik/feature.hpp>
#include <mapnik/feature_factory.hpp>
#include <mapnik/feature_kv_iterator.hpp>
#include <mapnik/value/types.hpp>
#include <mapnik/geometry/box2d.hpp>
#include <mapnik/debug.hpp>
#include <mapnik/proj_transform.hpp>
#include <mapnik/projection.hpp>
#include <mapnik/util/variant.hpp>
#include <mapnik/util/file_io.hpp>
#include <mapnik/util/geometry_to_ds_type.hpp>
#include <mapnik/geometry/boost_adapters.hpp>
#include <mapnik/util/fs.hpp>
#include <mapnik/util/spatial_index.hpp>
#include <mapnik/geom_util.hpp>
#include <mapnik/json/parse_feature.hpp>
#include <mapnik/json/extract_bounding_boxes_x3.hpp>
#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#include <mapnik/warning.hpp>
MAPNIK_DISABLE_WARNING_PUSH
#include <mapnik/warning_ignore.hpp>
#include <boost/interprocess/mapped_region.hpp>
MAPNIK_DISABLE_WARNING_POP
#include <mapnik/mapped_memory_cache.hpp>
#endif
using mapnik::datasource;
using mapnik::parameters;
DATASOURCE_PLUGIN(geojson_datasource)
struct attr_value_converter
{
mapnik::eAttributeType operator()(mapnik::value_integer) const { return mapnik::Integer; }
mapnik::eAttributeType operator()(double) const { return mapnik::Double; }
mapnik::eAttributeType operator()(float) const { return mapnik::Double; }
mapnik::eAttributeType operator()(bool) const { return mapnik::Boolean; }
mapnik::eAttributeType operator()(std::string const&) const { return mapnik::String; }
mapnik::eAttributeType operator()(mapnik::value_unicode_string const&) const { return mapnik::String; }
mapnik::eAttributeType operator()(mapnik::value_null const&) const { return mapnik::String; }
};
geojson_datasource::geojson_datasource(parameters const& params)
: datasource(params)
, type_(datasource::Vector)
, desc_(geojson_datasource::name(), *params.get<std::string>("encoding", "utf-8"))
, filename_()
, from_inline_string_(false)
, extent_()
, features_()
, tree_(nullptr)
, num_features_to_query_(
std::max(mapnik::value_integer(1), *params.get<mapnik::value_integer>("num_features_to_query", 5)))
{
boost::optional<std::string> inline_string = params.get<std::string>("inline");
if (!inline_string)
{
boost::optional<std::string> file = params.get<std::string>("file");
if (!file)
throw mapnik::datasource_exception("GeoJSON Plugin: missing <file> parameter");
boost::optional<std::string> base = params.get<std::string>("base");
if (base)
filename_ = *base + "/" + *file;
else
filename_ = *file;
has_disk_index_ = mapnik::util::exists(filename_ + ".index");
}
if (inline_string)
{
from_inline_string_ = true;
char const* start = inline_string->c_str();
char const* end = start + inline_string->size();
parse_geojson(start, end);
}
else if (has_disk_index_)
{
initialise_disk_index(filename_);
}
else
{
cache_features_ = *params.get<mapnik::boolean_type>("cache_features", true);
#if !defined(MAPNIK_MEMORY_MAPPED_FILE)
mapnik::util::file file(filename_);
if (!file)
{
throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
}
std::string file_buffer;
file_buffer.resize(file.size());
auto count = std::fread(&file_buffer[0], file.size(), 1, file.get());
char const* start = file_buffer.c_str();
char const* end = (count == 1) ? start + file_buffer.length() : start;
#else
boost::optional<mapnik::mapped_region_ptr> mapped_region =
mapnik::mapped_memory_cache::instance().find(filename_, false);
if (!mapped_region)
{
throw std::runtime_error("could not get file mapping for " + filename_);
}
char const* start = reinterpret_cast<char const*>((*mapped_region)->get_address());
char const* end = start + (*mapped_region)->get_size();
#endif
if (cache_features_)
{
parse_geojson(start, end);
}
else
{
initialise_index(start, end);
}
}
}
namespace {
using box_type = box2d<double>;
using boxes_type = std::vector<std::pair<box_type, std::pair<std::uint64_t, std::uint64_t>>>;
using base_iterator_type = char const*;
const mapnik::transcoder geojson_datasource_static_tr("utf8");
} // namespace
void geojson_datasource::initialise_descriptor(mapnik::feature_ptr const& feature)
{
for (auto const& kv : *feature)
{
auto const& name = std::get<0>(kv);
if (!desc_.has_name(name))
{
desc_.add_descriptor(
mapnik::attribute_descriptor(name, mapnik::util::apply_visitor(attr_value_converter(), std::get<1>(kv))));
}
}
}
void geojson_datasource::initialise_disk_index(std::string const& filename)
{
// read extent
using value_type = mapnik::util::index_record;
std::ifstream index(filename_ + ".index", std::ios::binary);
if (!index)
throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + ".index'");
auto ext_f =
mapnik::util::spatial_index<value_type, mapnik::bounding_box_filter<float>, std::ifstream, mapnik::box2d<float>>::
bounding_box(index);
extent_ = {ext_f.minx(), ext_f.miny(), ext_f.maxx(), ext_f.maxy()};
mapnik::bounding_box_filter<float> filter(ext_f);
std::vector<value_type> positions;
mapnik::util::spatial_index<value_type, mapnik::bounding_box_filter<float>, std::ifstream, mapnik::box2d<float>>::
query_first_n(filter, index, positions, num_features_to_query_);
mapnik::util::file file(filename_);
if (!file)
throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
for (auto const& pos : positions)
{
std::fseek(file.get(), pos.off, SEEK_SET);
std::vector<char> record;
record.resize(pos.size);
auto count = std::fread(record.data(), pos.size, 1, file.get());
auto const* start = record.data();
auto const* end = (count == 1) ? start + record.size() : start;
mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, -1));
try
{
mapnik::json::parse_feature(start, end, *feature, geojson_datasource_static_tr);
} catch (...)
{
throw std::runtime_error("Failed to parse geojson feature");
}
initialise_descriptor(feature);
}
desc_.order_by_name();
}
namespace mapnik {
namespace json {
template<typename Iterator>
void parse_geometry_or_feature(Iterator& itr, Iterator const& end, feature_impl& feature, transcoder const& tr)
{
Iterator start = itr;
try
{
parse_feature(start, end, feature, tr);
} catch (...)
{
start = itr;
parse_geometry(start, end, feature);
}
}
} // namespace json
} // namespace mapnik
template<typename Iterator>
void geojson_datasource::initialise_index(Iterator start, Iterator end)
{
boxes_type boxes;
mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
Iterator itr = start;
try
{
mapnik::json::extract_bounding_boxes(itr, end, boxes);
if (itr != end || boxes.empty())
throw std::exception();
// bulk insert initialise r-tree
tree_ = std::make_unique<spatial_index_type>(boxes);
// calculate total extent
std::size_t feature_count = 0;
for (auto const& item : boxes)
{
auto const& box = std::get<0>(item);
auto const& geometry_index = std::get<1>(item);
if (!extent_.valid())
extent_ = box;
else
extent_.expand_to_include(box);
if (feature_count++ < num_features_to_query_)
{
// parse first N features to extract attributes schema.
// NOTE: this doesn't yield correct answer for geoJSON in general, just an indication
mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, -1)); // temp feature
Iterator itr2 = start + geometry_index.first;
Iterator end2 = itr2 + geometry_index.second;
try
{
mapnik::json::parse_feature(itr2, end2, *feature, geojson_datasource_static_tr);
} catch (...)
{
throw std::runtime_error("Failed to parse geojson feature");
}
initialise_descriptor(feature);
}
}
} catch (...)
{
cache_features_ = true; // force caching single feature
itr = start; // reset iteraror
std::size_t start_id = 1;
mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, start_id)); // single feature
try
{
mapnik::json::parse_geometry_or_feature(itr, end, *feature, geojson_datasource_static_tr);
} catch (...)
{
if (from_inline_string_)
throw mapnik::datasource_exception(
"geojson_datasource: Failed to parse GeoJSON file from in-memory string");
else
throw mapnik::datasource_exception("geojson_datasource: Failed to parse GeoJSON file '" + filename_ +
"'");
}
features_.push_back(std::move(feature));
using values_container = std::vector<std::pair<box_type, std::pair<std::uint64_t, std::uint64_t>>>;
values_container values;
values.reserve(features_.size());
std::size_t geometry_index = 0;
for (mapnik::feature_ptr const& f : features_)
{
mapnik::box2d<double> box = f->envelope();
if (box.valid())
{
if (geometry_index == 0)
{
extent_ = box;
}
else
{
extent_.expand_to_include(box);
}
values.emplace_back(box, std::make_pair(geometry_index, 0));
}
if (geometry_index++ < num_features_to_query_)
{
initialise_descriptor(f);
}
}
// packing algorithm
tree_ = std::make_unique<spatial_index_type>(values);
}
desc_.order_by_name();
}
template<typename Iterator>
void geojson_datasource::parse_geojson(Iterator start, Iterator end)
{
mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
std::size_t start_id = 1;
Iterator itr = start;
try
{
boxes_type boxes;
mapnik::json::extract_bounding_boxes(itr, end, boxes);
if (itr != end || boxes.empty())
throw std::exception(); // ensure we've consumed all input and we extracted at least one bbox;
for (auto const& item : boxes)
{
auto const& geometry_index = std::get<1>(item);
Iterator itr2 = start + geometry_index.first;
Iterator end2 = itr2 + geometry_index.second;
mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, start_id++));
mapnik::json::parse_feature(itr2, end2, *feature, geojson_datasource_static_tr);
features_.push_back(std::move(feature));
}
} catch (...)
{
itr = start;
// try parsing as single Feature or single Geometry JSON
mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, start_id)); // single feature
try
{
mapnik::json::parse_geometry_or_feature(itr, end, *feature, geojson_datasource_static_tr);
} catch (...)
{
if (from_inline_string_)
throw mapnik::datasource_exception(
"geojson_datasource: Failed to parse GeoJSON file from in-memory string");
else
throw mapnik::datasource_exception("geojson_datasource: Failed to parse GeoJSON file '" + filename_ +
"'");
}
features_.push_back(std::move(feature));
}
using values_container = std::vector<std::pair<box_type, std::pair<std::uint64_t, std::uint64_t>>>;
values_container values;
values.reserve(features_.size());
std::size_t geometry_index = 0;
for (mapnik::feature_ptr const& f : features_)
{
mapnik::box2d<double> box = f->envelope();
if (box.valid())
{
if (geometry_index == 0)
{
extent_ = box;
}
else
{
extent_.expand_to_include(box);
}
values.emplace_back(box, std::make_pair(geometry_index, 0));
}
if (geometry_index < num_features_to_query_)
{
initialise_descriptor(f);
}
++geometry_index;
}
// packing algorithm
tree_ = std::make_unique<spatial_index_type>(values);
}
geojson_datasource::~geojson_datasource() {}
const char* geojson_datasource::name()
{
return "geojson";
}
mapnik::datasource::datasource_t geojson_datasource::type() const
{
return type_;
}
mapnik::box2d<double> geojson_datasource::envelope() const
{
return extent_;
}
mapnik::layer_descriptor geojson_datasource::get_descriptor() const
{
return desc_;
}
boost::optional<mapnik::datasource_geometry_t> geojson_datasource::get_geometry_type() const
{
boost::optional<mapnik::datasource_geometry_t> result;
int multi_type = 0;
if (has_disk_index_)
{
using value_type = mapnik::util::index_record;
std::ifstream index(filename_ + ".index", std::ios::binary);
if (!index)
throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + ".index'");
mapnik::bounding_box_filter<float> filter(
mapnik::box2d<float>(extent_.minx(), extent_.miny(), extent_.maxx(), extent_.maxy()));
std::vector<value_type> positions;
mapnik::util::spatial_index<value_type,
mapnik::bounding_box_filter<float>,
std::ifstream,
mapnik::box2d<float>>::query_first_n(filter,
index,
positions,
num_features_to_query_);
mapnik::util::file file(filename_);
if (!file)
throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
for (auto const& pos : positions)
{
std::fseek(file.get(), pos.off, SEEK_SET);
std::vector<char> record;
record.resize(pos.size);
auto count = std::fread(record.data(), pos.size, 1, file.get());
auto const* start = record.data();
auto const* end = (count == 1) ? start + record.size() : start;
mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, -1)); // temp feature
try
{
mapnik::json::parse_feature(start, end, *feature, geojson_datasource_static_tr);
} catch (...)
{
throw std::runtime_error("Failed to parse geojson feature");
}
result = mapnik::util::to_ds_type(feature->get_geometry());
if (result)
{
int type = static_cast<int>(*result);
if (multi_type > 0 && multi_type != type)
{
result.reset(mapnik::datasource_geometry_t::Collection);
return result;
}
multi_type = type;
}
}
}
else if (cache_features_)
{
std::size_t num_features = features_.size();
for (std::size_t i = 0; i < num_features && i < num_features_to_query_; ++i)
{
result = mapnik::util::to_ds_type(features_[i]->get_geometry());
if (result)
{
int type = static_cast<int>(*result);
if (multi_type > 0 && multi_type != type)
{
result.reset(mapnik::datasource_geometry_t::Collection);
return result;
}
multi_type = type;
}
}
}
else
{
mapnik::util::file file(filename_);
if (!file)
{
throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
}
auto itr = tree_->qbegin(boost::geometry::index::intersects(extent_));
auto end = tree_->qend();
mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
for (std::size_t count = 0; itr != end && count < num_features_to_query_; ++itr, ++count)
{
geojson_datasource::item_type const& item = *itr;
std::uint64_t file_offset = item.second.first;
std::uint64_t size = item.second.second;
std::fseek(file.get(), file_offset, SEEK_SET);
std::vector<char> json;
json.resize(size);
auto count_objects = std::fread(json.data(), size, 1, file.get());
using chr_iterator_type = char const*;
chr_iterator_type start2 = json.data();
chr_iterator_type end2 = (count_objects == 1) ? start2 + json.size() : start2;
mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, -1)); // temp feature
try
{
mapnik::json::parse_feature(start2, end2, *feature, geojson_datasource_static_tr);
} catch (...)
{
throw std::runtime_error("Failed to parse geojson feature");
}
result = mapnik::util::to_ds_type(feature->get_geometry());
if (result)
{
int type = static_cast<int>(*result);
if (multi_type > 0 && multi_type != type)
{
result.reset(mapnik::datasource_geometry_t::Collection);
return result;
}
multi_type = type;
}
}
}
return result;
}
mapnik::featureset_ptr geojson_datasource::features(mapnik::query const& q) const
{
// if the query box intersects our world extent then query for features
mapnik::box2d<double> const& box = q.get_bbox();
if (extent_.intersects(box))
{
geojson_featureset::array_type index_array;
if (tree_)
{
tree_->query(boost::geometry::index::intersects(box), std::back_inserter(index_array));
// sort index array to preserve original feature ordering in GeoJSON
std::sort(index_array.begin(), index_array.end(), [](item_type const& item0, item_type const& item1) {
return item0.second.first < item1.second.first;
});
if (cache_features_)
{
return std::make_shared<geojson_featureset>(features_, std::move(index_array));
}
else
{
return std::make_shared<geojson_memory_index_featureset>(filename_, std::move(index_array));
}
}
else if (has_disk_index_)
{
auto const& bbox = q.get_bbox();
mapnik::bounding_box_filter<float> const filter(
mapnik::box2d<float>(bbox.minx(), bbox.miny(), bbox.maxx(), bbox.maxy()));
return std::make_shared<geojson_index_featureset>(filename_, filter);
}
}
// otherwise return an empty featureset
return mapnik::make_invalid_featureset();
}
mapnik::featureset_ptr geojson_datasource::features_at_point(mapnik::coord2d const& pt, double tol) const
{
mapnik::box2d<double> query_bbox(pt, pt);
query_bbox.pad(tol);
mapnik::query q(query_bbox);
for (auto const& attr_info : desc_.get_descriptors())
{
q.add_property_name(attr_info.get_name());
}
return features(q);
}
Reference in a new issue View git blame Copy permalink