sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

large_geojson.input - initial implementation

Browse source
This commit is contained in:
artemp 2015-01-16 16:22:46 +01:00
parent 8b19793dc8
commit 788fd8c80b
6 changed files with 689 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
SConstruct
View file

@ -119,6 +119,7 @@ PLUGINS = { # plugins with external dependencies
'csv': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
'raster': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
'geojson': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
'large_geojson': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
'topojson':{'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
'python': {'default':False,'path':None,'inc':None,'lib':None,'lang':'C++'},
}

65
plugins/input/large_geojson/build.py Normal file
View file

@ -0,0 +1,65 @@
#
# This file is part of Mapnik (c++ mapping toolkit)
#
# Copyright (C) 2013 Artem Pavlenko
#
# Mapnik is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
#
Import ('env')
Import ('plugin_base')
PLUGIN_NAME = 'large_geojson'
plugin_env = plugin_base.Clone()
plugin_sources = Split(
"""
%(PLUGIN_NAME)s_datasource.cpp
%(PLUGIN_NAME)s_featureset.cpp
""" % locals()
)
# Link Library to Dependencies
libraries = []
libraries.append(env['ICU_LIB_NAME'])
libraries.append('boost_system%s' % env['BOOST_APPEND'])
libraries.append('mapnik-json')
if env['PLUGIN_LINKING'] == 'shared':
libraries.append(env['MAPNIK_NAME'])
TARGET = plugin_env.SharedLibrary('../%s' % PLUGIN_NAME,
SHLIBPREFIX='',
SHLIBSUFFIX='.input',
source=plugin_sources,
LIBS=libraries)
# if the plugin links to libmapnik ensure it is built first
Depends(TARGET, env.subst('../../../src/%s' % env['MAPNIK_LIB_NAME']))
Depends(TARGET, env.subst('../../../src/json/libmapnik-json${LIBSUFFIX}'))
if 'uninstall' not in COMMAND_LINE_TARGETS:
env.Install(env['MAPNIK_INPUT_PLUGINS_DEST'], TARGET)
env.Alias('install', env['MAPNIK_INPUT_PLUGINS_DEST'])
plugin_obj = {
'LIBS': libraries,
'SOURCES': plugin_sources,
}
Return('plugin_obj')

349
plugins/input/large_geojson/large_geojson_datasource.cpp Normal file
View file

@ -0,0 +1,349 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2014 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#include "large_geojson_datasource.hpp"
#include "large_geojson_featureset.hpp"
#include <fstream>
#include <algorithm>
// boost
#include <boost/algorithm/string.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/support_multi_pass.hpp>
#include <boost/spirit/home/support/iterators/detail/functor_input_policy.hpp>
// mapnik
#include <mapnik/unicode.hpp>
#include <mapnik/utils.hpp>
#include <mapnik/feature.hpp>
#include <mapnik/feature_kv_iterator.hpp>
#include <mapnik/value_types.hpp>
#include <mapnik/box2d.hpp>
#include <mapnik/debug.hpp>
#include <mapnik/proj_transform.hpp>
#include <mapnik/projection.hpp>
#include <mapnik/util/geometry_to_ds_type.hpp>
#include <mapnik/util/variant.hpp>
#include <mapnik/util/file_io.hpp>
#include <mapnik/make_unique.hpp>
#include <mapnik/json/feature_collection_grammar.hpp>
#include <mapnik/json/extract_bounding_box_grammar_impl.hpp>
#include <mapnik/polygon_clipper.hpp> // boost::geometry - register box2d<double>
using mapnik::datasource;
using mapnik::parameters;
DATASOURCE_PLUGIN(large_geojson_datasource)
struct attr_value_converter
{
mapnik::eAttributeType operator() (mapnik::value_integer) const
{
return mapnik::Integer;
}
mapnik::eAttributeType operator() (double) const
{
return mapnik::Double;
}
mapnik::eAttributeType operator() (float) const
{
return mapnik::Double;
}
mapnik::eAttributeType operator() (bool) const
{
return mapnik::Boolean;
}
mapnik::eAttributeType operator() (std::string const& ) const
{
return mapnik::String;
}
mapnik::eAttributeType operator() (mapnik::value_unicode_string const&) const
{
return mapnik::String;
}
mapnik::eAttributeType operator() (mapnik::value_null const& ) const
{
return mapnik::String;
}
};
large_geojson_datasource::large_geojson_datasource(parameters const& params)
: datasource(params),
type_(datasource::Vector),
desc_(large_geojson_datasource::name(),
*params.get<std::string>("encoding","utf-8")),
filename_(),
inline_string_(),
extent_(),
features_(),
tree_(nullptr)
{
boost::optional<std::string> inline_string = params.get<std::string>("inline");
if (inline_string)
{
inline_string_ = *inline_string;
}
else
{
boost::optional<std::string> file = params.get<std::string>("file");
if (!file) throw mapnik::datasource_exception("GeoJSON Plugin: missing <file> parameter");
boost::optional<std::string> base = params.get<std::string>("base");
if (base)
filename_ = *base + "/" + *file;
else
filename_ = *file;
}
if (!inline_string_.empty())
{
parse_geojson(inline_string_);
}
else
{
std::ifstream file(filename_.c_str(), std::ios::binary);
if (!file)
{
throw mapnik::datasource_exception("Large GeoJSON Plugin: could not open: '" + filename_ + "'");
}
/*
using base_iterator_type = std::istreambuf_iterator<char>;
using chr_iterator_type =
boost::spirit::multi_pass
<base_iterator_type
, boost::spirit::iterator_policies::default_policy
< boost::spirit::iterator_policies::ref_counted
, boost::spirit::iterator_policies::no_check
//, boost::spirit::iterator_policies::functor_input
//, boost::spirit::iterator_policies::split_std_deque
>
> ;
base_iterator_type in(file);
chr_iterator_type start(in);
chr_iterator_type end;
*/
std::string json((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
std::string::const_iterator start = json.begin();
std::string::const_iterator end = json.end();
initialise_index(start, end);
/*
mapnik::util::file file(filename_);
if (!file.open())
{
throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
}
std::string file_buffer;
file_buffer.resize(file.size());
std::fread(&file_buffer[0], file.size(), 1, file.get());
parse_geojson(file_buffer);
*/
}
}
namespace {
using base_iterator_type = std::string::const_iterator;
const mapnik::transcoder tr("utf8");
const mapnik::json::feature_collection_grammar<base_iterator_type,mapnik::feature_impl> fc_grammar(tr);
}
template <typename T>
void large_geojson_datasource::parse_geojson(T const& buffer)
{
boost::spirit::standard_wide::space_type space;
mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
std::size_t start_id = 1;
bool result = boost::spirit::qi::phrase_parse(buffer.begin(), buffer.end(), (fc_grammar)
(boost::phoenix::ref(ctx),boost::phoenix::ref(start_id)),
space, features_);
if (!result)
{
if (!inline_string_.empty()) throw mapnik::datasource_exception("large_geojson_datasource: Failed parse GeoJSON file from in-memory string");
else throw mapnik::datasource_exception("large_geojson_datasource: Failed parse GeoJSON file '" + filename_ + "'");
}
#if BOOST_VERSION >= 105600
using values_container = std::vector< std::pair<box_type, std::size_t> >;
values_container values;
values.reserve(features_.size());
#else
tree_ = std::make_unique<spatial_index_type>(16, 4);
#endif
std::size_t geometry_index = 0;
for (mapnik::feature_ptr const& f : features_)
{
mapnik::box2d<double> box = f->envelope();
if (box.valid())
{
if (geometry_index == 0)
{
extent_ = box;
for ( auto const& kv : *f)
{
desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
mapnik::util::apply_visitor(attr_value_converter(),
std::get<1>(kv))));
}
}
else
{
extent_.expand_to_include(box);
}
}
#if BOOST_VERSION >= 105600
values.emplace_back(box, geometry_index);
#else
tree_->insert(box ,geometry_index);
#endif
++geometry_index;
}
#if BOOST_VERSION >= 105600
// packing algorithm
tree_ = std::make_unique<spatial_index_type>(values);
#endif
}
template <typename Iterator>
void large_geojson_datasource::initialise_index(Iterator start, Iterator end)
{
mapnik::json::boxes boxes;
mapnik::json::extract_bounding_box_grammar<Iterator> g;
boost::spirit::standard_wide::space_type space;
if (!boost::spirit::qi::phrase_parse(start, end, (g)(boost::phoenix::ref(boxes)) , space))
{
throw mapnik::datasource_exception("GeoJSON Plugin: could not parse: '" + filename_ + "'");
}
std::cerr << "OK size=" << boxes.size() << std::endl;
std::cerr << "Populate index" << std::endl;
tree_ = std::make_unique<spatial_index_type>(boxes);
std::cerr << "Calculate total extent" << std::endl;
for (auto const& item : boxes)
{
auto const& box = std::get<0>(item);
if (!extent_.valid())
{
extent_ = box;
}
else
{
extent_.expand_to_include(box);
}
}
}
large_geojson_datasource::~large_geojson_datasource() { }
const char * large_geojson_datasource::name()
{
return "large-geojson";
}
boost::optional<mapnik::datasource::geometry_t> large_geojson_datasource::get_geometry_type() const
{
boost::optional<mapnik::datasource::geometry_t> result;
int multi_type = 0;
unsigned num_features = features_.size();
for (unsigned i = 0; i < num_features && i < 5; ++i)
{
mapnik::util::to_ds_type(features_[i]->paths(),result);
if (result)
{
int type = static_cast<int>(*result);
if (multi_type > 0 && multi_type != type)
{
result.reset(mapnik::datasource::Collection);
return result;
}
multi_type = type;
}
}
return result;
}
mapnik::datasource::datasource_t large_geojson_datasource::type() const
{
return type_;
}
mapnik::box2d<double> large_geojson_datasource::envelope() const
{
return extent_;
}
mapnik::layer_descriptor large_geojson_datasource::get_descriptor() const
{
return desc_;
}
mapnik::featureset_ptr large_geojson_datasource::features(mapnik::query const& q) const
{
// if the query box intersects our world extent then query for features
mapnik::box2d<double> const& box = q.get_bbox();
if (extent_.intersects(box))
{
std::cerr << "extent_.intersects(box)" << std::endl;
//box_type box(point_type(b.minx(),b.miny()),point_type(b.maxx(),b.maxy()));
#if BOOST_VERSION >= 105600
large_geojson_featureset::array_type index_array;
if (tree_)
{
tree_->query(boost::geometry::index::intersects(box),std::back_inserter(index_array));
std::cerr << "Query size=" << index_array.size() << std::endl;
return std::make_shared<large_geojson_featureset>(filename_, std::move(index_array));
}
#else
if (tree_)
{
return std::make_shared<large_geojson_featureset>(features_, tree_->find(box));
}
#endif
}
// otherwise return an empty featureset pointer
return mapnik::featureset_ptr();
}
mapnik::featureset_ptr large_geojson_datasource::features_at_point(mapnik::coord2d const& pt, double tol) const
{
mapnik::box2d<double> query_bbox(pt, pt);
query_bbox.pad(tol);
mapnik::query q(query_bbox);
std::vector<mapnik::attribute_descriptor> const& desc = desc_.get_descriptors();
std::vector<mapnik::attribute_descriptor>::const_iterator itr = desc.begin();
std::vector<mapnik::attribute_descriptor>::const_iterator end = desc.end();
for ( ;itr!=end;++itr)
{
q.add_property_name(itr->get_name());
}
return features(q);
}

125
plugins/input/large_geojson/large_geojson_datasource.hpp Normal file
View file

@ -0,0 +1,125 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2014 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#ifndef LARGE_GEOJSON_DATASOURCE_HPP
#define LARGE_GEOJSON_DATASOURCE_HPP
// mapnik
#include <mapnik/datasource.hpp>
#include <mapnik/params.hpp>
#include <mapnik/query.hpp>
#include <mapnik/feature.hpp>
#include <mapnik/box2d.hpp>
#include <mapnik/coord.hpp>
#include <mapnik/feature_layer_desc.hpp>
#include <mapnik/unicode.hpp>
// boost
#include <boost/optional.hpp>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wunused-local-typedef"
#include <boost/geometry/geometries/point_xy.hpp>
#include <boost/geometry/geometries/box.hpp>
#include <boost/geometry/geometries/geometries.hpp>
#include <boost/geometry.hpp>
#include <boost/version.hpp>
#if BOOST_VERSION >= 105600
#include <boost/geometry/index/rtree.hpp>
#else
#include <boost/geometry/extensions/index/rtree/rtree.hpp>
#endif
#pragma GCC diagnostic pop
// stl
#include <memory>
#include <vector>
#include <string>
#include <map>
#include <deque>
#if BOOST_VERSION >= 105600
template <std::size_t Max, std::size_t Min>
struct geojson_linear : boost::geometry::index::linear<Max,Min> {};
namespace boost { namespace geometry { namespace index { namespace detail { namespace rtree {
template <std::size_t Max, std::size_t Min>
struct options_type<geojson_linear<Max,Min> >
{
using type = options<geojson_linear<Max, Min>,
insert_default_tag,
choose_by_content_diff_tag,
split_default_tag,
linear_tag,
#if BOOST_VERSION >= 105700
node_variant_static_tag>;
#else
node_s_mem_static_tag>;
#endif
};
}}}}}
#endif //BOOST_VERSION >= 105600
class large_geojson_datasource : public mapnik::datasource
{
public:
//using point_type = boost::geometry::model::point<double, 2, boost::geometry::cs::cartesian>;
using box_type = mapnik::box2d<double>;//boost::geometry::model::box<point_type>;
using item_type = std::pair<box_type,std::size_t>;
using spatial_index_type = boost::geometry::index::rtree<item_type,geojson_linear<16,4> >;
// constructor
large_geojson_datasource(mapnik::parameters const& params);
virtual ~large_geojson_datasource ();
mapnik::datasource::datasource_t type() const;
static const char * name();
mapnik::featureset_ptr features(mapnik::query const& q) const;
mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt, double tol = 0) const;
mapnik::box2d<double> envelope() const;
mapnik::layer_descriptor get_descriptor() const;
boost::optional<mapnik::datasource::geometry_t> get_geometry_type() const;
template <typename T>
void parse_geojson(T const& buffer);
template <typename Iterator>
void initialise_index(Iterator start, Iterator end);
private:
mapnik::datasource::datasource_t type_;
mapnik::layer_descriptor desc_;
std::string filename_;
std::string inline_string_;
mapnik::box2d<double> extent_;
std::vector<mapnik::feature_ptr> features_;
std::unique_ptr<spatial_index_type> tree_;
};
#endif // LARGE_GEOJSON_DATASOURCE_HPP

99
plugins/input/large_geojson/large_geojson_featureset.cpp Normal file
View file

@ -0,0 +1,99 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2014 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
// mapnik
#include <mapnik/feature.hpp>
#include <mapnik/feature_factory.hpp>
#include <mapnik/json/geometry_grammar_impl.hpp>
#include <mapnik/json/feature_grammar_impl.hpp>
// boost
#include <boost/spirit/include/support_multi_pass.hpp>
#include <boost/spirit/home/support/iterators/detail/functor_input_policy.hpp>
// stl
#include <string>
#include <vector>
#include <deque>
#include "large_geojson_featureset.hpp"
//namespace {
//using base_iterator_type = std::string::const_iterator;
//const mapnik::transcoder tr("utf8");
//const mapnik::json::feature_collection_grammar<base_iterator_type,mapnik::feature_impl> fc_grammar(tr);
//}
large_geojson_featureset::large_geojson_featureset(std::string const& filename,
array_type && index_array)
: file_(filename, std::ios::binary),
index_array_(std::move(index_array)),
index_itr_(index_array_.begin()),
index_end_(index_array_.end()),
ctx_(std::make_shared<mapnik::context_type>())
{
if (!file_) throw std::runtime_error("Can't open " + filename);
}
large_geojson_featureset::~large_geojson_featureset() {}
mapnik::feature_ptr large_geojson_featureset::next()
{
if (index_itr_ != index_end_)
{
#if BOOST_VERSION >= 105600
large_geojson_datasource::item_type const& item = *index_itr_++;
std::size_t file_offset = item.second;
//std::cerr << file_offset << " -- " << item.first << std::endl;
#else
std::size_t index = *index_itr_++;
#endif
using base_iterator_type = std::istreambuf_iterator<char>;
using chr_iterator_type =
boost::spirit::multi_pass
< base_iterator_type
, boost::spirit::iterator_policies::default_policy
< boost::spirit::iterator_policies::ref_counted
, boost::spirit::iterator_policies::no_check
//, boost::spirit::iterator_policies::functor_input
//, boost::spirit::iterator_policies::split_std_deque
>
> ;
file_.seekg(file_offset);
base_iterator_type in(file_);
chr_iterator_type start(in);
chr_iterator_type end;
static const mapnik::transcoder tr("utf8");
static const mapnik::json::feature_grammar<chr_iterator_type,mapnik::feature_impl> grammar(tr);
using namespace boost::spirit;
standard_wide::space_type space;
mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx_,1));
if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space))
{
throw std::runtime_error("Failed to parse geojson feature");
}
return feature;
}
return mapnik::feature_ptr();
}

50
plugins/input/large_geojson/large_geojson_featureset.hpp Normal file
View file

@ -0,0 +1,50 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2014 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#ifndef LARGE_GEOJSON_FEATURESET_HPP
#define LARGE_GEOJSON_FEATURESET_HPP
#include <mapnik/feature.hpp>
#include "large_geojson_datasource.hpp"
#include <vector>
#include <deque>
#include <fstream>
class large_geojson_featureset : public mapnik::Featureset
{
public:
typedef std::deque<large_geojson_datasource::item_type> array_type;
large_geojson_featureset(std::string const& filename,
array_type && index_array);
virtual ~large_geojson_featureset();
mapnik::feature_ptr next();
private:
std::ifstream file_;
const array_type index_array_;
array_type::const_iterator index_itr_;
array_type::const_iterator index_end_;
mapnik::context_ptr ctx_;
};
#endif // LARGE_GEOJSON_FEATURESET_HPP