sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into spirit-x3

Browse source
This commit is contained in:
artemp 2016-01-05 10:25:23 +00:00
commit 11c6896520
12 changed files with 225 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
include/mapnik/quad_tree.hpp
View file

@ -181,12 +181,7 @@ public:
"Values stored in quad-tree must be standard layout types to allow serialisation");
char header[16];
std::memset(header,0,16);
header[0]='m';
header[1]='a';
header[2]='p';
header[3]='n';
header[4]='i';
header[5]='k';
std::strcpy(header,"mapnik-index");
out.write(header,16);
write_node(out,root_);
}

19
include/mapnik/util/spatial_index.hpp
View file

@ -30,6 +30,7 @@
#include <mapnik/geom_util.hpp>
// stl
#include <type_traits>
#include <cstring>
using mapnik::box2d;
using mapnik::query;
@ -44,7 +45,6 @@ public:
static box2d<double> bounding_box( InputStream& in );
static void query_first_n(Filter const& filter, InputStream & in, std::vector<Value>& pos, std::size_t count);
private:
spatial_index();
~spatial_index();
spatial_index(spatial_index const&);
@ -53,12 +53,23 @@ private:
static void read_envelope(InputStream& in, box2d<double>& envelope);
static void query_node(Filter const& filter, InputStream& in, std::vector<Value> & results);
static void query_first_n_impl(Filter const& filter, InputStream& in, std::vector<Value> & results, std::size_t count);
static bool check_header(InputStream& in);
};
template <typename Value, typename Filter, typename InputStream>
bool spatial_index<Value, Filter, InputStream>::check_header(InputStream& in)
{
static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout type");
char header[17]; // mapnik-index
std::memset(header, 0, 17);
in.read(header,16);
return (std::strncmp(header, "mapnik-index",12) == 0);
}
template <typename Value, typename Filter, typename InputStream>
box2d<double> spatial_index<Value, Filter, InputStream>::bounding_box(InputStream& in)
{
static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout type");
if (!check_header(in)) throw std::runtime_error("Invalid index file");
in.seekg(16 + 4, std::ios::beg);
box2d<double> box;
read_envelope(in, box);
@ -69,7 +80,7 @@ box2d<double> spatial_index<Value, Filter, InputStream>::bounding_box(InputStrea
template <typename Value, typename Filter, typename InputStream>
void spatial_index<Value, Filter, InputStream>::query(Filter const& filter, InputStream& in, std::vector<Value>& results)
{
static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout types");
if (!check_header(in)) throw std::runtime_error("Invalid index file");
in.seekg(16, std::ios::beg);
query_node(filter, in, results);
}
@ -104,7 +115,7 @@ void spatial_index<Value, Filter, InputStream>::query_node(Filter const& filter,
template <typename Value, typename Filter, typename InputStream>
void spatial_index<Value, Filter, InputStream>::query_first_n(Filter const& filter, InputStream& in, std::vector<Value>& results, std::size_t count)
{
static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout types");
if (!check_header(in)) throw std::runtime_error("Invalid index file");
in.seekg(16, std::ios::beg);
query_first_n_impl(filter, in, results, count);
}

23
plugins/input/shape/shape_index_featureset.cpp
View file

@ -63,12 +63,13 @@ shape_index_featureset<filterT>::shape_index_featureset(filterT const& filter,
if (index)
{
#if defined(MAPNIK_MEMORY_MAPPED_FILE)
mapnik::util::spatial_index<int, filterT,boost::interprocess::ibufferstream>::query(filter, index->file(), offsets_);
mapnik::util::spatial_index<mapnik::detail::node, filterT,boost::interprocess::ibufferstream>::query(filter, index->file(), offsets_);
#else
mapnik::util::spatial_index<int, filterT, std::ifstream>::query(filter, index->file(), offsets_);
mapnik::util::spatial_index<mapnik::detail::node, filterT, std::ifstream>::query(filter, index->file(), offsets_);
#endif
}
std::sort(offsets_.begin(), offsets_.end());
std::sort(offsets_.begin(), offsets_.end(), [](mapnik::detail::node const& n0, mapnik::detail::node const& n1)
{return n0.offset != n1.offset ? n0.offset < n1.offset : n0.start < n1.start;});
MAPNIK_LOG_DEBUG(shape) << "shape_index_featureset: Query size=" << offsets_.size();
itr_ = offsets_.begin();
}
@ -83,7 +84,15 @@ feature_ptr shape_index_featureset<filterT>::next()
while ( itr_ != offsets_.end())
{
shape_ptr_->move_to(*itr_++);
int offset = itr_->offset;
shape_ptr_->move_to(offset);
std::vector<std::pair<int,int>> parts;
while (itr_->offset == offset && itr_ != offsets_.end())
{
if (itr_->start!= -1) parts.emplace_back(itr_->start, itr_->end);
++itr_;
}
//std::cerr << "PARTS SIZE=" << parts.size() <<" offset=" << offset << std::endl;
mapnik::value_integer feature_id = shape_ptr_->id();
shape_file::record_type record(shape_ptr_->reclength_ * 2);
shape_ptr_->shp().read_record(record);
@ -124,7 +133,8 @@ feature_ptr shape_index_featureset<filterT>::next()
{
shape_io::read_bbox(record, feature_bbox_);
if (!filter_.pass(feature_bbox_)) continue;
feature->set_geometry(shape_io::read_polyline(record));
if (parts.size() < 2) feature->set_geometry(shape_io::read_polyline(record));
else feature->set_geometry(shape_io::read_polyline_parts(record, parts));
break;
}
case shape_io::shape_polygon:
@ -133,7 +143,8 @@ feature_ptr shape_index_featureset<filterT>::next()
{
shape_io::read_bbox(record, feature_bbox_);
if (!filter_.pass(feature_bbox_)) continue;
feature->set_geometry(shape_io::read_polygon(record));
if (parts.size() < 2) feature->set_geometry(shape_io::read_polygon(record));
else feature->set_geometry(shape_io::read_polygon_parts(record, parts));
break;
}
default :

19
plugins/input/shape/shape_index_featureset.hpp
View file

@ -45,6 +45,21 @@ using mapnik::box2d;
using mapnik::feature_ptr;
using mapnik::context_ptr;
namespace mapnik { namespace detail
{
struct node
{
node() = default;
node(int offset_, int start_, int end_)
: offset(offset_),
start(start_),
end(end_) {}
int offset;
int start;
int end;
};
}} // ns
template <typename filterT>
class shape_index_featureset : public Featureset
{
@ -63,8 +78,8 @@ private:
context_ptr ctx_;
std::unique_ptr<shape_io> shape_ptr_;
const std::unique_ptr<mapnik::transcoder> tr_;
std::vector<int> offsets_;
std::vector<int>::iterator itr_;
std::vector<mapnik::detail::node> offsets_;
std::vector<mapnik::detail::node>::iterator itr_;
std::vector<int> attr_ids_;
mapnik::value_integer row_limit_;
mutable int count_;

79
plugins/input/shape/shape_io.cpp
View file

@ -125,6 +125,7 @@ mapnik::geometry::geometry<double> shape_io::read_polyline(shape_file::record_ty
std::for_each(parts.begin(), parts.end(), [&](int & part) { part = record.read_ndr_integer();});
int start, end;
mapnik::geometry::multi_line_string<double> multi_line;
multi_line.reserve(num_parts);
for (int k = 0; k < num_parts; ++k)
{
start = parts[k];
@ -152,6 +153,34 @@ mapnik::geometry::geometry<double> shape_io::read_polyline(shape_file::record_ty
return geom;
}
mapnik::geometry::geometry<double> shape_io::read_polyline_parts(shape_file::record_type & record, std::vector<std::pair<int, int>> const& parts)
{
mapnik::geometry::geometry<double> geom; // default empty
int total_num_parts = record.read_ndr_integer();
int num_parts = parts.size();
mapnik::geometry::multi_line_string<double> multi_line;
multi_line.reserve(num_parts);
for (int k = 0; k < num_parts; ++k)
{
int start = parts[k].first;
int end = parts[k].second;
unsigned pos = 4 + 32 + 8 + 4 * total_num_parts + start * 16;
record.set_pos(pos);
mapnik::geometry::line_string<double> line;
line.reserve(end - start);
for (int j = start; j < end; ++j)
{
double x = record.read_double();
double y = record.read_double();
line.emplace_back(x, y);
}
multi_line.push_back(std::move(line));
}
geom = std::move(multi_line);
return geom;
}
mapnik::geometry::geometry<double> shape_io::read_polygon(shape_file::record_type & record)
{
@ -207,3 +236,53 @@ mapnik::geometry::geometry<double> shape_io::read_polygon(shape_file::record_typ
mapnik::geometry::correct(geom);
return geom;
}
mapnik::geometry::geometry<double> shape_io::read_polygon_parts(shape_file::record_type & record, std::vector<std::pair<int,int>> const& parts)
{
mapnik::geometry::geometry<double> geom; // default empty
int total_num_parts = record.read_ndr_integer();
mapnik::geometry::polygon<double> poly;
mapnik::geometry::multi_polygon<double> multi_poly;
int num_parts = parts.size();
for (int k = 0; k < num_parts; ++k)
{
int start = parts[k].first;
int end = parts[k].second;
unsigned pos = 4 + 32 + 8 + 4 * total_num_parts + start * 16;
record.set_pos(pos);
mapnik::geometry::linear_ring<double> ring;
ring.reserve(end - start);
for (int j = start; j < end; ++j)
{
double x = record.read_double();
double y = record.read_double();
ring.emplace_back(x, y);
}
if (k == 0)
{
poly.set_exterior_ring(std::move(ring));
}
else if (mapnik::util::is_clockwise(ring))
{
multi_poly.emplace_back(std::move(poly));
poly.interior_rings.clear();
poly.set_exterior_ring(std::move(ring));
}
else
{
poly.add_hole(std::move(ring));
}
}
if (multi_poly.size() > 0) // multi
{
multi_poly.emplace_back(std::move(poly));
geom = std::move(multi_poly);
}
else
{
geom = std::move(poly);
}
mapnik::geometry::correct(geom);
return geom;
}

2
plugins/input/shape/shape_io.hpp
View file

@ -80,6 +80,8 @@ public:
static void read_bbox(shape_file::record_type & record, mapnik::box2d<double> & bbox);
static mapnik::geometry::geometry<double> read_polyline(shape_file::record_type & record);
static mapnik::geometry::geometry<double> read_polygon(shape_file::record_type & record);
static mapnik::geometry::geometry<double> read_polyline_parts(shape_file::record_type & record,std::vector<std::pair<int,int>> const& parts);
static mapnik::geometry::geometry<double> read_polygon_parts(shape_file::record_type & record, std::vector<std::pair<int,int>> const& parts);
shapeType type_;
shape_file shp_;

5
plugins/input/shape/shapefile.hpp
View file

@ -105,6 +105,11 @@ struct shape_record
pos += n;
}
void set_pos(unsigned pos_)
{
pos = pos_;
}
int read_ndr_integer()
{
std::int32_t val;

2
test/unit/datasource/csv.cpp
View file

@ -80,7 +80,7 @@ int create_disk_index(std::string const& filename, bool silent = true)
std::string cmd;
if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
{
cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
cmd += std::string("DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " ";
}
cmd += "mapnik-index " + filename;
if (silent)

2
test/unit/datasource/geojson.cpp
View file

@ -66,7 +66,7 @@ int create_disk_index(std::string const& filename, bool silent = true)
std::string cmd;
if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
{
cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
cmd += std::string("DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " ";
}
cmd += "mapnik-index " + filename;
if (silent)

4
test/unit/datasource/postgis.cpp
View file

@ -43,7 +43,7 @@ int run(std::string const& command, bool silent = false)
std::string cmd;
if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
{
cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
cmd += std::string("DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " ";
}
cmd += command;
if (silent)
@ -91,4 +91,4 @@ TEST_CASE("postgis") {
}
}
}

54
test/unit/datasource/shapeindex.cpp
View file

@ -58,14 +58,17 @@ std::size_t count_shapefile_features(std::string const& filename)
return feature_count;
}
int create_shapefile_index(std::string const& filename, bool silent = true)
int create_shapefile_index(std::string const& filename, bool index_parts, bool silent = true)
{
std::string cmd;
if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
{
cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
cmd += std::string("DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " ";
}
cmd += "shapeindex " + filename;
cmd += "shapeindex";
if (index_parts) cmd+= " --index-parts ";
cmd += filename;
if (silent)
{
#ifndef _WINDOWS
@ -90,28 +93,31 @@ TEST_CASE("shapeindex")
{
if (boost::iends_with(path,".shp"))
{
std::string index_path = path.substr(0, path.rfind(".")) + ".index";
// remove *.index if present
if (mapnik::util::exists(index_path))
for (bool index_parts : {false, true} )
{
mapnik::util::remove(index_path);
}
// count features
std::size_t feature_count = count_shapefile_features(path);
// create *.index
create_shapefile_index(path);
if (feature_count == 0)
{
REQUIRE(!mapnik::util::exists(index_path)); // index won't be created if there's no features
}
// count features
std::size_t feature_count_indexed = count_shapefile_features(path);
// ensure number of features are the same
REQUIRE(feature_count == feature_count_indexed);
// remove *.index if present
if (mapnik::util::exists(index_path))
{
mapnik::util::remove(index_path);
std::string index_path = path.substr(0, path.rfind(".")) + ".index";
// remove *.index if present
if (mapnik::util::exists(index_path))
{
mapnik::util::remove(index_path);
}
// count features
std::size_t feature_count = count_shapefile_features(path);
// create *.index
create_shapefile_index(path, index_parts);
if (feature_count == 0)
{
REQUIRE(!mapnik::util::exists(index_path)); // index won't be created if there's no features
}
// count features
std::size_t feature_count_indexed = count_shapefile_features(path);
// ensure number of features are the same
REQUIRE(feature_count == feature_count_indexed);
// remove *.index if present
if (mapnik::util::exists(index_path))
{
mapnik::util::remove(index_path);
}
}
}
}

64
utils/shapeindex/shapeindex.cpp
View file

@ -26,9 +26,10 @@
#include <string>
#include <mapnik/util/fs.hpp>
#include <mapnik/quad_tree.hpp>
#include <mapnik/geometry_envelope.hpp>
#include "shapefile.hpp"
#include "shape_io.hpp"
#include "shape_index_featureset.hpp"
#pragma GCC diagnostic push
#include <mapnik/warning_ignore.hpp>
#include <boost/algorithm/string.hpp>
@ -44,8 +45,9 @@ int main (int argc,char** argv)
namespace po = boost::program_options;
bool verbose=false;
unsigned int depth=DEFAULT_DEPTH;
double ratio=DEFAULT_RATIO;
bool index_parts = false;
unsigned int depth = DEFAULT_DEPTH;
double ratio = DEFAULT_RATIO;
std::vector<std::string> shape_files;
try
@ -54,6 +56,7 @@ int main (int argc,char** argv)
desc.add_options()
("help,h", "produce usage message")
("version,V","print version string")
("index-parts","index individual shape parts (default: no)")
("verbose,v","verbose output")
("depth,d", po::value<unsigned int>(), "max tree depth\n(default 8)")
("ratio,r",po::value<double>(),"split ratio (default 0.55)")
@ -81,6 +84,10 @@ int main (int argc,char** argv)
{
verbose = true;
}
if (vm.count("index-parts"))
{
index_parts = true;
}
if (vm.count("depth"))
{
depth = vm["depth"].as<unsigned int>();
@ -159,7 +166,7 @@ int main (int argc,char** argv)
int pos = 50;
shx.seek(pos * 2);
mapnik::quad_tree<int> tree(extent, depth, ratio);
mapnik::quad_tree<mapnik::detail::node> tree(extent, depth, ratio);
int count = 0;
if (shape_type != shape_io::shape_null)
@ -188,17 +195,56 @@ int main (int argc,char** argv)
double y=shp.read_double();
item_ext=box2d<double>(x,y,x,y);
}
else if (index_parts &&
(shape_type == shape_io::shape_polygon || shape_type == shape_io::shape_polygonm || shape_type == shape_io::shape_polygonz
|| shape_type == shape_io::shape_polyline || shape_type == shape_io::shape_polylinem || shape_type == shape_io::shape_polylinez))
{
shp.read_envelope(item_ext);
int num_parts = shp.read_ndr_integer();
int num_points = shp.read_ndr_integer();
std::vector<int> parts;
parts.resize(num_parts);
std::for_each(parts.begin(), parts.end(), [&](int & part) { part = shp.read_ndr_integer();});
for (int k = 0; k < num_parts; ++k)
{
int start = parts[k];
int end;
if (k == num_parts - 1) end = num_points;
else end = parts[k + 1];
mapnik::geometry::linear_ring<double> ring;
ring.reserve(end - start);
for (int j = start; j < end; ++j)
{
double x = shp.read_double();
double y = shp.read_double();
ring.emplace_back(x, y);
}
item_ext = mapnik::geometry::envelope(ring);
if (item_ext.valid())
{
if (verbose)
{
std::clog << "record number " << record_number << " box=" << item_ext << std::endl;
}
tree.insert(mapnik::detail::node(offset * 2, start, end),item_ext);
++count;
}
}
item_ext = mapnik::box2d<double>(); //invalid
}
else
{
shp.read_envelope(item_ext);
}
if (verbose)
{
std::clog << "record number " << record_number << " box=" << item_ext << std::endl;
}
if (item_ext.valid())
{
tree.insert(offset * 2,item_ext);
if (verbose)
{
std::clog << "record number " << record_number << " box=" << item_ext << std::endl;
}
tree.insert(mapnik::detail::node(offset * 2,-1,0),item_ext);
++count;
}
}