From d29f60905256e50c835ba4ce2ce6dd857a4681d0 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Wed, 12 Oct 2011 01:08:03 +0100 Subject: [PATCH 1/2] Added a 'multi' tiled raster plugin reader for virtual images already present as tiles on disk. --- plugins/input/raster/raster_datasource.cpp | 76 ++++++++---- plugins/input/raster/raster_datasource.hpp | 3 + plugins/input/raster/raster_featureset.cpp | 25 +++- plugins/input/raster/raster_featureset.hpp | 131 ++++++++++++++++++++- 4 files changed, 208 insertions(+), 27 deletions(-) diff --git a/plugins/input/raster/raster_datasource.cpp b/plugins/input/raster/raster_datasource.cpp index e7bf0b445..589f269c8 100644 --- a/plugins/input/raster/raster_datasource.cpp +++ b/plugins/input/raster/raster_datasource.cpp @@ -65,6 +65,10 @@ raster_datasource::raster_datasource(const parameters& params, bool bind) else filename_ = *file; + multi_tiles_ = *params_.get("multi", false); + tile_size_ = *params_.get("tile-size", 256); + tile_stride_ = *params_.get("tile-stride", 1); + format_=*params_.get("format","tiff"); boost::optional lox = params_.get("lox"); @@ -96,33 +100,50 @@ void raster_datasource::bind() const { if (is_bound_) return; - if (! boost::filesystem::exists(filename_)) - throw datasource_exception("Raster Plugin: " + filename_ + " does not exist"); - - try - { - std::auto_ptr reader(mapnik::get_image_reader(filename_, format_)); - if (reader.get()) - { - width_ = reader->width(); - height_ = reader->height(); + if (multi_tiles_) + { + boost::optional x_width = params_.get("x-width"); + boost::optional y_width = params_.get("y-width"); -#ifdef MAPNIK_DEBUG - std::clog << "Raster Plugin: RASTER SIZE(" << width_ << "," << height_ << ")" << std::endl; -#endif - } + if (!x_width) + throw datasource_exception("Raster Plugin: x-width parameter not supplied for multi-tiled data source."); + + if (!y_width) + throw datasource_exception("Raster Plugin: y-width parameter not supplied for multi-tiled data source."); + + width_ = x_width.get() * tile_size_; + height_ = y_width.get() * tile_size_; } - catch (mapnik::image_reader_exception const& ex) + else { - std::cerr << "Raster Plugin: image reader exception caught: " << ex.what() << std::endl; - throw; - } - catch (...) - { - std::cerr << "Raster Plugin: exception caught" << std::endl; - throw; + if (! boost::filesystem::exists(filename_)) + throw datasource_exception("Raster Plugin: " + filename_ + " does not exist"); + + try + { + std::auto_ptr reader(mapnik::get_image_reader(filename_, format_)); + if (reader.get()) + { + width_ = reader->width(); + height_ = reader->height(); + } + } + catch (mapnik::image_reader_exception const& ex) + { + std::cerr << "Raster Plugin: image reader exception caught: " << ex.what() << std::endl; + throw; + } + catch (...) + { + std::cerr << "Raster Plugin: exception caught" << std::endl; + throw; + } } +#ifdef MAPNIK_DEBUG + std::clog << "Raster Plugin: RASTER SIZE(" << width_ << "," << height_ << ")" << std::endl; +#endif + is_bound_ = true; } @@ -165,7 +186,16 @@ featureset_ptr raster_datasource::features(query const& q) const std::clog << "Raster Plugin: BOX SIZE(" << width << " " << height << ")" << std::endl; #endif - if (width * height > 512*512) + if (multi_tiles_) + { +#ifdef MAPNIK_DEBUG + std::clog << "Raster Plugin: MULTI-TILED policy" << std::endl; +#endif + + tiled_multi_file_policy policy(filename_, format_, tile_size_, extent_, q.get_bbox(), width_, height_, tile_stride_); + return boost::make_shared >(policy, extent_, q); + } + else if (width * height > 512*512) { #ifdef MAPNIK_DEBUG std::clog << "Raster Plugin: TILED policy" << std::endl; diff --git a/plugins/input/raster/raster_datasource.hpp b/plugins/input/raster/raster_datasource.hpp index c8f5f4745..d88c45a96 100644 --- a/plugins/input/raster/raster_datasource.hpp +++ b/plugins/input/raster/raster_datasource.hpp @@ -37,6 +37,9 @@ class raster_datasource : public mapnik::datasource std::string format_; mapnik::box2d extent_; bool extent_initialized_; + bool multi_tiles_; + unsigned tile_size_; + unsigned tile_stride_; mutable unsigned width_; mutable unsigned height_; public: diff --git a/plugins/input/raster/raster_featureset.cpp b/plugins/input/raster/raster_featureset.cpp index 57a797f50..1c463a92b 100644 --- a/plugins/input/raster/raster_featureset.cpp +++ b/plugins/input/raster/raster_featureset.cpp @@ -27,6 +27,7 @@ #include "raster_featureset.hpp" +#include using mapnik::query; using mapnik::CoordTransform; @@ -67,16 +68,18 @@ feature_ptr raster_featureset::next() std::clog << "Raster Plugin: READER = " << curIter_->format() << " " << curIter_->file() << " size(" << curIter_->width() << "," << curIter_->height() << ")" << std::endl; #endif + if (reader.get()) { - int image_width=reader->width(); - int image_height=reader->height(); + int image_width = policy_.img_width(reader->width()); + int image_height = policy_.img_height(reader->height()); if (image_width>0 && image_height>0) { CoordTransform t(image_width,image_height,extent_,0,0); box2d intersect=bbox_.intersect(curIter_->envelope()); box2d ext=t.forward(intersect); + box2d rem=policy_.transform(ext); if ( ext.width()>0.5 && ext.height()>0.5 ) { //select minimum raster containing whole ext @@ -96,7 +99,8 @@ feature_ptr raster_featureset::next() int width = end_x - x_off; int height = end_y - y_off; //calculate actual box2d of returned raster - box2d feature_raster_extent(x_off, y_off, x_off+width, y_off+height); + box2d feature_raster_extent(rem.minx() + x_off, rem.miny() + y_off, + rem.maxx() + x_off + width, rem.maxy() + y_off + height); intersect = t.backward(feature_raster_extent); image_data_32 image(width,height); @@ -121,5 +125,20 @@ feature_ptr raster_featureset::next() return feature_ptr(); } +std::string +tiled_multi_file_policy::interpolate(std::string const &pattern, int x, int y) const +{ + // TODO: make from some sort of configurable interpolation + int tms_y = tile_stride_ * ((image_height_ / tile_size_) - y - 1); + int tms_x = tile_stride_ * x; + std::string xs = (boost::format("%03d/%03d/%03d") % (tms_x / 1000000) % ((tms_x / 1000) % 1000) % (tms_x % 1000)).str(); + std::string ys = (boost::format("%03d/%03d/%03d") % (tms_y / 1000000) % ((tms_y / 1000) % 1000) % (tms_y % 1000)).str(); + std::string rv(pattern); + boost::algorithm::replace_all(rv, "${x}", xs); + boost::algorithm::replace_all(rv, "${y}", ys); + return rv; +} + template class raster_featureset; template class raster_featureset; +template class raster_featureset; diff --git a/plugins/input/raster/raster_featureset.hpp b/plugins/input/raster/raster_featureset.hpp index 879b2e848..4c2bb9816 100644 --- a/plugins/input/raster/raster_featureset.hpp +++ b/plugins/input/raster/raster_featureset.hpp @@ -29,7 +29,7 @@ // boost #include - +#include class single_file_policy { @@ -95,6 +95,21 @@ public: { return const_iterator(); } + + inline int img_width(int reader_width) const + { + return reader_width; + } + + inline int img_height(int reader_height) const + { + return reader_height; + } + + inline box2d transform(box2d &) const + { + return box2d(0, 0, 0, 0); + } }; class tiled_file_policy @@ -154,11 +169,125 @@ public: return infos_.end(); } + inline int img_width(int reader_width) const + { + return reader_width; + } + + inline int img_height(int reader_height) const + { + return reader_height; + } + + inline box2d transform(box2d &) const + { + return box2d(0, 0, 0, 0); + } + private: std::vector infos_; }; +class tiled_multi_file_policy +{ +public: + + typedef std::vector::const_iterator const_iterator; + + tiled_multi_file_policy(std::string const& file_pattern, std::string const& format, unsigned tile_size, + box2d extent, box2d bbox,unsigned width, unsigned height, + unsigned tile_stride) + : image_width_(width), image_height_(height), tile_size_(tile_size), tile_stride_(tile_stride) + { + + double lox = extent.minx(); + double loy = extent.miny(); + + int max_x = int(std::ceil(double(width)/double(tile_size))); + int max_y = int(std::ceil(double(height)/double(tile_size))); + + double pixel_x = extent.width()/double(width); + double pixel_y = extent.height()/double(height); + +#ifdef MAPNIK_DEBUG + std::clog << "Raster Plugin: PIXEL SIZE("<< pixel_x << "," << pixel_y << ")" << std::endl; +#endif + + // intersection of query with extent => new query + box2d e = bbox.intersect(extent); + + const int x_min = int(std::floor((e.minx() - lox) / (tile_size * pixel_x))); + const int y_min = int(std::floor((e.miny() - loy) / (tile_size * pixel_y))); + const int x_max = int(std::ceil((e.maxx() - lox) / (tile_size * pixel_x))); + const int y_max = int(std::ceil((e.maxy() - loy) / (tile_size * pixel_y))); + + for (int x = x_min ; x < x_max ; ++x) + { + for (int y = y_min ; y < y_max ; ++y) + { + // x0, y0, x1, y1 => projection-space image coordinates. + double x0 = lox + x*tile_size*pixel_x; + double y0 = loy + y*tile_size*pixel_y; + double x1 = x0 + tile_size*pixel_x; + double y1 = y0 + tile_size*pixel_y; + + // check if it intersects the query + if (e.intersects(box2d(x0,y0,x1,y1))) + { + // tile_box => intersection of tile with query in projection-space. + box2d tile_box = e.intersect(box2d(x0,y0,x1,y1)); + std::string file = interpolate(file_pattern, x, y); + raster_info info(file,format,tile_box,tile_size,tile_size); + infos_.push_back(info); + } + } + } +#ifdef MAPNIK_DEBUG + std::clog << "Raster Plugin: INFO SIZE=" << infos_.size() << " " << file_pattern << std::endl; +#endif + } + + const_iterator begin() + { + return infos_.begin(); + } + + const_iterator end() + { + return infos_.end(); + } + + inline int img_width(int) const + { + return image_width_; + } + + inline int img_height(int) const + { + return image_height_; + } + + inline box2d transform(box2d &box) const + { + int x_offset = int(std::floor(box.minx() / tile_size_)); + int y_offset = int(std::floor(box.miny() / tile_size_)); + box2d rem(x_offset * tile_size_, y_offset * tile_size_, + x_offset * tile_size_, y_offset * tile_size_); + box.init(box.minx() - rem.minx(), + box.miny() - rem.miny(), + box.maxx() - rem.maxx(), + box.maxy() - rem.maxy()); + return rem; + } + +private: + + std::string interpolate(std::string const &pattern, int x, int y) const; + + unsigned int image_width_, image_height_, tile_size_, tile_stride_; + std::vector infos_; +}; template class raster_featureset : public mapnik::Featureset From 20ca69c3ea88cf54e7d277287f3b9947eed5a600 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 18 Oct 2011 14:34:58 +0100 Subject: [PATCH 2/2] Added Python test for multi-tile raster policy. --- bindings/python/mapnik/__init__.py | 7 ++ plugins/input/raster/raster_datasource.cpp | 8 +- .../raster_tiles/000/000/000/000/000/000.tif | Bin 0 -> 3427 bytes .../raster_tiles/000/000/000/000/000/001.tif | Bin 0 -> 3427 bytes .../raster_tiles/000/000/001/000/000/000.tif | Bin 0 -> 3427 bytes .../raster_tiles/000/000/001/000/000/001.tif | Bin 0 -> 3423 bytes tests/python_tests/multi_tile_raster_test.py | 70 ++++++++++++++++++ 7 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 tests/data/raster_tiles/000/000/000/000/000/000.tif create mode 100644 tests/data/raster_tiles/000/000/000/000/000/001.tif create mode 100644 tests/data/raster_tiles/000/000/001/000/000/000.tif create mode 100644 tests/data/raster_tiles/000/000/001/000/000/001.tif create mode 100644 tests/python_tests/multi_tile_raster_test.py diff --git a/bindings/python/mapnik/__init__.py b/bindings/python/mapnik/__init__.py index 6eac1d6d9..2ed760e73 100644 --- a/bindings/python/mapnik/__init__.py +++ b/bindings/python/mapnik/__init__.py @@ -410,6 +410,13 @@ def Raster(**keywords): Optional keyword arguments: base -- path prefix (default None) + multi -- whether the image is in tiles on disk (default False) + + Multi-tiled keyword arguments: + x_width -- virtual image number of tiles in X direction (required) + y_width -- virtual image number of tiles in Y direction (required) + tile_size -- if an image is in tiles, how large are the tiles (default 256) + tile_stride -- if an image is in tiles, what's the increment between rows/cols (default 1) >>> from mapnik import Raster, Layer >>> raster = Raster(base='/home/mapnik/data',file='elevation.tif',lox=-122.8,loy=48.5,hix=-122.7,hiy=48.6) diff --git a/plugins/input/raster/raster_datasource.cpp b/plugins/input/raster/raster_datasource.cpp index 589f269c8..20b363650 100644 --- a/plugins/input/raster/raster_datasource.cpp +++ b/plugins/input/raster/raster_datasource.cpp @@ -66,8 +66,8 @@ raster_datasource::raster_datasource(const parameters& params, bool bind) filename_ = *file; multi_tiles_ = *params_.get("multi", false); - tile_size_ = *params_.get("tile-size", 256); - tile_stride_ = *params_.get("tile-stride", 1); + tile_size_ = *params_.get("tile_size", 256); + tile_stride_ = *params_.get("tile_stride", 1); format_=*params_.get("format","tiff"); @@ -102,8 +102,8 @@ void raster_datasource::bind() const if (multi_tiles_) { - boost::optional x_width = params_.get("x-width"); - boost::optional y_width = params_.get("y-width"); + boost::optional x_width = params_.get("x_width"); + boost::optional y_width = params_.get("y_width"); if (!x_width) throw datasource_exception("Raster Plugin: x-width parameter not supplied for multi-tiled data source."); diff --git a/tests/data/raster_tiles/000/000/000/000/000/000.tif b/tests/data/raster_tiles/000/000/000/000/000/000.tif new file mode 100644 index 0000000000000000000000000000000000000000..d43cebc820fdc297a711e7a12f0085f948d94f8c GIT binary patch literal 3427 zcmebD)M7Zo#lZ0Y#{b(4&I}xEZ0u~T9PI4uoSYn7JR-b2+}u2pLc;tavQqMLvQjcK z3MzW)3Q9W4GBO(GnmPuCCMG8G>Xx<^MmBoJCPx1cFbHxmI509WGYB#;3NkPWGW@^A zz{AYIz{tSFz+lht^WWZo`|1~UX^f)L5Fj%I_W!>*np{V7&`=41(d0T*VrtZ#zz`Ts zuA|9d2!#MJxe7D>Vtc+j)G#m^V znV4Bv+1NQaxw!uyVc06b0E|UuCKhH^Ru*7ECr+Nabot8FYu9hwy!G(W<0ns_J%91?)yGetzkL1n{m0K=|8D{S!C&GN literal 0 HcmV?d00001 diff --git a/tests/data/raster_tiles/000/000/000/000/000/001.tif b/tests/data/raster_tiles/000/000/000/000/000/001.tif new file mode 100644 index 0000000000000000000000000000000000000000..67d3bb67aba24b66cf30cc7f7c7a5e7b4c6c4118 GIT binary patch literal 3427 zcmebD)M7Zo#lZ0Y#{b(4&I}xEZ0u~T9PI4uoSYn7JR-b2+}u2pLc;tavQqMLvQjcK z3MzW)3Q9W4GBO(GnmPuCCMG8G>Xx<^MmBoJCPx1cFbHxmI509WGYB#;3NkPWGW@^A zz{AYIz{tSFz+lht{nyWb`|1~UX^f)L5Fj%I_W!>*np{V7&`=41(d0T*VrtZ#zz`Ts zuA|9d2!#MJxe7DR;@QBE$ z5`HGdRHf`Rrb=&qG zJ9iyAeB|h{<0np@x^(%<)oa&p+`RSh(c>pipFMx^^3}&rpTB(l_Wj4tU;l3c07(7f A>;M1& literal 0 HcmV?d00001 diff --git a/tests/data/raster_tiles/000/000/001/000/000/000.tif b/tests/data/raster_tiles/000/000/001/000/000/000.tif new file mode 100644 index 0000000000000000000000000000000000000000..0868eeaf089e0a01fc469945ec40551861f20080 GIT binary patch literal 3427 zcmebD)M7Zo#lZ0Y#{b(4&I}xEZ0u~T9PI4uoSYn7JR-b2+}u2pLc;tavQqMLvQjcK z3MzW)3Q9W4GBO(GnmPuCCMG8G>Xx<^MmBoJCPx1cFbHxmI509WGYB#;3NkPWGW@^A zz{AYIz{tSFz+lht^WVPu_4SLoG)B>A2#^^9`~Tk@O|GLkXsCq1XmTAYF*WK=US{3Nli=7$jmA(DJ?6nsH|#kX>Duo= literal 0 HcmV?d00001 diff --git a/tests/data/raster_tiles/000/000/001/000/000/001.tif b/tests/data/raster_tiles/000/000/001/000/000/001.tif new file mode 100644 index 0000000000000000000000000000000000000000..95eee548c7acef62840e02321af776a67dd1f7b2 GIT binary patch literal 3423 zcmebD)M7Zm#lZ0Y#{b(4&I}xEZ0u~T9PI4uoSYn7JR-b2+}u2pLc;tavQqMLvQjcK z3MzW)3Q9W4GBO(GnmPuCCMG8G>Xx<^MmBoJCPx1cFbHxmI509WGYB#;3NkPWGW@^A zz{AYIz{tSFz+lht{nyWbd;fK5EE+|lAwXsb{J%MxT1PX`Pziz2)H+($51kMgO|3&G zu7*kpNIQ4_zbVY{kAVf)xP$^mW(XT7z=*_V0KO&NF@ znHao*^ah|BJ|J5N$o>Ol3o^0*Z2*e#Ffa%~)r0I4g|b2Bib2^9K(;tky&O=z6eBCx zJ$gX4G?Z-yWXm9#qXA`u+z`#k&Y%c1+zweFFmnC^kf9>y>1tF)%n6r6!i7 zrYMwWmSiZnd-?_dZD3&72DC1P6=*+@vku5c#z1KgAO?9KOjD~Jc7#D5Xeuis7yu0i z14brh7FITP4o)ua|3?_M3NQd;k(r5wnU$3V7$1zaKzRlhK~^C}Lq|5@z(jVXLJ_0J zi3>TDoi-j64Z8S2#W<;`iIYoATtZSxRZU$(Q_IBE%-q7#%Gt%$&E3P(D>x)HEIcAI zDmf)JEj=SMtGJ}Jth}PKs=1}Lt-YhOYtrN?Q>RUzF>}_U#Y>hhTfSoDs!f}>Y~8kf z$Ie}c4j(ys?D&b3r!HN-a`oEv8#iw~eDwIq(`V0LynOZX)8{W=zkUDl^Vk2I0HclK AS^xk5 literal 0 HcmV?d00001 diff --git a/tests/python_tests/multi_tile_raster_test.py b/tests/python_tests/multi_tile_raster_test.py new file mode 100644 index 000000000..65a29675f --- /dev/null +++ b/tests/python_tests/multi_tile_raster_test.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +from nose.tools import * +from utilities import execution_path, save_data, contains_word + +import os, mapnik2 + +def setup(): + # All of the paths used are relative, if we run the tests + # from another directory we need to chdir() + os.chdir(execution_path('.')) + +def test_multi_tile_policy(): + srs = '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs' + lyr = mapnik2.Layer('raster') + lyr.datasource = mapnik2.Raster( + file = '../data/raster_tiles/${x}/${y}.tif', + lox = -180, + loy = -90, + hix = 180, + hiy = 90, + multi = 1, + tile_size = 256, + x_width = 2, + y_width = 2 + ) + lyr.srs = srs + _map = mapnik2.Map(256, 256, srs) + style = mapnik2.Style() + rule = mapnik2.Rule() + sym = mapnik2.RasterSymbolizer() + rule.symbols.append(sym) + style.rules.append(rule) + _map.append_style('foo', style) + lyr.styles.append('foo') + _map.layers.append(lyr) + _map.zoom_to_box(lyr.envelope()) + + im = mapnik2.Image(_map.width, _map.height) + mapnik2.render(_map, im) + + save_data('test_multi_tile_policy.png', im.tostring('png')) + + # test green chunk + assert im.view(0,64,1,1).tostring() == '\x00\xff\x00\xff' + assert im.view(127,64,1,1).tostring() == '\x00\xff\x00\xff' + assert im.view(0,127,1,1).tostring() == '\x00\xff\x00\xff' + assert im.view(127,127,1,1).tostring() == '\x00\xff\x00\xff' + + # test blue chunk + assert im.view(128,64,1,1).tostring() == '\x00\x00\xff\xff' + assert im.view(255,64,1,1).tostring() == '\x00\x00\xff\xff' + assert im.view(128,127,1,1).tostring() == '\x00\x00\xff\xff' + assert im.view(255,127,1,1).tostring() == '\x00\x00\xff\xff' + + # test red chunk + assert im.view(0,128,1,1).tostring() == '\xff\x00\x00\xff' + assert im.view(127,128,1,1).tostring() == '\xff\x00\x00\xff' + assert im.view(0,191,1,1).tostring() == '\xff\x00\x00\xff' + assert im.view(127,191,1,1).tostring() == '\xff\x00\x00\xff' + + # test magenta chunk + assert im.view(128,128,1,1).tostring() == '\xff\x00\xff\xff' + assert im.view(255,128,1,1).tostring() == '\xff\x00\xff\xff' + assert im.view(128,191,1,1).tostring() == '\xff\x00\xff\xff' + assert im.view(255,191,1,1).tostring() == '\xff\x00\xff\xff' + +if __name__ == "__main__": + setup() + [eval(run)() for run in dir() if 'test_' in run]