From c604e925f5c2fb3e519e54dd4a94424ad353ba3d Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 12 Aug 2014 11:44:34 -0700 Subject: [PATCH 01/10] apply #2350 to master/3.x too --- plugins/input/ogr/ogr_datasource.cpp | 14 +++++++++++--- tests/python_tests/ogr_test.py | 9 +++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/plugins/input/ogr/ogr_datasource.cpp b/plugins/input/ogr/ogr_datasource.cpp index f538984ad..8e02ece24 100644 --- a/plugins/input/ogr/ogr_datasource.cpp +++ b/plugins/input/ogr/ogr_datasource.cpp @@ -253,9 +253,17 @@ void ogr_datasource::init(mapnik::parameters const& params) OGRLayer* layer = layer_.layer(); // initialize envelope - OGREnvelope envelope; - layer->GetExtent(&envelope); - extent_.init(envelope.MinX, envelope.MinY, envelope.MaxX, envelope.MaxY); + boost::optional ext = params.get("extent"); + if (ext && !ext->empty()) + { + extent_.from_string(*ext); + } + else + { + OGREnvelope envelope; + layer->GetExtent(&envelope); + extent_.init(envelope.MinX, envelope.MinY, envelope.MaxX, envelope.MaxY); + } // scan for index file // TODO - layer names don't match dataset name, so this will break for diff --git a/tests/python_tests/ogr_test.py b/tests/python_tests/ogr_test.py index cf6a9f0c3..f87d63d4a 100644 --- a/tests/python_tests/ogr_test.py +++ b/tests/python_tests/ogr_test.py @@ -60,6 +60,15 @@ if 'ogr' in mapnik.DatasourceCache.plugin_names(): # fs = ds.all_features() # eq_(len(fs),1) + # OGR plugin extent parameter + def test_ogr_extent_parameter(): + ds = mapnik.Ogr(file='../data/shp/world_merc.shp',layer_by_index=0,extent='-1,-1,1,1') + e = ds.envelope() + eq_(e.minx,-1) + eq_(e.miny,-1) + eq_(e.maxx,1) + eq_(e.maxy,1) + if __name__ == "__main__": setup() exit(run_all(eval(x) for x in dir() if x.startswith("test_"))) From 9b789a398bb7e730dbdd8a4c8f3444b46b8d82aa Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 12 Aug 2014 13:43:37 -0700 Subject: [PATCH 02/10] remove bilinear8 from master/3.x - closes #2076 --- bindings/python/mapnik_scaling_method.cpp | 1 - include/mapnik/image_scaling.hpp | 15 +- .../process_raster_symbolizer.hpp | 30 +-- src/image_scaling.cpp | 171 ------------------ src/warp.cpp | 1 - 5 files changed, 11 insertions(+), 207 deletions(-) diff --git a/bindings/python/mapnik_scaling_method.cpp b/bindings/python/mapnik_scaling_method.cpp index 7d7fdb064..372b20d90 100644 --- a/bindings/python/mapnik_scaling_method.cpp +++ b/bindings/python/mapnik_scaling_method.cpp @@ -46,6 +46,5 @@ void export_scaling_method() .value("SINC", mapnik::SCALING_SINC) .value("LANCZOS", mapnik::SCALING_LANCZOS) .value("BLACKMAN", mapnik::SCALING_BLACKMAN) - .value("BILINEAR8", mapnik::SCALING_BILINEAR8) ; } diff --git a/include/mapnik/image_scaling.hpp b/include/mapnik/image_scaling.hpp index 0b85ce37a..97d7c2d30 100644 --- a/include/mapnik/image_scaling.hpp +++ b/include/mapnik/image_scaling.hpp @@ -54,8 +54,7 @@ enum scaling_method_e SCALING_MITCHELL, SCALING_SINC, SCALING_LANCZOS, - SCALING_BLACKMAN, - SCALING_BILINEAR8 + SCALING_BLACKMAN }; MAPNIK_DECL boost::optional scaling_method_from_string(std::string const& name); @@ -83,17 +82,5 @@ template MAPNIK_DECL void scale_image_agg( double filter_radius); #endif -template -void scale_image_bilinear_old(Image & target, - Image const& source, - double x_off_f=0, - double y_off_f=0); - -template -void scale_image_bilinear8(Image & target, - Image const& source, - double x_off_f=0, - double y_off_f=0); - } #endif // MAPNIK_IMAGE_SCALING_HPP diff --git a/include/mapnik/renderer_common/process_raster_symbolizer.hpp b/include/mapnik/renderer_common/process_raster_symbolizer.hpp index deddb74bc..2149d096a 100644 --- a/include/mapnik/renderer_common/process_raster_symbolizer.hpp +++ b/include/mapnik/renderer_common/process_raster_symbolizer.hpp @@ -111,26 +111,16 @@ void render_raster_symbolizer(raster_symbolizer const &sym, else { raster target(target_ext, raster_width, raster_height, source->get_filter_factor()); - if (scaling_method == SCALING_BILINEAR8) - { - scale_image_bilinear8(target.data_, - source->data_, - 0.0, - 0.0); - } - else - { - double image_ratio_x = ext.width() / source->data_.width(); - double image_ratio_y = ext.height() / source->data_.height(); - scale_image_agg(target.data_, - source->data_, - scaling_method, - image_ratio_x, - image_ratio_y, - 0.0, - 0.0, - source->get_filter_factor()); - } + double image_ratio_x = ext.width() / source->data_.width(); + double image_ratio_y = ext.height() / source->data_.height(); + scale_image_agg(target.data_, + source->data_, + scaling_method, + image_ratio_x, + image_ratio_y, + 0.0, + 0.0, + source->get_filter_factor()); composite(target.data_, comp_op, opacity, start_x, start_y); } } diff --git a/src/image_scaling.cpp b/src/image_scaling.cpp index 99ad9f088..0bb7adfed 100644 --- a/src/image_scaling.cpp +++ b/src/image_scaling.cpp @@ -66,7 +66,6 @@ static const scaling_method_lookup_type scaling_lookup = boost::assign::list_of< (SCALING_SINC,"sinc") (SCALING_LANCZOS,"lanczos") (SCALING_BLACKMAN,"blackman") - (SCALING_BILINEAR8,"bilinear8") ; boost::optional scaling_method_from_string(std::string const& name) @@ -91,170 +90,6 @@ boost::optional scaling_method_to_string(scaling_method_e scaling_m return mode; } -// this has been replaced by agg impl - see https://github.com/mapnik/mapnik/issues/656 -template -void scale_image_bilinear_old (Image & target,Image const& source, double x_off_f, double y_off_f) -{ - - int source_width=source.width(); - int source_height=source.height(); - - int target_width=target.width(); - int target_height=target.height(); - - if (source_width<1 || source_height<1 || - target_width<1 || target_height<1) return; - int x=0,y=0,xs=0,ys=0; - int tw2 = target_width/2; - int th2 = target_height/2; - int offs_x = rint((source_width-target_width-x_off_f*2*source_width)/2); - int offs_y = rint((source_height-target_height-y_off_f*2*source_height)/2); - unsigned yprt, yprt1, xprt, xprt1; - - //no scaling or subpixel offset - if (target_height == source_height && target_width == source_width && offs_x == 0 && offs_y == 0){ - for (y=0;y=source_height) - ys1--; - if (ys<0) - ys=ys1=0; - if (source_height/2=source_width) - xs1--; - if (xs<0) - xs=xs1=0; - - unsigned a = source(xs,ys); - unsigned b = source(xs1,ys); - unsigned c = source(xs,ys1); - unsigned d = source(xs1,ys1); - unsigned out=0; - unsigned t = 0; - - for(int i=0; i<4; i++){ - unsigned p,r,s; - // X axis - p = a&0xff; - r = b&0xff; - if (p!=r) - r = (r*xprt+p*xprt1+tw2)/target_width; - p = c&0xff; - s = d&0xff; - if (p!=s) - s = (s*xprt+p*xprt1+tw2)/target_width; - // Y axis - if (r!=s) - r = (s*yprt+r*yprt1+th2)/target_height; - // channel up - out |= r << t; - t += 8; - a >>= 8; - b >>= 8; - c >>= 8; - d >>= 8; - } - target(x,y)=out; - } - } -} - - -template -void scale_image_bilinear8 (Image & target,Image const& source, double x_off_f, double y_off_f) -{ - - int source_width=source.width(); - int source_height=source.height(); - - int target_width=target.width(); - int target_height=target.height(); - - if (source_width<1 || source_height<1 || - target_width<1 || target_height<1) return; - int x=0,y=0,xs=0,ys=0; - int tw2 = target_width/2; - int th2 = target_height/2; - int offs_x = rint((source_width-target_width-x_off_f*2*source_width)/2); - int offs_y = rint((source_height-target_height-y_off_f*2*source_height)/2); - unsigned yprt, yprt1, xprt, xprt1; - - //no scaling or subpixel offset - if (target_height == source_height && target_width == source_width && offs_x == 0 && offs_y == 0){ - for (y=0;y=source_height) - ys1--; - if (ys<0) - ys=ys1=0; - if (source_height/2=source_width) - xs1--; - if (xs<0) - xs=xs1=0; - - unsigned a = source(xs,ys); - unsigned b = source(xs1,ys); - unsigned c = source(xs,ys1); - unsigned d = source(xs1,ys1); - unsigned p,r,s; - // X axis - p = a&0xff; - r = b&0xff; - if (p!=r) - r = (r*xprt+p*xprt1+tw2)/target_width; - p = c&0xff; - s = d&0xff; - if (p!=s) - s = (s*xprt+p*xprt1+tw2)/target_width; - // Y axis - if (r!=s) - r = (s*yprt+r*yprt1+th2)/target_height; - target(x,y)=(0xff<<24) | (r<<16) | (r<<8) | r; - } - } -} - template void scale_image_agg(Image & target, Image const& source, @@ -316,7 +151,6 @@ void scale_image_agg(Image & target, return; } case SCALING_BILINEAR: - case SCALING_BILINEAR8: filter.calculate(agg::image_filter_bilinear(), true); break; case SCALING_BICUBIC: filter.calculate(agg::image_filter_bicubic(), true); break; @@ -376,9 +210,4 @@ template void scale_image_agg(image_data_32& target, double y_off_f, double filter_factor); -template void scale_image_bilinear_old (image_data_32& target,const image_data_32& source, double x_off_f, double y_off_f); - -template void scale_image_bilinear8 (image_data_32& target,const image_data_32& source, double x_off_f, double y_off_f); - - } diff --git a/src/warp.cpp b/src/warp.cpp index 9bb457872..9983591c3 100644 --- a/src/warp.cpp +++ b/src/warp.cpp @@ -108,7 +108,6 @@ void reproject_and_scale_raster(raster & target, raster const& source, switch(scaling_method) { case SCALING_NEAR: break; - case SCALING_BILINEAR8: // TODO - impl this or remove? case SCALING_BILINEAR: filter.calculate(agg::image_filter_bilinear(), true); break; case SCALING_BICUBIC: From 04a9a050a7c8354a430febe4f601ebd39946376d Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 12 Aug 2014 13:50:56 -0700 Subject: [PATCH 03/10] note png8 default change in master changelog still (no longer 2.3.x) - refs #2110 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f75d920b..83823ddc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Developers: Please commit along with changes. For a complete change history, see the git log. +## 3.x / Future + +- Default PNG encoding method when `png` is supplied is now `png8:m=h`, so paletted png using hextree color quantization (#2028) + Use `png32` now for full color png. More details at https://github.com/mapnik/mapnik/wiki/Image-IO. + ## 2.3.0 Released ... From a000df5e46a6865eb602daf04b1156216c77d0b9 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 12 Aug 2014 14:33:59 -0700 Subject: [PATCH 04/10] add testcase from #2137 --- ...r-position-500-100-1.0-grid-reference.json | 34 ++++++++++++++++++ ...ior-position-500-100-1.0-agg-reference.png | Bin 0 -> 455 bytes ...r-position-500-100-1.0-cairo-reference.png | Bin 0 -> 447 bytes ...ior-position-500-100-2.0-agg-reference.png | Bin 0 -> 711 bytes ...r-position-500-100-2.0-cairo-reference.png | Bin 0 -> 727 bytes .../styles/marker-interior-position.xml | 24 +++++++++++++ tests/visual_tests/test.py | 1 + 7 files changed, 59 insertions(+) create mode 100644 tests/visual_tests/grids/marker-interior-position-500-100-1.0-grid-reference.json create mode 100644 tests/visual_tests/images/marker-interior-position-500-100-1.0-agg-reference.png create mode 100644 tests/visual_tests/images/marker-interior-position-500-100-1.0-cairo-reference.png create mode 100644 tests/visual_tests/images/marker-interior-position-500-100-2.0-agg-reference.png create mode 100644 tests/visual_tests/images/marker-interior-position-500-100-2.0-cairo-reference.png create mode 100644 tests/visual_tests/styles/marker-interior-position.xml diff --git a/tests/visual_tests/grids/marker-interior-position-500-100-1.0-grid-reference.json b/tests/visual_tests/grids/marker-interior-position-500-100-1.0-grid-reference.json new file mode 100644 index 000000000..4449b111c --- /dev/null +++ b/tests/visual_tests/grids/marker-interior-position-500-100-1.0-grid-reference.json @@ -0,0 +1,34 @@ +{ + "keys": [ + "", + "1" + ], + "data": {}, + "grid": [ + " ", + " ", + " !!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!! ", + " !!! !!!!!!!! ", + " !!! !!!!!!!!!! ", + " !!! !!!!!!!!!! ", + " !!! !!!!!!!!! ", + " !!! !!!!!!!! ", + " !!! !!!!!!!! ", + " !!! !!!!!!!! ", + " !!! !!!!!!!! ", + " !!! !!!!!!!! ", + " !!! !!!!!!!! ", + " !!! !!!!!!!! ", + " !!! !!!!!!!! ", + " !!! !!!!!!!! ", + " !!! !!!!!!!! ", + " ", + " " + ] +} \ No newline at end of file diff --git a/tests/visual_tests/images/marker-interior-position-500-100-1.0-agg-reference.png b/tests/visual_tests/images/marker-interior-position-500-100-1.0-agg-reference.png new file mode 100644 index 0000000000000000000000000000000000000000..fa3ca3049917895f95d1a41acc0ade5c31f48b3a GIT binary patch literal 455 zcmV;&0XY7NP)003kJ0{{R3g6ZE*0000XP)t-s0Du7h z0093625Nw6fPjGbU0tQ6rM`+1c*y?upgr=>Px$VM#<jxb=bm-8b zTU`WQKj_e*Lx=8}{<0C8{w6-@-L?gE=te=;uU*V8x_bP1zjjZ5NGWyI^t;J(_w@5Z zzpSROIp;Iy^uEZ`Q8jJ~{lq!_XQUrYr+ty<)424xbNcT{Kdi=0fTuo;^y6xJ8{ltl zSQh$CHT@{`wRiesO0%w-_IJ~V?vUPIEuW=huaSW-L^Y)IR-{AnBhQ!SZ30X&Z zl`I&Xbb3VZP7IK|A;7huM^?guk^RM?Qvnm_OqyxP`99w3a=OsudvBv0+TV$k{JypNr*C+_dHLxNU(zQoUN zr504hKPigp?cTGK!|#6Im&Y%+w9C!Nq0#je;{Qa;1lA?(7?df%=Vvw!EC0P zbA$7LPR_?NGNH>u`-+Nc8)_Tt>KYmvo@8b&UAlDd-n~1o?c8@`->Fllu0Fc@{^reh z@7_Hihpu(bvjCZk$&(_B*Ti{~wzL-+n>zwn>`3lsC+0}lEI7r1%A%&_)*%co!8|J#1=uB!`|A3p8<{-6JP zzLE=nnPb_lUi@oZ$1Gd&-{E>A->&)vuN~O$+FvML(ELvRh1CVe@BAgQFBbe}--TUy z=KtiJ`F8VGygydg^zLqEaO|x)Pi`#Cp1pos|4qm9yJJgp)sOvCp8jS*`rWlVgWk`& zy=}k4cb(AhwwI)CJ&ThqIqmnAWpCg0uZg#PUOANKg!^{uf4F;PPo`sej@8BgIS&tS zEdHW)=XB$|{O2<(L+36&xOuhpi)p^w3zLs0yfHBE-(Ixf`R(!-6UAR>zub|qtbSYZ zmxAAXRmaYn7nNE}Gp$w4O?%N@_Wq%<cavG=9OcnZa$Gdx}!*={ZMq&u#RsQ<&(z=i8hvxm) lnjF^dX9bIIq^PfBc5Nq0#je<0-%;1lA?(7?dn%+A@uY1?4? zpMl{&JG<3vt7npu+q}Gjmj$m13!9sq+*?%CR9oB7(C{oSZpo4*d-v|$bA8XLQ>U&! zx&GnGmG^hTnehFutmMper%NVhtwY0$U)ML(*A$M5_lvM(0=X5YnEa^Wv?EW6cW?y6l!^``_N`4?Dj=dV8MV+M_r8Kk!i+V@2zX}glA=RDaLdHJw%#RcwnhR*do$Di%~u4gludG7mdKI;Vst E0J)cUr~m)} literal 0 HcmV?d00001 diff --git a/tests/visual_tests/styles/marker-interior-position.xml b/tests/visual_tests/styles/marker-interior-position.xml new file mode 100644 index 000000000..53332127c --- /dev/null +++ b/tests/visual_tests/styles/marker-interior-position.xml @@ -0,0 +1,24 @@ + + + + + + poly + + csv + -1 -1 11 11 + + id|name|wkt + 1|Example|Polygon((0 10, 0 0, 1 0, 1 7, 6 7, 6 0, 10 0, 10 10, 0 10)) + + + + \ No newline at end of file diff --git a/tests/visual_tests/test.py b/tests/visual_tests/test.py index e9b70e9fb..afbf87161 100755 --- a/tests/visual_tests/test.py +++ b/tests/visual_tests/test.py @@ -109,6 +109,7 @@ files = { 'geometry-transform-translate': {'sizes':[(200,200)]}, 'geometry-transform-translate-patterns': {'sizes':[(200,200)]}, 'geometry-transform-translate-patterns-svg': {'sizes':[(200,200)]}, + 'marker-interior-position':{}, 'marker-svg-opacity':{}, 'marker-svg-opacity2':{}, 'marker-svg-empty-g-element':{}, From 8b6a0a32f6947d54c46e6f6b3b9d13e8764b5031 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 12 Aug 2014 14:42:51 -0700 Subject: [PATCH 05/10] improve interior position algorithm - patch from @mrwojo - closes #2137 --- include/mapnik/geom_util.hpp | 15 ++- ...r-position-500-100-1.0-grid-reference.json | 34 ------ ...r-position-600-400-1.0-grid-reference.json | 109 ++++++++++++++++++ ...ior-position-500-100-1.0-agg-reference.png | Bin 455 -> 0 bytes ...r-position-500-100-1.0-cairo-reference.png | Bin 447 -> 0 bytes ...ior-position-500-100-2.0-agg-reference.png | Bin 711 -> 0 bytes ...r-position-500-100-2.0-cairo-reference.png | Bin 727 -> 0 bytes ...ior-position-600-400-1.0-agg-reference.png | Bin 0 -> 1211 bytes ...r-position-600-400-1.0-cairo-reference.png | Bin 0 -> 1196 bytes ...ior-position-600-400-2.0-agg-reference.png | Bin 0 -> 1330 bytes ...r-position-600-400-2.0-cairo-reference.png | Bin 0 -> 1978 bytes .../styles/marker-interior-position.xml | 5 +- tests/visual_tests/test.py | 2 +- 13 files changed, 120 insertions(+), 45 deletions(-) delete mode 100644 tests/visual_tests/grids/marker-interior-position-500-100-1.0-grid-reference.json create mode 100644 tests/visual_tests/grids/marker-interior-position-600-400-1.0-grid-reference.json delete mode 100644 tests/visual_tests/images/marker-interior-position-500-100-1.0-agg-reference.png delete mode 100644 tests/visual_tests/images/marker-interior-position-500-100-1.0-cairo-reference.png delete mode 100644 tests/visual_tests/images/marker-interior-position-500-100-2.0-agg-reference.png delete mode 100644 tests/visual_tests/images/marker-interior-position-500-100-2.0-cairo-reference.png create mode 100644 tests/visual_tests/images/marker-interior-position-600-400-1.0-agg-reference.png create mode 100644 tests/visual_tests/images/marker-interior-position-600-400-1.0-cairo-reference.png create mode 100644 tests/visual_tests/images/marker-interior-position-600-400-2.0-agg-reference.png create mode 100644 tests/visual_tests/images/marker-interior-position-600-400-2.0-cairo-reference.png diff --git a/include/mapnik/geom_util.hpp b/include/mapnik/geom_util.hpp index 9bc663a30..3114e59cb 100644 --- a/include/mapnik/geom_util.hpp +++ b/include/mapnik/geom_util.hpp @@ -563,18 +563,17 @@ bool interior_position(PathType & path, double & x, double & y) // no intersections we just return the default if (intersections.empty()) return true; - x0=intersections[0]; + std::sort(intersections.begin(), intersections.end()); double max_width = 0; - for (unsigned ii = 1; ii < intersections.size(); ++ii) + for (unsigned ii = 1; ii < intersections.size(); ii += 2) { - double xi=intersections[ii]; - double xc=(x0+xi)/2.0; - double width = std::fabs(xi-x0); - if (width > max_width && hit_test(path,xc,y,0)) + double xlow = intersections[ii-1]; + double xhigh = intersections[ii]; + double width = xhigh - xlow; + if (width > max_width) { - x=xc; + x = (xlow + xhigh) / 2.0; max_width = width; - break; } } return true; diff --git a/tests/visual_tests/grids/marker-interior-position-500-100-1.0-grid-reference.json b/tests/visual_tests/grids/marker-interior-position-500-100-1.0-grid-reference.json deleted file mode 100644 index 4449b111c..000000000 --- a/tests/visual_tests/grids/marker-interior-position-500-100-1.0-grid-reference.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "keys": [ - "", - "1" - ], - "data": {}, - "grid": [ - " ", - " ", - " !!!!!!!!!!!!!!!!!!!!! ", - " !!!!!!!!!!!!!!!!!!!!! ", - " !!!!!!!!!!!!!!!!!!!!! ", - " !!!!!!!!!!!!!!!!!!!!! ", - " !!!!!!!!!!!!!!!!!!!!! ", - " !!!!!!!!!!!!!!!!!!!!! ", - " !!!!!!!!!!!!!!!!!!!!! ", - " !!! !!!!!!!! ", - " !!! !!!!!!!!!! ", - " !!! !!!!!!!!!! ", - " !!! !!!!!!!!! ", - " !!! !!!!!!!! ", - " !!! !!!!!!!! ", - " !!! !!!!!!!! ", - " !!! !!!!!!!! ", - " !!! !!!!!!!! ", - " !!! !!!!!!!! ", - " !!! !!!!!!!! ", - " !!! !!!!!!!! ", - " !!! !!!!!!!! ", - " !!! !!!!!!!! ", - " ", - " " - ] -} \ No newline at end of file diff --git a/tests/visual_tests/grids/marker-interior-position-600-400-1.0-grid-reference.json b/tests/visual_tests/grids/marker-interior-position-600-400-1.0-grid-reference.json new file mode 100644 index 000000000..1c614a36b --- /dev/null +++ b/tests/visual_tests/grids/marker-interior-position-600-400-1.0-grid-reference.json @@ -0,0 +1,109 @@ +{ + "keys": [ + "", + "1" + ], + "data": {}, + "grid": [ + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " !!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " " + ] +} \ No newline at end of file diff --git a/tests/visual_tests/images/marker-interior-position-500-100-1.0-agg-reference.png b/tests/visual_tests/images/marker-interior-position-500-100-1.0-agg-reference.png deleted file mode 100644 index fa3ca3049917895f95d1a41acc0ade5c31f48b3a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 455 zcmV;&0XY7NP)003kJ0{{R3g6ZE*0000XP)t-s0Du7h z0093625Nw6fPjGbU0tQ6rM`+1c*y?upgr=>Px$VM#<jxb=bm-8b zTU`WQKj_e*Lx=8}{<0C8{w6-@-L?gE=te=;uU*V8x_bP1zjjZ5NGWyI^t;J(_w@5Z zzpSROIp;Iy^uEZ`Q8jJ~{lq!_XQUrYr+ty<)424xbNcT{Kdi=0fTuo;^y6xJ8{ltl zSQh$CHT@{`wRiesO0%w-_IJ~V?vUPIEuW=huaSW-L^Y)IR-{AnBhQ!SZ30X&Z zl`I&Xbb3VZP7IK|A;7huM^?guk^RM?Qvnm_OqyxP`99w3a=OsudvBv0+TV$k{JypNr*C+_dHLxNU(zQoUN zr504hKPigp?cTGK!|#6Im&Y%+w9C!Nq0#je;{Qa;1lA?(7?df%=Vvw!EC0P zbA$7LPR_?NGNH>u`-+Nc8)_Tt>KYmvo@8b&UAlDd-n~1o?c8@`->Fllu0Fc@{^reh z@7_Hihpu(bvjCZk$&(_B*Ti{~wzL-+n>zwn>`3lsC+0}lEI7r1%A%&_)*%co!8|J#1=uB!`|A3p8<{-6JP zzLE=nnPb_lUi@oZ$1Gd&-{E>A->&)vuN~O$+FvML(ELvRh1CVe@BAgQFBbe}--TUy z=KtiJ`F8VGygydg^zLqEaO|x)Pi`#Cp1pos|4qm9yJJgp)sOvCp8jS*`rWlVgWk`& zy=}k4cb(AhwwI)CJ&ThqIqmnAWpCg0uZg#PUOANKg!^{uf4F;PPo`sej@8BgIS&tS zEdHW)=XB$|{O2<(L+36&xOuhpi)p^w3zLs0yfHBE-(Ixf`R(!-6UAR>zub|qtbSYZ zmxAAXRmaYn7nNE}Gp$w4O?%N@_Wq%<cavG=9OcnZa$Gdx}!*={ZMq&u#RsQ<&(z=i8hvxm) lnjF^dX9bIIq^PfBc5Nq0#je<0-%;1lA?(7?dn%+A@uY1?4? zpMl{&JG<3vt7npu+q}Gjmj$m13!9sq+*?%CR9oB7(C{oSZpo4*d-v|$bA8XLQ>U&! zx&GnGmG^hTnehFutmMper%NVhtwY0$U)ML(*A$M5_lvM(0=X5YnEa^Wv?EW6cW?y6l!^``_N`4?Dj=dV8MV+M_r8Kk!i+V@2zX}glA=RDaLdHJw%#RcwnhR*do$Di%~u4gludG7mdKI;Vst E0J)cUr~m)} diff --git a/tests/visual_tests/images/marker-interior-position-600-400-1.0-agg-reference.png b/tests/visual_tests/images/marker-interior-position-600-400-1.0-agg-reference.png new file mode 100644 index 0000000000000000000000000000000000000000..47aaa38d18f2b06a52d671896b900b69c8ad709b GIT binary patch literal 1211 zcmeAS@N?(olHy`uVBq!ia0y~yV2WU1V4T3h3>0aQW?l`Xqyv0HTp1Yt|7T!eW&QP^ zm9?RvVd>JPd-v`=b?Vf+ckk-nUHZ$wz!L50;uumf=j}B^ufqupZ4X~?_LLP|QQq-F zr-HG4(Qa>jg>!se$IUMIPWf`QA#8p=U+sUp5A8hp+dj9=w>x0R8z!;FAR=+Yp@i0h zT+O0v+B~ERY-70gx=MfB&wuu@46mQ2>6h_me<#x@GHj=ZmfuSw%Kf|Fe>|V0Td?LP z+nNgvqEZ{2n8Pv-aPdYgWYjiEXr+#z-QCX?eanj@E*mSa;e4IAXV%fExw3JGXKm0s zHaT+5<3x-0t&dG@q}rGxHe6C$^D~0aQW?l`Xqyv0HTp1Yt|7T!eW&QP^ zjjf@fVabvud-v`=b?Vf+ckeca9+hHXVDa&EaSW-L^Y)sd*O34Q*NZQ>dZHp;39VVf zy4@f{+bqvv&GLeRn{01fJoJPwT>SiM!tY`Q#lyRG?|rz;Zn5MbSFHKf|ms}C~EB#dd=SOEZ#X{{%yNo2lVH~x9^p%-H@Q0NfT$$!mFcZQj~NG zS48YSJKgqu{rQ`2_qmowF~0s!Nfw4>pEu0ae5HHR>$+Kh#Seq0tDnm{r-UW|(=xII literal 0 HcmV?d00001 diff --git a/tests/visual_tests/images/marker-interior-position-600-400-2.0-agg-reference.png b/tests/visual_tests/images/marker-interior-position-600-400-2.0-agg-reference.png new file mode 100644 index 0000000000000000000000000000000000000000..e280621ac90ed177e02e3c6ae72f7af6863f79a0 GIT binary patch literal 1330 zcmeAS@N?(olHy`uVBq!ia0y~yV2WU1V4T3h3>0aQW?l`X3<7*YTp1Yt|7T#};P@dZ z`2Ig9=RYyAKbo4Usj~_T`x+V=mM&eockkX)r%ru1dGh^@8}Ht|d%Ex|69WUwUQZXt zkcv5PuW!r>PGo3%cpGe6o-m0u1`&xH z4v{RNeZa~#s#`Yx-|Te^ua7^Bdc(ZtBV{I%;Ru>)`PQGb_y3kpwSV$Ah80A6OK)&u z4$C;e#T&7ZQQIV;m2J(122m-B1i#MqT_2NkA+i39>+A;8-$vmTHnPX6HXF~(*?;eT z#hT4`Ew61|e>g&4{qx=CuW$db-j+UgD>5PXu$5$Sn(w*qDUUaPJC=O+X2qJ=h{fi6 z`Ln<5X-mAko$trR*>!f?Z^<6pwR6L*s*N8b(z~5szdC7f?K11N-2I!=y4_ztoV4GX z{j+=aYs)n^Z|-LA-=}}js;2b&wav~E;?IBgCa&r3UYBomZN60PhJ@Zss(PBjFdNlQ z-83jF^}4IH>dWaq{r@w)rE9f!tQJkkJx6kJfF&o^F|BA8inTgZ;sPuL89ZJ6T-G@y GGywqKS_2dS literal 0 HcmV?d00001 diff --git a/tests/visual_tests/images/marker-interior-position-600-400-2.0-cairo-reference.png b/tests/visual_tests/images/marker-interior-position-600-400-2.0-cairo-reference.png new file mode 100644 index 0000000000000000000000000000000000000000..22f8de24a52509f2d6c379e1bbc18732b17d6398 GIT binary patch literal 1978 zcmeAS@N?(olHy`uVBq!ia0y~yV2WU1V4T3g3>0yF{caVIG7Intab;lm|DS<@i|ZRd z|3@LAcmKG!{)&obWK7S?n^;`j+tARkWXY1RJw1E(?mczt)Q8ijKU}%;?%lhj3!%Fh z7})JST^vIy=DfXiJMYFZM%TdGfls1i-2eaIoO|h;57&}{fVV4Elh>N>6FB_v8LRWY z$|ml6HSZZeBnyd!b98IB92GtEh%2$Mb;Ay)h!{nkbpjL%e>6Ju>(k6L)8E&c)qe-F z|8$;t4q~gXk?-5Um)P1;%Au{Y#yLVUQT7lQ=WBs5kBIgSPKT^oM1`&?u9+~ZWKe`l zjq#fE_0^(rM_85CJieV+`(E&$=^?4asV%05v^cX{LWQnztX5eg80HzFyk^n{w+Pk5 z-VI)~S1|AS+xjXu{+sjSj>T}kuDoYwTU+}lovVMpovrp7e|h&2@?ep*bIqfgsa<$@U;@@8ta`FA=xn4g!Yf~c<_DHn-tM|9Quh01J zach6*NhW>n(K3*VwDCdH(U!zjzLyy>)*NZ~jnu zn)kh5_bGS|NAOvkhU_9$!zdyY#a>^8X93h~9A7YnQLb z%hlJ_{CRqPd3xK|mxq71m%o?0SN}t0%?WM3)AxDn{UTKVC(|Vy=@3<;R?sb142PgT zdz$0*>F4=BKJ9h?alPiV#Xosq7T?>m;m1STR8x?;?vJd`$HEue#P|6DD`N&vS3j3^ HP6 - - + + + diff --git a/tests/visual_tests/test.py b/tests/visual_tests/test.py index afbf87161..e25dfd97c 100755 --- a/tests/visual_tests/test.py +++ b/tests/visual_tests/test.py @@ -109,7 +109,7 @@ files = { 'geometry-transform-translate': {'sizes':[(200,200)]}, 'geometry-transform-translate-patterns': {'sizes':[(200,200)]}, 'geometry-transform-translate-patterns-svg': {'sizes':[(200,200)]}, - 'marker-interior-position':{}, + 'marker-interior-position': {'sizes':[(600,400)]}, 'marker-svg-opacity':{}, 'marker-svg-opacity2':{}, 'marker-svg-empty-g-element':{}, From 4e12b999f030ca85b920553a41484eabd546ad44 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 12 Aug 2014 15:03:51 -0700 Subject: [PATCH 06/10] pull in pgraster plugin by @strk from 2.3.x to master/3.x - refs #2329 #1660 --- SConstruct | 3 +- bindings/python/mapnik/__init__.py | 45 + plugins/input/pgraster/README | 14 + plugins/input/pgraster/TODO | 23 + plugins/input/pgraster/build.py | 84 ++ .../input/pgraster/pgraster_datasource.cpp | 1200 +++++++++++++++++ .../input/pgraster/pgraster_datasource.hpp | 147 ++ .../input/pgraster/pgraster_featureset.cpp | 360 +++++ .../input/pgraster/pgraster_featureset.hpp | 71 + .../input/pgraster/pgraster_wkb_reader.cpp | 497 +++++++ .../input/pgraster/pgraster_wkb_reader.hpp | 87 ++ tests/python_tests/pgraster_test.py | 747 ++++++++++ 12 files changed, 3277 insertions(+), 1 deletion(-) create mode 100644 plugins/input/pgraster/README create mode 100644 plugins/input/pgraster/TODO create mode 100644 plugins/input/pgraster/build.py create mode 100644 plugins/input/pgraster/pgraster_datasource.cpp create mode 100644 plugins/input/pgraster/pgraster_datasource.hpp create mode 100644 plugins/input/pgraster/pgraster_featureset.cpp create mode 100644 plugins/input/pgraster/pgraster_featureset.hpp create mode 100644 plugins/input/pgraster/pgraster_wkb_reader.cpp create mode 100644 plugins/input/pgraster/pgraster_wkb_reader.hpp create mode 100644 tests/python_tests/pgraster_test.py diff --git a/SConstruct b/SConstruct index 6e594967b..b70739f21 100644 --- a/SConstruct +++ b/SConstruct @@ -103,6 +103,7 @@ pretty_dep_names = { PLUGINS = { # plugins with external dependencies # configured by calling project, hence 'path':None 'postgis': {'default':True,'path':None,'inc':'libpq-fe.h','lib':'pq','lang':'C'}, + 'pgraster': {'default':True,'path':None,'inc':'libpq-fe.h','lib':'pq','lang':'C'}, 'gdal': {'default':True,'path':None,'inc':'gdal_priv.h','lib':'gdal','lang':'C++'}, 'ogr': {'default':True,'path':None,'inc':'ogrsf_frmts.h','lib':'gdal','lang':'C++'}, # configured with custom paths, hence 'path': PREFIX/INCLUDES/LIBS @@ -1429,7 +1430,7 @@ if not preconfigured: env['LIBS'].remove(libname) else: details['lib'] = libname - elif plugin == 'postgis': + elif plugin == 'postgis' or plugin == 'pgraster': conf.parse_pg_config('PG_CONFIG') elif plugin == 'ogr': if conf.ogr_enabled(): diff --git a/bindings/python/mapnik/__init__.py b/bindings/python/mapnik/__init__.py index 3f94a62bb..c03ea46d6 100644 --- a/bindings/python/mapnik/__init__.py +++ b/bindings/python/mapnik/__init__.py @@ -440,6 +440,51 @@ def PostGIS(**keywords): keywords['type'] = 'postgis' return CreateDatasource(keywords) +def PgRaster(**keywords): + """Create a PgRaster Datasource. + + Required keyword arguments: + dbname -- database name to connect to + table -- table name or subselect query + + *Note: if using subselects for the 'table' value consider also + passing the 'raster_field' and 'srid' and 'extent_from_subquery' + options and/or specifying the 'raster_table' option. + + Optional db connection keyword arguments: + user -- database user to connect as (default: see postgres docs) + password -- password for database user (default: see postgres docs) + host -- portgres hostname (default: see postgres docs) + port -- postgres port (default: see postgres docs) + initial_size -- integer size of connection pool (default: 1) + max_size -- integer max of connection pool (default: 10) + persist_connection -- keep connection open (default: True) + + Optional table-level keyword arguments: + extent -- manually specified data extent (comma delimited string, default: None) + estimate_extent -- boolean, direct PostGIS to use the faster, less accurate `estimate_extent` over `extent` (default: False) + extent_from_subquery -- boolean, direct Mapnik to query Postgis for the extent of the raw 'table' value (default: uses 'geometry_table') + raster_table -- specify geometry table to use to look up metadata (default: automatically parsed from 'table' value) + raster_field -- specify geometry field to use (default: first entry in raster_columns) + srid -- specify srid to use (default: auto-detected from geometry_field) + row_limit -- integer limit of rows to return (default: 0) + cursor_size -- integer size of binary cursor to use (default: 0, no binary cursor is used) + use_overviews -- boolean, use overviews when available (default: false) + prescale_rasters -- boolean, scale rasters on the db side (default: false) + clip_rasters -- boolean, clip rasters on the db side (default: false) + band -- integer, if non-zero interprets the given band (1-based offset) as a data raster (default: 0) + + >>> from mapnik import PgRaster, Layer + >>> params = dict(dbname='mapnik',table='osm',user='postgres',password='gis') + >>> params['estimate_extent'] = False + >>> params['extent'] = '-20037508,-19929239,20037508,19929239' + >>> pgraster = PgRaster(**params) + >>> lyr = Layer('PgRaster Layer') + >>> lyr.datasource = pgraster + + """ + keywords['type'] = 'pgraster' + return CreateDatasource(keywords) def Raster(**keywords): """Create a Raster (Tiff) Datasource. diff --git a/plugins/input/pgraster/README b/plugins/input/pgraster/README new file mode 100644 index 000000000..baf2814a9 --- /dev/null +++ b/plugins/input/pgraster/README @@ -0,0 +1,14 @@ +This plugin shares directives with the "postgis" one, +with the following differences: + + - "raster_field" replaces "geometry_field" + + - "raster_table" replaces "geometry_table" + + - "prescale_rasters" replaces "simplify_geometries" + + - "use_overviews" introduced + + - "clip_rasters" boolean introduced, defaults to false + + - "band" introduced, with same semantic of the GDAL driver diff --git a/plugins/input/pgraster/TODO b/plugins/input/pgraster/TODO new file mode 100644 index 000000000..730c846fa --- /dev/null +++ b/plugins/input/pgraster/TODO @@ -0,0 +1,23 @@ +- Automated tests + - Test subqueries [x] + - Test Data input [x] + - Test RGB input [x] + - Test RGBA input [x] + - Test Grayscale input [x] +- Support for all band types: + - PT_1BB data[x] rgb[ ] grayscale[x] + - PT_2BUI data[x] rgb[ ] grayscale[x] + - PT_4BUI data[x] rgb[ ] grayscale[x] + - PT_8BSI data[x] rgb[x] grayscale[x] + - PT_8BUI data[x] rgb[x] grayscale[x] + - PT_16BSI data[x] rgb[ ] grayscale[x] + - PT_16BUI data[x] rgb[ ] grayscale[x] + - PT_32BSI data[x] rgb[ ] grayscale[x] + - PT_32BUI data[x] rgb[ ] grayscale[x] + - PT_32BF data[x] rgb[ ] grayscale[ ] + - PT_64BF data[x] rgb[ ] grayscale[ ] +- Have pgraster and postgis plugins share the same connection pool +- Make overviews enabled by default if table is not a subquery ? +- Make clipping enabled automatically when needed ? +- Allow more flexible band layout specification, see + http://github.com/mapnik/mapnik/wiki/RFC:-Raster-color-interpretation diff --git a/plugins/input/pgraster/build.py b/plugins/input/pgraster/build.py new file mode 100644 index 000000000..e4b083806 --- /dev/null +++ b/plugins/input/pgraster/build.py @@ -0,0 +1,84 @@ +# +# This file is part of Mapnik (c++ mapping toolkit) +# +# Copyright (C) 2013 Artem Pavlenko +# +# Mapnik is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# + +Import ('plugin_base') +Import ('env') +from copy import copy + +PLUGIN_NAME = 'pgraster' + +plugin_env = plugin_base.Clone() + +plugin_sources = Split( + """ + %(PLUGIN_NAME)s_datasource.cpp + %(PLUGIN_NAME)s_featureset.cpp + %(PLUGIN_NAME)s_wkb_reader.cpp + """ % locals() +) + +cxxflags = [] +plugin_env['LIBS'] = [] + +# TODO: use variable for top-level dir +plugin_env['CXXFLAGS'].append('-Iplugins/input/postgis') + +if env['RUNTIME_LINK'] == 'static': + # pkg-config is more reliable than pg_config across platforms + cmd = 'pkg-config libpq --libs --static' + try: + plugin_env.ParseConfig(cmd) + except OSError, e: + plugin_env.Append(LIBS='pq') +else: + plugin_env.Append(LIBS='pq') + +# Link Library to Dependencies +libraries = copy(plugin_env['LIBS']) + +if env['THREADING'] == 'multi': + libraries.append('boost_thread%s' % env['BOOST_APPEND']) + +if env['PLUGIN_LINKING'] == 'shared': + libraries.insert(0,env['MAPNIK_NAME']) + libraries.append(env['ICU_LIB_NAME']) + libraries.append('boost_system%s' % env['BOOST_APPEND']) + + TARGET = plugin_env.SharedLibrary('../%s' % PLUGIN_NAME, + SHLIBPREFIX='', + SHLIBSUFFIX='.input', + source=plugin_sources, + LIBS=libraries, + LINKFLAGS=env['CUSTOM_LDFLAGS']) + + # if the plugin links to libmapnik ensure it is built first + Depends(TARGET, env.subst('../../../src/%s' % env['MAPNIK_LIB_NAME'])) + + if 'uninstall' not in COMMAND_LINE_TARGETS: + env.Install(env['MAPNIK_INPUT_PLUGINS_DEST'], TARGET) + env.Alias('install', env['MAPNIK_INPUT_PLUGINS_DEST']) + +plugin_obj = { + 'LIBS': libraries, + 'SOURCES': plugin_sources, +} + +Return('plugin_obj') diff --git a/plugins/input/pgraster/pgraster_datasource.cpp b/plugins/input/pgraster/pgraster_datasource.cpp new file mode 100644 index 000000000..9d15b7ef1 --- /dev/null +++ b/plugins/input/pgraster/pgraster_datasource.cpp @@ -0,0 +1,1200 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#include "connection_manager.hpp" +#include "pgraster_datasource.hpp" +#include "pgraster_featureset.hpp" +#include "asyncresultset.hpp" + + +// mapnik +#include +#include // for byte +#include +#include +#include +#include +#include + +// boost +#include +#include +#include + +// stl +#include +#include +#include +#include +#include + +DATASOURCE_PLUGIN(pgraster_datasource) + +const double pgraster_datasource::FMAX = std::numeric_limits::max(); +const std::string pgraster_datasource::RASTER_COLUMNS = "raster_columns"; +const std::string pgraster_datasource::RASTER_OVERVIEWS = "raster_overviews"; +const std::string pgraster_datasource::SPATIAL_REF_SYS = "spatial_ref_system"; + +using boost::shared_ptr; +using mapnik::attribute_descriptor; + +namespace { + // TODO: move to sql_utils + std::string quote_literal(std::string& s) { + return "'" + s + "'"; // TODO: escape internal quotes + } + + // TODO: move to sql_utils + std::string quote_ident(std::string& s) { + return "\"" + s + "\""; // TODO: escape internal quotes + } + +}; + +pgraster_datasource::pgraster_datasource(parameters const& params) + : datasource(params), + table_(*params.get("table", "")), + schema_(""), + raster_table_(*params.get("raster_table", "")), + raster_field_(*params.get("raster_field", "")), + key_field_(*params.get("key_field", "")), + cursor_fetch_size_(*params.get("cursor_size", 0)), + row_limit_(*params.get("row_limit", 0)), + type_(datasource::Raster), + srid_(*params.get("srid", 0)), + band_(*params.get("band", 0)), + extent_initialized_(false), + prescale_rasters_(*params.get("prescale_rasters", false)), + use_overviews_(*params.get("use_overviews", false)), + clip_rasters_(*params.get("clip_rasters", false)), + desc_(*params.get("type"), "utf-8"), + creator_(params.get("host"), + params.get("port"), + params.get("dbname"), + params.get("user"), + params.get("password"), + params.get("connect_timeout", "4")), + bbox_token_("!bbox!"), + scale_denom_token_("!scale_denominator!"), + pixel_width_token_("!pixel_width!"), + pixel_height_token_("!pixel_height!"), + pool_max_size_(*params_.get("max_size", 10)), + persist_connection_(*params.get("persist_connection", true)), + extent_from_subquery_(*params.get("extent_from_subquery", false)), + estimate_extent_(*params.get("estimate_extent", false)), + max_async_connections_(*params_.get("max_async_connection", 1)), + asynchronous_request_(false), + // params below are for testing purposes only and may be removed at any time + intersect_min_scale_(*params.get("intersect_min_scale", 0)), + intersect_max_scale_(*params.get("intersect_max_scale", 0)) +{ +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, "pgraster_datasource::init"); +#endif + if (table_.empty()) + { + throw mapnik::datasource_exception("Pgraster Plugin: missing parameter"); + } + + boost::optional ext = params.get("extent"); + if (ext && !ext->empty()) + { + extent_initialized_ = extent_.from_string(*ext); + } + + // NOTE: In multithread environment, pool_max_size_ should be + // max_async_connections_ * num_threads + if(max_async_connections_ > 1) + { + if(max_async_connections_ > pool_max_size_) + { + std::ostringstream err; + err << "PostGIS Plugin: Error: 'max_async_connections (" + << max_async_connections_ << ") must be <= max_size(" << pool_max_size_ << ")"; + throw mapnik::datasource_exception(err.str()); + } + asynchronous_request_ = true; + } + + boost::optional initial_size = params.get("initial_size", 1); + boost::optional autodetect_key_field = params.get("autodetect_key_field", false); + + ConnectionManager::instance().registerPool(creator_, *initial_size, pool_max_size_); + CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); + if (pool) + { + shared_ptr conn = pool->borrowObject(); + if (!conn) return; + + if (conn->isOK()) + { + + desc_.set_encoding(conn->client_encoding()); + + if (raster_table_.empty()) + { + raster_table_ = mapnik::sql_utils::table_from_sql(table_); + // non-trivial subqueries (having no FROM) make it + // impossible to use overviews + // TODO: improve "table_from_sql" ? + if ( raster_table_[raster_table_.find_first_not_of(" \t\r\n")] == '(' ) + { + raster_table_.clear(); + if ( use_overviews_ ) + { + std::ostringstream err; + err << "Pgraster Plugin: overviews cannot be used " + "with non-trivial subqueries"; + MAPNIK_LOG_WARN(pgraster) << err.str(); + use_overviews_ = false; + } + if ( ! extent_from_subquery_ ) { + std::ostringstream err; + err << "Pgraster Plugin: extent can only be computed " + "from subquery as we could not found table source"; + MAPNIK_LOG_WARN(pgraster) << err.str(); + extent_from_subquery_ = true; + } + + } + } + + std::string::size_type idx = raster_table_.find_last_of('.'); + if (idx != std::string::npos) + { + schema_ = raster_table_.substr(0, idx); + raster_table_ = raster_table_.substr(idx + 1); + } + + // If we do not know either the geometry_field or the srid or we + // want to use overviews but do not know about schema, or + // no extent was specified, then attempt to fetch the missing + // information from a raster_columns entry. + // + // This will return no records if we are querying a bogus table returned + // from the simplistic table parsing in table_from_sql() or if + // the table parameter references a table, view, or subselect not + // registered in the geometry columns. + // + geometryColumn_ = mapnik::sql_utils::unquote_double(raster_field_); + if (!raster_table_.empty() && ( + geometryColumn_.empty() || srid_ == 0 || + (schema_.empty() && use_overviews_) || + ! extent_initialized_ + )) + { +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::init(get_srid_and_geometry_column)"); +#endif + std::ostringstream s; + + try + { + s << "SELECT r_raster_column col, srid, r_table_schema"; + if ( ! extent_initialized_ ) { + s << ", st_xmin(extent) xmin, st_ymin(extent) ymin" + << ", st_xmax(extent) xmax, st_ymax(extent) ymax"; + } + s << " FROM " + << RASTER_COLUMNS << " WHERE r_table_name='" + << mapnik::sql_utils::unquote_double(raster_table_) + << "'"; + if (! schema_.empty()) + { + s << " AND r_table_schema='" + << mapnik::sql_utils::unquote_double(schema_) + << "'"; + } + if (! raster_field_.empty()) + { + s << " AND r_raster_column='" + << mapnik::sql_utils::unquote_double(raster_field_) + << "'"; + } + MAPNIK_LOG_DEBUG(pgraster) << + "pgraster_datasource: running query " << s.str(); + shared_ptr rs = conn->executeQuery(s.str()); + if (rs->next()) + { + geometryColumn_ = rs->getValue("col"); + if ( ! extent_initialized_ ) + { + double lox, loy, hix, hiy; + if (mapnik::util::string2double(rs->getValue("xmin"), lox) && + mapnik::util::string2double(rs->getValue("ymin"), loy) && + mapnik::util::string2double(rs->getValue("xmax"), hix) && + mapnik::util::string2double(rs->getValue("ymax"), hiy)) + { + extent_.init(lox, loy, hix, hiy); + extent_initialized_ = true; + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Layer extent=" << extent_; + } + else + { + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Could not determine extent from query: " << s.str(); + } + } + if (srid_ == 0) + { + const char* srid_c = rs->getValue("srid"); + if (srid_c != NULL) + { + int result = 0; + const char * end = srid_c + std::strlen(srid_c); + if (mapnik::util::string2int(srid_c, end, result)) + { + srid_ = result; + } + } + } + if ( schema_.empty() ) + { + schema_ = rs->getValue("r_table_schema"); + } + } + else + { + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: no response from metadata query " << s.str(); + } + rs->close(); + } + catch (mapnik::datasource_exception const& ex) { + // let this pass on query error and use the fallback below + MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: metadata query failed: " << ex.what(); + } + + // If we still do not know the srid then we can try to fetch + // it from the 'table_' parameter, which should work even if it is + // a subselect as long as we know the geometry_field to query + if (! geometryColumn_.empty() && srid_ <= 0) + { + s.str(""); + + s << "SELECT ST_SRID(\"" << geometryColumn_ << "\") AS srid FROM " + << populate_tokens(table_) << " WHERE \"" << geometryColumn_ << "\" IS NOT NULL LIMIT 1;"; + + shared_ptr rs = conn->executeQuery(s.str()); + if (rs->next()) + { + const char* srid_c = rs->getValue("srid"); + if (srid_c != NULL) + { + int result = 0; + const char * end = srid_c + std::strlen(srid_c); + if (mapnik::util::string2int(srid_c, end, result)) + { + srid_ = result; + } + } + } + rs->close(); + } + } + + // If overviews were requested, take note of the max scale + // of each available overview, sorted by scale descending + if ( use_overviews_ ) + { + std::ostringstream err; + if ( schema_.empty() ) + { + err << "Pgraster Plugin: unable to lookup available table" + << " overviews due to unknown schema"; + throw mapnik::datasource_exception(err.str()); + } + if ( geometryColumn_.empty() ) + { + err << "Pgraster Plugin: unable to lookup available table" + << " overviews due to unknown column name"; + throw mapnik::datasource_exception(err.str()); + } + + std::ostringstream s; + s << "select " + "r.r_table_schema sch, " + "r.r_table_name tab, " + "r.r_raster_column col, " + "greatest(abs(r.scale_x), abs(r.scale_y)) scl " + "from" + " raster_overviews o," + " raster_columns r " + "where" + " o.r_table_schema = '" + << mapnik::sql_utils::unquote_double(schema_) + << "' and o.r_table_name = '" + << mapnik::sql_utils::unquote_double(raster_table_) + << "' and o.r_raster_column = '" + << mapnik::sql_utils::unquote_double(geometryColumn_) + << "' and r.r_table_schema = o.o_table_schema" + " and r.r_table_name = o.o_table_name" + " and r.r_raster_column = o.o_raster_column" + " ORDER BY scl ASC"; + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: running query " << s.str(); + shared_ptr rs = conn->executeQuery(s.str()); + while (rs->next()) + { + overviews_.resize(overviews_.size()+1); + pgraster_overview& ov = overviews_.back(); + ov.schema = rs->getValue("sch"); + ov.table = rs->getValue("tab"); + ov.column = rs->getValue("col"); + ov.scale = atof(rs->getValue("scl")); + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: found overview " << ov.schema << "." << ov.table << "." << ov.column << " with scale " << ov.scale; + } + rs->close(); + if ( overviews_.empty() ) { + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: no overview found for " << schema_ << "." << raster_table_ << "." << geometryColumn_; + } + } + + // detect primary key + if (*autodetect_key_field && key_field_.empty()) + { +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::bind(get_primary_key)"); +#endif + + std::ostringstream s; + s << "SELECT a.attname, a.attnum, t.typname, t.typname in ('int2','int4','int8') " + "AS is_int FROM pg_class c, pg_attribute a, pg_type t, pg_namespace n, pg_index i " + "WHERE a.attnum > 0 AND a.attrelid = c.oid " + "AND a.atttypid = t.oid AND c.relnamespace = n.oid " + "AND c.oid = i.indrelid AND i.indisprimary = 't' " + "AND t.typname !~ '^geom' AND c.relname =" + << " '" << mapnik::sql_utils::unquote_double(raster_table_) << "' " + //"AND a.attnum = ANY (i.indkey) " // postgres >= 8.1 + << "AND (i.indkey[0]=a.attnum OR i.indkey[1]=a.attnum OR i.indkey[2]=a.attnum " + "OR i.indkey[3]=a.attnum OR i.indkey[4]=a.attnum OR i.indkey[5]=a.attnum " + "OR i.indkey[6]=a.attnum OR i.indkey[7]=a.attnum OR i.indkey[8]=a.attnum " + "OR i.indkey[9]=a.attnum) "; + if (! schema_.empty()) + { + s << "AND n.nspname='" + << mapnik::sql_utils::unquote_double(schema_) + << "' "; + } + s << "ORDER BY a.attnum"; + + shared_ptr rs_key = conn->executeQuery(s.str()); + if (rs_key->next()) + { + unsigned int result_rows = rs_key->size(); + if (result_rows == 1) + { + bool is_int = (std::string(rs_key->getValue(3)) == "t"); + if (is_int) + { + const char* key_field_string = rs_key->getValue(0); + if (key_field_string) + { + key_field_ = std::string(key_field_string); + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: auto-detected key field of '" + << key_field_ << "' on table '" << raster_table_ << "'"; + } + } + else + { + // throw for cases like a numeric primary key, which is invalid + // as it should be floating point (int numerics are useless) + std::ostringstream err; + err << "PostGIS Plugin: Error: '" + << rs_key->getValue(0) + << "' on table '" + << raster_table_ + << "' is not a valid integer primary key field\n"; + throw mapnik::datasource_exception(err.str()); + } + } + else if (result_rows > 1) + { + std::ostringstream err; + err << "PostGIS Plugin: Error: '" + << "multi column primary key detected but is not supported"; + throw mapnik::datasource_exception(err.str()); + } + } + rs_key->close(); + } + + // if a globally unique key field/primary key is required + // but still not known at this point, then throw + if (*autodetect_key_field && key_field_.empty()) + { + throw mapnik::datasource_exception(std::string("PostGIS Plugin: Error: primary key required") + + " but could not be detected for table '" + + raster_table_ + "', please supply 'key_field' option to specify field to use for primary key"); + } + + if (srid_ == 0) + { + srid_ = -1; + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Table " << table_ << " is using SRID=" << srid_; + } + + // At this point the geometry_field may still not be known + // but we'll catch that where more useful... + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Using SRID=" << srid_; + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Using geometry_column=" << geometryColumn_; + + // collect attribute desc +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats2__(std::clog, "pgraster_datasource::bind(get_column_description)"); +#endif + + std::ostringstream s; + s << "SELECT * FROM " << populate_tokens(table_) << " LIMIT 0"; + + shared_ptr rs = conn->executeQuery(s.str()); + int count = rs->getNumFields(); + bool found_key_field = false; + for (int i = 0; i < count; ++i) + { + std::string fld_name = rs->getFieldName(i); + int type_oid = rs->getTypeOID(i); + + // validate type of key_field + if (! found_key_field && ! key_field_.empty() && fld_name == key_field_) + { + if (type_oid == 20 || type_oid == 21 || type_oid == 23) + { + found_key_field = true; + desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); + } + else + { + std::ostringstream error_s; + error_s << "invalid type '"; + + std::ostringstream type_s; + type_s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; + + shared_ptr rs_oid = conn->executeQuery(type_s.str()); + if (rs_oid->next()) + { + error_s << rs_oid->getValue("typname") + << "' (oid:" << rs_oid->getValue("oid") << ")"; + } + else + { + error_s << "oid:" << type_oid << "'"; + } + + rs_oid->close(); + error_s << " for key_field '" << fld_name << "' - " + << "must be an integer primary key"; + + rs->close(); + throw mapnik::datasource_exception(error_s.str()); + } + } + else + { + switch (type_oid) + { + case 16: // bool + desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Boolean)); + break; + case 20: // int8 + case 21: // int2 + case 23: // int4 + desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer)); + break; + case 700: // float4 + case 701: // float8 + case 1700: // numeric + desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double)); + break; + case 1042: // bpchar + case 1043: // varchar + case 25: // text + case 705: // literal + desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String)); + break; + default: // should not get here +#ifdef MAPNIK_LOG + s.str(""); + s << "SELECT oid, typname FROM pg_type WHERE oid = " << type_oid; + + shared_ptr rs_oid = conn->executeQuery(s.str()); + if (rs_oid->next()) + { + std::string typname(rs_oid->getValue("typname")); + if (typname != "geometry" && typname != "raster") + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: Unknown type=" << typname + << " (oid:" << rs_oid->getValue("oid") << ")"; + } + } + else + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_datasource: Unknown type_oid=" << type_oid; + } + rs_oid->close(); +#endif + break; + } + } + } + + rs->close(); + + } + + // Close explicitly the connection so we can 'fork()' without sharing open connections + conn->close(); + + } +} + +pgraster_datasource::~pgraster_datasource() +{ + if (! persist_connection_) + { + CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); + if (pool) + { + try { + shared_ptr conn = pool->borrowObject(); + if (conn) + { + conn->close(); + } + } catch (mapnik::datasource_exception const& ex) { + // happens when borrowObject tries to + // create a new connection and fails. + // In turn, new connection would be needed + // when our broke and was thus no good to + // be borrowed + // See https://github.com/mapnik/mapnik/issues/2191 + } + } + } +} + +const char * pgraster_datasource::name() +{ + return "pgraster"; +} + +mapnik::datasource::datasource_t pgraster_datasource::type() const +{ + return type_; +} + +layer_descriptor pgraster_datasource::get_descriptor() const +{ + return desc_; +} + +std::string pgraster_datasource::sql_bbox(box2d const& env) const +{ + std::ostringstream b; + + if (srid_ > 0) + { + b << "ST_SetSRID("; + } + + b << "'BOX3D("; + b << std::setprecision(16); + b << env.minx() << " " << env.miny() << ","; + b << env.maxx() << " " << env.maxy() << ")'::box3d"; + + if (srid_ > 0) + { + b << ", " << srid_ << ")"; + } + + return b.str(); +} + +std::string pgraster_datasource::populate_tokens(std::string const& sql) const +{ + std::string populated_sql = sql; + + if (boost::algorithm::icontains(sql, bbox_token_)) + { + box2d max_env(-1.0 * FMAX, -1.0 * FMAX, FMAX, FMAX); + const std::string max_box = sql_bbox(max_env); + boost::algorithm::replace_all(populated_sql, bbox_token_, max_box); + } + + if (boost::algorithm::icontains(sql, scale_denom_token_)) + { + std::ostringstream ss; + ss << FMAX; + boost::algorithm::replace_all(populated_sql, scale_denom_token_, ss.str()); + } + + if (boost::algorithm::icontains(sql, pixel_width_token_)) + { + std::ostringstream ss; + ss << 0; + boost::algorithm::replace_all(populated_sql, pixel_width_token_, ss.str()); + } + + if (boost::algorithm::icontains(sql, pixel_height_token_)) + { + std::ostringstream ss; + ss << 0; + boost::algorithm::replace_all(populated_sql, pixel_height_token_, ss.str()); + } + + return populated_sql; +} + +std::string pgraster_datasource::populate_tokens(std::string const& sql, double scale_denom, box2d const& env, double pixel_width, double pixel_height) const +{ + std::string populated_sql = sql; + std::string box = sql_bbox(env); + + if (boost::algorithm::icontains(populated_sql, scale_denom_token_)) + { + std::ostringstream ss; + ss << scale_denom; + boost::algorithm::replace_all(populated_sql, scale_denom_token_, ss.str()); + } + + if (boost::algorithm::icontains(sql, pixel_width_token_)) + { + std::ostringstream ss; + ss << pixel_width; + boost::algorithm::replace_all(populated_sql, pixel_width_token_, ss.str()); + } + + if (boost::algorithm::icontains(sql, pixel_height_token_)) + { + std::ostringstream ss; + ss << pixel_height; + boost::algorithm::replace_all(populated_sql, pixel_height_token_, ss.str()); + } + + if (boost::algorithm::icontains(populated_sql, bbox_token_)) + { + boost::algorithm::replace_all(populated_sql, bbox_token_, box); + return populated_sql; + } + else + { + std::ostringstream s; + + if (intersect_min_scale_ > 0 && (scale_denom <= intersect_min_scale_)) + { + s << " WHERE ST_Intersects(\"" << geometryColumn_ << "\"," << box << ")"; + } + else if (intersect_max_scale_ > 0 && (scale_denom >= intersect_max_scale_)) + { + // do no bbox restriction + } + else + { + s << " WHERE \"" << geometryColumn_ << "\" && " << box; + } + + return populated_sql + s.str(); + } +} + + +boost::shared_ptr pgraster_datasource::get_resultset(boost::shared_ptr &conn, std::string const& sql, CnxPool_ptr const& pool, processor_context_ptr ctx) const +{ + + if (!ctx) + { + // ! asynchronous_request_ + if (cursor_fetch_size_ > 0) + { + // cursor + std::ostringstream csql; + std::string cursor_name = conn->new_cursor_name(); + + csql << "DECLARE " << cursor_name << " BINARY INSENSITIVE NO SCROLL CURSOR WITH HOLD FOR " << sql << " FOR READ ONLY"; + + if (! conn->execute(csql.str())) + { + // TODO - better error + throw mapnik::datasource_exception("Pgraster Plugin: error creating cursor for data select." ); + } + + return boost::make_shared(conn, cursor_name, cursor_fetch_size_); + + } + else + { + // no cursor + return conn->executeQuery(sql, 1); + } + } + else + { // asynchronous requests + + boost::shared_ptr pgis_ctxt = boost::static_pointer_cast(ctx); + if (conn) + { + // lauch async req & create asyncresult with conn + conn->executeAsyncQuery(sql, 1); + return boost::make_shared(pgis_ctxt, pool, conn, sql); + } + else + { + // create asyncresult with null connection + boost::shared_ptr res = boost::make_shared(pgis_ctxt, pool, conn, sql); + pgis_ctxt->add_request(res); + return res; + } + } +} + +processor_context_ptr pgraster_datasource::get_context(feature_style_context_map & ctx) const +{ + if (!asynchronous_request_) + { + return processor_context_ptr(); + } + + std::string ds_name(name()); + feature_style_context_map::const_iterator itr = ctx.find(ds_name); + if (itr != ctx.end()) + { + return itr->second; + } + else + { + return ctx.insert(std::make_pair(ds_name,boost::make_shared())).first->second; + } +} + +featureset_ptr pgraster_datasource::features(query const& q) const +{ + // if the driver is in asynchronous mode, return the appropriate fetaures + if (asynchronous_request_ ) + { + return features_with_context(q,boost::make_shared()); + } + else + { + return features_with_context(q,processor_context_ptr()); + } +} + +featureset_ptr pgraster_datasource::features_with_context(query const& q,processor_context_ptr proc_ctx) const +{ + +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, "pgraster_datasource::features_with_context"); +#endif + + + box2d const& box = q.get_bbox(); + double scale_denom = q.scale_denominator(); + + CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); + + if (pool) + { + shared_ptr conn; + + if ( asynchronous_request_ ) + { + // limit use to num_async_request_ => if reached don't borrow the last connexion object + boost::shared_ptr pgis_ctxt = boost::static_pointer_cast(proc_ctx); + if ( pgis_ctxt->num_async_requests_ < max_async_connections_ ) + { + conn = pool->borrowObject(); + pgis_ctxt->num_async_requests_++; + } + } + else + { + // Always get a connection in synchronous mode + conn = pool->borrowObject(); + if(!conn ) + { + throw mapnik::datasource_exception("Pgraster Plugin: Null connection"); + } + } + + + if (geometryColumn_.empty()) + { + std::ostringstream s_error; + s_error << "PostGIS: geometry name lookup failed for table '"; + + if (! schema_.empty()) + { + s_error << schema_ << "."; + } + s_error << raster_table_ + << "'. Please manually provide the 'geometry_field' parameter or add an entry " + << "in the geometry_columns for '"; + + if (! schema_.empty()) + { + s_error << schema_ << "."; + } + s_error << raster_table_ << "'."; + + throw mapnik::datasource_exception(s_error.str()); + } + + const double px_gw = 1.0 / boost::get<0>(q.resolution()); + const double px_gh = 1.0 / boost::get<1>(q.resolution()); + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: px_gw=" << px_gw + << " px_gh=" << px_gh; + + std::string table_with_bbox; + std::string col = geometryColumn_; + + if ( use_overviews_ ) { + std::string sch = schema_; + std::string tab = mapnik::sql_utils::unquote_double(raster_table_); + const double scale = std::min(px_gw, px_gh); + std::vector::const_reverse_iterator i; + for (i=overviews_.rbegin(); i!=overviews_.rend(); ++i) { + const pgraster_overview& o = *i; + if ( o.scale < scale ) { + sch = o.schema; + tab = o.table; + col = o.column; + MAPNIK_LOG_DEBUG(pgraster) + << "pgraster_datasource: using overview " + << o.schema << "." << o.table << "." << o.column + << " with scale=" << o.scale + << " for min out scale=" << scale; + break; + } else { + MAPNIK_LOG_DEBUG(pgraster) + << "pgraster_datasource: overview " + << o.schema << "." << o.table << "." << o.column + << " with scale=" << o.scale + << " not good for min out scale " << scale; + } + } + table_with_bbox = table_; // possibly a subquery + boost::algorithm::replace_all(table_with_bbox, + mapnik::sql_utils::unquote_double(raster_table_), tab); + boost::algorithm::replace_all(table_with_bbox, + mapnik::sql_utils::unquote_double(schema_), sch); + boost::algorithm::replace_all(table_with_bbox, + mapnik::sql_utils::unquote_double(geometryColumn_), col); + table_with_bbox = populate_tokens(table_with_bbox, + scale_denom, box, px_gw, px_gh); + } else { + table_with_bbox = populate_tokens(table_, scale_denom, box, px_gw, px_gh); + } + + std::ostringstream s; + + s << "SELECT ST_AsBinary("; + + if (band_) s << "ST_Band("; + + if (prescale_rasters_) s << "ST_Resize("; + + if (clip_rasters_) s << "ST_Clip("; + + s << "\"" << col << "\""; + + if (clip_rasters_) { + s << ", ST_Expand(" << sql_bbox(box) + << ", greatest(abs(ST_ScaleX(\"" + << col << "\")), abs(ST_ScaleY(\"" + << col << "\")))))"; + } + + if (prescale_rasters_) { + const double scale = std::min(px_gw, px_gh); + s << ", least(abs(ST_ScaleX(\"" << col + << "\"))::float8/" << scale + << ", 1.0), least(abs(ST_ScaleY(\"" << col + << "\"))::float8/" << scale << ", 1.0))"; + // TODO: if band_ is given, we'll interpret as indexed so + // the rescaling must NOT ruin it (use algorithm mode!) + } + + if (band_) s << ", " << band_ << ")"; + + s << ") AS geom"; + + mapnik::context_ptr ctx = boost::make_shared(); + std::set const& props = q.property_names(); + std::set::const_iterator pos = props.begin(); + std::set::const_iterator end = props.end(); + + if (! key_field_.empty()) + { + mapnik::sql_utils::quote_attr(s, key_field_); + ctx->push(key_field_); + + for (; pos != end; ++pos) + { + if (*pos != key_field_) + { + mapnik::sql_utils::quote_attr(s, *pos); + ctx->push(*pos); + } + } + } + else + { + for (; pos != end; ++pos) + { + mapnik::sql_utils::quote_attr(s, *pos); + ctx->push(*pos); + } + } + + s << " FROM " << table_with_bbox; + + if (row_limit_ > 0) + { + s << " LIMIT " << row_limit_; + } + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: " + "features query: " << s.str(); + + boost::shared_ptr rs = get_resultset(conn, s.str(), pool, proc_ctx); + return boost::make_shared(rs, ctx, + desc_.get_encoding(), !key_field_.empty(), + band_ ? 1 : 0 // whatever band number is given we'd have + // extracted with ST_Band above so it becomes + // band number 1 + ); + + } + + return featureset_ptr(); +} + + +featureset_ptr pgraster_datasource::features_at_point(coord2d const& pt, double tol) const +{ +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, "pgraster_datasource::features_at_point"); +#endif + CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); + if (pool) + { + shared_ptr conn = pool->borrowObject(); + if (!conn) return featureset_ptr(); + + if (conn->isOK()) + { + if (geometryColumn_.empty()) + { + std::ostringstream s_error; + s_error << "PostGIS: geometry name lookup failed for table '"; + + if (! schema_.empty()) + { + s_error << schema_ << "."; + } + s_error << raster_table_ + << "'. Please manually provide the 'geometry_field' parameter or add an entry " + << "in the geometry_columns for '"; + + if (! schema_.empty()) + { + s_error << schema_ << "."; + } + s_error << raster_table_ << "'."; + + throw mapnik::datasource_exception(s_error.str()); + } + + std::ostringstream s; + s << "SELECT ST_AsBinary(\"" << geometryColumn_ << "\") AS geom"; + + mapnik::context_ptr ctx = boost::make_shared(); + std::vector::const_iterator itr = desc_.get_descriptors().begin(); + std::vector::const_iterator end = desc_.get_descriptors().end(); + + if (! key_field_.empty()) + { + mapnik::sql_utils::quote_attr(s, key_field_); + ctx->push(key_field_); + for (; itr != end; ++itr) + { + if (itr->get_name() != key_field_) + { + mapnik::sql_utils::quote_attr(s, itr->get_name()); + ctx->push(itr->get_name()); + } + } + } + else + { + for (; itr != end; ++itr) + { + mapnik::sql_utils::quote_attr(s, itr->get_name()); + ctx->push(itr->get_name()); + } + } + + box2d box(pt.x - tol, pt.y - tol, pt.x + tol, pt.y + tol); + std::string table_with_bbox = populate_tokens(table_, FMAX, box, 0, 0); + + s << " FROM " << table_with_bbox; + + if (row_limit_ > 0) + { + s << " LIMIT " << row_limit_; + } + + boost::shared_ptr rs = get_resultset(conn, s.str(), pool); + return boost::make_shared(rs, ctx, desc_.get_encoding(), !key_field_.empty()); + } + } + + return featureset_ptr(); +} + +box2d pgraster_datasource::envelope() const +{ + if (extent_initialized_) + { + return extent_; + } + + CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); + if (pool) + { + shared_ptr conn = pool->borrowObject(); + if (!conn) return extent_; + if (conn->isOK()) + { + std::ostringstream s; + + std::string col = mapnik::sql_utils::unquote_double(geometryColumn_); + std::string sch = mapnik::sql_utils::unquote_double(schema_); + std::string tab = mapnik::sql_utils::unquote_double(raster_table_); + + if ( ! overviews_.empty() ) + { + // query from highest-factor overview instead + const pgraster_overview& o = overviews_.back(); + sch = o.schema; + tab = o.table; + col = o.column; + } + + if (col.empty()) + { + std::ostringstream s_error; + s_error << "PostGIS: unable to query the layer extent of table '"; + + if (! sch.empty()) + { + s_error << sch << "."; + } + s_error << raster_table_ << "' because we cannot determine the raster field name." + << "\nPlease provide either an 'extent' parameter to skip this query, " + << "a 'raster_field' and/or 'raster_table' parameter, or add " + << "standard constraints to your raster table."; + + throw mapnik::datasource_exception("Pgraster Plugin: " + s_error.str()); + } + + if (estimate_extent_) + { + if (tab.empty()) + { + std::ostringstream s_error; + s_error << "PostGIS: unable to query the layer extent as " + << "we couldn't determine the raster table name.\n" + << "Please provide either an 'extent' parameter to skip this query, " + << "a 'raster_table' parameter, or do not set 'estimate_extent'"; + throw mapnik::datasource_exception("Pgraster Plugin: " + s_error.str()); + } + s << "SELECT ST_XMin(ext),ST_YMin(ext),ST_XMax(ext),ST_YMax(ext)" + << " FROM (SELECT ST_Estimated_Extent('"; + + if (! sch.empty()) + { + s << mapnik::sql_utils::unquote_double(sch) << "','"; + } + + s << mapnik::sql_utils::unquote_double(tab) << "','" + << mapnik::sql_utils::unquote_double(col) << "') as ext) as tmp"; + } + else + { + s << "SELECT ST_XMin(ext),ST_YMin(ext),ST_XMax(ext),ST_YMax(ext)" + << " FROM (SELECT ST_Extent(" << quote_ident(col) << "::geometry) as ext from "; + + if (extent_from_subquery_) + { + // if a subselect limits records then calculating the extent upon the + // subquery will be faster and the bounds will be more accurate + s << populate_tokens(table_) << ") as tmpx"; + } + else + { + if (! sch.empty()) + { + s << quote_ident(sch) << "."; + } + + // but if the subquery does not limit records then querying the + // actual table will be faster as indexes can be used + s << quote_ident(tab) << ") as tmp"; + } + } + + shared_ptr rs = conn->executeQuery(s.str()); + if (rs->next() && ! rs->isNull(0)) + { + double lox, loy, hix, hiy; + if (mapnik::util::string2double(rs->getValue(0), lox) && + mapnik::util::string2double(rs->getValue(1), loy) && + mapnik::util::string2double(rs->getValue(2), hix) && + mapnik::util::string2double(rs->getValue(3), hiy)) + { + extent_.init(lox, loy, hix, hiy); + extent_initialized_ = true; + } + else + { + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: Could not determine extent from query: " << s.str(); + } + } + rs->close(); + } + } + + return extent_; +} + +boost::optional pgraster_datasource::get_geometry_type() const +{ + return boost::optional(); +} + + diff --git a/plugins/input/pgraster/pgraster_datasource.hpp b/plugins/input/pgraster/pgraster_datasource.hpp new file mode 100644 index 000000000..e4f1ae38f --- /dev/null +++ b/plugins/input/pgraster/pgraster_datasource.hpp @@ -0,0 +1,147 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#ifndef PGRASTER_DATASOURCE_HPP +#define PGRASTER_DATASOURCE_HPP + +// mapnik +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// boost +#include +#include +#include + +// stl +#include +#include + +#include "connection_manager.hpp" +#include "resultset.hpp" +#include "cursorresultset.hpp" + +using mapnik::transcoder; +using mapnik::datasource; +using mapnik::feature_style_context_map; +using mapnik::processor_context_ptr; +using mapnik::box2d; +using mapnik::layer_descriptor; +using mapnik::featureset_ptr; +using mapnik::feature_ptr; +using mapnik::query; +using mapnik::parameters; +using mapnik::coord2d; + +typedef boost::shared_ptr< ConnectionManager::PoolType> CnxPool_ptr; + +struct pgraster_overview +{ + std::string schema; + std::string table; + std::string column; + float scale; // max absolute scale between x and y +}; + + +class pgraster_datasource : public datasource +{ +public: + pgraster_datasource(const parameters ¶ms); + ~pgraster_datasource(); + mapnik::datasource::datasource_t type() const; + static const char * name(); + processor_context_ptr get_context(feature_style_context_map &) const; + featureset_ptr features_with_context(query const& q, processor_context_ptr ctx) const; + featureset_ptr features(query const& q) const; + featureset_ptr features_at_point(coord2d const& pt, double tol = 0) const; + mapnik::box2d envelope() const; + boost::optional get_geometry_type() const; + layer_descriptor get_descriptor() const; + +private: + std::string sql_bbox(box2d const& env) const; + std::string populate_tokens(std::string const& sql, double scale_denom, box2d const& env, double pixel_width, double pixel_height) const; + std::string populate_tokens(std::string const& sql) const; + boost::shared_ptr get_resultset(boost::shared_ptr &conn, std::string const& sql, CnxPool_ptr const& pool, processor_context_ptr ctx= processor_context_ptr()) const; + static const std::string RASTER_COLUMNS; + static const std::string RASTER_OVERVIEWS; + static const std::string SPATIAL_REF_SYS; + static const double FMAX; + + const std::string uri_; + const std::string username_; + const std::string password_; + // table name (schema qualified or not) or subquery + const std::string table_; + // schema name (possibly extracted from table_) + std::string schema_; + // table name (possibly extracted from table_) + std::string raster_table_; + const std::string raster_field_; + std::string key_field_; + mapnik::value_integer cursor_fetch_size_; + mapnik::value_integer row_limit_; + std::string geometryColumn_; + mapnik::datasource::datasource_t type_; + int srid_; + // 1-based index of band to extract from the raster + // 0 means fetch all bands + // any index also forces color interpretation off so that values + // arrives untouched into the resulting mapnik raster, for threatment + // by raster colorizer + int band_; + // Available overviews, ordered by max scale, ascending + std::vector overviews_; + mutable bool extent_initialized_; + mutable mapnik::box2d extent_; + bool prescale_rasters_; + bool use_overviews_; + bool clip_rasters_; + layer_descriptor desc_; + ConnectionCreator creator_; + const std::string bbox_token_; + const std::string scale_denom_token_; + const std::string pixel_width_token_; + const std::string pixel_height_token_; + int pool_max_size_; + bool persist_connection_; + bool extent_from_subquery_; + bool estimate_extent_; + int max_async_connections_; + bool asynchronous_request_; + int intersect_min_scale_; + int intersect_max_scale_; +}; + +#endif // PGRASTER_DATASOURCE_HPP diff --git a/plugins/input/pgraster/pgraster_featureset.cpp b/plugins/input/pgraster/pgraster_featureset.cpp new file mode 100644 index 000000000..8e4ded1fd --- /dev/null +++ b/plugins/input/pgraster/pgraster_featureset.cpp @@ -0,0 +1,360 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#include "pgraster_featureset.hpp" +#include "pgraster_wkb_reader.hpp" +#include "resultset.hpp" +#include "cursorresultset.hpp" + +// mapnik +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // for int2net +#include + + +// boost +#include // for boost::int16_t + +// stl +#include +#include + +using mapnik::geometry_type; +using mapnik::byte; +using mapnik::feature_factory; +using mapnik::context_ptr; + +pgraster_featureset::pgraster_featureset(boost::shared_ptr const& rs, + context_ptr const& ctx, + std::string const& encoding, + bool key_field, int bandno) + : rs_(rs), + ctx_(ctx), + tr_(new transcoder(encoding)), + feature_id_(1), + key_field_(key_field), + band_(bandno) +{ +} + +std::string numeric2string(const char* buf); + +feature_ptr pgraster_featureset::next() +{ + while (rs_->next()) + { + // new feature + unsigned pos = 1; + feature_ptr feature; + + if (key_field_) + { + std::string name = rs_->getFieldName(pos); + + // null feature id is not acceptable + if (rs_->isNull(pos)) + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: null value encountered for key_field: " << name; + continue; + } + // create feature with user driven id from attribute + int oid = rs_->getTypeOID(pos); + const char* buf = rs_->getValue(pos); + + // validation happens of this type at initialization + mapnik::value_integer val; + + if (oid == 20) + { + val = int8net(buf); + } + else if (oid == 21) + { + val = int2net(buf); + } + else + { + val = int4net(buf); + } + + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: feature key: " << val; + + feature = feature_factory::create(ctx_, val); + // TODO - extend feature class to know + // that its id is also an attribute to avoid + // this duplication + feature->put(name,val); + ++pos; + } + else + { + // fallback to auto-incrementing id + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: feature id: " << feature_id_; + + feature = feature_factory::create(ctx_, feature_id_); + ++feature_id_; + } + + // null geometry is not acceptable + if (rs_->isNull(0)) + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: null value encountered for raster"; + continue; + } + + // parse geometry + int size = rs_->getFieldLength(0); + const uint8_t *data = (const uint8_t*)rs_->getValue(0); + + mapnik::raster_ptr raster = pgraster_wkb_reader::read(data, size, band_); + if (!raster) + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: could not parse raster wkb"; + // TODO: throw an exception ? + continue; + } + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: raster of " << raster->data_.width() << "x" << raster->data_.height() << " pixels covering extent " << raster->ext_; + feature->set_raster(raster); + + unsigned num_attrs = ctx_->size() + 1; + for (; pos < num_attrs; ++pos) + { + std::string name = rs_->getFieldName(pos); + + // NOTE: we intentionally do not store null here + // since it is equivalent to the attribute not existing + if (!rs_->isNull(pos)) + { + const char* buf = rs_->getValue(pos); + const int oid = rs_->getTypeOID(pos); + switch (oid) + { + case 16: //bool + { + feature->put(name, (buf[0] != 0)); + break; + } + + case 23: //int4 + { + feature->put(name, int4net(buf)); + break; + } + + case 21: //int2 + { + feature->put(name, int2net(buf)); + break; + } + + case 20: //int8/BigInt + { + feature->put(name, int8net(buf)); + break; + } + + case 700: //float4 + { + float val; + float4net(val, buf); + feature->put(name, static_cast(val)); + break; + } + + case 701: //float8 + { + double val; + float8net(val, buf); + feature->put(name, val); + break; + } + + case 25: //text + case 1043: //varchar + case 705: //literal + { + feature->put(name, tr_->transcode(buf)); + break; + } + + case 1042: //bpchar + { + std::string str = mapnik::util::trim_copy(buf); + feature->put(name, tr_->transcode(str.c_str())); + break; + } + + case 1700: //numeric + { + double val; + std::string str = numeric2string(buf); + if (mapnik::util::string2double(str, val)) + { + feature->put(name, val); + } + break; + } + + default: + { + MAPNIK_LOG_WARN(pgraster) << "pgraster_featureset: Unknown type_oid=" << oid; + + break; + } + } + } + } + return feature; + } + return feature_ptr(); +} + + +pgraster_featureset::~pgraster_featureset() +{ + rs_->close(); +} + +std::string numeric2string(const char* buf) +{ + boost::int16_t ndigits = int2net(buf); + boost::int16_t weight = int2net(buf+2); + boost::int16_t sign = int2net(buf+4); + boost::int16_t dscale = int2net(buf+6); + + boost::scoped_array digits(new boost::int16_t[ndigits]); + for (int n=0; n < ndigits ;++n) + { + digits[n] = int2net(buf+8+n*2); + } + + std::ostringstream ss; + + if (sign == 0x4000) ss << "-"; + + int i = std::max(weight,boost::int16_t(0)); + int d = 0; + + // Each numeric "digit" is actually a value between 0000 and 9999 stored in a 16 bit field. + // For example, the number 1234567809990001 is stored as four digits: [1234] [5678] [999] [1]. + // Note that the last two digits show that the leading 0's are lost when the number is split. + // We must be careful to re-insert these 0's when building the string. + + while ( i >= 0) + { + if (i <= weight && d < ndigits) + { + // All digits after the first must be padded to make the field 4 characters long + if (d != 0) + { +#ifdef _WINDOWS + int dig = digits[d]; + if (dig < 10) + { + ss << "000"; // 0000 - 0009 + } + else if (dig < 100) + { + ss << "00"; // 0010 - 0099 + } + else + { + ss << "0"; // 0100 - 0999; + } +#else + switch(digits[d]) + { + case 0 ... 9: + ss << "000"; // 0000 - 0009 + break; + case 10 ... 99: + ss << "00"; // 0010 - 0099 + break; + case 100 ... 999: + ss << "0"; // 0100 - 0999 + break; + } +#endif + } + ss << digits[d++]; + } + else + { + if (d == 0) + ss << "0"; + else + ss << "0000"; + } + + i--; + } + if (dscale > 0) + { + ss << '.'; + // dscale counts the number of decimal digits following the point, not the numeric digits + while (dscale > 0) + { + int value; + if (i <= weight && d < ndigits) + value = digits[d++]; + else + value = 0; + + // Output up to 4 decimal digits for this value + if (dscale > 0) { + ss << (value / 1000); + value %= 1000; + dscale--; + } + if (dscale > 0) { + ss << (value / 100); + value %= 100; + dscale--; + } + if (dscale > 0) { + ss << (value / 10); + value %= 10; + dscale--; + } + if (dscale > 0) { + ss << value; + dscale--; + } + + i--; + } + } + return ss.str(); +} diff --git a/plugins/input/pgraster/pgraster_featureset.hpp b/plugins/input/pgraster/pgraster_featureset.hpp new file mode 100644 index 000000000..b4652bf48 --- /dev/null +++ b/plugins/input/pgraster/pgraster_featureset.hpp @@ -0,0 +1,71 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#ifndef PGRASTER_FEATURESET_HPP +#define PGRASTER_FEATURESET_HPP + +// mapnik +#include +#include +#include +#include + +// boost +#include + +using mapnik::Featureset; +using mapnik::box2d; +using mapnik::feature_ptr; +using mapnik::raster_ptr; +using mapnik::transcoder; +using mapnik::context_ptr; + +class IResultSet; + +class pgraster_featureset : public mapnik::Featureset +{ +public: + /// @param bandno band number (1-based). 0 (default) reads all bands. + /// Anything else forces interpretation of colors off + /// (values copied verbatim) + pgraster_featureset(boost::shared_ptr const& rs, + context_ptr const& ctx, + std::string const& encoding, + bool key_field = false, + int bandno = 0); + feature_ptr next(); + ~pgraster_featureset(); + +private: + boost::shared_ptr rs_; + context_ptr ctx_; + boost::scoped_ptr tr_; + mapnik::value_integer feature_id_; + bool key_field_; + int band_; +}; + +#endif // PGRASTER_FEATURESET_HPP diff --git a/plugins/input/pgraster/pgraster_wkb_reader.cpp b/plugins/input/pgraster/pgraster_wkb_reader.cpp new file mode 100644 index 000000000..f61d27ba6 --- /dev/null +++ b/plugins/input/pgraster/pgraster_wkb_reader.cpp @@ -0,0 +1,497 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#include "pgraster_wkb_reader.hpp" + +// mapnik +#include // for datasource_exception +#include +#include +#include +#include +#include +#include +#include +#include // for box2d + +// boost +#include // for boost::int8_t +#include +#include + +namespace { + +uint8_t +read_uint8(const uint8_t** from) { + return *(*from)++; +} + +uint16_t +read_uint16(const boost::uint8_t** from, boost::uint8_t littleEndian) { + uint16_t ret = 0; + + if (littleEndian) { + ret = (*from)[0] | + (*from)[1] << 8; + } else { + /* big endian */ + ret = (*from)[0] << 8 | + (*from)[1]; + } + *from += 2; + return ret; +} + +int16_t +read_int16(const uint8_t** from, uint8_t littleEndian) { + assert(NULL != from); + + return read_uint16(from, littleEndian); +} + +double +read_float64(const boost::uint8_t** from, boost::uint8_t littleEndian) { + + union { + double d; + uint64_t i; + } ret; + + if (littleEndian) { + ret.i = (uint64_t) ((*from)[0] & 0xff) | + (uint64_t) ((*from)[1] & 0xff) << 8 | + (uint64_t) ((*from)[2] & 0xff) << 16 | + (uint64_t) ((*from)[3] & 0xff) << 24 | + (uint64_t) ((*from)[4] & 0xff) << 32 | + (uint64_t) ((*from)[5] & 0xff) << 40 | + (uint64_t) ((*from)[6] & 0xff) << 48 | + (uint64_t) ((*from)[7] & 0xff) << 56; + } else { + /* big endian */ + ret.i = (uint64_t) ((*from)[7] & 0xff) | + (uint64_t) ((*from)[6] & 0xff) << 8 | + (uint64_t) ((*from)[5] & 0xff) << 16 | + (uint64_t) ((*from)[4] & 0xff) << 24 | + (uint64_t) ((*from)[3] & 0xff) << 32 | + (uint64_t) ((*from)[2] & 0xff) << 40 | + (uint64_t) ((*from)[1] & 0xff) << 48 | + (uint64_t) ((*from)[0] & 0xff) << 56; + } + + *from += 8; + return ret.d; +} + +uint32_t +read_uint32(const boost::uint8_t** from, boost::uint8_t littleEndian) { + uint32_t ret = 0; + + if (littleEndian) { + ret = (uint32_t) ((*from)[0] & 0xff) | + (uint32_t) ((*from)[1] & 0xff) << 8 | + (uint32_t) ((*from)[2] & 0xff) << 16 | + (uint32_t) ((*from)[3] & 0xff) << 24; + } else { + /* big endian */ + ret = (uint32_t) ((*from)[3] & 0xff) | + (uint32_t) ((*from)[2] & 0xff) << 8 | + (uint32_t) ((*from)[1] & 0xff) << 16 | + (uint32_t) ((*from)[0] & 0xff) << 24; + } + + *from += 4; + return ret; +} + +int32_t +read_int32(const boost::uint8_t** from, boost::uint8_t littleEndian) { + + return read_uint32(from, littleEndian); +} + +float +read_float32(const uint8_t** from, uint8_t littleEndian) { + + union { + float f; + uint32_t i; + } ret; + + ret.i = read_uint32(from, littleEndian); + + return ret.f; +} + + +typedef enum { + PT_1BB=0, /* 1-bit boolean */ + PT_2BUI=1, /* 2-bit unsigned integer */ + PT_4BUI=2, /* 4-bit unsigned integer */ + PT_8BSI=3, /* 8-bit signed integer */ + PT_8BUI=4, /* 8-bit unsigned integer */ + PT_16BSI=5, /* 16-bit signed integer */ + PT_16BUI=6, /* 16-bit unsigned integer */ + PT_32BSI=7, /* 32-bit signed integer */ + PT_32BUI=8, /* 32-bit unsigned integer */ + PT_32BF=10, /* 32-bit float */ + PT_64BF=11, /* 64-bit float */ + PT_END=13 +} rt_pixtype; + +#define BANDTYPE_FLAGS_MASK 0xF0 +#define BANDTYPE_PIXTYPE_MASK 0x0F +#define BANDTYPE_FLAG_OFFDB (1<<7) +#define BANDTYPE_FLAG_HASNODATA (1<<6) +#define BANDTYPE_FLAG_ISNODATA (1<<5) +#define BANDTYPE_FLAG_RESERVED3 (1<<4) + +#define BANDTYPE_PIXTYPE_MASK 0x0F +#define BANDTYPE_PIXTYPE(x) ((x)&BANDTYPE_PIXTYPE_MASK) +#define BANDTYPE_IS_OFFDB(x) ((x)&BANDTYPE_FLAG_OFFDB) +#define BANDTYPE_HAS_NODATA(x) ((x)&BANDTYPE_FLAG_HASNODATA) +#define BANDTYPE_IS_NODATA(x) ((x)&BANDTYPE_FLAG_ISNODATA) + +} + +using mapnik::box2d; + +template +void read_data_band(mapnik::raster_ptr raster, + uint16_t width, uint16_t height, + bool hasnodata, T reader) +{ + mapnik::image_data_32 & image = raster->data_; + + // Start with plain white (ABGR or RGBA depending on endiannes) + // TODO: set to transparent instead? + image.set(0xffffffff); + + raster->premultiplied_alpha_ = true; + + float* data = (float*)image.getBytes(); + double val; + val = reader(); // nodata value, need to read anyway + if ( hasnodata ) raster->set_nodata(val); + for (int y=0; ydata_; + + // Start with all zeroes + image.set(0); + + uint8_t type = read_uint8(&ptr_); + + int pixtype = BANDTYPE_PIXTYPE(type); + int offline = BANDTYPE_IS_OFFDB(type) ? 1 : 0; + int hasnodata = BANDTYPE_HAS_NODATA(type) ? 1 : 0; + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_wkb_reader: band type:" + << pixtype << " offline:" << offline + << " hasnodata:" << hasnodata; + + if ( offline ) { + MAPNIK_LOG_WARN(pgraster) << "pgraster_wkb_reader: offline band " + " unsupported"; + return; + } + + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_wkb_reader: reading " << height_ << "x" << width_ << " pixels"; + + switch (pixtype) { + case PT_1BB: + case PT_2BUI: + case PT_4BUI: + // all <8BPP values are wrote in full bytes anyway + case PT_8BSI: + // mapnik does not support signed anyway + case PT_8BUI: + read_data_band(raster, width_, height_, hasnodata, + boost::bind(read_uint8, &ptr_)); + break; + case PT_16BSI: + // mapnik does not support signed anyway + case PT_16BUI: + read_data_band(raster, width_, height_, hasnodata, + boost::bind(read_uint16, &ptr_, endian_)); + break; + case PT_32BSI: + // mapnik does not support signed anyway + case PT_32BUI: + read_data_band(raster, width_, height_, hasnodata, + boost::bind(read_uint32, &ptr_, endian_)); + break; + case PT_32BF: + read_data_band(raster, width_, height_, hasnodata, + boost::bind(read_float32, &ptr_, endian_)); + break; + case PT_64BF: + read_data_band(raster, width_, height_, hasnodata, + boost::bind(read_float64, &ptr_, endian_)); + break; + default: + std::ostringstream err; + err << "pgraster_wkb_reader: data band type " << pixtype << " unsupported"; + // TODO: accept policy to decide on throw-or-skip ? + //MAPNIK_LOG_WARN(pgraster) << err.str(); + throw mapnik::datasource_exception(err.str()); + } + +} + +template +void read_grayscale_band(mapnik::raster_ptr raster, + uint16_t width, uint16_t height, + bool hasnodata, T reader) +{ + mapnik::image_data_32 & image = raster->data_; + + // Start with plain white (ABGR or RGBA depending on endiannes) + // TODO: set to transparent instead? + image.set(0xffffffff); + + raster->premultiplied_alpha_ = true; + + int val; + uint8_t * data = image.getBytes(); + int ps = 4; // sizeof(image_data::pixel_type) + int off; + val = reader(); // nodata value, need to read anyway + if ( hasnodata ) raster->set_nodata(val); + for (int y=0; ydata_; + + // Start with plain white (ABGR or RGBA depending on endiannes) + image.set(0xffffffff); + //raster->set_nodata(0xffffffff); + + raster->premultiplied_alpha_ = true; + + uint8_t nodataval; + for (int bn=0; bn PT_8BUI || pixtype < PT_8BSI ) { + MAPNIK_LOG_WARN(pgraster) << "pgraster_wkb_reader: band " << bn + << " type " << type << " unsupported"; + continue; + } + + uint8_t tmp = read_uint8(&ptr_); + if ( ! bn ) nodataval = tmp; + else if ( tmp != nodataval ) { + MAPNIK_LOG_WARN(pgraster) << "pgraster_wkb_reader: band " << bn + << " nodataval " << tmp << " != band 0 nodataval " << nodataval; + } + + int ps = 4; // sizeof(image_data::pixel_type) + uint8_t * image_data = image.getBytes(); + for (int y=0; y ext(ipX,ipY,ipX+(width_*scaleX),ipY+(height_*scaleY)); + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_wkb_reader: Raster extent=" << ext; + + mapnik::raster_ptr raster = boost::make_shared(ext, width_, height_); + + if ( bandno_ ) { + if ( bandno_ != 1 ) { + MAPNIK_LOG_WARN(pgraster) << "pgraster_wkb_reader: " + "reading bands other than 1st as indexed is unsupported"; + return mapnik::raster_ptr(); + } + MAPNIK_LOG_DEBUG(pgraster) << "pgraster_wkb_reader: requested band " << bandno_; + read_indexed(raster); + } + else { + switch (numBands_) { + case 1: + read_grayscale(raster); + break; + case 3: + case 4: + read_rgba(raster); + break; + default: + std::ostringstream err; + err << "pgraster_wkb_reader: raster with " + << numBands_ + << " bands is not supported, specify a band number"; + //MAPNIK_LOG_WARN(pgraster) << err.str(); + throw mapnik::datasource_exception(err.str()); + return mapnik::raster_ptr(); + } + } + + return raster; + +} + + diff --git a/plugins/input/pgraster/pgraster_wkb_reader.hpp b/plugins/input/pgraster/pgraster_wkb_reader.hpp new file mode 100644 index 000000000..3a571ffd5 --- /dev/null +++ b/plugins/input/pgraster/pgraster_wkb_reader.hpp @@ -0,0 +1,87 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2014 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + ***************************************************************************** + * + * Initially developed by Sandro Santilli + * + *****************************************************************************/ + +#ifndef PGRASTER_WKB_READER_HPP +#define PGRASTER_WKB_READER_HPP + +// mapnik +#include // for raster_ptr + +// boost +#include // for boost::uint8_t + +enum pgraster_color_interp { + // Automatic color interpretation: + // uses grayscale for single band, rgb for 3 bands + // rgba for 4 bands + pgr_auto, + // Grayscale interpretation + pgr_grayscale, + pgr_indexed, + pgr_rgb, + pgr_rgba +}; + +class pgraster_wkb_reader +{ +public: + + pgraster_wkb_reader(const uint8_t* wkb, int size, int bnd=0) + : wkbsize_(size), wkb_(wkb), wkbend_(wkb+size), ptr_(wkb), bandno_(bnd) + {} + + mapnik::raster_ptr get_raster(); + + /// @param bnd band number. If 0 (default) it'll try to read all bands + /// with automatic color interpretation (rgb for 3 bands, + /// rgba for 4 bands, grayscale for 1 band). + /// Any other value results in pixel + /// values being copied verbatim into the returned raster + /// for interpretation by the caller. + static mapnik::raster_ptr read(const uint8_t* wkb, int size, int bnd=0) + { + pgraster_wkb_reader reader(wkb,size,bnd); + return reader.get_raster(); + } + +private: + void read_indexed(mapnik::raster_ptr raster); + void read_grayscale(mapnik::raster_ptr raster); + void read_rgba(mapnik::raster_ptr raster); + + int wkbsize_; + const uint8_t* wkb_; + const uint8_t* wkbend_; + const uint8_t* ptr_; + uint8_t endian_; + int bandno_; + uint16_t numBands_; + uint16_t width_; + uint16_t height_; +}; + + +#endif // PGRASTER_WKB_READER_HPP diff --git a/tests/python_tests/pgraster_test.py b/tests/python_tests/pgraster_test.py new file mode 100644 index 000000000..f958b55b4 --- /dev/null +++ b/tests/python_tests/pgraster_test.py @@ -0,0 +1,747 @@ +#!/usr/bin/env python + +from nose.tools import * +import atexit +import cProfile, pstats, io +import time +from utilities import execution_path, run_all +from subprocess import Popen, PIPE +import os, mapnik +from Queue import Queue +import threading +import sys +import re +from binascii import hexlify, unhexlify + + +MAPNIK_TEST_DBNAME = 'mapnik-tmp-pgraster-test-db' +POSTGIS_TEMPLATE_DBNAME = 'template_postgis' + +def setup(): + # All of the paths used are relative, if we run the tests + # from another directory we need to chdir() + os.chdir(execution_path('.')) + +def call(cmd,silent=False): + proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + stdout, stderr = proc.communicate() + if stderr and not silent: + print stderr.strip() + if proc.returncode: + raise RuntimeError(stderr.strip()) + +def psql_can_connect(): + """Test ability to connect to a postgis template db with no options. + + Basically, to run these tests your user must have full read + access over unix sockets without supplying a password. This + keeps these tests simple and focused on postgis not on postgres + auth issues. + """ + try: + call('psql %s -c "select postgis_version()"' % POSTGIS_TEMPLATE_DBNAME) + return True + except RuntimeError, e: + print 'Notice: skipping postgis tests (connection)' + return False + +def psql_run(cmd): + cmd = 'psql --set ON_ERROR_STOP=1 %s -c "%s"' % \ + (MAPNIK_TEST_DBNAME, cmd.replace('"', '\\"')) + print 'DEBUG: running ' + cmd + call(cmd) + +def raster2pgsql_on_path(): + """Test for presence of raster2pgsql on the user path. + + We require this program to load test data into a temporarily database. + """ + try: + call('raster2pgsql') + return True + except RuntimeError, e: + print 'Notice: skipping postgis tests (raster2pgsql)' + return False + +def createdb_and_dropdb_on_path(): + """Test for presence of dropdb/createdb on user path. + + We require these programs to setup and teardown the testing db. + """ + try: + call('createdb --help') + call('dropdb --help') + return True + except RuntimeError, e: + print 'Notice: skipping postgis tests (createdb/dropdb)' + return False + +def postgis_setup(): + call('dropdb %s' % MAPNIK_TEST_DBNAME,silent=True) + call('createdb -T %s %s' % (POSTGIS_TEMPLATE_DBNAME,MAPNIK_TEST_DBNAME),silent=False) + +def postgis_takedown(): + pass + # fails as the db is in use: https://github.com/mapnik/mapnik/issues/960 + #call('dropdb %s' % MAPNIK_TEST_DBNAME) + +def import_raster(filename, tabname, tilesize, constraint, overview): + print 'tile: ' + tilesize + ' constraints: ' + str(constraint) \ + + ' overviews: ' + overview + cmd = 'raster2pgsql -Y -I -q' + if constraint: + cmd += ' -C' + if tilesize: + cmd += ' -t ' + tilesize + if overview: + cmd += ' -l ' + overview + cmd += ' %s %s | psql --set ON_ERROR_STOP=1 -q %s' % (filename,tabname,MAPNIK_TEST_DBNAME) + print 'Import call: ' + cmd + call(cmd) + +def drop_imported(tabname, overview): + psql_run('DROP TABLE IF EXISTS "' + tabname + '";') + if overview: + for of in overview.split(','): + psql_run('DROP TABLE IF EXISTS "o_' + of + '_' + tabname + '";') + +if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ + and createdb_and_dropdb_on_path() \ + and psql_can_connect() \ + and raster2pgsql_on_path(): + + # initialize test database + postgis_setup() + + # dataraster.tif, 2283x1913 int16 single-band + def _test_dataraster_16bsi_rendering(lbl, overview, rescale, clip): + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME,table='"dataRaster"', + band=1,use_overviews=1 if overview else 0, + prescale_rasters=rescale,clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['rid'],1) + lyr = mapnik.Layer('dataraster_16bsi') + lyr.datasource = ds + expenv = mapnik.Box2d(-14637, 3903178, 1126863, 4859678) + env = lyr.envelope() + # As the input size is a prime number both horizontally + # and vertically, we expect the extent of the overview + # tables to be a pixel wider than the original, whereas + # the pixel size in geographical units depends on the + # overview factor. So we start with the original pixel size + # as base scale and multiply by the overview factor. + # NOTE: the overview table extent only grows north and east + pixsize = 500 # see gdalinfo dataraster.tif + tol = pixsize * max(overview.split(',')) if overview else 0 + assert_almost_equal(env.minx, expenv.minx) + assert_almost_equal(env.miny, expenv.miny, delta=tol) + assert_almost_equal(env.maxx, expenv.maxx, delta=tol) + assert_almost_equal(env.maxy, expenv.maxy) + mm = mapnik.Map(256, 256) + style = mapnik.Style() + col = mapnik.RasterColorizer(); + col.default_mode = mapnik.COLORIZER_DISCRETE; + col.add_stop(0, mapnik.Color(0x40,0x40,0x40,255)); + col.add_stop(10, mapnik.Color(0x80,0x80,0x80,255)); + col.add_stop(20, mapnik.Color(0xa0,0xa0,0xa0,255)); + sym = mapnik.RasterSymbolizer() + sym.colorizer = col + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + # no data + eq_(im.view(1,1,1,1).tostring(), '\x00\x00\x00\x00') + eq_(im.view(255,255,1,1).tostring(), '\x00\x00\x00\x00') + eq_(im.view(195,116,1,1).tostring(), '\x00\x00\x00\x00') + # A0A0A0 + eq_(im.view(100,120,1,1).tostring(), '\xa0\xa0\xa0\xff') + eq_(im.view( 75, 80,1,1).tostring(), '\xa0\xa0\xa0\xff') + # 808080 + eq_(im.view( 74,170,1,1).tostring(), '\x80\x80\x80\xff') + eq_(im.view( 30, 50,1,1).tostring(), '\x80\x80\x80\xff') + # 404040 + eq_(im.view(190, 70,1,1).tostring(), '\x40\x40\x40\xff') + eq_(im.view(140,170,1,1).tostring(), '\x40\x40\x40\xff') + + # Now zoom over a portion of the env (1/10) + newenv = mapnik.Box2d(273663,4024478,330738,4072303) + mm.zoom_to_box(newenv) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:1/10' + # nodata + eq_(hexlify(im.view(255,255,1,1).tostring()), '00000000') + eq_(hexlify(im.view(200,254,1,1).tostring()), '00000000') + # A0A0A0 + eq_(hexlify(im.view(90,232,1,1).tostring()), 'a0a0a0ff') + eq_(hexlify(im.view(96,245,1,1).tostring()), 'a0a0a0ff') + # 808080 + eq_(hexlify(im.view(1,1,1,1).tostring()), '808080ff') + eq_(hexlify(im.view(128,128,1,1).tostring()), '808080ff') + # 404040 + eq_(hexlify(im.view(255, 0,1,1).tostring()), '404040ff') + + def _test_dataraster_16bsi(lbl, tilesize, constraint, overview): + rf = os.path.join(execution_path('.'),'../data/raster/dataraster.tif') + import_raster(rf, 'dataRaster', tilesize, constraint, overview) + if constraint: + lbl += ' C' + if tilesize: + lbl += ' T:' + tilesize + if overview: + lbl += ' O:' + overview + for prescale in [0,1]: + for clip in [0,1]: + _test_dataraster_16bsi_rendering(lbl, overview, prescale, clip) + drop_imported('dataRaster', overview) + + def test_dataraster_16bsi(): + for tilesize in ['','256x256']: + for constraint in [0,1]: + for overview in ['','4','2,16']: + _test_dataraster_16bsi('data_16bsi', tilesize, constraint, overview) + + # river.tiff, RGBA 8BUI + def _test_rgba_8bui_rendering(lbl, overview, rescale, clip): + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME,table='(select * from "River") foo', + use_overviews=1 if overview else 0, + prescale_rasters=rescale,clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['rid'],1) + lyr = mapnik.Layer('rgba_8bui') + lyr.datasource = ds + expenv = mapnik.Box2d(0, -210, 256, 0) + env = lyr.envelope() + # As the input size is a prime number both horizontally + # and vertically, we expect the extent of the overview + # tables to be a pixel wider than the original, whereas + # the pixel size in geographical units depends on the + # overview factor. So we start with the original pixel size + # as base scale and multiply by the overview factor. + # NOTE: the overview table extent only grows north and east + pixsize = 1 # see gdalinfo river.tif + tol = pixsize * max(overview.split(',')) if overview else 0 + assert_almost_equal(env.minx, expenv.minx) + assert_almost_equal(env.miny, expenv.miny, delta=tol) + assert_almost_equal(env.maxx, expenv.maxx, delta=tol) + assert_almost_equal(env.maxy, expenv.maxy) + mm = mapnik.Map(256, 256) + style = mapnik.Style() + sym = mapnik.RasterSymbolizer() + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + #im.save('/tmp/xfull.png') # for debugging + # no data + eq_(hexlify(im.view(3,3,1,1).tostring()), '00000000') + eq_(hexlify(im.view(250,250,1,1).tostring()), '00000000') + # full opaque river color + eq_(hexlify(im.view(175,118,1,1).tostring()), 'b9d8f8ff') + # half-transparent pixel + pxstr = hexlify(im.view(122,138,1,1).tostring()) + apat = ".*(..)$" + match = re.match(apat, pxstr) + assert match, 'pixel ' + pxstr + ' does not match pattern ' + apat + alpha = match.group(1) + assert alpha != 'ff' and alpha != '00', \ + 'unexpected full transparent/opaque pixel: ' + alpha + + # Now zoom over a portion of the env (1/10) + newenv = mapnik.Box2d(166,-105,191,-77) + mm.zoom_to_box(newenv) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:1/10' + #im.save('/tmp/xtenth.png') # for debugging + # no data + eq_(hexlify(im.view(255,255,1,1).tostring()), '00000000') + eq_(hexlify(im.view(200,40,1,1).tostring()), '00000000') + # full opaque river color + eq_(hexlify(im.view(100,168,1,1).tostring()), 'b9d8f8ff') + # half-transparent pixel + pxstr = hexlify(im.view(122,138,1,1).tostring()) + apat = ".*(..)$" + match = re.match(apat, pxstr) + assert match, 'pixel ' + pxstr + ' does not match pattern ' + apat + alpha = match.group(1) + assert alpha != 'ff' and alpha != '00', \ + 'unexpected full transparent/opaque pixel: ' + alpha + + def _test_rgba_8bui(lbl, tilesize, constraint, overview): + rf = os.path.join(execution_path('.'),'../data/raster/river.tiff') + import_raster(rf, 'River', tilesize, constraint, overview) + if constraint: + lbl += ' C' + if tilesize: + lbl += ' T:' + tilesize + if overview: + lbl += ' O:' + overview + for prescale in [0,1]: + for clip in [0,1]: + _test_rgba_8bui_rendering(lbl, overview, prescale, clip) + drop_imported('River', overview) + + def test_rgba_8bui(): + for tilesize in ['','16x16']: + for constraint in [0,1]: + for overview in ['2']: + _test_rgba_8bui('rgba_8bui', tilesize, constraint, overview) + + # nodata-edge.tif, RGB 8BUI + def _test_rgb_8bui_rendering(lbl, tnam, overview, rescale, clip): + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME,table=tnam, + use_overviews=1 if overview else 0, + prescale_rasters=rescale,clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['rid'],1) + lyr = mapnik.Layer('rgba_8bui') + lyr.datasource = ds + expenv = mapnik.Box2d(-12329035.7652168,4508650.39854396, \ + -12328653.0279471,4508957.34625536) + env = lyr.envelope() + # As the input size is a prime number both horizontally + # and vertically, we expect the extent of the overview + # tables to be a pixel wider than the original, whereas + # the pixel size in geographical units depends on the + # overview factor. So we start with the original pixel size + # as base scale and multiply by the overview factor. + # NOTE: the overview table extent only grows north and east + pixsize = 2 # see gdalinfo nodata-edge.tif + tol = pixsize * max(overview.split(',')) if overview else 0 + assert_almost_equal(env.minx, expenv.minx, places=0) + assert_almost_equal(env.miny, expenv.miny, delta=tol) + assert_almost_equal(env.maxx, expenv.maxx, delta=tol) + assert_almost_equal(env.maxy, expenv.maxy, places=0) + mm = mapnik.Map(256, 256) + style = mapnik.Style() + sym = mapnik.RasterSymbolizer() + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + #im.save('/tmp/xfull.png') # for debugging + # no data + eq_(hexlify(im.view(3,16,1,1).tostring()), '00000000') + eq_(hexlify(im.view(128,16,1,1).tostring()), '00000000') + eq_(hexlify(im.view(250,16,1,1).tostring()), '00000000') + eq_(hexlify(im.view(3,240,1,1).tostring()), '00000000') + eq_(hexlify(im.view(128,240,1,1).tostring()), '00000000') + eq_(hexlify(im.view(250,240,1,1).tostring()), '00000000') + # dark brown + eq_(hexlify(im.view(174,39,1,1).tostring()), 'c3a698ff') + # dark gray + eq_(hexlify(im.view(195,132,1,1).tostring()), '575f62ff') + # Now zoom over a portion of the env (1/10) + newenv = mapnik.Box2d(-12329035.7652168, 4508926.651484220, \ + -12328997.49148983,4508957.34625536) + mm.zoom_to_box(newenv) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:1/10' + #im.save('/tmp/xtenth.png') # for debugging + # no data + eq_(hexlify(im.view(3,16,1,1).tostring()), '00000000') + eq_(hexlify(im.view(128,16,1,1).tostring()), '00000000') + eq_(hexlify(im.view(250,16,1,1).tostring()), '00000000') + # black + eq_(hexlify(im.view(3,42,1,1).tostring()), '000000ff') + eq_(hexlify(im.view(3,134,1,1).tostring()), '000000ff') + eq_(hexlify(im.view(3,244,1,1).tostring()), '000000ff') + # gray + eq_(hexlify(im.view(135,157,1,1).tostring()), '4e555bff') + # brown + eq_(hexlify(im.view(195,223,1,1).tostring()), 'f2cdbaff') + + def _test_rgb_8bui(lbl, tilesize, constraint, overview): + rf = os.path.join(execution_path('.'),'../data/raster/nodata-edge.tif') + tnam = 'nodataedge' + import_raster(rf, tnam, tilesize, constraint, overview) + if constraint: + lbl += ' C' + if tilesize: + lbl += ' T:' + tilesize + if overview: + lbl += ' O:' + overview + for prescale in [0,1]: + for clip in [0,1]: + _test_rgb_8bui_rendering(lbl, tnam, overview, prescale, clip) + #drop_imported(tnam, overview) + + def test_rgb_8bui(): + for tilesize in ['64x64']: + for constraint in [1]: + for overview in ['']: + _test_rgb_8bui('rgb_8bui', tilesize, constraint, overview) + + def _test_grayscale_subquery(lbl,pixtype,value): + # + # 3 8 13 + # +---+---+---+ + # 3 | v | v | v | NOTE: writes different color + # +---+---+---+ in 13,8 and 8,13 + # 8 | v | v | a | + # +---+---+---+ + # 13 | v | b | v | + # +---+---+---+ + # + val_a = value/3; + val_b = val_a*2; + sql = "(select 3 as i, " \ + " ST_SetValues(" \ + " ST_SetValues(" \ + " ST_AsRaster(" \ + " ST_MakeEnvelope(0,0,14,14), " \ + " 1.0, -1.0, '%s', %s" \ + " ), " \ + " 11, 6, 4, 5, %s::float8" \ + " )," \ + " 6, 11, 5, 4, %s::float8" \ + " ) as \"R\"" \ + ") as foo" % (pixtype,value, val_a, val_b) + rescale = 0 + clip = 0 + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME, table=sql, + raster_field='"R"', use_overviews=1, + prescale_rasters=rescale,clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['i'],3) + lyr = mapnik.Layer('grayscale_subquery') + lyr.datasource = ds + expenv = mapnik.Box2d(0,0,14,14) + env = lyr.envelope() + assert_almost_equal(env.minx, expenv.minx, places=0) + assert_almost_equal(env.miny, expenv.miny, places=0) + assert_almost_equal(env.maxx, expenv.maxx, places=0) + assert_almost_equal(env.maxy, expenv.maxy, places=0) + mm = mapnik.Map(15, 15) + style = mapnik.Style() + sym = mapnik.RasterSymbolizer() + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + #im.save('/tmp/xfull.png') # for debugging + h = format(value, '02x') + hex_v = h+h+h+'ff' + h = format(val_a, '02x') + hex_a = h+h+h+'ff' + h = format(val_b, '02x') + hex_b = h+h+h+'ff' + eq_(hexlify(im.view( 3, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 3, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 8,1,1).tostring()), hex_a); + eq_(hexlify(im.view( 3,13,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8,13,1,1).tostring()), hex_b); + eq_(hexlify(im.view(13,13,1,1).tostring()), hex_v); + + def test_grayscale_2bui_subquery(): + _test_grayscale_subquery('grayscale_2bui_subquery', '2BUI', 3) + + def test_grayscale_4bui_subquery(): + _test_grayscale_subquery('grayscale_4bui_subquery', '4BUI', 15) + + def test_grayscale_8bui_subquery(): + _test_grayscale_subquery('grayscale_8bui_subquery', '8BUI', 63) + + def test_grayscale_8bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_grayscale_subquery('grayscale_8bsi_subquery', '8BSI', 69) + + def test_grayscale_16bui_subquery(): + _test_grayscale_subquery('grayscale_16bui_subquery', '16BUI', 126) + + def test_grayscale_16bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_grayscale_subquery('grayscale_16bsi_subquery', '16BSI', 144) + + def test_grayscale_32bui_subquery(): + _test_grayscale_subquery('grayscale_32bui_subquery', '32BUI', 255) + + def test_grayscale_32bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_grayscale_subquery('grayscale_32bsi_subquery', '32BSI', 129) + + def _test_data_subquery(lbl, pixtype, value): + # + # 3 8 13 + # +---+---+---+ + # 3 | v | v | v | NOTE: writes different values + # +---+---+---+ in 13,8 and 8,13 + # 8 | v | v | a | + # +---+---+---+ + # 13 | v | b | v | + # +---+---+---+ + # + val_a = value/3; + val_b = val_a*2; + sql = "(select 3 as i, " \ + " ST_SetValues(" \ + " ST_SetValues(" \ + " ST_AsRaster(" \ + " ST_MakeEnvelope(0,0,14,14), " \ + " 1.0, -1.0, '%s', %s" \ + " ), " \ + " 11, 6, 5, 5, %s::float8" \ + " )," \ + " 6, 11, 5, 5, %s::float8" \ + " ) as \"R\"" \ + ") as foo" % (pixtype,value, val_a, val_b) + overview = '' + rescale = 0 + clip = 0 + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME, table=sql, + raster_field='R', use_overviews=0 if overview else 0, + band=1, prescale_rasters=rescale, clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['i'],3) + lyr = mapnik.Layer('data_subquery') + lyr.datasource = ds + expenv = mapnik.Box2d(0,0,14,14) + env = lyr.envelope() + assert_almost_equal(env.minx, expenv.minx, places=0) + assert_almost_equal(env.miny, expenv.miny, places=0) + assert_almost_equal(env.maxx, expenv.maxx, places=0) + assert_almost_equal(env.maxy, expenv.maxy, places=0) + mm = mapnik.Map(15, 15) + style = mapnik.Style() + col = mapnik.RasterColorizer(); + col.default_mode = mapnik.COLORIZER_DISCRETE; + col.add_stop(val_a, mapnik.Color(0xff,0x00,0x00,255)); + col.add_stop(val_b, mapnik.Color(0x00,0xff,0x00,255)); + col.add_stop(value, mapnik.Color(0x00,0x00,0xff,255)); + sym = mapnik.RasterSymbolizer() + sym.colorizer = col + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + #im.save('/tmp/xfull.png') # for debugging + h = format(value, '02x') + hex_v = '0000ffff' + hex_a = 'ff0000ff' + hex_b = '00ff00ff' + eq_(hexlify(im.view( 3, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 3, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 8,1,1).tostring()), hex_a); + eq_(hexlify(im.view( 3,13,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8,13,1,1).tostring()), hex_b); + eq_(hexlify(im.view(13,13,1,1).tostring()), hex_v); + + def test_data_2bui_subquery(): + _test_data_subquery('data_2bui_subquery', '2BUI', 3) + + def test_data_4bui_subquery(): + _test_data_subquery('data_4bui_subquery', '4BUI', 15) + + def test_data_8bui_subquery(): + _test_data_subquery('data_8bui_subquery', '8BUI', 63) + + def test_data_8bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_data_subquery('data_8bsi_subquery', '8BSI', 69) + + def test_data_16bui_subquery(): + _test_data_subquery('data_16bui_subquery', '16BUI', 126) + + def test_data_16bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_data_subquery('data_16bsi_subquery', '16BSI', 135) + + def test_data_32bui_subquery(): + _test_data_subquery('data_32bui_subquery', '32BUI', 255) + + def test_data_32bsi_subquery(): + # NOTE: we're using a positive integer because Mapnik + # does not support negative data values anyway + _test_data_subquery('data_32bsi_subquery', '32BSI', 264) + + def test_data_32bf_subquery(): + _test_data_subquery('data_32bf_subquery', '32BF', 450) + + def test_data_64bf_subquery(): + _test_data_subquery('data_64bf_subquery', '64BF', 3072) + + def _test_rgba_subquery(lbl, pixtype, r, g, b, a, g1, b1): + # + # 3 8 13 + # +---+---+---+ + # 3 | v | v | h | NOTE: writes different alpha + # +---+---+---+ in 13,8 and 8,13 + # 8 | v | v | a | + # +---+---+---+ + # 13 | v | b | v | + # +---+---+---+ + # + sql = "(select 3 as i, " \ + " ST_SetValues(" \ + " ST_SetValues(" \ + " ST_AddBand(" \ + " ST_AddBand(" \ + " ST_AddBand(" \ + " ST_AsRaster(" \ + " ST_MakeEnvelope(0,0,14,14), " \ + " 1.0, -1.0, '%s', %s" \ + " )," \ + " '%s', %d::float" \ + " ), " \ + " '%s', %d::float" \ + " ), " \ + " '%s', %d::float" \ + " ), " \ + " 2, 11, 6, 4, 5, %s::float8" \ + " )," \ + " 3, 6, 11, 5, 4, %s::float8" \ + " ) as r" \ + ") as foo" % (pixtype, r, pixtype, g, pixtype, b, pixtype, a, g1, b1) + overview = '' + rescale = 0 + clip = 0 + if rescale: + lbl += ' Sc' + if clip: + lbl += ' Cl' + ds = mapnik.PgRaster(dbname=MAPNIK_TEST_DBNAME, table=sql, + raster_field='r', use_overviews=0 if overview else 0, + prescale_rasters=rescale, clip_rasters=clip) + fs = ds.featureset() + feature = fs.next() + eq_(feature['i'],3) + lyr = mapnik.Layer('rgba_subquery') + lyr.datasource = ds + expenv = mapnik.Box2d(0,0,14,14) + env = lyr.envelope() + assert_almost_equal(env.minx, expenv.minx, places=0) + assert_almost_equal(env.miny, expenv.miny, places=0) + assert_almost_equal(env.maxx, expenv.maxx, places=0) + assert_almost_equal(env.maxy, expenv.maxy, places=0) + mm = mapnik.Map(15, 15) + style = mapnik.Style() + sym = mapnik.RasterSymbolizer() + rule = mapnik.Rule() + rule.symbols.append(sym) + style.rules.append(rule) + mm.append_style('foo', style) + lyr.styles.append('foo') + mm.layers.append(lyr) + mm.zoom_to_box(expenv) + im = mapnik.Image(mm.width, mm.height) + t0 = time.time() # we want wall time to include IO waits + mapnik.render(mm, im) + lap = time.time() - t0 + print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + im.save('/tmp/xfull.png') # for debugging + hex_v = format(r << 24 | g << 16 | b << 8 | a, '08x') + hex_a = format(r << 24 | g1 << 16 | b << 8 | a, '08x') + hex_b = format(r << 24 | g << 16 | b1 << 8 | a, '08x') + eq_(hexlify(im.view( 3, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 3,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 3, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8, 8,1,1).tostring()), hex_v); + eq_(hexlify(im.view(13, 8,1,1).tostring()), hex_a); + eq_(hexlify(im.view( 3,13,1,1).tostring()), hex_v); + eq_(hexlify(im.view( 8,13,1,1).tostring()), hex_b); + eq_(hexlify(im.view(13,13,1,1).tostring()), hex_v); + + def test_rgba_8bui_subquery(): + _test_rgba_subquery('rgba_8bui_subquery', '8BUI', 255, 0, 0, 255, 255, 255) + + #def test_rgba_16bui_subquery(): + # _test_rgba_subquery('rgba_16bui_subquery', '16BUI', 65535, 0, 0, 65535, 65535, 65535) + + #def test_rgba_32bui_subquery(): + # _test_rgba_subquery('rgba_32bui_subquery', '32BUI') + + atexit.register(postgis_takedown) + +def enabled(tname): + enabled = len(sys.argv) < 2 or tname in sys.argv + if not enabled: + print "Skipping " + tname + " as not explicitly enabled" + return enabled + +if __name__ == "__main__": + setup() + fail = run_all(eval(x) for x in dir() if x.startswith("test_") and enabled(x)) + exit(fail) From e03448ecb9e680bb20289833488b9ed1594004ad Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 12 Aug 2014 15:14:25 -0700 Subject: [PATCH 07/10] port pg_raster plugin to c++11/recent 3.x changes --- plugins/input/pgraster/README | 2 + .../input/pgraster/pgraster_datasource.cpp | 84 ++++++++----------- .../input/pgraster/pgraster_datasource.hpp | 11 +-- .../input/pgraster/pgraster_featureset.cpp | 10 +-- .../input/pgraster/pgraster_featureset.hpp | 16 ++-- .../input/pgraster/pgraster_wkb_reader.cpp | 12 +-- .../input/pgraster/pgraster_wkb_reader.hpp | 7 -- 7 files changed, 51 insertions(+), 91 deletions(-) diff --git a/plugins/input/pgraster/README b/plugins/input/pgraster/README index baf2814a9..fcc0e63b1 100644 --- a/plugins/input/pgraster/README +++ b/plugins/input/pgraster/README @@ -1,3 +1,5 @@ +Initially developed by Sandro Santilli + This plugin shares directives with the "postgis" one, with the following differences: diff --git a/plugins/input/pgraster/pgraster_datasource.cpp b/plugins/input/pgraster/pgraster_datasource.cpp index 9d15b7ef1..3c4fe2729 100644 --- a/plugins/input/pgraster/pgraster_datasource.cpp +++ b/plugins/input/pgraster/pgraster_datasource.cpp @@ -18,10 +18,6 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * - ***************************************************************************** - * - * Initially developed by Sandro Santilli - * *****************************************************************************/ #include "connection_manager.hpp" @@ -29,7 +25,6 @@ #include "pgraster_featureset.hpp" #include "asyncresultset.hpp" - // mapnik #include #include // for byte @@ -58,14 +53,11 @@ const std::string pgraster_datasource::RASTER_COLUMNS = "raster_columns"; const std::string pgraster_datasource::RASTER_OVERVIEWS = "raster_overviews"; const std::string pgraster_datasource::SPATIAL_REF_SYS = "spatial_ref_system"; -using boost::shared_ptr; +using std::shared_ptr; using mapnik::attribute_descriptor; +using mapnik::value_integer; namespace { - // TODO: move to sql_utils - std::string quote_literal(std::string& s) { - return "'" + s + "'"; // TODO: escape internal quotes - } // TODO: move to sql_utils std::string quote_ident(std::string& s) { @@ -82,14 +74,14 @@ pgraster_datasource::pgraster_datasource(parameters const& params) raster_field_(*params.get("raster_field", "")), key_field_(*params.get("key_field", "")), cursor_fetch_size_(*params.get("cursor_size", 0)), - row_limit_(*params.get("row_limit", 0)), + row_limit_(*params.get("row_limit", 0)), type_(datasource::Raster), - srid_(*params.get("srid", 0)), - band_(*params.get("band", 0)), + srid_(*params.get("srid", 0)), + band_(*params.get("band", 0)), extent_initialized_(false), - prescale_rasters_(*params.get("prescale_rasters", false)), - use_overviews_(*params.get("use_overviews", false)), - clip_rasters_(*params.get("clip_rasters", false)), + prescale_rasters_(*params.get("prescale_rasters", false)), + use_overviews_(*params.get("use_overviews", false)), + clip_rasters_(*params.get("clip_rasters", false)), desc_(*params.get("type"), "utf-8"), creator_(params.get("host"), params.get("port"), @@ -101,15 +93,15 @@ pgraster_datasource::pgraster_datasource(parameters const& params) scale_denom_token_("!scale_denominator!"), pixel_width_token_("!pixel_width!"), pixel_height_token_("!pixel_height!"), - pool_max_size_(*params_.get("max_size", 10)), - persist_connection_(*params.get("persist_connection", true)), - extent_from_subquery_(*params.get("extent_from_subquery", false)), - estimate_extent_(*params.get("estimate_extent", false)), - max_async_connections_(*params_.get("max_async_connection", 1)), + pool_max_size_(*params_.get("max_size", 10)), + persist_connection_(*params.get("persist_connection", true)), + extent_from_subquery_(*params.get("extent_from_subquery", false)), + estimate_extent_(*params.get("estimate_extent", false)), + max_async_connections_(*params_.get("max_async_connection", 1)), asynchronous_request_(false), // params below are for testing purposes only and may be removed at any time - intersect_min_scale_(*params.get("intersect_min_scale", 0)), - intersect_max_scale_(*params.get("intersect_max_scale", 0)) + intersect_min_scale_(*params.get("intersect_min_scale", 0)), + intersect_max_scale_(*params.get("intersect_max_scale", 0)) { #ifdef MAPNIK_STATS mapnik::progress_timer __stats__(std::clog, "pgraster_datasource::init"); @@ -139,8 +131,8 @@ pgraster_datasource::pgraster_datasource(parameters const& params) asynchronous_request_ = true; } - boost::optional initial_size = params.get("initial_size", 1); - boost::optional autodetect_key_field = params.get("autodetect_key_field", false); + boost::optional initial_size = params.get("initial_size", 1); + boost::optional autodetect_key_field = params.get("autodetect_key_field", false); ConnectionManager::instance().registerPool(creator_, *initial_size, pool_max_size_); CnxPool_ptr pool = ConnectionManager::instance().getPool(creator_.id()); @@ -358,7 +350,7 @@ pgraster_datasource::pgraster_datasource(parameters const& params) { overviews_.resize(overviews_.size()+1); pgraster_overview& ov = overviews_.back(); - ov.schema = rs->getValue("sch"); + ov.schema = rs->getValue("sch"); ov.table = rs->getValue("tab"); ov.column = rs->getValue("col"); ov.scale = atof(rs->getValue("scl")); @@ -721,7 +713,7 @@ std::string pgraster_datasource::populate_tokens(std::string const& sql, double } -boost::shared_ptr pgraster_datasource::get_resultset(boost::shared_ptr &conn, std::string const& sql, CnxPool_ptr const& pool, processor_context_ptr ctx) const +std::shared_ptr pgraster_datasource::get_resultset(std::shared_ptr &conn, std::string const& sql, CnxPool_ptr const& pool, processor_context_ptr ctx) const { if (!ctx) @@ -741,7 +733,7 @@ boost::shared_ptr pgraster_datasource::get_resultset(boost::shared_p throw mapnik::datasource_exception("Pgraster Plugin: error creating cursor for data select." ); } - return boost::make_shared(conn, cursor_name, cursor_fetch_size_); + return std::make_shared(conn, cursor_name, cursor_fetch_size_); } else @@ -753,17 +745,17 @@ boost::shared_ptr pgraster_datasource::get_resultset(boost::shared_p else { // asynchronous requests - boost::shared_ptr pgis_ctxt = boost::static_pointer_cast(ctx); + std::shared_ptr pgis_ctxt = std::static_pointer_cast(ctx); if (conn) { // lauch async req & create asyncresult with conn conn->executeAsyncQuery(sql, 1); - return boost::make_shared(pgis_ctxt, pool, conn, sql); + return std::make_shared(pgis_ctxt, pool, conn, sql); } else { // create asyncresult with null connection - boost::shared_ptr res = boost::make_shared(pgis_ctxt, pool, conn, sql); + std::shared_ptr res = std::make_shared(pgis_ctxt, pool, conn, sql); pgis_ctxt->add_request(res); return res; } @@ -785,7 +777,7 @@ processor_context_ptr pgraster_datasource::get_context(feature_style_context_map } else { - return ctx.insert(std::make_pair(ds_name,boost::make_shared())).first->second; + return ctx.insert(std::make_pair(ds_name,std::make_shared())).first->second; } } @@ -794,7 +786,7 @@ featureset_ptr pgraster_datasource::features(query const& q) const // if the driver is in asynchronous mode, return the appropriate fetaures if (asynchronous_request_ ) { - return features_with_context(q,boost::make_shared()); + return features_with_context(q,std::make_shared()); } else { @@ -822,7 +814,7 @@ featureset_ptr pgraster_datasource::features_with_context(query const& q,process if ( asynchronous_request_ ) { // limit use to num_async_request_ => if reached don't borrow the last connexion object - boost::shared_ptr pgis_ctxt = boost::static_pointer_cast(proc_ctx); + std::shared_ptr pgis_ctxt = std::static_pointer_cast(proc_ctx); if ( pgis_ctxt->num_async_requests_ < max_async_connections_ ) { conn = pool->borrowObject(); @@ -862,8 +854,8 @@ featureset_ptr pgraster_datasource::features_with_context(query const& q,process throw mapnik::datasource_exception(s_error.str()); } - const double px_gw = 1.0 / boost::get<0>(q.resolution()); - const double px_gh = 1.0 / boost::get<1>(q.resolution()); + const double px_gw = 1.0 / std::get<0>(q.resolution()); + const double px_gh = 1.0 / std::get<1>(q.resolution()); MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: px_gw=" << px_gw << " px_gh=" << px_gh; @@ -901,7 +893,7 @@ featureset_ptr pgraster_datasource::features_with_context(query const& q,process mapnik::sql_utils::unquote_double(raster_table_), tab); boost::algorithm::replace_all(table_with_bbox, mapnik::sql_utils::unquote_double(schema_), sch); - boost::algorithm::replace_all(table_with_bbox, + boost::algorithm::replace_all(table_with_bbox, mapnik::sql_utils::unquote_double(geometryColumn_), col); table_with_bbox = populate_tokens(table_with_bbox, scale_denom, box, px_gw, px_gh); @@ -925,7 +917,7 @@ featureset_ptr pgraster_datasource::features_with_context(query const& q,process s << ", ST_Expand(" << sql_bbox(box) << ", greatest(abs(ST_ScaleX(\"" << col << "\")), abs(ST_ScaleY(\"" - << col << "\")))))"; + << col << "\")))))"; } if (prescale_rasters_) { @@ -942,7 +934,7 @@ featureset_ptr pgraster_datasource::features_with_context(query const& q,process s << ") AS geom"; - mapnik::context_ptr ctx = boost::make_shared(); + mapnik::context_ptr ctx = std::make_shared(); std::set const& props = q.property_names(); std::set::const_iterator pos = props.begin(); std::set::const_iterator end = props.end(); @@ -980,10 +972,10 @@ featureset_ptr pgraster_datasource::features_with_context(query const& q,process MAPNIK_LOG_DEBUG(pgraster) << "pgraster_datasource: " "features query: " << s.str(); - boost::shared_ptr rs = get_resultset(conn, s.str(), pool, proc_ctx); - return boost::make_shared(rs, ctx, + std::shared_ptr rs = get_resultset(conn, s.str(), pool, proc_ctx); + return std::make_shared(rs, ctx, desc_.get_encoding(), !key_field_.empty(), - band_ ? 1 : 0 // whatever band number is given we'd have + band_ ? 1 : 0 // whatever band number is given we'd have // extracted with ST_Band above so it becomes // band number 1 ); @@ -1032,7 +1024,7 @@ featureset_ptr pgraster_datasource::features_at_point(coord2d const& pt, double std::ostringstream s; s << "SELECT ST_AsBinary(\"" << geometryColumn_ << "\") AS geom"; - mapnik::context_ptr ctx = boost::make_shared(); + mapnik::context_ptr ctx = std::make_shared(); std::vector::const_iterator itr = desc_.get_descriptors().begin(); std::vector::const_iterator end = desc_.get_descriptors().end(); @@ -1068,8 +1060,8 @@ featureset_ptr pgraster_datasource::features_at_point(coord2d const& pt, double s << " LIMIT " << row_limit_; } - boost::shared_ptr rs = get_resultset(conn, s.str(), pool); - return boost::make_shared(rs, ctx, desc_.get_encoding(), !key_field_.empty()); + std::shared_ptr rs = get_resultset(conn, s.str(), pool); + return std::make_shared(rs, ctx, desc_.get_encoding(), !key_field_.empty()); } } @@ -1196,5 +1188,3 @@ boost::optional pgraster_datasource::get_geometr { return boost::optional(); } - - diff --git a/plugins/input/pgraster/pgraster_datasource.hpp b/plugins/input/pgraster/pgraster_datasource.hpp index e4f1ae38f..b58114139 100644 --- a/plugins/input/pgraster/pgraster_datasource.hpp +++ b/plugins/input/pgraster/pgraster_datasource.hpp @@ -18,10 +18,6 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * - ***************************************************************************** - * - * Initially developed by Sandro Santilli - * *****************************************************************************/ #ifndef PGRASTER_DATASOURCE_HPP @@ -40,12 +36,11 @@ // boost #include -#include -#include // stl #include #include +#include #include "connection_manager.hpp" #include "resultset.hpp" @@ -63,7 +58,7 @@ using mapnik::query; using mapnik::parameters; using mapnik::coord2d; -typedef boost::shared_ptr< ConnectionManager::PoolType> CnxPool_ptr; +typedef std::shared_ptr< ConnectionManager::PoolType> CnxPool_ptr; struct pgraster_overview { @@ -93,7 +88,7 @@ private: std::string sql_bbox(box2d const& env) const; std::string populate_tokens(std::string const& sql, double scale_denom, box2d const& env, double pixel_width, double pixel_height) const; std::string populate_tokens(std::string const& sql) const; - boost::shared_ptr get_resultset(boost::shared_ptr &conn, std::string const& sql, CnxPool_ptr const& pool, processor_context_ptr ctx= processor_context_ptr()) const; + std::shared_ptr get_resultset(std::shared_ptr &conn, std::string const& sql, CnxPool_ptr const& pool, processor_context_ptr ctx= processor_context_ptr()) const; static const std::string RASTER_COLUMNS; static const std::string RASTER_OVERVIEWS; static const std::string SPATIAL_REF_SYS; diff --git a/plugins/input/pgraster/pgraster_featureset.cpp b/plugins/input/pgraster/pgraster_featureset.cpp index 8e4ded1fd..e75b97f07 100644 --- a/plugins/input/pgraster/pgraster_featureset.cpp +++ b/plugins/input/pgraster/pgraster_featureset.cpp @@ -18,10 +18,6 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * - ***************************************************************************** - * - * Initially developed by Sandro Santilli - * *****************************************************************************/ #include "pgraster_featureset.hpp" @@ -43,10 +39,6 @@ #include // for int2net #include - -// boost -#include // for boost::int16_t - // stl #include #include @@ -56,7 +48,7 @@ using mapnik::byte; using mapnik::feature_factory; using mapnik::context_ptr; -pgraster_featureset::pgraster_featureset(boost::shared_ptr const& rs, +pgraster_featureset::pgraster_featureset(std::shared_ptr const& rs, context_ptr const& ctx, std::string const& encoding, bool key_field, int bandno) diff --git a/plugins/input/pgraster/pgraster_featureset.hpp b/plugins/input/pgraster/pgraster_featureset.hpp index b4652bf48..5665be6e7 100644 --- a/plugins/input/pgraster/pgraster_featureset.hpp +++ b/plugins/input/pgraster/pgraster_featureset.hpp @@ -2,7 +2,7 @@ * * This file is part of Mapnik (c++ mapping toolkit) * - * Copyright (C) 2014 Artem Pavlenko + * Copyright (C) 2011 Artem Pavlenko * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -18,10 +18,6 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * - ***************************************************************************** - * - * Initially developed by Sandro Santilli - * *****************************************************************************/ #ifndef PGRASTER_FEATURESET_HPP @@ -33,8 +29,8 @@ #include #include -// boost -#include +// stl +#include using mapnik::Featureset; using mapnik::box2d; @@ -51,7 +47,7 @@ public: /// @param bandno band number (1-based). 0 (default) reads all bands. /// Anything else forces interpretation of colors off /// (values copied verbatim) - pgraster_featureset(boost::shared_ptr const& rs, + pgraster_featureset(std::shared_ptr const& rs, context_ptr const& ctx, std::string const& encoding, bool key_field = false, @@ -60,9 +56,9 @@ public: ~pgraster_featureset(); private: - boost::shared_ptr rs_; + std::shared_ptr rs_; context_ptr ctx_; - boost::scoped_ptr tr_; + const std::unique_ptr tr_; mapnik::value_integer feature_id_; bool key_field_; int band_; diff --git a/plugins/input/pgraster/pgraster_wkb_reader.cpp b/plugins/input/pgraster/pgraster_wkb_reader.cpp index f61d27ba6..2d73145b8 100644 --- a/plugins/input/pgraster/pgraster_wkb_reader.cpp +++ b/plugins/input/pgraster/pgraster_wkb_reader.cpp @@ -18,10 +18,6 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * - ***************************************************************************** - * - * Initially developed by Sandro Santilli - * *****************************************************************************/ #include "pgraster_wkb_reader.hpp" @@ -38,8 +34,6 @@ #include // for box2d // boost -#include // for boost::int8_t -#include #include namespace { @@ -456,10 +450,10 @@ pgraster_wkb_reader::get_raster() { return mapnik::raster_ptr(); } - box2d ext(ipX,ipY,ipX+(width_*scaleX),ipY+(height_*scaleY)); + box2d ext(ipX,ipY,ipX+(width_*scaleX),ipY+(height_*scaleY)); MAPNIK_LOG_DEBUG(pgraster) << "pgraster_wkb_reader: Raster extent=" << ext; - mapnik::raster_ptr raster = boost::make_shared(ext, width_, height_); + mapnik::raster_ptr raster = std::make_shared(ext, width_, height_, 1.0); if ( bandno_ ) { if ( bandno_ != 1 ) { @@ -493,5 +487,3 @@ pgraster_wkb_reader::get_raster() { return raster; } - - diff --git a/plugins/input/pgraster/pgraster_wkb_reader.hpp b/plugins/input/pgraster/pgraster_wkb_reader.hpp index 3a571ffd5..64799d2da 100644 --- a/plugins/input/pgraster/pgraster_wkb_reader.hpp +++ b/plugins/input/pgraster/pgraster_wkb_reader.hpp @@ -18,10 +18,6 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * - ***************************************************************************** - * - * Initially developed by Sandro Santilli - * *****************************************************************************/ #ifndef PGRASTER_WKB_READER_HPP @@ -30,9 +26,6 @@ // mapnik #include // for raster_ptr -// boost -#include // for boost::uint8_t - enum pgraster_color_interp { // Automatic color interpretation: // uses grayscale for single band, rgb for 3 bands From e4c7e2419c9295362c7e8bd16dcbe43bafbc5d30 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 12 Aug 2014 20:19:35 -0700 Subject: [PATCH 08/10] make sure pgraster is rebuilt if headers change inside the postgis plugin dir --- plugins/input/pgraster/build.py | 3 --- plugins/input/pgraster/pgraster_datasource.cpp | 4 ++-- plugins/input/pgraster/pgraster_datasource.hpp | 6 +++--- plugins/input/pgraster/pgraster_featureset.cpp | 4 ++-- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/plugins/input/pgraster/build.py b/plugins/input/pgraster/build.py index e4b083806..9f1f40fe2 100644 --- a/plugins/input/pgraster/build.py +++ b/plugins/input/pgraster/build.py @@ -38,9 +38,6 @@ plugin_sources = Split( cxxflags = [] plugin_env['LIBS'] = [] -# TODO: use variable for top-level dir -plugin_env['CXXFLAGS'].append('-Iplugins/input/postgis') - if env['RUNTIME_LINK'] == 'static': # pkg-config is more reliable than pg_config across platforms cmd = 'pkg-config libpq --libs --static' diff --git a/plugins/input/pgraster/pgraster_datasource.cpp b/plugins/input/pgraster/pgraster_datasource.cpp index 3c4fe2729..cc4d08d77 100644 --- a/plugins/input/pgraster/pgraster_datasource.cpp +++ b/plugins/input/pgraster/pgraster_datasource.cpp @@ -20,10 +20,10 @@ * *****************************************************************************/ -#include "connection_manager.hpp" +#include "../postgis/connection_manager.hpp" +#include "../postgis/asyncresultset.hpp" #include "pgraster_datasource.hpp" #include "pgraster_featureset.hpp" -#include "asyncresultset.hpp" // mapnik #include diff --git a/plugins/input/pgraster/pgraster_datasource.hpp b/plugins/input/pgraster/pgraster_datasource.hpp index b58114139..e00825131 100644 --- a/plugins/input/pgraster/pgraster_datasource.hpp +++ b/plugins/input/pgraster/pgraster_datasource.hpp @@ -42,9 +42,9 @@ #include #include -#include "connection_manager.hpp" -#include "resultset.hpp" -#include "cursorresultset.hpp" +#include "../postgis/connection_manager.hpp" +#include "../postgis/resultset.hpp" +#include "../postgis/cursorresultset.hpp" using mapnik::transcoder; using mapnik::datasource; diff --git a/plugins/input/pgraster/pgraster_featureset.cpp b/plugins/input/pgraster/pgraster_featureset.cpp index e75b97f07..17e14c5cd 100644 --- a/plugins/input/pgraster/pgraster_featureset.cpp +++ b/plugins/input/pgraster/pgraster_featureset.cpp @@ -22,8 +22,8 @@ #include "pgraster_featureset.hpp" #include "pgraster_wkb_reader.hpp" -#include "resultset.hpp" -#include "cursorresultset.hpp" +#include "../postgis/resultset.hpp" +#include "../postgis/cursorresultset.hpp" // mapnik #include From 78258a9459f6ecc869674b3f7d5995460e170511 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 12 Aug 2014 20:19:59 -0700 Subject: [PATCH 09/10] quiet pgraster tests by default --- tests/python_tests/pgraster_test.py | 51 +++++++++++++++-------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/python_tests/pgraster_test.py b/tests/python_tests/pgraster_test.py index f958b55b4..15045dc74 100644 --- a/tests/python_tests/pgraster_test.py +++ b/tests/python_tests/pgraster_test.py @@ -16,6 +16,11 @@ from binascii import hexlify, unhexlify MAPNIK_TEST_DBNAME = 'mapnik-tmp-pgraster-test-db' POSTGIS_TEMPLATE_DBNAME = 'template_postgis' +DEBUG_OUTPUT=False + +def log(msg): + if DEBUG_OUTPUT: + print msg def setup(): # All of the paths used are relative, if we run the tests @@ -23,11 +28,10 @@ def setup(): os.chdir(execution_path('.')) def call(cmd,silent=False): - proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) - stdout, stderr = proc.communicate() - if stderr and not silent: - print stderr.strip() - if proc.returncode: + stdin, stderr = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE).communicate() + if not stderr: + return stdin.strip() + elif not silent and 'ERROR' in stderr: raise RuntimeError(stderr.strip()) def psql_can_connect(): @@ -48,7 +52,7 @@ def psql_can_connect(): def psql_run(cmd): cmd = 'psql --set ON_ERROR_STOP=1 %s -c "%s"' % \ (MAPNIK_TEST_DBNAME, cmd.replace('"', '\\"')) - print 'DEBUG: running ' + cmd + log('DEBUG: running ' + cmd) call(cmd) def raster2pgsql_on_path(): @@ -86,8 +90,8 @@ def postgis_takedown(): #call('dropdb %s' % MAPNIK_TEST_DBNAME) def import_raster(filename, tabname, tilesize, constraint, overview): - print 'tile: ' + tilesize + ' constraints: ' + str(constraint) \ - + ' overviews: ' + overview + log('tile: ' + tilesize + ' constraints: ' + str(constraint) \ + + ' overviews: ' + overview) cmd = 'raster2pgsql -Y -I -q' if constraint: cmd += ' -C' @@ -95,8 +99,8 @@ def import_raster(filename, tabname, tilesize, constraint, overview): cmd += ' -t ' + tilesize if overview: cmd += ' -l ' + overview - cmd += ' %s %s | psql --set ON_ERROR_STOP=1 -q %s' % (filename,tabname,MAPNIK_TEST_DBNAME) - print 'Import call: ' + cmd + cmd += ' %s %s | psql --set ON_ERROR_STOP=1 -q %s' % (os.path.abspath(os.path.normpath(filename)),tabname,MAPNIK_TEST_DBNAME) + log('Import call: ' + cmd) call(cmd) def drop_imported(tabname, overview): @@ -162,7 +166,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ t0 = time.time() # we want wall time to include IO waits mapnik.render(mm, im) lap = time.time() - t0 - print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + log('T ' + str(lap) + ' -- ' + lbl + ' E:full') # no data eq_(im.view(1,1,1,1).tostring(), '\x00\x00\x00\x00') eq_(im.view(255,255,1,1).tostring(), '\x00\x00\x00\x00') @@ -183,7 +187,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ t0 = time.time() # we want wall time to include IO waits mapnik.render(mm, im) lap = time.time() - t0 - print 'T ' + str(lap) + ' -- ' + lbl + ' E:1/10' + log('T ' + str(lap) + ' -- ' + lbl + ' E:1/10') # nodata eq_(hexlify(im.view(255,255,1,1).tostring()), '00000000') eq_(hexlify(im.view(200,254,1,1).tostring()), '00000000') @@ -197,8 +201,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ eq_(hexlify(im.view(255, 0,1,1).tostring()), '404040ff') def _test_dataraster_16bsi(lbl, tilesize, constraint, overview): - rf = os.path.join(execution_path('.'),'../data/raster/dataraster.tif') - import_raster(rf, 'dataRaster', tilesize, constraint, overview) + import_raster('../data/raster/dataraster.tif', 'dataRaster', tilesize, constraint, overview) if constraint: lbl += ' C' if tilesize: @@ -259,7 +262,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ t0 = time.time() # we want wall time to include IO waits mapnik.render(mm, im) lap = time.time() - t0 - print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + log('T ' + str(lap) + ' -- ' + lbl + ' E:full') #im.save('/tmp/xfull.png') # for debugging # no data eq_(hexlify(im.view(3,3,1,1).tostring()), '00000000') @@ -281,7 +284,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ t0 = time.time() # we want wall time to include IO waits mapnik.render(mm, im) lap = time.time() - t0 - print 'T ' + str(lap) + ' -- ' + lbl + ' E:1/10' + log('T ' + str(lap) + ' -- ' + lbl + ' E:1/10') #im.save('/tmp/xtenth.png') # for debugging # no data eq_(hexlify(im.view(255,255,1,1).tostring()), '00000000') @@ -298,8 +301,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ 'unexpected full transparent/opaque pixel: ' + alpha def _test_rgba_8bui(lbl, tilesize, constraint, overview): - rf = os.path.join(execution_path('.'),'../data/raster/river.tiff') - import_raster(rf, 'River', tilesize, constraint, overview) + import_raster('../data/raster/river.tiff', 'River', tilesize, constraint, overview) if constraint: lbl += ' C' if tilesize: @@ -361,7 +363,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ t0 = time.time() # we want wall time to include IO waits mapnik.render(mm, im) lap = time.time() - t0 - print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + log('T ' + str(lap) + ' -- ' + lbl + ' E:full') #im.save('/tmp/xfull.png') # for debugging # no data eq_(hexlify(im.view(3,16,1,1).tostring()), '00000000') @@ -381,7 +383,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ t0 = time.time() # we want wall time to include IO waits mapnik.render(mm, im) lap = time.time() - t0 - print 'T ' + str(lap) + ' -- ' + lbl + ' E:1/10' + log('T ' + str(lap) + ' -- ' + lbl + ' E:1/10') #im.save('/tmp/xtenth.png') # for debugging # no data eq_(hexlify(im.view(3,16,1,1).tostring()), '00000000') @@ -397,9 +399,8 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ eq_(hexlify(im.view(195,223,1,1).tostring()), 'f2cdbaff') def _test_rgb_8bui(lbl, tilesize, constraint, overview): - rf = os.path.join(execution_path('.'),'../data/raster/nodata-edge.tif') tnam = 'nodataedge' - import_raster(rf, tnam, tilesize, constraint, overview) + import_raster('../data/raster/nodata-edge.tif', tnam, tilesize, constraint, overview) if constraint: lbl += ' C' if tilesize: @@ -476,7 +477,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ t0 = time.time() # we want wall time to include IO waits mapnik.render(mm, im) lap = time.time() - t0 - print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + log('T ' + str(lap) + ' -- ' + lbl + ' E:full') #im.save('/tmp/xfull.png') # for debugging h = format(value, '02x') hex_v = h+h+h+'ff' @@ -590,7 +591,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ t0 = time.time() # we want wall time to include IO waits mapnik.render(mm, im) lap = time.time() - t0 - print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + log('T ' + str(lap) + ' -- ' + lbl + ' E:full') #im.save('/tmp/xfull.png') # for debugging h = format(value, '02x') hex_v = '0000ffff' @@ -709,7 +710,7 @@ if 'pgraster' in mapnik.DatasourceCache.plugin_names() \ t0 = time.time() # we want wall time to include IO waits mapnik.render(mm, im) lap = time.time() - t0 - print 'T ' + str(lap) + ' -- ' + lbl + ' E:full' + log('T ' + str(lap) + ' -- ' + lbl + ' E:full') im.save('/tmp/xfull.png') # for debugging hex_v = format(r << 24 | g << 16 | b << 8 | a, '08x') hex_a = format(r << 24 | g1 << 16 | b << 8 | a, '08x') From 44844f7bd0b11b9b68887e3e4a678d3dd82eda8a Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 12 Aug 2014 20:26:52 -0700 Subject: [PATCH 10/10] postgis: disable NOTICES by default - refs #2296 --- plugins/input/postgis/connection.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/input/postgis/connection.hpp b/plugins/input/postgis/connection.hpp index 19f34a408..479472109 100644 --- a/plugins/input/postgis/connection.hpp +++ b/plugins/input/postgis/connection.hpp @@ -65,7 +65,7 @@ public: close(); throw mapnik::datasource_exception(err_msg); } - PGresult *result = PQexec(conn_, "SET DEFAULT_TRANSACTION_READ_ONLY = TRUE;"); + PGresult *result = PQexec(conn_, "SET DEFAULT_TRANSACTION_READ_ONLY = TRUE; SET CLIENT_MIN_MESSAGES = WARNING;"); bool ok = (result && (PQresultStatus(result) == PGRES_COMMAND_OK)); if ( result ) PQclear(result); if ( ! ok ) { @@ -74,6 +74,7 @@ public: err_msg += "\nConnection string: '"; err_msg += connection_str; err_msg += "'\n"; + close(); throw mapnik::datasource_exception(err_msg); } }