From 8fe932b7d1c63f74569b570ff64ee141b1531d14 Mon Sep 17 00:00:00 2001 From: artemp Date: Mon, 24 Sep 2012 13:35:22 +0100 Subject: [PATCH 01/10] + tiff_reader: make TIFF* 'cacheable', to avoid initialization under certain conditions. --- src/tiff_reader.cpp | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/tiff_reader.cpp b/src/tiff_reader.cpp index a536ee232..effb6551b 100644 --- a/src/tiff_reader.cpp +++ b/src/tiff_reader.cpp @@ -23,6 +23,8 @@ // mapnik #include #include +// boost +#include #include // stl @@ -41,6 +43,18 @@ using std::max; class tiff_reader : public image_reader { + typedef boost::shared_ptr tiff_ptr; + struct tiff_closer + { + void operator() (TIFF * tif) + { + if (tif != 0) + { + TIFFClose(tif); + } + } + }; + private: std::string file_name_; int read_method_; @@ -49,6 +63,8 @@ private: int rows_per_strip_; int tile_width_; int tile_height_; + tiff_ptr tif_; + public: enum TiffType { generic=1, @@ -116,11 +132,9 @@ void tiff_reader::init() { read_method_=stripped; } - TIFFClose(tif); } else { - TIFFClose(tif); throw image_reader_exception(msg); } } @@ -128,7 +142,6 @@ void tiff_reader::init() tiff_reader::~tiff_reader() { - // } @@ -167,8 +180,6 @@ void tiff_reader::read_generic(unsigned /*x*/,unsigned /*y*/,image_data_32& /*im if (tif) { MAPNIK_LOG_DEBUG(tiff_reader) << "tiff_reader: TODO - tiff is not stripped or tiled"; - - TIFFClose(tif); } } @@ -213,7 +224,6 @@ void tiff_reader::read_tiled(unsigned x0,unsigned y0,image_data_32& image) } } _TIFFfree(buf); - TIFFClose(tif); } } @@ -254,21 +264,21 @@ void tiff_reader::read_stripped(unsigned x0,unsigned y0,image_data_32& image) } } _TIFFfree(buf); - TIFFClose(tif); } } TIFF* tiff_reader::load_if_exists(std::string const& filename) { - TIFF * tif = 0; - boost::filesystem::path path(file_name_); - if (exists(path)) // && is_regular(path)) { -- not supported in boost-1.33.* + if (!tif_) { - // File path is a full file path and does exist - tif = TIFFOpen(filename.c_str(), "rb"); + boost::filesystem::path path(file_name_); + if (boost::filesystem::is_regular(path)) // exists and regular file + { + // File path is a full file path and does exist + tif_ = tiff_ptr(TIFFOpen(filename.c_str(), "rb"), tiff_closer()); + } } - - return tif; -} + return tif_.get(); } +} // namespace mapnik From ac735e82c2c05114505abcba5632eba62e5bf3a3 Mon Sep 17 00:00:00 2001 From: artemp Date: Mon, 24 Sep 2012 13:39:03 +0100 Subject: [PATCH 02/10] + make tile_size configurable for 'tiled' read policy --- plugins/input/raster/raster_datasource.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/input/raster/raster_datasource.cpp b/plugins/input/raster/raster_datasource.cpp index 1c66c462e..f758ae736 100644 --- a/plugins/input/raster/raster_datasource.cpp +++ b/plugins/input/raster/raster_datasource.cpp @@ -45,7 +45,7 @@ using mapnik::image_reader; DATASOURCE_PLUGIN(raster_datasource) -raster_datasource::raster_datasource(const parameters& params, bool bind) +raster_datasource::raster_datasource(parameters const& params, bool bind) : datasource(params), desc_(*params.get("type"), "utf-8"), extent_initialized_(false) @@ -201,11 +201,11 @@ featureset_ptr raster_datasource::features(query const& q) const return boost::make_shared >(policy, extent_, q); } - else if (width * height > 512*512) + else if (width * height > (tile_size_ * tile_size_ << 2)) { MAPNIK_LOG_DEBUG(raster) << "raster_datasource: Tiled policy"; - tiled_file_policy policy(filename_, format_, 256, extent_, q.get_bbox(), width_, height_); + tiled_file_policy policy(filename_, format_, tile_size_, extent_, q.get_bbox(), width_, height_); return boost::make_shared >(policy, extent_, q); } From 5c5c4fb76ed73ad56fb15de794e9dd7ae3825ab1 Mon Sep 17 00:00:00 2001 From: artemp Date: Mon, 24 Sep 2012 16:14:17 +0100 Subject: [PATCH 03/10] + fix parameters names to be more css-ish --- plugins/input/raster/raster_datasource.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/input/raster/raster_datasource.cpp b/plugins/input/raster/raster_datasource.cpp index f758ae736..1a33952c8 100644 --- a/plugins/input/raster/raster_datasource.cpp +++ b/plugins/input/raster/raster_datasource.cpp @@ -62,8 +62,8 @@ raster_datasource::raster_datasource(parameters const& params, bool bind) filename_ = *file; multi_tiles_ = *params_.get("multi", false); - tile_size_ = *params_.get("tile_size", 256); - tile_stride_ = *params_.get("tile_stride", 1); + tile_size_ = *params_.get("tile-size", 256); + tile_stride_ = *params_.get("tile-stride", 1); format_ = *params_.get("format","tiff"); @@ -100,8 +100,8 @@ void raster_datasource::bind() const if (multi_tiles_) { - boost::optional x_width = params_.get("x_width"); - boost::optional y_width = params_.get("y_width"); + boost::optional x_width = params_.get("x-width"); + boost::optional y_width = params_.get("y-width"); if (! x_width) { From ae176d1d193b8b264d1493b84f8f7a8082967200 Mon Sep 17 00:00:00 2001 From: artemp Date: Mon, 24 Sep 2012 17:12:03 +0100 Subject: [PATCH 04/10] + use 'extent' for both raster.input and gdal.input --- plugins/input/gdal/gdal_datasource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/input/gdal/gdal_datasource.cpp b/plugins/input/gdal/gdal_datasource.cpp index a324b3825..048edf1c0 100644 --- a/plugins/input/gdal/gdal_datasource.cpp +++ b/plugins/input/gdal/gdal_datasource.cpp @@ -121,7 +121,7 @@ void gdal_datasource::bind() const double tr[6]; bool bbox_override = false; - boost::optional bbox_s = params_.get("bbox"); + boost::optional bbox_s = params_.get("extent"); if (bbox_s) { MAPNIK_LOG_DEBUG(gdal) << "gdal_datasource: BBox Parameter=" << *bbox_s; From e8abc8eef1665c1fa62de4e9f5dd4a0e58f5258a Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 25 Sep 2012 14:08:07 -0700 Subject: [PATCH 05/10] csv: stop adding WKT/JSON geometry data to attributes --- plugins/input/csv/csv_datasource.cpp | 4 +++- tests/python_tests/csv_test.py | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/plugins/input/csv/csv_datasource.cpp b/plugins/input/csv/csv_datasource.cpp index 9352bbf71..a08dc833a 100644 --- a/plugins/input/csv/csv_datasource.cpp +++ b/plugins/input/csv/csv_datasource.cpp @@ -665,7 +665,9 @@ void csv_datasource::parse_csv(T & stream, } } - // now, add all values as attributes + // now, add attributes, skipping any WKT or JSON fiels + if ((has_wkt_field) && (i == wkt_idx)) continue; + if ((has_json_field) && (i == json_idx)) continue; /* First we detect likely strings, then try parsing likely numbers, finally falling back to string type * We intentionally do not try to detect boolean or null types diff --git a/tests/python_tests/csv_test.py b/tests/python_tests/csv_test.py index 211be6b39..65db6a654 100644 --- a/tests/python_tests/csv_test.py +++ b/tests/python_tests/csv_test.py @@ -137,9 +137,9 @@ if 'csv' in mapnik.DatasourceCache.plugin_names(): def test_wkt_field(**kwargs): ds = get_csv_ds('wkt.csv') - eq_(len(ds.fields()),2) - eq_(ds.fields(),['type','WKT']) - eq_(ds.field_types(),['str','str']) + eq_(len(ds.fields()),1) + eq_(ds.fields(),['type']) + eq_(ds.field_types(),['str']) fs = ds.all_features() eq_(len(fs[0].geometries()),1) eq_(fs[0].geometries()[0].type(),mapnik.DataGeometryType.Point) @@ -396,9 +396,9 @@ if 'csv' in mapnik.DatasourceCache.plugin_names(): eq_(feat['Name'],u"Winthrop, WA") def validate_geojson_datasource(ds): - eq_(len(ds.fields()),2) - eq_(ds.fields(),['type','GeoJSON']) - eq_(ds.field_types(),['str','str']) + eq_(len(ds.fields()),1) + eq_(ds.fields(),['type']) + eq_(ds.field_types(),['str']) fs = ds.all_features() eq_(len(fs[0].geometries()),1) eq_(fs[0].geometries()[0].type(),mapnik.DataGeometryType.Point) From 1af2faf56b1357fae39c9caee16a2ffde74c977c Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 25 Sep 2012 14:52:32 -0700 Subject: [PATCH 06/10] formatting --- src/agg/process_raster_symbolizer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/agg/process_raster_symbolizer.cpp b/src/agg/process_raster_symbolizer.cpp index dbdcc88b7..2830b2564 100644 --- a/src/agg/process_raster_symbolizer.cpp +++ b/src/agg/process_raster_symbolizer.cpp @@ -83,7 +83,8 @@ void agg_renderer::process(raster_symbolizer const& sym, if (scaling_method == SCALING_BILINEAR8) { scale_image_bilinear8(target.data_,source->data_, 0.0, 0.0); - } else + } + else { double scaling_ratio = ext.width() / source->data_.width(); scale_image_agg(target.data_, From a63df933d522e4090fb98bc46baa8ee1f95eb022 Mon Sep 17 00:00:00 2001 From: artemp Date: Wed, 26 Sep 2012 14:24:26 +0100 Subject: [PATCH 07/10] + only log transform evaluation if trans_expr is not null --- src/symbolizer.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/symbolizer.cpp b/src/symbolizer.cpp index eab65cf58..298c29abb 100644 --- a/src/symbolizer.cpp +++ b/src/symbolizer.cpp @@ -30,15 +30,12 @@ namespace mapnik { void evaluate_transform(agg::trans_affine& tr, Feature const& feature, transform_list_ptr const& trans_expr) { - #ifdef MAPNIK_LOG - MAPNIK_LOG_DEBUG(transform) << "transform: evaluate " - << (trans_expr - ? transform_processor_type::to_string(*trans_expr) - : std::string("null")); - #endif - if (trans_expr) { +#ifdef MAPNIK_LOG + MAPNIK_LOG_DEBUG(transform) << "transform: evaluate " + << transform_processor_type::to_string(*trans_expr); +#endif transform_processor_type::evaluate(tr, feature, *trans_expr); } } From d91cd510aa5a43981e01724aaa0adff4336e42a7 Mon Sep 17 00:00:00 2001 From: artemp Date: Wed, 26 Sep 2012 14:27:42 +0100 Subject: [PATCH 08/10] + ensure warp polygons are aligned exactly (avoid padding!) - #1501 --- src/warp.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/warp.cpp b/src/warp.cpp index 60232b01e..4fa185d94 100644 --- a/src/warp.cpp +++ b/src/warp.cpp @@ -47,11 +47,11 @@ namespace mapnik { void reproject_and_scale_raster(raster & target, raster const& source, - proj_transform const& prj_trans, - double offset_x, double offset_y, - unsigned mesh_size, - double filter_radius, - scaling_method_e scaling_method) + proj_transform const& prj_trans, + double offset_x, double offset_y, + unsigned mesh_size, + double filter_radius, + scaling_method_e scaling_method) { CoordTransform ts(source.data_.width(), source.data_.height(), source.ext_); @@ -160,10 +160,10 @@ void reproject_and_scale_raster(raster & target, raster const& source, tt.forward(polygon+6, polygon+7); rasterizer.reset(); - rasterizer.move_to_d(polygon[0]-1, polygon[1]-1); - rasterizer.line_to_d(polygon[2]+1, polygon[3]-1); - rasterizer.line_to_d(polygon[4]+1, polygon[5]+1); - rasterizer.line_to_d(polygon[6]-1, polygon[7]+1); + rasterizer.move_to_d(std::floor(polygon[0]), std::floor(polygon[1])); + rasterizer.line_to_d(std::floor(polygon[2]), std::floor(polygon[3])); + rasterizer.line_to_d(std::floor(polygon[4]), std::floor(polygon[5])); + rasterizer.line_to_d(std::floor(polygon[6]), std::floor(polygon[7])); unsigned x0 = i * mesh_size; unsigned y0 = j * mesh_size; From ef94bc8db8b7ac1052cc947cefd341bd747d91fa Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Wed, 26 Sep 2012 11:26:51 -0700 Subject: [PATCH 09/10] support python 2.5 --- tests/python_tests/json_feature_properties_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/python_tests/json_feature_properties_test.py b/tests/python_tests/json_feature_properties_test.py index 2eb92a20a..0f27810c3 100644 --- a/tests/python_tests/json_feature_properties_test.py +++ b/tests/python_tests/json_feature_properties_test.py @@ -3,7 +3,10 @@ from nose.tools import * import os,sys import mapnik -import json +try: + import json +except ImportError: + import simplejson as json chars = [ { From 7cf4a00af9530ba2aa46ba1a4a5cb2cb41fe166a Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Wed, 26 Sep 2012 11:27:25 -0700 Subject: [PATCH 10/10] add ability to generate expected test results on the fly --- tests/visual_tests/compare.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/visual_tests/compare.py b/tests/visual_tests/compare.py index 8fa26360e..43bd251d9 100644 --- a/tests/visual_tests/compare.py +++ b/tests/visual_tests/compare.py @@ -49,7 +49,7 @@ def compare(actual, expected): passed += 1 return diff -def summary(): +def summary(generate=False): global errors global passed print "-"*80 @@ -57,7 +57,13 @@ def summary(): if len(errors) != 0: for error in errors: if (error[0] is None): - print "Could not verify %s: No reference image found!" % error[1] + if generate: + actual = open(error[1],'r').read() + open(error[2],'wb').write(actual) + print "Generating reference image: '%s'" % error[2] + continue + else: + print "Could not verify %s: No reference image found!" % error[1] else: print "Failed: %d different pixels:\n\t%s (actual)\n\t%s (expected)" % error sys.exit(1)