From a8d7fc1cfbd0bc2f388d6f132064fccfc62a95ec Mon Sep 17 00:00:00 2001 From: Artem Pavlenko Date: Thu, 15 Mar 2018 15:58:23 +0100 Subject: [PATCH 1/5] Revert "avoid symbols duplication" This reverts commit c737f4d56f2f3b56237c1d1eef500321bb6b7c47. --- include/mapnik/csv/csv_grammar_x3_def.hpp | 8 ++++---- include/mapnik/expression_grammar_x3_def.hpp | 4 ++-- .../mapnik/json/generic_json_grammar_x3_def.hpp | 2 +- include/mapnik/json/positions_grammar_x3_def.hpp | 2 +- include/mapnik/json/topojson_grammar_x3_def.hpp | 2 +- include/mapnik/json/unicode_string_grammar_x3.hpp | 3 --- .../mapnik/json/unicode_string_grammar_x3_def.hpp | 4 ++-- src/expression_grammar_x3.cpp | 14 ++------------ 8 files changed, 13 insertions(+), 26 deletions(-) diff --git a/include/mapnik/csv/csv_grammar_x3_def.hpp b/include/mapnik/csv/csv_grammar_x3_def.hpp index a4f5e8942..792175e96 100644 --- a/include/mapnik/csv/csv_grammar_x3_def.hpp +++ b/include/mapnik/csv/csv_grammar_x3_def.hpp @@ -38,9 +38,9 @@ using x3::lit; using x3::lexeme; using ascii::char_; -struct csv_unesc_chars_ : x3::symbols +struct unesc_char_ : x3::symbols { - csv_unesc_chars_() + unesc_char_() { add("\\a", '\a') ("\\b", '\b') @@ -55,7 +55,7 @@ struct csv_unesc_chars_ : x3::symbols ("\"\"", '\"') // double quote ; } -} csv_unesc_chars; +} unesc_char; template struct literal : x3::parser> @@ -97,7 +97,7 @@ auto const column_def = quoted_text | *(char_ - separator) auto const quoted_text_def = quote > text > quote // support unmatched quotes or not (??) ; -auto const text_def = *(csv_unesc_chars | (char_ - quote)) +auto const text_def = *(unesc_char | (char_ - quote)) ; BOOST_SPIRIT_DEFINE ( diff --git a/include/mapnik/expression_grammar_x3_def.hpp b/include/mapnik/expression_grammar_x3_def.hpp index f35ccdb45..dd7d40eba 100644 --- a/include/mapnik/expression_grammar_x3_def.hpp +++ b/include/mapnik/expression_grammar_x3_def.hpp @@ -24,7 +24,7 @@ #define MAPNIK_EXPRESSIONS_GRAMMAR_X3_DEF_HPP #include -#include +#include #include #include #include @@ -67,7 +67,7 @@ namespace mapnik { namespace grammar { x3::uint_parser const hex2 {}; namespace { - auto const& escaped_unicode = json::escaped_unicode_grammar(); + auto const& escaped_unicode = json::grammar::escaped_unicode; } auto append = [](auto const& ctx) diff --git a/include/mapnik/json/generic_json_grammar_x3_def.hpp b/include/mapnik/json/generic_json_grammar_x3_def.hpp index 9f6e882d5..ea4f52f3c 100644 --- a/include/mapnik/json/generic_json_grammar_x3_def.hpp +++ b/include/mapnik/json/generic_json_grammar_x3_def.hpp @@ -74,7 +74,7 @@ namespace { auto const& json_string = mapnik::json::unicode_string_grammar(); } // exported rules // start -generic_json_grammar_type const value MAPNIK_INIT_PRIORITY(103) ("JSON Value"); +generic_json_grammar_type const value MAPNIK_INIT_PRIORITY(102) ("JSON Value"); generic_json_key_value_type const key_value("JSON Object element"); // rules x3::rule const object("JSON Object"); diff --git a/include/mapnik/json/positions_grammar_x3_def.hpp b/include/mapnik/json/positions_grammar_x3_def.hpp index e786b121c..206d22251 100644 --- a/include/mapnik/json/positions_grammar_x3_def.hpp +++ b/include/mapnik/json/positions_grammar_x3_def.hpp @@ -43,7 +43,7 @@ auto assign_helper = [](auto const& ctx) } // anonymous ns // start rule -positions_grammar_type const positions MAPNIK_INIT_PRIORITY(104) ("Positions"); +positions_grammar_type const positions MAPNIK_INIT_PRIORITY(103) ("Positions"); // rules x3::rule const point("Position"); x3::rule const ring("Ring"); diff --git a/include/mapnik/json/topojson_grammar_x3_def.hpp b/include/mapnik/json/topojson_grammar_x3_def.hpp index 49d7ee5d7..54ba23963 100644 --- a/include/mapnik/json/topojson_grammar_x3_def.hpp +++ b/include/mapnik/json/topojson_grammar_x3_def.hpp @@ -306,7 +306,7 @@ struct topojson_geometry_type_ : x3::symbols } topojson_geometry_type; // start rule -topojson_grammar_type const topology MAPNIK_INIT_PRIORITY(105) ("Topology"); +topojson_grammar_type const topology MAPNIK_INIT_PRIORITY(104) ("Topology"); // rules x3::rule const transform = "Transform"; x3::rule const bbox = "Bounding Box"; diff --git a/include/mapnik/json/unicode_string_grammar_x3.hpp b/include/mapnik/json/unicode_string_grammar_x3.hpp index 740b13613..9cc9f3f34 100644 --- a/include/mapnik/json/unicode_string_grammar_x3.hpp +++ b/include/mapnik/json/unicode_string_grammar_x3.hpp @@ -33,15 +33,12 @@ namespace mapnik { namespace json { namespace grammar { namespace x3 = boost::spirit::x3; using unicode_string_grammar_type = x3::rule; -using escaped_unicode_type = x3::rule; BOOST_SPIRIT_DECLARE(unicode_string_grammar_type); -BOOST_SPIRIT_DECLARE(escaped_unicode_type); } grammar::unicode_string_grammar_type const& unicode_string_grammar(); -grammar::escaped_unicode_type const& escaped_unicode_grammar(); }} diff --git a/include/mapnik/json/unicode_string_grammar_x3_def.hpp b/include/mapnik/json/unicode_string_grammar_x3_def.hpp index 2a6b1b2da..2335b657b 100644 --- a/include/mapnik/json/unicode_string_grammar_x3_def.hpp +++ b/include/mapnik/json/unicode_string_grammar_x3_def.hpp @@ -110,11 +110,11 @@ x3::uint_parser const hex4 {}; x3::uint_parser const hex8 {}; // start rule -unicode_string_grammar_type const unicode_string MAPNIK_INIT_PRIORITY(102) ("Unicode String"); +unicode_string_grammar_type const unicode_string MAPNIK_INIT_PRIORITY(101) ("Unicode String"); // rules x3::rule const double_quoted("Double-quoted string"); x3::rule const escaped("Escaped Character"); -escaped_unicode_type const escaped_unicode MAPNIK_INIT_PRIORITY(101) ("Escaped Unicode code point(s)"); +x3::rule const escaped_unicode("Escaped Unicode code point(s)"); x3::rule> const utf16_string("UTF16 encoded string"); auto unicode_string_def = double_quoted diff --git a/src/expression_grammar_x3.cpp b/src/expression_grammar_x3.cpp index c072ad75d..24f43f8a5 100644 --- a/src/expression_grammar_x3.cpp +++ b/src/expression_grammar_x3.cpp @@ -22,19 +22,9 @@ #include #include -#include - -namespace mapnik { namespace json { namespace grammar { -BOOST_SPIRIT_INSTANTIATE(unicode_string_grammar_type, mapnik::grammar::iterator_type, mapnik::grammar::context_type); -} - -grammar::escaped_unicode_type const& escaped_unicode_grammar() -{ - return grammar::escaped_unicode; -} - -}} namespace mapnik { namespace grammar { + BOOST_SPIRIT_INSTANTIATE(expression_grammar_type, iterator_type, context_type); + }} From ac50139c54419504894ccf1d38b64d2cb3764685 Mon Sep 17 00:00:00 2001 From: Jiri Drbalek Date: Fri, 23 Mar 2018 11:55:06 +0000 Subject: [PATCH 2/5] Raster colorizer: check image bounds --- src/raster_colorizer.cpp | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/raster_colorizer.cpp b/src/raster_colorizer.cpp index b30221e27..237dd9329 100644 --- a/src/raster_colorizer.cpp +++ b/src/raster_colorizer.cpp @@ -129,20 +129,25 @@ void raster_colorizer::colorize(image_rgba8 & out, T const& in, { using image_type = T; using pixel_type = typename image_type::pixel_type; - // TODO: assuming in/out have the same width/height for now - std::uint32_t * out_data = out.data(); - pixel_type const* in_data = in.data(); - int len = out.width() * out.height(); - for (int i=0; i Date: Thu, 22 Mar 2018 15:50:41 +0100 Subject: [PATCH 3/5] Fix build issue with MAPNIK_THREADSAFE disabled --- plugins/input/gdal/gdal_datasource.cpp | 2 ++ plugins/input/ogr/ogr_datasource.cpp | 1 + 2 files changed, 3 insertions(+) diff --git a/plugins/input/gdal/gdal_datasource.cpp b/plugins/input/gdal/gdal_datasource.cpp index 1ac6557fe..a5a6fc535 100644 --- a/plugins/input/gdal/gdal_datasource.cpp +++ b/plugins/input/gdal/gdal_datasource.cpp @@ -32,6 +32,8 @@ #include +#include + using mapnik::datasource; using mapnik::parameters; diff --git a/plugins/input/ogr/ogr_datasource.cpp b/plugins/input/ogr/ogr_datasource.cpp index 4c0cb5902..e9433d969 100644 --- a/plugins/input/ogr/ogr_datasource.cpp +++ b/plugins/input/ogr/ogr_datasource.cpp @@ -41,6 +41,7 @@ // stl #include +#include #include #include From 25e4bb3f6ca28f7c15e880530ea9b103005b9d7f Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Tue, 20 Mar 2018 17:01:32 -0400 Subject: [PATCH 4/5] A fix for two distinct issues associated with gdal featuresets, the first is overviews were not properly being utilized based on the resolution of the final image requested. The second is that allocation of far too much memory could be possible in GDAL to attempt to do resampling internally in mapnik. This now has a hard cap so that we allocate less memory in these situations but are still able to resample internally in mapnik. --- include/mapnik/image.hpp | 2 +- include/mapnik/image_impl.hpp | 6 +- plugins/input/gdal/gdal_featureset.cpp | 140 +++++++++++++++++++++++-- src/image.cpp | 2 +- test/data | 2 +- test/data-visual | 2 +- 6 files changed, 140 insertions(+), 14 deletions(-) diff --git a/include/mapnik/image.hpp b/include/mapnik/image.hpp index 6f7b6d730..2cc952681 100644 --- a/include/mapnik/image.hpp +++ b/include/mapnik/image.hpp @@ -74,7 +74,7 @@ public: static constexpr image_dtype dtype = T::id; static constexpr std::size_t pixel_size = sizeof(pixel_type); private: - detail::image_dimensions<65535> dimensions_; + detail::image_dimensions<4294836225> dimensions_; detail::buffer buffer_; double offset_; double scaling_; diff --git a/include/mapnik/image_impl.hpp b/include/mapnik/image_impl.hpp index 002cef08c..856bc6990 100644 --- a/include/mapnik/image_impl.hpp +++ b/include/mapnik/image_impl.hpp @@ -39,8 +39,10 @@ image_dimensions::image_dimensions(int width, int height) : width_(width), height_(height) { - if (width < 0 || static_cast(width) > max_size) throw std::runtime_error("Invalid width for image dimensions requested"); - if (height < 0 || static_cast(height) > max_size) throw std::runtime_error("Invalid height for image dimensions requested"); + int64_t area = (int64_t)width * (int64_t)height; + if (width < 0) throw std::runtime_error("Invalid width for image dimensions requested"); + if (height < 0) throw std::runtime_error("Invalid height for image dimensions requested"); + if (area > max_size) throw std::runtime_error("Image area too large based on image dimensions"); } template diff --git a/plugins/input/gdal/gdal_featureset.cpp b/plugins/input/gdal/gdal_featureset.cpp index eb3c78d06..4363e2f93 100644 --- a/plugins/input/gdal/gdal_featureset.cpp +++ b/plugins/input/gdal/gdal_featureset.cpp @@ -153,6 +153,8 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q) //size of resized output pixel in source image domain double margin_x = 1.0 / (std::fabs(dx_) * std::get<0>(q.resolution())); double margin_y = 1.0 / (std::fabs(dy_) * std::get<1>(q.resolution())); + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: margin_x=" << margin_x; + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: margin_y=" << margin_y; if (margin_x < 1) { margin_x = 1.0; @@ -167,6 +169,10 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q) int y_off = rint(box.miny() - margin_y); int end_x = rint(box.maxx() + margin_x); int end_y = rint(box.maxy() + margin_y); + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: x_off=" << x_off; + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: y_off=" << y_off; + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: end_x=" << end_x; + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: end_y=" << end_y; //clip to available data if (x_off < 0) @@ -185,21 +191,139 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q) { end_y = raster_height_; } + + // width and height of the portion of the source image we are requesting int width = end_x - x_off; int height = end_y - y_off; + // In many cases we want GDAL to simply return the exact image so we + // can handle resampling internally in mapnik. In other cases such as + // when overviews exist or when the image allocated might be too large + // we want to utilize some resampling in GDAL instead. + int im_height = height; + int im_width = width; + double im_offset_x = x_off; + double im_offset_y = y_off; + int current_width = (int)raster_width_; + int current_height = (int)raster_height_; + + // loop through overviews -- snap up in resolution to closest overview if necessary + // we find an image size that most resembles the resolution of our output image. + double width_res = std::get<0>(q.resolution()); + double height_res = std::get<1>(q.resolution()); + int res_adjusted_raster_width = static_cast(std::floor(((double)raster_width_ * width_res) + .5)); + int res_adjusted_raster_height = static_cast(std::floor(((double)raster_height_ * height_res) + .5)); + if (band_ > 0 && band_ < nbands_) + { + GDALRasterBand * band = dataset_.GetRasterBand(band_); + int band_overviews = band->GetOverviewCount(); + if (band_overviews > 0) + { + for (int b = 0; b < band_overviews; b++) + { + GDALRasterBand * overview = band->GetOverview(b); + int overview_width = overview->GetXSize(); + int overview_height = overview->GetYSize(); + if ((overview_width < current_width || overview_height < current_height) && + res_adjusted_raster_width <= overview_width && + res_adjusted_raster_height <= overview_height) + { + current_width = overview_width; + current_height = overview_height; + } + } + } + } + else + { + for (int i = 0; i < nbands_; ++i) + { + GDALRasterBand * band = dataset_.GetRasterBand(i + 1); + int band_overviews = band->GetOverviewCount(); + if (band_overviews > 0) + { + for (int b = 0; b < band_overviews; b++) + { + GDALRasterBand * overview = band->GetOverview(b); + int overview_width = overview->GetXSize(); + int overview_height = overview->GetYSize(); + if ((overview_width < current_width || overview_height < current_height) && + res_adjusted_raster_width <= overview_width && + res_adjusted_raster_height <= overview_height) + { + current_width = overview_width; + current_height = overview_height; + } + } + } + } + } + if (current_width != (int)raster_width_ || current_height != (int)raster_height_) + { + if (current_width != (int)raster_width_) + { + double ratio = (double)current_width / (double)raster_width_; + int adjusted_width = static_cast(std::floor((ratio * im_width) + 0.5)); + double adjusted_ratio = (double)adjusted_width / (double)im_width; + im_offset_x = adjusted_ratio * im_offset_x; + im_width = adjusted_width; + } + if (current_height != (int)raster_height_) + { + double ratio = (double)current_height / (double)raster_height_; + int adjusted_height = static_cast(std::floor((ratio * im_height) + 0.5)); + double adjusted_ratio = (double)adjusted_height / (double)im_height; + im_offset_y = adjusted_ratio * im_offset_y; + im_height = adjusted_height; + } + } + + // Maximum memory limitation for image will be simply based on the maximum + // area we allow for an image. The true memory footprint therefore will vary based + // on the type of imagery that exists. + // max_im_area based on 50 mb limit for RGBA + constexpr int64_t max_im_area = (50*1024*1024) / 4; + int64_t im_area = (int64_t)im_width * (int64_t)im_height; + if (im_area > max_im_area) + { + int adjusted_width = static_cast(std::round(std::sqrt(max_im_area * ((double)im_width / (double)im_height)))); + int adjusted_height = static_cast(std::round(std::sqrt(max_im_area * ((double)im_height / (double)im_width)))); + if (adjusted_width < 1) + { + adjusted_width = 1; + } + if (adjusted_height < 1) + { + adjusted_height = 1; + } + double ratio_x = (double)adjusted_width / (double)im_width; + double ratio_y = (double)adjusted_height / (double)im_height; + im_offset_x = ratio_x * im_offset_x; + im_offset_y = ratio_y * im_offset_y; + im_width = adjusted_width; + im_height = adjusted_height; + current_width = static_cast(std::floor((ratio_x * current_width) + 0.5)); + current_height = static_cast(std::floor((ratio_y * current_height) + 0.5)); + } + //calculate actual box2d of returned raster - box2d feature_raster_extent(x_off, y_off, x_off + width, y_off + height); - feature_raster_extent = t.backward(feature_raster_extent); + view_transform t2(current_width, current_height, raster_extent_, 0, 0); + box2d feature_raster_extent(im_offset_x, im_offset_y, im_offset_x + im_width, im_offset_y + im_height); + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Feature Raster extent=" << feature_raster_extent; + feature_raster_extent = t2.backward(feature_raster_extent); MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Raster extent=" << raster_extent_; + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Feature Raster extent=" << feature_raster_extent; MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: View extent=" << intersect; MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Query resolution=" << std::get<0>(q.resolution()) << "," << std::get<1>(q.resolution()); MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: StartX=" << x_off << " StartY=" << y_off << " Width=" << width << " Height=" << height; + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: IM StartX=" << im_offset_x << " StartY=" << im_offset_y << " Width=" << im_width << " Height=" << im_height; if (width > 0 && height > 0) { - MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Image Size=(" << width << "," << height << ")"; + + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Requested Image Size=(" << width << "," << height << ")"; + MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Image Size=(" << im_width << "," << im_height << ")"; MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Reading band=" << band_; if (band_ > 0) // we are querying a single band { @@ -215,7 +339,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q) { case GDT_Byte: { - mapnik::image_gray8 image(width, height); + mapnik::image_gray8 image(im_width, im_height); image.set(std::numeric_limits::max()); raster_nodata = band->GetNoDataValue(&raster_has_nodata); raster_io_error = band->RasterIO(GF_Read, x_off, y_off, width, height, @@ -235,7 +359,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q) case GDT_Float64: case GDT_Float32: { - mapnik::image_gray32f image(width, height); + mapnik::image_gray32f image(im_width, im_height); image.set(std::numeric_limits::max()); raster_nodata = band->GetNoDataValue(&raster_has_nodata); raster_io_error = band->RasterIO(GF_Read, x_off, y_off, width, height, @@ -254,7 +378,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q) } case GDT_UInt16: { - mapnik::image_gray16 image(width, height); + mapnik::image_gray16 image(im_width, im_height); image.set(std::numeric_limits::max()); raster_nodata = band->GetNoDataValue(&raster_has_nodata); raster_io_error = band->RasterIO(GF_Read, x_off, y_off, width, height, @@ -274,7 +398,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q) default: case GDT_Int16: { - mapnik::image_gray16s image(width, height); + mapnik::image_gray16s image(im_width, im_height); image.set(std::numeric_limits::max()); raster_nodata = band->GetNoDataValue(&raster_has_nodata); raster_io_error = band->RasterIO(GF_Read, x_off, y_off, width, height, @@ -295,7 +419,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q) } else // working with all bands { - mapnik::image_rgba8 image(width, height); + mapnik::image_rgba8 image(im_width, im_height); image.set(std::numeric_limits::max()); for (int i = 0; i < nbands_; ++i) { diff --git a/src/image.cpp b/src/image.cpp index 8eceb1da9..d1aca683f 100644 --- a/src/image.cpp +++ b/src/image.cpp @@ -84,7 +84,7 @@ void buffer::swap(buffer & rhs) std::swap(owns_, rhs.owns_); } -template struct MAPNIK_DECL image_dimensions<65535>; +template struct MAPNIK_DECL image_dimensions<4294836225>; } // end ns detail diff --git a/test/data b/test/data index 96d52a79d..3a636431a 160000 --- a/test/data +++ b/test/data @@ -1 +1 @@ -Subproject commit 96d52a79df7e20c5543321baa177811dc72bc93a +Subproject commit 3a636431ab189285378686c596be16a440c0808f diff --git a/test/data-visual b/test/data-visual index d374baac6..a6ec8c091 160000 --- a/test/data-visual +++ b/test/data-visual @@ -1 +1 @@ -Subproject commit d374baac6cd72e7de665a27ded9b129e92661e18 +Subproject commit a6ec8c0919eda8ac2f34ede5c4a53240d0bb275a From a217b38fd5873e977d3698b2b05e84c900fe9719 Mon Sep 17 00:00:00 2001 From: Blake Thompson Date: Fri, 30 Mar 2018 15:40:30 -0500 Subject: [PATCH 5/5] Updated to use max_image_area as a datasource parameter for GDAL plugin --- plugins/input/gdal/gdal_datasource.cpp | 13 +++++++++++-- plugins/input/gdal/gdal_datasource.hpp | 1 + plugins/input/gdal/gdal_featureset.cpp | 15 ++++++--------- plugins/input/gdal/gdal_featureset.hpp | 4 +++- test/data | 2 +- test/data-visual | 2 +- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/plugins/input/gdal/gdal_datasource.cpp b/plugins/input/gdal/gdal_datasource.cpp index a5a6fc535..f47c26331 100644 --- a/plugins/input/gdal/gdal_datasource.cpp +++ b/plugins/input/gdal/gdal_datasource.cpp @@ -84,6 +84,13 @@ gdal_datasource::gdal_datasource(parameters const& params) shared_dataset_ = *params.get("shared", false); band_ = *params.get("band", -1); + + // Maximum memory limitation for image will be simply based on the maximum + // area we allow for an image. The true memory footprint therefore will vary based + // on the type of imagery that exists. This is not the maximum size of an image + // on disk but rather the maximum size we will load into mapnik from GDAL. + // max_im_area based on 50 mb limit for RGBA + max_image_area_ = *params.get("max_image_area", (50*1024*1024) / 4); #if GDAL_VERSION_NUM >= 1600 if (shared_dataset_) @@ -237,7 +244,8 @@ featureset_ptr gdal_datasource::features(query const& q) const dx_, dy_, nodata_value_, - nodata_tolerance_); + nodata_tolerance_, + max_image_area_); } featureset_ptr gdal_datasource::features_at_point(coord2d const& pt, double tol) const @@ -256,5 +264,6 @@ featureset_ptr gdal_datasource::features_at_point(coord2d const& pt, double tol) dx_, dy_, nodata_value_, - nodata_tolerance_); + nodata_tolerance_, + max_image_area_); } diff --git a/plugins/input/gdal/gdal_datasource.hpp b/plugins/input/gdal/gdal_datasource.hpp index 7fc74975b..bbd2f7607 100644 --- a/plugins/input/gdal/gdal_datasource.hpp +++ b/plugins/input/gdal/gdal_datasource.hpp @@ -68,6 +68,7 @@ private: bool shared_dataset_; boost::optional nodata_value_; double nodata_tolerance_; + int64_t max_image_area_; }; #endif // GDAL_DATASOURCE_HPP diff --git a/plugins/input/gdal/gdal_featureset.cpp b/plugins/input/gdal/gdal_featureset.cpp index 4363e2f93..7ec336f55 100644 --- a/plugins/input/gdal/gdal_featureset.cpp +++ b/plugins/input/gdal/gdal_featureset.cpp @@ -89,7 +89,8 @@ gdal_featureset::gdal_featureset(GDALDataset& dataset, double dx, double dy, boost::optional const& nodata, - double nodata_tolerance) + double nodata_tolerance, + int64_t max_image_area) : dataset_(dataset), ctx_(std::make_shared()), band_(band), @@ -102,6 +103,7 @@ gdal_featureset::gdal_featureset(GDALDataset& dataset, nbands_(nbands), nodata_value_(nodata), nodata_tolerance_(nodata_tolerance), + max_image_area_(max_image_area), first_(true) { ctx_->push("nodata"); @@ -278,16 +280,11 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q) } } - // Maximum memory limitation for image will be simply based on the maximum - // area we allow for an image. The true memory footprint therefore will vary based - // on the type of imagery that exists. - // max_im_area based on 50 mb limit for RGBA - constexpr int64_t max_im_area = (50*1024*1024) / 4; int64_t im_area = (int64_t)im_width * (int64_t)im_height; - if (im_area > max_im_area) + if (im_area > max_image_area_) { - int adjusted_width = static_cast(std::round(std::sqrt(max_im_area * ((double)im_width / (double)im_height)))); - int adjusted_height = static_cast(std::round(std::sqrt(max_im_area * ((double)im_height / (double)im_width)))); + int adjusted_width = static_cast(std::round(std::sqrt(max_image_area_ * ((double)im_width / (double)im_height)))); + int adjusted_height = static_cast(std::round(std::sqrt(max_image_area_ * ((double)im_height / (double)im_width)))); if (adjusted_width < 1) { adjusted_width = 1; diff --git a/plugins/input/gdal/gdal_featureset.hpp b/plugins/input/gdal/gdal_featureset.hpp index beabb7df1..325c6a999 100644 --- a/plugins/input/gdal/gdal_featureset.hpp +++ b/plugins/input/gdal/gdal_featureset.hpp @@ -66,7 +66,8 @@ public: double dx, double dy, boost::optional const& nodata, - double nodata_tolerance); + double nodata_tolerance, + int64_t max_image_area); virtual ~gdal_featureset(); mapnik::feature_ptr next(); @@ -85,6 +86,7 @@ private: int nbands_; boost::optional nodata_value_; double nodata_tolerance_; + int64_t max_image_area_; bool first_; }; diff --git a/test/data b/test/data index 3a636431a..a44457cc0 160000 --- a/test/data +++ b/test/data @@ -1 +1 @@ -Subproject commit 3a636431ab189285378686c596be16a440c0808f +Subproject commit a44457cc063249e0d204cde7249d37706bf8844c diff --git a/test/data-visual b/test/data-visual index a6ec8c091..c113ce132 160000 --- a/test/data-visual +++ b/test/data-visual @@ -1 +1 @@ -Subproject commit a6ec8c0919eda8ac2f34ede5c4a53240d0bb275a +Subproject commit c113ce13267124332cc2ecd049d7d2d7397f9a51