sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #3872 from mapnik/gdal_overview_fix_v3.0.x

Browse source 
GDAL Driver Overview Fix and Memory Reduction
This commit is contained in:
Blake Thompson 2018-03-30 15:44:07 -05:00 committed by GitHub
commit d4f95642ca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 170 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
include/mapnik/image.hpp
View file

@ -74,7 +74,7 @@ public:
static constexpr image_dtype dtype = T::id;
static constexpr std::size_t pixel_size = sizeof(pixel_type);
private:
detail::image_dimensions<65535> dimensions_;
detail::image_dimensions<4294836225> dimensions_;
detail::buffer buffer_;
double offset_;
double scaling_;

6
include/mapnik/image_impl.hpp
View file

@ -39,8 +39,10 @@ image_dimensions<max_size>::image_dimensions(int width, int height)
: width_(width),
height_(height)
{
if (width < 0 || static_cast<std::size_t>(width) > max_size) throw std::runtime_error("Invalid width for image dimensions requested");
if (height < 0 || static_cast<std::size_t>(height) > max_size) throw std::runtime_error("Invalid height for image dimensions requested");
int64_t area = (int64_t)width * (int64_t)height;
if (width < 0) throw std::runtime_error("Invalid width for image dimensions requested");
if (height < 0) throw std::runtime_error("Invalid height for image dimensions requested");
if (area > max_size) throw std::runtime_error("Image area too large based on image dimensions");
}
template <std::size_t max_size>

13
plugins/input/gdal/gdal_datasource.cpp
View file

@ -83,6 +83,13 @@ gdal_datasource::gdal_datasource(parameters const& params)
shared_dataset_ = *params.get<mapnik::boolean_type>("shared", false);
band_ = *params.get<mapnik::value_integer>("band", -1);
// Maximum memory limitation for image will be simply based on the maximum
// area we allow for an image. The true memory footprint therefore will vary based
// on the type of imagery that exists. This is not the maximum size of an image
// on disk but rather the maximum size we will load into mapnik from GDAL.
// max_im_area based on 50 mb limit for RGBA
max_image_area_ = *params.get<mapnik::value_integer>("max_image_area", (50*1024*1024) / 4);
#if GDAL_VERSION_NUM >= 1600
if (shared_dataset_)
{
@ -235,7 +242,8 @@ featureset_ptr gdal_datasource::features(query const& q) const
dx_,
dy_,
nodata_value_,
nodata_tolerance_);
nodata_tolerance_,
max_image_area_);
}
featureset_ptr gdal_datasource::features_at_point(coord2d const& pt, double tol) const
@ -254,5 +262,6 @@ featureset_ptr gdal_datasource::features_at_point(coord2d const& pt, double tol)
dx_,
dy_,
nodata_value_,
nodata_tolerance_);
nodata_tolerance_,
max_image_area_);
}

1
plugins/input/gdal/gdal_datasource.hpp
View file

@ -68,6 +68,7 @@ private:
bool shared_dataset_;
boost::optional<double> nodata_value_;
double nodata_tolerance_;
int64_t max_image_area_;
};
#endif // GDAL_DATASOURCE_HPP

139
plugins/input/gdal/gdal_featureset.cpp
View file

@ -89,7 +89,8 @@ gdal_featureset::gdal_featureset(GDALDataset& dataset,
double dx,
double dy,
boost::optional<double> const& nodata,
double nodata_tolerance)
double nodata_tolerance,
int64_t max_image_area)
: dataset_(dataset),
ctx_(std::make_shared<mapnik::context_type>()),
band_(band),
@ -102,6 +103,7 @@ gdal_featureset::gdal_featureset(GDALDataset& dataset,
nbands_(nbands),
nodata_value_(nodata),
nodata_tolerance_(nodata_tolerance),
max_image_area_(max_image_area),
first_(true)
{
ctx_->push("nodata");
@ -155,6 +157,8 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
//size of resized output pixel in source image domain
double margin_x = 1.0 / (std::fabs(dx_) * std::get<0>(q.resolution()));
double margin_y = 1.0 / (std::fabs(dy_) * std::get<1>(q.resolution()));
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: margin_x=" << margin_x;
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: margin_y=" << margin_y;
if (margin_x < 1)
{
margin_x = 1.0;
@ -169,6 +173,10 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
int y_off = rint(box.miny() - margin_y);
int end_x = rint(box.maxx() + margin_x);
int end_y = rint(box.maxy() + margin_y);
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: x_off=" << x_off;
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: y_off=" << y_off;
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: end_x=" << end_x;
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: end_y=" << end_y;
//clip to available data
if (x_off < 0)
@ -187,21 +195,134 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
{
end_y = raster_height_;
}
// width and height of the portion of the source image we are requesting
int width = end_x - x_off;
int height = end_y - y_off;
// In many cases we want GDAL to simply return the exact image so we
// can handle resampling internally in mapnik. In other cases such as
// when overviews exist or when the image allocated might be too large
// we want to utilize some resampling in GDAL instead.
int im_height = height;
int im_width = width;
double im_offset_x = x_off;
double im_offset_y = y_off;
int current_width = (int)raster_width_;
int current_height = (int)raster_height_;
// loop through overviews -- snap up in resolution to closest overview if necessary
// we find an image size that most resembles the resolution of our output image.
double width_res = std::get<0>(q.resolution());
double height_res = std::get<1>(q.resolution());
int res_adjusted_raster_width = static_cast<int>(std::floor(((double)raster_width_ * width_res) + .5));
int res_adjusted_raster_height = static_cast<int>(std::floor(((double)raster_height_ * height_res) + .5));
if (band_ > 0 && band_ < nbands_)
{
GDALRasterBand * band = dataset_.GetRasterBand(band_);
int band_overviews = band->GetOverviewCount();
if (band_overviews > 0)
{
for (int b = 0; b < band_overviews; b++)
{
GDALRasterBand * overview = band->GetOverview(b);
int overview_width = overview->GetXSize();
int overview_height = overview->GetYSize();
if ((overview_width < current_width || overview_height < current_height) &&
res_adjusted_raster_width <= overview_width &&
res_adjusted_raster_height <= overview_height)
{
current_width = overview_width;
current_height = overview_height;
}
}
}
}
else
{
for (int i = 0; i < nbands_; ++i)
{
GDALRasterBand * band = dataset_.GetRasterBand(i + 1);
int band_overviews = band->GetOverviewCount();
if (band_overviews > 0)
{
for (int b = 0; b < band_overviews; b++)
{
GDALRasterBand * overview = band->GetOverview(b);
int overview_width = overview->GetXSize();
int overview_height = overview->GetYSize();
if ((overview_width < current_width || overview_height < current_height) &&
res_adjusted_raster_width <= overview_width &&
res_adjusted_raster_height <= overview_height)
{
current_width = overview_width;
current_height = overview_height;
}
}
}
}
}
if (current_width != (int)raster_width_ || current_height != (int)raster_height_)
{
if (current_width != (int)raster_width_)
{
double ratio = (double)current_width / (double)raster_width_;
int adjusted_width = static_cast<int>(std::floor((ratio * im_width) + 0.5));
double adjusted_ratio = (double)adjusted_width / (double)im_width;
im_offset_x = adjusted_ratio * im_offset_x;
im_width = adjusted_width;
}
if (current_height != (int)raster_height_)
{
double ratio = (double)current_height / (double)raster_height_;
int adjusted_height = static_cast<int>(std::floor((ratio * im_height) + 0.5));
double adjusted_ratio = (double)adjusted_height / (double)im_height;
im_offset_y = adjusted_ratio * im_offset_y;
im_height = adjusted_height;
}
}
int64_t im_area = (int64_t)im_width * (int64_t)im_height;
if (im_area > max_image_area_)
{
int adjusted_width = static_cast<int>(std::round(std::sqrt(max_image_area_ * ((double)im_width / (double)im_height))));
int adjusted_height = static_cast<int>(std::round(std::sqrt(max_image_area_ * ((double)im_height / (double)im_width))));
if (adjusted_width < 1)
{
adjusted_width = 1;
}
if (adjusted_height < 1)
{
adjusted_height = 1;
}
double ratio_x = (double)adjusted_width / (double)im_width;
double ratio_y = (double)adjusted_height / (double)im_height;
im_offset_x = ratio_x * im_offset_x;
im_offset_y = ratio_y * im_offset_y;
im_width = adjusted_width;
im_height = adjusted_height;
current_width = static_cast<int>(std::floor((ratio_x * current_width) + 0.5));
current_height = static_cast<int>(std::floor((ratio_y * current_height) + 0.5));
}
//calculate actual box2d of returned raster
box2d<double> feature_raster_extent(x_off, y_off, x_off + width, y_off + height);
feature_raster_extent = t.backward(feature_raster_extent);
view_transform t2(current_width, current_height, raster_extent_, 0, 0);
box2d<double> feature_raster_extent(im_offset_x, im_offset_y, im_offset_x + im_width, im_offset_y + im_height);
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Feature Raster extent=" << feature_raster_extent;
feature_raster_extent = t2.backward(feature_raster_extent);
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Raster extent=" << raster_extent_;
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Feature Raster extent=" << feature_raster_extent;
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: View extent=" << intersect;
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Query resolution=" << std::get<0>(q.resolution()) << "," << std::get<1>(q.resolution());
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: StartX=" << x_off << " StartY=" << y_off << " Width=" << width << " Height=" << height;
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: IM StartX=" << im_offset_x << " StartY=" << im_offset_y << " Width=" << im_width << " Height=" << im_height;
if (width > 0 && height > 0)
{
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Image Size=(" << width << "," << height << ")";
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Requested Image Size=(" << width << "," << height << ")";
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Image Size=(" << im_width << "," << im_height << ")";
MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Reading band=" << band_;
if (band_ > 0) // we are querying a single band
{
@ -217,7 +338,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
{
case GDT_Byte:
{
mapnik::image_gray8 image(width, height);
mapnik::image_gray8 image(im_width, im_height);
image.set(std::numeric_limits<std::uint8_t>::max());
raster_nodata = band->GetNoDataValue(&raster_has_nodata);
raster_io_error = band->RasterIO(GF_Read, x_off, y_off, width, height,
@ -237,7 +358,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
case GDT_Float64:
case GDT_Float32:
{
mapnik::image_gray32f image(width, height);
mapnik::image_gray32f image(im_width, im_height);
image.set(std::numeric_limits<float>::max());
raster_nodata = band->GetNoDataValue(&raster_has_nodata);
raster_io_error = band->RasterIO(GF_Read, x_off, y_off, width, height,
@ -256,7 +377,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
}
case GDT_UInt16:
{
mapnik::image_gray16 image(width, height);
mapnik::image_gray16 image(im_width, im_height);
image.set(std::numeric_limits<std::uint16_t>::max());
raster_nodata = band->GetNoDataValue(&raster_has_nodata);
raster_io_error = band->RasterIO(GF_Read, x_off, y_off, width, height,
@ -276,7 +397,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
default:
case GDT_Int16:
{
mapnik::image_gray16s image(width, height);
mapnik::image_gray16s image(im_width, im_height);
image.set(std::numeric_limits<std::int16_t>::max());
raster_nodata = band->GetNoDataValue(&raster_has_nodata);
raster_io_error = band->RasterIO(GF_Read, x_off, y_off, width, height,
@ -297,7 +418,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
}
else // working with all bands
{
mapnik::image_rgba8 image(width, height);
mapnik::image_rgba8 image(im_width, im_height);
image.set(std::numeric_limits<std::uint32_t>::max());
for (int i = 0; i < nbands_; ++i)
{

4
plugins/input/gdal/gdal_featureset.hpp
View file

@ -66,7 +66,8 @@ public:
double dx,
double dy,
boost::optional<double> const& nodata,
double nodata_tolerance);
double nodata_tolerance,
int64_t max_image_area);
virtual ~gdal_featureset();
mapnik::feature_ptr next();
@ -85,6 +86,7 @@ private:
int nbands_;
boost::optional<double> nodata_value_;
double nodata_tolerance_;
int64_t max_image_area_;
bool first_;
};

2
src/image.cpp
View file

@ -84,7 +84,7 @@ void buffer::swap(buffer & rhs)
std::swap(owns_, rhs.owns_);
}
template struct MAPNIK_DECL image_dimensions<65535>;
template struct MAPNIK_DECL image_dimensions<4294836225>;
} // end ns detail

23
src/raster_colorizer.cpp
View file

@ -129,20 +129,25 @@ void raster_colorizer::colorize(image_rgba8 & out, T const& in,
{
using image_type = T;
using pixel_type = typename image_type::pixel_type;
// TODO: assuming in/out have the same width/height for now
std::uint32_t * out_data = out.data();
pixel_type const* in_data = in.data();
int len = out.width() * out.height();
for (int i=0; i<len; ++i)
const std::size_t width = std::min(in.width(), out.width());
const std::size_t height = std::min(in.height(), out.height());
for (std::size_t y = 0; y < height; ++y)
{
pixel_type value = in_data[i];
if (nodata && (std::fabs(value - *nodata) < epsilon_))
pixel_type const * in_row = in.get_row(y);
image_rgba8::pixel_type * out_row = out.get_row(y);
for (std::size_t x = 0; x < width; ++x)
{
out_data[i] = 0; // rgba(0,0,0,0)
pixel_type val = in_row[x];
if (nodata && (std::fabs(val - *nodata) < epsilon_))
{
out_row[x] = 0; // rgba(0,0,0,0)
}
else
{
out_data[i] = get_color(value);
out_row[x] = get_color(val);
}
}
}
}

2
test/data

@ -1 +1 @@
Subproject commit da98ee10eb7ac8cd6d3b1441c3470dc5d3748c6e
Subproject commit 18f610c211efa95c9d689e350d7ebe635d9c0b36

2
test/data-visual

@ -1 +1 @@
Subproject commit 9e0887293e7f6195e1eecc6fed13ecb64e300259
Subproject commit 6ece97e8d43915527b4bd83507a9cd9ea9ef910e