From 26052486ee644e7f643bdf46d62be2b87d33b4cf Mon Sep 17 00:00:00 2001 From: Mathis Logemann Date: Thu, 20 Jan 2022 22:52:43 +0100 Subject: [PATCH 1/9] remove cxx11_support.hpp --- include/mapnik/cxx11_support.hpp | 43 -------------------------------- include/mapnik/value/types.hpp | 11 ++++---- 2 files changed, 5 insertions(+), 49 deletions(-) delete mode 100644 include/mapnik/cxx11_support.hpp diff --git a/include/mapnik/cxx11_support.hpp b/include/mapnik/cxx11_support.hpp deleted file mode 100644 index f298a2074..000000000 --- a/include/mapnik/cxx11_support.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/***************************************************************************** - * - * This file is part of Mapnik (c++ mapping toolkit) - * - * Copyright (C) 2021 Artem Pavlenko - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *****************************************************************************/ - -#ifndef MAPNIK_CXX11_SUPPORT_HPP -#define MAPNIK_CXX11_SUPPORT_HPP - -#include - -namespace mapnik { -namespace detail { - -template -using conditional_t = typename std::conditional::type; - -template -using decay_t = typename std::decay::type; - -template -using enable_if_t = typename std::enable_if::type; - -} // namespace detail -} // namespace mapnik - -#endif // MAPNIK_CXX11_SUPPORT_HPP diff --git a/include/mapnik/value/types.hpp b/include/mapnik/value/types.hpp index a69c01fdb..39932125b 100644 --- a/include/mapnik/value/types.hpp +++ b/include/mapnik/value/types.hpp @@ -25,9 +25,8 @@ // mapnik #include -#include #include - +#include #include MAPNIK_DISABLE_WARNING_PUSH @@ -161,13 +160,13 @@ namespace detail { // value_double if T is a floating-point type // T && otherwise -template > +template > using mapnik_value_type_t = - conditional_t< + std::conditional_t< std::is_same::value, value_bool, - conditional_t< + std::conditional_t< std::is_integral
::value, value_integer, - conditional_t< + std::conditional_t< std::is_floating_point
::value, value_double, T && >>>; From c5d4bfe680db86be62a43537805f62a02ce80195 Mon Sep 17 00:00:00 2001 From: Mathis Logemann Date: Thu, 27 Jan 2022 18:28:44 +0100 Subject: [PATCH 2/9] [CMake] add msvc utf-8 flag --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a9be165f0..f2dc1f02d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -344,6 +344,11 @@ if(NOT WIN32) list(APPEND MAPNIK_OPTIONAL_LIBS ${CMAKE_DL_LIBS}) endif() +# force utf-8 source code processing +# see https://docs.microsoft.com/de-de/cpp/build/reference/utf-8-set-source-and-executable-character-sets-to-utf-8?view=msvc-170 +add_compile_options("$<$:/utf-8>") +add_compile_options("$<$:/utf-8>") + add_library(core INTERFACE) add_library(mapnik::core ALIAS core) From ccd44da4a2fc7d0189b276887931649b4c4f67c2 Mon Sep 17 00:00:00 2001 From: Mathis Logemann Date: Thu, 27 Jan 2022 18:30:13 +0100 Subject: [PATCH 3/9] [CMake] fix installation --- cmake/MapnikInstall.cmake | 11 ++++++----- cmake/pack.cmake | 3 +++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmake/MapnikInstall.cmake b/cmake/MapnikInstall.cmake index eb343032d..0953c09db 100644 --- a/cmake/MapnikInstall.cmake +++ b/cmake/MapnikInstall.cmake @@ -70,13 +70,13 @@ function(mapnik_install_targets) set(_internal_libraries "") foreach(_target IN LISTS _installed_utilities) - list(APPEND _internal_executables "${CMAKE_INSTALL_PREFIX}/${MAPNIK_BIN_DIR}/$") + list(APPEND _internal_executables "\${CMAKE_INSTALL_PREFIX}/${MAPNIK_BIN_DIR}/$") endforeach() foreach(_target IN LISTS _installed_targets) - list(APPEND _internal_libraries "${CMAKE_INSTALL_PREFIX}/${MAPNIK_BIN_DIR}/$") + list(APPEND _internal_libraries "\${CMAKE_INSTALL_PREFIX}/${MAPNIK_BIN_DIR}/$") endforeach() foreach(_target IN LISTS _installed_plugins) - list(APPEND _internal_libraries "${CMAKE_INSTALL_PREFIX}/${PLUGINS_INSTALL_DIR}/$") + list(APPEND _internal_libraries "\${CMAKE_INSTALL_PREFIX}/${PLUGINS_INSTALL_DIR}/$") endforeach() # all other executables get auto detected and fixed. if(_internal_executables) @@ -84,8 +84,9 @@ function(mapnik_install_targets) endif() INSTALL(CODE " - message(STATUS \"${_internal_executables}\") - message(STATUS \"${_internal_libraries}\") + message(STATUS \"internal_executables: ${_internal_executables}\") + message(STATUS \"internal_libraries: ${_internal_libraries}\") + message(STATUS \"ADDITIONAL_LIBARIES_PATHS: ${ADDITIONAL_LIBARIES_PATHS}\") include(BundleUtilities) fixup_bundle(\"${_internal_executables}\" \"${_internal_libraries}\" \"${ADDITIONAL_LIBARIES_PATHS}\") diff --git a/cmake/pack.cmake b/cmake/pack.cmake index eb27bda41..1544e5018 100644 --- a/cmake/pack.cmake +++ b/cmake/pack.cmake @@ -1,4 +1,7 @@ include(InstallRequiredSystemLibraries) +set(CPACK_PACKAGE_NAME "mapnik") +set(CPACK_PACKAGE_HOMEPAGE_URL "https://mapnik.org") +set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/COPYING") set(CPACK_SOURCE_GENERATOR "TGZ") set(CPACK_GENERATOR "TGZ") set(CPACK_SOURCE_IGNORE_FILES From 08dcab617ebe556512ec929f9dacc88fa89d709f Mon Sep 17 00:00:00 2001 From: Mathis Logemann Date: Thu, 27 Jan 2022 18:31:33 +0100 Subject: [PATCH 4/9] [CMake] add benchmark run script --- benchmark/CMakeLists.txt | 7 +++++ benchmark/run_benchmarks | 65 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 benchmark/run_benchmarks diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 60c4ff85f..87809b55e 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -36,9 +36,16 @@ function(mapnik_create_benchmark) LIBRARY_OUTPUT_DIRECTORY "${MAPNIK_OUTPUT_DIR}" RUNTIME_OUTPUT_DIRECTORY "${MAPNIK_OUTPUT_DIR}" ARCHIVE_OUTPUT_DIRECTORY "${MAPNIK_OUTPUT_DIR}/lib" + OUTPUT_NAME "${BENCHNAME}" ) endfunction() foreach(benchmark ${BENCHMARK_SRCS}) mapnik_create_benchmark(${benchmark}) endforeach() + +file(COPY data DESTINATION "${MAPNIK_OUTPUT_DIR}/benchmark") +file(COPY run_benchmarks + DESTINATION "${MAPNIK_OUTPUT_DIR}" + FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE OWNER_WRITE GROUP_WRITE GROUP_READ GROUP_EXECUTE WORLD_READ +) diff --git a/benchmark/run_benchmarks b/benchmark/run_benchmarks new file mode 100644 index 000000000..4476f3a9d --- /dev/null +++ b/benchmark/run_benchmarks @@ -0,0 +1,65 @@ +#!/bin/bash + +BASE=. +function run { + local runner="$BASE/$1 --log=none" + local threads="$2" + local iters="$3" + shift 3 + $runner --threads 0 --iterations $iters "$@" + if test $threads -gt 0; then + $runner --threads $threads --iterations $((iters/threads)) "$@" + fi +} +run test_getline 30 10000000 +#run test_array_allocation 20 100000 +#run test_png_encoding1 10 1000 +#run test_png_encoding2 10 50 +#run test_to_string1 10 100000 +#run test_to_string2 10 100000 +#run test_polygon_clipping 10 1000 +#run test_polygon_clipping_rendering 10 100 +run test_proj_transform1 10 100 +run test_expression_parse 10 10000 +run test_face_ptr_creation 10 1000 +run test_font_registration 10 100 +run test_offset_converter 10 1000 +#run normalize_angle 0 1000000 --min-duration=0.2 + +# commented since this is really slow on travis +: ' +$BASE/test_rendering \ + --name "text rendering" \ + --map benchmark/data/roads.xml \ + --extent 1477001.12245,6890242.37746,1480004.49012,6892244.62256 \ + --width 600 \ + --height 600 \ + --iterations 20 \ + --threads 10 +' + +$BASE/test_rendering \ + --name "gdal tiff rendering" \ + --map benchmark/data/gdal-wgs.xml \ + --extent -180.0,-120.0,180.0,120.0 \ + --width 600 \ + --height 600 \ + --iterations 20 \ + --threads 10 + +$BASE/test_rendering \ + --name "raster tiff rendering" \ + --map benchmark/data/raster-wgs.xml \ + --extent -180.0,-120.0,180.0,120.0 \ + --width 600 \ + --height 600 \ + --iterations 20 \ + --threads 10 + +$BASE/test_quad_tree \ + --iterations 10000 \ + --threads 1 + +$BASE/test_quad_tree \ + --iterations 1000 \ + --threads 10 From ba271492325cf8dbe41ad6338440e208b5ea3b76 Mon Sep 17 00:00:00 2001 From: Mathis Logemann Date: Thu, 27 Jan 2022 18:32:21 +0100 Subject: [PATCH 5/9] add missing includes --- include/mapnik/util/timer.hpp | 1 + include/mapnik/util/variant_io.hpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/include/mapnik/util/timer.hpp b/include/mapnik/util/timer.hpp index 7541a584f..c51962474 100644 --- a/include/mapnik/util/timer.hpp +++ b/include/mapnik/util/timer.hpp @@ -25,6 +25,7 @@ #include #include +#include namespace mapnik { diff --git a/include/mapnik/util/variant_io.hpp b/include/mapnik/util/variant_io.hpp index 220d8893b..cb5abb37b 100644 --- a/include/mapnik/util/variant_io.hpp +++ b/include/mapnik/util/variant_io.hpp @@ -23,6 +23,8 @@ #ifndef MAPNIK_UTIL_VARIANT_IO_HPP #define MAPNIK_UTIL_VARIANT_IO_HPP +#include +#include namespace mapbox { namespace util { From aee8519bac9bbdca8fdfa4eab341ffee6898628d Mon Sep 17 00:00:00 2001 From: Mathis Logemann Date: Thu, 27 Jan 2022 18:37:03 +0100 Subject: [PATCH 6/9] [CMake] visual test in own dir --- test/CMakeLists.txt | 35 ++++++++--------------------------- test/visual/CMakeLists.txt | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 27 deletions(-) create mode 100644 test/visual/CMakeLists.txt diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 847511f49..a7aa3f98e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,6 +15,8 @@ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.19.0") cmake_policy(SET CMP0110 OLD) endif() +add_subdirectory(visual) + add_executable(mapnik-test-unit unit/run.cpp unit/color/css_color.cpp @@ -114,7 +116,7 @@ target_link_libraries(mapnik-test-unit PUBLIC # workaround since the "offical" include dir would be file(COPY catch_ext.hpp DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") file(COPY cleanup.hpp DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") -target_include_directories(mapnik-test-unit PRIVATE "${catch2_SOURCE_DIR}/single_include/catch2" ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(mapnik-test-unit PRIVATE "${Catch2_SOURCE_DIR}/single_include/catch2" "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}") add_executable(agg_rasterizer_integer_overflow_test standalone/agg_rasterizer_integer_overflow_test.cpp) set_target_properties(agg_rasterizer_integer_overflow_test PROPERTIES @@ -128,7 +130,7 @@ target_link_libraries(agg_rasterizer_integer_overflow_test PUBLIC mapnik::agg mapnik::json ) -target_include_directories(agg_rasterizer_integer_overflow_test PRIVATE "${catch2_SOURCE_DIR}/single_include/catch2" ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(agg_rasterizer_integer_overflow_test PRIVATE "${Catch2_SOURCE_DIR}/single_include/catch2" "${CMAKE_CURRENT_BINARY_DIR}") add_executable(datasource_registration_test standalone/datasource_registration_test.cpp) set_target_properties(datasource_registration_test PROPERTIES @@ -141,7 +143,7 @@ target_link_libraries(datasource_registration_test PUBLIC mapnik::mapnik mapnik::agg ) -target_include_directories(datasource_registration_test PRIVATE "${catch2_SOURCE_DIR}/single_include/catch2" ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(datasource_registration_test PRIVATE "${Catch2_SOURCE_DIR}/single_include/catch2" "${CMAKE_CURRENT_BINARY_DIR}") add_executable(font_registration_test standalone/font_registration_test.cpp) set_target_properties(font_registration_test PROPERTIES @@ -155,7 +157,7 @@ target_link_libraries(font_registration_test PUBLIC mapnik::agg mapnik::json ) -target_include_directories(font_registration_test PRIVATE "${catch2_SOURCE_DIR}/single_include/catch2" ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(font_registration_test PRIVATE "${Catch2_SOURCE_DIR}/single_include/catch2" "${CMAKE_CURRENT_BINARY_DIR}") #not workable since boost::filesystem native returns a wstring and the function taskes a std::string add_executable(map_xml_test standalone/map_xml_test.cpp) @@ -170,30 +172,9 @@ target_link_libraries(map_xml_test PUBLIC mapnik::agg mapnik::json ) -target_include_directories(map_xml_test PRIVATE "${catch2_SOURCE_DIR}/single_include/catch2" ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(map_xml_test PRIVATE "${Catch2_SOURCE_DIR}/single_include/catch2" "${CMAKE_CURRENT_BINARY_DIR}") -add_executable(mapnik-test-visual - visual/parse_map_sizes.cpp - visual/report.cpp - visual/runner.cpp - visual/run.cpp -) -set_target_properties(mapnik-test-visual PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${MAPNIK_OUTPUT_DIR}" - RUNTIME_OUTPUT_DIRECTORY "${MAPNIK_OUTPUT_DIR}" - ARCHIVE_OUTPUT_DIRECTORY "${MAPNIK_OUTPUT_DIR}/lib" -) -target_link_libraries(mapnik-test-visual PRIVATE - Catch2::Catch2 - Boost::program_options - Boost::filesystem - mapnik::mapnik - mapnik::agg -) -target_include_directories(mapnik-test-visual PRIVATE "${catch2_SOURCE_DIR}/single_include/catch2" ${CMAKE_CURRENT_BINARY_DIR}) - -include("${catch2_SOURCE_DIR}/contrib/Catch.cmake") -include("${catch2_SOURCE_DIR}/contrib/ParseAndAddCatchTests.cmake") +include("${Catch2_SOURCE_DIR}/contrib/Catch.cmake") file(COPY data DESTINATION "${MAPNIK_OUTPUT_DIR}/test") file(COPY data-visual DESTINATION "${MAPNIK_OUTPUT_DIR}/test") diff --git a/test/visual/CMakeLists.txt b/test/visual/CMakeLists.txt new file mode 100644 index 000000000..a1d561c7e --- /dev/null +++ b/test/visual/CMakeLists.txt @@ -0,0 +1,22 @@ +add_executable(mapnik-test-visual + parse_map_sizes.cpp + report.cpp + runner.cpp + run.cpp +) +set_target_properties(mapnik-test-visual PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${MAPNIK_OUTPUT_DIR}" + RUNTIME_OUTPUT_DIRECTORY "${MAPNIK_OUTPUT_DIR}" + ARCHIVE_OUTPUT_DIRECTORY "${MAPNIK_OUTPUT_DIR}/lib" +) +target_link_libraries(mapnik-test-visual PRIVATE + Catch2::Catch2 + Boost::program_options + Boost::filesystem + mapnik::mapnik + mapnik::agg +) +# needed for cleanup.hpp +target_include_directories(mapnik-test-visual PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..") + +mapnik_install_utility(mapnik-test-visual) From 3012ea43ee87eed7d617010fb721f963cfdfad69 Mon Sep 17 00:00:00 2001 From: Mathis Logemann Date: Thu, 27 Jan 2022 18:37:19 +0100 Subject: [PATCH 7/9] [visual-test] enable utf-8 console output for windows --- test/visual/run.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/visual/run.cpp b/test/visual/run.cpp index 93e855422..84b47e9a7 100644 --- a/test/visual/run.cpp +++ b/test/visual/run.cpp @@ -31,6 +31,10 @@ #include "cleanup.hpp" // run_cleanup() +#if defined(_WIN32) +#include +#endif + #ifdef MAPNIK_LOG using log_levels_map = std::map; @@ -105,6 +109,11 @@ runner::renderer_container create_renderers(po::variables_map const & args, int main(int argc, char** argv) { +#ifdef _WIN32 + SetConsoleCP(CP_UTF8); + SetConsoleOutputCP(CP_UTF8); +#endif + po::options_description desc("visual test runner"); desc.add_options() ("help,h", "produce usage message") From 3b1d09702c5750b604c6880147cf9fcf3fc68190 Mon Sep 17 00:00:00 2001 From: Mathis Logemann Date: Thu, 27 Jan 2022 18:40:22 +0100 Subject: [PATCH 8/9] [unit-test] enable tiff test --- src/tiff_reader.cpp | 730 +------------------------------- src/tiff_reader.hpp | 753 ++++++++++++++++++++++++++++++++++ test/unit/imaging/tiff_io.cpp | 4 +- 3 files changed, 764 insertions(+), 723 deletions(-) create mode 100644 src/tiff_reader.hpp diff --git a/src/tiff_reader.cpp b/src/tiff_reader.cpp index b435da4b4..9d154e09e 100644 --- a/src/tiff_reader.cpp +++ b/src/tiff_reader.cpp @@ -38,6 +38,7 @@ MAPNIK_DISABLE_WARNING_PUSH MAPNIK_DISABLE_WARNING_POP #include #endif +#include "tiff_reader.hpp" // stl #include @@ -46,7 +47,7 @@ MAPNIK_DISABLE_WARNING_POP namespace mapnik { namespace detail { -static toff_t tiff_seek_proc(thandle_t handle, toff_t off, int whence) +toff_t tiff_seek_proc(thandle_t handle, toff_t off, int whence) { std::istream* in = reinterpret_cast(handle); @@ -65,12 +66,12 @@ static toff_t tiff_seek_proc(thandle_t handle, toff_t off, int whence) return static_cast(in->tellg()); } -static int tiff_close_proc(thandle_t) +int tiff_close_proc(thandle_t) { return 0; } -static toff_t tiff_size_proc(thandle_t handle) +toff_t tiff_size_proc(thandle_t handle) { std::istream* in = reinterpret_cast(handle); std::ios::pos_type pos = in->tellg(); @@ -80,7 +81,7 @@ static toff_t tiff_size_proc(thandle_t handle) return static_cast(len); } -static tsize_t tiff_read_proc(thandle_t handle, tdata_t buf, tsize_t size) +tsize_t tiff_read_proc(thandle_t handle, tdata_t buf, tsize_t size) { std::istream * in = reinterpret_cast(handle); std::streamsize request_size = size; @@ -90,121 +91,24 @@ static tsize_t tiff_read_proc(thandle_t handle, tdata_t buf, tsize_t size) return static_cast(in->gcount()); } -static tsize_t tiff_write_proc(thandle_t , tdata_t , tsize_t) +tsize_t tiff_write_proc(thandle_t , tdata_t , tsize_t) { return 0; } -static void tiff_unmap_proc(thandle_t, tdata_t, toff_t) +void tiff_unmap_proc(thandle_t, tdata_t, toff_t) { } -static int tiff_map_proc(thandle_t, tdata_t* , toff_t*) +int tiff_map_proc(thandle_t, tdata_t* , toff_t*) { return 0; } - -template -struct tiff_io_traits -{ - using input_stream_type = std::istream; -}; - -#if defined(MAPNIK_MEMORY_MAPPED_FILE) -template <> -struct tiff_io_traits -{ - using input_stream_type = boost::interprocess::ibufferstream; -}; -#endif } -template -class tiff_reader : public image_reader -{ - using tiff_ptr = std::shared_ptr; - using source_type = T; - using input_stream = typename detail::tiff_io_traits::input_stream_type; -#if defined(MAPNIK_MEMORY_MAPPED_FILE) - mapnik::mapped_region_ptr mapped_region_; -#endif - - struct tiff_closer - { - void operator() (TIFF * tif) - { - if (tif != 0) TIFFClose(tif); - } - }; - -private: - source_type source_; - input_stream stream_; - tiff_ptr tif_; - int read_method_; - int rows_per_strip_; - int tile_width_; - int tile_height_; - std::size_t width_; - std::size_t height_; - boost::optional > bbox_; - unsigned bps_; - unsigned sample_format_; - unsigned photometric_; - unsigned bands_; - unsigned planar_config_; - unsigned compression_; - bool has_alpha_; - bool is_tiled_; - -public: - enum TiffType { - generic=1, - stripped, - tiled - }; - explicit tiff_reader(std::string const& filename); - tiff_reader(char const* data, std::size_t size); - virtual ~tiff_reader(); - unsigned width() const final; - unsigned height() const final; - boost::optional > bounding_box() const final; - inline bool has_alpha() const final { return has_alpha_; } - void read(unsigned x,unsigned y,image_rgba8& image) final; - image_any read(unsigned x, unsigned y, unsigned width, unsigned height) final; - // methods specific to tiff reader - unsigned bits_per_sample() const { return bps_; } - unsigned sample_format() const { return sample_format_; } - unsigned photometric() const { return photometric_; } - bool is_tiled() const { return is_tiled_; } - unsigned tile_width() const { return tile_width_; } - unsigned tile_height() const { return tile_height_; } - unsigned rows_per_strip() const { return rows_per_strip_; } - unsigned planar_config() const { return planar_config_; } - unsigned compression() const { return compression_; } -private: - tiff_reader(const tiff_reader&); - tiff_reader& operator=(const tiff_reader&); - void init(); - - template - void read_generic(std::size_t x,std::size_t y, ImageData & image); - - template - void read_stripped(std::size_t x,std::size_t y, ImageData & image); - - template - void read_tiled(std::size_t x,std::size_t y, ImageData & image); - - template - image_any read_any_gray(std::size_t x, std::size_t y, std::size_t width, std::size_t height); - - TIFF* open(std::istream & input); -}; - namespace { - + image_reader* create_tiff_reader(std::string const& filename) { #if defined(MAPNIK_MEMORY_MAPPED_FILE) @@ -224,622 +128,6 @@ const bool registered2 = register_image_reader("tiff", create_tiff_reader2); } -template -tiff_reader::tiff_reader(std::string const& filename) - : -#if defined(MAPNIK_MEMORY_MAPPED_FILE) - stream_(), -#else - source_(), - stream_(&source_), -#endif - tif_(nullptr), - read_method_(generic), - rows_per_strip_(0), - tile_width_(0), - tile_height_(0), - width_(0), - height_(0), - bps_(0), - sample_format_(SAMPLEFORMAT_UINT), - photometric_(0), - bands_(1), - planar_config_(PLANARCONFIG_CONTIG), - compression_(COMPRESSION_NONE), - has_alpha_(false), - is_tiled_(false) -{ - -#if defined(MAPNIK_MEMORY_MAPPED_FILE) - boost::optional memory = - mapnik::mapped_memory_cache::instance().find(filename,true); - - if (memory) - { - mapped_region_ = *memory; - stream_.buffer(static_cast(mapped_region_->get_address()),mapped_region_->get_size()); - } - else - { - throw image_reader_exception("could not create file mapping for " + filename); - } -#else - source_.open(filename, std::ios_base::in | std::ios_base::binary); -#endif - if (!stream_) throw image_reader_exception("TIFF reader: cannot open file " + filename); - init(); -} - -template -tiff_reader::tiff_reader(char const* data, std::size_t size) - : source_(data, size), - stream_(&source_), - tif_(nullptr), - read_method_(generic), - rows_per_strip_(0), - tile_width_(0), - tile_height_(0), - width_(0), - height_(0), - bps_(0), - sample_format_(SAMPLEFORMAT_UINT), - photometric_(0), - bands_(1), - planar_config_(PLANARCONFIG_CONTIG), - compression_(COMPRESSION_NONE), - has_alpha_(false), - is_tiled_(false) -{ - if (!stream_) throw image_reader_exception("TIFF reader: cannot open image stream "); - init(); -} - -template -void tiff_reader::init() -{ - // avoid calling TIFFs global structures - TIFFSetWarningHandler(0); - TIFFSetErrorHandler(0); - - TIFF* tif = open(stream_); - - if (!tif) throw image_reader_exception("Can't open tiff file"); - - TIFFGetField(tif,TIFFTAG_BITSPERSAMPLE,&bps_); - TIFFGetField(tif,TIFFTAG_SAMPLEFORMAT,&sample_format_); - TIFFGetField(tif,TIFFTAG_PHOTOMETRIC,&photometric_); - TIFFGetField(tif, TIFFTAG_SAMPLESPERPIXEL, &bands_); - - MAPNIK_LOG_DEBUG(tiff_reader) << "bits per sample: " << bps_ ; - MAPNIK_LOG_DEBUG(tiff_reader) << "sample format: " << sample_format_ ; - MAPNIK_LOG_DEBUG(tiff_reader) << "photometric: " << photometric_ ; - MAPNIK_LOG_DEBUG(tiff_reader) << "bands: " << bands_ ; - - TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, &width_); - TIFFGetField(tif, TIFFTAG_IMAGELENGTH, &height_); - - TIFFGetField(tif, TIFFTAG_PLANARCONFIG, &planar_config_); - TIFFGetField(tif, TIFFTAG_COMPRESSION, &compression_ ); - - std::uint16_t orientation; - if (TIFFGetField(tif, TIFFTAG_ORIENTATION, &orientation) == 0) - { - orientation = 1; - } - MAPNIK_LOG_DEBUG(tiff_reader) << "orientation: " << orientation ; - MAPNIK_LOG_DEBUG(tiff_reader) << "planar-config: " << planar_config_ ; - is_tiled_ = TIFFIsTiled(tif); - - if (is_tiled_) - { - TIFFGetField(tif, TIFFTAG_TILEWIDTH, &tile_width_); - TIFFGetField(tif, TIFFTAG_TILELENGTH, &tile_height_); - MAPNIK_LOG_DEBUG(tiff_reader) << "tiff is tiled"; - read_method_ = tiled; - } - else if (TIFFGetField(tif, TIFFTAG_ROWSPERSTRIP, &rows_per_strip_) != 0) - { - MAPNIK_LOG_DEBUG(tiff_reader) << "tiff is stripped"; - read_method_ = stripped; - } - //TIFFTAG_EXTRASAMPLES - uint16 extrasamples = 0; - uint16* sampleinfo = nullptr; - if (TIFFGetField(tif, TIFFTAG_EXTRASAMPLES, - &extrasamples, &sampleinfo)) - { - has_alpha_ = true; - if (extrasamples > 0 && - sampleinfo[0] == EXTRASAMPLE_UNSPECIFIED) - { - throw image_reader_exception("Unspecified provided for extra samples to tiff reader."); - } - } - // Try extracting bounding box from geoTIFF tags - { - uint16 count = 0; - double *pixelscale; - double *tilepoint; - if (TIFFGetField(tif, 33550, &count, &pixelscale) == 1 && count == 3 - && TIFFGetField(tif, 33922 , &count, &tilepoint) == 1 && count == 6) - { - MAPNIK_LOG_DEBUG(tiff_reader) << "PixelScale:" << pixelscale[0] << "," << pixelscale[1] << "," << pixelscale[2] ; - MAPNIK_LOG_DEBUG(tiff_reader) << "TilePoint:" << tilepoint[0] << "," << tilepoint[1] << "," << tilepoint[2] ; - MAPNIK_LOG_DEBUG(tiff_reader) << " " << tilepoint[3] << "," << tilepoint[4] << "," << tilepoint[5] ; - - // assuming upper-left - double lox = tilepoint[3]; - double loy = tilepoint[4]; - double hix = lox + pixelscale[0] * width_; - double hiy = loy - pixelscale[1] * height_; - bbox_.reset(box2d(lox, loy, hix, hiy)); - MAPNIK_LOG_DEBUG(tiff_reader) << "Bounding Box:" << *bbox_ ; - } - - } - if (!is_tiled_ && - compression_ == COMPRESSION_NONE && - planar_config_ == PLANARCONFIG_CONTIG) - { - if (height_ > 128 * 1024 * 1024) - { - std::size_t line_size = (bands_ * width_ * bps_ + 7) / 8; - std::size_t default_strip_height = 8192 / line_size; - if (default_strip_height == 0) default_strip_height = 1; - std::size_t num_strips = height_ / default_strip_height; - if (num_strips > 128 * 1024 * 1024) - { - throw image_reader_exception("Can't allocate tiff"); - } - } - } -} - -template -tiff_reader::~tiff_reader() -{ -} - -template -unsigned tiff_reader::width() const -{ - return width_; -} - -template -unsigned tiff_reader::height() const -{ - return height_; -} - -template -boost::optional > tiff_reader::bounding_box() const -{ - return bbox_; -} - -template -void tiff_reader::read(unsigned x,unsigned y,image_rgba8& image) -{ - if (read_method_==stripped) - { - read_stripped(static_cast(x),static_cast(y),image); - } - else if (read_method_==tiled) - { - read_tiled(static_cast(x),static_cast(y),image); - } - else - { - read_generic(static_cast(x),static_cast(y),image); - } -} - -template -template -image_any tiff_reader::read_any_gray(std::size_t x0, std::size_t y0, std::size_t width, std::size_t height) -{ - using image_type = ImageData; - using pixel_type = typename image_type::pixel_type; - if (read_method_ == tiled) - { - image_type data(width, height); - read_tiled(x0, y0, data); - return image_any(std::move(data)); - } - else if (read_method_ == stripped) - { - image_type data(width, height); - read_stripped(x0, y0, data); - return image_any(std::move(data)); - } - else - { - TIFF* tif = open(stream_); - if (tif) - { - image_type data(width, height); - std::size_t block_size = rows_per_strip_ > 0 ? rows_per_strip_ : tile_height_ ; - std::size_t start_y = y0 - y0 % block_size; - std::size_t end_y = std::min(y0 + height, height_); - std::size_t start_x = x0; - std::size_t end_x = std::min(x0 + width, width_); - std::size_t element_size = sizeof(pixel_type); - MAPNIK_LOG_DEBUG(tiff_reader) << "SCANLINE SIZE=" << TIFFScanlineSize(tif); - std::size_t size_to_allocate = (TIFFScanlineSize(tif) + element_size - 1)/element_size; - std::unique_ptr const scanline(new pixel_type[size_to_allocate]); - if (planar_config_ == PLANARCONFIG_CONTIG) - { - for (std::size_t y = start_y; y < end_y; ++y) - { - // we have to read all scanlines sequentially from start_y - // to be able to use scanline interface with compressed blocks. - if (-1 != TIFFReadScanline(tif, scanline.get(), y) && (y >= y0)) - { - pixel_type * row = data.get_row(y - y0); - if (bands_ == 1) - { - std::transform(scanline.get() + start_x, scanline.get() + end_x, row, [](pixel_type const& p) { return p;}); - } - else if (size_to_allocate == bands_ * width_) - { - // bands_ > 1 => packed bands in grayscale image e.g an extra alpha channel. - // Just pick first one for now. - pixel_type * buf = scanline.get() + start_x * bands_; - std::size_t x_index = 0; - for (std::size_t j = 0; j < end_x * bands_; ++j) - { - if (x_index >= width) break; - if (j % bands_ == 0) - { - row[x_index++] = buf[j]; - } - } - } - } - } - } - else if (planar_config_ == PLANARCONFIG_SEPARATE) - { - for (std::size_t s = 0 ; s < bands_ ; ++s) - { - for (std::size_t y = start_y; y < end_y; ++y) - { - if (-1 != TIFFReadScanline(tif, scanline.get(), y) && (y >= y0)) - { - pixel_type * row = data.get_row(y - y0); - std::transform(scanline.get() + start_x, scanline.get() + end_x, row, [](pixel_type const& p) { return p;}); - } - } - } - } - return image_any(std::move(data)); - } - } - return image_any(); -} - - -namespace detail { - -struct rgb8 -{ - std::uint8_t r; - std::uint8_t g; - std::uint8_t b; -}; - -struct rgb8_to_rgba8 -{ - std::uint32_t operator() (rgb8 const& in) const - { - return ((255 << 24) | (in.r) | (in.g << 8) | (in.b << 16)); - } -}; - -template -struct tiff_reader_traits -{ - using image_type = T; - using pixel_type = typename image_type::pixel_type; - - constexpr static bool reverse = false; - static bool read_tile(TIFF * tif, std::size_t x, std::size_t y, pixel_type* buf, std::size_t tile_width, std::size_t tile_height) - { - std::uint32_t tile_size = TIFFTileSize(tif); - return (TIFFReadEncodedTile(tif, TIFFComputeTile(tif, x, y, 0, 0), buf, tile_size) != -1); - } - - static bool read_strip(TIFF * tif, std::size_t y, std::size_t rows_per_strip, std::size_t strip_width, pixel_type * buf) - { - return (TIFFReadEncodedStrip(tif, y/rows_per_strip, buf, -1) != -1); - } -}; - -// default specialization that expands into RGBA -template <> -struct tiff_reader_traits -{ - using image_type = image_rgba8; - using pixel_type = std::uint32_t; - constexpr static bool reverse = true; - static bool read_tile(TIFF * tif, std::size_t x0, std::size_t y0, pixel_type* buf, std::size_t tile_width, std::size_t tile_height) - { - return (TIFFReadRGBATile(tif, x0, y0, buf) != 0); - } - - static bool read_strip(TIFF * tif, std::size_t y, std::size_t rows_per_strip, std::size_t strip_width, pixel_type * buf) - { - return (TIFFReadRGBAStrip(tif, y, buf) != 0); - } -}; - -} - -template -image_any tiff_reader::read(unsigned x, unsigned y, unsigned width, unsigned height) -{ - if (width > 10000 || height > 10000) - { - throw image_reader_exception("Can't allocate tiff > 10000x10000"); - } - std::size_t x0 = static_cast(x); - std::size_t y0 = static_cast(y); - switch (photometric_) - { - case PHOTOMETRIC_MINISBLACK: - case PHOTOMETRIC_MINISWHITE: - { - switch (bps_) - { - case 8: - { - switch (sample_format_) - { - case SAMPLEFORMAT_UINT: - { - return read_any_gray(x0, y0, width, height); - } - case SAMPLEFORMAT_INT: - { - return read_any_gray(x0, y0, width, height); - } - default: - { - throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample"); - } - } - } - case 16: - { - switch (sample_format_) - { - case SAMPLEFORMAT_UINT: - { - return read_any_gray(x0, y0, width, height); - } - case SAMPLEFORMAT_INT: - { - return read_any_gray(x0, y0, width, height); - } - default: - { - throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample"); - } - } - } - case 32: - { - switch (sample_format_) - { - case SAMPLEFORMAT_UINT: - { - return read_any_gray(x0, y0, width, height); - } - case SAMPLEFORMAT_INT: - { - return read_any_gray(x0, y0, width, height); - } - case SAMPLEFORMAT_IEEEFP: - { - return read_any_gray(x0, y0, width, height); - } - default: - { - throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample"); - } - } - } - case 64: - { - switch (sample_format_) - { - case SAMPLEFORMAT_UINT: - { - return read_any_gray(x0, y0, width, height); - } - case SAMPLEFORMAT_INT: - { - return read_any_gray(x0, y0, width, height); - } - case SAMPLEFORMAT_IEEEFP: - { - return read_any_gray(x0, y0, width, height); - } - default: - { - throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample"); - } - } - } - } - } - default: - { - //PHOTOMETRIC_PALETTE = 3; - //PHOTOMETRIC_MASK = 4; - //PHOTOMETRIC_SEPARATED = 5; - //PHOTOMETRIC_YCBCR = 6; - //PHOTOMETRIC_CIELAB = 8; - //PHOTOMETRIC_ICCLAB = 9; - //PHOTOMETRIC_ITULAB = 10; - //PHOTOMETRIC_LOGL = 32844; - //PHOTOMETRIC_LOGLUV = 32845; - image_rgba8 data(width,height, true, true); - read(x0, y0, data); - return image_any(std::move(data)); - } - } - return image_any(); -} - -template -template -void tiff_reader::read_generic(std::size_t, std::size_t, ImageData &) -{ - throw image_reader_exception("tiff_reader: TODO - tiff is not stripped or tiled"); -} - -template -template -void tiff_reader::read_tiled(std::size_t x0,std::size_t y0, ImageData & image) -{ - using pixel_type = typename detail::tiff_reader_traits::pixel_type; - - TIFF* tif = open(stream_); - if (tif) - { - std::uint32_t tile_size = TIFFTileSize(tif); - std::unique_ptr tile(new pixel_type[tile_size]); - std::size_t width = image.width(); - std::size_t height = image.height(); - std::size_t start_y = (y0 / tile_height_) * tile_height_; - std::size_t end_y = ((y0 + height) / tile_height_ + 1) * tile_height_; - std::size_t start_x = (x0 / tile_width_) * tile_width_; - std::size_t end_x = ((x0 + width) / tile_width_ + 1) * tile_width_; - end_y = std::min(end_y, height_); - end_x = std::min(end_x, width_); - bool pick_first_band = (bands_ > 1) && (tile_size / (tile_width_ * tile_height_ * sizeof(pixel_type)) == bands_); - for (std::size_t y = start_y; y < end_y; y += tile_height_) - { - std::size_t ty0 = std::max(y0, y) - y; - std::size_t ty1 = std::min(height + y0, y + tile_height_) - y; - - for (std::size_t x = start_x; x < end_x; x += tile_width_) - { - if (!detail::tiff_reader_traits::read_tile(tif, x, y, tile.get(), tile_width_, tile_height_)) - { - MAPNIK_LOG_DEBUG(tiff_reader) << "read_tile(...) failed at " << x << "/" << y << " for " << width_ << "/" << height_ << "\n"; - break; - } - if (pick_first_band) - { - std::uint32_t size = tile_width_ * tile_height_ * sizeof(pixel_type); - for (std::uint32_t n = 0; n < size; ++n) - { - tile[n] = tile[n * bands_]; - } - } - std::size_t tx0 = std::max(x0, x); - std::size_t tx1 = std::min(width + x0, x + tile_width_); - std::size_t row_index = y + ty0 - y0; - - if (detail::tiff_reader_traits::reverse) - { - for (std::size_t ty = ty0; ty < ty1; ++ty, ++row_index) - { - // This is in reverse because the TIFFReadRGBATile reads are inverted - image.set_row(row_index, tx0 - x0, tx1 - x0, &tile[(tile_height_ - ty - 1) * tile_width_ + tx0 - x]); - } - } - else - { - for (std::size_t ty = ty0; ty < ty1; ++ty, ++row_index) - { - image.set_row(row_index, tx0 - x0, tx1 - x0, &tile[ty * tile_width_ + tx0 - x]); - } - } - } - } - } -} - -template -template -void tiff_reader::read_stripped(std::size_t x0, std::size_t y0, ImageData & image) -{ - using pixel_type = typename detail::tiff_reader_traits::pixel_type; - TIFF* tif = open(stream_); - if (tif) - { - std::uint32_t strip_size = TIFFStripSize(tif); - std::unique_ptr strip(new pixel_type[strip_size]); - std::size_t width = image.width(); - std::size_t height = image.height(); - - std::size_t start_y = (y0 / rows_per_strip_) * rows_per_strip_; - std::size_t end_y = std::min(y0 + height, height_); - std::size_t tx0, tx1, ty0, ty1; - tx0 = x0; - tx1 = std::min(width + x0, width_); - std::size_t row = 0; - bool pick_first_band = (bands_ > 1) && (strip_size / (width_ * rows_per_strip_ * sizeof(pixel_type)) == bands_); - for (std::size_t y = start_y; y < end_y; y += rows_per_strip_) - { - ty0 = std::max(y0, y) - y; - ty1 = std::min(end_y, y + rows_per_strip_) - y; - - if (!detail::tiff_reader_traits::read_strip(tif, y, rows_per_strip_, width_, strip.get())) - { - MAPNIK_LOG_DEBUG(tiff_reader) << "TIFFRead(Encoded|RGBA)Strip failed at " << y << " for " << width_ << "/" << height_ << "\n"; - break; - } - if (pick_first_band) - { - std::uint32_t size = width_ * rows_per_strip_ * sizeof(pixel_type); - for (std::uint32_t n = 0; n < size; ++n) - { - strip[n] = strip[bands_ * n]; - } - } - - if (detail::tiff_reader_traits::reverse) - { - std::size_t num_rows = std::min(height_ - y, static_cast(rows_per_strip_)); - for (std::size_t ty = ty0; ty < ty1; ++ty) - { - // This is in reverse because the TIFFReadRGBAStrip reads are inverted - image.set_row(row++, tx0 - x0, tx1 - x0, &strip[(num_rows - ty - 1) * width_ + tx0]); - } - } - else - { - for (std::size_t ty = ty0; ty < ty1; ++ty) - { - image.set_row(row++, tx0 - x0, tx1 - x0, &strip[ty * width_ + tx0]); - } - } - } - } -} - -template -TIFF* tiff_reader::open(std::istream & input) -{ - if (!tif_) - { - tif_ = tiff_ptr(TIFFClientOpen("tiff_input_stream", "rcm", - reinterpret_cast(&input), - detail::tiff_read_proc, - detail::tiff_write_proc, - detail::tiff_seek_proc, - detail::tiff_close_proc, - detail::tiff_size_proc, - detail::tiff_map_proc, - detail::tiff_unmap_proc), tiff_closer()); - } - return tif_.get(); -} } // namespace mapnik diff --git a/src/tiff_reader.hpp b/src/tiff_reader.hpp new file mode 100644 index 000000000..d5e410b7b --- /dev/null +++ b/src/tiff_reader.hpp @@ -0,0 +1,753 @@ +#pragma once +// mapnik +#include +#include +#include +extern "C" +{ +#include +} + +#if defined(MAPNIK_MEMORY_MAPPED_FILE) +#include +MAPNIK_DISABLE_WARNING_PUSH +#include +#include +#include +MAPNIK_DISABLE_WARNING_POP +#include +#endif +#include "tiff_reader.hpp" + +// stl +#include +#include +#include + +namespace mapnik { namespace detail { + +MAPNIK_DECL toff_t tiff_seek_proc(thandle_t handle, toff_t off, int whence); +MAPNIK_DECL int tiff_close_proc(thandle_t); +MAPNIK_DECL toff_t tiff_size_proc(thandle_t handle); +MAPNIK_DECL tsize_t tiff_read_proc(thandle_t handle, tdata_t buf, tsize_t size); +MAPNIK_DECL tsize_t tiff_write_proc(thandle_t , tdata_t , tsize_t); +MAPNIK_DECL void tiff_unmap_proc(thandle_t, tdata_t, toff_t); +MAPNIK_DECL int tiff_map_proc(thandle_t, tdata_t* , toff_t*); + +template +struct tiff_io_traits +{ + using input_stream_type = std::istream; +}; + +#if defined(MAPNIK_MEMORY_MAPPED_FILE) +template <> +struct tiff_io_traits +{ + using input_stream_type = boost::interprocess::ibufferstream; +}; +#endif +} + +template +class tiff_reader : public image_reader +{ + using tiff_ptr = std::shared_ptr; + using source_type = T; + using input_stream = typename detail::tiff_io_traits::input_stream_type; +#if defined(MAPNIK_MEMORY_MAPPED_FILE) + mapnik::mapped_region_ptr mapped_region_; +#endif + + struct tiff_closer + { + void operator() (TIFF * tif) + { + if (tif != 0) TIFFClose(tif); + } + }; + +private: + source_type source_; + input_stream stream_; + tiff_ptr tif_; + int read_method_; + int rows_per_strip_; + int tile_width_; + int tile_height_; + std::size_t width_; + std::size_t height_; + boost::optional > bbox_; + unsigned bps_; + unsigned sample_format_; + unsigned photometric_; + unsigned bands_; + unsigned planar_config_; + unsigned compression_; + bool has_alpha_; + bool is_tiled_; + +public: + enum TiffType { + generic=1, + stripped, + tiled + }; + explicit tiff_reader(std::string const& filename); + tiff_reader(char const* data, std::size_t size); + virtual ~tiff_reader(); + unsigned width() const final; + unsigned height() const final; + boost::optional > bounding_box() const final; + inline bool has_alpha() const final { return has_alpha_; } + void read(unsigned x,unsigned y,image_rgba8& image) final; + image_any read(unsigned x, unsigned y, unsigned width, unsigned height) final; + // methods specific to tiff reader + unsigned bits_per_sample() const { return bps_; } + unsigned sample_format() const { return sample_format_; } + unsigned photometric() const { return photometric_; } + bool is_tiled() const { return is_tiled_; } + unsigned tile_width() const { return tile_width_; } + unsigned tile_height() const { return tile_height_; } + unsigned rows_per_strip() const { return rows_per_strip_; } + unsigned planar_config() const { return planar_config_; } + unsigned compression() const { return compression_; } +private: + tiff_reader(const tiff_reader&); + tiff_reader& operator=(const tiff_reader&); + void init(); + + template + void read_generic(std::size_t x,std::size_t y, ImageData & image); + + template + void read_stripped(std::size_t x,std::size_t y, ImageData & image); + + template + void read_tiled(std::size_t x,std::size_t y, ImageData & image); + + template + image_any read_any_gray(std::size_t x, std::size_t y, std::size_t width, std::size_t height); + + TIFF* open(std::istream & input); +}; + +template +tiff_reader::tiff_reader(std::string const& filename) + : +#if defined(MAPNIK_MEMORY_MAPPED_FILE) + stream_(), +#else + source_(), + stream_(&source_), +#endif + + tif_(nullptr), + read_method_(generic), + rows_per_strip_(0), + tile_width_(0), + tile_height_(0), + width_(0), + height_(0), + bps_(0), + sample_format_(SAMPLEFORMAT_UINT), + photometric_(0), + bands_(1), + planar_config_(PLANARCONFIG_CONTIG), + compression_(COMPRESSION_NONE), + has_alpha_(false), + is_tiled_(false) +{ + +#if defined(MAPNIK_MEMORY_MAPPED_FILE) + boost::optional memory = + mapnik::mapped_memory_cache::instance().find(filename,true); + + if (memory) + { + mapped_region_ = *memory; + stream_.buffer(static_cast(mapped_region_->get_address()),mapped_region_->get_size()); + } + else + { + throw image_reader_exception("could not create file mapping for " + filename); + } +#else + source_.open(filename, std::ios_base::in | std::ios_base::binary); +#endif + if (!stream_) throw image_reader_exception("TIFF reader: cannot open file " + filename); + init(); +} + +template +tiff_reader::tiff_reader(char const* data, std::size_t size) + : source_(data, size), + stream_(&source_), + tif_(nullptr), + read_method_(generic), + rows_per_strip_(0), + tile_width_(0), + tile_height_(0), + width_(0), + height_(0), + bps_(0), + sample_format_(SAMPLEFORMAT_UINT), + photometric_(0), + bands_(1), + planar_config_(PLANARCONFIG_CONTIG), + compression_(COMPRESSION_NONE), + has_alpha_(false), + is_tiled_(false) +{ + if (!stream_) throw image_reader_exception("TIFF reader: cannot open image stream "); + init(); +} + +template +void tiff_reader::init() +{ + // avoid calling TIFFs global structures + TIFFSetWarningHandler(0); + TIFFSetErrorHandler(0); + + TIFF* tif = open(stream_); + + if (!tif) throw image_reader_exception("Can't open tiff file"); + + TIFFGetField(tif,TIFFTAG_BITSPERSAMPLE,&bps_); + TIFFGetField(tif,TIFFTAG_SAMPLEFORMAT,&sample_format_); + TIFFGetField(tif,TIFFTAG_PHOTOMETRIC,&photometric_); + TIFFGetField(tif, TIFFTAG_SAMPLESPERPIXEL, &bands_); + + MAPNIK_LOG_DEBUG(tiff_reader) << "bits per sample: " << bps_ ; + MAPNIK_LOG_DEBUG(tiff_reader) << "sample format: " << sample_format_ ; + MAPNIK_LOG_DEBUG(tiff_reader) << "photometric: " << photometric_ ; + MAPNIK_LOG_DEBUG(tiff_reader) << "bands: " << bands_ ; + + TIFFGetField(tif, TIFFTAG_IMAGEWIDTH, &width_); + TIFFGetField(tif, TIFFTAG_IMAGELENGTH, &height_); + + TIFFGetField(tif, TIFFTAG_PLANARCONFIG, &planar_config_); + TIFFGetField(tif, TIFFTAG_COMPRESSION, &compression_ ); + + std::uint16_t orientation; + if (TIFFGetField(tif, TIFFTAG_ORIENTATION, &orientation) == 0) + { + orientation = 1; + } + MAPNIK_LOG_DEBUG(tiff_reader) << "orientation: " << orientation ; + MAPNIK_LOG_DEBUG(tiff_reader) << "planar-config: " << planar_config_ ; + is_tiled_ = TIFFIsTiled(tif); + + if (is_tiled_) + { + TIFFGetField(tif, TIFFTAG_TILEWIDTH, &tile_width_); + TIFFGetField(tif, TIFFTAG_TILELENGTH, &tile_height_); + MAPNIK_LOG_DEBUG(tiff_reader) << "tiff is tiled"; + read_method_ = tiled; + } + else if (TIFFGetField(tif, TIFFTAG_ROWSPERSTRIP, &rows_per_strip_) != 0) + { + MAPNIK_LOG_DEBUG(tiff_reader) << "tiff is stripped"; + read_method_ = stripped; + } + //TIFFTAG_EXTRASAMPLES + uint16 extrasamples = 0; + uint16* sampleinfo = nullptr; + if (TIFFGetField(tif, TIFFTAG_EXTRASAMPLES, + &extrasamples, &sampleinfo)) + { + has_alpha_ = true; + if (extrasamples > 0 && + sampleinfo[0] == EXTRASAMPLE_UNSPECIFIED) + { + throw image_reader_exception("Unspecified provided for extra samples to tiff reader."); + } + } + // Try extracting bounding box from geoTIFF tags + { + uint16 count = 0; + double *pixelscale; + double *tilepoint; + if (TIFFGetField(tif, 33550, &count, &pixelscale) == 1 && count == 3 + && TIFFGetField(tif, 33922 , &count, &tilepoint) == 1 && count == 6) + { + MAPNIK_LOG_DEBUG(tiff_reader) << "PixelScale:" << pixelscale[0] << "," << pixelscale[1] << "," << pixelscale[2] ; + MAPNIK_LOG_DEBUG(tiff_reader) << "TilePoint:" << tilepoint[0] << "," << tilepoint[1] << "," << tilepoint[2] ; + MAPNIK_LOG_DEBUG(tiff_reader) << " " << tilepoint[3] << "," << tilepoint[4] << "," << tilepoint[5] ; + + // assuming upper-left + double lox = tilepoint[3]; + double loy = tilepoint[4]; + double hix = lox + pixelscale[0] * width_; + double hiy = loy - pixelscale[1] * height_; + bbox_.reset(box2d(lox, loy, hix, hiy)); + MAPNIK_LOG_DEBUG(tiff_reader) << "Bounding Box:" << *bbox_ ; + } + + } + if (!is_tiled_ && + compression_ == COMPRESSION_NONE && + planar_config_ == PLANARCONFIG_CONTIG) + { + if (height_ > 128 * 1024 * 1024) + { + std::size_t line_size = (bands_ * width_ * bps_ + 7) / 8; + std::size_t default_strip_height = 8192 / line_size; + if (default_strip_height == 0) default_strip_height = 1; + std::size_t num_strips = height_ / default_strip_height; + if (num_strips > 128 * 1024 * 1024) + { + throw image_reader_exception("Can't allocate tiff"); + } + } + } +} + +template +tiff_reader::~tiff_reader() +{ +} + +template +unsigned tiff_reader::width() const +{ + return width_; +} + +template +unsigned tiff_reader::height() const +{ + return height_; +} + +template +boost::optional > tiff_reader::bounding_box() const +{ + return bbox_; +} + +template +void tiff_reader::read(unsigned x,unsigned y,image_rgba8& image) +{ + if (read_method_==stripped) + { + read_stripped(static_cast(x),static_cast(y),image); + } + else if (read_method_==tiled) + { + read_tiled(static_cast(x),static_cast(y),image); + } + else + { + read_generic(static_cast(x),static_cast(y),image); + } +} + +template +template +image_any tiff_reader::read_any_gray(std::size_t x0, std::size_t y0, std::size_t width, std::size_t height) +{ + using image_type = ImageData; + using pixel_type = typename image_type::pixel_type; + if (read_method_ == tiled) + { + image_type data(width, height); + read_tiled(x0, y0, data); + return image_any(std::move(data)); + } + else if (read_method_ == stripped) + { + image_type data(width, height); + read_stripped(x0, y0, data); + return image_any(std::move(data)); + } + else + { + TIFF* tif = open(stream_); + if (tif) + { + image_type data(width, height); + std::size_t block_size = rows_per_strip_ > 0 ? rows_per_strip_ : tile_height_ ; + std::size_t start_y = y0 - y0 % block_size; + std::size_t end_y = std::min(y0 + height, height_); + std::size_t start_x = x0; + std::size_t end_x = std::min(x0 + width, width_); + std::size_t element_size = sizeof(pixel_type); + MAPNIK_LOG_DEBUG(tiff_reader) << "SCANLINE SIZE=" << TIFFScanlineSize(tif); + std::size_t size_to_allocate = (TIFFScanlineSize(tif) + element_size - 1)/element_size; + std::unique_ptr const scanline(new pixel_type[size_to_allocate]); + if (planar_config_ == PLANARCONFIG_CONTIG) + { + for (std::size_t y = start_y; y < end_y; ++y) + { + // we have to read all scanlines sequentially from start_y + // to be able to use scanline interface with compressed blocks. + if (-1 != TIFFReadScanline(tif, scanline.get(), y) && (y >= y0)) + { + pixel_type * row = data.get_row(y - y0); + if (bands_ == 1) + { + std::transform(scanline.get() + start_x, scanline.get() + end_x, row, [](pixel_type const& p) { return p;}); + } + else if (size_to_allocate == bands_ * width_) + { + // bands_ > 1 => packed bands in grayscale image e.g an extra alpha channel. + // Just pick first one for now. + pixel_type * buf = scanline.get() + start_x * bands_; + std::size_t x_index = 0; + for (std::size_t j = 0; j < end_x * bands_; ++j) + { + if (x_index >= width) break; + if (j % bands_ == 0) + { + row[x_index++] = buf[j]; + } + } + } + } + } + } + else if (planar_config_ == PLANARCONFIG_SEPARATE) + { + for (std::size_t s = 0 ; s < bands_ ; ++s) + { + for (std::size_t y = start_y; y < end_y; ++y) + { + if (-1 != TIFFReadScanline(tif, scanline.get(), y) && (y >= y0)) + { + pixel_type * row = data.get_row(y - y0); + std::transform(scanline.get() + start_x, scanline.get() + end_x, row, [](pixel_type const& p) { return p;}); + } + } + } + } + return image_any(std::move(data)); + } + } + return image_any(); +} + + +template +image_any tiff_reader::read(unsigned x, unsigned y, unsigned width, unsigned height) +{ + if (width > 10000 || height > 10000) + { + throw image_reader_exception("Can't allocate tiff > 10000x10000"); + } + std::size_t x0 = static_cast(x); + std::size_t y0 = static_cast(y); + switch (photometric_) + { + case PHOTOMETRIC_MINISBLACK: + case PHOTOMETRIC_MINISWHITE: + { + switch (bps_) + { + case 8: + { + switch (sample_format_) + { + case SAMPLEFORMAT_UINT: + { + return read_any_gray(x0, y0, width, height); + } + case SAMPLEFORMAT_INT: + { + return read_any_gray(x0, y0, width, height); + } + default: + { + throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample"); + } + } + } + case 16: + { + switch (sample_format_) + { + case SAMPLEFORMAT_UINT: + { + return read_any_gray(x0, y0, width, height); + } + case SAMPLEFORMAT_INT: + { + return read_any_gray(x0, y0, width, height); + } + default: + { + throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample"); + } + } + } + case 32: + { + switch (sample_format_) + { + case SAMPLEFORMAT_UINT: + { + return read_any_gray(x0, y0, width, height); + } + case SAMPLEFORMAT_INT: + { + return read_any_gray(x0, y0, width, height); + } + case SAMPLEFORMAT_IEEEFP: + { + return read_any_gray(x0, y0, width, height); + } + default: + { + throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample"); + } + } + } + case 64: + { + switch (sample_format_) + { + case SAMPLEFORMAT_UINT: + { + return read_any_gray(x0, y0, width, height); + } + case SAMPLEFORMAT_INT: + { + return read_any_gray(x0, y0, width, height); + } + case SAMPLEFORMAT_IEEEFP: + { + return read_any_gray(x0, y0, width, height); + } + default: + { + throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample"); + } + } + } + } + } + default: + { + //PHOTOMETRIC_PALETTE = 3; + //PHOTOMETRIC_MASK = 4; + //PHOTOMETRIC_SEPARATED = 5; + //PHOTOMETRIC_YCBCR = 6; + //PHOTOMETRIC_CIELAB = 8; + //PHOTOMETRIC_ICCLAB = 9; + //PHOTOMETRIC_ITULAB = 10; + //PHOTOMETRIC_LOGL = 32844; + //PHOTOMETRIC_LOGLUV = 32845; + image_rgba8 data(width,height, true, true); + read(x0, y0, data); + return image_any(std::move(data)); + } + } + return image_any(); +} + +namespace detail { + +struct rgb8 +{ + std::uint8_t r; + std::uint8_t g; + std::uint8_t b; +}; + +struct rgb8_to_rgba8 +{ + std::uint32_t operator() (rgb8 const& in) const + { + return ((255 << 24) | (in.r) | (in.g << 8) | (in.b << 16)); + } +}; + +template +struct tiff_reader_traits +{ + using image_type = T; + using pixel_type = typename image_type::pixel_type; + + constexpr static bool reverse = false; + static bool read_tile(TIFF * tif, std::size_t x, std::size_t y, pixel_type* buf, std::size_t tile_width, std::size_t tile_height) + { + std::uint32_t tile_size = TIFFTileSize(tif); + return (TIFFReadEncodedTile(tif, TIFFComputeTile(tif, x, y, 0, 0), buf, tile_size) != -1); + } + + static bool read_strip(TIFF * tif, std::size_t y, std::size_t rows_per_strip, std::size_t strip_width, pixel_type * buf) + { + return (TIFFReadEncodedStrip(tif, y/rows_per_strip, buf, -1) != -1); + } +}; + +// default specialization that expands into RGBA +template <> +struct tiff_reader_traits +{ + using image_type = image_rgba8; + using pixel_type = std::uint32_t; + constexpr static bool reverse = true; + static bool read_tile(TIFF * tif, std::size_t x0, std::size_t y0, pixel_type* buf, std::size_t tile_width, std::size_t tile_height) + { + return (TIFFReadRGBATile(tif, x0, y0, buf) != 0); + } + + static bool read_strip(TIFF * tif, std::size_t y, std::size_t rows_per_strip, std::size_t strip_width, pixel_type * buf) + { + return (TIFFReadRGBAStrip(tif, y, buf) != 0); + } +}; + +} + +template +template +void tiff_reader::read_generic(std::size_t, std::size_t, ImageData &) +{ + throw image_reader_exception("tiff_reader: TODO - tiff is not stripped or tiled"); +} + +template +template +void tiff_reader::read_tiled(std::size_t x0,std::size_t y0, ImageData & image) +{ + using pixel_type = typename detail::tiff_reader_traits::pixel_type; + + TIFF* tif = open(stream_); + if (tif) + { + std::uint32_t tile_size = TIFFTileSize(tif); + std::unique_ptr tile(new pixel_type[tile_size]); + std::size_t width = image.width(); + std::size_t height = image.height(); + std::size_t start_y = (y0 / tile_height_) * tile_height_; + std::size_t end_y = ((y0 + height) / tile_height_ + 1) * tile_height_; + std::size_t start_x = (x0 / tile_width_) * tile_width_; + std::size_t end_x = ((x0 + width) / tile_width_ + 1) * tile_width_; + end_y = std::min(end_y, height_); + end_x = std::min(end_x, width_); + bool pick_first_band = (bands_ > 1) && (tile_size / (tile_width_ * tile_height_ * sizeof(pixel_type)) == bands_); + for (std::size_t y = start_y; y < end_y; y += tile_height_) + { + std::size_t ty0 = std::max(y0, y) - y; + std::size_t ty1 = std::min(height + y0, y + tile_height_) - y; + + for (std::size_t x = start_x; x < end_x; x += tile_width_) + { + if (!detail::tiff_reader_traits::read_tile(tif, x, y, tile.get(), tile_width_, tile_height_)) + { + MAPNIK_LOG_DEBUG(tiff_reader) << "read_tile(...) failed at " << x << "/" << y << " for " << width_ << "/" << height_ << "\n"; + break; + } + if (pick_first_band) + { + std::uint32_t size = tile_width_ * tile_height_ * sizeof(pixel_type); + for (std::uint32_t n = 0; n < size; ++n) + { + tile[n] = tile[n * bands_]; + } + } + std::size_t tx0 = std::max(x0, x); + std::size_t tx1 = std::min(width + x0, x + tile_width_); + std::size_t row_index = y + ty0 - y0; + + if (detail::tiff_reader_traits::reverse) + { + for (std::size_t ty = ty0; ty < ty1; ++ty, ++row_index) + { + // This is in reverse because the TIFFReadRGBATile reads are inverted + image.set_row(row_index, tx0 - x0, tx1 - x0, &tile[(tile_height_ - ty - 1) * tile_width_ + tx0 - x]); + } + } + else + { + for (std::size_t ty = ty0; ty < ty1; ++ty, ++row_index) + { + image.set_row(row_index, tx0 - x0, tx1 - x0, &tile[ty * tile_width_ + tx0 - x]); + } + } + } + } + } +} + +template +template +void tiff_reader::read_stripped(std::size_t x0, std::size_t y0, ImageData & image) +{ + using pixel_type = typename detail::tiff_reader_traits::pixel_type; + TIFF* tif = open(stream_); + if (tif) + { + std::uint32_t strip_size = TIFFStripSize(tif); + std::unique_ptr strip(new pixel_type[strip_size]); + std::size_t width = image.width(); + std::size_t height = image.height(); + + std::size_t start_y = (y0 / rows_per_strip_) * rows_per_strip_; + std::size_t end_y = std::min(y0 + height, height_); + std::size_t tx0, tx1, ty0, ty1; + tx0 = x0; + tx1 = std::min(width + x0, width_); + std::size_t row = 0; + bool pick_first_band = (bands_ > 1) && (strip_size / (width_ * rows_per_strip_ * sizeof(pixel_type)) == bands_); + for (std::size_t y = start_y; y < end_y; y += rows_per_strip_) + { + ty0 = std::max(y0, y) - y; + ty1 = std::min(end_y, y + rows_per_strip_) - y; + + if (!detail::tiff_reader_traits::read_strip(tif, y, rows_per_strip_, width_, strip.get())) + { + MAPNIK_LOG_DEBUG(tiff_reader) << "TIFFRead(Encoded|RGBA)Strip failed at " << y << " for " << width_ << "/" << height_ << "\n"; + break; + } + if (pick_first_band) + { + std::uint32_t size = width_ * rows_per_strip_ * sizeof(pixel_type); + for (std::uint32_t n = 0; n < size; ++n) + { + strip[n] = strip[bands_ * n]; + } + } + + if (detail::tiff_reader_traits::reverse) + { + std::size_t num_rows = std::min(height_ - y, static_cast(rows_per_strip_)); + for (std::size_t ty = ty0; ty < ty1; ++ty) + { + // This is in reverse because the TIFFReadRGBAStrip reads are inverted + image.set_row(row++, tx0 - x0, tx1 - x0, &strip[(num_rows - ty - 1) * width_ + tx0]); + } + } + else + { + for (std::size_t ty = ty0; ty < ty1; ++ty) + { + image.set_row(row++, tx0 - x0, tx1 - x0, &strip[ty * width_ + tx0]); + } + } + } + } +} + +template +TIFF* tiff_reader::open(std::istream & input) +{ + if (!tif_) + { + tif_ = tiff_ptr(TIFFClientOpen("tiff_input_stream", "rcm", + reinterpret_cast(&input), + detail::tiff_read_proc, + detail::tiff_write_proc, + detail::tiff_seek_proc, + detail::tiff_close_proc, + detail::tiff_size_proc, + detail::tiff_map_proc, + detail::tiff_unmap_proc), tiff_closer()); + } + return tif_.get(); +} + +} diff --git a/test/unit/imaging/tiff_io.cpp b/test/unit/imaging/tiff_io.cpp index 4145285fa..567c3d854 100644 --- a/test/unit/imaging/tiff_io.cpp +++ b/test/unit/imaging/tiff_io.cpp @@ -1,7 +1,7 @@ // disabled on windows due to https://github.com/mapnik/mapnik/issues/2838 // TODO - get to the bottom of why including `tiff_reader.cpp` breaks windows // or re-write image_readers to allow `#include tiff_reader.hpp` -#if !defined(_MSC_VER) && defined(HAVE_TIFF) +#ifdef HAVE_TIFF #include "catch.hpp" @@ -12,7 +12,7 @@ #include #include #include -#include "../../../src/tiff_reader.cpp" +#include "../../../src/tiff_reader.hpp" #if defined(MAPNIK_MEMORY_MAPPED_FILE) using source_type = boost::interprocess::ibufferstream; From ef062ac3144addfeab90b340fd29c95c4e024fa3 Mon Sep 17 00:00:00 2001 From: Mathis Logemann Date: Thu, 27 Jan 2022 18:42:40 +0100 Subject: [PATCH 9/9] [unit-tests] mark text/nested script runs as unicode 16 --- test/unit/text/script_runs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/text/script_runs.cpp b/test/unit/text/script_runs.cpp index 10eec295f..8b101f456 100644 --- a/test/unit/text/script_runs.cpp +++ b/test/unit/text/script_runs.cpp @@ -5,7 +5,7 @@ TEST_CASE("nested script runs") { - mapnik::value_unicode_string text("Nested text runs(первый(second(третий)))"); //mixed scripts + mapnik::value_unicode_string text(u"Nested text runs(первый(second(третий)))"); //mixed scripts ScriptRun runs(text.getBuffer(), text.length()); std::size_t count = 0; std::size_t size = 0;