From 2e8c0d36c2237f2815d8004c1b96bad909056eb9 Mon Sep 17 00:00:00 2001 From: artemp Date: Wed, 22 Jun 2016 17:21:25 +0100 Subject: [PATCH 1/4] implement custom `char_array_bufrer` and remove boost::iostreams dependency --- include/mapnik/util/char_array_buffer.hpp | 89 +++++++++++++++++++++++ src/jpeg_reader.cpp | 30 ++++---- src/png_reader.cpp | 32 ++++---- src/tiff_reader.cpp | 34 ++++----- src/webp_reader.cpp | 3 - test/unit/imaging/tiff_io.cpp | 19 ++--- 6 files changed, 139 insertions(+), 68 deletions(-) create mode 100644 include/mapnik/util/char_array_buffer.hpp diff --git a/include/mapnik/util/char_array_buffer.hpp b/include/mapnik/util/char_array_buffer.hpp new file mode 100644 index 000000000..11d6801c1 --- /dev/null +++ b/include/mapnik/util/char_array_buffer.hpp @@ -0,0 +1,89 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2016 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef MAPNIK_UTIL_CHAR_ARRAY_BUFFER_HPP +#define MAPNIK_UTIL_CHAR_ARRAY_BUFFER_HPP + +#include +#include + +namespace mapnik { namespace util { + +// ref https://artofcode.wordpress.com/2010/12/12/deriving-from-stdstreambuf/ + +class char_array_buffer : public std::streambuf +{ +public: + char_array_buffer(char const* data, std::size_t size) + : begin_(data), end_(data + size), current_(data) {} + +private: + int_type underflow() + { + if (current_ == end_) + { + return traits_type::eof(); + } + return traits_type::to_int_type(*current_); + } + + int_type uflow() + { + if (current_ == end_) + { + return traits_type::eof(); + } + return traits_type::to_int_type(*current_++); + } + + int_type pbackfail(int_type ch) + { + std::cerr << "pbackfail" << std::endl; + if (current_ == begin_ || (ch != traits_type::eof() && ch != current_[-1])) + { + return traits_type::eof(); + } + return traits_type::to_int_type(*--current_); + } + + std::streamsize showmanyc() + { + return end_ - current_; + } + + pos_type seekoff(off_type off, std::ios_base::seekdir dir, + std::ios_base::openmode which = std::ios_base::in | std::ios_base::out ) + { + if (dir == std::ios_base::beg) current_ = std::min(begin_ + off, end_); + else if (dir == std::ios_base::cur) current_ = std::min(current_ + off, end_); + else current_ = std::max(end_ - off, begin_); // dir == std::ios_base::end + return pos_type(off_type(current_ - begin_)); + } + char const * const begin_; + char const * const end_; + char const * current_; +}; + +}} + + +#endif // MAPNIK_UTIL_CHAR_ARRAY_BUFFER_HPP diff --git a/src/jpeg_reader.cpp b/src/jpeg_reader.cpp index 84c805ee8..6b6e1b127 100644 --- a/src/jpeg_reader.cpp +++ b/src/jpeg_reader.cpp @@ -22,6 +22,7 @@ // mapnik #include +#include #include // jpeg @@ -30,16 +31,10 @@ extern "C" #include } -#pragma GCC diagnostic push -#include -#include -#include -#include -#pragma GCC diagnostic pop - // std #include #include +#include namespace mapnik { @@ -49,7 +44,7 @@ class jpeg_reader : public image_reader { public: using source_type = T; - using input_stream = boost::iostreams::stream; + using input_stream = std::iostream; const static unsigned BUF_SIZE = 4096; private: struct jpeg_stream_wrapper @@ -77,7 +72,7 @@ private: unsigned width_; unsigned height_; public: - explicit jpeg_reader(std::string const& file_name); + explicit jpeg_reader(std::string const& filename); explicit jpeg_reader(char const* data, size_t size); ~jpeg_reader(); unsigned width() const final; @@ -99,14 +94,14 @@ private: namespace { -image_reader* create_jpeg_reader(std::string const& file) +image_reader* create_jpeg_reader(std::string const& filename) { - return new jpeg_reader(file); + return new jpeg_reader(filename); } image_reader* create_jpeg_reader2(char const* data, size_t size) { - return new jpeg_reader(data, size); + return new jpeg_reader(data, size); } const bool registered = register_image_reader("jpeg",create_jpeg_reader); @@ -115,20 +110,21 @@ const bool registered2 = register_image_reader("jpeg",create_jpeg_reader2); // ctors template -jpeg_reader::jpeg_reader(std::string const& file_name) - : source_(file_name,std::ios_base::in | std::ios_base::binary), - stream_(source_), +jpeg_reader::jpeg_reader(std::string const& filename) + : source_(), + stream_(&source_), width_(0), height_(0) { - if (!stream_) throw image_reader_exception("cannot open image file "+ file_name); + source_.open(filename, std::ios_base::in | std::ios_base::binary); + if (!stream_) throw image_reader_exception("cannot open image file "+ filename); init(); } template jpeg_reader::jpeg_reader(char const* data, size_t size) : source_(data, size), - stream_(source_), + stream_(&source_), width_(0), height_(0) { diff --git a/src/png_reader.cpp b/src/png_reader.cpp index 30b470fd6..8c58b2238 100644 --- a/src/png_reader.cpp +++ b/src/png_reader.cpp @@ -23,22 +23,17 @@ // mapnik #include #include +#include extern "C" { #include } -#pragma GCC diagnostic push -#include -#include -#include -#include -#pragma GCC diagnostic pop - // stl #include #include +#include namespace mapnik { @@ -47,7 +42,7 @@ template class png_reader : public image_reader { using source_type = T; - using input_stream = boost::iostreams::stream; + using input_stream = std::istream; struct png_struct_guard { @@ -73,7 +68,7 @@ private: int color_type_; bool has_alpha_; public: - explicit png_reader(std::string const& file_name); + explicit png_reader(std::string const& filename); png_reader(char const* data, std::size_t size); ~png_reader(); unsigned width() const final; @@ -90,14 +85,14 @@ private: namespace { -image_reader* create_png_reader(std::string const& file) +image_reader* create_png_reader(std::string const& filename) { - return new png_reader(file); + return new png_reader(filename); } image_reader* create_png_reader2(char const * data, std::size_t size) { - return new png_reader(data, size); + return new png_reader(data, size); } const bool registered = register_image_reader("png",create_png_reader); @@ -128,31 +123,30 @@ void png_reader::png_read_data(png_structp png_ptr, png_bytep data, png_size_ } template -png_reader::png_reader(std::string const& file_name) - : source_(file_name,std::ios_base::in | std::ios_base::binary), - stream_(source_), +png_reader::png_reader(std::string const& filename) + : source_(), + stream_(&source_), width_(0), height_(0), bit_depth_(0), color_type_(0), has_alpha_(false) { - if (!source_.is_open()) throw image_reader_exception("PNG reader: cannot open file '"+ file_name + "'"); - if (!stream_) throw image_reader_exception("PNG reader: cannot open file '"+ file_name + "'"); + source_.open(filename, std::ios_base::in | std::ios_base::binary); + if (!source_.is_open()) throw image_reader_exception("PNG reader: cannot open file '"+ filename + "'"); init(); } template png_reader::png_reader(char const* data, std::size_t size) : source_(data,size), - stream_(source_), + stream_(&source_), width_(0), height_(0), bit_depth_(0), color_type_(0), has_alpha_(false) { - if (!stream_) throw image_reader_exception("PNG reader: cannot open image stream"); init(); } diff --git a/src/tiff_reader.cpp b/src/tiff_reader.cpp index 1f885407e..e58faf14a 100644 --- a/src/tiff_reader.cpp +++ b/src/tiff_reader.cpp @@ -23,21 +23,15 @@ // mapnik #include #include - +#include extern "C" { #include } -#pragma GCC diagnostic push -#include -#include -#include -#include -#pragma GCC diagnostic pop - // stl #include +#include namespace mapnik { namespace impl { @@ -106,7 +100,7 @@ class tiff_reader : public image_reader { using tiff_ptr = std::shared_ptr; using source_type = T; - using input_stream = boost::iostreams::stream; + using input_stream = std::istream; struct tiff_closer { @@ -145,7 +139,7 @@ public: stripped, tiled }; - explicit tiff_reader(std::string const& file_name); + explicit tiff_reader(std::string const& filename); tiff_reader(char const* data, std::size_t size); virtual ~tiff_reader(); unsigned width() const final; @@ -183,14 +177,14 @@ private: namespace { -image_reader* create_tiff_reader(std::string const& file) +image_reader* create_tiff_reader(std::string const& filename) { - return new tiff_reader(file); + return new tiff_reader(filename); } image_reader* create_tiff_reader2(char const * data, std::size_t size) { - return new tiff_reader(data, size); + return new tiff_reader(data, size); } const bool registered = register_image_reader("tiff",create_tiff_reader); @@ -199,9 +193,9 @@ const bool registered2 = register_image_reader("tiff", create_tiff_reader2); } template -tiff_reader::tiff_reader(std::string const& file_name) - : source_(file_name, std::ios_base::in | std::ios_base::binary), - stream_(source_), +tiff_reader::tiff_reader(std::string const& filename) + : source_(), + stream_(&source_), tif_(nullptr), read_method_(generic), rows_per_strip_(0), @@ -218,14 +212,16 @@ tiff_reader::tiff_reader(std::string const& file_name) has_alpha_(false), is_tiled_(false) { - if (!stream_) throw image_reader_exception("TIFF reader: cannot open file "+ file_name); + source_.open(filename, std::ios_base::in | std::ios_base::binary); + if (!source_.is_open()) throw image_reader_exception("TIFF reader: cannot open file "+ filename); + if (!stream_) throw image_reader_exception("TIFF reader: cannot open file "+ filename); init(); } template tiff_reader::tiff_reader(char const* data, std::size_t size) : source_(data, size), - stream_(source_), + stream_(&source_), tif_(nullptr), read_method_(generic), rows_per_strip_(0), @@ -243,8 +239,6 @@ tiff_reader::tiff_reader(char const* data, std::size_t size) is_tiled_(false) { if (!stream_) throw image_reader_exception("TIFF reader: cannot open image stream "); - stream_.rdbuf()->pubsetbuf(0, 0); - stream_.seekg(0, std::ios::beg); init(); } diff --git a/src/webp_reader.cpp b/src/webp_reader.cpp index c806df9c1..ca611aeef 100644 --- a/src/webp_reader.cpp +++ b/src/webp_reader.cpp @@ -33,9 +33,6 @@ extern "C" #include } -#include -#include -#include #pragma GCC diagnostic pop // stl diff --git a/test/unit/imaging/tiff_io.cpp b/test/unit/imaging/tiff_io.cpp index 1ace38fa1..14f9e5f29 100644 --- a/test/unit/imaging/tiff_io.cpp +++ b/test/unit/imaging/tiff_io.cpp @@ -1,4 +1,3 @@ - // disabled on windows due to https://github.com/mapnik/mapnik/issues/2838 // TODO - get to the bottom of why including `tiff_reader.cpp` breaks windows // or re-write image_readers to allow `#include tiff_reader.hpp` @@ -12,7 +11,7 @@ #include "../../../src/tiff_reader.cpp" #define TIFF_ASSERT(filename) \ - mapnik::tiff_reader tiff_reader(filename); \ + mapnik::tiff_reader tiff_reader(filename); \ REQUIRE( tiff_reader.width() == 256 ); \ REQUIRE( tiff_reader.height() == 256 ); \ REQUIRE( tiff_reader.planar_config() == PLANARCONFIG_CONTIG ); \ @@ -20,7 +19,7 @@ REQUIRE( reader->width() == 256 ); \ REQUIRE( reader->height() == 256 ); \ mapnik::util::file file(filename); \ - mapnik::tiff_reader tiff_reader2(file.data().get(),file.size()); \ + mapnik::tiff_reader tiff_reader2(file.data().get(),file.size()); \ REQUIRE( tiff_reader2.width() == 256 ); \ REQUIRE( tiff_reader2.height() == 256 ); \ std::unique_ptr reader2(mapnik::get_image_reader(file.data().get(),file.size())); \ @@ -57,11 +56,13 @@ REQUIRE( subimage.width() == 1 ); \ REQUIRE( subimage.height() == 1 ); \ -TEST_CASE("tiff io") { +TEST_CASE("tiff io") +{ -SECTION("scan rgb8 striped") { +SECTION("scan rgb8 striped") +{ std::string filename("./test/data/tiff/scan_512x512_rgb8_striped.tif"); - mapnik::tiff_reader tiff_reader(filename); + mapnik::tiff_reader tiff_reader(filename); REQUIRE( tiff_reader.width() == 512 ); REQUIRE( tiff_reader.height() == 512 ); REQUIRE( tiff_reader.planar_config() == PLANARCONFIG_CONTIG ); @@ -76,7 +77,7 @@ SECTION("scan rgb8 striped") { REQUIRE( reader->width() == 512 ); REQUIRE( reader->height() == 512 ); mapnik::util::file file(filename); - mapnik::tiff_reader tiff_reader2(file.data().get(),file.size()); + mapnik::tiff_reader tiff_reader2(file.data().get(),file.size()); REQUIRE( tiff_reader2.width() == 512 ); REQUIRE( tiff_reader2.height() == 512 ); std::unique_ptr reader2(mapnik::get_image_reader(file.data().get(),file.size())); @@ -91,7 +92,7 @@ SECTION("scan rgb8 striped") { SECTION("scan rgb8 tiled") { std::string filename("./test/data/tiff/scan_512x512_rgb8_tiled.tif"); - mapnik::tiff_reader tiff_reader(filename); + mapnik::tiff_reader tiff_reader(filename); REQUIRE( tiff_reader.width() == 512 ); REQUIRE( tiff_reader.height() == 512 ); REQUIRE( tiff_reader.planar_config() == PLANARCONFIG_CONTIG ); @@ -106,7 +107,7 @@ SECTION("scan rgb8 tiled") { REQUIRE( reader->width() == 512 ); REQUIRE( reader->height() == 512 ); mapnik::util::file file(filename); - mapnik::tiff_reader tiff_reader2(file.data().get(),file.size()); + mapnik::tiff_reader tiff_reader2(file.data().get(),file.size()); REQUIRE( tiff_reader2.width() == 512 ); REQUIRE( tiff_reader2.height() == 512 ); std::unique_ptr reader2(mapnik::get_image_reader(file.data().get(),file.size())); From 7de7464cafa4d566c2de6587b7038b2dc3990e53 Mon Sep 17 00:00:00 2001 From: artemp Date: Wed, 22 Jun 2016 17:25:37 +0100 Subject: [PATCH 2/4] tidy --- src/tiff_reader.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tiff_reader.cpp b/src/tiff_reader.cpp index e58faf14a..157e4d0cb 100644 --- a/src/tiff_reader.cpp +++ b/src/tiff_reader.cpp @@ -213,8 +213,7 @@ tiff_reader::tiff_reader(std::string const& filename) is_tiled_(false) { source_.open(filename, std::ios_base::in | std::ios_base::binary); - if (!source_.is_open()) throw image_reader_exception("TIFF reader: cannot open file "+ filename); - if (!stream_) throw image_reader_exception("TIFF reader: cannot open file "+ filename); + if (!stream_) throw image_reader_exception("TIFF reader: cannot open file "+ filename); init(); } From 3536ad09028e982c1289025eb9c97446b71248c8 Mon Sep 17 00:00:00 2001 From: artemp Date: Wed, 22 Jun 2016 17:35:09 +0100 Subject: [PATCH 3/4] even more tidy - remove left over stderr --- include/mapnik/util/char_array_buffer.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/mapnik/util/char_array_buffer.hpp b/include/mapnik/util/char_array_buffer.hpp index 11d6801c1..e5b834baa 100644 --- a/include/mapnik/util/char_array_buffer.hpp +++ b/include/mapnik/util/char_array_buffer.hpp @@ -24,7 +24,6 @@ #define MAPNIK_UTIL_CHAR_ARRAY_BUFFER_HPP #include -#include namespace mapnik { namespace util { @@ -57,7 +56,6 @@ private: int_type pbackfail(int_type ch) { - std::cerr << "pbackfail" << std::endl; if (current_ == begin_ || (ch != traits_type::eof() && ch != current_[-1])) { return traits_type::eof(); From 00e5d005304b5429a2e36334a64f5c84441d7d10 Mon Sep 17 00:00:00 2001 From: Artem Pavlenko Date: Thu, 23 Jun 2016 09:46:45 +0100 Subject: [PATCH 4/4] Fix unbound variable when number of JOBS is omitted --- test/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run b/test/run index 54c4d6fb4..014b79861 100755 --- a/test/run +++ b/test/run @@ -42,7 +42,7 @@ if [ -d "test/data" ]; then if [ -d "test/data-visual/styles" ]; then run_substep "Running visual tests..." - if [ -z "$JOBS" ]; then + if [ -z "${JOBS:-}" ]; then JOBS=1 fi if [[ -f ./test/visual/run ]]; then