new visual test runner

test/build.py

@@ -23,6 +23,9 @@ else:
         test_env.PrependUnique(CPPPATH=test_env['CAIRO_CPPPATHS'])
         test_env.Append(CPPDEFINES = '-DHAVE_CAIRO')
     test_env.PrependUnique(CPPPATH=['./'])
+    if test_env['PLATFORM'] == 'Linux':
+        test_env['LINKFLAGS'].append('-pthread')
+    test_env.AppendUnique(LIBS='boost_program_options%s' % env['BOOST_APPEND'])
     test_env_local = test_env.Clone()
 
 
@@ -39,6 +42,18 @@ else:
         test_program = test_env_local.Program(standalone.replace('.cpp','-bin'), source=standalone)
         Depends(test_program, env.subst('../src/%s' % env['MAPNIK_LIB_NAME']))
 
+    # visual tests
+    source = Split(
+        """
+        visual/config.cpp
+        visual/report.cpp
+        visual/runner.cpp
+        visual/run.cpp
+        """
+        )
+    test_program = test_env_local.Program('visual/run', source=source)
+    Depends(test_program, env.subst('../src/%s' % env['MAPNIK_LIB_NAME']))
+
     # build locally if installing
     if 'install' in COMMAND_LINE_TARGETS:
         env.Alias('install',test_program)

test/run

@@ -23,7 +23,7 @@ if [[ -f mapnik-settings.env ]]; then
 fi
 
 run_substep "Running C++ Unit tests..."
-./test/unit/run 
+./test/unit/run
 failures=$((failures+$?))
 
 run_substep "Running standalone C++ tests..."
@@ -34,4 +34,8 @@ if [ -n "$(find test/standalone/ -maxdepth 1 -name '*-bin' -print -quit)" ]; the
     done
 fi
 
-exit $failures
+run_substep "Running visual tests..."
+./test/visual/run
+failures=$((failures+$?))
+
+exit $failures

test/visual/compare_images.hpp

@@ -0,0 +1,53 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef COMPARE_IMAGES_HPP
+#define COMPARE_IMAGES_HPP
+
+// stl
+#include <memory>
+
+// mapnik
+#include <mapnik/image_util.hpp>
+#include <mapnik/image_reader.hpp>
+
+namespace visual_tests
+{
+
+template <typename Image>
+unsigned compare_images(Image const & actual, std::string const & reference)
+{
+    std::unique_ptr<mapnik::image_reader> reader(mapnik::get_image_reader(reference, "png"));
+    if (!reader.get())
+    {
+        throw mapnik::image_reader_exception("Failed to load: " + reference);
+    }
+
+    mapnik::image_any ref_image_any = reader->read(0, 0, reader->width(), reader->height());
+    Image const & reference_image = mapnik::util::get<Image>(ref_image_any);
+
+    return mapnik::compare(actual, reference_image, 0, true);
+}
+
+}
+
+#endif

test/visual/config.cpp

@@ -0,0 +1,40 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+// mapnik
+#include <mapnik/util/fs.hpp>
+
+#include "config.hpp"
+
+namespace visual_tests
+{
+
+bool ensure_dir(boost::filesystem::path const & dir)
+{
+    if (!mapnik::util::exists(dir.string()))
+    {
+        return boost::filesystem::create_directories(dir);
+    }
+    return true;
+}
+
+}

test/visual/config.hpp

@@ -0,0 +1,82 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef CONFIG_HPP
+#define CONFIG_HPP
+
+// stl
+#include <vector>
+#include <string>
+
+// boost
+#include <boost/filesystem.hpp>
+
+namespace visual_tests
+{
+
+struct map_size
+{
+    map_size(int width, int height) : width(width), height(height) { }
+    map_size() { }
+    unsigned width;
+    unsigned height;
+};
+
+struct config
+{
+    config() : status(true),
+               scales({ 1.0, 2.0 }),
+               sizes({ { 500, 100 } }) { }
+
+    bool status;
+    std::vector<double> scales;
+    std::vector<map_size> sizes;
+};
+
+enum result_state : std::uint8_t
+{
+    STATE_OK,
+    STATE_FAIL,
+    STATE_ERROR,
+    STATE_OVERWRITE
+};
+
+struct result
+{
+    std::string name;
+    result_state state;
+    char const * renderer_name;
+    map_size size;
+    double scale_factor;
+    boost::filesystem::path actual_image_path;
+    boost::filesystem::path reference_image_path;
+    std::string error_message;
+    unsigned diff;
+};
+
+using result_list = std::vector<result>;
+
+bool ensure_dir(boost::filesystem::path const & dir);
+
+}
+
+#endif

test/visual/map_sizes_grammar.hpp

@@ -0,0 +1,62 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef MAP_SIZES_GRAMMAR_HPP
+#define MAP_SIZES_GRAMMAR_HPP
+
+// boost
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#pragma GCC diagnostic ignored "-Wunused-local-typedef"
+#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+#include <boost/spirit/include/qi.hpp>
+#include <boost/spirit/include/phoenix.hpp>
+#pragma GCC diagnostic pop
+
+namespace visual_tests
+{
+
+namespace qi = boost::spirit::qi;
+namespace ascii = boost::spirit::ascii;
+
+template <typename Iterator>
+struct map_sizes_grammar : qi::grammar<Iterator, std::vector<map_size>(), ascii::space_type>
+{
+    map_sizes_grammar() : map_sizes_grammar::base_type(start)
+    {
+        using namespace boost::spirit::qi;
+        using namespace boost::phoenix;
+
+        int_type int_;
+        _1_type _1;
+        _2_type _2;
+        _val_type _val;
+
+        start = (int_ >> ',' >> int_)[push_back(_val, construct<map_size>(_1, _2))] % ';';
+    }
+
+    qi::rule<Iterator, std::vector<map_size>(), ascii::space_type> start;
+};
+
+}
+
+#endif

test/visual/renderer.hpp

@@ -0,0 +1,170 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef RENDERER_HPP
+#define RENDERER_HPP
+
+#include "compare_images.hpp"
+
+// stl
+#include <sstream>
+#include <iomanip>
+
+// mapnik
+#include <mapnik/map.hpp>
+#include <mapnik/agg_renderer.hpp>
+#if defined(HAVE_CAIRO)
+#include <mapnik/cairo/cairo_renderer.hpp>
+#include <mapnik/cairo/cairo_image_util.hpp>
+#endif
+
+// boost
+#include <boost/filesystem.hpp>
+
+namespace visual_tests
+{
+
+template <typename ImageType>
+struct renderer_base
+{
+    using image_type = ImageType;
+
+    unsigned compare(image_type const & actual, boost::filesystem::path const& reference) const
+    {
+        return compare_images(actual, reference.string());
+    }
+
+    void save(image_type const & image, boost::filesystem::path const& path) const
+    {
+        mapnik::save_to_file(image, path.string(), "png32");
+    }
+};
+
+struct agg_renderer : renderer_base<mapnik::image_rgba8>
+{
+    static constexpr const char * name = "agg";
+
+    image_type render(mapnik::Map const & map, double scale_factor) const
+    {
+        image_type image(map.width(), map.height());
+        mapnik::agg_renderer<image_type> ren(map, image, scale_factor);
+        ren.apply();
+        return image;
+    }
+};
+
+#if defined(HAVE_CAIRO)
+struct cairo_renderer : renderer_base<mapnik::image_rgba8>
+{
+    static constexpr const char * name = "cairo";
+
+    image_type render(mapnik::Map const & map, double scale_factor) const
+    {
+        mapnik::cairo_surface_ptr image_surface(
+            cairo_image_surface_create(CAIRO_FORMAT_ARGB32, map.width(), map.height()),
+            mapnik::cairo_surface_closer());
+        mapnik::cairo_ptr image_context(mapnik::create_context(image_surface));
+        mapnik::cairo_renderer<mapnik::cairo_ptr> ren(map, image_context, scale_factor);
+        ren.apply();
+        image_type image(map.width(), map.height());
+        mapnik::cairo_image_to_rgba8(image, image_surface);
+        return image;
+    }
+};
+#endif
+
+struct grid_renderer : renderer_base<mapnik::image_gray8>
+{
+    static constexpr const char * name = "grid";
+
+    image_type render(mapnik::Map const & map, double scale_factor) const
+    {
+        image_type image(map.width(), map.height());
+        // TODO: Render grid here.
+        return image;
+    }
+};
+
+template <typename Renderer>
+class renderer
+{
+public:
+    renderer(boost::filesystem::path const & output_dir, boost::filesystem::path const & reference_dir, bool overwrite)
+        : output_dir(output_dir), reference_dir(reference_dir), overwrite(overwrite)
+    {
+    }
+
+    result test(std::string const & name, mapnik::Map const & map, double scale_factor) const
+    {
+        typename Renderer::image_type image(ren.render(map, scale_factor));
+        boost::filesystem::path reference = reference_dir / image_file_name(name, map.width(), map.height(), scale_factor, true);
+        result res;
+
+        res.state = STATE_OK;
+        res.name = name;
+        res.renderer_name = Renderer::name;
+        res.scale_factor = scale_factor;
+        res.size = map_size(map.width(), map.height());
+        res.reference_image_path = reference;
+        res.diff = ren.compare(image, reference);
+
+        if (res.diff)
+        {
+            ensure_dir(output_dir);
+            boost::filesystem::path path = output_dir / image_file_name(name, map.width(), map.height(), scale_factor);
+            res.actual_image_path = path;
+            res.state = STATE_FAIL;
+            ren.save(image, path);
+
+            if (overwrite)
+            {
+                ren.save(image, reference);
+                res.state = STATE_OVERWRITE;
+            }
+        }
+
+        return res;
+    }
+
+private:
+    std::string image_file_name(std::string const & test_name,
+                                double width,
+                                double height,
+                                double scale_factor,
+                                bool reference=false) const
+    {
+        std::stringstream s;
+        s << test_name << '-' << width << '-' << height << '-'
+          << std::fixed << std::setprecision(1) << scale_factor
+          << '-' << Renderer::name << (reference ? "-reference" : "") << ".png";
+        return s.str();
+    }
+
+    Renderer ren;
+    boost::filesystem::path const & output_dir;
+    boost::filesystem::path const & reference_dir;
+    const bool overwrite;
+};
+
+}
+
+#endif

test/visual/report.cpp

@@ -0,0 +1,170 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+// stl
+#include <iomanip>
+#include <fstream>
+#include <numeric>
+
+#include "report.hpp"
+
+namespace visual_tests
+{
+
+void console_report::report(result const & r)
+{
+    s << '"' << r.name << '-' << r.size.width << '-' << r.size.height << '-' << std::fixed <<
+        std::setprecision(1) << r.scale_factor << "\" with " << r.renderer_name << "... ";
+
+    switch (r.state)
+    {
+        case STATE_OK:
+            s << "OK";
+            break;
+        case STATE_FAIL:
+            s << "FAILED (" << r.diff << " different pixels)";
+            break;
+        case STATE_OVERWRITE:
+            s << "OVERWRITTEN (" << r.diff << " different pixels)";
+            break;
+        case STATE_ERROR:
+            s << "ERROR (" << r.error_message << ")";
+            break;
+    }
+
+    s << std::endl;
+}
+
+unsigned console_report::summary(result_list const & results)
+{
+    unsigned ok = 0;
+    unsigned error = 0;
+    unsigned fail = 0;
+    unsigned overwrite = 0;
+
+    for (auto const & r : results)
+    {
+        switch (r.state)
+        {
+            case STATE_OK: ok++; break;
+            case STATE_FAIL: fail++; break;
+            case STATE_ERROR: error++; break;
+            case STATE_OVERWRITE: overwrite++; break;
+        }
+    }
+
+    s << std::endl;
+    s << "Visual rendering: " << fail << " failed / " << ok << " passed / "
+        << overwrite << " overwritten / " << error << " errors" << std::endl;
+
+    return fail + error;
+}
+
+void console_short_report::report(result const & r)
+{
+    s << '.';
+}
+
+void html_report::report(result const & r, boost::filesystem::path const & output_dir)
+{
+    using namespace boost::filesystem;
+
+    path reference = output_dir / r.reference_image_path.filename();
+    path actual = output_dir / r.actual_image_path.filename();
+
+    copy_file(r.reference_image_path, reference, copy_option::overwrite_if_exists);
+    copy_file(r.actual_image_path, actual, copy_option::overwrite_if_exists);
+
+     s << "<div class=\"expected\">\n"
+          "  <a href=" << reference.filename() << ">\n"
+          "    <img src=" << reference.filename() << " width=\"100\%\">\n"
+          "  </a>\n"
+          "</div>\n"
+          "<div class=\"text\">" << r.diff << "</div>\n"
+          "<div class=\"actual\">\n"
+          "  <a href=" << actual.filename() << ">\n"
+          "    <img src=" << actual.filename() << " width=\"100\%\">\n"
+          "  </a>\n"
+          "</div>\n";
+}
+
+constexpr const char * html_header = R"template(<!DOCTYPE html>
+<html>
+<head>
+  <style>
+    body { margin:10; padding:10; }
+    .expected {
+       float:left;
+       border-width:1px;
+       border-style:solid;
+       width:45%;
+    }    
+    .actual {
+       float:right;
+       border-width:1px;
+       border-style:solid;
+       width:45%;
+    }
+    .text {
+       float:left;
+    }
+  </style>
+</head>
+<body>
+<script>
+</script>
+<div id='results'>
+     <div class="expected">expected</div>
+     <div class="text">% difference</div>
+     <div class="actual">actual</div>
+)template";
+
+constexpr const char * html_footer = R"template(</div>
+</body>
+</html>)template";
+
+void html_report::summary(result_list const & results, boost::filesystem::path const & output_dir)
+{
+    s << html_header;
+
+    for (auto const & r : results)
+    {
+        if (r.state != STATE_OK)
+        {
+            report(r, output_dir);
+        }
+    }
+
+    s << html_footer;
+}
+
+void html_summary(result_list const & results, boost::filesystem::path output_dir)
+{
+    boost::filesystem::path html_root = output_dir / "visual-test-results";
+    ensure_dir(html_root);
+    boost::filesystem::path html_report_path = html_root / "comparison.html";
+    std::ofstream output_file(html_report_path.string());
+    html_report report(output_file);
+    report.summary(results, html_root);
+}
+
+}

test/visual/report.hpp

@@ -0,0 +1,124 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef CONSOLE_REPORT_HPP
+#define CONSOLE_REPORT_HPP
+
+// stl
+#include <iostream>
+
+#include <mapnik/util/variant.hpp>
+
+#include "config.hpp"
+
+namespace visual_tests
+{
+
+class console_report
+{
+public:
+    console_report() : s(std::clog)
+    {
+    }
+
+    console_report(std::ostream & s) : s(s)
+    {
+    }
+
+    void report(result const & r);
+    unsigned summary(result_list const & results);
+
+protected:
+    std::ostream & s;
+};
+
+class console_short_report : public console_report
+{
+public:
+    console_short_report() : console_report()
+    {
+    }
+
+    console_short_report(std::ostream & s) : console_report(s)
+    {
+    }
+
+    void report(result const & r);
+};
+
+class html_report
+{
+public:
+    html_report(std::ostream & s) : s(s)
+    {
+    }
+
+    void report(result const & r, boost::filesystem::path const & output_dir);
+    void summary(result_list const & results, boost::filesystem::path const & output_dir);
+
+protected:
+    std::ostream & s;
+};
+
+using report_type = mapnik::util::variant<console_report, console_short_report>;
+
+class report_visitor
+{
+public:
+    report_visitor(result const & r)
+        : result_(r)
+    {
+    }
+
+    template <typename T>
+    void operator()(T & report) const
+    {
+        return report.report(result_);
+    }
+
+private:
+    result const & result_;
+};
+
+class summary_visitor
+{
+public:
+    summary_visitor(result_list const & r)
+        : result_(r)
+    {
+    }
+
+    template <typename T>
+    unsigned operator()(T & report) const
+    {
+        return report.summary(result_);
+    }
+
+private:
+    result_list const & result_;
+};
+
+void html_summary(result_list const & results, boost::filesystem::path output_dir);
+
+}
+
+#endif

test/visual/run.cpp

@@ -0,0 +1,103 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#include "runner.hpp"
+#include "config.hpp"
+
+#include <mapnik/datasource_cache.hpp>
+#include <mapnik/font_engine_freetype.hpp>
+
+// boost
+#include <boost/program_options.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <boost/uuid/uuid_generators.hpp>
+#include <boost/uuid/uuid_io.hpp>
+
+int main(int argc, char** argv)
+{
+    using namespace visual_tests;
+    namespace po = boost::program_options;
+
+    po::options_description desc("visual test runner");
+    desc.add_options()
+        ("help,h", "produce usage message")
+        ("verbose,v", "verbose output")
+        ("overwrite,o", "overwrite reference image")
+        ("jobs,j", po::value<std::size_t>()->default_value(1), "number of paralel threads")
+        ("styles-dir", po::value<std::string>()->default_value("test/data-visual/styles"), "directory with styles")
+        ("images-dir", po::value<std::string>()->default_value("test/data-visual/images"), "directory with reference images")
+        ("output-dir", po::value<std::string>()->default_value("/tmp/mapnik-visual-images"), "directory for output files")
+        ("uuid-subdir,u", "write output files to subdirectory with unique name")
+        ("styles", po::value<std::vector<std::string>>(), "selected styles to test")
+        ("fonts", po::value<std::string>()->default_value("fonts"), "font search path")
+        ("plugins", po::value<std::string>()->default_value("plugins/input"), "input plugins search path")
+        ;
+
+    po::positional_options_description p;
+    p.add("styles", -1);
+    po::variables_map vm;
+    po::store(po::command_line_parser(argc, argv).options(desc).positional(p).run(), vm);
+    po::notify(vm);
+
+    if (vm.count("help"))
+    {
+        std::clog << desc << std::endl;
+        return 1;
+    }
+
+    mapnik::freetype_engine::register_fonts(vm["fonts"].as<std::string>(), true);
+    mapnik::datasource_cache::instance().register_datasources(vm["plugins"].as<std::string>());
+
+    boost::filesystem::path output_dir(vm["output-dir"].as<std::string>());
+
+    if (vm.count("uuid-subdir"))
+    {
+        boost::uuids::uuid uuid = boost::uuids::random_generator()();
+        output_dir /= boost::lexical_cast<std::string>(uuid);
+    }
+
+    runner run(vm["styles-dir"].as<std::string>(),
+               output_dir,
+               vm["images-dir"].as<std::string>(),
+               vm.count("overwrite"),
+               vm["jobs"].as<std::size_t>());
+    report_type report = vm.count("verbose") ? report_type((console_report())) : report_type((console_short_report()));
+    result_list results;
+
+    if (vm.count("styles"))
+    {
+        results = run.test(vm["styles"].as<std::vector<std::string>>(), report);
+    }
+    else
+    {
+        results = run.test_all(report);
+    }
+
+    unsigned failed_count = mapnik::util::apply_visitor(summary_visitor(results), report);
+
+    if (failed_count)
+    {
+        html_summary(results, output_dir);
+    }
+
+    return failed_count;
+}

test/visual/runner.cpp

@@ -0,0 +1,248 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+// stl
+#include <algorithm>
+#include <future>
+
+#include <mapnik/load_map.hpp>
+#include <mapnik/util/fs.hpp>
+
+#include "runner.hpp"
+
+namespace visual_tests
+{
+
+class renderer_visitor
+{
+public:
+    renderer_visitor(std::string const & name, mapnik::Map const & map, double scale_factor)
+        : name_(name), map_(map), scale_factor_(scale_factor)
+    {
+    }
+
+    template <typename T>
+    result operator()(T const & renderer) const
+    {
+        return renderer.test(name_, map_, scale_factor_);
+    }
+
+private:
+    std::string const & name_;
+    mapnik::Map const & map_;
+    double scale_factor_;
+};
+
+runner::runner(boost::filesystem::path const & styles_dir,
+               boost::filesystem::path const & output_dir,
+               boost::filesystem::path const & reference_dir,
+               bool overwrite,
+               std::size_t jobs)
+    : styles_dir_(styles_dir),
+      output_dir_(output_dir),
+      reference_dir_(reference_dir),
+      jobs_(jobs),
+      renderers_{ renderer<agg_renderer>(output_dir_, reference_dir_, overwrite),
+                  renderer<cairo_renderer>(output_dir_, reference_dir_, overwrite)/*,
+                  renderer<grid_renderer>(output_dir_, reference_dir_, overwrite)*/ }
+{
+}
+
+result_list runner::test_all(report_type & report) const
+{
+    std::vector<std::string> files = mapnik::util::list_directory(styles_dir_.string());
+    return test_paralel(files, report, jobs_);
+}
+
+result_list runner::test(std::vector<std::string> const & style_names, report_type & report) const
+{
+    std::vector<std::string> files(style_names.size());
+    std::transform(style_names.begin(), style_names.end(), std::back_inserter(files),
+        [&](std::string const & name)
+        {
+            return (styles_dir_ / (name + ".xml")).string();
+        });
+    return test_paralel(files, report, jobs_);
+}
+
+result_list runner::test_paralel(std::vector<std::string> const & files, report_type & report, std::size_t jobs) const
+{
+    result_list results;
+
+    if (files.empty())
+    {
+        return results;
+    }
+
+    if (jobs == 0)
+    {
+        jobs = 1;
+    }
+
+    std::size_t chunk_size = files.size() / jobs;
+
+    if (chunk_size == 0)
+    {
+        chunk_size = files.size();
+        jobs = 1;
+    }
+
+    std::launch launch(jobs == 1 ? std::launch::deferred : std::launch::async);
+    std::vector<std::future<result_list>> futures(jobs);
+
+    for (std::size_t i = 0; i < jobs; i++)
+    {
+        files_iterator begin(files.begin() + i * chunk_size);
+        files_iterator end(files.begin() + (i + 1) * chunk_size);
+
+        // Handle remainder of files.size() / jobs
+        if (i == jobs - 1)
+        {
+            end = files.end();
+        }
+
+        futures[i] = std::async(launch, &runner::test_range, this, begin, end, std::ref(report));
+    }
+
+    for (auto & f : futures)
+    {
+        result_list r = f.get();
+        std::move(r.begin(), r.end(), std::back_inserter(results));
+    }
+
+    return results;
+}
+
+result_list runner::test_range(files_iterator begin, files_iterator end, std::reference_wrapper<report_type> report) const
+{
+    config defaults;
+    result_list results;
+
+    for (std::vector<std::string>::const_iterator i = begin; i != end; i++)
+    {
+        std::string const & file = *i;
+        if (file.size() >= 3 && file.substr(file.size() - 3) == "xml")
+        {
+            try
+            {
+                result_list r = test_one(file, defaults, report);
+                std::move(r.begin(), r.end(), std::back_inserter(results));
+            }
+            catch (std::exception const& ex)
+            {
+                result r;
+                r.state = STATE_ERROR;
+                r.name = file;
+                r.error_message = ex.what();
+                results.emplace_back(r);
+                mapnik::util::apply_visitor(report_visitor(r), report.get());
+            }
+        }
+    }
+
+    return results;
+}
+
+result_list runner::test_one(std::string const& style_path, config cfg, report_type & report) const
+{
+    mapnik::Map m(cfg.sizes.front().width, cfg.sizes.front().height);
+    result_list results;
+
+    try
+    {
+        mapnik::load_map(m, style_path, true);
+    }
+    catch (std::exception const& ex)
+    {
+        std::string what = ex.what();
+        if (what.find("Could not create datasource") != std::string::npos ||
+            what.find("Postgis Plugin: could not connect to server") != std::string::npos)
+        {
+            return results;
+        }
+        throw;
+    }
+
+    mapnik::parameters const & params = m.get_extra_parameters();
+
+    boost::optional<mapnik::value_integer> status = params.get<mapnik::value_integer>("status", cfg.status);
+
+    if (!*status)
+    {
+        return results;
+    }
+
+    boost::optional<std::string> sizes = params.get<std::string>("sizes");
+
+    if (sizes)
+    {
+        cfg.sizes.clear();
+        parse_map_sizes(*sizes, cfg.sizes);
+    }
+
+    boost::filesystem::path p(style_path);
+    std::string name(p.stem().string());
+
+    for (map_size const & size : cfg.sizes)
+    {
+        m.resize(size.width, size.height);
+
+        boost::optional<std::string> bbox_string = params.get<std::string>("bbox");
+
+        if (bbox_string)
+        {
+            mapnik::box2d<double> bbox;
+            bbox.from_string(*bbox_string);
+            m.zoom_to_box(bbox);
+        }
+        else
+        {
+            m.zoom_all();
+        }
+
+        for (double const & scale_factor : cfg.scales)
+        {
+            for(auto const& ren : renderers_)
+            {
+                result r = mapnik::util::apply_visitor(renderer_visitor(name, m, scale_factor), ren);
+                results.emplace_back(r);
+                mapnik::util::apply_visitor(report_visitor(r), report);
+            }
+        }
+    }
+
+    return results;
+}
+
+void runner::parse_map_sizes(std::string const & str, std::vector<map_size> & sizes) const
+{
+    boost::spirit::ascii::space_type space;
+    std::string::const_iterator iter = str.begin();
+    std::string::const_iterator end = str.end();
+    if (!boost::spirit::qi::phrase_parse(iter, end, map_sizes_parser_, space, sizes))
+    {
+        throw std::runtime_error("Failed to parse list of sizes: '" + str + "'");
+    }
+}
+
+}
+

test/visual/runner.hpp

@@ -0,0 +1,70 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef VISUAL_TEST_RUNNER_HPP
+#define VISUAL_TEST_RUNNER_HPP
+
+#include <mapnik/util/variant.hpp>
+
+#include "config.hpp"
+#include "report.hpp"
+#include "renderer.hpp"
+#include "map_sizes_grammar.hpp"
+
+namespace visual_tests
+{
+
+class runner
+{
+    using renderer_type = mapnik::util::variant<renderer<agg_renderer>,
+                                                renderer<cairo_renderer>/*,
+                                                renderer<grid_renderer>*/>;
+
+public:
+    runner(boost::filesystem::path const & styles_dir,
+           boost::filesystem::path const & output_dir,
+           boost::filesystem::path const & reference_dir,
+           bool overwrite,
+           std::size_t jobs);
+
+    result_list test_all(report_type & report) const;
+    result_list test(std::vector<std::string> const & style_names, report_type & report) const;
+
+private:
+    using files_iterator = std::vector<std::string>::const_iterator;
+
+    result_list test_paralel(std::vector<std::string> const & files, report_type & report, std::size_t jobs) const;
+    result_list test_range(files_iterator begin, files_iterator end, std::reference_wrapper<report_type> report) const;
+    result_list test_one(std::string const & style_path, config cfg, report_type & report) const;
+    void parse_map_sizes(std::string const & str, std::vector<map_size> & sizes) const;
+
+    const map_sizes_grammar<std::string::const_iterator> map_sizes_parser_;
+    const boost::filesystem::path styles_dir_;
+    const boost::filesystem::path output_dir_;
+    const boost::filesystem::path reference_dir_;
+    const std::size_t jobs_;
+    const renderer_type renderers_[boost::mpl::size<renderer_type::types>::value];
+};
+
+}
+
+#endif