diff --git a/test/visual/run.cpp b/test/visual/run.cpp index 62ca73b0e..833f97b60 100644 --- a/test/visual/run.cpp +++ b/test/visual/run.cpp @@ -56,6 +56,7 @@ int main(int argc, char** argv) ("duration,d", "output rendering duration") ("iterations,i", po::value()->default_value(1), "number of iterations for benchmarking") ("jobs,j", po::value()->default_value(1), "number of parallel threads") + ("limit,l", po::value()->default_value(0), "limit number of failures") ("styles-dir", po::value()->default_value("test/data-visual/styles"), "directory with styles") ("images-dir", po::value()->default_value("test/data-visual/images"), "directory with reference images") ("output-dir", po::value()->default_value("/tmp/mapnik-visual-images"), "directory for output files") @@ -111,6 +112,7 @@ int main(int argc, char** argv) vm["images-dir"].as(), vm.count("overwrite"), vm["iterations"].as(), + vm["limit"].as(), vm["jobs"].as()); bool show_duration = vm.count("duration"); report_type report(vm.count("verbose") ? report_type((console_report(show_duration))) : report_type((console_short_report(show_duration)))); diff --git a/test/visual/runner.cpp b/test/visual/runner.cpp index 45dc36656..a987d05d9 100644 --- a/test/visual/runner.cpp +++ b/test/visual/runner.cpp @@ -23,6 +23,7 @@ // stl #include #include +#include #include @@ -40,14 +41,18 @@ public: double scale_factor, result_list & results, report_type & report, - std::size_t iterations) + std::size_t iterations, + bool is_fail_limit, + std::atomic & fail_count) : name_(name), map_(map), tiles_(tiles), scale_factor_(scale_factor), results_(results), report_(report), - iterations_(iterations) + iterations_(iterations), + is_fail_limit_(is_fail_limit), + fail_count_(fail_count) { } @@ -82,6 +87,10 @@ private: r.duration = end - start; mapnik::util::apply_visitor(report_visitor(r), report_); results_.push_back(std::move(r)); + if (is_fail_limit_ && r.state == STATE_FAIL) + { + ++fail_count_; + } } } } @@ -112,6 +121,8 @@ private: result_list & results_; report_type & report_; std::size_t iterations_; + bool is_fail_limit_; + std::atomic & fail_count_; }; runner::runner(runner::path_type const & styles_dir, @@ -119,12 +130,14 @@ runner::runner(runner::path_type const & styles_dir, runner::path_type const & reference_dir, bool overwrite, std::size_t iterations, + std::size_t fail_limit, std::size_t jobs) : styles_dir_(styles_dir), output_dir_(output_dir), reference_dir_(reference_dir), jobs_(jobs), iterations_(iterations), + fail_limit_(fail_limit), renderers_{ renderer(output_dir_, reference_dir_, overwrite) #if defined(HAVE_CAIRO) ,renderer(output_dir_, reference_dir_, overwrite) @@ -182,6 +195,7 @@ result_list runner::test_parallel(std::vector const & files, std::launch launch(jobs == 1 ? std::launch::deferred : std::launch::async); std::vector> futures(jobs); + std::atomic fail_count(0); for (std::size_t i = 0; i < jobs; i++) { @@ -194,7 +208,7 @@ result_list runner::test_parallel(std::vector const & files, end = files.end(); } - futures[i] = std::async(launch, &runner::test_range, this, begin, end, std::ref(report)); + futures[i] = std::async(launch, &runner::test_range, this, begin, end, std::ref(report), std::ref(fail_count)); } for (auto & f : futures) @@ -206,7 +220,10 @@ result_list runner::test_parallel(std::vector const & files, return results; } -result_list runner::test_range(files_iterator begin, files_iterator end, std::reference_wrapper report) const +result_list runner::test_range(files_iterator begin, + files_iterator end, + std::reference_wrapper report, + std::reference_wrapper> fail_count) const { config defaults; result_list results; @@ -218,7 +235,7 @@ result_list runner::test_range(files_iterator begin, files_iterator end, std::re { try { - result_list r = test_one(file, defaults, report); + result_list r = test_one(file, defaults, report, fail_count.get()); std::move(r.begin(), r.end(), std::back_inserter(results)); } catch (std::exception const& ex) @@ -227,16 +244,25 @@ result_list runner::test_range(files_iterator begin, files_iterator end, std::re r.state = STATE_ERROR; r.name = file.string(); r.error_message = ex.what(); + r.duration = std::chrono::high_resolution_clock::duration::zero(); results.emplace_back(r); mapnik::util::apply_visitor(report_visitor(r), report.get()); + ++fail_count.get(); } } + if (fail_limit_ && fail_count.get() >= fail_limit_) + { + break; + } } return results; } -result_list runner::test_one(runner::path_type const& style_path, config cfg, report_type & report) const +result_list runner::test_one(runner::path_type const& style_path, + config cfg, + report_type & report, + std::atomic & fail_count) const { mapnik::Map map(cfg.sizes.front().width, cfg.sizes.front().height); result_list results; @@ -317,7 +343,19 @@ result_list runner::test_one(runner::path_type const& style_path, config cfg, re { map.zoom_all(); } - mapnik::util::apply_visitor(renderer_visitor(name, map, tiles_count, scale_factor, results, report, iterations_), ren); + mapnik::util::apply_visitor(renderer_visitor(name, + map, + tiles_count, + scale_factor, + results, + report, + iterations_, + fail_limit_, + fail_count), ren); + if (fail_limit_ && fail_count >= fail_limit_) + { + return results; + } } } } diff --git a/test/visual/runner.hpp b/test/visual/runner.hpp index a4c91baef..65b19bb45 100644 --- a/test/visual/runner.hpp +++ b/test/visual/runner.hpp @@ -55,6 +55,7 @@ public: path_type const & reference_dir, bool overwrite, std::size_t iterations, + std::size_t fail_limit, std::size_t jobs); result_list test_all(report_type & report) const; @@ -62,8 +63,13 @@ public: private: result_list test_parallel(std::vector const & files, report_type & report, std::size_t jobs) const; - result_list test_range(files_iterator begin, files_iterator end, std::reference_wrapper report) const; - result_list test_one(path_type const & style_path, config cfg, report_type & report) const; + result_list test_range(files_iterator begin, + files_iterator end, + std::reference_wrapper report, + std::reference_wrapper> fail_limit) const; + result_list test_one(path_type const & style_path, + config cfg, report_type & report, + std::atomic & fail_limit) const; void parse_map_sizes(std::string const & str, std::vector & sizes) const; const map_sizes_grammar map_sizes_parser_; @@ -72,6 +78,7 @@ private: const path_type reference_dir_; const std::size_t jobs_; const std::size_t iterations_; + const std::size_t fail_limit_; const renderer_type renderers_[boost::mpl::size::value]; };