benchmarks: refactor to allow early exit if tests do not validate

benchmark/bench_framework.hpp

@@ -37,7 +37,7 @@ public:
         return iterations_;
     }
     virtual bool validate() const = 0;
-    virtual void operator()() const = 0;
+    virtual bool operator()() const = 0;
     virtual ~test_case() {}
 };
 
@@ -85,35 +85,40 @@ int run(T const& test_runner, std::string const& name)
             std::clog << "test did not validate: " << name << "\n";
             return -1;
         }
-        std::chrono::high_resolution_clock::time_point start;
+        // run test once before timing
-        std::chrono::high_resolution_clock::duration elapsed;
+        // if it returns false then we'll abort timing
-        std::stringstream s;
+        if (test_runner())
-        s << name << ":"
-            << std::setw(45 - (int)s.tellp()) << std::right
-            << " t:" << test_runner.threads()
-            << " i:" << test_runner.iterations();
-        if (test_runner.threads() > 0)
         {
-            using thread_group = std::vector<std::unique_ptr<std::thread> >;
+            std::chrono::high_resolution_clock::time_point start;
-            using value_type = thread_group::value_type;
+            std::chrono::high_resolution_clock::duration elapsed;
-            thread_group tg;
+            std::stringstream s;
-            for (std::size_t i=0;i<test_runner.threads();++i)
+            s << name << ":"
+                << std::setw(45 - (int)s.tellp()) << std::right
+                << " t:" << test_runner.threads()
+                << " i:" << test_runner.iterations();
+            if (test_runner.threads() > 0)
             {
-                tg.emplace_back(new std::thread(test_runner));
+                using thread_group = std::vector<std::unique_ptr<std::thread> >;
+                using value_type = thread_group::value_type;
+                thread_group tg;
+                for (std::size_t i=0;i<test_runner.threads();++i)
+                {
+                    tg.emplace_back(new std::thread(test_runner));
+                }
+                start = std::chrono::high_resolution_clock::now();
+                std::for_each(tg.begin(), tg.end(), [](value_type & t) {if (t->joinable()) t->join();});
+                elapsed = std::chrono::high_resolution_clock::now() - start;
             }
-            start = std::chrono::high_resolution_clock::now();
+            else
-            std::for_each(tg.begin(), tg.end(), [](value_type & t) {if (t->joinable()) t->join();});
+            {
-            elapsed = std::chrono::high_resolution_clock::now() - start;
+                start = std::chrono::high_resolution_clock::now();
+                test_runner();
+                elapsed = std::chrono::high_resolution_clock::now() - start;
+            }
+            s << std::setw(65 - (int)s.tellp()) << std::right
+                << std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count() << " milliseconds\n";
+            std::clog << s.str();
         }
-        else
-        {
-            start = std::chrono::high_resolution_clock::now();
-            test_runner();
-            elapsed = std::chrono::high_resolution_clock::now() - start;
-        }
-        s << std::setw(65 - (int)s.tellp()) << std::right
-            << std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count() << " milliseconds\n";
-        std::clog << s.str();
         return 0;
     }
     catch (std::exception const& ex)

benchmark/test_array_allocation.cpp

@@ -40,7 +40,7 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              // NOTE: sizeof(uint8_t) == 1
@@ -49,6 +49,7 @@ public:
              ensure_zero(data,size_);
              free(data);
          }
+         return true;
     }
 };
 
@@ -65,7 +66,7 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              // NOTE: sizeof(uint8_t) == 1
@@ -74,6 +75,7 @@ public:
              ensure_zero(data,size_);
              free(data);
          }
+         return true;
     }
 };
 
@@ -90,7 +92,7 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              uint8_t *data = static_cast<uint8_t *>(::operator new(sizeof(uint8_t) * size_));
@@ -98,6 +100,7 @@ public:
              ensure_zero(data,size_);
              ::operator delete(data);
          }
+         return true;
     }
 };
 
@@ -115,7 +118,7 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              uint8_t * data = static_cast<uint8_t*>(::operator new(sizeof(uint8_t)*size_));
@@ -123,6 +126,7 @@ public:
              ensure_zero(data,size_);
              ::operator delete(data),data=0;
          }
+         return true;
     }
 };
 
@@ -139,12 +143,13 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              std::vector<uint8_t> data(size_);
              ensure_zero(&data[0],data.size());
          }
+         return true;
     }
 };
 
@@ -162,13 +167,14 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              std::vector<uint8_t> data(0);
              data.resize(size_,0);
              ensure_zero(&data[0],data.size());
          }
+         return true;
     }
 };
 
@@ -186,13 +192,14 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              std::vector<uint8_t> data(0);
              data.assign(size_,0);
              ensure_zero(&data[0],data.size());
          }
+         return true;
     }
 };
 
@@ -209,13 +216,14 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              uint8_t *data = (uint8_t *)calloc(size_,sizeof(uint8_t));
              ensure_zero(data,size_);
              free(data);
          }
+         return true;
     }
 };
 
@@ -232,12 +240,13 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              std::string data(array_.begin(),array_.end());
              ensure_zero((uint8_t *)&data[0],size_);
          }
+         return true;
     }
 };
 
@@ -254,12 +263,13 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              std::string data(&array_[0],array_.size());
              ensure_zero((uint8_t *)&data[0],size_);
          }
+         return true;
     }
 };
 
@@ -281,12 +291,13 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              std::valarray<uint8_t> data(static_cast<uint8_t>(0),static_cast<size_t>(size_));
              ensure_zero(&data[0],size_);
          }
+         return true;
     }
 };
 
@@ -307,12 +318,13 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              boost::container::static_vector<uint8_t,256*256> data(size_,0);
              ensure_zero(&data[0],size_);
          }
+         return true;
     }
 };
 #endif

benchmark/test_expression_parse.cpp

@@ -22,11 +22,12 @@ public:
         }
         return ret;
     }
-    void operator()() const
+    bool operator()() const
     {
          for (std::size_t i=0;i<iterations_;++i) {
              mapnik::expression_ptr expr = mapnik::parse_expression(expr_);
          }
+         return true;
     }
 };
 

benchmark/test_face_ptr_creation.cpp

@@ -26,7 +26,7 @@ public:
         }
         return count == expected_count;
     }
-    void operator()() const
+    bool operator()() const
     {
         std::size_t expected_count = mapnik::freetype_engine::face_names().size();
         for (unsigned i=0;i<iterations_;++i)
@@ -49,6 +49,7 @@ public:
                 std::clog << "warning: face creation not working as expected\n";
             }
         }
+        return true;
     }
 };
 

benchmark/test_font_registration.cpp

@@ -12,7 +12,7 @@ public:
     {
         return mapnik::freetype_engine::register_fonts("./fonts", true);
     }
-    void operator()() const
+    bool operator()() const
     {
         unsigned long count = 0;
         for (unsigned i=0;i<iterations_;++i)
@@ -20,6 +20,7 @@ public:
             mapnik::freetype_engine::register_fonts("./fonts", true);
             count++;
         }
+        return true;
     }
 };
 

benchmark/test_png_encoding1.cpp

@@ -13,7 +13,7 @@ public:
     {
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
         std::string out;
         for (std::size_t i=0;i<iterations_;++i) {
@@ -21,6 +21,7 @@ public:
             out = mapnik::save_to_string(im_,"png8:m=h:z=1");
         }
     }
+    return true;
 };
 
 BENCHMARK(test,"encoding blank png")

benchmark/test_png_encoding2.cpp

@@ -23,7 +23,7 @@ public:
         mapnik::save_to_file(im_->data(),actual, "png8:m=h:z=1");
         return benchmark::compare_images(actual,expected);
     }
-    void operator()() const
+    bool operator()() const
     {
         std::string out;
         for (std::size_t i=0;i<iterations_;++i) {
@@ -31,6 +31,7 @@ public:
             out = mapnik::save_to_string(im_->data(),"png8:m=h:z=1");
         }
     }
+    return true;
 };
 
 BENCHMARK(test,"encoding multicolor png")

benchmark/test_polygon_clipping.cpp

@@ -108,7 +108,7 @@ public:
         render(geom2,geom.envelope(),actual);
         return benchmark::compare_images(actual,expect);
     }
-    void operator()() const
+    bool operator()() const
     {
         boost::ptr_vector<mapnik::geometry_type> paths;
         if (!mapnik::from_wkt(wkt_in_, paths))
@@ -130,6 +130,7 @@ public:
                 while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {}
             }
         }
+        return true;
     }
 };
 
@@ -189,7 +190,7 @@ public:
         render(geom2,geom.envelope(),actual);
         return benchmark::compare_images(actual,expect);
     }
-    void operator()() const
+    bool operator()() const
     {
         boost::ptr_vector<mapnik::geometry_type> paths;
         if (!mapnik::from_wkt(wkt_in_, paths))
@@ -217,6 +218,7 @@ public:
                 while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {}
             }
         }
+        return true;
     }
 };
 
@@ -265,7 +267,7 @@ public:
         render(geom2,geom.envelope(),actual);
         return benchmark::compare_images(actual,expect);
     }
-    void operator()() const
+    bool operator()() const
     {
         boost::ptr_vector<mapnik::geometry_type> paths;
         if (!mapnik::from_wkt(wkt_in_, paths))
@@ -282,6 +284,7 @@ public:
                 while ((cmd = clipped.vertex(&x, &y)) != mapnik::SEG_END) {}
             }
         }
+        return true;
     }
 };
 

benchmark/test_polygon_clipping_rendering.cpp

@@ -28,7 +28,7 @@ public:
         //mapnik::save_to_file(im,"test.png");
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
         mapnik::Map m(256,256);
         mapnik::load_map(m,xml_);
@@ -39,6 +39,7 @@ public:
             mapnik::agg_renderer<mapnik::image_32> ren(m,im);
             ren.apply();
         }
+        return true;
     }
 };
 

benchmark/test_proj_transform1.cpp

@@ -36,7 +36,7 @@ public:
                 (std::fabs(bbox.maxy() - to_.maxy()) < .5)
                );
     }
-    void operator()() const
+    bool operator()() const
     {
         for (std::size_t i=0;i<iterations_;++i) {
             for (int i=-180;i<180;i=i+5)
@@ -51,6 +51,7 @@ public:
                 }
             }
         }
+        return true;
     }
 };
 

benchmark/test_rendering.cpp

@@ -58,11 +58,17 @@ public:
         mapnik::image_32 im(m.width(),m.height());
         mapnik::agg_renderer<mapnik::image_32> ren(m,im,scale_factor_);
         ren.apply();
-        if (!preview_.empty()) mapnik::save_to_file(im,preview_);
+        if (!preview_.empty()) {
+            std::clog << "preview available at " << preview_ << "\n";
+            mapnik::save_to_file(im,preview_);
+        }
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
+        if (!preview_.empty()) {
+            return false;
+        }
         mapnik::Map m(width_,height_);
         mapnik::load_map(m,xml_);
         if (extent_.valid()) {
@@ -76,6 +82,7 @@ public:
             mapnik::agg_renderer<mapnik::image_32> ren(m,im,scale_factor_);
             ren.apply();
         }
+        return true;
     }
 };
 

benchmark/test_rendering_shared_map.cpp

@@ -26,13 +26,7 @@ template <typename Renderer> void process_layers(Renderer & ren,
         if (lyr.visible(scale_denom))
         {
             std::set<std::string> names;
-            mapnik::parameters p;
-            p["type"]="csv";
-            p["file"]="benchmark/data/roads.csv";
-            mapnik::datasource_ptr ds = mapnik::datasource_cache::instance().create(p);
             mapnik::layer l(lyr);
-            l.set_datasource(ds);
-            l.add_style("labels");
             ren.apply_to_layer(l,
                                ren,
                                map_proj,
@@ -56,6 +50,7 @@ class test : public benchmark::test_case
     std::shared_ptr<mapnik::Map> m_;
     double scale_factor_;
     std::string preview_;
+    mutable mapnik::image_32 im_;
 public:
     test(mapnik::parameters const& params)
      : test_case(params),
@@ -65,7 +60,8 @@ public:
        height_(*params.get<mapnik::value_integer>("height",256)),
        m_(new mapnik::Map(width_,height_)),
        scale_factor_(*params.get<mapnik::value_double>("scale_factor",2.0)),
-       preview_(*params.get<std::string>("preview",""))
+       preview_(*params.get<std::string>("preview","")),
+       im_(m_->width(),m_->height())
     {
         boost::optional<std::string> map = params.get<std::string>("map");
         if (!map)
@@ -75,6 +71,7 @@ public:
         xml_ = *map;
 
         boost::optional<std::string> ext = params.get<std::string>("extent");
+        mapnik::load_map(*m_,xml_,true);
         if (ext && !ext->empty())
         {
             if (!extent_.from_string(*ext))
@@ -82,51 +79,67 @@ public:
         }
         else
         {
-            throw std::runtime_error("please provide a --extent=<minx,miny,maxx,maxy> arg");
+            m_->zoom_all();
+            extent_ = m_->get_current_extent();
+            std::clog << "Defaulting to max extent " << extent_ << "\n";
+            std::clog << "    (pass --extent=<minx,miny,maxx,maxy> to restrict bounds)\n";
         }
-        mapnik::load_map(*m_,xml_,true);
     }
 
     bool validate() const
     {
         mapnik::request m_req(width_,height_,extent_);
-        mapnik::image_32 im(m_->width(),m_->height());
         mapnik::attributes variables;
         m_req.set_buffer_size(m_->buffer_size());
         mapnik::projection map_proj(m_->srs(),true);
         double scale_denom = mapnik::scale_denominator(m_req.scale(),map_proj.is_geographic());
         scale_denom *= scale_factor_;
-        mapnik::agg_renderer<mapnik::image_32> ren(*m_,m_req,variables,im,scale_factor_);
+        mapnik::agg_renderer<mapnik::image_32> ren(*m_,m_req,variables,im_,scale_factor_);
         ren.start_map_processing(*m_);
         std::vector<mapnik::layer> const& layers = m_->layers();
         process_layers(ren,m_req,map_proj,layers,scale_denom);
         ren.end_map_processing(*m_);
         if (!preview_.empty()) {
             std::clog << "preview available at " << preview_ << "\n";
-            mapnik::save_to_file(im,preview_);
+            mapnik::save_to_file(im_,preview_);
         }
         return true;
     }
 
-    void operator()() const
+    bool operator()() const
     {
-        if (preview_.empty()) {
+        if (!preview_.empty()) {
-            for (unsigned i=0;i<iterations_;++i)
+            return false;
-            {
-                mapnik::request m_req(width_,height_,extent_);
-                mapnik::image_32 im(m_->width(),m_->height());
-                mapnik::attributes variables;
-                m_req.set_buffer_size(m_->buffer_size());
-                mapnik::projection map_proj(m_->srs(),true);
-                double scale_denom = mapnik::scale_denominator(m_req.scale(),map_proj.is_geographic());
-                scale_denom *= scale_factor_;
-                mapnik::agg_renderer<mapnik::image_32> ren(*m_,m_req,variables,im,scale_factor_);
-                ren.start_map_processing(*m_);
-                std::vector<mapnik::layer> const& layers = m_->layers();
-                process_layers(ren,m_req,map_proj,layers,scale_denom);
-                ren.end_map_processing(*m_);
-            }            
         }
+        for (unsigned i=0;i<iterations_;++i)
+        {
+            mapnik::request m_req(width_,height_,extent_);
+            mapnik::image_32 im(m_->width(),m_->height());
+            mapnik::attributes variables;
+            m_req.set_buffer_size(m_->buffer_size());
+            mapnik::projection map_proj(m_->srs(),true);
+            double scale_denom = mapnik::scale_denominator(m_req.scale(),map_proj.is_geographic());
+            scale_denom *= scale_factor_;
+            mapnik::agg_renderer<mapnik::image_32> ren(*m_,m_req,variables,im,scale_factor_);
+            ren.start_map_processing(*m_);
+            std::vector<mapnik::layer> const& layers = m_->layers();
+            process_layers(ren,m_req,map_proj,layers,scale_denom);
+            ren.end_map_processing(*m_);
+            bool diff = false;
+            mapnik::image_data_rgba8 const& dest = im.data();
+            mapnik::image_data_rgba8 const& src = im_.data();
+            for (unsigned int y = 0; y < height_; ++y)
+            {
+                const unsigned int* row_from = src.getRow(y);
+                const unsigned int* row_to = dest.getRow(y);
+                for (unsigned int x = 0; x < width_; ++x)
+                {
+                   if (row_from[x] != row_to[x]) diff = true;
+                }
+            }
+            if (diff) throw std::runtime_error("images differ");
+        }
+        return true;
     }
 };
 

benchmark/test_to_bool.cpp

@@ -16,13 +16,14 @@ public:
         mapnik::util::string2bool(value_,result);
         return (result == true);
     }
-    void operator()() const
+    bool operator()() const
     {
         for (std::size_t i=0;i<iterations_;++i) {
             bool result = false;
             mapnik::util::string2bool(value_,result);
             mapnik::util::string2bool(value_.data(),value_.data()+value_.size(),result);
         }
+        return true;
     }
 };
 

benchmark/test_to_double.cpp

@@ -18,13 +18,14 @@ public:
         if (result != 1.23456789) return false;
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
         for (std::size_t i=0;i<iterations_;++i) {
             double result = 0;
             mapnik::util::string2double(value_,result);
             mapnik::util::string2double(value_.data(),value_.data()+value_.size(),result);
         }
+        return true;
     }
 };
 

benchmark/test_to_int.cpp

@@ -18,13 +18,14 @@ public:
         if (result != 123456789) return false;
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
         for (std::size_t i=0;i<iterations_;++i) {
             mapnik::value_integer result = 0;
             mapnik::util::string2int(value_,result);
             mapnik::util::string2int(value_.data(),value_.data()+value_.size(),result);
         }
+        return true;
     }
 };
 

benchmark/test_to_string1.cpp

@@ -14,13 +14,14 @@ public:
         mapnik::util::to_string(s,value_);
         return (s == "-0.1234");
     }
-    void operator()() const
+    bool operator()() const
     {
         std::string out;
         for (std::size_t i=0;i<iterations_;++i) {
             out.clear();
             mapnik::util::to_string(out,value_);
         }
+        return true;
     }
 };
 

benchmark/test_to_string2.cpp

@@ -14,7 +14,7 @@ public:
         s << value_;
         return (s.str() == "-0.1234");
     }
-    void operator()() const
+    bool operator()() const
     {
         std::string out;
         for (std::size_t i=0;i<iterations_;++i) {
@@ -22,6 +22,7 @@ public:
             s << value_;
             out = s.str();
         }
+        return true;
     }
 };
 

benchmark/test_utf_encoding.cpp

@@ -23,13 +23,14 @@ public:
             utf32[3] != 0x5dd) return false;
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
         std::u32string utf32;
         std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf32conv;
         for (std::size_t i=0;i<iterations_;++i) {
              utf32 = utf32conv.from_bytes(utf8_);
         }
+        return true;
     }
 };
 
@@ -52,12 +53,13 @@ public:
             utf32[3] != 0x5dd) return false;
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
          std::u32string utf32;
          for (std::size_t i=0;i<iterations_;++i) {
              utf32 = boost::locale::conv::utf_to_utf<char32_t>(utf8_);
          }
+        return true;
     }
 };
 
@@ -80,13 +82,14 @@ public:
             utf32[3] != 0x5dd) return false;
         return true;
     }
-    void operator()() const
+    bool operator()() const
     {
         mapnik::transcoder tr_("utf-8");
         mapnik::value_unicode_string utf32;
         for (std::size_t i=0;i<iterations_;++i) {
             utf32 = tr_.transcode(utf8_.data(),utf8_.size());
         }
+        return true;
     }
 };