Merge branch 'master' into value-cpp

CHANGELOG.md

@@ -6,6 +6,37 @@ Developers: Please commit along with changes.
 
 For a complete change history, see the git log.
 
+## 3.0.10
+
+Released: February 25, 2016
+
+(Packaged from 5c0d496)
+
+#### Summary
+
+ - The `shapeindex` command now has a `--index-parts` option. When used the index will be bigger
+   but will allow the Shapefile datasource to only parse polygon parts within the query bounds.
+ - WARNING: index files generated with this newer Mapnik are invalid for older versions of Mapnik.
+ - Any `.index` files accompanying a `.shp` must now be regenerated otherwise
+   it will be skipped. To avoid this problem you can delete the existing `.index` files, or ideally run `shapeindex` to recreate the `.index`. (https://github.com/mapnik/mapnik/pull/3300)
+   The trigger for this change was an optimization that required a new binary format for the shapefile indexes (https://github.com/mapnik/mapnik/pull/3217).
+ - Shapeindex - another fix for skipping `null` shapes (#3288)
+ - Fixed support for filter expressions starting with `not` (https://github.com/mapnik/mapnik/issues/3017)
+ - Ensure `mapped_memory_cache` acts as singleton across shared objects (#3306)
+ - Removed miniz support in PNG encoder (#3281)
+ - Added `-fvisibility=hidden -fvisibility-inlines-hidden` to default compiler flags
+ - Fixed parsing of SVG `PathElement` (https://github.com/mapnik/mapnik/issues/3225)
+ - JSON parsing now supports arbitrary (nested) attributes in `geometry`
+ - Support for rendering `dash-array` in SVGs
+ - SVG parser is now stricter (fails is all input is not parsable) (#3251)
+ - SVG parser now correctly handles optional separator `(,)` between multiple command parts
+ - Optimized parsing of `png` format string
+ - The `memory_datasource` now dynamically reports correct datasource type (vector or raster)
+ - Upgraded `mapbox::variant v1.1.0`
+ - Compare: https://github.com/mapnik/mapnik/compare/v3.0.9...v3.0.10
+
+
+
 ## 3.0.9
 
 Released: November 26, 2015
@@ -18,12 +49,12 @@ Released: November 26, 2015
  - Fixed mapnik.util.variant issue when compiling with gcc-5.x and SSO enabled by default (https://github.com/mapnik/mapnik/issues/3103) (via @nkovacs)
  - Fixed issue with complex scripts where some character sequences weren't rendered correctly (https://github.com/mapnik/mapnik/issues/3050) (via @jkroll20)
  - Revived postgis.input tests
- - JSON: geometry grammar has been refactored and optimized to have expectation points
+ - JSON: geometry grammar has been re-factored and optimized to have expectation points
  - Filled missing specializations for value_bool in `mapnik::value` comparison operators
  - `mapnik.Image` - fixed copy semantics implementation for internal buffer
  - JSON parsing: unified error_handler across all grammars
  - Improved unit test coverage
- - Raster scaling: fixed nodata handling, acurracy when working with small floats and clipping floats by \[0; 255\] (https://github.com/mapnik/mapnik/pull/3147)
+ - Raster scaling: fixed nodata handling, accuracy when working with small floats and clipping floats by \[0; 255\] (https://github.com/mapnik/mapnik/pull/3147)
  - Added [`code of conduct`](http://contributor-covenant.org)
  - GeoJSON plug-in is updated to skip feature with empty geometries
  - GeoJSON plug-in : ensure original order of features is preserved (fixed) (https://github.com/mapnik/mapnik/issues/3182)
@@ -41,9 +72,9 @@ Released: October 23, 2015
 
  - Renamed `SHAPE_MEMORY_MAPPED_FILE` define to `MAPNIK_MEMORY_MAPPED_FILE`. Pass `./configure MEMORY_MAPPED_FILE=True|False` to request
    support for memory mapped files across Mapnik plugins (currently shape, csv, and geojson).
- - Unified `mapnik-index` utility supporing GeoJSON and CSV formats
+ - Unified `mapnik-index` utility supporting GeoJSON and CSV formats
  - Increased unit test coverage for GeoJSON and CSV plugins
- - shape.input - refactor to support *.shx and improve handling various bogus shapefiles
+ - shape.input - re-factor to support *.shx and improve handling various bogus shapefiles
  - geojson.input - make JSON parser stricter + support single Feature/Geometry as well as FeatureCollection
  - maintain 'FT_LOAD_NO_HINTING' + support >= harfbuzz 1.0.5
  - geojson.input - implement on-disk-index support
@@ -109,7 +140,7 @@ Released: August 26, 2015
 
 #### Summary
 
-- CSV.input: plug-in has been refactored to minimise memory usage and to improve handling of larger input.
+- CSV.input: plug-in has been re-factored to minimise memory usage and to improve handling of larger input.
   (NOTE: [large_csv](https://github.com/mapnik/mapnik/tree/large_csv) branch adds experimental trunsduction parser with deferred string initialisation)
 - CSV.input: added internal spatial index (boost::geometry::index::tree) for fast `bounding box` queries (https://github.com/mapnik/mapnik/pull/3010)
 - Fixed deadlock in recursive datasource registration via @zerebubuth (https://github.com/mapnik/mapnik/pull/3038)
@@ -1200,7 +1231,7 @@ Released April 1, 2009
 
 - Plugins: Use memory mapped files for reading shape file (r628)
 
-- Core: Use streams to write images (i/o refactor) (r628) (#15)
+- Core: Use streams to write images (i/o re-factor) (r628) (#15)
 
 # Mapnik 0.5.1
 

Makefile

@@ -24,7 +24,8 @@ release:
 	git clone --depth 1 --branch v$${MAPNIK_VERSION} git@github.com:mapnik/mapnik.git $${TARBALL_NAME} && \
 	cd $${TARBALL_NAME} && \
 	git checkout "tags/v$${MAPNIK_VERSION}" && \
-	git submodule update --depth 1 --init && \
+	git submodule update --depth 100 --init && \
+	rm -rf deps/mapbox/variant/.git && \
 	rm -rf test/data/.git && \
 	rm -rf test/data/.gitignore && \
 	rm -rf test/data-visual/.git && \

appveyor.yml

@@ -12,7 +12,13 @@ os: Visual Studio 2015
 # limit clone to latest 5 commits
 clone_depth: 5
 
+services:
+  - postgresql94 #if changing this, also change PATH below
+
 install:
+  - SET PGUSER=postgres
+  - SET PGPASSWORD=Password12!
+  - SET PATH=C:\Program Files\PostgreSQL\9.4\bin\;%PATH%
   - scripts\build-appveyor.bat
 
 artifacts:

benchmark/bench_framework.hpp

@@ -10,6 +10,7 @@
 
 // stl
 #include <chrono>
+#include <cmath> // log10, round
 #include <cstdio> // snprintf
 #include <iostream>
 #include <set>
@@ -19,6 +20,12 @@
 
 namespace benchmark {
 
+template <typename T>
+using milliseconds = std::chrono::duration<T, std::milli>;
+
+template <typename T>
+using seconds = std::chrono::duration<T>;
+
 class test_case
 {
 protected:
@@ -92,7 +99,7 @@ inline int parse_args(int argc, char** argv, mapnik::parameters & params)
 
 inline void handle_common_args(mapnik::parameters const& params)
 {
-    if (auto severity = params.get<std::string>("log-severity")) {
+    if (auto severity = params.get<std::string>("log")) {
         if (*severity == "debug")
             mapnik::logger::set_severity(mapnik::logger::debug);
         else if (*severity == "warn")
@@ -102,7 +109,7 @@ inline void handle_common_args(mapnik::parameters const& params)
         else if (*severity == "none")
             mapnik::logger::set_severity(mapnik::logger::none);
         else
-            std::clog << "ignoring option --log-severity='" << *severity
+            std::clog << "ignoring option --log='" << *severity
                       << "' (allowed values are: debug, warn, error, none)\n";
     }
 }
@@ -134,6 +141,29 @@ inline int handle_args(int argc, char** argv, mapnik::parameters & params)
         }                                               \
     }                                                   \
 
+struct big_number_fmt
+{
+    int w;
+    double v;
+    const char* u;
+
+    big_number_fmt(int width, double value, int base = 1000)
+        : w(width), v(value), u("")
+    {
+        static const char* suffixes = "\0\0k\0M\0G\0T\0P\0E\0Z\0Y\0\0";
+        u = suffixes;
+
+        while (v > 1 && std::log10(std::round(v)) >= width && u[2])
+        {
+            v /= base;
+            u += 2;
+        }
+
+        // adjust width for proper alignment without suffix
+        w += (u == suffixes);
+    }
+};
+
 template <typename T>
 int run(T const& test_runner, std::string const& name)
 {
@@ -156,21 +186,43 @@ int run(T const& test_runner, std::string const& name)
         auto opt_min_duration = test_runner.params().template get<double>("min-duration", 0.0);
         std::chrono::duration<double> min_seconds(*opt_min_duration);
         auto min_duration = std::chrono::duration_cast<decltype(elapsed)>(min_seconds);
-        std::size_t loops = 0;
+        auto num_iters = test_runner.iterations();
+        auto num_threads = test_runner.threads();
+        auto total_iters = 0;
 
-        if (test_runner.threads() > 0)
+        if (num_threads > 0)
         {
-            using thread_group = std::vector<std::unique_ptr<std::thread> >;
+            std::mutex mtx_ready;
-            using value_type = thread_group::value_type;
+            std::unique_lock<std::mutex> lock_ready(mtx_ready);
-            thread_group tg;
+
-            for (std::size_t i=0;i<test_runner.threads();++i)
+            auto stub = [&](T const& test_copy)
             {
-                tg.emplace_back(new std::thread(test_runner));
+                // workers will wait on this mutex until the main thread
+                // constructs all of them and starts measuring time
+                std::unique_lock<std::mutex> my_lock(mtx_ready);
+                my_lock.unlock();
+                test_copy();
+            };
+
+            std::vector<std::thread> tg;
+            tg.reserve(num_threads);
+            for (auto i = num_threads; i-- > 0; )
+            {
+                tg.emplace_back(stub, test_runner);
             }
             start = std::chrono::high_resolution_clock::now();
-            std::for_each(tg.begin(), tg.end(), [](value_type & t) {if (t->joinable()) t->join();});
+            lock_ready.unlock();
+            // wait for all workers to finish
+            for (auto & t : tg)
+            {
+                if (t.joinable())
+                    t.join();
+            }
             elapsed = std::chrono::high_resolution_clock::now() - start;
-            loops = 1;
+            // this is actually per-thread count, not total, but I think
+            // reporting average 'iters/thread/second' is more useful
+            // than 'iters/second' multiplied by the number of threads
+            total_iters += num_iters;
         }
         else
         {
@@ -178,39 +230,25 @@ int run(T const& test_runner, std::string const& name)
             do {
                 test_runner();
                 elapsed = std::chrono::high_resolution_clock::now() - start;
-                ++loops;
+                total_iters += num_iters;
             } while (elapsed < min_duration);
         }
 
-        double iters = loops * test_runner.iterations();
-        double dur_total = std::chrono::duration<double, std::milli>(elapsed).count();
-        double dur_avg = dur_total / iters;
-        char iters_unit = ' ';
         char msg[200];
-
+        double dur_total = milliseconds<double>(elapsed).count();
-        if (iters >= 1e7) iters *= 1e-6, iters_unit = 'M';
+        auto elapsed_nonzero = std::max(elapsed, decltype(elapsed){1});
-        else if (iters >= 1e4) iters *= 1e-3, iters_unit = 'k';
+        big_number_fmt itersf(4, total_iters);
+        big_number_fmt ips(5, total_iters / seconds<double>(elapsed_nonzero).count());
 
         std::snprintf(msg, sizeof(msg),
-                "%-43s %3zu threads %4.0f%c iters %6.0f milliseconds",
+                "%-43s %3zu threads %*.0f%s iters %6.0f milliseconds %*.0f%s i/s\n",
                 name.c_str(),
-                test_runner.threads(),
+                num_threads,
-                iters, iters_unit,
+                itersf.w, itersf.v, itersf.u,
-                dur_total);
+                dur_total,
+                ips.w, ips.v, ips.u
+                );
         std::clog << msg;
-
-        // log average time per iteration, currently only for non-threaded runs
-        if (test_runner.threads() == 0)
-        {
-            char unit = 'm';
-            if (dur_avg < 1e-5) dur_avg *= 1e+9, unit = 'p';
-            else if (dur_avg < 1e-2) dur_avg *= 1e+6, unit = 'n';
-            else if (dur_avg < 1e+1) dur_avg *= 1e+3, unit = 'u';
-            std::snprintf(msg, sizeof(msg), " %4.0f%cs/iter", dur_avg, unit);
-            std::clog << msg;
-        }
-
-        std::clog << "\n";
         return 0;
     }
     catch (std::exception const& ex)

benchmark/run

@@ -6,8 +6,14 @@ source ./localize.sh
 
 BASE=./benchmark/out
 function run {
-    ${BASE}/$1 --threads 0 --iterations $3;
+    local runner="$BASE/$1 --log=none"
-    ${BASE}/$1 --threads $2 --iterations $(expr $3 / $2);
+    local threads="$2"
+    local iters="$3"
+    shift 3
+    $runner --threads 0 --iterations $iters "$@"
+    if test $threads -gt 0; then
+        $runner --threads $threads --iterations $((iters/threads)) "$@"
+    fi
 }
 run test_getline 30 10000000
 #run test_array_allocation 20 100000

include/mapnik/csv/csv_grammar.hpp

@@ -90,7 +90,7 @@ struct csv_line_grammar : qi::grammar<Iterator, csv_line(char, char), Skipper>
             ("\\\"", '\"')
             ("\"\"", '\"') // double quote
             ;
-        line = -lit("\n\r") >> column(_r1, _r2) % lit(_r1)
+        line = -lit("\r") >> -lit("\n") >> column(_r1, _r2) % lit(_r1)
             ;
         column = quoted(_r2) | *(char_ - lit(_r1))
             ;

include/mapnik/marker_helpers.hpp

@@ -159,21 +159,68 @@ void setup_transform_scaling(agg::trans_affine & tr,
                              symbolizer_base const& sym);
 
 // Apply markers to a feature with multiple geometries
+
+template <typename Converter, typename Processor>
+void apply_markers_single(Converter & converter, Processor & proc, geometry::geometry<double> const& geom)
+{
+    geometry::geometry_types type = geometry::geometry_type(geom);
+
+    if (type == geometry::geometry_types::Point)
+    {
+        geometry::point_vertex_adapter<double> va(geom.get<geometry::point<double>>());
+        converter.apply(va, proc);
+    }
+    else if (type == geometry::geometry_types::LineString)
+    {
+        geometry::line_string_vertex_adapter<double> va(geom.get<geometry::line_string<double>>());
+        converter.apply(va, proc);
+    }
+    else if (type == geometry::geometry_types::Polygon)
+    {
+        geometry::polygon_vertex_adapter<double> va(geom.get<geometry::polygon<double>>());
+        converter.apply(va, proc);
+    }
+    else if (type == geometry::geometry_types::MultiPoint)
+    {
+        for (auto const& pt : geom.get<geometry::multi_point<double>>())
+        {
+            geometry::point_vertex_adapter<double> va(pt);
+            converter.apply(va, proc);
+        }
+    }
+    else if (type == geometry::geometry_types::MultiLineString)
+    {
+        for (auto const& line : geom.get<geometry::multi_line_string<double>>())
+        {
+            geometry::line_string_vertex_adapter<double> va(line);
+            converter.apply(va, proc);
+        }
+    }
+    else if (type == geometry::geometry_types::MultiPolygon)
+    {
+        for (auto const& poly : geom.get<geometry::multi_polygon<double>>())
+        {
+            geometry::polygon_vertex_adapter<double> va(poly);
+            converter.apply(va, proc);
+        }
+    }
+}
+
 template <typename Converter, typename Processor>
 void apply_markers_multi(feature_impl const& feature, attributes const& vars, Converter & converter, Processor & proc, symbolizer_base const& sym)
 {
     using apply_vertex_converter_type = detail::apply_vertex_converter<Converter,Processor>;
-    using vertex_processor_type = geometry::vertex_processor<apply_vertex_converter_type>;
+    apply_vertex_converter_type apply(converter, proc);
-
     auto const& geom = feature.get_geometry();
     geometry::geometry_types type = geometry::geometry_type(geom);
 
     if (type == geometry::geometry_types::Point
-        || type == geometry::geometry_types::LineString
+        ||
-        || type == geometry::geometry_types::Polygon)
+        type == geometry::geometry_types::LineString
+        ||
+        type == geometry::geometry_types::Polygon)
     {
-        apply_vertex_converter_type apply(converter, proc);
+        apply_markers_single(converter, proc, geom);
-        mapnik::util::apply_visitor(vertex_processor_type(apply), geom);
     }
     else
     {
@@ -231,8 +278,17 @@ void apply_markers_multi(feature_impl const& feature, attributes const& vars, Co
             {
                 MAPNIK_LOG_WARN(marker_symbolizer) << "marker_multi_policy != 'each' has no effect with marker_placement != 'point'";
             }
-            apply_vertex_converter_type apply(converter, proc);
+            if (type == geometry::geometry_types::GeometryCollection)
-            mapnik::util::apply_visitor(vertex_processor_type(apply), geom);
+            {
+                for (auto const& g : geom.get<geometry::geometry_collection<double>>())
+                {
+                    apply_markers_single(converter, proc, g);
+                }
+            }
+            else
+            {
+                apply_markers_single(converter, proc, geom);
+            }
         }
     }
 }

include/mapnik/version.hpp

@@ -25,7 +25,7 @@
 
 #define MAPNIK_MAJOR_VERSION 3
 #define MAPNIK_MINOR_VERSION 0
-#define MAPNIK_PATCH_VERSION 9
+#define MAPNIK_PATCH_VERSION 10
 
 #define MAPNIK_VERSION (MAPNIK_MAJOR_VERSION*100000) + (MAPNIK_MINOR_VERSION*100) + (MAPNIK_PATCH_VERSION)
 

scripts/build-appveyor.bat

@@ -22,6 +22,8 @@ ECHO msvs_toolset^: %msvs_toolset%
 SET BUILD_TYPE=%configuration%
 SET BUILDPLATFORM=%platform%
 SET TOOLS_VERSION=%msvs_toolset%.0
+SET ICU_VERSION=56.1
+ECHO ICU_VERSION^: %ICU_VERSION%
 IF DEFINED APPVEYOR (ECHO on AppVeyor) ELSE (ECHO NOT on AppVeyor)
 ECHO ========
 

test/data-visual

@@ -1 +1 @@
-Subproject commit f5623b5a312c58cd9c1926008fcf114944c769a3
+Subproject commit 93e70a5ae91b2bd4b8fc3912a7e6e4b017021b2b

utils/mapnik-index/process_csv_file.cpp

@@ -35,6 +35,7 @@
 #endif
 
 #include <fstream>
+#include <iomanip>
 
 namespace mapnik { namespace detail {
 
@@ -89,7 +90,12 @@ std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& fil
 
     ::detail::geometry_column_locator locator;
     std::vector<std::string> headers;
-    std::clog << "Parsing CSV using SEPARATOR=" << separator << " QUOTE=" << quote << std::endl;
+    std::clog << std::showbase << std::internal << std::setfill('0') ;
+    std::clog << "Parsing CSV using"
+              << " NEWLINE=" << std::hex << std::setw(4) << int(newline) << std::dec
+              << " SEPARATOR=" << separator
+              << " QUOTE=" << quote << std::endl;
+
     if (!manual_headers.empty())
     {
         std::size_t index = 0;