Merge branch 'master' of github.com:mapnik/mapnik into faster-csv-compile

.travis.yml

@@ -52,64 +52,42 @@ matrix:
       env: JOBS=8 COVERAGE=true HEAVY_JOBS=3
 
 before_install:
- - export PYTHONUSERBASE=${PYTHONUSERBASE}
+ - source scripts/travis-common.sh
+ - export PYTHONUSERBASE=$(pwd)/mason_packages/.link
  - export PATH=${PYTHONUSERBASE}/bin:${PATH}
  - export COVERAGE=${COVERAGE:-false}
  - export MASON_PUBLISH=${MASON_PUBLISH:-false}
  - export BENCH=${BENCH:-false}
  - if [[ ${TRAVIS_BRANCH} != 'master' ]]; then export MASON_PUBLISH=false; fi
  - if [[ ${TRAVIS_PULL_REQUEST} != 'false' ]]; then export MASON_PUBLISH=false; fi
- - git submodule update --init --depth=10 ||
+ - git_submodule_update --init --depth=10
-     git submodule foreach 'test "$sha1" = "`git rev-parse HEAD`" ||
-                            git ls-remote origin "refs/pull/*/head" |
-                            while read hash ref; do
-                                if test "$hash" = "$sha1"; then
-                                    git config --add remote.origin.fetch "+$ref:$ref";
-                                fi
-                            done'
- - git submodule update --init --depth=10
 
 install:
- - if [[ $(uname -s) == 'Linux' ]]; then
+ - on 'linux' export PYTHONPATH=${PYTHONUSERBASE}/lib/python2.7/site-packages
-     export PYTHONPATH=${PYTHONUSERBASE}/lib/python2.7/site-packages;
+ - on 'osx' export PYTHONPATH=${PYTHONUSERBASE}/lib/python/site-packages
-   else
+ - on 'osx' brew rm postgis --force
-     brew rm postgis --force;
+ - on 'osx' brew install postgis --force
-     brew install postgis --force;
+ - on 'osx' pg_ctl -w start -l postgres.log --pgdata /usr/local/var/postgres
-     pg_ctl -w start -l postgres.log --pgdata /usr/local/var/postgres;
+ - on 'osx' createuser -s postgres
-     createuser -s postgres;
+ - psql -c 'create database template_postgis;' -U postgres
-     export PYTHONPATH=${PYTHONUSERBASE}/lib/python/site-packages;
+ - psql -c 'create extension postgis;' -d template_postgis -U postgres
-   fi
+ - enabled ${COVERAGE} pip install --user cpp-coveralls
- - psql -c 'create database template_postgis;' -U postgres;
+
- - psql -c 'create extension postgis;' -d template_postgis -U postgres;
+before_script:
- - if [[ ${COVERAGE} == true ]]; then
+ - source bootstrap.sh
-     if [[ ! $(which pip) ]]; then easy_install --user pip && export PATH=/Users/travis/Library/Python/2.7/bin:${PATH}; fi;
+ - commit_message_parse
-     pip install --user cpp-coveralls;
-   fi
 
 script:
- - source bootstrap.sh
+ - export SCONSFLAGS='--debug=time'
- - if [[ ${COVERAGE} == true ]]; then
+ - configure BENCHMARK=${BENCH}
-     ./configure PGSQL2SQLITE=False SVG2PNG=False SAMPLE_INPUT_PLUGINS=False SVG_RENDERER=False BENCHMARK=${BENCH} CUSTOM_LDFLAGS='--coverage' CUSTOM_CXXFLAGS='--coverage' CUSTOM_CFLAGS='--coverage' DEBUG=True;
+ - make
-   elif [[ ${MASON_PUBLISH} == true ]]; then
+ - make test
-     export MASON_NAME=mapnik;
+ - enabled ${COVERAGE} coverage
-     export MASON_VERSION=latest;
+ - enabled ${BENCH} make bench
-     export MASON_LIB_FILE=lib/libmapnik-wkt.a;
+
-     source ./.mason/mason.sh;
+after_success:
-     ./configure BENCHMARK=${BENCH} PREFIX=${MASON_PREFIX} PATH_REPLACE='' MAPNIK_BUNDLED_SHARE_DIRECTORY=True RUNTIME_LINK='static';
+ - if enabled ${MASON_PUBLISH}; then
-   else
+     ./mason_latest.sh build &&
-     ./configure BENCHMARK=${BENCH};
+     ./mason_latest.sh link &&
-   fi
- - SCONSFLAGS='--debug=time' make
- - make test || TEST_RESULT=$?
- - if [[ ${COVERAGE} == true ]]; then
-     ./mason_packages/.link/bin/cpp-coveralls --build-root . --gcov-options '\-lp' --exclude mason_packages --exclude .sconf_temp --exclude benchmark --exclude deps --exclude scons --exclude test --exclude demo --exclude docs --exclude fonts --exclude utils > /dev/null; 
-   fi
- - if [[ ${BENCH} == True ]]; then
-     make bench;
-   fi
- - if [[ ${TEST_RESULT:-0} != 0 ]]; then exit $TEST_RESULT ; fi;
- - if [[ ${MASON_PUBLISH} == true ]]; then
-     ./mason_latest.sh build;
-     ./mason_latest.sh link;
      ./mason_latest.sh publish;
    fi

benchmark/bench_framework.hpp

@@ -2,6 +2,7 @@
 #define __MAPNIK_BENCH_FRAMEWORK_HPP__
 
 // mapnik
+#include <mapnik/debug.hpp>
 #include <mapnik/params.hpp>
 #include <mapnik/value_types.hpp>
 #include <mapnik/safe_cast.hpp>
@@ -9,7 +10,7 @@
 
 // stl
 #include <chrono>
-#include <iomanip>
+#include <cstdio> // snprintf
 #include <iostream>
 #include <set>
 #include <sstream>
@@ -38,26 +39,79 @@ public:
     {
         return iterations_;
     }
+    mapnik::parameters const& params() const
+    {
+        return params_;
+    }
     virtual bool validate() const = 0;
     virtual bool operator()() const = 0;
     virtual ~test_case() {}
 };
 
-void handle_args(int argc, char** argv, mapnik::parameters & params)
+// gathers --long-option values in 'params';
+// returns the index of the first non-option argument,
+// or negated index of an ill-formed option argument
+inline int parse_args(int argc, char** argv, mapnik::parameters & params)
 {
-    if (argc > 0) {
+    for (int i = 1; i < argc; ++i) {
-        for (int i=1;i<argc;++i) {
+        const char* opt = argv[i];
-            std::string opt(argv[i]);
+        if (opt[0] != '-') {
-            // parse --foo bar
+            // non-option argument, return its index
-            if (!opt.empty() && (opt.find("--") != 0)) {
+            return i;
-                std::string key = std::string(argv[i-1]);
+        }
-                if (!key.empty() && (key.find("--") == 0)) {
+        if (opt[1] != '-') {
-                    key = key.substr(key.find_first_not_of("-"));
+            // we only accept --long-options, but instead of throwing,
-                    params[key] = opt;
+            // just issue a warning and let the caller decide what to do
-                }
+            std::clog << argv[0] << ": invalid option '" << opt << "'\n";
-            }
+            return -i; // negative means ill-formed option #i
+        }
+        if (opt[2] == '\0') {
+            // option-list terminator '--'
+            return i + 1;
+        }
+
+        // take option name without the leading '--'
+        std::string key(opt + 2);
+        size_t eq = key.find('=');
+        if (eq != std::string::npos) {
+            // one-argument form '--foo=bar'
+            params[key.substr(0, eq)] = key.substr(eq + 1);
+        }
+        else if (i + 1 < argc) {
+            // two-argument form '--foo' 'bar'
+            params[key] = std::string(argv[++i]);
+        }
+        else {
+            // missing second argument
+            std::clog << argv[0] << ": missing option '" << opt << "' value\n";
+            return -i; // negative means ill-formed option #i
         }
     }
+    return argc; // there were no non-option arguments
+}
+
+inline void handle_common_args(mapnik::parameters const& params)
+{
+    if (auto severity = params.get<std::string>("log-severity")) {
+        if (*severity == "debug")
+            mapnik::logger::set_severity(mapnik::logger::debug);
+        else if (*severity == "warn")
+            mapnik::logger::set_severity(mapnik::logger::warn);
+        else if (*severity == "error")
+            mapnik::logger::set_severity(mapnik::logger::error);
+        else if (*severity == "none")
+            mapnik::logger::set_severity(mapnik::logger::none);
+        else
+            std::clog << "ignoring option --log-severity='" << *severity
+                      << "' (allowed values are: debug, warn, error, none)\n";
+    }
+}
+
+inline int handle_args(int argc, char** argv, mapnik::parameters & params)
+{
+    int res = parse_args(argc, argv, params);
+    handle_common_args(params);
+    return res;
 }
 
 #define BENCHMARK(test_class,name)                      \
@@ -88,52 +142,112 @@ int run(T const& test_runner, std::string const& name)
         if (!test_runner.validate())
         {
             std::clog << "test did not validate: " << name << "\n";
-            return -1;
+            return 1;
         }
         // run test once before timing
         // if it returns false then we'll abort timing
-        if (test_runner())
+        if (!test_runner())
         {
-            std::chrono::high_resolution_clock::time_point start;
+            return 2;
-            std::chrono::high_resolution_clock::duration elapsed;
+        }
-            std::stringstream s;
+
-            s << name << ":"
+        std::chrono::high_resolution_clock::time_point start;
-                << std::setw(45 - (int)s.tellp()) << std::right
+        std::chrono::high_resolution_clock::duration elapsed;
-                << " t:" << test_runner.threads()
+        auto opt_min_duration = test_runner.params().template get<double>("min-duration", 0.0);
-                << " i:" << test_runner.iterations();
+        std::chrono::duration<double> min_seconds(*opt_min_duration);
-            if (test_runner.threads() > 0)
+        auto min_duration = std::chrono::duration_cast<decltype(elapsed)>(min_seconds);
+        std::size_t loops = 0;
+
+        if (test_runner.threads() > 0)
+        {
+            using thread_group = std::vector<std::unique_ptr<std::thread> >;
+            using value_type = thread_group::value_type;
+            thread_group tg;
+            for (std::size_t i=0;i<test_runner.threads();++i)
             {
-                using thread_group = std::vector<std::unique_ptr<std::thread> >;
+                tg.emplace_back(new std::thread(test_runner));
-                using value_type = thread_group::value_type;
-                thread_group tg;
-                for (std::size_t i=0;i<test_runner.threads();++i)
-                {
-                    tg.emplace_back(new std::thread(test_runner));
-                }
-                start = std::chrono::high_resolution_clock::now();
-                std::for_each(tg.begin(), tg.end(), [](value_type & t) {if (t->joinable()) t->join();});
-                elapsed = std::chrono::high_resolution_clock::now() - start;
             }
-            else
+            start = std::chrono::high_resolution_clock::now();
-            {
+            std::for_each(tg.begin(), tg.end(), [](value_type & t) {if (t->joinable()) t->join();});
-                start = std::chrono::high_resolution_clock::now();
+            elapsed = std::chrono::high_resolution_clock::now() - start;
+            loops = 1;
+        }
+        else
+        {
+            start = std::chrono::high_resolution_clock::now();
+            do {
                 test_runner();
                 elapsed = std::chrono::high_resolution_clock::now() - start;
-            }
+                ++loops;
-            s << std::setw(65 - (int)s.tellp()) << std::right
+            } while (elapsed < min_duration);
-                << std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count() << " milliseconds\n";
-            std::clog << s.str();
         }
+
+        double iters = loops * test_runner.iterations();
+        double dur_total = std::chrono::duration<double, std::milli>(elapsed).count();
+        double dur_avg = dur_total / iters;
+        char iters_unit = ' ';
+        char msg[200];
+
+        if (iters >= 1e7) iters *= 1e-6, iters_unit = 'M';
+        else if (iters >= 1e4) iters *= 1e-3, iters_unit = 'k';
+
+        std::snprintf(msg, sizeof(msg),
+                "%-43s %3zu threads %4.0f%c iters %6.0f milliseconds",
+                name.c_str(),
+                test_runner.threads(),
+                iters, iters_unit,
+                dur_total);
+        std::clog << msg;
+
+        // log average time per iteration, currently only for non-threaded runs
+        if (test_runner.threads() == 0)
+        {
+            char unit = 'm';
+            if (dur_avg < 1e-5) dur_avg *= 1e+9, unit = 'p';
+            else if (dur_avg < 1e-2) dur_avg *= 1e+6, unit = 'n';
+            else if (dur_avg < 1e+1) dur_avg *= 1e+3, unit = 'u';
+            std::snprintf(msg, sizeof(msg), " %4.0f%cs/iter", dur_avg, unit);
+            std::clog << msg;
+        }
+
+        std::clog << "\n";
         return 0;
     }
     catch (std::exception const& ex)
     {
         std::clog << "test runner did not complete: " << ex.what() << "\n";
-        return -1;
+        return 4;
     }
-    return 0;
 }
 
+struct sequencer
+{
+    sequencer(int argc, char** argv)
+      : exit_code_(0)
+    {
+        benchmark::handle_args(argc, argv, params_);
+    }
+
+    int done() const
+    {
+        return exit_code_;
+    }
+
+    template <typename Test, typename... Args>
+    sequencer & run(std::string const& name, Args && ...args)
+    {
+        // Test instance lifetime is confined to this function
+        Test test_runner(params_, std::forward<Args>(args)...);
+        // any failing test run will make exit code non-zero
+        exit_code_ |= benchmark::run(test_runner, name);
+        return *this; // allow chaining calls
+    }
+
+protected:
+    mapnik::parameters params_;
+    int exit_code_;
+};
+
 }
 
 #endif // __MAPNIK_BENCH_FRAMEWORK_HPP__

benchmark/test_array_allocation.cpp

@@ -231,33 +231,6 @@ public:
     }
 };
 
-class test3d : public benchmark::test_case
-{
-public:
-    uint32_t size_;
-    std::vector<uint8_t> array_;
-    test3d(mapnik::parameters const& params)
-     : test_case(params),
-       size_(*params.get<mapnik::value_integer>("size",256*256)),
-       array_(size_,0) { }
-    bool validate() const
-    {
-        return true;
-    }
-    bool operator()() const
-    {
-         for (std::size_t i=0;i<iterations_;++i) {
-             std::deque<uint8_t> data(size_);
-             for (std::size_t i=0;i<size_;++i) {
-                 if (data[i] != 0) {
-                     throw std::runtime_error("found non zero value");
-                 }
-             }
-         }
-         return true;
-    }
-};
-
 class test4 : public benchmark::test_case
 {
 public:
@@ -386,62 +359,21 @@ public:
 
 int main(int argc, char** argv)
 {
-    int return_value = 0;
+    return benchmark::sequencer(argc, argv)
-    mapnik::parameters params;
+        .run<test4>("calloc")
-    benchmark::handle_args(argc,argv,params);
+        .run<test1>("malloc/memcpy")
-    {
+        .run<test1b>("malloc/memset")
-        test4 test_runner4(params);
+        .run<test1c>("operator new/std::fill")
-        return_value = return_value | run(test_runner4,"calloc");
+        .run<test2>("operator new/memcpy")
-    }
+        .run<test3>("vector(N)")
-    {
+        .run<test3b>("vector/resize")
-        test1 test_runner(params);
+        .run<test3c>("vector/assign")
-        return_value = return_value | run(test_runner,"malloc/memcpy");
+        .run<test3d>("deque(N)")
-    }
+        .run<test5>("std::string range")
-    {
+        .run<test5b>("std::string &[0]")
-        test1b test_runner(params);
+        .run<test6>("valarray")
-        return_value = return_value | run(test_runner,"malloc/memset");
-    }
-    {
-        test1c test_runner(params);
-        return_value = return_value | run(test_runner,"operator new/std::fill");
-    }
-    {
-        test2 test_runner(params);
-        return_value = return_value | run(test_runner,"operator new/memcpy");
-    }
-    {
-        test3 test_runner(params);
-        return_value = return_value | run(test_runner,"vector(N)");
-    }
-    {
-        test3b test_runner(params);
-        return_value = return_value | run(test_runner,"vector/resize");
-    }
-    {
-        test3c test_runner(params);
-        return_value = return_value | run(test_runner,"vector/assign");
-    }
-    {
-        test3d test_runner(params);
-        return_value = return_value | run(test_runner,"deque(N)");
-    }
-    {
-        test5 test_runner(params);
-        return_value = return_value | run(test_runner,"std::string range");
-    }
-    {
-        test5b test_runner(params);
-        return_value = return_value | run(test_runner,"std::string &[0]");
-    }
-    {
-        test6 test_runner(params);
-        return_value = return_value | run(test_runner,"valarray");
-    }
 #if BOOST_VERSION >= 105400
-    {
+        .run<test7>("static_vector")
-        test7 test_runner(params);
-        return_value = return_value | run(test_runner,"static_vector");
-    }
 #endif
-    return return_value;
+        .done();
 }

benchmark/test_numeric_cast_vs_static_cast.cpp

@@ -73,16 +73,8 @@ public:
 
 int main(int argc, char** argv)
 {
-    mapnik::parameters params;
+    return benchmark::sequencer(argc, argv)
-    benchmark::handle_args(argc,argv,params);
+        .run<test_static>("static_cast")
-    int return_value = 0;
+        .run<test_numeric>("numeric_cast")
-    {
+        .done();
-        test_static test_runner(params);
-        return_value = return_value | run(test_runner,"static_cast");
-    }
-    {
-        test_numeric test_runner(params);
-        return_value = return_value | run(test_runner,"numeric_cast");
-    }
-    return return_value;
 }

benchmark/test_png_encoding1.cpp

@@ -19,8 +19,8 @@ public:
             out.clear();
             out = mapnik::save_to_string(im_,"png8:m=h:z=1");
         }
+        return true;
     }
-    return true;
 };
 
 BENCHMARK(test,"encoding blank png")

benchmark/test_png_encoding2.cpp

@@ -30,8 +30,8 @@ public:
             out.clear();
             out = mapnik::save_to_string(*im_,"png8:m=h:z=1");
         }
+        return true;
     }
-    return true;
 };
 
 BENCHMARK(test,"encoding multicolor png")

benchmark/test_polygon_clipping_rendering.cpp

@@ -51,30 +51,10 @@ int main(int argc, char** argv)
     mapnik::box2d<double> z1(-20037508.3428,-8317435.0606,20037508.3428,18399242.7298);
     // bbox for 16/10491/22911.png
     mapnik::box2d<double> z16(-13622912.929097254,6026906.8062295765,-13621689.93664469,6028129.79868214);
-    int return_value = 0;
+    return benchmark::sequencer(argc, argv)
-    {
+        .run<test>("polygon clip render z1", "benchmark/data/polygon_rendering_clip.xml", z1)
-        test test_runner(params,
+        .run<test>("polygon noclip render z1", "benchmark/data/polygon_rendering_no_clip.xml", z1)
-                          "benchmark/data/polygon_rendering_clip.xml",
+        .run<test>("polygon clip render z16", "benchmark/data/polygon_rendering_clip.xml", z16)
-                          z1);
+        .run<test>("polygon noclip render z16", "benchmark/data/polygon_rendering_no_clip.xml", z16)
-        return_value = return_value | run(test_runner,"polygon clip render z1");
+        .done();
-    }
-    {
-        test test_runner(params,
-                          "benchmark/data/polygon_rendering_no_clip.xml",
-                          z1);
-        return_value = return_value | run(test_runner,"polygon noclip render z1");
-    }
-    {
-        test test_runner(params,
-                          "benchmark/data/polygon_rendering_clip.xml",
-                          z16);
-        return_value = return_value | run(test_runner,"polygon clip render z16");
-    }
-    {
-        test test_runner(params,
-                          "benchmark/data/polygon_rendering_no_clip.xml",
-                          z16);
-        return_value = return_value | run(test_runner,"polygon noclip render z16");
-    }
-    return return_value;
 }

benchmark/test_proj_transform1.cpp

@@ -59,42 +59,16 @@ public:
 // echo -180 -60 | cs2cs -f "%.10f" +init=epsg:4326 +to +init=epsg:3857
 int main(int argc, char** argv)
 {
-    mapnik::parameters params;
-    benchmark::handle_args(argc,argv,params);
     mapnik::box2d<double> from(-180,-80,180,80);
     mapnik::box2d<double> to(-20037508.3427892476,-15538711.0963092316,20037508.3427892476,15538711.0963092316);
     std::string from_str("+init=epsg:4326");
     std::string to_str("+init=epsg:3857");
     std::string from_str2("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs");
     std::string to_str2("+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over");
-    int return_value = 0;
+    return benchmark::sequencer(argc, argv)
-    test test_runner(params,
+        .run<test>("lonlat->merc epsg", from_str, to_str, from, to, true)
-                     from_str,
+        .run<test>("lonlat->merc literal", from_str2, to_str2, from, to, true)
-                     to_str,
+        .run<test>("merc->lonlat epsg", to_str, from_str, to, from, true)
-                     from,
+        .run<test>("merc->lonlat literal", to_str2, from_str2, to, from, true)
-                     to,
+        .done();
-                     true);
-    return_value = return_value | run(test_runner,"lonlat->merc epsg");
-    test test_runner2(params,
-                     from_str2,
-                     to_str2,
-                     from,
-                     to,
-                     true);
-    return_value = return_value | run(test_runner2,"lonlat->merc literal");
-    test test_runner3(params,
-                     to_str,
-                     from_str,
-                     to,
-                     from,
-                     true);
-    return_value = return_value | run(test_runner3,"merc->lonlat epsg");
-    test test_runner4(params,
-                     to_str2,
-                     from_str2,
-                     to,
-                     from,
-                     true);
-    return_value = return_value | run(test_runner4,"merc->lonlat literal");
-    return return_value;
 }

bootstrap.sh

@@ -112,7 +112,6 @@ CPP_TESTS = True
 PGSQL2SQLITE = True
 XMLPARSER = 'ptree'
 SVG2PNG = True
-SAMPLE_INPUT_PLUGINS = True
 "
 }
 

deps/mapbox/variant

@@ -1 +1 @@
-Subproject commit 0bd70f39a1d12dd8caea055f5ba970d404f4b825
+Subproject commit 5aab5df0dc899b484c04ce9c649645787ee0bc5c

include/mapnik/csv/csv_grammar.hpp

@@ -35,34 +35,48 @@ using csv_value  = std::string;
 using csv_line = std::vector<csv_value>;
 using csv_data = std::vector<csv_line>;
 
-template <typename Iterator>
+struct csv_white_space_skipper : qi::primitive_parser<csv_white_space_skipper>
-struct csv_white_space_skipper : qi::grammar<Iterator>
 {
-    csv_white_space_skipper()
+    template <typename Context, typename Iterator>
-        : csv_white_space_skipper::base_type(skip)
+    struct attribute
     {
-        using namespace qi;
+        typedef qi::unused_type type;
-        qi::lit_type lit;
+    };
-        skip = +lit(' ')
+
-            ;
+    template <typename Iterator, typename Context
+      , typename Skipper, typename Attribute>
+    bool parse(Iterator& first, Iterator const& last
+      , Context& /*context*/, Skipper const& skipper
+      , Attribute& /*attr*/) const
+    {
+        qi::skip_over(first, last, skipper);
+        if (first != last && *first == ' ')
+        {
+            while (++first != last && *first == ' ')
+                ;
+            return true;
+        }
+        return false;
+    }
+
+    template <typename Context>
+    qi::info what(Context& /*context*/) const
+    {
+        return qi::info("csv_white_space_skipper");
     }
-    qi::rule<Iterator> skip;
 };
 
-template <typename Iterator, typename Skipper = csv_white_space_skipper<Iterator> >
+
+template <typename Iterator, typename Skipper = csv_white_space_skipper>
 struct csv_line_grammar : qi::grammar<Iterator, csv_line(char, char), Skipper>
 {
     csv_line_grammar()
         : csv_line_grammar::base_type(line)
     {
-        using namespace qi;
-        qi::_a_type _a;
         qi::_r1_type _r1;
         qi::_r2_type _r2;
         qi::lit_type lit;
-        qi::_1_type _1;
         qi::char_type char_;
-        qi::omit_type omit;
         unesc_char.add
             ("\\a", '\a')
             ("\\b", '\b')
@@ -76,11 +90,11 @@ struct csv_line_grammar : qi::grammar<Iterator, csv_line(char, char), Skipper>
             ("\\\"", '\"')
             ("\"\"", '\"') // double quote
             ;
-        line = -omit[char_("\n\r")] >> column(_r1, _r2) % lit(_r1)
+        line = -lit("\n\r") >> column(_r1, _r2) % lit(_r1)
             ;
-        column = quoted(_r2) | *(char_ - (lit(_r1)))
+        column = quoted(_r2) | *(char_ - lit(_r1))
             ;
-        quoted = omit[char_(_r1)[_a = _1]] > text(_a) > -lit(_a) // support unmatched quotes or not (??)
+        quoted = lit(_r1) > text(_r1) > lit(_r1) // support unmatched quotes or not (??)
             ;
         text = *(unesc_char | (char_ - lit(_r1)))
             ;
@@ -90,7 +104,7 @@ private:
     qi::rule<Iterator, csv_line(char, char),  Skipper> line;
     qi::rule<Iterator, csv_value(char, char)> column; // no-skip
     qi::rule<Iterator, csv_value(char)> text; // no-skip
-    qi::rule<Iterator, qi::locals<char>, csv_value(char)> quoted; //no-skip
+    qi::rule<Iterator, csv_value(char)> quoted; // no-skip
     qi::symbols<char const, char const> unesc_char;
 };
 

include/mapnik/mapped_memory_cache.hpp

@@ -52,6 +52,8 @@ public:
     void clear();
 };
 
+extern template class MAPNIK_DECL singleton<mapped_memory_cache, CreateStatic>;
+
 }
 
 #endif // MAPNIK_MAPPED_MEMORY_CACHE_HPP

include/mapnik/pool.hpp

@@ -24,7 +24,6 @@
 #define MAPNIK_POOL_HPP
 
 // mapnik
-#include <mapnik/util/singleton.hpp>
 #include <mapnik/util/noncopyable.hpp>
 
 // boost

include/mapnik/text/face.hpp

@@ -23,8 +23,8 @@
 #define MAPNIK_FACE_HPP
 
 //mapnik
-#include <mapnik/text/glyph_info.hpp>
 #include <mapnik/config.hpp>
+#include <mapnik/text/glyph_info.hpp>
 #include <mapnik/util/noncopyable.hpp>
 
 // freetype2
@@ -36,7 +36,6 @@ extern "C"
 }
 
 //stl
-#include <unordered_map>
 #include <memory>
 #include <string>
 #include <vector>

include/mapnik/util/spatial_index.hpp

@@ -37,6 +37,16 @@ using mapnik::query;
 
 namespace mapnik { namespace util {
 
+
+template <typename InputStream>
+bool check_spatial_index(InputStream& in)
+{
+    char header[17]; // mapnik-index
+    std::memset(header, 0, 17);
+    in.read(header,16);
+    return (std::strncmp(header, "mapnik-index",12) == 0);
+}
+
 template <typename Value, typename Filter, typename InputStream>
 class spatial_index
 {
@@ -53,23 +63,13 @@ private:
     static void read_envelope(InputStream& in, box2d<double>& envelope);
     static void query_node(Filter const& filter, InputStream& in, std::vector<Value> & results);
     static void query_first_n_impl(Filter const& filter, InputStream& in, std::vector<Value> & results, std::size_t count);
-    static bool check_header(InputStream& in);
 };
 
-template <typename Value, typename Filter, typename InputStream>
-bool spatial_index<Value, Filter, InputStream>::check_header(InputStream& in)
-{
-    static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout type");
-    char header[17]; // mapnik-index
-    std::memset(header, 0, 17);
-    in.read(header,16);
-    return (std::strncmp(header, "mapnik-index",12) == 0);
-}
-
 template <typename Value, typename Filter, typename InputStream>
 box2d<double> spatial_index<Value, Filter, InputStream>::bounding_box(InputStream& in)
 {
-    if (!check_header(in)) throw std::runtime_error("Invalid index file (regenerate with shapeindex)");
+    static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout type");
+    if (!check_spatial_index(in)) throw std::runtime_error("Invalid index file (regenerate with shapeindex)");
     in.seekg(16 + 4, std::ios::beg);
     box2d<double> box;
     read_envelope(in, box);
@@ -80,7 +80,8 @@ box2d<double> spatial_index<Value, Filter, InputStream>::bounding_box(InputStrea
 template <typename Value, typename Filter, typename InputStream>
 void spatial_index<Value, Filter, InputStream>::query(Filter const& filter, InputStream& in, std::vector<Value>& results)
 {
-    if (!check_header(in)) throw std::runtime_error("Invalid index file (regenerate with shapeindex)");
+    static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout type");
+    if (!check_spatial_index(in)) throw std::runtime_error("Invalid index file (regenerate with shapeindex)");
     in.seekg(16, std::ios::beg);
     query_node(filter, in, results);
 }
@@ -115,7 +116,8 @@ void spatial_index<Value, Filter, InputStream>::query_node(Filter const& filter,
 template <typename Value, typename Filter, typename InputStream>
 void spatial_index<Value, Filter, InputStream>::query_first_n(Filter const& filter, InputStream& in, std::vector<Value>& results, std::size_t count)
 {
-    if (!check_header(in)) throw std::runtime_error("Invalid index file (regenerate with shapeindex)");
+    static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout type");
+    if (!check_spatial_index(in)) throw std::runtime_error("Invalid index file (regenerate with shapeindex)");
     in.seekg(16, std::ios::beg);
     query_first_n_impl(filter, in, results, count);
 }

include/mapnik/value.hpp

@@ -43,33 +43,16 @@
 #include <unicode/unistr.h>
 #include <unicode/ustring.h>
 
-namespace mapnik  {
+namespace mapnik {
+
+using value_base = util::variant<value_null, value_bool, value_integer,value_double, value_unicode_string>;
 
 inline void to_utf8(mapnik::value_unicode_string const& input, std::string & target)
 {
-    if (input.isEmpty()) return;
+    target.clear(); // mimic previous target.assign(...) semantics
-
+    input.toUTF8String(target); // this appends to target
-    const int BUF_SIZE = 256;
-    char buf [BUF_SIZE];
-    int len;
-
-    UErrorCode err = U_ZERO_ERROR;
-    u_strToUTF8(buf, BUF_SIZE, &len, input.getBuffer(), input.length(), &err);
-    if (err == U_BUFFER_OVERFLOW_ERROR || err == U_STRING_NOT_TERMINATED_WARNING )
-    {
-        const std::unique_ptr<char[]> buf_ptr(new char [len+1]);
-        err = U_ZERO_ERROR;
-        u_strToUTF8(buf_ptr.get() , len + 1, &len, input.getBuffer(), input.length(), &err);
-        target.assign(buf_ptr.get() , static_cast<std::size_t>(len));
-    }
-    else
-    {
-        target.assign(buf, static_cast<std::size_t>(len));
-    }
 }
 
-using value_base = util::variant<value_null, value_bool, value_integer,value_double, value_unicode_string>;
-
 namespace detail {
 
 namespace {
@@ -80,6 +63,11 @@ struct both_arithmetic : std::integral_constant<bool,
 
 struct equals
 {
+    static bool apply(value_null, value_unicode_string const& rhs)
+    {
+        return false;
+    }
+
     template <typename T>
     static auto apply(T const& lhs, T const& rhs)
         -> decltype(lhs == rhs)
@@ -90,6 +78,15 @@ struct equals
 
 struct not_equal
 {
+    // back compatibility shim to equate empty string with null for != test
+    // https://github.com/mapnik/mapnik/issues/1859
+    // TODO - consider removing entire specialization at Mapnik 3.1.x
+    static bool apply(value_null, value_unicode_string const& rhs)
+    {
+        if (rhs.isEmpty()) return false;
+        return true;
+    }
+
     template <typename T>
     static auto apply(T const& lhs, T const& rhs)
         ->decltype(lhs != rhs)
@@ -100,6 +97,11 @@ struct not_equal
 
 struct greater_than
 {
+    static bool apply(value_null, value_unicode_string const& rhs)
+    {
+        return false;
+    }
+
     template <typename T>
     static auto  apply(T const& lhs, T const& rhs)
         ->decltype(lhs > rhs)
@@ -110,6 +112,11 @@ struct greater_than
 
 struct greater_or_equal
 {
+    static bool apply(value_null, value_unicode_string const& rhs)
+    {
+        return false;
+    }
+
     template <typename T>
     static auto apply(T const& lhs, T const& rhs)
         ->decltype(lhs >= rhs)
@@ -120,6 +127,11 @@ struct greater_or_equal
 
 struct less_than
 {
+    static bool apply(value_null, value_unicode_string const& rhs)
+    {
+        return false;
+    }
+
     template <typename T>
     static auto apply(T const& lhs, T const& rhs)
          ->decltype(lhs < rhs)
@@ -130,6 +142,11 @@ struct less_than
 
 struct less_or_equal
 {
+    static bool apply(value_null, value_unicode_string const& rhs)
+    {
+        return false;
+    }
+
     template <typename T>
     static auto apply(T const& lhs, T const& rhs)
         ->decltype(lhs <= rhs)
@@ -147,9 +164,20 @@ struct comparison
     bool operator() (value_unicode_string const& lhs,
                      value_unicode_string const& rhs) const
     {
-        return Op::apply(lhs, rhs) ? true: false;
+        return Op::apply(lhs, rhs) ? true : false;
     }
 
+    //////////////////////////////////////////////////////////////////////////
+    // special case for unicode_string and value_null
+    //////////////////////////////////////////////////////////////////////////
+
+    bool operator() (value_null const& lhs, value_unicode_string const& rhs) const
+    {
+        return Op::apply(lhs, rhs);
+    }
+    //////////////////////////////////////////////////////////////////////////
+
+
     // same types
     template <typename T>
     bool operator() (T lhs, T rhs) const
@@ -582,7 +610,7 @@ struct convert<value_double>
     value_double operator() (value_unicode_string const& val) const
     {
         std::string utf8;
-        to_utf8(val,utf8);
+        val.toUTF8String(utf8);
         return operator()(utf8);
     }
 
@@ -621,7 +649,7 @@ struct convert<value_integer>
     value_integer operator() (value_unicode_string const& val) const
     {
         std::string utf8;
-        to_utf8(val,utf8);
+        val.toUTF8String(utf8);
         return operator()(utf8);
     }
 
@@ -646,7 +674,7 @@ struct convert<std::string>
     std::string operator() (value_unicode_string const& val) const
     {
         std::string utf8;
-        to_utf8(val,utf8);
+        val.toUTF8String(utf8);
         return utf8;
     }
 
@@ -664,7 +692,7 @@ struct convert<std::string>
 
     std::string operator() (value_null const&) const
     {
-        return "";
+        return std::string();
     }
 };
 
@@ -694,30 +722,66 @@ struct to_unicode_impl
 
     value_unicode_string operator() (value_bool val) const
     {
-        if (val) {
+        return value_unicode_string(val ? "true" : "false");
-            std::string str("true");
-            return value_unicode_string(str.c_str());
-        }
-        std::string str("false");
-        return value_unicode_string(str.c_str());
     }
 
     value_unicode_string operator() (value_null const&) const
     {
-        return value_unicode_string("");
+        return value_unicode_string();
     }
 };
 
 struct to_expression_string_impl
 {
+    struct EscapingByteSink : U_NAMESPACE_QUALIFIER ByteSink
+    {
+        std::string dest_;
+        char quote_;
+
+        explicit EscapingByteSink(char quote)
+            : quote_(quote)
+        {}
+
+        virtual void Append(const char* data, int32_t n)
+        {
+            // reserve enough room to hold the appended chunk and quotes;
+            // if another chunk follows, or any character needs escaping,
+            // the string will grow naturally
+            if (dest_.empty())
+            {
+                dest_.reserve(2 + static_cast<std::size_t>(n));
+                dest_.append(1, quote_);
+            }
+            else
+            {
+                dest_.reserve(dest_.size() + n + 1);
+            }
+
+            for (auto end = data + n; data < end; ++data)
+            {
+                if (*data == '\\' || *data == quote_)
+                    dest_.append(1, '\\');
+                dest_.append(1, *data);
+            }
+        }
+
+        virtual void Flush()
+        {
+            if (dest_.empty())
+                dest_.append(2, quote_);
+            else
+                dest_.append(1, quote_);
+        }
+    };
+
     explicit to_expression_string_impl(char quote = '\'')
         : quote_(quote) {}
 
     std::string operator() (value_unicode_string const& val) const
     {
-        std::string utf8;
+        EscapingByteSink sink(quote_);
-        to_utf8(val,utf8);
+        val.toUTF8(sink);
-        return quote_ + utf8 + quote_;
+        return sink.dest_;
     }
 
     std::string operator() (value_integer val) const

plugins/input/csv/csv_datasource.cpp

@@ -180,21 +180,21 @@ void csv_datasource::parse_csv(T & stream)
     char newline;
     bool has_newline;
     char detected_quote;
-    std::tie(newline, has_newline, detected_quote) = detail::autodect_newline_and_quote(stream, file_length);
+    char detected_separator;
+    std::tie(newline, has_newline, detected_separator, detected_quote) = detail::autodect_csv_flavour(stream, file_length);
     if (quote_ == 0) quote_ = detected_quote;
-    // set back to start
+    if (separator_ == 0) separator_ = detected_separator;
-    stream.seekg(0, std::ios::beg);
-    std::string csv_line;
-    csv_utils::getline_csv(stream, csv_line, newline, quote_);
-    if (separator_ == 0)
-    {
-        separator_ = detail::detect_separator(csv_line);
-    }
 
+    // set back to start
     MAPNIK_LOG_DEBUG(csv) << "csv_datasource: separator: '" << separator_
                           << "' quote: '" << quote_ << "'";
-    stream.seekg(0, std::ios::beg);
 
+    // rewind stream
+    stream.seekg(0, std::ios::beg);
+    //
+    std::string csv_line;
+    csv_utils::getline_csv(stream, csv_line, newline, quote_);
+    stream.seekg(0, std::ios::beg);
     int line_number = 0;
     if (!manual_headers_.empty())
     {

plugins/input/csv/csv_utils.hpp

@@ -47,7 +47,7 @@ namespace csv_utils
 {
 
 static const mapnik::csv_line_grammar<char const*> line_g;
-static const mapnik::csv_white_space_skipper<char const*> skipper;
+static const mapnik::csv_white_space_skipper skipper{};
 
 static mapnik::csv_line parse_line(char const* start, char const* end, char separator, char quote, std::size_t num_columns)
 {
@@ -98,54 +98,28 @@ std::size_t file_length(T & stream)
     return stream.tellg();
 }
 
-static inline char detect_separator(std::string const& str)
-{
-    char separator = ','; // default
-    int num_commas = std::count(str.begin(), str.end(), ',');
-    // detect tabs
-    int num_tabs = std::count(str.begin(), str.end(), '\t');
-    if (num_tabs > 0)
-    {
-        if (num_tabs > num_commas)
-        {
-            separator = '\t';
-            MAPNIK_LOG_DEBUG(csv) << "csv_datasource: auto detected tab separator";
-        }
-    }
-    else // pipes
-    {
-        int num_pipes = std::count(str.begin(), str.end(), '|');
-        if (num_pipes > num_commas)
-        {
-            separator = '|';
-            MAPNIK_LOG_DEBUG(csv) << "csv_datasource: auto detected '|' separator";
-        }
-        else // semicolons
-        {
-            int num_semicolons = std::count(str.begin(), str.end(), ';');
-            if (num_semicolons > num_commas)
-            {
-                separator = ';';
-                MAPNIK_LOG_DEBUG(csv) << "csv_datasource: auto detected ';' separator";
-            }
-        }
-    }
-    return separator;
-}
-
 template <typename T>
-std::tuple<char,bool,char> autodect_newline_and_quote(T & stream, std::size_t file_length)
+std::tuple<char, bool, char, char> autodect_csv_flavour(T & stream, std::size_t file_length)
 {
-    // autodetect newlines
+    // autodetect newlines/quotes/separators
-    char newline = '\n';
+    char newline = '\n'; // default
     bool has_newline = false;
-    bool has_quote = false;
+    bool has_single_quote = false;
-    char quote = '"';
+    char quote = '"'; // default
+    char separator = ','; // default
+    // local counters
+    int num_commas = 0;
+    int num_tabs = 0;
+    int num_pipes = 0;
+    int num_semicolons = 0;
+
     static std::size_t const max_size = 4000;
     std::size_t size = std::min(file_length, max_size);
-    for (std::size_t lidx = 0; lidx < size; ++lidx)
+    std::vector<char> buffer;
+    buffer.resize(size);
+    stream.read(buffer.data(), size);
+    for (auto c : buffer)
     {
-        char c = static_cast<char>(stream.get());
         switch (c)
         {
         case '\r':
@@ -156,18 +130,76 @@ std::tuple<char,bool,char> autodect_newline_and_quote(T & stream, std::size_t fi
             has_newline = true;
             break;
         case '\'':
-        case '"':
+            if (!has_single_quote)
-            if (!has_quote)
             {
                 quote = c;
-                has_quote = true;
+                has_single_quote = true;
             }
             break;
+        case ',':
+            if (!has_newline) ++num_commas;
+            break;
+        case '\t':
+            if (!has_newline) ++num_tabs;
+            break;
+        case '|':
+            if (!has_newline) ++num_pipes;
+            break;
+        case ';':
+           if (!has_newline) ++num_semicolons;
+            break;
+        }
+    }
+    // detect separator
+    if (num_tabs > 0 && num_tabs > num_commas)
+    {
+        separator = '\t';
+        MAPNIK_LOG_DEBUG(csv) << "csv_datasource: auto detected tab separator";
+    }
+    else // pipes/semicolons
+    {
+        if (num_pipes > num_commas)
+        {
+            separator = '|';
+            MAPNIK_LOG_DEBUG(csv) << "csv_datasource: auto detected '|' separator";
+        }
+        else  if (num_semicolons > num_commas)
+        {
+            separator = ';';
+            MAPNIK_LOG_DEBUG(csv) << "csv_datasource: auto detected ';' separator";
         }
     }
-    return std::make_tuple(newline, has_newline, quote);
-}
 
+    if (has_newline && has_single_quote)
+    {
+        std::istringstream ss(std::string(buffer.begin(), buffer.end()));
+        std::size_t num_columns = 0;
+        for (std::string line; csv_utils::getline_csv(ss, line, newline, quote); )
+        {
+            if (size < file_length && ss.eof())
+            {
+                // we can't be sure last line
+                // is not truncated so skip it
+                break;
+            }
+            if (line.size() == 0) continue; // empty lines are not interesting
+            auto num_quotes = std::count(line.begin(), line.end(), quote);
+            if (num_quotes % 2 != 0)
+            {
+                quote = '"';
+                break;
+            }
+            auto columns = csv_utils::parse_line(line, separator, quote);
+            if (num_columns > 0 && num_columns != columns.size())
+            {
+                quote = '"';
+                break;
+            }
+            num_columns = columns.size();
+        }
+    }
+    return std::make_tuple(newline, has_newline, separator, quote);
+}
 
 struct geometry_column_locator
 {

plugins/input/shape/shape_datasource.cpp

@@ -98,11 +98,11 @@ shape_datasource::shape_datasource(parameters const& params)
         mapnik::progress_timer __stats2__(std::clog, "shape_datasource::init(get_column_description)");
 #endif
 
-        std::unique_ptr<shape_io> shape_ref = std::make_unique<shape_io>(shape_name_);
+        shape_io shape(shape_name_);
-        init(*shape_ref);
+        init(shape);
-        for (int i=0;i<shape_ref->dbf().num_fields();++i)
+        for (int i = 0; i < shape.dbf().num_fields(); ++i)
         {
-            field_descriptor const& fd = shape_ref->dbf().descriptor(i);
+            field_descriptor const& fd = shape.dbf().descriptor(i);
             std::string fld_name=fd.name_;
             switch (fd.type_)
             {

plugins/input/shape/shape_io.hpp

@@ -23,16 +23,16 @@
 #ifndef SHAPE_IO_HPP
 #define SHAPE_IO_HPP
 
+// stl
+#include <memory>
+#include <ios>
 // mapnik
 #include <mapnik/box2d.hpp>
 #include <mapnik/util/noncopyable.hpp>
-
+#include <mapnik/util/spatial_index.hpp>
 // boost
 #include <boost/optional.hpp>
-
+//
-// stl
-#include <memory>
-
 #include "dbfile.hpp"
 #include "shapefile.hpp"
 
@@ -72,7 +72,13 @@ public:
 
     inline bool has_index() const
     {
-        return (index_ && index_->is_open());
+        if (index_ && index_->is_open())
+        {
+            bool status = mapnik::util::check_spatial_index(index_->file());
+            index_->seek(0);// rewind
+            return status;
+        }
+        return false;
     }
 
     inline int id() const { return id_;}

scripts/build-appveyor.bat

@@ -32,6 +32,16 @@ SET PATH=C:\Program Files\7-Zip;%PATH%
 git submodule update --init deps/mapbox/variant
 IF %ERRORLEVEL% NEQ 0 GOTO ERROR
 
+
+::python bindings, including test data
+IF NOT EXIST bindings\python git clone --recursive https://github.com/mapnik/python-mapnik.git bindings/python
+IF %ERRORLEVEL% NEQ 0 GOTO ERROR
+
+CD bindings\python & IF %ERRORLEVEL% NEQ 0 GOTO ERROR
+git fetch & IF %ERRORLEVEL% NEQ 0 GOTO ERROR
+git pull & IF %ERRORLEVEL% NEQ 0 GOTO ERROR
+CD ..\.. & IF %ERRORLEVEL% NEQ 0 GOTO ERROR
+
 ::cloning mapnik-gyp
 if EXIST mapnik-gyp ECHO mapnik-gyp already cloned && GOTO MAPNIK_GYP_ALREADY_HERE
 CALL git clone https://github.com/mapnik/mapnik-gyp.git

scripts/build-local.bat

@@ -33,14 +33,6 @@ SET msvs_toolset=14
 SET platform=x64
 SET APPVEYOR_BUILD_FOLDER=%CD%
 
-IF NOT EXIST bindings\python git clone https://github.com/mapnik/python-mapnik.git bindings/python
-IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-
-CD bindings\python & IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-git fetch & IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-git pull & IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-CD ..\.. & IF %ERRORLEVEL% NEQ 0 GOTO ERROR
-
 ECHO pulling test data
 CALL git submodule update --init
 IF %ERRORLEVEL% NEQ 0 GOTO ERROR

scripts/travis-common.sh

@@ -0,0 +1,109 @@
+#! /bin/bash
+
+# enabled VALUE
+#   - if VALUE is empty or falsy, returns 1 (false)
+#   - otherwise returns 0 (true)
+# enabled VALUE COMMAND ...
+#   - if VALUE is empty or falsy, returns 0 (true)
+#   - otherwise runs COMMAND and returns its result
+enabled () {
+    local value="$1"; shift
+    case $value in
+        ''|'0'|[Ff]alse|[Nn]o) test $# -ne 0;;
+        *) test $# -eq 0 || "$@";;
+    esac
+}
+
+# on NAME
+#   - if NAME == $TRAVIS_OS_NAME, returns 0 (true)
+#   - otherwise returns 1 (false)
+# on NAME COMMAND ...
+#   - if NAME == $TRAVIS_OS_NAME, runs COMMAND and returns its result
+#   - otherwise returns 0 (true)
+on () {
+    local name="$1"; shift
+    case $name in
+        $TRAVIS_OS_NAME) test $# -eq 0 || "$@";;
+        *) test $# -ne 0;;
+    esac
+}
+
+git_submodule_update () {
+    git submodule update "$@" && return
+    # failed, search pull requests for matching commits
+    git submodule foreach \
+        '
+        test "$sha1" = "`git rev-parse HEAD`" ||
+        git ls-remote origin "refs/pull/*/head" |
+        while read hash ref; do
+            if test "$hash" = "$sha1"; then
+                git config --add remote.origin.fetch "+$ref:$ref";
+            fi
+        done
+        '
+    # try again with added fetch refs
+    git submodule update "$@"
+}
+
+# install and call pip
+pip () {
+    if ! which pip >/dev/null; then
+        easy_install --user pip && \
+        export PATH="$HOME/Library/Python/2.7/bin:$PATH"
+    fi
+    command pip "$@"
+}
+
+# commit_message_contains TEXT
+#   - returns 0 (true) if TEXT is found in commit message
+#   - case-insensitive, plain-text search, not regex
+commit_message_contains () {
+    git log -1 --pretty='%B' "$TRAVIS_COMMIT" | grep -qiFe "$*"
+}
+
+commit_message_parse () {
+    if commit_message_contains '[skip tests]'; then
+        config_override "CPP_TESTS = False"
+    fi
+    if commit_message_contains '[skip utils]'; then
+        config_override "MAPNIK_INDEX = False"
+        config_override "MAPNIK_RENDER = False"
+        config_override "PGSQL2SQLITE = False"
+        config_override "SHAPEINDEX = False"
+        config_override "SVG2PNG = False"
+    fi
+}
+
+config_override () {
+    echo "Appending to config.py:" "$@"
+    echo "$@" >> ./config.py
+}
+
+configure () {
+    if enabled ${COVERAGE}; then
+        ./configure "$@" PGSQL2SQLITE=False SVG2PNG=False SVG_RENDERER=False \
+            CUSTOM_LDFLAGS='--coverage' CUSTOM_CXXFLAGS='--coverage' \
+            CUSTOM_CFLAGS='--coverage' DEBUG=True
+    elif enabled ${MASON_PUBLISH}; then
+        export MASON_NAME=mapnik
+        export MASON_VERSION=latest
+        export MASON_LIB_FILE=lib/libmapnik-wkt.a
+        source ./.mason/mason.sh
+        ./configure "$@" PREFIX=${MASON_PREFIX} \
+            PATH_REPLACE='' MAPNIK_BUNDLED_SHARE_DIRECTORY=True \
+            RUNTIME_LINK='static'
+    else
+        ./configure "$@"
+    fi
+    # print final config values, sorted and indented
+    sort -sk1,1 ./config.py | sed -e 's/^/	/'
+}
+
+coverage () {
+    ./mason_packages/.link/bin/cpp-coveralls \
+        --build-root . --gcov-options '\-lp' --exclude mason_packages \
+        --exclude .sconf_temp --exclude benchmark --exclude deps \
+        --exclude scons --exclude test --exclude demo --exclude docs \
+        --exclude fonts --exclude utils \
+        > /dev/null
+}

src/color.cpp

@@ -25,22 +25,11 @@
 #include <mapnik/color_factory.hpp>
 #include <mapnik/config_error.hpp>
 
-// agg
-#include "agg_color_rgba.h"
-
 #pragma GCC diagnostic push
 #include <mapnik/warning_ignore.hpp>
 #include <boost/spirit/include/karma.hpp>
-#include <boost/spirit/include/phoenix_statement.hpp>
-#include <boost/spirit/include/phoenix_core.hpp>
-#include <boost/spirit/include/phoenix_operator.hpp>
-#include <boost/spirit/include/phoenix_fusion.hpp>
-#include <boost/spirit/include/phoenix_function.hpp>
 #pragma GCC diagnostic pop
 
-// stl
-#include <sstream>
-
 namespace mapnik {
 
 color::color(std::string const& str, bool premultiplied)
@@ -52,22 +41,23 @@ color::color(std::string const& str, bool premultiplied)
 std::string color::to_string() const
 {
     namespace karma = boost::spirit::karma;
-    boost::spirit::karma::_1_type _1;
     boost::spirit::karma::eps_type eps;
     boost::spirit::karma::double_type double_;
-    boost::spirit::karma::string_type kstring;
+    boost::spirit::karma::uint_generator<uint8_t,10> color_;
-    boost::spirit::karma::uint_generator<uint8_t,10> color_generator;
     std::string str;
     std::back_insert_iterator<std::string> sink(str);
-    karma::generate(sink,
+    karma::generate(sink, eps(alpha() < 255)
                     // begin grammar
-                    kstring[ boost::phoenix::if_(alpha()==255) [_1="rgb("].else_[_1="rgba("]]
+                    << "rgba("
-                    << color_generator[_1 = red()] << ','
+                    << color_(red()) << ','
-                    << color_generator[_1 = green()] << ','
+                    << color_(green()) << ','
-                    << color_generator[_1 = blue()]
+                    << color_(blue()) << ','
-                    << kstring[ boost::phoenix::if_(alpha()==255) [_1 = ')'].else_[_1 =',']]
+                    << double_(alpha()/255.0) << ')'
-                    << eps(alpha()<255) << double_ [_1 = alpha()/255.0]
+                    |
-                    << ')'
+                    "rgb("
+                    << color_(red()) << ','
+                    << color_(green()) << ','
+                    << color_(blue()) << ')'
                     // end grammar
         );
     return str;
@@ -76,7 +66,6 @@ std::string color::to_string() const
 std::string color::to_hex_string() const
 {
     namespace karma = boost::spirit::karma;
-    boost::spirit::karma::_1_type _1;
     boost::spirit::karma::hex_type hex;
     boost::spirit::karma::eps_type eps;
     boost::spirit::karma::right_align_type right_align;
@@ -85,23 +74,34 @@ std::string color::to_hex_string() const
     karma::generate(sink,
                     // begin grammar
                     '#'
-                    << right_align(2,'0')[hex[_1 = red()]]
+                    << right_align(2,'0')[hex(red())]
-                    << right_align(2,'0')[hex[_1 = green()]]
+                    << right_align(2,'0')[hex(green())]
-                    << right_align(2,'0')[hex[_1 = blue()]]
+                    << right_align(2,'0')[hex(blue())]
-                    << eps(alpha() < 255) <<  right_align(2,'0')[hex [_1 = alpha()]]
+                    << eps(alpha() < 255) << right_align(2,'0')[hex(alpha())]
                     // end grammar
         );
     return str;
 }
 
+namespace  {
+
+static std::uint8_t multiply(std::uint8_t c, std::uint8_t a)
+{
+    std::uint32_t t = c * a + 128;
+    return std::uint8_t(((t >> 8) + t) >> 8);
+}
+
+}
+
 bool color::premultiply()
 {
     if (premultiplied_) return false;
-    agg::rgba8 pre_c = agg::rgba8(red_,green_,blue_,alpha_);
+    if (alpha_ != 255)
-    pre_c.premultiply();
+    {
-    red_ = pre_c.r;
+        red_ = multiply(red_, alpha_);
-    green_ = pre_c.g;
+        green_ = multiply(green_, alpha_);
-    blue_ = pre_c.b;
+        blue_ = multiply(blue_, alpha_);
+    }
     premultiplied_ = true;
     return true;
 }
@@ -109,13 +109,23 @@ bool color::premultiply()
 bool color::demultiply()
 {
     if (!premultiplied_) return false;
-    agg::rgba8 pre_c = agg::rgba8(red_,green_,blue_,alpha_);
+    if (alpha_ < 255)
-    pre_c.demultiply();
+    {
-    red_ = pre_c.r;
+        if (alpha_ == 0)
-    green_ = pre_c.g;
+        {
-    blue_ = pre_c.b;
+            red_ = green_ = blue_ = 0;
+        }
+        else
+        {
+            std::uint32_t r = (std::uint32_t(red_) * 255) / alpha_;
+            std::uint32_t g = (std::uint32_t(green_) * 255) / alpha_;
+            std::uint32_t b = (std::uint32_t(blue_) * 255) / alpha_;
+            red_ = (r > 255) ? 255 : r;
+            green_ = (g > 255) ? 255 : g;
+            blue_ = (b > 255) ? 255 : b;
+        }
+    }
     premultiplied_ = false;
     return true;
 }
-
 }

src/datasource_cache.cpp

@@ -88,7 +88,7 @@ datasource_ptr datasource_cache::create(parameters const& params)
 #ifdef MAPNIK_THREADSAFE
         std::lock_guard<std::recursive_mutex> lock(instance_mutex_);
 #endif
-        itr=plugins_.find(*type);
+        itr = plugins_.find(*type);
         if (itr == plugins_.end())
         {
             std::string s("Could not create datasource for type: '");
@@ -105,7 +105,7 @@ datasource_ptr datasource_cache::create(parameters const& params)
         }
     }
 
-    if (! itr->second->valid())
+    if (!itr->second->valid())
     {
         throw std::runtime_error(std::string("Cannot load library: ") +
                                  itr->second->get_error());

src/expression_node.cpp

@@ -37,6 +37,14 @@
 namespace mapnik
 {
 
+#if defined(BOOST_REGEX_HAS_ICU)
+static void fromUTF32toUTF8(std::basic_string<UChar32> const& src, std::string & dst)
+{
+    int32_t len = safe_cast<int32_t>(src.length());
+    value_unicode_string::fromUTF32(src.data(), len).toUTF8String(dst);
+}
+#endif
+
 struct _regex_match_impl : util::noncopyable {
 #if defined(BOOST_REGEX_HAS_ICU)
     _regex_match_impl(value_unicode_string const& ustr) :
@@ -94,10 +102,7 @@ std::string regex_match_node::to_string() const
     str_ +=".match('";
     auto const& pattern = impl_.get()->pattern_;
 #if defined(BOOST_REGEX_HAS_ICU)
-    std::string utf8;
+    fromUTF32toUTF8(pattern.str(), str_);
-    value_unicode_string ustr = value_unicode_string::fromUTF32( &pattern.str()[0], safe_cast<int>(pattern.str().length()));
-    to_utf8(ustr,utf8);
-    str_ += utf8;
 #else
     str_ += pattern.str();
 #endif
@@ -141,13 +146,9 @@ std::string regex_replace_node::to_string() const
     auto const& pattern = impl_.get()->pattern_;
     auto const& format = impl_.get()->format_;
 #if defined(BOOST_REGEX_HAS_ICU)
-    std::string utf8;
+    fromUTF32toUTF8(pattern.str(), str_);
-    value_unicode_string ustr = value_unicode_string::fromUTF32( &pattern.str()[0], safe_cast<int>(pattern.str().length()));
-    to_utf8(ustr,utf8);
-    str_ += utf8;
     str_ +="','";
-    to_utf8(format ,utf8);
+    format.toUTF8String(str_);
-    str_ += utf8;
 #else
     str_ += pattern.str();
     str_ +="','";

src/font_engine_freetype.cpp

@@ -27,7 +27,6 @@
 #include <mapnik/text/face.hpp>
 #include <mapnik/util/fs.hpp>
 #include <mapnik/util/file_io.hpp>
-#include <mapnik/util/singleton.hpp>
 #include <mapnik/make_unique.hpp>
 
 #pragma GCC diagnostic push

src/mapped_memory_cache.cpp

@@ -37,6 +37,8 @@
 namespace mapnik
 {
 
+template class singleton<mapped_memory_cache, CreateStatic>;
+
 void mapped_memory_cache::clear()
 {
 #ifdef MAPNIK_THREADSAFE
@@ -58,6 +60,7 @@ boost::optional<mapped_region_ptr> mapped_memory_cache::find(std::string const&
 #ifdef MAPNIK_THREADSAFE
     std::lock_guard<std::mutex> lock(mutex_);
 #endif
+
     using iterator_type = std::unordered_map<std::string, mapped_region_ptr>::const_iterator;
     boost::optional<mapped_region_ptr> result;
     iterator_type itr = cache_.find(uri);
@@ -76,7 +79,7 @@ boost::optional<mapped_region_ptr> mapped_memory_cache::find(std::string const&
             result.reset(region);
             if (update_cache)
             {
-                cache_.emplace(uri,*result);
+                cache_.emplace(uri, *result);
             }
             return result;
         }

test/data

@@ -1 +1 @@
-Subproject commit a49ef259427514faa2cc21242ee840c0caa1e290
+Subproject commit 2a8261be8cca79a4b6fd62e8f4a93b2808613fef

test/unit/core/expressions_test.cpp

@@ -89,6 +89,8 @@ TEST_CASE("expressions")
     // unicode
     TRY_CHECK(parse_and_dump("'single-quoted string'") == "'single-quoted string'");
     TRY_CHECK(parse_and_dump("\"double-quoted string\"") == "'double-quoted string'");
+    TRY_CHECK(parse_and_dump("'escaped \\' apostrophe'") == "'escaped \\' apostrophe'");
+    TRY_CHECK(parse_and_dump("'escaped \\\\ backslash'") == "'escaped \\\\ backslash'");
 
     // floating point constants
     TRY_CHECK(parse_and_dump("pi") == "3.14159");
@@ -159,8 +161,13 @@ TEST_CASE("expressions")
     // regex
     // replace
     TRY_CHECK(eval(" [foo].replace('(\\B)|( )','$1 ') ") == tr.transcode("b a r"));
+    // 'foo' =~ s:(\w)\1:$1x:r
+    TRY_CHECK(eval(" 'foo'.replace('(\\w)\\1', '$1x') ") == tr.transcode("fox"));
+    TRY_CHECK(parse_and_dump(" 'foo'.replace('(\\w)\\1', '$1x') ") == "'foo'.replace('(\\w)\\1','$1x')");
 
     // match
     TRY_CHECK(eval(" [name].match('Québec') ") == true);
-
+    // 'Québec' =~ m:^Q\S*$:
+    TRY_CHECK(eval(" [name].match('^Q\\S*$') ") == true);
+    TRY_CHECK(parse_and_dump(" [name].match('^Q\\S*$') ") == "[name].match('^Q\\S*$')");
 }

test/unit/datasource/csv.cpp

@@ -987,7 +987,7 @@ TEST_CASE("csv") {
             using ustring = mapnik::value_unicode_string;
             using row = std::pair<std::string, std::size_t>;
 
-            for (auto const &r : {
+            for (auto const& r : {
                     row{"test/data/csv/fails/needs_headers_two_lines.csv", 2},
                         row{"test/data/csv/fails/needs_headers_one_line.csv", 1},
                             row{"test/data/csv/fails/needs_headers_one_line_no_newline.csv", 1}})

test/unit/datasource/shapeindex.cpp

@@ -24,8 +24,10 @@
 
 #include <mapnik/datasource.hpp>
 #include <mapnik/datasource_cache.hpp>
+#include <mapnik/mapped_memory_cache.hpp>
 #include <mapnik/util/fs.hpp>
 #include <cstdlib>
+#include <fstream>
 #pragma GCC diagnostic push
 #include <mapnik/warning_ignore.hpp>
 #include <boost/algorithm/string.hpp>
@@ -35,6 +37,9 @@ namespace {
 
 std::size_t count_shapefile_features(std::string const& filename)
 {
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
+    mapnik::mapped_memory_cache::instance().clear();
+#endif
     mapnik::parameters params;
     params["type"] = "shape";
     params["file"] = filename;
@@ -57,6 +62,7 @@ std::size_t count_shapefile_features(std::string const& filename)
         ++feature_count;
         feature = features->next();
     }
+
     return feature_count;
 }
 
@@ -84,12 +90,59 @@ int create_shapefile_index(std::string const& filename, bool index_parts, bool s
 
 }
 
+TEST_CASE("invalid shapeindex")
+{
+    std::string shape_plugin("./plugins/input/shape.input");
+    if (mapnik::util::exists(shape_plugin))
+    {
+        SECTION("Invalid index")
+        {
+            for (auto val : {std::make_tuple(true, std::string("mapnik-invalid-index.................")), // invalid header
+                           std::make_tuple(false, std::string("mapnik-index................."))})       // valid header + invalid index
+            {
+                std::string path = "test/data/shp/boundaries.shp";
+                std::string index_path = path.substr(0, path.rfind(".")) + ".index";
+                // remove *.index if present
+                if (mapnik::util::exists(index_path))
+                {
+                    mapnik::util::remove(index_path);
+                }
+                // count features
+
+                std::size_t feature_count = count_shapefile_features(path);
+
+                // create index
+                std::ofstream index(index_path.c_str(), std::ios::binary);
+                index.write(std::get<1>(val).c_str(), std::get<1>(val).size());
+                index.close();
+
+                // count features
+                std::size_t feature_count_indexed = count_shapefile_features(path);
+                if (std::get<0>(val)) // fallback to un-indexed access
+                {
+                    // ensure number of features are the same
+                    CHECK(feature_count == feature_count_indexed);
+                }
+                else // the header is valid but index file itself is not - expect datasource to fail and return 0 features.
+                {
+                    CHECK(feature_count_indexed == 0);
+                }
+                // remove *.index if present
+                if (mapnik::util::exists(index_path))
+                {
+                    mapnik::util::remove(index_path);
+                }
+            }
+        }
+    }
+}
+
 TEST_CASE("shapeindex")
 {
     std::string shape_plugin("./plugins/input/shape.input");
     if (mapnik::util::exists(shape_plugin))
     {
-        SECTION("Shapefile index")
+        SECTION("Index")
         {
             for (auto const& path : mapnik::util::list_directory("test/data/shp/"))
             {

test/unit/numerics/enumeration.cpp

@@ -1,36 +1,16 @@
 #include "catch.hpp"
 #include <mapnik/enumeration.hpp>
+#include <mapnik/symbolizer_enumerations.hpp>
 #include <sstream>
 
-namespace mapnik {
-
-    enum _test_enumeration_enum : std::uint8_t
-    {
-        TEST_ONE,
-        TEST_TWO,
-        _test_enumeration_enum_MAX
-    };
-
-    DEFINE_ENUM( _test_enumeration_e, _test_enumeration_enum );
-
-    static const char * _test_enumeration_strings[] = {
-        "test_one",
-        "test_two",
-        ""
-    };
-
-    IMPLEMENT_ENUM( _test_enumeration_e, _test_enumeration_strings )
-
-}
-
 TEST_CASE("enumeration") {
 
-        mapnik::_test_enumeration_e e(mapnik::TEST_ONE);
+        mapnik::line_cap_e e(mapnik::ROUND_CAP);
-        CHECK( e.as_string() == "test_one" );
+        CHECK( e.as_string() == "round" );
-        // test the << operator, which calls `as_string` internally
+        // note: test the << operator, which calls `as_string` internally
-        // this is not used in mapnik, but kept for back compat
+        // is not used in mapnik, but kept for back compat
         std::stringstream s;
         s << e;
-        CHECK( s.str() == "test_one" );
+        CHECK( s.str() == "round" );
 
 }

utils/mapnik-index/process_csv_file.cpp

@@ -77,16 +77,17 @@ std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& fil
     char newline;
     bool has_newline;
     char detected_quote;
-    std::tie(newline, has_newline, detected_quote) = ::detail::autodect_newline_and_quote(csv_file, file_length);
+    char detected_separator;
+    std::tie(newline, has_newline, detected_separator, detected_quote) = ::detail::autodect_csv_flavour(csv_file, file_length);
     if (quote == 0) quote = detected_quote;
+    if (separator == 0) separator = detected_separator;
     // set back to start
     csv_file.seekg(0, std::ios::beg);
-    // get first line
     std::string csv_line;
     csv_utils::getline_csv(csv_file, csv_line, newline, quote);
-    if (separator == 0) separator = ::detail::detect_separator(csv_line);
     csv_file.seekg(0, std::ios::beg);
     int line_number = 0;
+
     ::detail::geometry_column_locator locator;
     std::vector<std::string> headers;
     std::clog << "Parsing CSV using SEPARATOR=" << separator << " QUOTE=" << quote << std::endl;

utils/shapeindex/shapeindex.cpp

@@ -255,8 +255,7 @@ int main (int argc,char** argv)
         if (count > 0)
         {
             std::clog << " number shapes=" << count << std::endl;
-            std::fstream file((shapename+".index").c_str(),
+            std::ofstream file((shapename+".index").c_str(), std::ios::trunc | std::ios::binary);
-                              std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary);
             if (!file)
             {
                 std::clog << "cannot open index file for writing file \""