port #3759 to master

.travis.yml

@@ -18,6 +18,8 @@ cache:
   directories:
   - $HOME/.ccache
 
+dist: precise
+
 matrix:
   include:
     - os: linux

SConstruct

@@ -99,7 +99,10 @@ pretty_dep_names = {
     'osm':'more info: https://github.com/mapnik/mapnik/wiki/OsmPlugin',
     'boost_regex_icu':'libboost_regex built with optional ICU unicode support is needed for unicode regex support in mapnik.',
     'sqlite_rtree':'The SQLite plugin requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)',
-    'pgsql2sqlite_rtree':'The pgsql2sqlite program requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)'
+    'pgsql2sqlite_rtree':'The pgsql2sqlite program requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)',
+    'PROJ_LIB':'The directory where proj4 stores its data files. Must exist for proj4 to work correctly',
+    'GDAL_DATA':'The directory where GDAL stores its data files. Must exist for GDAL to work correctly',
+    'ICU_DATA':'The directory where icu stores its data files. If ICU reports a path, it must exist. ICU can also be built without .dat files and in that case this path is empty'
     }
 
 # Core plugin build configuration
@@ -473,7 +476,10 @@ pickle_store = [# Scons internal variables
         'SQLITE_LINKFLAGS',
         'BOOST_LIB_VERSION_FROM_HEADER',
         'BIGINT',
-        'HOST'
+        'HOST',
+        'QUERIED_GDAL_DATA',
+        'QUERIED_ICU_DATA',
+        'QUERIED_PROJ_LIB'
         ]
 
 # Add all other user configurable options to pickle pickle_store
@@ -800,6 +806,117 @@ int main()
     context.Result(ret)
     return ret
 
+def CheckIcuData(context, silent=False):
+ 
+    if not silent:
+        context.Message('Checking for ICU data directory...')
+    ret = context.TryRun("""
+
+#include <unicode/putil.h>
+#include <iostream>
+
+int main() {
+    std::string result = u_getDataDirectory();
+    std::cout << result;
+    if (result.empty()) {
+        return -1;
+    }
+    return 0;
+}
+
+""", '.cpp')
+    if silent:
+        context.did_show_result=1
+    if ret[0]:
+        context.Result('u_getDataDirectory returned %s' % ret[1])
+    else:
+        context.Result('Failed to detect (mapnik-config will have null value)')
+    return ret[1].strip()
+
+def CheckGdalData(context, silent=False):
+ 
+    if not silent:
+        context.Message('Checking for GDAL data directory...')
+    ret = context.TryRun("""
+
+#include "cpl_config.h"
+#include <iostream>
+
+int main() {
+    std::cout << GDAL_PREFIX << "/share/gdal" << std::endl;
+    return 0;
+}
+
+""", '.cpp')
+    if silent:
+        context.did_show_result=1
+    if ret[0]:
+        context.Result('GDAL_PREFIX returned %s' % ret[1])
+    else:
+        context.Result('Failed to detect (mapnik-config will have null value)')
+    return ret[1].strip()
+
+def CheckProjData(context, silent=False):
+ 
+    if not silent:
+        context.Message('Checking for PROJ_LIB directory...')
+    ret = context.TryRun("""
+
+// This is narly, could eventually be replaced using https://github.com/OSGeo/proj.4/pull/551]
+#include <proj_api.h>
+#include <iostream>
+
+static void my_proj4_logger(void * user_data, int /*level*/, const char * msg)
+{
+    std::string* posMsg = static_cast<std::string*>(user_data);
+    *posMsg += msg;
+}
+
+// https://github.com/OSGeo/gdal/blob/ddbf6d39aa4b005a77ca4f27c2d61a3214f336f8/gdal/alg/gdalapplyverticalshiftgrid.cpp#L616-L633
+
+std::string find_proj_path(const char * pszFilename) {
+    std::string osMsg;
+    std::string osFilename;
+    projCtx ctx = pj_ctx_alloc();
+    pj_ctx_set_app_data(ctx, &osMsg);
+    pj_ctx_set_debug(ctx, PJ_LOG_DEBUG_MAJOR);
+    pj_ctx_set_logger(ctx, my_proj4_logger);
+    PAFile f = pj_open_lib(ctx, pszFilename, "rb");
+    if( f )
+    {
+        pj_ctx_fclose(ctx, f);
+    }
+    size_t nPos = osMsg.find("fopen(");
+    if( nPos != std::string::npos )
+    {
+        osFilename = osMsg.substr(nPos + strlen("fopen("));
+        nPos = osFilename.find(")");
+        if( nPos != std::string::npos )
+            osFilename = osFilename.substr(0, nPos);
+    }
+    pj_ctx_free(ctx);
+    return osFilename;
+}
+
+
+int main() {
+    std::string result = find_proj_path(" ");
+    std::cout << result;
+    if (result.empty()) {
+        return -1;
+    }
+    return 0;
+}
+
+""", '.cpp')
+    if silent:
+        context.did_show_result=1
+    if ret[0]:
+        context.Result('pj_open_lib returned %s' % ret[1])
+    else:
+        context.Result('Failed to detect (mapnik-config will have null value)')
+    return ret[1].strip()
+
 def CheckCairoHasFreetype(context, silent=False):
     if not silent:
         context.Message('Checking for cairo freetype font support ... ')
@@ -1068,6 +1185,9 @@ conf_tests = { 'prioritize_paths'      : prioritize_paths,
                'CheckPKGVersion'       : CheckPKGVersion,
                'FindBoost'             : FindBoost,
                'CheckBoost'            : CheckBoost,
+               'CheckIcuData'          : CheckIcuData,
+               'CheckProjData'         : CheckProjData,
+               'CheckGdalData'         : CheckGdalData,
                'CheckCairoHasFreetype' : CheckCairoHasFreetype,
                'CheckHasDlfcn'         : CheckHasDlfcn,
                'GetBoostLibVersion'    : GetBoostLibVersion,
@@ -1164,6 +1284,10 @@ if not preconfigured:
     env['PLUGINS'] = PLUGINS
     env['EXTRA_FREETYPE_LIBS'] = []
     env['SQLITE_LINKFLAGS'] = []
+    env['QUERIED_PROJ_LIB'] = None
+    env['QUERIED_ICU_DATA'] = None
+    env['QUERIED_GDAL_DATA'] = None
+
     # previously a leading / was expected for LIB_DIR_NAME
     # now strip it to ensure expected behavior
     if env['LIB_DIR_NAME'].startswith(os.path.sep):
@@ -1481,6 +1605,31 @@ if not preconfigured:
     if env['HOST']:
         SQLITE_HAS_RTREE = True
 
+    if not env['HOST']:
+        env['QUERIED_PROJ_LIB'] = conf.CheckProjData()
+        if os.environ.get('PROJ_LIB'):
+            env['QUERIED_PROJ_LIB'] = os.environ['PROJ_LIB']
+            color_print(4,'Detected PROJ_LIB in environ, using env value instead: %s' % os.environ['PROJ_LIB'] )
+        env['QUERIED_ICU_DATA'] = conf.CheckIcuData()
+        if os.environ.get('ICU_DATA'):
+            env['QUERIED_ICU_DATA'] = os.environ['ICU_DATA']
+            color_print(4,'Detected ICU_DATA in environ, using env value instead: %s' % os.environ['ICU_DATA'] )
+        env['QUERIED_GDAL_DATA'] = conf.CheckGdalData()
+        if os.environ.get('GDAL_DATA'):
+            env['QUERIED_GDAL_DATA'] = os.environ['GDAL_DATA']
+            color_print(4,'Detected GDAL_DATA in environ, using env value instead: %s' % os.environ['GDAL_DATA'] )
+        # now validate the paths actually exist
+        if env['QUERIED_PROJ_LIB'] and not os.path.exists(env['QUERIED_PROJ_LIB']):
+            color_print(1,'%s not detected on your system' % env['QUERIED_PROJ_LIB'] )
+            env['MISSING_DEPS'].append('PROJ_LIB')
+        if env['QUERIED_GDAL_DATA'] and not os.path.exists(env['QUERIED_GDAL_DATA']):
+            color_print(1,'%s not detected on your system' % env['QUERIED_GDAL_DATA'] )
+            env['MISSING_DEPS'].append('GDAL_DATA')
+        if env['QUERIED_ICU_DATA'] and not os.path.exists(env['QUERIED_ICU_DATA']):
+            color_print(1,'%s not detected on your system' % env['QUERIED_ICU_DATA'] )
+            env['MISSING_DEPS'].append('ICU_DATA')
+
+
     CHECK_PKG_CONFIG = conf.CheckPKGConfig('0.15.0')
 
     if len(env['REQUESTED_PLUGINS']):

bootstrap.sh

@@ -59,9 +59,6 @@ function install_mason_deps() {
     install proj 4.9.3 libproj
     install pixman 0.34.0 libpixman-1
     install cairo 1.14.8 libcairo
-    install protobuf 3.2.0
-    # technically protobuf is not a mapnik core dep, but installing
-    # here by default helps make mapnik-vector-tile builds easier
     install webp 0.6.0 libwebp
     install libgdal 2.1.3 libgdal
     install boost 1.63.0

configure

@@ -1,5 +1,18 @@
 #!/bin/sh
 
+set -eu
+
 PYTHON=${PYTHON:-python}
 
+# mapnik-settings.env is an optional file to store
+# environment variables that should be used before
+# running tests like PROJ_LIB, GDAL_DATA, and ICU_DATA
+# These do not normally need to be set except when
+# building against binary versions of dependencies like
+# done via bootstrap.sh
+if [[ -f mapnik-settings.env ]]; then
+    echo "Inheriting from mapnik-settings.env"
+    source mapnik-settings.env
+fi
+
 $PYTHON scons/scons.py --implicit-deps-changed configure "$@"

deps/mapbox/protozero

@@ -1 +1 @@
-Subproject commit a1eb7327e853161e939a409a8d74a8302dd1ecc5
+Subproject commit a0e9109cca7a8d28711a82c8264e93ab86374451

plugins/input/geobuf/geobuf.hpp

@@ -83,7 +83,7 @@ struct geobuf : util::noncopyable
     double precision = std::pow(10,6);
     bool is_topo = false;
     bool transformed = false;
-    std::size_t lengths = 0;
+    //std::size_t lengths = 0;
     std::vector<std::string> keys_;
     std::vector<value_type> values_;
     protozero::pbf_reader reader_;
@@ -429,6 +429,11 @@ private:
     template <typename T>
     geometry::multi_line_string<double> read_multi_linestring(T & reader, boost::optional<std::vector<std::uint32_t>> const& lengths)
     {
+        if (!lengths) {
+            std::clog << "rml not set!\n";
+        } else {
+            std::clog << "rml lengths: " << lengths->size() << "\n";
+        }
         geometry::multi_line_string<double> multi_line;
         multi_line.reserve(!lengths ? 1 : lengths->size());
         auto pi = reader.get_packed_sint64();
@@ -455,6 +460,11 @@ private:
     geometry::polygon<double> read_polygon(T & reader, boost::optional<std::vector<std::uint32_t>> const& lengths)
     {
         geometry::polygon<double> poly;
+        if (!lengths) {
+            std::clog << "rp not set!\n";
+        } else {
+            std::clog << "rp lengths: " << lengths->size() << "\n";
+        }
         poly.reserve(!lengths ? 1 : lengths->size());
         auto pi = reader.get_packed_sint64();
         if (!lengths)
@@ -480,6 +490,11 @@ private:
     geometry::multi_polygon<double> read_multi_polygon(T & reader, boost::optional<std::vector<std::uint32_t>> const& lengths)
     {
         geometry::multi_polygon<double> multi_poly;
+        if (!lengths) {
+            std::clog << "rmp not set!\n";
+        } else {
+            std::clog << "rmp lengths: " << lengths->size() << "\n";
+        }
         if (!lengths)
         {
             auto poly = read_polygon(reader, lengths);
@@ -524,8 +539,12 @@ private:
             }
             case 2:
             {
+                std::clog << "here\n";
                 auto val = read_lengths(reader);
-                if (!val.empty()) lengths = std::move(val);
+                if (!val.empty()) {
+                    std::clog << "lenghts " << val.size() << "\n";
+                    lengths = std::move(val);
+                }
                 break;
             }
             case 3:

utils/mapnik-config/build.py

@@ -67,9 +67,9 @@ CONFIG_MAPNIK_INCLUDE="${CONFIG_PREFIX}/include -I${CONFIG_PREFIX}/include/mapni
 CONFIG_DEP_INCLUDES="%(dep_includes)s"
 CONFIG_CXXFLAGS="%(cxxflags)s"
 CONFIG_CXX='%(cxx)s'
-CONFIG_MAPNIK_GDAL_DATA='%(mapnik_bundled_gdal_data)s'
+CONFIG_MAPNIK_GDAL_DATA='%(found_gdal_data)s'
-CONFIG_MAPNIK_PROJ_LIB='%(mapnik_bundled_proj_data)s'
+CONFIG_MAPNIK_PROJ_LIB='%(found_proj_data)s'
-CONFIG_MAPNIK_ICU_DATA='%(mapnik_bundled_icu_data)s'
+CONFIG_MAPNIK_ICU_DATA='%(found_icu_data)s'
 
 '''
 
@@ -135,9 +135,9 @@ if lib_root in inputpluginspath:
 
 lib_path = "${CONFIG_PREFIX}/" + config_env['LIBDIR_SCHEMA']
 
-mapnik_bundled_gdal_data = ''
+found_gdal_data = config_env['QUERIED_GDAL_DATA']
-mapnik_bundled_proj_data = ''
+found_proj_data = config_env['QUERIED_PROJ_LIB']
-mapnik_bundled_icu_data = ''
+found_icu_data = config_env['QUERIED_ICU_DATA']
 
 configuration = {
     "git_revision": git_revision,
@@ -154,9 +154,9 @@ configuration = {
     "defines":defines,
     "cxxflags":cxxflags,
     "cxx":env['CXX'],
-    "mapnik_bundled_gdal_data":mapnik_bundled_gdal_data,
+    "found_gdal_data":found_gdal_data,
-    "mapnik_bundled_proj_data":mapnik_bundled_proj_data,
+    "found_proj_data":found_proj_data,
-    "mapnik_bundled_icu_data":mapnik_bundled_icu_data,
+    "found_icu_data":found_icu_data,
 }
 
 ## if we are statically linking dependencies

utils/mapnik-config/mapnik-config.template.sh

@@ -27,9 +27,9 @@ Known values for OPTION are:
   --cflags          all include paths, compiler flags, and pre-processor defines (for back-compatibility)
   --cxx             c++ compiler used to build mapnik (new in 2.2.0)
   --all-flags       all compile and link flags (new in 2.2.0)
-  --gdal-data       path to GDAL_DATA directory, if known (relevant only for packaged builds of Mapnik) (new in 3.0.0)
+  --gdal-data       path to GDAL_DATA directory, if detected at build time (new in 3.0.16)
-  --proj-lib        path to PROJ_LIB directory, if known (relevant only for packaged builds of Mapnik) (new in 3.0.0)
+  --proj-lib        path to PROJ_LIB directory, if detected at build time (new in 3.0.16)
-  --icu-data        path to ICU_DATA directory, if known (relevant only for packaged builds of Mapnik) (new in 3.0.0)
+  --icu-data        path to ICU_DATA directory, if detected at build time (new in 3.0.16)
 EOF
 
     exit $1
@@ -132,15 +132,15 @@ while test $# -gt 0; do
       ;;
 
     --gdal-data)
-      if [[ ${CONFIG_MAPNIK_GDAL_DATA:-unset} != "unset" ]]; then echo ${CONFIG_PREFIX}/${CONFIG_MAPNIK_GDAL_DATA}; fi;
+      if [[ ${CONFIG_MAPNIK_GDAL_DATA:-unset} != "unset" ]]; then echo ${CONFIG_MAPNIK_GDAL_DATA}; fi;
       ;;
 
     --proj-lib)
-      if [[ ${CONFIG_MAPNIK_PROJ_LIB:-unset} != "unset" ]]; then echo ${CONFIG_PREFIX}/${CONFIG_MAPNIK_PROJ_LIB}; fi;
+      if [[ ${CONFIG_MAPNIK_PROJ_LIB:-unset} != "unset" ]]; then echo ${CONFIG_MAPNIK_PROJ_LIB}; fi;
       ;;
 
     --icu-data)
-      if [[ ${CONFIG_MAPNIK_ICU_DATA:-unset} != "unset" ]]; then echo ${CONFIG_PREFIX}/${CONFIG_MAPNIK_ICU_DATA}; fi;
+      if [[ ${CONFIG_MAPNIK_ICU_DATA:-unset} != "unset" ]]; then echo ${CONFIG_MAPNIK_ICU_DATA}; fi;
       ;;
 
     *)