From 8689048ba6882448c355f85c63e99a99b9e81964 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Sun, 10 Sep 2017 08:19:04 -0700 Subject: [PATCH] port #3759 to master --- .travis.yml | 2 + SConstruct | 153 +++++++++++++++++- bootstrap.sh | 3 - configure | 13 ++ deps/mapbox/protozero | 2 +- plugins/input/geobuf/geobuf.hpp | 23 ++- utils/mapnik-config/build.py | 18 +-- utils/mapnik-config/mapnik-config.template.sh | 12 +- 8 files changed, 203 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 174406b30..32718dd4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,8 @@ cache: directories: - $HOME/.ccache +dist: precise + matrix: include: - os: linux diff --git a/SConstruct b/SConstruct index db090ba97..cb1152815 100644 --- a/SConstruct +++ b/SConstruct @@ -99,7 +99,10 @@ pretty_dep_names = { 'osm':'more info: https://github.com/mapnik/mapnik/wiki/OsmPlugin', 'boost_regex_icu':'libboost_regex built with optional ICU unicode support is needed for unicode regex support in mapnik.', 'sqlite_rtree':'The SQLite plugin requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)', - 'pgsql2sqlite_rtree':'The pgsql2sqlite program requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)' + 'pgsql2sqlite_rtree':'The pgsql2sqlite program requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)', + 'PROJ_LIB':'The directory where proj4 stores its data files. Must exist for proj4 to work correctly', + 'GDAL_DATA':'The directory where GDAL stores its data files. Must exist for GDAL to work correctly', + 'ICU_DATA':'The directory where icu stores its data files. If ICU reports a path, it must exist. ICU can also be built without .dat files and in that case this path is empty' } # Core plugin build configuration @@ -473,7 +476,10 @@ pickle_store = [# Scons internal variables 'SQLITE_LINKFLAGS', 'BOOST_LIB_VERSION_FROM_HEADER', 'BIGINT', - 'HOST' + 'HOST', + 'QUERIED_GDAL_DATA', + 'QUERIED_ICU_DATA', + 'QUERIED_PROJ_LIB' ] # Add all other user configurable options to pickle pickle_store @@ -800,6 +806,117 @@ int main() context.Result(ret) return ret +def CheckIcuData(context, silent=False): + + if not silent: + context.Message('Checking for ICU data directory...') + ret = context.TryRun(""" + +#include +#include + +int main() { + std::string result = u_getDataDirectory(); + std::cout << result; + if (result.empty()) { + return -1; + } + return 0; +} + +""", '.cpp') + if silent: + context.did_show_result=1 + if ret[0]: + context.Result('u_getDataDirectory returned %s' % ret[1]) + else: + context.Result('Failed to detect (mapnik-config will have null value)') + return ret[1].strip() + +def CheckGdalData(context, silent=False): + + if not silent: + context.Message('Checking for GDAL data directory...') + ret = context.TryRun(""" + +#include "cpl_config.h" +#include + +int main() { + std::cout << GDAL_PREFIX << "/share/gdal" << std::endl; + return 0; +} + +""", '.cpp') + if silent: + context.did_show_result=1 + if ret[0]: + context.Result('GDAL_PREFIX returned %s' % ret[1]) + else: + context.Result('Failed to detect (mapnik-config will have null value)') + return ret[1].strip() + +def CheckProjData(context, silent=False): + + if not silent: + context.Message('Checking for PROJ_LIB directory...') + ret = context.TryRun(""" + +// This is narly, could eventually be replaced using https://github.com/OSGeo/proj.4/pull/551] +#include +#include + +static void my_proj4_logger(void * user_data, int /*level*/, const char * msg) +{ + std::string* posMsg = static_cast(user_data); + *posMsg += msg; +} + +// https://github.com/OSGeo/gdal/blob/ddbf6d39aa4b005a77ca4f27c2d61a3214f336f8/gdal/alg/gdalapplyverticalshiftgrid.cpp#L616-L633 + +std::string find_proj_path(const char * pszFilename) { + std::string osMsg; + std::string osFilename; + projCtx ctx = pj_ctx_alloc(); + pj_ctx_set_app_data(ctx, &osMsg); + pj_ctx_set_debug(ctx, PJ_LOG_DEBUG_MAJOR); + pj_ctx_set_logger(ctx, my_proj4_logger); + PAFile f = pj_open_lib(ctx, pszFilename, "rb"); + if( f ) + { + pj_ctx_fclose(ctx, f); + } + size_t nPos = osMsg.find("fopen("); + if( nPos != std::string::npos ) + { + osFilename = osMsg.substr(nPos + strlen("fopen(")); + nPos = osFilename.find(")"); + if( nPos != std::string::npos ) + osFilename = osFilename.substr(0, nPos); + } + pj_ctx_free(ctx); + return osFilename; +} + + +int main() { + std::string result = find_proj_path(" "); + std::cout << result; + if (result.empty()) { + return -1; + } + return 0; +} + +""", '.cpp') + if silent: + context.did_show_result=1 + if ret[0]: + context.Result('pj_open_lib returned %s' % ret[1]) + else: + context.Result('Failed to detect (mapnik-config will have null value)') + return ret[1].strip() + def CheckCairoHasFreetype(context, silent=False): if not silent: context.Message('Checking for cairo freetype font support ... ') @@ -1068,6 +1185,9 @@ conf_tests = { 'prioritize_paths' : prioritize_paths, 'CheckPKGVersion' : CheckPKGVersion, 'FindBoost' : FindBoost, 'CheckBoost' : CheckBoost, + 'CheckIcuData' : CheckIcuData, + 'CheckProjData' : CheckProjData, + 'CheckGdalData' : CheckGdalData, 'CheckCairoHasFreetype' : CheckCairoHasFreetype, 'CheckHasDlfcn' : CheckHasDlfcn, 'GetBoostLibVersion' : GetBoostLibVersion, @@ -1164,6 +1284,10 @@ if not preconfigured: env['PLUGINS'] = PLUGINS env['EXTRA_FREETYPE_LIBS'] = [] env['SQLITE_LINKFLAGS'] = [] + env['QUERIED_PROJ_LIB'] = None + env['QUERIED_ICU_DATA'] = None + env['QUERIED_GDAL_DATA'] = None + # previously a leading / was expected for LIB_DIR_NAME # now strip it to ensure expected behavior if env['LIB_DIR_NAME'].startswith(os.path.sep): @@ -1481,6 +1605,31 @@ if not preconfigured: if env['HOST']: SQLITE_HAS_RTREE = True + if not env['HOST']: + env['QUERIED_PROJ_LIB'] = conf.CheckProjData() + if os.environ.get('PROJ_LIB'): + env['QUERIED_PROJ_LIB'] = os.environ['PROJ_LIB'] + color_print(4,'Detected PROJ_LIB in environ, using env value instead: %s' % os.environ['PROJ_LIB'] ) + env['QUERIED_ICU_DATA'] = conf.CheckIcuData() + if os.environ.get('ICU_DATA'): + env['QUERIED_ICU_DATA'] = os.environ['ICU_DATA'] + color_print(4,'Detected ICU_DATA in environ, using env value instead: %s' % os.environ['ICU_DATA'] ) + env['QUERIED_GDAL_DATA'] = conf.CheckGdalData() + if os.environ.get('GDAL_DATA'): + env['QUERIED_GDAL_DATA'] = os.environ['GDAL_DATA'] + color_print(4,'Detected GDAL_DATA in environ, using env value instead: %s' % os.environ['GDAL_DATA'] ) + # now validate the paths actually exist + if env['QUERIED_PROJ_LIB'] and not os.path.exists(env['QUERIED_PROJ_LIB']): + color_print(1,'%s not detected on your system' % env['QUERIED_PROJ_LIB'] ) + env['MISSING_DEPS'].append('PROJ_LIB') + if env['QUERIED_GDAL_DATA'] and not os.path.exists(env['QUERIED_GDAL_DATA']): + color_print(1,'%s not detected on your system' % env['QUERIED_GDAL_DATA'] ) + env['MISSING_DEPS'].append('GDAL_DATA') + if env['QUERIED_ICU_DATA'] and not os.path.exists(env['QUERIED_ICU_DATA']): + color_print(1,'%s not detected on your system' % env['QUERIED_ICU_DATA'] ) + env['MISSING_DEPS'].append('ICU_DATA') + + CHECK_PKG_CONFIG = conf.CheckPKGConfig('0.15.0') if len(env['REQUESTED_PLUGINS']): diff --git a/bootstrap.sh b/bootstrap.sh index 030c0dc0b..c5a4dc50c 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -59,9 +59,6 @@ function install_mason_deps() { install proj 4.9.3 libproj install pixman 0.34.0 libpixman-1 install cairo 1.14.8 libcairo - install protobuf 3.2.0 - # technically protobuf is not a mapnik core dep, but installing - # here by default helps make mapnik-vector-tile builds easier install webp 0.6.0 libwebp install libgdal 2.1.3 libgdal install boost 1.63.0 diff --git a/configure b/configure index b468f8587..c4c33400f 100755 --- a/configure +++ b/configure @@ -1,5 +1,18 @@ #!/bin/sh +set -eu + PYTHON=${PYTHON:-python} +# mapnik-settings.env is an optional file to store +# environment variables that should be used before +# running tests like PROJ_LIB, GDAL_DATA, and ICU_DATA +# These do not normally need to be set except when +# building against binary versions of dependencies like +# done via bootstrap.sh +if [[ -f mapnik-settings.env ]]; then + echo "Inheriting from mapnik-settings.env" + source mapnik-settings.env +fi + $PYTHON scons/scons.py --implicit-deps-changed configure "$@" diff --git a/deps/mapbox/protozero b/deps/mapbox/protozero index a1eb7327e..a0e9109cc 160000 --- a/deps/mapbox/protozero +++ b/deps/mapbox/protozero @@ -1 +1 @@ -Subproject commit a1eb7327e853161e939a409a8d74a8302dd1ecc5 +Subproject commit a0e9109cca7a8d28711a82c8264e93ab86374451 diff --git a/plugins/input/geobuf/geobuf.hpp b/plugins/input/geobuf/geobuf.hpp index 55fa25532..cf5b5bbb5 100644 --- a/plugins/input/geobuf/geobuf.hpp +++ b/plugins/input/geobuf/geobuf.hpp @@ -83,7 +83,7 @@ struct geobuf : util::noncopyable double precision = std::pow(10,6); bool is_topo = false; bool transformed = false; - std::size_t lengths = 0; + //std::size_t lengths = 0; std::vector keys_; std::vector values_; protozero::pbf_reader reader_; @@ -429,6 +429,11 @@ private: template geometry::multi_line_string read_multi_linestring(T & reader, boost::optional> const& lengths) { + if (!lengths) { + std::clog << "rml not set!\n"; + } else { + std::clog << "rml lengths: " << lengths->size() << "\n"; + } geometry::multi_line_string multi_line; multi_line.reserve(!lengths ? 1 : lengths->size()); auto pi = reader.get_packed_sint64(); @@ -455,6 +460,11 @@ private: geometry::polygon read_polygon(T & reader, boost::optional> const& lengths) { geometry::polygon poly; + if (!lengths) { + std::clog << "rp not set!\n"; + } else { + std::clog << "rp lengths: " << lengths->size() << "\n"; + } poly.reserve(!lengths ? 1 : lengths->size()); auto pi = reader.get_packed_sint64(); if (!lengths) @@ -480,6 +490,11 @@ private: geometry::multi_polygon read_multi_polygon(T & reader, boost::optional> const& lengths) { geometry::multi_polygon multi_poly; + if (!lengths) { + std::clog << "rmp not set!\n"; + } else { + std::clog << "rmp lengths: " << lengths->size() << "\n"; + } if (!lengths) { auto poly = read_polygon(reader, lengths); @@ -524,8 +539,12 @@ private: } case 2: { + std::clog << "here\n"; auto val = read_lengths(reader); - if (!val.empty()) lengths = std::move(val); + if (!val.empty()) { + std::clog << "lenghts " << val.size() << "\n"; + lengths = std::move(val); + } break; } case 3: diff --git a/utils/mapnik-config/build.py b/utils/mapnik-config/build.py index 9ffd73132..cc34908bc 100644 --- a/utils/mapnik-config/build.py +++ b/utils/mapnik-config/build.py @@ -67,9 +67,9 @@ CONFIG_MAPNIK_INCLUDE="${CONFIG_PREFIX}/include -I${CONFIG_PREFIX}/include/mapni CONFIG_DEP_INCLUDES="%(dep_includes)s" CONFIG_CXXFLAGS="%(cxxflags)s" CONFIG_CXX='%(cxx)s' -CONFIG_MAPNIK_GDAL_DATA='%(mapnik_bundled_gdal_data)s' -CONFIG_MAPNIK_PROJ_LIB='%(mapnik_bundled_proj_data)s' -CONFIG_MAPNIK_ICU_DATA='%(mapnik_bundled_icu_data)s' +CONFIG_MAPNIK_GDAL_DATA='%(found_gdal_data)s' +CONFIG_MAPNIK_PROJ_LIB='%(found_proj_data)s' +CONFIG_MAPNIK_ICU_DATA='%(found_icu_data)s' ''' @@ -135,9 +135,9 @@ if lib_root in inputpluginspath: lib_path = "${CONFIG_PREFIX}/" + config_env['LIBDIR_SCHEMA'] -mapnik_bundled_gdal_data = '' -mapnik_bundled_proj_data = '' -mapnik_bundled_icu_data = '' +found_gdal_data = config_env['QUERIED_GDAL_DATA'] +found_proj_data = config_env['QUERIED_PROJ_LIB'] +found_icu_data = config_env['QUERIED_ICU_DATA'] configuration = { "git_revision": git_revision, @@ -154,9 +154,9 @@ configuration = { "defines":defines, "cxxflags":cxxflags, "cxx":env['CXX'], - "mapnik_bundled_gdal_data":mapnik_bundled_gdal_data, - "mapnik_bundled_proj_data":mapnik_bundled_proj_data, - "mapnik_bundled_icu_data":mapnik_bundled_icu_data, + "found_gdal_data":found_gdal_data, + "found_proj_data":found_proj_data, + "found_icu_data":found_icu_data, } ## if we are statically linking dependencies diff --git a/utils/mapnik-config/mapnik-config.template.sh b/utils/mapnik-config/mapnik-config.template.sh index 27dc32cdf..a1ef44b60 100755 --- a/utils/mapnik-config/mapnik-config.template.sh +++ b/utils/mapnik-config/mapnik-config.template.sh @@ -27,9 +27,9 @@ Known values for OPTION are: --cflags all include paths, compiler flags, and pre-processor defines (for back-compatibility) --cxx c++ compiler used to build mapnik (new in 2.2.0) --all-flags all compile and link flags (new in 2.2.0) - --gdal-data path to GDAL_DATA directory, if known (relevant only for packaged builds of Mapnik) (new in 3.0.0) - --proj-lib path to PROJ_LIB directory, if known (relevant only for packaged builds of Mapnik) (new in 3.0.0) - --icu-data path to ICU_DATA directory, if known (relevant only for packaged builds of Mapnik) (new in 3.0.0) + --gdal-data path to GDAL_DATA directory, if detected at build time (new in 3.0.16) + --proj-lib path to PROJ_LIB directory, if detected at build time (new in 3.0.16) + --icu-data path to ICU_DATA directory, if detected at build time (new in 3.0.16) EOF exit $1 @@ -132,15 +132,15 @@ while test $# -gt 0; do ;; --gdal-data) - if [[ ${CONFIG_MAPNIK_GDAL_DATA:-unset} != "unset" ]]; then echo ${CONFIG_PREFIX}/${CONFIG_MAPNIK_GDAL_DATA}; fi; + if [[ ${CONFIG_MAPNIK_GDAL_DATA:-unset} != "unset" ]]; then echo ${CONFIG_MAPNIK_GDAL_DATA}; fi; ;; --proj-lib) - if [[ ${CONFIG_MAPNIK_PROJ_LIB:-unset} != "unset" ]]; then echo ${CONFIG_PREFIX}/${CONFIG_MAPNIK_PROJ_LIB}; fi; + if [[ ${CONFIG_MAPNIK_PROJ_LIB:-unset} != "unset" ]]; then echo ${CONFIG_MAPNIK_PROJ_LIB}; fi; ;; --icu-data) - if [[ ${CONFIG_MAPNIK_ICU_DATA:-unset} != "unset" ]]; then echo ${CONFIG_PREFIX}/${CONFIG_MAPNIK_ICU_DATA}; fi; + if [[ ${CONFIG_MAPNIK_ICU_DATA:-unset} != "unset" ]]; then echo ${CONFIG_MAPNIK_ICU_DATA}; fi; ;; *)