Merge branch 'master' into large-geojson

.travis.yml

@@ -1,53 +1,19 @@
 language: cpp
 
-compiler:
+sudo: false
- - clang
- - gcc
 
-env:
+addons:
-  matrix:
+  postgresql: "9.3"
-   - DEBUG=False PLUGIN_LINKING=static ENABLE_LOG=True  DEFAULT_LOG_SEVERITY=debug XMLPARSER="libxml2" DEMO=False BENCHMARK=False CUSTOM_CXXFLAGS="" CUSTOM_LDFLAGS=""
-   - DEBUG=False ENABLE_LOG=False DEFAULT_LOG_SEVERITY=none  XMLPARSER="ptree"   DEMO=False BENCHMARK=True CUSTOM_CXXFLAGS="" CUSTOM_LDFLAGS=""
-
-# travis + ubuntugis with gdal and postggis leads to many potential dead-end conflicts
-# the below is thanks to https://github.com/CartoDB/Windshaft/blob/d82fe08b32fc7907bbe907ab290f8a082215ae26/.travis.yml#L1
-before_install:
-  - export PGUSER=postgres
-  - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
-  - sudo apt-get -qq purge postgis* postgresql*
-  - sudo apt-add-repository -y ppa:cartodb/postgresql-9.3
-  - sudo apt-add-repository -y ppa:cartodb/gis
-  # grab harfbuzz from ppa
-  - sudo apt-add-repository -y ppa:fontforge/fontforge
-  # we need at least g++-4.7 for c++11 features
-  - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-  # enable to test against latest boost rather that v1.48
-  #- sudo add-apt-repository -y ppa:boost-latest/ppa
-  - sudo rm -Rf /var/lib/postgresql /etc/postgresql
-  - sudo apt-get update -qq
-  - sudo apt-get install -q libharfbuzz-dev postgresql-9.3-postgis-2.1 postgresql-contrib-9.3 gdal-bin libgdal-dev
-  - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.3/main/pg_hba.conf
-  - sudo service postgresql restart
 
 install:
- # enable to test against boost ppa
+ - psql -U postgres -c 'create database template_postgis;' -U postgres
- #- sudo apt-get install -y boost1.55
+ - psql -U postgres -c 'create extension postgis;' -d template_postgis -U postgres
- - sudo apt-get install -y libboost-python1.48-dev libboost-thread1.48-dev libboost-filesystem1.48-dev libboost-regex1.48-dev libboost-program-options1.48-dev
- - sudo apt-get install -y ttf-wqy-microhei make libstdc++6 libstdc++-4.8-dev valgrind python-nose libicu-dev libproj-dev libcairo-dev python-cairo-dev libcairo-dev python-cairo-dev libpng-dev libjpeg-dev libtiff-dev libwebp-dev libz-dev libfreetype6-dev libxml2-dev libsqlite3-dev
-
-before_script:
- - psql -U postgres -c 'create database template_postgis'
- - psql -U postgres -c 'create extension postgis' -d template_postgis
- - if [[ "${CXX}" == 'g++' ]]; then export JOBS=2; sudo apt-get install gcc-4.8 g++-4.8; export CXX="$(which g++-4.8)"; export CC="$(which gcc-4.8)"; fi;
- - if [[ "${CXX}" == 'clang++' ]]; then export JOBS=4; export CXX="$(which clang++)"; export CC="$(which clang)"; fi;
 
 script:
- - rm -rf ./git
+ - source bootstrap.sh
- - ./configure CXX="${CXX}" CC="${CC}" CUSTOM_CXXFLAGS="${CUSTOM_CXXFLAGS}" CUSTOM_LDFLAGS="${CUSTOM_LDFLAGS}" XML_PARSER="${XML_PARSER}" ENABLE_LOG="${ENABLE_LOG}" DEBUG="${DEBUG}" DEMO="${DEMO}" BENCHMARK="${BENCHMARK}" CPP_TESTS=True CAIRO=True FAST=True || cat config.log
+ - ./configure
- - if [[ "${DEBUG}" == True ]]; then export JOBS=$((JOBS/2)); fi;
+ - JOBS=6 make
- - make
  - git clone --depth=1 https://github.com/mapbox/mapnik-test-data tests/data/mapnik-test-data
  - make test
- - source localize.sh && make grind
+ #- source localize.sh && make grind
- - if [[ ${BENCHMARK} != False ]]; then make bench; fi;
+ #- make bench
- - if [[ ${BENCHMARK} != False ]]; then for i in $(ls tests/visual_tests/styles/*xml); do echo $i;./benchmark/out/test_rendering --name "text rendering" --map $i --width 1024 --height 1024 --iterations 2 --threads 5;done;fi

Makefile

@@ -13,6 +13,20 @@ install:
 	$(PYTHON) scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1 install
 
 mapnik:
+	# we first build memory intensive files with -j1
+	$(PYTHON) scons/scons.py -j1 \
+		--config=cache --implicit-cache --max-drift=1 \
+		src/json/libmapnik-json.a \
+		src/wkt/libmapnik-wkt.a \
+		src/css_color_grammar.os \
+		src/expression_grammar.os \
+		src/transform_expression_grammar.os \
+		src/image_filter_types.os \
+		src/renderer_common/process_group_symbolizer.cpp \
+		src/agg/process_markers_symbolizer.cpp \
+		src/grid/process_markers_symbolizer.cpp \
+		src/cairo/process_markers_symbolizer.cpp
+	# then install the rest with -j$(JOBS)
 	$(PYTHON) scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1
 
 clean:

SConstruct

@@ -88,10 +88,11 @@ pretty_dep_names = {
     'm':'Basic math library, part of C++ stlib',
     'pkg-config':'pkg-config tool | more info: http://pkg-config.freedesktop.org',
     'pg_config':'pg_config program | try setting PG_CONFIG SCons option',
-    'xml2-config':'xml2-config program | try setting XML2_CONFIG SCons option',
+    'xml2-config':'xml2-config program | try setting XML2_CONFIG SCons option or avoid the need for xml2-config command by configuring with XML2_LIBS & XML2_INCLUDES',
-    'libxml2':'libxml2 library | try setting XML2_CONFIG SCons option to point to location of xml2-config program',
+    'libxml2':'libxml2 library | try setting XML2_CONFIG SCons option to point to location of xml2-config program or configure with XML2_LIBS & XML2_INCLUDES',
     'gdal-config':'gdal-config program | try setting GDAL_CONFIG SCons option',
-    'freetype-config':'freetype-config program | try setting FREETYPE_CONFIG SCons option',
+    'freetype-config':'freetype-config program | try setting FREETYPE_CONFIG SCons option or configure with FREETYPE_LIBS & FREETYPE_INCLUDES',
+    'freetype':'libfreetype library | try setting FREETYPE_CONFIG SCons option or configure with FREETYPE_LIBS & FREETYPE_INCLUDES',
     'osm':'more info: https://github.com/mapnik/mapnik/wiki/OsmPlugin',
     'boost_regex_icu':'libboost_regex built with optional ICU unicode support is needed for unicode regex support in mapnik.',
     'sqlite_rtree':'The SQLite plugin requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)',
@@ -352,6 +353,10 @@ opts.AddVariables(
     BoolVariable('PROJ', 'Build Mapnik with proj4 support to enable transformations between many different projections', 'True'),
     PathVariable('PROJ_INCLUDES', 'Search path for PROJ.4 include files', '/usr/include', PathVariable.PathAccept),
     PathVariable('PROJ_LIBS', 'Search path for PROJ.4 library files', '/usr/' + LIBDIR_SCHEMA_DEFAULT, PathVariable.PathAccept),
+    ('FREETYPE_INCLUDES', 'Search path for Freetype include files', ''),
+    ('FREETYPE_LIBS', 'Search path for Freetype library files', ''),
+    ('XML2_INCLUDES', 'Search path for libxml2 include files', ''),
+    ('XML2_LIBS', 'Search path for libxml2 library files', ''),
     ('PKG_CONFIG_PATH', 'Use this path to point pkg-config to .pc files instead of the PKG_CONFIG_PATH environment setting',''),
 
     # Variables affecting rendering back-ends
@@ -1194,7 +1199,21 @@ if not preconfigured:
         env.AppendUnique(CPPPATH = os.path.realpath(inc_path))
         env.AppendUnique(LIBPATH = os.path.realpath(lib_path))
 
-    if conf.parse_config('FREETYPE_CONFIG'):
+    REQUIRED_LIBSHEADERS = [
+        ['z', 'zlib.h', True,'C'],
+        [env['ICU_LIB_NAME'],'unicode/unistr.h',True,'C++'],
+        ['harfbuzz', 'harfbuzz/hb.h',True,'C++']
+    ]
+
+    if env.get('FREETYPE_LIBS') or env.get('FREETYPE_INCLUDES'):
+        REQUIRED_LIBSHEADERS.append(['freetype','ft2build.h',True,'C'])
+        if env.get('FREETYPE_INCLUDES'):
+            inc_path = env['FREETYPE_INCLUDES']
+            env.AppendUnique(CPPPATH = os.path.realpath(inc_path))
+        if env.get('FREETYPE_LIBS'):
+            lib_path = env['FREETYPE_LIBS']
+            env.AppendUnique(LIBPATH = os.path.realpath(lib_path))
+    elif conf.parse_config('FREETYPE_CONFIG'):
         # check if freetype links to bz2
         if env['RUNTIME_LINK'] == 'static':
             temp_env = env.Clone()
@@ -1209,17 +1228,19 @@ if not preconfigured:
 
     # libxml2 should be optional but is currently not
     # https://github.com/mapnik/mapnik/issues/913
-    if conf.parse_config('XML2_CONFIG',checks='--cflags'):
+    if env.get('XML2_LIBS') or env.get('XML2_INCLUDES'):
+        REQUIRED_LIBSHEADERS.append(['libxml2','libxml/parser.h',True,'C'])
+        if env.get('XML2_INCLUDES'):
+            inc_path = env['XML2_INCLUDES']
+            env.AppendUnique(CPPPATH = os.path.realpath(inc_path))
+        if env.get('XML2_LIBS'):
+            lib_path = env['XML2_LIBS']
+            env.AppendUnique(LIBPATH = os.path.realpath(lib_path))
+    elif conf.parse_config('XML2_CONFIG',checks='--cflags'):
         env['HAS_LIBXML2'] = True
     else:
         env['MISSING_DEPS'].append('libxml2')
 
-    REQUIRED_LIBSHEADERS = [
-        ['z', 'zlib.h', True,'C'],
-        [env['ICU_LIB_NAME'],'unicode/unistr.h',True,'C++'],
-        ['harfbuzz', 'harfbuzz/hb.h',True,'C++']
-    ]
-
     if conf.CheckHasDlfcn():
         env.Append(CPPDEFINES = '-DMAPNIK_HAS_DLCFN')
     else:
@@ -1442,7 +1463,7 @@ if not preconfigured:
                         sqlite_backup = env.Clone().Dictionary()
                         # if statically linking, on linux we likely
                         # need to link sqlite to pthreads and dl
-                        if env['RUNTIME_LINK'] == 'static':
+                        if env['RUNTIME_LINK'] == 'static' and not env['PLATFORM'] == 'Darwin':
                             if CHECK_PKG_CONFIG and conf.CheckPKG('sqlite3'):
                                 sqlite_env = env.Clone()
                                 try:
@@ -1452,7 +1473,15 @@ if not preconfigured:
                                             env["SQLITE_LINKFLAGS"].append(lib)
                                             env.Append(LIBS=lib)
                                 except OSError,e:
-                                    pass
+                                    for lib in ["sqlite3","dl","pthread"]:
+                                        if not lib in env['LIBS']:
+                                            env["SQLITE_LINKFLAGS"].append("lib")
+                                            env.Append(LIBS=lib)
+                            else:
+                                for lib in ["sqlite3","dl","pthread"]:
+                                    if not lib in env['LIBS']:
+                                        env["SQLITE_LINKFLAGS"].append("lib")
+                                        env.Append(LIBS=lib)
                         SQLITE_HAS_RTREE = conf.sqlite_has_rtree()
                         if not SQLITE_HAS_RTREE:
                             env.Replace(**sqlite_backup)

bootstrap.sh

@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+
+#set -eu
+
+# NOTE: requires at least bash >= 4.0
+# brew install bash
+
+: '
+
+todo
+
+- clang debs to s3
+- docs for base setup: sudo apt-get -y install zlib1g-dev python-dev make git python-dev
+- boost_python_patch
+- shrink icu data
+- cairo/pycairo
+- clang + libc++
+- pkg-config-less
+- gdal shared lib?
+'
+
+declare -A DEPS
+DEPS["freetype"]="2.5.4"
+DEPS["harfbuzz"]="2cd5323"
+DEPS["jpeg"]="v8d"
+DEPS["libxml2"]="2.9.2"
+DEPS["libpng"]="1.6.13"
+DEPS["webp"]="0.4.2"
+DEPS["icu"]="54.1"
+DEPS["proj"]="4.8.0"
+DEPS["libtiff"]="dev"
+DEPS["boost"]="1.57.0"
+DEPS["boost_libsystem"]="1.57.0"
+DEPS["boost_libthread"]="1.57.0"
+DEPS["boost_libfilesystem"]="1.57.0"
+DEPS["boost_libprogram_options"]="1.57.0"
+DEPS["boost_libregex"]="1.57.0"
+DEPS["boost_libpython"]="1.57.0"
+DEPS["libpq"]="9.4.0"
+DEPS["sqlite"]="3.8.6"
+DEPS["gdal"]="1.11.1"
+DEPS["expat"]="2.1.0"
+
+CPP11_TOOLCHAIN="$(pwd)/toolchain"
+
+function dpack() {
+    if [[ ! -f $2 ]]; then
+        wget -q $1/$(echo $2 | sed 's/+/%2B/g')
+        dpkg -x $2 ${CPP11_TOOLCHAIN}
+    fi
+}
+
+function setup_cpp11_toolchain() {
+    if [[ $(uname -s) == 'Linux' ]]; then
+        local PPA="https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test/+files"
+        # http://llvm.org/apt/precise/dists/llvm-toolchain-precise-3.5/main/binary-amd64/Packages
+        # TODO: cache these for faster downloads
+        local LLVM_DIST="http://llvm.org/apt/precise/pool/main/l/llvm-toolchain-3.5"
+        dpack ${LLVM_DIST} clang-3.5_3.5~svn217304-1~exp1_amd64.deb
+        dpack ${LLVM_DIST} libllvm3.5_3.5~svn217304-1~exp1_amd64.deb
+        dpack ${LLVM_DIST} libclang-common-3.5-dev_3.5~svn215019-1~exp1_amd64.deb
+        dpack ${PPA} libstdc++6_4.8.1-2ubuntu1~12.04_amd64.deb
+        dpack ${PPA} libstdc++-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb
+        dpack ${PPA} libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb
+        export CPLUS_INCLUDE_PATH="${CPP11_TOOLCHAIN}/usr/include/c++/4.8:${CPP11_TOOLCHAIN}/usr/include/x86_64-linux-gnu/c++/4.8:${CPLUS_INCLUDE_PATH}"
+        export LD_LIBRARY_PATH="${CPP11_TOOLCHAIN}/usr/lib/x86_64-linux-gnu:${CPP11_TOOLCHAIN}/usr/lib/gcc/x86_64-linux-gnu/4.8/:${LD_LIBRARY_PATH}"
+        export LIBRARY_PATH="${LD_LIBRARY_PATH}"
+        export PATH="${CPP11_TOOLCHAIN}/usr/bin":${PATH}
+        export CXX="${CPP11_TOOLCHAIN}/usr/bin/clang++-3.5"
+        export CC="${CPP11_TOOLCHAIN}/usr/bin/clang-3.5"
+    else
+        export CXX=clang++
+        export CC=clang
+    fi
+}
+
+function setup_mason() {
+    if [[ -d ~/.mason ]]; then
+        export PATH=~/.mason:$PATH
+    else
+        if [[ ! -d ./.mason ]]; then
+            git clone --depth 1 https://github.com/mapbox/mason.git ./.mason
+        fi
+        export MASON_DIR=$(pwd)/.mason
+        export PATH=$(pwd)/.mason:$PATH
+    fi
+}
+
+function install_mason_deps() {
+    if [[ ! -d ./mason_packages ]]; then
+        for DEP in "${!DEPS[@]}"; do
+            mason install ${DEP} ${DEPS[$DEP]}
+        done
+    fi
+    if [[ ! -d ./mason_packages/.link ]]; then
+        for DEP in "${!DEPS[@]}"; do
+            mason link ${DEP} ${DEPS[$DEP]}
+        done
+    fi
+}
+
+function setup_nose() {
+    if [[ ! -d $(pwd)/nose-1.3.4 ]]; then
+        wget -q https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz
+        tar -xzf nose-1.3.4.tar.gz
+    fi
+    export PYTHONPATH=$(pwd)/nose-1.3.4:${PYTHONPATH}
+}
+
+function make_config() {
+    local MASON_LINKED_REL=./mason_packages/.link
+    export PKG_CONFIG_PATH="${MASON_LINKED_REL}/lib/pkgconfig"
+    export C_INCLUDE_PATH="${MASON_LINKED_REL}/include"
+    export CPLUS_INCLUDE_PATH="${MASON_LINKED_REL}/include"
+    export LIBRARY_PATH="${MASON_LINKED_REL}/lib"
+    export PATH="${MASON_LINKED_REL}/bin":${PATH}
+
+    local CUSTOM_CXXFLAGS="-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1"
+    local MASON_LIBS="${MASON_LINKED_REL}/lib"
+    local MASON_INCLUDES="${MASON_LINKED_REL}/include"
+    echo "
+CXX = '$CXX'
+CC = '$CC'
+CUSTOM_CXXFLAGS = '-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1'
+CUSTOM_LDFLAGS = '-L${MASON_LINKED_REL}/lib'
+RUNTIME_LINK = 'static'
+INPUT_PLUGINS = 'csv,gdal,geojson,occi,ogr,osm,pgraster,postgis,python,raster,rasterlite,shape,sqlite,topojson'
+PREFIX = '/opt/mapnik-3.x'
+PATH = '${MASON_LINKED_REL}/bin'
+PATH_REMOVE = '/usr:/usr/local'
+MAPNIK_NAME = 'mapnik_3-0-0'
+BOOST_INCLUDES = '${MASON_LINKED_REL}/include'
+BOOST_LIBS = '${MASON_LINKED_REL}/lib'
+ICU_INCLUDES = '${MASON_LINKED_REL}/include'
+ICU_LIBS = '${MASON_LINKED_REL}/lib'
+HB_INCLUDES = '${MASON_LINKED_REL}/include'
+HB_LIBS = '${MASON_LINKED_REL}/lib'
+PNG_INCLUDES = '${MASON_LINKED_REL}/include/libpng16'
+PNG_LIBS = '${MASON_LINKED_REL}/lib'
+JPEG_INCLUDES = '${MASON_LINKED_REL}/include'
+JPEG_LIBS = '${MASON_LINKED_REL}/lib'
+TIFF_INCLUDES = '${MASON_LINKED_REL}/include'
+TIFF_LIBS = '${MASON_LINKED_REL}/lib'
+WEBP_INCLUDES = '${MASON_LINKED_REL}/include'
+WEBP_LIBS = '${MASON_LINKED_REL}/lib'
+PROJ_INCLUDES = '${MASON_LINKED_REL}/include'
+PROJ_LIBS = '${MASON_LINKED_REL}/lib'
+FREETYPE_INCLUDES = '${MASON_LINKED_REL}/include/freetype2'
+FREETYPE_LIBS = '${MASON_LINKED_REL}/lib'
+XML2_INCLUDES = '${MASON_LINKED_REL}/include/libxml2'
+XML2_LIBS = '${MASON_LINKED_REL}/lib'
+SVG_RENDERER = True
+CAIRO_INCLUDES = '${MASON_LINKED_REL}/include'
+CAIRO_LIBS = '${MASON_LINKED_REL}/lib'
+SQLITE_INCLUDES = '${MASON_LINKED_REL}/include'
+SQLITE_LIBS = '${MASON_LINKED_REL}/lib'
+FRAMEWORK_PYTHON = False
+BINDINGS = 'python'
+XMLPARSER = 'ptree'
+SVG2PNG = True
+SAMPLE_INPUT_PLUGINS = True
+" > ./config.py
+}
+
+function setup_runtime_settings() {
+    local MASON_LINKED_ABS=$(pwd)/mason_packages/.link
+    export PROJ_LIB=${MASON_LINKED_ABS}/share/proj/
+    export ICU_DATA=${MASON_LINKED_ABS}/share/icu/54.1/
+    export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal
+}
+
+function main() {
+    setup_mason
+    install_mason_deps
+    setup_nose
+    setup_cpp11_toolchain
+    make_config
+    setup_runtime_settings
+}
+
+main