diff --git a/.travis.yml b/.travis.yml index e0a579db1..5074f5658 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,53 +1,19 @@ language: cpp -compiler: - - clang - - gcc +sudo: false -env: - matrix: - - DEBUG=False PLUGIN_LINKING=static ENABLE_LOG=True DEFAULT_LOG_SEVERITY=debug XMLPARSER="libxml2" DEMO=False BENCHMARK=False CUSTOM_CXXFLAGS="" CUSTOM_LDFLAGS="" - - DEBUG=False ENABLE_LOG=False DEFAULT_LOG_SEVERITY=none XMLPARSER="ptree" DEMO=False BENCHMARK=True CUSTOM_CXXFLAGS="" CUSTOM_LDFLAGS="" - -# travis + ubuntugis with gdal and postggis leads to many potential dead-end conflicts -# the below is thanks to https://github.com/CartoDB/Windshaft/blob/d82fe08b32fc7907bbe907ab290f8a082215ae26/.travis.yml#L1 -before_install: - - export PGUSER=postgres - - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp - - sudo apt-get -qq purge postgis* postgresql* - - sudo apt-add-repository -y ppa:cartodb/postgresql-9.3 - - sudo apt-add-repository -y ppa:cartodb/gis - # grab harfbuzz from ppa - - sudo apt-add-repository -y ppa:fontforge/fontforge - # we need at least g++-4.7 for c++11 features - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - # enable to test against latest boost rather that v1.48 - #- sudo add-apt-repository -y ppa:boost-latest/ppa - - sudo rm -Rf /var/lib/postgresql /etc/postgresql - - sudo apt-get update -qq - - sudo apt-get install -q libharfbuzz-dev postgresql-9.3-postgis-2.1 postgresql-contrib-9.3 gdal-bin libgdal-dev - - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.3/main/pg_hba.conf - - sudo service postgresql restart +addons: + postgresql: "9.3" install: - # enable to test against boost ppa - #- sudo apt-get install -y boost1.55 - - sudo apt-get install -y libboost-python1.48-dev libboost-thread1.48-dev libboost-filesystem1.48-dev libboost-regex1.48-dev libboost-program-options1.48-dev - - sudo apt-get install -y ttf-wqy-microhei make libstdc++6 libstdc++-4.8-dev valgrind python-nose libicu-dev libproj-dev libcairo-dev python-cairo-dev libcairo-dev python-cairo-dev libpng-dev libjpeg-dev libtiff-dev libwebp-dev libz-dev libfreetype6-dev libxml2-dev libsqlite3-dev - -before_script: - - psql -U postgres -c 'create database template_postgis' - - psql -U postgres -c 'create extension postgis' -d template_postgis - - if [[ "${CXX}" == 'g++' ]]; then export JOBS=2; sudo apt-get install gcc-4.8 g++-4.8; export CXX="$(which g++-4.8)"; export CC="$(which gcc-4.8)"; fi; - - if [[ "${CXX}" == 'clang++' ]]; then export JOBS=4; export CXX="$(which clang++)"; export CC="$(which clang)"; fi; + - psql -U postgres -c 'create database template_postgis;' -U postgres + - psql -U postgres -c 'create extension postgis;' -d template_postgis -U postgres script: - - rm -rf ./git - - ./configure CXX="${CXX}" CC="${CC}" CUSTOM_CXXFLAGS="${CUSTOM_CXXFLAGS}" CUSTOM_LDFLAGS="${CUSTOM_LDFLAGS}" XML_PARSER="${XML_PARSER}" ENABLE_LOG="${ENABLE_LOG}" DEBUG="${DEBUG}" DEMO="${DEMO}" BENCHMARK="${BENCHMARK}" CPP_TESTS=True CAIRO=True FAST=True || cat config.log - - if [[ "${DEBUG}" == True ]]; then export JOBS=$((JOBS/2)); fi; - - make + - source bootstrap.sh + - ./configure + - JOBS=6 make - git clone --depth=1 https://github.com/mapbox/mapnik-test-data tests/data/mapnik-test-data - make test - - source localize.sh && make grind - - if [[ ${BENCHMARK} != False ]]; then make bench; fi; - - if [[ ${BENCHMARK} != False ]]; then for i in $(ls tests/visual_tests/styles/*xml); do echo $i;./benchmark/out/test_rendering --name "text rendering" --map $i --width 1024 --height 1024 --iterations 2 --threads 5;done;fi + #- source localize.sh && make grind + #- make bench diff --git a/Makefile b/Makefile index 0ae5d4160..e0e229dc5 100755 --- a/Makefile +++ b/Makefile @@ -13,6 +13,20 @@ install: $(PYTHON) scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1 install mapnik: + # we first build memory intensive files with -j1 + $(PYTHON) scons/scons.py -j1 \ + --config=cache --implicit-cache --max-drift=1 \ + src/json/libmapnik-json.a \ + src/wkt/libmapnik-wkt.a \ + src/css_color_grammar.os \ + src/expression_grammar.os \ + src/transform_expression_grammar.os \ + src/image_filter_types.os \ + src/renderer_common/process_group_symbolizer.cpp \ + src/agg/process_markers_symbolizer.cpp \ + src/grid/process_markers_symbolizer.cpp \ + src/cairo/process_markers_symbolizer.cpp + # then install the rest with -j$(JOBS) $(PYTHON) scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1 clean: diff --git a/SConstruct b/SConstruct index 7026acc0b..a83807525 100644 --- a/SConstruct +++ b/SConstruct @@ -88,10 +88,11 @@ pretty_dep_names = { 'm':'Basic math library, part of C++ stlib', 'pkg-config':'pkg-config tool | more info: http://pkg-config.freedesktop.org', 'pg_config':'pg_config program | try setting PG_CONFIG SCons option', - 'xml2-config':'xml2-config program | try setting XML2_CONFIG SCons option', - 'libxml2':'libxml2 library | try setting XML2_CONFIG SCons option to point to location of xml2-config program', + 'xml2-config':'xml2-config program | try setting XML2_CONFIG SCons option or avoid the need for xml2-config command by configuring with XML2_LIBS & XML2_INCLUDES', + 'libxml2':'libxml2 library | try setting XML2_CONFIG SCons option to point to location of xml2-config program or configure with XML2_LIBS & XML2_INCLUDES', 'gdal-config':'gdal-config program | try setting GDAL_CONFIG SCons option', - 'freetype-config':'freetype-config program | try setting FREETYPE_CONFIG SCons option', + 'freetype-config':'freetype-config program | try setting FREETYPE_CONFIG SCons option or configure with FREETYPE_LIBS & FREETYPE_INCLUDES', + 'freetype':'libfreetype library | try setting FREETYPE_CONFIG SCons option or configure with FREETYPE_LIBS & FREETYPE_INCLUDES', 'osm':'more info: https://github.com/mapnik/mapnik/wiki/OsmPlugin', 'boost_regex_icu':'libboost_regex built with optional ICU unicode support is needed for unicode regex support in mapnik.', 'sqlite_rtree':'The SQLite plugin requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)', @@ -352,6 +353,10 @@ opts.AddVariables( BoolVariable('PROJ', 'Build Mapnik with proj4 support to enable transformations between many different projections', 'True'), PathVariable('PROJ_INCLUDES', 'Search path for PROJ.4 include files', '/usr/include', PathVariable.PathAccept), PathVariable('PROJ_LIBS', 'Search path for PROJ.4 library files', '/usr/' + LIBDIR_SCHEMA_DEFAULT, PathVariable.PathAccept), + ('FREETYPE_INCLUDES', 'Search path for Freetype include files', ''), + ('FREETYPE_LIBS', 'Search path for Freetype library files', ''), + ('XML2_INCLUDES', 'Search path for libxml2 include files', ''), + ('XML2_LIBS', 'Search path for libxml2 library files', ''), ('PKG_CONFIG_PATH', 'Use this path to point pkg-config to .pc files instead of the PKG_CONFIG_PATH environment setting',''), # Variables affecting rendering back-ends @@ -1194,7 +1199,21 @@ if not preconfigured: env.AppendUnique(CPPPATH = os.path.realpath(inc_path)) env.AppendUnique(LIBPATH = os.path.realpath(lib_path)) - if conf.parse_config('FREETYPE_CONFIG'): + REQUIRED_LIBSHEADERS = [ + ['z', 'zlib.h', True,'C'], + [env['ICU_LIB_NAME'],'unicode/unistr.h',True,'C++'], + ['harfbuzz', 'harfbuzz/hb.h',True,'C++'] + ] + + if env.get('FREETYPE_LIBS') or env.get('FREETYPE_INCLUDES'): + REQUIRED_LIBSHEADERS.append(['freetype','ft2build.h',True,'C']) + if env.get('FREETYPE_INCLUDES'): + inc_path = env['FREETYPE_INCLUDES'] + env.AppendUnique(CPPPATH = os.path.realpath(inc_path)) + if env.get('FREETYPE_LIBS'): + lib_path = env['FREETYPE_LIBS'] + env.AppendUnique(LIBPATH = os.path.realpath(lib_path)) + elif conf.parse_config('FREETYPE_CONFIG'): # check if freetype links to bz2 if env['RUNTIME_LINK'] == 'static': temp_env = env.Clone() @@ -1209,17 +1228,19 @@ if not preconfigured: # libxml2 should be optional but is currently not # https://github.com/mapnik/mapnik/issues/913 - if conf.parse_config('XML2_CONFIG',checks='--cflags'): + if env.get('XML2_LIBS') or env.get('XML2_INCLUDES'): + REQUIRED_LIBSHEADERS.append(['libxml2','libxml/parser.h',True,'C']) + if env.get('XML2_INCLUDES'): + inc_path = env['XML2_INCLUDES'] + env.AppendUnique(CPPPATH = os.path.realpath(inc_path)) + if env.get('XML2_LIBS'): + lib_path = env['XML2_LIBS'] + env.AppendUnique(LIBPATH = os.path.realpath(lib_path)) + elif conf.parse_config('XML2_CONFIG',checks='--cflags'): env['HAS_LIBXML2'] = True else: env['MISSING_DEPS'].append('libxml2') - REQUIRED_LIBSHEADERS = [ - ['z', 'zlib.h', True,'C'], - [env['ICU_LIB_NAME'],'unicode/unistr.h',True,'C++'], - ['harfbuzz', 'harfbuzz/hb.h',True,'C++'] - ] - if conf.CheckHasDlfcn(): env.Append(CPPDEFINES = '-DMAPNIK_HAS_DLCFN') else: @@ -1442,7 +1463,7 @@ if not preconfigured: sqlite_backup = env.Clone().Dictionary() # if statically linking, on linux we likely # need to link sqlite to pthreads and dl - if env['RUNTIME_LINK'] == 'static': + if env['RUNTIME_LINK'] == 'static' and not env['PLATFORM'] == 'Darwin': if CHECK_PKG_CONFIG and conf.CheckPKG('sqlite3'): sqlite_env = env.Clone() try: @@ -1452,7 +1473,15 @@ if not preconfigured: env["SQLITE_LINKFLAGS"].append(lib) env.Append(LIBS=lib) except OSError,e: - pass + for lib in ["sqlite3","dl","pthread"]: + if not lib in env['LIBS']: + env["SQLITE_LINKFLAGS"].append("lib") + env.Append(LIBS=lib) + else: + for lib in ["sqlite3","dl","pthread"]: + if not lib in env['LIBS']: + env["SQLITE_LINKFLAGS"].append("lib") + env.Append(LIBS=lib) SQLITE_HAS_RTREE = conf.sqlite_has_rtree() if not SQLITE_HAS_RTREE: env.Replace(**sqlite_backup) diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 000000000..5b661430a --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash + +#set -eu + +# NOTE: requires at least bash >= 4.0 +# brew install bash + +: ' + +todo + +- clang debs to s3 +- docs for base setup: sudo apt-get -y install zlib1g-dev python-dev make git python-dev +- boost_python_patch +- shrink icu data +- cairo/pycairo +- clang + libc++ +- pkg-config-less +- gdal shared lib? +' + +declare -A DEPS +DEPS["freetype"]="2.5.4" +DEPS["harfbuzz"]="2cd5323" +DEPS["jpeg"]="v8d" +DEPS["libxml2"]="2.9.2" +DEPS["libpng"]="1.6.13" +DEPS["webp"]="0.4.2" +DEPS["icu"]="54.1" +DEPS["proj"]="4.8.0" +DEPS["libtiff"]="dev" +DEPS["boost"]="1.57.0" +DEPS["boost_libsystem"]="1.57.0" +DEPS["boost_libthread"]="1.57.0" +DEPS["boost_libfilesystem"]="1.57.0" +DEPS["boost_libprogram_options"]="1.57.0" +DEPS["boost_libregex"]="1.57.0" +DEPS["boost_libpython"]="1.57.0" +DEPS["libpq"]="9.4.0" +DEPS["sqlite"]="3.8.6" +DEPS["gdal"]="1.11.1" +DEPS["expat"]="2.1.0" + +CPP11_TOOLCHAIN="$(pwd)/toolchain" + +function dpack() { + if [[ ! -f $2 ]]; then + wget -q $1/$(echo $2 | sed 's/+/%2B/g') + dpkg -x $2 ${CPP11_TOOLCHAIN} + fi +} + +function setup_cpp11_toolchain() { + if [[ $(uname -s) == 'Linux' ]]; then + local PPA="https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test/+files" + # http://llvm.org/apt/precise/dists/llvm-toolchain-precise-3.5/main/binary-amd64/Packages + # TODO: cache these for faster downloads + local LLVM_DIST="http://llvm.org/apt/precise/pool/main/l/llvm-toolchain-3.5" + dpack ${LLVM_DIST} clang-3.5_3.5~svn217304-1~exp1_amd64.deb + dpack ${LLVM_DIST} libllvm3.5_3.5~svn217304-1~exp1_amd64.deb + dpack ${LLVM_DIST} libclang-common-3.5-dev_3.5~svn215019-1~exp1_amd64.deb + dpack ${PPA} libstdc++6_4.8.1-2ubuntu1~12.04_amd64.deb + dpack ${PPA} libstdc++-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb + dpack ${PPA} libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb + export CPLUS_INCLUDE_PATH="${CPP11_TOOLCHAIN}/usr/include/c++/4.8:${CPP11_TOOLCHAIN}/usr/include/x86_64-linux-gnu/c++/4.8:${CPLUS_INCLUDE_PATH}" + export LD_LIBRARY_PATH="${CPP11_TOOLCHAIN}/usr/lib/x86_64-linux-gnu:${CPP11_TOOLCHAIN}/usr/lib/gcc/x86_64-linux-gnu/4.8/:${LD_LIBRARY_PATH}" + export LIBRARY_PATH="${LD_LIBRARY_PATH}" + export PATH="${CPP11_TOOLCHAIN}/usr/bin":${PATH} + export CXX="${CPP11_TOOLCHAIN}/usr/bin/clang++-3.5" + export CC="${CPP11_TOOLCHAIN}/usr/bin/clang-3.5" + else + export CXX=clang++ + export CC=clang + fi +} + +function setup_mason() { + if [[ -d ~/.mason ]]; then + export PATH=~/.mason:$PATH + else + if [[ ! -d ./.mason ]]; then + git clone --depth 1 https://github.com/mapbox/mason.git ./.mason + fi + export MASON_DIR=$(pwd)/.mason + export PATH=$(pwd)/.mason:$PATH + fi +} + +function install_mason_deps() { + if [[ ! -d ./mason_packages ]]; then + for DEP in "${!DEPS[@]}"; do + mason install ${DEP} ${DEPS[$DEP]} + done + fi + if [[ ! -d ./mason_packages/.link ]]; then + for DEP in "${!DEPS[@]}"; do + mason link ${DEP} ${DEPS[$DEP]} + done + fi +} + +function setup_nose() { + if [[ ! -d $(pwd)/nose-1.3.4 ]]; then + wget -q https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz + tar -xzf nose-1.3.4.tar.gz + fi + export PYTHONPATH=$(pwd)/nose-1.3.4:${PYTHONPATH} +} + +function make_config() { + local MASON_LINKED_REL=./mason_packages/.link + export PKG_CONFIG_PATH="${MASON_LINKED_REL}/lib/pkgconfig" + export C_INCLUDE_PATH="${MASON_LINKED_REL}/include" + export CPLUS_INCLUDE_PATH="${MASON_LINKED_REL}/include" + export LIBRARY_PATH="${MASON_LINKED_REL}/lib" + export PATH="${MASON_LINKED_REL}/bin":${PATH} + + local CUSTOM_CXXFLAGS="-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1" + local MASON_LIBS="${MASON_LINKED_REL}/lib" + local MASON_INCLUDES="${MASON_LINKED_REL}/include" + echo " +CXX = '$CXX' +CC = '$CC' +CUSTOM_CXXFLAGS = '-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1' +CUSTOM_LDFLAGS = '-L${MASON_LINKED_REL}/lib' +RUNTIME_LINK = 'static' +INPUT_PLUGINS = 'csv,gdal,geojson,occi,ogr,osm,pgraster,postgis,python,raster,rasterlite,shape,sqlite,topojson' +PREFIX = '/opt/mapnik-3.x' +PATH = '${MASON_LINKED_REL}/bin' +PATH_REMOVE = '/usr:/usr/local' +MAPNIK_NAME = 'mapnik_3-0-0' +BOOST_INCLUDES = '${MASON_LINKED_REL}/include' +BOOST_LIBS = '${MASON_LINKED_REL}/lib' +ICU_INCLUDES = '${MASON_LINKED_REL}/include' +ICU_LIBS = '${MASON_LINKED_REL}/lib' +HB_INCLUDES = '${MASON_LINKED_REL}/include' +HB_LIBS = '${MASON_LINKED_REL}/lib' +PNG_INCLUDES = '${MASON_LINKED_REL}/include/libpng16' +PNG_LIBS = '${MASON_LINKED_REL}/lib' +JPEG_INCLUDES = '${MASON_LINKED_REL}/include' +JPEG_LIBS = '${MASON_LINKED_REL}/lib' +TIFF_INCLUDES = '${MASON_LINKED_REL}/include' +TIFF_LIBS = '${MASON_LINKED_REL}/lib' +WEBP_INCLUDES = '${MASON_LINKED_REL}/include' +WEBP_LIBS = '${MASON_LINKED_REL}/lib' +PROJ_INCLUDES = '${MASON_LINKED_REL}/include' +PROJ_LIBS = '${MASON_LINKED_REL}/lib' +FREETYPE_INCLUDES = '${MASON_LINKED_REL}/include/freetype2' +FREETYPE_LIBS = '${MASON_LINKED_REL}/lib' +XML2_INCLUDES = '${MASON_LINKED_REL}/include/libxml2' +XML2_LIBS = '${MASON_LINKED_REL}/lib' +SVG_RENDERER = True +CAIRO_INCLUDES = '${MASON_LINKED_REL}/include' +CAIRO_LIBS = '${MASON_LINKED_REL}/lib' +SQLITE_INCLUDES = '${MASON_LINKED_REL}/include' +SQLITE_LIBS = '${MASON_LINKED_REL}/lib' +FRAMEWORK_PYTHON = False +BINDINGS = 'python' +XMLPARSER = 'ptree' +SVG2PNG = True +SAMPLE_INPUT_PLUGINS = True +" > ./config.py +} + +function setup_runtime_settings() { + local MASON_LINKED_ABS=$(pwd)/mason_packages/.link + export PROJ_LIB=${MASON_LINKED_ABS}/share/proj/ + export ICU_DATA=${MASON_LINKED_ABS}/share/icu/54.1/ + export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal +} + +function main() { + setup_mason + install_mason_deps + setup_nose + setup_cpp11_toolchain + make_config + setup_runtime_settings +} + +main