Merge branch 'master' into large-geojson

This commit is contained in:
artemp 2015-01-26 09:16:06 +01:00
commit 2f42cb91cc
4 changed files with 247 additions and 57 deletions

View file

@ -1,53 +1,19 @@
language: cpp
compiler:
- clang
- gcc
sudo: false
env:
matrix:
- DEBUG=False PLUGIN_LINKING=static ENABLE_LOG=True DEFAULT_LOG_SEVERITY=debug XMLPARSER="libxml2" DEMO=False BENCHMARK=False CUSTOM_CXXFLAGS="" CUSTOM_LDFLAGS=""
- DEBUG=False ENABLE_LOG=False DEFAULT_LOG_SEVERITY=none XMLPARSER="ptree" DEMO=False BENCHMARK=True CUSTOM_CXXFLAGS="" CUSTOM_LDFLAGS=""
# travis + ubuntugis with gdal and postggis leads to many potential dead-end conflicts
# the below is thanks to https://github.com/CartoDB/Windshaft/blob/d82fe08b32fc7907bbe907ab290f8a082215ae26/.travis.yml#L1
before_install:
- export PGUSER=postgres
- sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
- sudo apt-get -qq purge postgis* postgresql*
- sudo apt-add-repository -y ppa:cartodb/postgresql-9.3
- sudo apt-add-repository -y ppa:cartodb/gis
# grab harfbuzz from ppa
- sudo apt-add-repository -y ppa:fontforge/fontforge
# we need at least g++-4.7 for c++11 features
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
# enable to test against latest boost rather that v1.48
#- sudo add-apt-repository -y ppa:boost-latest/ppa
- sudo rm -Rf /var/lib/postgresql /etc/postgresql
- sudo apt-get update -qq
- sudo apt-get install -q libharfbuzz-dev postgresql-9.3-postgis-2.1 postgresql-contrib-9.3 gdal-bin libgdal-dev
- echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.3/main/pg_hba.conf
- sudo service postgresql restart
addons:
postgresql: "9.3"
install:
# enable to test against boost ppa
#- sudo apt-get install -y boost1.55
- sudo apt-get install -y libboost-python1.48-dev libboost-thread1.48-dev libboost-filesystem1.48-dev libboost-regex1.48-dev libboost-program-options1.48-dev
- sudo apt-get install -y ttf-wqy-microhei make libstdc++6 libstdc++-4.8-dev valgrind python-nose libicu-dev libproj-dev libcairo-dev python-cairo-dev libcairo-dev python-cairo-dev libpng-dev libjpeg-dev libtiff-dev libwebp-dev libz-dev libfreetype6-dev libxml2-dev libsqlite3-dev
before_script:
- psql -U postgres -c 'create database template_postgis'
- psql -U postgres -c 'create extension postgis' -d template_postgis
- if [[ "${CXX}" == 'g++' ]]; then export JOBS=2; sudo apt-get install gcc-4.8 g++-4.8; export CXX="$(which g++-4.8)"; export CC="$(which gcc-4.8)"; fi;
- if [[ "${CXX}" == 'clang++' ]]; then export JOBS=4; export CXX="$(which clang++)"; export CC="$(which clang)"; fi;
- psql -U postgres -c 'create database template_postgis;' -U postgres
- psql -U postgres -c 'create extension postgis;' -d template_postgis -U postgres
script:
- rm -rf ./git
- ./configure CXX="${CXX}" CC="${CC}" CUSTOM_CXXFLAGS="${CUSTOM_CXXFLAGS}" CUSTOM_LDFLAGS="${CUSTOM_LDFLAGS}" XML_PARSER="${XML_PARSER}" ENABLE_LOG="${ENABLE_LOG}" DEBUG="${DEBUG}" DEMO="${DEMO}" BENCHMARK="${BENCHMARK}" CPP_TESTS=True CAIRO=True FAST=True || cat config.log
- if [[ "${DEBUG}" == True ]]; then export JOBS=$((JOBS/2)); fi;
- make
- source bootstrap.sh
- ./configure
- JOBS=6 make
- git clone --depth=1 https://github.com/mapbox/mapnik-test-data tests/data/mapnik-test-data
- make test
- source localize.sh && make grind
- if [[ ${BENCHMARK} != False ]]; then make bench; fi;
- if [[ ${BENCHMARK} != False ]]; then for i in $(ls tests/visual_tests/styles/*xml); do echo $i;./benchmark/out/test_rendering --name "text rendering" --map $i --width 1024 --height 1024 --iterations 2 --threads 5;done;fi
#- source localize.sh && make grind
#- make bench

View file

@ -13,6 +13,20 @@ install:
$(PYTHON) scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1 install
mapnik:
# we first build memory intensive files with -j1
$(PYTHON) scons/scons.py -j1 \
--config=cache --implicit-cache --max-drift=1 \
src/json/libmapnik-json.a \
src/wkt/libmapnik-wkt.a \
src/css_color_grammar.os \
src/expression_grammar.os \
src/transform_expression_grammar.os \
src/image_filter_types.os \
src/renderer_common/process_group_symbolizer.cpp \
src/agg/process_markers_symbolizer.cpp \
src/grid/process_markers_symbolizer.cpp \
src/cairo/process_markers_symbolizer.cpp
# then install the rest with -j$(JOBS)
$(PYTHON) scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1
clean:

View file

@ -88,10 +88,11 @@ pretty_dep_names = {
'm':'Basic math library, part of C++ stlib',
'pkg-config':'pkg-config tool | more info: http://pkg-config.freedesktop.org',
'pg_config':'pg_config program | try setting PG_CONFIG SCons option',
'xml2-config':'xml2-config program | try setting XML2_CONFIG SCons option',
'libxml2':'libxml2 library | try setting XML2_CONFIG SCons option to point to location of xml2-config program',
'xml2-config':'xml2-config program | try setting XML2_CONFIG SCons option or avoid the need for xml2-config command by configuring with XML2_LIBS & XML2_INCLUDES',
'libxml2':'libxml2 library | try setting XML2_CONFIG SCons option to point to location of xml2-config program or configure with XML2_LIBS & XML2_INCLUDES',
'gdal-config':'gdal-config program | try setting GDAL_CONFIG SCons option',
'freetype-config':'freetype-config program | try setting FREETYPE_CONFIG SCons option',
'freetype-config':'freetype-config program | try setting FREETYPE_CONFIG SCons option or configure with FREETYPE_LIBS & FREETYPE_INCLUDES',
'freetype':'libfreetype library | try setting FREETYPE_CONFIG SCons option or configure with FREETYPE_LIBS & FREETYPE_INCLUDES',
'osm':'more info: https://github.com/mapnik/mapnik/wiki/OsmPlugin',
'boost_regex_icu':'libboost_regex built with optional ICU unicode support is needed for unicode regex support in mapnik.',
'sqlite_rtree':'The SQLite plugin requires libsqlite3 built with RTREE support (-DSQLITE_ENABLE_RTREE=1)',
@ -352,6 +353,10 @@ opts.AddVariables(
BoolVariable('PROJ', 'Build Mapnik with proj4 support to enable transformations between many different projections', 'True'),
PathVariable('PROJ_INCLUDES', 'Search path for PROJ.4 include files', '/usr/include', PathVariable.PathAccept),
PathVariable('PROJ_LIBS', 'Search path for PROJ.4 library files', '/usr/' + LIBDIR_SCHEMA_DEFAULT, PathVariable.PathAccept),
('FREETYPE_INCLUDES', 'Search path for Freetype include files', ''),
('FREETYPE_LIBS', 'Search path for Freetype library files', ''),
('XML2_INCLUDES', 'Search path for libxml2 include files', ''),
('XML2_LIBS', 'Search path for libxml2 library files', ''),
('PKG_CONFIG_PATH', 'Use this path to point pkg-config to .pc files instead of the PKG_CONFIG_PATH environment setting',''),
# Variables affecting rendering back-ends
@ -1194,7 +1199,21 @@ if not preconfigured:
env.AppendUnique(CPPPATH = os.path.realpath(inc_path))
env.AppendUnique(LIBPATH = os.path.realpath(lib_path))
if conf.parse_config('FREETYPE_CONFIG'):
REQUIRED_LIBSHEADERS = [
['z', 'zlib.h', True,'C'],
[env['ICU_LIB_NAME'],'unicode/unistr.h',True,'C++'],
['harfbuzz', 'harfbuzz/hb.h',True,'C++']
]
if env.get('FREETYPE_LIBS') or env.get('FREETYPE_INCLUDES'):
REQUIRED_LIBSHEADERS.append(['freetype','ft2build.h',True,'C'])
if env.get('FREETYPE_INCLUDES'):
inc_path = env['FREETYPE_INCLUDES']
env.AppendUnique(CPPPATH = os.path.realpath(inc_path))
if env.get('FREETYPE_LIBS'):
lib_path = env['FREETYPE_LIBS']
env.AppendUnique(LIBPATH = os.path.realpath(lib_path))
elif conf.parse_config('FREETYPE_CONFIG'):
# check if freetype links to bz2
if env['RUNTIME_LINK'] == 'static':
temp_env = env.Clone()
@ -1209,17 +1228,19 @@ if not preconfigured:
# libxml2 should be optional but is currently not
# https://github.com/mapnik/mapnik/issues/913
if conf.parse_config('XML2_CONFIG',checks='--cflags'):
if env.get('XML2_LIBS') or env.get('XML2_INCLUDES'):
REQUIRED_LIBSHEADERS.append(['libxml2','libxml/parser.h',True,'C'])
if env.get('XML2_INCLUDES'):
inc_path = env['XML2_INCLUDES']
env.AppendUnique(CPPPATH = os.path.realpath(inc_path))
if env.get('XML2_LIBS'):
lib_path = env['XML2_LIBS']
env.AppendUnique(LIBPATH = os.path.realpath(lib_path))
elif conf.parse_config('XML2_CONFIG',checks='--cflags'):
env['HAS_LIBXML2'] = True
else:
env['MISSING_DEPS'].append('libxml2')
REQUIRED_LIBSHEADERS = [
['z', 'zlib.h', True,'C'],
[env['ICU_LIB_NAME'],'unicode/unistr.h',True,'C++'],
['harfbuzz', 'harfbuzz/hb.h',True,'C++']
]
if conf.CheckHasDlfcn():
env.Append(CPPDEFINES = '-DMAPNIK_HAS_DLCFN')
else:
@ -1442,7 +1463,7 @@ if not preconfigured:
sqlite_backup = env.Clone().Dictionary()
# if statically linking, on linux we likely
# need to link sqlite to pthreads and dl
if env['RUNTIME_LINK'] == 'static':
if env['RUNTIME_LINK'] == 'static' and not env['PLATFORM'] == 'Darwin':
if CHECK_PKG_CONFIG and conf.CheckPKG('sqlite3'):
sqlite_env = env.Clone()
try:
@ -1452,7 +1473,15 @@ if not preconfigured:
env["SQLITE_LINKFLAGS"].append(lib)
env.Append(LIBS=lib)
except OSError,e:
pass
for lib in ["sqlite3","dl","pthread"]:
if not lib in env['LIBS']:
env["SQLITE_LINKFLAGS"].append("lib")
env.Append(LIBS=lib)
else:
for lib in ["sqlite3","dl","pthread"]:
if not lib in env['LIBS']:
env["SQLITE_LINKFLAGS"].append("lib")
env.Append(LIBS=lib)
SQLITE_HAS_RTREE = conf.sqlite_has_rtree()
if not SQLITE_HAS_RTREE:
env.Replace(**sqlite_backup)

181
bootstrap.sh Executable file
View file

@ -0,0 +1,181 @@
#!/usr/bin/env bash
#set -eu
# NOTE: requires at least bash >= 4.0
# brew install bash
: '
todo
- clang debs to s3
- docs for base setup: sudo apt-get -y install zlib1g-dev python-dev make git python-dev
- boost_python_patch
- shrink icu data
- cairo/pycairo
- clang + libc++
- pkg-config-less
- gdal shared lib?
'
declare -A DEPS
DEPS["freetype"]="2.5.4"
DEPS["harfbuzz"]="2cd5323"
DEPS["jpeg"]="v8d"
DEPS["libxml2"]="2.9.2"
DEPS["libpng"]="1.6.13"
DEPS["webp"]="0.4.2"
DEPS["icu"]="54.1"
DEPS["proj"]="4.8.0"
DEPS["libtiff"]="dev"
DEPS["boost"]="1.57.0"
DEPS["boost_libsystem"]="1.57.0"
DEPS["boost_libthread"]="1.57.0"
DEPS["boost_libfilesystem"]="1.57.0"
DEPS["boost_libprogram_options"]="1.57.0"
DEPS["boost_libregex"]="1.57.0"
DEPS["boost_libpython"]="1.57.0"
DEPS["libpq"]="9.4.0"
DEPS["sqlite"]="3.8.6"
DEPS["gdal"]="1.11.1"
DEPS["expat"]="2.1.0"
CPP11_TOOLCHAIN="$(pwd)/toolchain"
function dpack() {
if [[ ! -f $2 ]]; then
wget -q $1/$(echo $2 | sed 's/+/%2B/g')
dpkg -x $2 ${CPP11_TOOLCHAIN}
fi
}
function setup_cpp11_toolchain() {
if [[ $(uname -s) == 'Linux' ]]; then
local PPA="https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test/+files"
# http://llvm.org/apt/precise/dists/llvm-toolchain-precise-3.5/main/binary-amd64/Packages
# TODO: cache these for faster downloads
local LLVM_DIST="http://llvm.org/apt/precise/pool/main/l/llvm-toolchain-3.5"
dpack ${LLVM_DIST} clang-3.5_3.5~svn217304-1~exp1_amd64.deb
dpack ${LLVM_DIST} libllvm3.5_3.5~svn217304-1~exp1_amd64.deb
dpack ${LLVM_DIST} libclang-common-3.5-dev_3.5~svn215019-1~exp1_amd64.deb
dpack ${PPA} libstdc++6_4.8.1-2ubuntu1~12.04_amd64.deb
dpack ${PPA} libstdc++-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb
dpack ${PPA} libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb
export CPLUS_INCLUDE_PATH="${CPP11_TOOLCHAIN}/usr/include/c++/4.8:${CPP11_TOOLCHAIN}/usr/include/x86_64-linux-gnu/c++/4.8:${CPLUS_INCLUDE_PATH}"
export LD_LIBRARY_PATH="${CPP11_TOOLCHAIN}/usr/lib/x86_64-linux-gnu:${CPP11_TOOLCHAIN}/usr/lib/gcc/x86_64-linux-gnu/4.8/:${LD_LIBRARY_PATH}"
export LIBRARY_PATH="${LD_LIBRARY_PATH}"
export PATH="${CPP11_TOOLCHAIN}/usr/bin":${PATH}
export CXX="${CPP11_TOOLCHAIN}/usr/bin/clang++-3.5"
export CC="${CPP11_TOOLCHAIN}/usr/bin/clang-3.5"
else
export CXX=clang++
export CC=clang
fi
}
function setup_mason() {
if [[ -d ~/.mason ]]; then
export PATH=~/.mason:$PATH
else
if [[ ! -d ./.mason ]]; then
git clone --depth 1 https://github.com/mapbox/mason.git ./.mason
fi
export MASON_DIR=$(pwd)/.mason
export PATH=$(pwd)/.mason:$PATH
fi
}
function install_mason_deps() {
if [[ ! -d ./mason_packages ]]; then
for DEP in "${!DEPS[@]}"; do
mason install ${DEP} ${DEPS[$DEP]}
done
fi
if [[ ! -d ./mason_packages/.link ]]; then
for DEP in "${!DEPS[@]}"; do
mason link ${DEP} ${DEPS[$DEP]}
done
fi
}
function setup_nose() {
if [[ ! -d $(pwd)/nose-1.3.4 ]]; then
wget -q https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz
tar -xzf nose-1.3.4.tar.gz
fi
export PYTHONPATH=$(pwd)/nose-1.3.4:${PYTHONPATH}
}
function make_config() {
local MASON_LINKED_REL=./mason_packages/.link
export PKG_CONFIG_PATH="${MASON_LINKED_REL}/lib/pkgconfig"
export C_INCLUDE_PATH="${MASON_LINKED_REL}/include"
export CPLUS_INCLUDE_PATH="${MASON_LINKED_REL}/include"
export LIBRARY_PATH="${MASON_LINKED_REL}/lib"
export PATH="${MASON_LINKED_REL}/bin":${PATH}
local CUSTOM_CXXFLAGS="-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1"
local MASON_LIBS="${MASON_LINKED_REL}/lib"
local MASON_INCLUDES="${MASON_LINKED_REL}/include"
echo "
CXX = '$CXX'
CC = '$CC'
CUSTOM_CXXFLAGS = '-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1'
CUSTOM_LDFLAGS = '-L${MASON_LINKED_REL}/lib'
RUNTIME_LINK = 'static'
INPUT_PLUGINS = 'csv,gdal,geojson,occi,ogr,osm,pgraster,postgis,python,raster,rasterlite,shape,sqlite,topojson'
PREFIX = '/opt/mapnik-3.x'
PATH = '${MASON_LINKED_REL}/bin'
PATH_REMOVE = '/usr:/usr/local'
MAPNIK_NAME = 'mapnik_3-0-0'
BOOST_INCLUDES = '${MASON_LINKED_REL}/include'
BOOST_LIBS = '${MASON_LINKED_REL}/lib'
ICU_INCLUDES = '${MASON_LINKED_REL}/include'
ICU_LIBS = '${MASON_LINKED_REL}/lib'
HB_INCLUDES = '${MASON_LINKED_REL}/include'
HB_LIBS = '${MASON_LINKED_REL}/lib'
PNG_INCLUDES = '${MASON_LINKED_REL}/include/libpng16'
PNG_LIBS = '${MASON_LINKED_REL}/lib'
JPEG_INCLUDES = '${MASON_LINKED_REL}/include'
JPEG_LIBS = '${MASON_LINKED_REL}/lib'
TIFF_INCLUDES = '${MASON_LINKED_REL}/include'
TIFF_LIBS = '${MASON_LINKED_REL}/lib'
WEBP_INCLUDES = '${MASON_LINKED_REL}/include'
WEBP_LIBS = '${MASON_LINKED_REL}/lib'
PROJ_INCLUDES = '${MASON_LINKED_REL}/include'
PROJ_LIBS = '${MASON_LINKED_REL}/lib'
FREETYPE_INCLUDES = '${MASON_LINKED_REL}/include/freetype2'
FREETYPE_LIBS = '${MASON_LINKED_REL}/lib'
XML2_INCLUDES = '${MASON_LINKED_REL}/include/libxml2'
XML2_LIBS = '${MASON_LINKED_REL}/lib'
SVG_RENDERER = True
CAIRO_INCLUDES = '${MASON_LINKED_REL}/include'
CAIRO_LIBS = '${MASON_LINKED_REL}/lib'
SQLITE_INCLUDES = '${MASON_LINKED_REL}/include'
SQLITE_LIBS = '${MASON_LINKED_REL}/lib'
FRAMEWORK_PYTHON = False
BINDINGS = 'python'
XMLPARSER = 'ptree'
SVG2PNG = True
SAMPLE_INPUT_PLUGINS = True
" > ./config.py
}
function setup_runtime_settings() {
local MASON_LINKED_ABS=$(pwd)/mason_packages/.link
export PROJ_LIB=${MASON_LINKED_ABS}/share/proj/
export ICU_DATA=${MASON_LINKED_ABS}/share/icu/54.1/
export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal
}
function main() {
setup_mason
install_mason_deps
setup_nose
setup_cpp11_toolchain
make_config
setup_runtime_settings
}
main