diff --git a/.travis.yml b/.travis.yml index b52df1bc6..5074f5658 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,8 @@ install: script: - source bootstrap.sh - - JOBS=4 make + - ./configure + - JOBS=6 make - git clone --depth=1 https://github.com/mapbox/mapnik-test-data tests/data/mapnik-test-data - make test #- source localize.sh && make grind diff --git a/Makefile b/Makefile index 0ae5d4160..e0e229dc5 100755 --- a/Makefile +++ b/Makefile @@ -13,6 +13,20 @@ install: $(PYTHON) scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1 install mapnik: + # we first build memory intensive files with -j1 + $(PYTHON) scons/scons.py -j1 \ + --config=cache --implicit-cache --max-drift=1 \ + src/json/libmapnik-json.a \ + src/wkt/libmapnik-wkt.a \ + src/css_color_grammar.os \ + src/expression_grammar.os \ + src/transform_expression_grammar.os \ + src/image_filter_types.os \ + src/renderer_common/process_group_symbolizer.cpp \ + src/agg/process_markers_symbolizer.cpp \ + src/grid/process_markers_symbolizer.cpp \ + src/cairo/process_markers_symbolizer.cpp + # then install the rest with -j$(JOBS) $(PYTHON) scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1 clean: diff --git a/bootstrap.sh b/bootstrap.sh index aee8a79c5..ca9f60a95 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -9,96 +9,16 @@ todo -- gdal shared lib +- clang debs to s3 +- docs for base setup: sudo apt-get -y install zlib1g-dev python-dev make git python-dev - boost_python_patch - shrink icu data -- cairo -- benchmarks not linking to chrono +- cairo/pycairo - clang + libc++ - pkg-config-less +- gdal shared lib? ' -TOOLCHAIN="$(pwd)/toolchain" -PPA="https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test/+files" -LLVM_DIST="http://llvm.org/apt/precise/pool/main/l/llvm-toolchain-3.5" -SECURITY="http://security.ubuntu.com/ubuntu/pool/main/e/eglibc/" - -function setup_linux_cpp11_toolchain() { - if [[ ! -d ${TOOLCHAIN} ]]; then - wget -q ${PPA}/libstdc%2B%2B6_4.8.1-2ubuntu1~12.04_amd64.deb - dpkg -x libstdc++6_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN} - wget -q ${PPA}/gcc-4.8_4.8.1-2ubuntu1~12.04_amd64.deb - dpkg -x gcc-4.8_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN} - wget -q ${PPA}/g%2B%2B-4.8_4.8.1-2ubuntu1~12.04_amd64.deb - dpkg -x g++-4.8_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN} - wget -q ${PPA}/libisl10_0.12.2-2~12.04_amd64.deb - dpkg -x libisl10_0.12.2-2~12.04_amd64.deb ${TOOLCHAIN} - wget -q ${PPA}/libcloog-isl4_0.18.2-1~12.04_amd64.deb - dpkg -x libcloog-isl4_0.18.2-1~12.04_amd64.deb ${TOOLCHAIN} - wget -q ${PPA}/libstdc%2B%2B-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb - dpkg -x libstdc++-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN} - ${PPA}/gcc-4.8-base_4.8.1-2ubuntu1~12.04_amd64.deb - dpkg -x gcc-4.8-base_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN} - wget -q ${PPA}/libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb - dpkg -x libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN} - wget -q ${PPA}/libgcc1_4.8.1-2ubuntu1~12.04_amd64.deb - dpkg -x libgcc1_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN} - wget -q ${PPA}/cpp-4.8_4.8.1-2ubuntu1~12.04_amd64.deb - dpkg -x cpp-4.8_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN} - fi -} - -function dpack() { - wget -q $1/$(echo $2 | sed 's/+/%2B/g') - dpkg -x $2 ${TOOLCHAIN} -} - -function test_cpp11() { - echo '#include ' > test.cpp - echo ' ' >> test.cpp - echo 'int main() {' >> test.cpp - echo ' ' >> test.cpp - echo 'std::move("hello");' >> test.cpp - echo 'return 0;' >> test.cpp - echo '}' >> test.cpp - echo ' ' >> test.cpp - # ensure no gcc 4.6 paths and that - # "./toolchain/usr/bin/../lib/gcc/x86_64-linux-gnu/4.8" is chosen - ./toolchain/usr/bin/clang++-3.5 -o test test.cpp -std=c++11 -v -} - -# http://llvm.org/apt/precise/dists/llvm-toolchain-precise-3.5/main/binary-amd64/Packages -function setup_clang_toolchain() { - dpack ${LLVM_DIST} clang-3.5_3.5~svn217304-1~exp1_amd64.deb - dpack ${LLVM_DIST} libllvm3.5_3.5~svn217304-1~exp1_amd64.deb - dpack ${LLVM_DIST} libclang-common-3.5-dev_3.5~svn215019-1~exp1_amd64.deb - dpack ${PPA} libstdc++6_4.8.1-2ubuntu1~12.04_amd64.deb - dpack ${PPA} libstdc++-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb - dpack ${PPA} libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb - #dpack ${PPA} cpp-4.8_4.8.1-2ubuntu1~12.04_amd64.deb - #dpack ${PPA} g++-4.8-multilib_4.8.1-2ubuntu1~12.04_amd64.deb - #dpack ${PPA} gcc-4.8_4.8.1-2ubuntu1~12.04_amd64.deb - #dpack ${PPA} gcc-4.8-base_4.8.1-2ubuntu1~12.04_amd64.deb - #dpack ${PPA} libgcc1_4.8.1-2ubuntu1~12.04_amd64.deb - #dpack ${LLVM_DIST} libclang1-3.5_3.5~svn215019-1~exp1_amd64.deb - #dpack ${LLVM_DIST} libclang-3.5-dev_3.5~svn215019-1~exp1_amd64.deb - #dpack ${LLVM_DIST} llvm-3.5-runtime_3.5~svn215019-1~exp1_amd64.deb - #dpack ${PPA} g++-4.8_4.8.1-2ubuntu1~12.04_amd64.deb - #dpack ${PPA} libisl10_0.12.2-2~12.04_amd64.deb - #dpack ${PPA} libcloog-isl4_0.18.2-1~12.04_amd64.deb - #dpack ${PPA} g++-4.8_4.8.1-2ubuntu1~12.04_amd64.deb - #dpack ${PPA} gcc-4.8-locales_4.8.1-2ubuntu1~12.04_all.deb - #export CPLUS_INCLUDE_PATH="${TOOLCHAIN}/usr/include/:${TOOLCHAIN}/usr/include/x86_64-linux-gnu:${TOOLCHAIN}/usr/include/c++/4.8:${TOOLCHAIN}/usr/include/x86_64-linux-gnu/c++/4.8:${CPLUS_INCLUDE_PATH}" - #dpack ${SECURITY} libc6-dev_2.15-0ubuntu10.9_amd64.deb - #wget -q http://mirrors.kernel.org/ubuntu/pool/main/l/linux/linux-libc-dev_3.2.0-75.110_amd64.deb - #dpkg -x linux-libc-dev_3.2.0-75.110_amd64.deb ${TOOLCHAIN} - #wget -q http://mirrors.kernel.org/ubuntu/pool/main/b/binutils/binutils-gold_2.22-6ubuntu1.1_amd64.deb - #dpkg -x binutils-gold_2.22-6ubuntu1.1_amd64.deb ${TOOLCHAIN} - #wget -q http://security.ubuntu.com/ubuntu/pool/main/b/binutils/binutils_2.22-6ubuntu1.1_amd64.deb - #dpkg -x binutils_2.22-6ubuntu1.1_amd64.deb ${TOOLCHAIN} - # build-essential gets: binutils build-essential cpp cpp-4.6 dpkg-dev fakeroot g++ g++-4.6 gcc gcc-4.6 libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl libc-dev-bin libc6-dev libdpkg-perl libgomp1 libmpc2 libmpfr4 libquadmath0 libstdc++6-4.6-dev linux-libc-dev make manpages-dev -} - declare -A DEPS DEPS["freetype"]="2.5.4" DEPS["harfbuzz"]="2cd5323" @@ -116,98 +36,140 @@ DEPS["boost_libfilesystem"]="1.57.0" DEPS["boost_libprogram_options"]="1.57.0" DEPS["boost_libregex"]="1.57.0" DEPS["boost_libpython"]="1.57.0" -DEPS["gdal"]="1.11.1" DEPS["libpq"]="9.4.0" DEPS["sqlite"]="3.8.6" -# gdal deps +DEPS["gdal"]="1.11.1" DEPS["expat"]="2.1.0" +CPP11_TOOLCHAIN="$(pwd)/toolchain" -if [[ -d ~/.mason ]]; then - export PATH=~/.mason:$PATH -else - if [[ ! -d mason ]]; then - git clone --depth 1 https://github.com/mapbox/mason.git ./.mason +function dpack() { + if [[ ! -f $2 ]]; then + wget -q $1/$(echo $2 | sed 's/+/%2B/g') + dpkg -x $2 ${CPP11_TOOLCHAIN} fi - export MASON_DIR=$(pwd)/.mason - export PATH=$(pwd)/.mason:$PATH -fi +} -for DEP in "${!DEPS[@]}"; do - mason install ${DEP} ${DEPS[$DEP]} -done +function setup_cpp11_toolchain() { + if [[ $(uname -s) == 'Linux' ]]; then + local PPA="https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test/+files" + # http://llvm.org/apt/precise/dists/llvm-toolchain-precise-3.5/main/binary-amd64/Packages + # TODO: cache these for faster downloads + local LLVM_DIST="http://llvm.org/apt/precise/pool/main/l/llvm-toolchain-3.5" + dpack ${LLVM_DIST} clang-3.5_3.5~svn217304-1~exp1_amd64.deb + dpack ${LLVM_DIST} libllvm3.5_3.5~svn217304-1~exp1_amd64.deb + dpack ${LLVM_DIST} libclang-common-3.5-dev_3.5~svn215019-1~exp1_amd64.deb + dpack ${PPA} libstdc++6_4.8.1-2ubuntu1~12.04_amd64.deb + dpack ${PPA} libstdc++-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb + dpack ${PPA} libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb + export CPLUS_INCLUDE_PATH="${CPP11_TOOLCHAIN}/usr/include/c++/4.8:${CPP11_TOOLCHAIN}/usr/include/x86_64-linux-gnu/c++/4.8:${CPLUS_INCLUDE_PATH}" + export LD_LIBRARY_PATH="${CPP11_TOOLCHAIN}/usr/lib/x86_64-linux-gnu:${CPP11_TOOLCHAIN}/usr/lib/gcc/x86_64-linux-gnu/4.8/:${LD_LIBRARY_PATH}" + export LIBRARY_PATH="${LD_LIBRARY_PATH}" + export PATH="${CPP11_TOOLCHAIN}/usr/bin":${PATH} + export CXX="${CPP11_TOOLCHAIN}/usr/bin/clang++-3.5" + export CC="${CPP11_TOOLCHAIN}/usr/bin/clang-3.5" + else + export CXX=clang++ + export CC=clang + fi +} -for DEP in "${!DEPS[@]}"; do - mason link ${DEP} ${DEPS[$DEP]} -done +function setup_mason() { + if [[ -d ~/.mason ]]; then + export PATH=~/.mason:$PATH + else + if [[ ! -d mason ]]; then + git clone --depth 1 https://github.com/mapbox/mason.git ./.mason + fi + export MASON_DIR=$(pwd)/.mason + export PATH=$(pwd)/.mason:$PATH + fi +} -MASON_LINKED=$(pwd)/mason_packages/.link +function install_mason_deps() { + if [[ ! -d ./mason_packages ]]; then + for DEP in "${!DEPS[@]}"; do + mason install ${DEP} ${DEPS[$DEP]} + done + fi + if [[ ! -d ./mason_packages/.link ]]; then + for DEP in "${!DEPS[@]}"; do + mason link ${DEP} ${DEPS[$DEP]} + done + fi +} -export PROJ_LIB=${MASON_LINKED}/share/proj/ -export ICU_DATA=${MASON_LINKED}/share/icu/54.1/ -export GDAL_DATA=${MASON_LINKED}/share/gdal -export PKG_CONFIG_PATH="${MASON_LINKED}/lib/pkgconfig" -export C_INCLUDE_PATH="${MASON_LINKED}/include" -export CPLUS_INCLUDE_PATH="${MASON_LINKED}/include" -export LIBRARY_PATH="${MASON_LINKED}/lib" -export PATH="${MASON_LINKED}/bin":${PATH} +function setup_nose() { + if [[ ! -d $(pwd)/nose-1.3.4 ]]; then + wget -q https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz + tar -xzf nose-1.3.4.tar.gz + fi + export PYTHONPATH=$(pwd)/nose-1.3.4:${PYTHONPATH} +} -if [[ $(uname -s) == 'Linux' ]]; then - #setup_linux_cpp11_toolchain - setup_clang_toolchain - export CPLUS_INCLUDE_PATH="${TOOLCHAIN}/usr/include/c++/4.8:${TOOLCHAIN}/usr/include/x86_64-linux-gnu/c++/4.8:${CPLUS_INCLUDE_PATH}" - export LD_LIBRARY_PATH="${TOOLCHAIN}/usr/lib/x86_64-linux-gnu:${TOOLCHAIN}/usr/lib/gcc/x86_64-linux-gnu/4.8/:${LD_LIBRARY_PATH}" - export LIBRARY_PATH="${LD_LIBRARY_PATH}" - export PATH=${TOOLCHAIN}/usr/bin:$PATH - #sudo apt-get -y install zlib1g-dev python-dev make git python-dev python-nose -fi +function make_config() { + local MASON_LINKED=./mason_packages/.link + export PROJ_LIB=${MASON_LINKED}/share/proj/ + export ICU_DATA=${MASON_LINKED}/share/icu/54.1/ + export GDAL_DATA=${MASON_LINKED}/share/gdal + export PKG_CONFIG_PATH="${MASON_LINKED}/lib/pkgconfig" + export C_INCLUDE_PATH="${MASON_LINKED}/include" + export CPLUS_INCLUDE_PATH="${MASON_LINKED}/include" + export LIBRARY_PATH="${MASON_LINKED}/lib" + export PATH="${MASON_LINKED}/bin":${PATH} -wget -q https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz -tar -xzf nose-1.3.4.tar.gz -export PYTHONPATH=$(pwd)/nose-1.3.4:${PYTHONPATH} + local CUSTOM_CXXFLAGS="-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1" + local MASON_LIBS="${MASON_LINKED}/lib" + local MASON_INCLUDES="${MASON_LINKED}/include" + echo " +CUSTOM_CXXFLAGS = '-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1' +CUSTOM_LDFLAGS = '-L${MASON_LINKED}/lib' +RUNTIME_LINK = 'static' +INPUT_PLUGINS = 'csv,gdal,geojson,occi,ogr,osm,pgraster,postgis,python,raster,rasterlite,shape,sqlite,topojson' +PREFIX = '/opt/mapnik-3.x' +PATH = '${MASON_LINKED}/bin' +PATH_REMOVE = '/usr:/usr/local' +MAPNIK_NAME = 'mapnik_3-0-0' +BOOST_INCLUDES = '${MASON_LINKED}/include' +BOOST_LIBS = '${MASON_LINKED}/lib' +BOOST_PYTHON_LIB = 'boost_python-2.7' +ICU_INCLUDES = '${MASON_LINKED}/include' +ICU_LIBS = '${MASON_LINKED}/lib' +HB_INCLUDES = '${MASON_LINKED}/include' +HB_LIBS = '${MASON_LINKED}/lib' +PNG_INCLUDES = '${MASON_LINKED}/include/libpng16' +PNG_LIBS = '${MASON_LINKED}/lib' +JPEG_INCLUDES = '${MASON_LINKED}/include' +JPEG_LIBS = '${MASON_LINKED}/lib' +TIFF_INCLUDES = '${MASON_LINKED}/include' +TIFF_LIBS = '${MASON_LINKED}/lib' +WEBP_INCLUDES = '${MASON_LINKED}/include' +WEBP_LIBS = '${MASON_LINKED}/lib' +PROJ_INCLUDES = '${MASON_LINKED}/include' +PROJ_LIBS = '${MASON_LINKED}/lib' +FREETYPE_INCLUDES = '${MASON_LINKED}/include/freetype2' +FREETYPE_LIBS = '${MASON_LINKED}/lib' +XML2_INCLUDES = '${MASON_LINKED}/include/libxml2' +XML2_LIBS = '${MASON_LINKED}/lib' +SVG_RENDERER = True +CAIRO_INCLUDES = '${MASON_LINKED}/include' +CAIRO_LIBS = '${MASON_LINKED}/lib' +SQLITE_INCLUDES = '${MASON_LINKED}/include' +SQLITE_LIBS = '${MASON_LINKED}/lib' +FRAMEWORK_PYTHON = False +BINDINGS = 'python' +XMLPARSER = 'ptree' +SVG2PNG = True +SAMPLE_INPUT_PLUGINS = True +" > ./config.py +} -CXXFLAGS="-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1" -MASON_LIBS="${MASON_LINKED}/lib" -MASON_INCLUDES="${MASON_LINKED}/include" +function main() { + setup_mason + install_mason_deps + setup_nose + setup_cpp11_toolchain + make_config +} -./configure \ - CC=${TOOLCHAIN}/usr/bin/clang-3.5 \ - CXX=${TOOLCHAIN}/usr/bin/clang++-3.5 \ - INPUT_PLUGINS=all \ - SAMPLE_INPUT_PLUGINS=True \ - SVG2PNG=True \ - BENCHMARK=False \ - SVG_RENDERER=True \ - PATH="${MASON_LINKED}/bin" \ - PATH_REMOVE=/usr:/usr/local \ - RUNTIME_LINK=static \ - CUSTOM_LDFLAGS="-L${MASON_LIBS}" \ - CUSTOM_CXXFLAGS="${CXXFLAGS}" \ - BOOST_INCLUDES="${MASON_INCLUDES}" \ - BOOST_LIBS="${MASON_LIBS}" \ - ICU_INCLUDES="${MASON_INCLUDES}" \ - ICU_LIBS="${MASON_LIBS}" \ - FREETYPE_INCLUDES="${MASON_INCLUDES}/freetype2" \ - FREETYPE_LIBS="${MASON_LIBS}" \ - XML2_INCLUDES="${MASON_INCLUDES}/libxml2" \ - XML2_LIBS="${MASON_LIBS}" \ - PNG_INCLUDES="${MASON_INCLUDES}/libpng16" \ - PNG_LIBS="${MASON_LIBS}" \ - JPEG_INCLUDES="${MASON_INCLUDES}" \ - JPEG_LIBS="${MASON_LIBS}" \ - WEBP_INCLUDES="${MASON_INCLUDES}" \ - WEBP_LIBS="${MASON_LIBS}" \ - TIFF_INCLUDES="${MASON_INCLUDES}" \ - TIFF_LIBS="${MASON_LIBS}" \ - PROJ_INCLUDES="${MASON_INCLUDES}" \ - PROJ_LIBS="${MASON_LIBS}" \ - CAIRO_INCLUDES="${MASON_INCLUDES}" \ - CAIRO_LIBS="${MASON_LIBS}" \ - SQLITE_INCLUDES="${MASON_INCLUDES}" \ - SQLITE_LIBS="${MASON_LIBS}" \ - HB_INCLUDES="${MASON_INCLUDES}" \ - HB_LIBS="${MASON_LIBS}" \ - SQLITE_INCLUDES="${MASON_INCLUDES}" \ - SQLITE_LIBS="${MASON_LIBS}" \ - SQLITE_INCLUDES="${MASON_INCLUDES}" \ - SQLITE_LIBS="${MASON_LIBS}" \ +main