bootstrap cleanups

.travis.yml

@@ -11,7 +11,8 @@ install:
 
 script:
  - source bootstrap.sh
- - JOBS=4 make
+ - ./configure
+ - JOBS=6 make
  - git clone --depth=1 https://github.com/mapbox/mapnik-test-data tests/data/mapnik-test-data
  - make test
  #- source localize.sh && make grind

Makefile

@@ -13,6 +13,20 @@ install:
 	$(PYTHON) scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1 install
 
 mapnik:
+	# we first build memory intensive files with -j1
+	$(PYTHON) scons/scons.py -j1 \
+		--config=cache --implicit-cache --max-drift=1 \
+		src/json/libmapnik-json.a \
+		src/wkt/libmapnik-wkt.a \
+		src/css_color_grammar.os \
+		src/expression_grammar.os \
+		src/transform_expression_grammar.os \
+		src/image_filter_types.os \
+		src/renderer_common/process_group_symbolizer.cpp \
+		src/agg/process_markers_symbolizer.cpp \
+		src/grid/process_markers_symbolizer.cpp \
+		src/cairo/process_markers_symbolizer.cpp
+	# then install the rest with -j$(JOBS)
 	$(PYTHON) scons/scons.py -j$(JOBS) --config=cache --implicit-cache --max-drift=1
 
 clean:

bootstrap.sh

@@ -9,96 +9,16 @@
 
 todo
 
-- gdal shared lib
+- clang debs to s3
+- docs for base setup: sudo apt-get -y install zlib1g-dev python-dev make git python-dev
 - boost_python_patch
 - shrink icu data
-- cairo
+- cairo/pycairo
-- benchmarks not linking to chrono
 - clang + libc++
 - pkg-config-less
+- gdal shared lib?
 '
 
-TOOLCHAIN="$(pwd)/toolchain"
-PPA="https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test/+files"
-LLVM_DIST="http://llvm.org/apt/precise/pool/main/l/llvm-toolchain-3.5"
-SECURITY="http://security.ubuntu.com/ubuntu/pool/main/e/eglibc/"
-
-function setup_linux_cpp11_toolchain() {
-  if [[ ! -d ${TOOLCHAIN} ]]; then
-      wget -q ${PPA}/libstdc%2B%2B6_4.8.1-2ubuntu1~12.04_amd64.deb
-      dpkg -x libstdc++6_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN}
-      wget -q ${PPA}/gcc-4.8_4.8.1-2ubuntu1~12.04_amd64.deb
-      dpkg -x gcc-4.8_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN}
-      wget -q ${PPA}/g%2B%2B-4.8_4.8.1-2ubuntu1~12.04_amd64.deb
-      dpkg -x g++-4.8_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN}
-      wget -q ${PPA}/libisl10_0.12.2-2~12.04_amd64.deb
-      dpkg -x libisl10_0.12.2-2~12.04_amd64.deb ${TOOLCHAIN}
-      wget -q ${PPA}/libcloog-isl4_0.18.2-1~12.04_amd64.deb
-      dpkg -x libcloog-isl4_0.18.2-1~12.04_amd64.deb ${TOOLCHAIN}
-      wget -q ${PPA}/libstdc%2B%2B-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb
-      dpkg -x libstdc++-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN}
-      ${PPA}/gcc-4.8-base_4.8.1-2ubuntu1~12.04_amd64.deb
-      dpkg -x gcc-4.8-base_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN}
-      wget -q ${PPA}/libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb
-      dpkg -x libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN}
-      wget -q ${PPA}/libgcc1_4.8.1-2ubuntu1~12.04_amd64.deb
-      dpkg -x libgcc1_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN}
-      wget -q ${PPA}/cpp-4.8_4.8.1-2ubuntu1~12.04_amd64.deb
-      dpkg -x cpp-4.8_4.8.1-2ubuntu1~12.04_amd64.deb ${TOOLCHAIN}
-  fi
-}
-
-function dpack() {
-  wget -q $1/$(echo $2 | sed 's/+/%2B/g')
-  dpkg -x $2 ${TOOLCHAIN}
-}
-
-function test_cpp11() {
-  echo '#include <iostream>' > test.cpp
-  echo ' ' >> test.cpp
-  echo 'int main() {' >> test.cpp
-  echo ' ' >> test.cpp
-  echo 'std::move("hello");' >> test.cpp
-  echo 'return 0;' >> test.cpp
-  echo '}' >> test.cpp
-  echo ' ' >> test.cpp
-  # ensure no gcc 4.6 paths and that
-  # "./toolchain/usr/bin/../lib/gcc/x86_64-linux-gnu/4.8" is chosen
-  ./toolchain/usr/bin/clang++-3.5 -o test test.cpp -std=c++11 -v
-}
-
-# http://llvm.org/apt/precise/dists/llvm-toolchain-precise-3.5/main/binary-amd64/Packages
-function setup_clang_toolchain() {
-    dpack ${LLVM_DIST} clang-3.5_3.5~svn217304-1~exp1_amd64.deb
-    dpack ${LLVM_DIST} libllvm3.5_3.5~svn217304-1~exp1_amd64.deb
-    dpack ${LLVM_DIST} libclang-common-3.5-dev_3.5~svn215019-1~exp1_amd64.deb
-    dpack ${PPA} libstdc++6_4.8.1-2ubuntu1~12.04_amd64.deb
-    dpack ${PPA} libstdc++-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb
-    dpack ${PPA} libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb
-    #dpack ${PPA} cpp-4.8_4.8.1-2ubuntu1~12.04_amd64.deb
-    #dpack ${PPA} g++-4.8-multilib_4.8.1-2ubuntu1~12.04_amd64.deb
-    #dpack ${PPA} gcc-4.8_4.8.1-2ubuntu1~12.04_amd64.deb
-    #dpack ${PPA} gcc-4.8-base_4.8.1-2ubuntu1~12.04_amd64.deb
-    #dpack ${PPA} libgcc1_4.8.1-2ubuntu1~12.04_amd64.deb
-    #dpack ${LLVM_DIST} libclang1-3.5_3.5~svn215019-1~exp1_amd64.deb
-    #dpack ${LLVM_DIST} libclang-3.5-dev_3.5~svn215019-1~exp1_amd64.deb
-    #dpack ${LLVM_DIST} llvm-3.5-runtime_3.5~svn215019-1~exp1_amd64.deb
-    #dpack ${PPA} g++-4.8_4.8.1-2ubuntu1~12.04_amd64.deb
-    #dpack ${PPA} libisl10_0.12.2-2~12.04_amd64.deb
-    #dpack ${PPA} libcloog-isl4_0.18.2-1~12.04_amd64.deb
-    #dpack ${PPA} g++-4.8_4.8.1-2ubuntu1~12.04_amd64.deb
-    #dpack ${PPA} gcc-4.8-locales_4.8.1-2ubuntu1~12.04_all.deb
-    #export CPLUS_INCLUDE_PATH="${TOOLCHAIN}/usr/include/:${TOOLCHAIN}/usr/include/x86_64-linux-gnu:${TOOLCHAIN}/usr/include/c++/4.8:${TOOLCHAIN}/usr/include/x86_64-linux-gnu/c++/4.8:${CPLUS_INCLUDE_PATH}"
-    #dpack ${SECURITY} libc6-dev_2.15-0ubuntu10.9_amd64.deb
-    #wget -q http://mirrors.kernel.org/ubuntu/pool/main/l/linux/linux-libc-dev_3.2.0-75.110_amd64.deb
-    #dpkg -x linux-libc-dev_3.2.0-75.110_amd64.deb ${TOOLCHAIN}
-    #wget -q http://mirrors.kernel.org/ubuntu/pool/main/b/binutils/binutils-gold_2.22-6ubuntu1.1_amd64.deb
-    #dpkg -x binutils-gold_2.22-6ubuntu1.1_amd64.deb ${TOOLCHAIN}
-    #wget -q http://security.ubuntu.com/ubuntu/pool/main/b/binutils/binutils_2.22-6ubuntu1.1_amd64.deb
-    #dpkg -x binutils_2.22-6ubuntu1.1_amd64.deb ${TOOLCHAIN}
-    # build-essential gets: binutils build-essential cpp cpp-4.6 dpkg-dev fakeroot g++ g++-4.6 gcc gcc-4.6 libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl libc-dev-bin libc6-dev libdpkg-perl libgomp1 libmpc2 libmpfr4 libquadmath0 libstdc++6-4.6-dev linux-libc-dev make manpages-dev
-}
-
 declare -A DEPS
 DEPS["freetype"]="2.5.4"
 DEPS["harfbuzz"]="2cd5323"
@@ -116,98 +36,140 @@ DEPS["boost_libfilesystem"]="1.57.0"
 DEPS["boost_libprogram_options"]="1.57.0"
 DEPS["boost_libregex"]="1.57.0"
 DEPS["boost_libpython"]="1.57.0"
-DEPS["gdal"]="1.11.1"
 DEPS["libpq"]="9.4.0"
 DEPS["sqlite"]="3.8.6"
-# gdal deps
+DEPS["gdal"]="1.11.1"
 DEPS["expat"]="2.1.0"
 
+CPP11_TOOLCHAIN="$(pwd)/toolchain"
 
-if [[ -d ~/.mason ]]; then
+function dpack() {
-    export PATH=~/.mason:$PATH
+    if [[ ! -f $2 ]]; then
-else
+        wget -q $1/$(echo $2 | sed 's/+/%2B/g')
-    if [[ ! -d mason ]]; then
+        dpkg -x $2 ${CPP11_TOOLCHAIN}
-        git clone --depth 1 https://github.com/mapbox/mason.git ./.mason
     fi
-    export MASON_DIR=$(pwd)/.mason
+}
-    export PATH=$(pwd)/.mason:$PATH
-fi
 
-for DEP in "${!DEPS[@]}"; do
+function setup_cpp11_toolchain() {
-    mason install ${DEP} ${DEPS[$DEP]}
+    if [[ $(uname -s) == 'Linux' ]]; then
-done
+        local PPA="https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test/+files"
+        # http://llvm.org/apt/precise/dists/llvm-toolchain-precise-3.5/main/binary-amd64/Packages
+        # TODO: cache these for faster downloads
+        local LLVM_DIST="http://llvm.org/apt/precise/pool/main/l/llvm-toolchain-3.5"
+        dpack ${LLVM_DIST} clang-3.5_3.5~svn217304-1~exp1_amd64.deb
+        dpack ${LLVM_DIST} libllvm3.5_3.5~svn217304-1~exp1_amd64.deb
+        dpack ${LLVM_DIST} libclang-common-3.5-dev_3.5~svn215019-1~exp1_amd64.deb
+        dpack ${PPA} libstdc++6_4.8.1-2ubuntu1~12.04_amd64.deb
+        dpack ${PPA} libstdc++-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb
+        dpack ${PPA} libgcc-4.8-dev_4.8.1-2ubuntu1~12.04_amd64.deb
+        export CPLUS_INCLUDE_PATH="${CPP11_TOOLCHAIN}/usr/include/c++/4.8:${CPP11_TOOLCHAIN}/usr/include/x86_64-linux-gnu/c++/4.8:${CPLUS_INCLUDE_PATH}"
+        export LD_LIBRARY_PATH="${CPP11_TOOLCHAIN}/usr/lib/x86_64-linux-gnu:${CPP11_TOOLCHAIN}/usr/lib/gcc/x86_64-linux-gnu/4.8/:${LD_LIBRARY_PATH}"
+        export LIBRARY_PATH="${LD_LIBRARY_PATH}"
+        export PATH="${CPP11_TOOLCHAIN}/usr/bin":${PATH}
+        export CXX="${CPP11_TOOLCHAIN}/usr/bin/clang++-3.5"
+        export CC="${CPP11_TOOLCHAIN}/usr/bin/clang-3.5"
+    else
+        export CXX=clang++
+        export CC=clang
+    fi
+}
 
-for DEP in "${!DEPS[@]}"; do
+function setup_mason() {
-    mason link ${DEP} ${DEPS[$DEP]}
+    if [[ -d ~/.mason ]]; then
-done
+        export PATH=~/.mason:$PATH
+    else
+        if [[ ! -d mason ]]; then
+            git clone --depth 1 https://github.com/mapbox/mason.git ./.mason
+        fi
+        export MASON_DIR=$(pwd)/.mason
+        export PATH=$(pwd)/.mason:$PATH
+    fi
+}
 
-MASON_LINKED=$(pwd)/mason_packages/.link
+function install_mason_deps() {
+    if [[ ! -d ./mason_packages ]]; then
+        for DEP in "${!DEPS[@]}"; do
+            mason install ${DEP} ${DEPS[$DEP]}
+        done
+    fi
+    if [[ ! -d ./mason_packages/.link ]]; then
+        for DEP in "${!DEPS[@]}"; do
+            mason link ${DEP} ${DEPS[$DEP]}
+        done
+    fi
+}
 
-export PROJ_LIB=${MASON_LINKED}/share/proj/
+function setup_nose() {
-export ICU_DATA=${MASON_LINKED}/share/icu/54.1/
+    if [[ ! -d $(pwd)/nose-1.3.4 ]]; then
-export GDAL_DATA=${MASON_LINKED}/share/gdal
+        wget -q https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz
-export PKG_CONFIG_PATH="${MASON_LINKED}/lib/pkgconfig"
+        tar -xzf nose-1.3.4.tar.gz
-export C_INCLUDE_PATH="${MASON_LINKED}/include"
+    fi
-export CPLUS_INCLUDE_PATH="${MASON_LINKED}/include"
+    export PYTHONPATH=$(pwd)/nose-1.3.4:${PYTHONPATH}
-export LIBRARY_PATH="${MASON_LINKED}/lib"
+}
-export PATH="${MASON_LINKED}/bin":${PATH}
 
-if [[ $(uname -s) == 'Linux' ]]; then
+function make_config() {
-  #setup_linux_cpp11_toolchain
+    local MASON_LINKED=./mason_packages/.link
-  setup_clang_toolchain
+    export PROJ_LIB=${MASON_LINKED}/share/proj/
-  export CPLUS_INCLUDE_PATH="${TOOLCHAIN}/usr/include/c++/4.8:${TOOLCHAIN}/usr/include/x86_64-linux-gnu/c++/4.8:${CPLUS_INCLUDE_PATH}"
+    export ICU_DATA=${MASON_LINKED}/share/icu/54.1/
-  export LD_LIBRARY_PATH="${TOOLCHAIN}/usr/lib/x86_64-linux-gnu:${TOOLCHAIN}/usr/lib/gcc/x86_64-linux-gnu/4.8/:${LD_LIBRARY_PATH}"
+    export GDAL_DATA=${MASON_LINKED}/share/gdal
-  export LIBRARY_PATH="${LD_LIBRARY_PATH}"
+    export PKG_CONFIG_PATH="${MASON_LINKED}/lib/pkgconfig"
-  export PATH=${TOOLCHAIN}/usr/bin:$PATH
+    export C_INCLUDE_PATH="${MASON_LINKED}/include"
-  #sudo apt-get -y install zlib1g-dev python-dev make git python-dev python-nose
+    export CPLUS_INCLUDE_PATH="${MASON_LINKED}/include"
-fi
+    export LIBRARY_PATH="${MASON_LINKED}/lib"
+    export PATH="${MASON_LINKED}/bin":${PATH}
 
-wget -q https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz
+    local CUSTOM_CXXFLAGS="-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1"
-tar -xzf nose-1.3.4.tar.gz
+    local MASON_LIBS="${MASON_LINKED}/lib"
-export PYTHONPATH=$(pwd)/nose-1.3.4:${PYTHONPATH}
+    local MASON_INCLUDES="${MASON_LINKED}/include"
+    echo "
+CUSTOM_CXXFLAGS = '-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1'
+CUSTOM_LDFLAGS = '-L${MASON_LINKED}/lib'
+RUNTIME_LINK = 'static'
+INPUT_PLUGINS = 'csv,gdal,geojson,occi,ogr,osm,pgraster,postgis,python,raster,rasterlite,shape,sqlite,topojson'
+PREFIX = '/opt/mapnik-3.x'
+PATH = '${MASON_LINKED}/bin'
+PATH_REMOVE = '/usr:/usr/local'
+MAPNIK_NAME = 'mapnik_3-0-0'
+BOOST_INCLUDES = '${MASON_LINKED}/include'
+BOOST_LIBS = '${MASON_LINKED}/lib'
+BOOST_PYTHON_LIB = 'boost_python-2.7'
+ICU_INCLUDES = '${MASON_LINKED}/include'
+ICU_LIBS = '${MASON_LINKED}/lib'
+HB_INCLUDES = '${MASON_LINKED}/include'
+HB_LIBS = '${MASON_LINKED}/lib'
+PNG_INCLUDES = '${MASON_LINKED}/include/libpng16'
+PNG_LIBS = '${MASON_LINKED}/lib'
+JPEG_INCLUDES = '${MASON_LINKED}/include'
+JPEG_LIBS = '${MASON_LINKED}/lib'
+TIFF_INCLUDES = '${MASON_LINKED}/include'
+TIFF_LIBS = '${MASON_LINKED}/lib'
+WEBP_INCLUDES = '${MASON_LINKED}/include'
+WEBP_LIBS = '${MASON_LINKED}/lib'
+PROJ_INCLUDES = '${MASON_LINKED}/include'
+PROJ_LIBS = '${MASON_LINKED}/lib'
+FREETYPE_INCLUDES = '${MASON_LINKED}/include/freetype2'
+FREETYPE_LIBS = '${MASON_LINKED}/lib'
+XML2_INCLUDES = '${MASON_LINKED}/include/libxml2'
+XML2_LIBS = '${MASON_LINKED}/lib'
+SVG_RENDERER = True
+CAIRO_INCLUDES = '${MASON_LINKED}/include'
+CAIRO_LIBS = '${MASON_LINKED}/lib'
+SQLITE_INCLUDES = '${MASON_LINKED}/include'
+SQLITE_LIBS = '${MASON_LINKED}/lib'
+FRAMEWORK_PYTHON = False
+BINDINGS = 'python'
+XMLPARSER = 'ptree'
+SVG2PNG = True
+SAMPLE_INPUT_PLUGINS = True
+" > ./config.py
+}
 
-CXXFLAGS="-fvisibility=hidden -fvisibility-inlines-hidden -DU_CHARSET_IS_UTF8=1"
+function main() {
-MASON_LIBS="${MASON_LINKED}/lib"
+    setup_mason
-MASON_INCLUDES="${MASON_LINKED}/include"
+    install_mason_deps
+    setup_nose
+    setup_cpp11_toolchain
+    make_config
+}
 
-./configure \
+main
-  CC=${TOOLCHAIN}/usr/bin/clang-3.5 \
-  CXX=${TOOLCHAIN}/usr/bin/clang++-3.5 \
-  INPUT_PLUGINS=all \
-  SAMPLE_INPUT_PLUGINS=True \
-  SVG2PNG=True \
-  BENCHMARK=False \
-  SVG_RENDERER=True \
-  PATH="${MASON_LINKED}/bin" \
-  PATH_REMOVE=/usr:/usr/local \
-  RUNTIME_LINK=static \
-  CUSTOM_LDFLAGS="-L${MASON_LIBS}" \
-  CUSTOM_CXXFLAGS="${CXXFLAGS}" \
-  BOOST_INCLUDES="${MASON_INCLUDES}" \
-  BOOST_LIBS="${MASON_LIBS}" \
-  ICU_INCLUDES="${MASON_INCLUDES}" \
-  ICU_LIBS="${MASON_LIBS}" \
-  FREETYPE_INCLUDES="${MASON_INCLUDES}/freetype2" \
-  FREETYPE_LIBS="${MASON_LIBS}" \
-  XML2_INCLUDES="${MASON_INCLUDES}/libxml2" \
-  XML2_LIBS="${MASON_LIBS}" \
-  PNG_INCLUDES="${MASON_INCLUDES}/libpng16" \
-  PNG_LIBS="${MASON_LIBS}" \
-  JPEG_INCLUDES="${MASON_INCLUDES}" \
-  JPEG_LIBS="${MASON_LIBS}" \
-  WEBP_INCLUDES="${MASON_INCLUDES}" \
-  WEBP_LIBS="${MASON_LIBS}" \
-  TIFF_INCLUDES="${MASON_INCLUDES}" \
-  TIFF_LIBS="${MASON_LIBS}" \
-  PROJ_INCLUDES="${MASON_INCLUDES}" \
-  PROJ_LIBS="${MASON_LIBS}" \
-  CAIRO_INCLUDES="${MASON_INCLUDES}" \
-  CAIRO_LIBS="${MASON_LIBS}" \
-  SQLITE_INCLUDES="${MASON_INCLUDES}" \
-  SQLITE_LIBS="${MASON_LIBS}" \
-  HB_INCLUDES="${MASON_INCLUDES}" \
-  HB_LIBS="${MASON_LIBS}" \
-  SQLITE_INCLUDES="${MASON_INCLUDES}" \
-  SQLITE_LIBS="${MASON_LIBS}" \
-  SQLITE_INCLUDES="${MASON_INCLUDES}" \
-  SQLITE_LIBS="${MASON_LIBS}" \