From aa77eb4550d29158c74d479f873d672ba156aa3b Mon Sep 17 00:00:00 2001 From: artemp Date: Tue, 9 Aug 2016 16:13:12 +0100 Subject: [PATCH 1/6] Expressions - add more regex tests to cover #3483 --- src/expression_node.cpp | 4 ++-- test/unit/core/expressions_test.cpp | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/expression_node.cpp b/src/expression_node.cpp index 380c36a4e..549f4c891 100644 --- a/src/expression_node.cpp +++ b/src/expression_node.cpp @@ -130,9 +130,9 @@ value regex_replace_node::apply(value const& v) const auto const& pattern = impl_.get()->pattern_; auto const& format = impl_.get()->format_; #if defined(BOOST_REGEX_HAS_ICU) - return boost::u32regex_replace(v.to_unicode(),pattern,format); + return boost::u32regex_replace(v.to_unicode(), pattern, format); #else - std::string repl = boost::regex_replace(v.to_string(),pattern,format); + std::string repl = boost::regex_replace(v.to_string(), pattern, format); transcoder tr_("utf8"); return tr_.transcode(repl.c_str()); #endif diff --git a/test/unit/core/expressions_test.cpp b/test/unit/core/expressions_test.cpp index e60d6127b..9f6a919f3 100644 --- a/test/unit/core/expressions_test.cpp +++ b/test/unit/core/expressions_test.cpp @@ -1,4 +1,3 @@ - #include "catch_ext.hpp" #include @@ -176,6 +175,18 @@ TEST_CASE("expressions") // regex // replace TRY_CHECK(eval(" [foo].replace('(\\B)|( )','$1 ') ") == tr.transcode("b a r")); + + // https://en.wikipedia.org/wiki/Chess_symbols_in_Unicode + //'\u265C\u265E\u265D\u265B\u265A\u265D\u265E\u265C' - black chess figures + // replace black knights with white knights + auto val0 = eval(u8"'\u265C\u265E\u265D\u265B\u265A\u265D\u265E\u265C'.replace('\u265E','\u2658')"); + auto val1 = eval(u8"'♜♞♝♛♚♝♞♜'.replace('♞','♘')"); // ==> expected ♜♘♝♛♚♝♘♜ + TRY_CHECK(val0 == val1); + TRY_CHECK(val0.to_string() == val1.to_string()); + + // following test will fail if boost_regex is built without ICU support (unpaired surrogates in output) + TRY_CHECK(eval("[name].replace('(\\B)|( )',' ') ") == tr.transcode(u8"Q u é b e c")); + TRY_CHECK(eval("'Москва'.replace('(? Date: Tue, 9 Aug 2016 16:25:59 +0100 Subject: [PATCH 2/6] Expressions - more tests (ref #3483) --- test/unit/core/expressions_test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/core/expressions_test.cpp b/test/unit/core/expressions_test.cpp index 9f6a919f3..c10982f78 100644 --- a/test/unit/core/expressions_test.cpp +++ b/test/unit/core/expressions_test.cpp @@ -182,7 +182,8 @@ TEST_CASE("expressions") auto val0 = eval(u8"'\u265C\u265E\u265D\u265B\u265A\u265D\u265E\u265C'.replace('\u265E','\u2658')"); auto val1 = eval(u8"'♜♞♝♛♚♝♞♜'.replace('♞','♘')"); // ==> expected ♜♘♝♛♚♝♘♜ TRY_CHECK(val0 == val1); - TRY_CHECK(val0.to_string() == val1.to_string()); + TRY_CHECK(val0.to_string() == val1.to_string()); // UTF-8 + TRY_CHECK(val0.to_unicode() == val1.to_unicode()); // Unicode (UTF-16) // following test will fail if boost_regex is built without ICU support (unpaired surrogates in output) TRY_CHECK(eval("[name].replace('(\\B)|( )',' ') ") == tr.transcode(u8"Q u é b e c")); From f56c45b86bc70c3cfd12e399716720eaecf6be20 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 9 Aug 2016 10:41:38 -0700 Subject: [PATCH 3/6] [bootstrap] upgrade to boost_regex package with icu + better error handling --- bootstrap.sh | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index fd877a644..6f4ba7ba7 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -#set -eu +set -eu +set -o pipefail : ' @@ -10,7 +11,16 @@ todo - shrink icu data ' -MASON_VERSION="b709931" +# allow sourcing of script without +# causing the terminal to bail on error +function finish { + set +eu + set +o pipefail +} + +trap finish EXIT + +MASON_VERSION="7ed8931" function setup_mason() { if [[ ! -d ./.mason ]]; then @@ -33,7 +43,7 @@ function install() { if [[ ${3:-false} != false ]]; then LA_FILE=$(mason prefix $1 $2)/lib/$3.la if [[ -f ${LA_FILE} ]]; then - perl -i -p -e 's:\Q$ENV{HOME}/build/mapbox/mason\E:$ENV{PWD}:g' ${LA_FILE} + perl -i -p -e 's:\Q$ENV{HOME}/build/mapbox/mason\E:$ENV{PWD}:g' ${LA_FILE} else echo "$LA_FILE not found" fi @@ -44,6 +54,7 @@ function install() { ICU_VERSION="55.1" function install_mason_deps() { + FAIL=0 install ccache 3.2.4 & install jpeg_turbo 1.4.0 libjpeg & install libpng 1.6.20 libpng & @@ -51,7 +62,6 @@ function install_mason_deps() { install libpq 9.4.1 & install sqlite 3.8.8.3 libsqlite3 & install expat 2.1.0 libexpat & - wait install icu ${ICU_VERSION} & install proj 4.8.0 libproj & install pixman 0.32.6 libpixman-1 & @@ -59,17 +69,22 @@ function install_mason_deps() { install protobuf 2.6.1 & # technically protobuf is not a mapnik core dep, but installing # here by default helps make mapnik-vector-tile builds easier - wait install webp 0.4.2 libwebp & install gdal 1.11.2 libgdal & install boost 1.61.0 & install boost_libsystem 1.61.0 & install boost_libfilesystem 1.61.0 & install boost_libprogram_options 1.61.0 & - install boost_libregex 1.61.0 & + install boost_libregex_icu 1.61.0 & install freetype 2.6 libfreetype & install harfbuzz 0.9.41 libharfbuzz & - wait + for job in $(jobs -p) + do + wait $job || let "FAIL+=1" + done + if [[ "$FAIL" != "0" ]]; then + exit ${FAIL} + fi } MASON_LINKED_ABS=$(pwd)/mason_packages/.link From 8e84b6d853d2653195c1773f9d8501c357c8de46 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 9 Aug 2016 10:46:18 -0700 Subject: [PATCH 4/6] stop caching mason_packages --- circle.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/circle.yml b/circle.yml index cf3e31cb1..a627527a9 100644 --- a/circle.yml +++ b/circle.yml @@ -21,7 +21,6 @@ dependencies: cache_directories: - "~/.ccache" - "~/.apt-cache" - - "mason_packages" pre: # https://discuss.circleci.com/t/add-ability-to-cache-apt-get-programs/598/3 - sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial From 181feef2534e3717bdca02e5f21b83f0dd2836bd Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 9 Aug 2016 12:17:39 -0700 Subject: [PATCH 5/6] bootstrap: exit trap not called when sourcing --- bootstrap.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 6f4ba7ba7..b0ebcf813 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -11,15 +11,6 @@ todo - shrink icu data ' -# allow sourcing of script without -# causing the terminal to bail on error -function finish { - set +eu - set +o pipefail -} - -trap finish EXIT - MASON_VERSION="7ed8931" function setup_mason() { @@ -155,3 +146,8 @@ function main() { } main + +# allow sourcing of script without +# causing the terminal to bail on error +set +eu +set +o pipefail From fc52a818d838109144c5d89ec492299e0900c453 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 9 Aug 2016 14:30:13 -0700 Subject: [PATCH 6/6] trigger new build