Merge remote-tracking branch 'origin/master'

.gitmodules

@@ -5,7 +5,7 @@
 [submodule "test/data-visual"]
 	path = test/data-visual
 	url = https://github.com/mapnik/test-data-visual.git
-	branch = master
+	branch = harfbuzz-shaper
 [submodule "deps/mapbox/variant"]
 	path = deps/mapbox/variant
 	url = https://github.com/mapbox/variant.git

SConstruct

@@ -1787,7 +1787,7 @@ if not preconfigured:
         # Common flags for g++/clang++ CXX compiler.
         # TODO: clean up code more to make -Wextra -Wsign-compare -Wsign-conversion -Wconversion viable
         # -Wfloat-equal -Wold-style-cast -Wexit-time-destructors -Wglobal-constructors -Wreserved-id-macro -Wheader-hygiene -Wmissing-noreturn
-        common_cxx_flags = '-fvisibility=hidden -fvisibility-inlines-hidden -Wall %s %s -ftemplate-depth-300 -Wsign-compare -Wshadow ' % (env['WARNING_CXXFLAGS'], pthread)
+        common_cxx_flags = '-fvisibility=hidden -fvisibility-inlines-hidden -Wall %s %s -ftemplate-depth-300 -Wsign-compare ' % (env['WARNING_CXXFLAGS'], pthread)
 
         if 'clang++' in env['CXX']:
             common_cxx_flags += ' -Wno-unsequenced  -Wtautological-compare -Wheader-hygiene -Wc++14-extensions '
@@ -1799,7 +1799,7 @@ if not preconfigured:
             env.Append(CXXFLAGS = '-fsanitize=undefined-trap -fsanitize-undefined-trap-on-error -ftrapv -fwrapv')
 
         if env['DEBUG_SANITIZE']:
-            env.Append(CXXFLAGS = ['-fsanitize=address'])
+            env.Append(CXXFLAGS = ['-fsanitize=address','-fno-omit-frame-pointer'])
             env.Append(LINKFLAGS = ['-fsanitize=address'])
 
 

include/mapnik/expression_grammar_x3_def.hpp

@@ -24,6 +24,7 @@
 #define MAPNIK_EXPRESSIONS_GRAMMAR_X3_DEF_HPP
 
 #include <mapnik/expression_grammar_x3.hpp>
+#include <mapnik/json/unicode_string_grammar_x3_def.hpp>
 #include <mapnik/expression_node.hpp>
 #include <mapnik/function_call.hpp>
 #include <mapnik/unicode.hpp>
@@ -65,6 +66,15 @@ namespace mapnik { namespace grammar {
     using x3::alnum;
     x3::uint_parser<char, 16, 2, 2> const hex2 {};
 
+    namespace {
+    auto const& escaped_unicode = json::grammar::escaped_unicode;
+    }
+
+    auto append = [](auto const& ctx)
+    {
+        _val(ctx) += _attr(ctx);
+    };
+
     auto do_assign = [] (auto const& ctx)
     {
         _val(ctx) = std::move(_attr(ctx));
@@ -302,8 +312,20 @@ namespace mapnik { namespace grammar {
     x3::rule<class regex_replace_expression, std::pair<std::string,std::string> > const regex_replace_expression("regex replace expression");
 
     // strings
-    auto const single_quoted_string = x3::rule<class single_quoted_string, std::string> {} = lit('\'') >> no_skip[*(unesc_char | ("\\x" > hex2) | (char_ - '\''))] > '\'';
+    auto const single_quoted_string = x3::rule<class single_quoted_string, std::string> {} = lit('\'')
-    auto const double_quoted_string = x3::rule<class double_quoted_string, std::string> {} = lit('"') >> no_skip[*(unesc_char | ("\\x" > hex2) | (char_ - '"'))] > '"';
+        >> no_skip[*(unesc_char[append]
+                     |
+                     //(lit('\\') > escaped_unicode[append]) // FIXME (!)
+                     //|
+                     (~char_('\''))[append])] > lit('\'');
+
+    auto const double_quoted_string = x3::rule<class double_quoted_string, std::string> {} = lit('"')
+        >> no_skip[*(unesc_char[append]
+                     |
+                     (lit('\\') > escaped_unicode[append])
+                     |
+                     (~char_('"'))[append])] > lit('"');
+
     auto const quoted_string = x3::rule<class quoted_string, std::string> {} = single_quoted_string | double_quoted_string;
 
     auto const unquoted_ustring = x3::rule<class ustring, std::string> {} = no_skip[alpha > *alnum] - lit("not");

include/mapnik/json/unicode_string_grammar_x3_def.hpp

@@ -24,8 +24,9 @@
 #define MAPNIK_JSON_UNICODE_STRING_GRAMMAR_X3_DEF_HPP
 
 #include <mapnik/json/unicode_string_grammar_x3.hpp>
-#include <iostream>
+// boost
-
+#include <boost/regex/pending/unicode_iterator.hpp>
+//
 namespace mapnik { namespace json { namespace grammar {
 
 namespace x3 = boost::spirit::x3;
@@ -52,6 +53,25 @@ auto push_char = [](auto const& ctx) { _val(ctx).push_back(_attr(ctx));};
 
 auto push_utf8 = [](auto const& ctx) { detail::push_utf8_impl(_val(ctx), _attr(ctx));};
 
+auto push_utf16 = [](auto const& ctx)
+{
+    using iterator_type = std::vector<std::uint16_t>::const_iterator;
+    auto const& utf16 = _attr(ctx);
+    try
+    {
+        boost::u16_to_u32_iterator<iterator_type> itr(utf16.begin());
+        boost::u16_to_u32_iterator<iterator_type> end(utf16.end());
+        for (; itr != end; ++itr)
+        {
+            detail::push_utf8_impl(_val(ctx), *itr);
+        }
+    }
+    catch( ... )
+    {
+        // caught
+    }
+};
+
 auto push_esc = [] (auto const& ctx)
 {
     std::string & utf8 = _val(ctx);
@@ -85,29 +105,35 @@ using x3::eol;
 using x3::no_skip;
 
 x3::uint_parser<char,  16, 2, 2> const hex2 {};
-x3::uint_parser<uchar, 16, 4, 4> const hex4 {};
+x3::uint_parser<std::uint16_t, 16, 4, 4> const hex4 {};
 x3::uint_parser<uchar, 16, 8, 8> const hex8 {};
 
 // start rule
 unicode_string_grammar_type const unicode_string("Unicode String");
 // rules
 x3::rule<class double_quoted_tag, std::string> const double_quoted("Double-quoted string");
-x3::rule<class escaped_tag, std::string> const escaped("Escaped Characted");
+x3::rule<class escaped_tag, std::string> const escaped("Escaped Character");
+x3::rule<class escaped_unicode_tag, std::string> const escaped_unicode("Escaped Unicode code point(s)");
+x3::rule<class utf16_string_tag, std::vector<std::uint16_t>> const utf16_string("UTF16 encoded string");
 
 auto unicode_string_def = double_quoted
     ;
-auto const escaped_def = lit('\\') >
+auto utf16_string_def = lit('u') > hex4 > *(lit("\\u") > hex4)
-    ((lit('x') > hex2[push_char])
+    ;
+auto escaped_unicode_def =
+    (lit('x') > hex2[push_char])
     |
-     (lit('u') > hex4[push_utf8])
+    utf16_string[push_utf16]
     |
     (lit('U') > hex8[push_utf8])
+    ;
+auto const escaped_def = lit('\\') >
+    (escaped_unicode[append]
      |
      char_("0abtnvfre\"/\\N_LP \t")[push_esc]
      |
      eol) // continue to next line
     ;
-
 auto const double_quoted_def = lit('"') > no_skip[*(escaped[append] | (~char_('"'))[append])] > lit('"');
 
 #pragma GCC diagnostic push
@@ -116,7 +142,9 @@ auto const double_quoted_def = lit('"') > no_skip[*(escaped[append] | (~char_('"
 BOOST_SPIRIT_DEFINE(
     unicode_string,
     double_quoted,
-    escaped
+    escaped,
+    escaped_unicode,
+    utf16_string
     );
 
 #pragma GCC diagnostic pop

include/mapnik/text/face.hpp

@@ -72,10 +72,13 @@ public:
 
     bool glyph_dimensions(glyph_info &glyph) const;
 
+    inline bool is_color() const { return color_font_;}
+
     ~font_face();
 
 private:
     FT_Face face_;
+    bool color_font_ = false;
 };
 using face_ptr = std::shared_ptr<font_face>;
 

include/mapnik/text/harfbuzz_shaper.hpp

@@ -83,14 +83,13 @@ static void shape_text(text_line & line,
     const std::unique_ptr<hb_buffer_t, decltype(hb_buffer_deleter)> buffer(hb_buffer_create(), hb_buffer_deleter);
     hb_buffer_pre_allocate(buffer.get(), safe_cast<int>(length));
     mapnik::value_unicode_string const& text = itemizer.text();
-
     for (auto const& text_item : list)
     {
         face_set_ptr face_set = font_manager.get_face_set(text_item.format_->face_name, text_item.format_->fontset);
         double size = text_item.format_->text_size * scale_factor;
         face_set->set_unscaled_character_sizes();
         std::size_t num_faces = face_set->size();
-        std::size_t pos = 0;
+
         font_feature_settings const& ff_settings = text_item.format_->ff_settings;
         int ff_count = safe_cast<int>(ff_settings.count());
 
@@ -101,11 +100,13 @@ static void shape_text(text_line & line,
             hb_glyph_info_t glyph;
             hb_glyph_position_t position;
         };
+
         // this table is filled with information for rendering each glyph, so that
         // several font faces can be used in a single text_item
-        std::vector<glyph_face_info> glyphinfos;
+        std::size_t pos = 0;
-        unsigned valid_glyphs = 0;
+        std::vector<std::vector<glyph_face_info>> glyphinfos;
 
+        glyphinfos.resize(text.length());
         for (auto const& face : *face_set)
         {
             ++pos;
@@ -115,71 +116,101 @@ static void shape_text(text_line & line,
             hb_buffer_set_script(buffer.get(), _icu_script_to_script(text_item.script));
             hb_font_t *font(hb_ft_font_create(face->get_face(), nullptr));
             // https://github.com/mapnik/test-data-visual/pull/25
-            #if HB_VERSION_MAJOR > 0
+
-             #if HB_VERSION_ATLEAST(1, 0 , 5)
+#if (HB_VERSION_MAJOR > 0 && HB_VERSION_ATLEAST(1, 0, 5))
             hb_ft_font_set_load_flags(font,FT_LOAD_DEFAULT | FT_LOAD_NO_HINTING);
 #endif
-            #endif
             hb_shape(font, buffer.get(), ff_settings.get_features(), ff_count);
             hb_font_destroy(font);
 
             unsigned num_glyphs = hb_buffer_get_length(buffer.get());
+            hb_glyph_info_t *glyphs = hb_buffer_get_glyph_infos(buffer.get(), &num_glyphs);
+            hb_glyph_position_t *positions = hb_buffer_get_glyph_positions(buffer.get(), &num_glyphs);
 
-            // if the number of rendered glyphs has increased, we need to resize the table 
+            unsigned cluster = 0;
-            if (num_glyphs > glyphinfos.size())
+            bool in_cluster = false;
-            {
+            std::vector<unsigned> clusters;
-                glyphinfos.resize(num_glyphs);
-            }
 
-            hb_glyph_info_t *glyphs = hb_buffer_get_glyph_infos(buffer.get(), nullptr);
-            hb_glyph_position_t *positions = hb_buffer_get_glyph_positions(buffer.get(), nullptr);
-
-            // Check if all glyphs are valid.
             for (unsigned i = 0; i < num_glyphs; ++i)
             {
-                // if we have a valid codepoint, save rendering info.
+                if (i == 0)
-                if (glyphs[i].codepoint)
                 {
-                    if (!glyphinfos[i].glyph.codepoint)
+                    cluster = glyphs[0].cluster;
+                    clusters.push_back(cluster);
+                }
+                if (cluster != glyphs[i].cluster)
                 {
-                        ++valid_glyphs;
+                    cluster = glyphs[i].cluster;
+                    clusters.push_back(cluster);
+                    in_cluster = false;
                 }
-                    glyphinfos[i] = { face, glyphs[i], positions[i] };
+                else if (i != 0)
+                {
+                    in_cluster = true;
+                }
+                if (glyphinfos.size() <= cluster)
+                {
+                    glyphinfos.resize(cluster + 1);
+                }
+                auto & c = glyphinfos[cluster];
+                if (c.empty())
+                {
+                    c.push_back({face, glyphs[i], positions[i]});
+                }
+                else if (c.front().glyph.codepoint == 0)
+                {
+                    c.front() = { face, glyphs[i], positions[i] };
+                }
+                else if (in_cluster)
+                {
+                    c.push_back({ face, glyphs[i], positions[i] });
                 }
             }
-            if (valid_glyphs < num_glyphs && (pos < num_faces))
+            bool all_set = true;
+            for (auto c_id : clusters)
+            {
+                auto const& c = glyphinfos[c_id];
+                if (c.empty() || c.front().glyph.codepoint == 0)
+                {
+                    all_set = false;
+                    break;
+                }
+            }
+            if (!all_set && (pos < num_faces))
             {
                 //Try next font in fontset
                 continue;
             }
-
             double max_glyph_height = 0;
-            for (unsigned i=0; i<num_glyphs; ++i)
+            for (auto const& c_id : clusters)
             {
-                auto& gpos = positions[i];
+                auto const& c = glyphinfos[c_id];
-                auto& glyph = glyphs[i];
+                for (auto const& info : c)
-                face_ptr theface = face;
-                if (glyphinfos[i].glyph.codepoint)
                 {
-                    gpos = glyphinfos[i].position;
+                    auto const& gpos = info.position;
-                    glyph = glyphinfos[i].glyph;
+                    auto const& glyph = info.glyph;
-                    theface = glyphinfos[i].face;
-                }
                     unsigned char_index = glyph.cluster;
                     glyph_info g(glyph.codepoint,char_index,text_item.format_);
-                if (theface->glyph_dimensions(g))
+                    if (info.glyph.codepoint != 0) g.face = info.face;
+                    else g.face = face;
+                    if (g.face->glyph_dimensions(g))
                     {
-                    g.face = theface;
+                        g.scale_multiplier = g.face->get_face()->units_per_EM > 0 ?
-                    g.scale_multiplier = size / theface->get_face()->units_per_EM;
+                            (size / g.face->get_face()->units_per_EM) : (size / 2048.0) ;
                         //Overwrite default advance with better value provided by HarfBuzz
                         g.unscaled_advance = gpos.x_advance;
                         g.offset.set(gpos.x_offset * g.scale_multiplier, gpos.y_offset * g.scale_multiplier);
                         double tmp_height = g.height();
+                        if (g.face->is_color())
+                        {
+                            tmp_height = size;
+                        }
                         if (tmp_height > max_glyph_height) max_glyph_height = tmp_height;
                         width_map[char_index] += g.advance();
                         line.add_glyph(std::move(g), scale_factor);
                     }
                 }
+            }
             line.update_max_char_height(max_glyph_height);
             break; //When we reach this point the current font had all glyphs.
         }

include/mapnik/text/renderer.hpp

@@ -28,6 +28,7 @@
 #include <mapnik/image_compositing.hpp>
 #include <mapnik/symbolizer_enumerations.hpp>
 #include <mapnik/util/noncopyable.hpp>
+#include <mapnik/pixel_position.hpp>
 
 #pragma GCC diagnostic push
 #include <mapnik/warning_ignore.hpp>
@@ -51,8 +52,13 @@ struct glyph_t
 {
     FT_Glyph image;
     detail::evaluated_format_properties const& properties;
-    glyph_t(FT_Glyph image_, detail::evaluated_format_properties const& properties_)
+    pixel_position pos;
-        : image(image_), properties(properties_) {}
+    double size;
+    glyph_t(FT_Glyph image_, detail::evaluated_format_properties const& properties_, pixel_position const& pos_, double size_)
+        : image(image_),
+          properties(properties_),
+          pos(pos_),
+          size(size_) {}
 };
 
 class text_renderer : private util::noncopyable

include/mapnik/text/text_layout.hpp

@@ -143,8 +143,8 @@ private:
     // note: this probably isn't the best solution. it would be better to have an object for each cluster, but
     // it needs to be implemented with no overhead.
     std::map<unsigned, double> width_map_;
-    double width_;
+    double width_ = 0.0;
-    double height_;
+    double height_ = 0.0;
     unsigned glyphs_count_;
 
     // output
@@ -173,7 +173,7 @@ private:
     bool rotate_displacement_ = false;
     double text_ratio_ = 0.0;
     pixel_position displacement_ = {0,0};
-    box2d<double> bounds_;
+    box2d<double> bounds_ = {0, 0, 0, 0};
 
     // children
     text_layout_vector child_layout_list_;

include/mapnik/wkt/wkt_grammar_x3_def.hpp

@@ -26,10 +26,11 @@
 #include <mapnik/wkt/wkt_grammar_x3.hpp>
 #include <mapnik/geometry/fusion_adapted.hpp>
 
-#if defined(__GNUC__) && BOOST_VERSION < 106300
+#if defined(__GNUC__)
 // instantiate `is_substitute` for reference T and reference Attribute
-// fixes gcc6 compilation issue with boost 1_61 and boost_1_62
+// fixes gcc6 compilation issue
 namespace boost { namespace spirit { namespace x3 { namespace traits {
+
 template <typename T, typename Attribute, typename Enable>
 struct is_substitute<T&, Attribute&, Enable>
   : is_substitute<T, Attribute, Enable> {};

src/text/face.cpp

@@ -29,6 +29,7 @@
 extern "C"
 {
 #include FT_GLYPH_H
+#include FT_TRUETYPE_TABLES_H
 }
 
 #pragma GCC diagnostic pop
@@ -37,7 +38,13 @@ namespace mapnik
 {
 
 font_face::font_face(FT_Face face)
-    : face_(face) {}
+    : face_(face)
+{
+    static const uint32_t tag = FT_MAKE_TAG('C', 'B', 'D', 'T');
+    unsigned long length = 0;
+    FT_Load_Sfnt_Table(face_, tag, 0, nullptr, &length);
+    if (length) color_font_ = true;
+}
 
 bool font_face::set_character_sizes(double size)
 {
@@ -46,7 +53,8 @@ bool font_face::set_character_sizes(double size)
 
 bool font_face::set_unscaled_character_sizes()
 {
-    return (FT_Set_Char_Size(face_,0,face_->units_per_EM,0,0) == 0);
+    FT_F26Dot6 char_height = face_->units_per_EM > 0 ? face_->units_per_EM : 2048.0;
+    return (FT_Set_Char_Size(face_, 0, char_height, 0, 0) == 0);
 }
 
 bool font_face::glyph_dimensions(glyph_info & glyph) const
@@ -54,11 +62,14 @@ bool font_face::glyph_dimensions(glyph_info & glyph) const
     FT_Vector pen;
     pen.x = 0;
     pen.y = 0;
+    if (color_font_) FT_Select_Size(face_, 0);
     FT_Set_Transform(face_, 0, &pen);
-
+    FT_Int32 load_flags = FT_LOAD_DEFAULT | FT_LOAD_NO_HINTING;
-    if (FT_Load_Glyph(face_, glyph.glyph_index, FT_LOAD_NO_HINTING))
+    if (color_font_) load_flags |= FT_LOAD_COLOR ;
+    if (FT_Load_Glyph(face_, glyph.glyph_index, load_flags))
     {
-        MAPNIK_LOG_ERROR(font_face) << "FT_Load_Glyph failed";
+        MAPNIK_LOG_ERROR(font_face) << "FT_Load_Glyph failed :( index=" << glyph.glyph_index << " " << load_flags
+                                    << " " << face_->family_name << " " << face_->style_name  ;
         return false;
     }
     FT_Glyph image;

src/text/renderer.cpp

@@ -25,6 +25,8 @@
 #include <mapnik/grid/grid.hpp>
 #include <mapnik/text/text_properties.hpp>
 #include <mapnik/font_engine_freetype.hpp>
+#include <mapnik/image_compositing.hpp>
+#include <mapnik/image_scaling.hpp>
 #include <mapnik/text/face.hpp>
 #include <mapnik/image_util.hpp>
 #include <mapnik/image_any.hpp>
@@ -66,8 +68,35 @@ void text_renderer::prepare_glyphs(glyph_positions const& positions)
     for (auto const& glyph_pos : positions)
     {
         glyph_info const& glyph = glyph_pos.glyph;
-        glyph.face->set_character_sizes(glyph.format->text_size * scale_factor_); //TODO: Optimize this?
+        FT_Int32 load_flags = FT_LOAD_DEFAULT | FT_LOAD_NO_HINTING;
 
+        FT_Face face = glyph.face->get_face();
+        if (glyph.face->is_color())
+        {
+            load_flags |= FT_LOAD_COLOR ;
+            if (face->num_fixed_sizes > 0)
+            {
+                int scaled_size = static_cast<int>(glyph.format->text_size * scale_factor_);
+                int best_match = 0;
+                int diff = std::abs(scaled_size - face->available_sizes[0].width);
+                for (int i = 1; i < face->num_fixed_sizes; ++i)
+                {
+                    int ndiff = std::abs(scaled_size - face->available_sizes[i].height);
+                    if (ndiff < diff)
+                    {
+                        best_match = i;
+                        diff = ndiff;
+                    }
+                }
+                error = FT_Select_Size(face, best_match);
+            }
+        }
+        else
+        {
+            glyph.face->set_character_sizes(glyph.format->text_size * scale_factor_);
+        }
+
+        double size = glyph.format->text_size * scale_factor_;
         matrix.xx = static_cast<FT_Fixed>( glyph_pos.rot.cos * 0x10000L);
         matrix.xy = static_cast<FT_Fixed>(-glyph_pos.rot.sin * 0x10000L);
         matrix.yx = static_cast<FT_Fixed>( glyph_pos.rot.sin * 0x10000L);
@@ -77,17 +106,13 @@ void text_renderer::prepare_glyphs(glyph_positions const& positions)
         pen.x = static_cast<FT_Pos>(pos.x * 64);
         pen.y = static_cast<FT_Pos>(pos.y * 64);
 
-        FT_Face face = glyph.face->get_face();
         FT_Set_Transform(face, &matrix, &pen);
-
+        error = FT_Load_Glyph(face, glyph.glyph_index, load_flags);
-        error = FT_Load_Glyph(face, glyph.glyph_index, FT_LOAD_NO_HINTING);
         if (error) continue;
-
         FT_Glyph image;
         error = FT_Get_Glyph(face->glyph, &image);
         if (error) continue;
-
+        glyphs_.emplace_back(image, *glyph.format, pos, size);
-        glyphs_.emplace_back(image, *glyph.format);
     }
 }
 
@@ -110,6 +135,31 @@ void composite_bitmap(T & pixmap, FT_Bitmap *bitmap, unsigned rgba, int x, int y
     }
 }
 
+template <typename T>
+void composite_color_bitmap(T & pixmap, FT_Bitmap *bitmap, int x, int y, double size, double opacity, composite_mode_e comp_op)
+{
+  image_rgba8 image(bitmap->width, bitmap->rows);
+
+  for (unsigned i = 0, p = 0; i < bitmap->width; ++i, p += 4)
+  {
+      for (unsigned j = 0, q = 0; j < bitmap->rows; ++j, ++q)
+      {
+          std::uint8_t b = bitmap->buffer[q * bitmap->width * 4 + p];
+          std::uint8_t g = bitmap->buffer[q * bitmap->width * 4 + p + 1];
+          std::uint8_t r = bitmap->buffer[q * bitmap->width * 4 + p + 2];
+          std::uint8_t a = bitmap->buffer[q * bitmap->width * 4 + p + 3];
+          unsigned c = static_cast<unsigned>((a << 24) | (b << 16) | (g << 8) | (r));
+          image(i, j) = c;
+      }
+  }
+  double scale = size/image.height();
+  int scaled_width = bitmap->width * scale;
+  int scaled_height = bitmap->rows * scale;
+  image_rgba8 scaled_image(scaled_width, scaled_height);
+  scale_image_agg(scaled_image, image , SCALING_BILINEAR , scale, scale, 0.0, 0.0, 1.0, 0);
+  composite(pixmap, scaled_image, comp_op, opacity, x, y);
+}
+
 template <typename T>
 agg_text_renderer<T>::agg_text_renderer (pixmap_type & pixmap,
                                          halo_rasterizer_e rasterizer,
@@ -211,11 +261,30 @@ void agg_text_renderer<T>::render(glyph_positions const& pos)
     {
         fill = glyph.properties.fill.rgba();
         text_opacity = glyph.properties.text_opacity;
+
         FT_Glyph_Transform(glyph.image, &matrix, &start);
+        error = 0;
+        if ( glyph.image->format != FT_GLYPH_FORMAT_BITMAP )
+        {
+
             error = FT_Glyph_To_Bitmap(&glyph.image ,FT_RENDER_MODE_NORMAL, 0, 1);
-        if (!error)
+        }
+        if (error == 0)
         {
             FT_BitmapGlyph bit = reinterpret_cast<FT_BitmapGlyph>(glyph.image);
+            int pixel_mode = bit->bitmap.pixel_mode;
+            if (pixel_mode == 7)
+            {
+                int x = (start.x >> 6) + glyph.pos.x;
+                int y = height - (start.y >> 6) + glyph.pos.y;
+                composite_color_bitmap(pixmap_,
+                                       &bit->bitmap,
+                                       x,y, glyph.size,
+                                       text_opacity,
+                                       comp_op_);
+            }
+            else
+            {
                 composite_bitmap(pixmap_,
                                  &bit->bitmap,
                                  fill,
@@ -224,6 +293,7 @@ void agg_text_renderer<T>::render(glyph_positions const& pos)
                                  text_opacity,
                                  comp_op_);
             }
+        }
         FT_Done_Glyph(glyph.image);
     }
 
@@ -257,7 +327,6 @@ void grid_text_renderer<T>::render(glyph_positions const& pos, value_integer fea
         error = FT_Glyph_To_Bitmap(&glyph.image, FT_RENDER_MODE_NORMAL, 0, 1);
         if (!error)
         {
-
             FT_BitmapGlyph bit = reinterpret_cast<FT_BitmapGlyph>(glyph.image);
             render_halo_id(&bit->bitmap,
                            feature_id,

src/text/text_line.cpp

@@ -64,7 +64,7 @@ void text_line::add_glyph(glyph_info && glyph, double scale_factor_)
         // Only add character spacing if the character is not a zero-width part of a cluster.
         width_ += advance + glyphs_.back().format->character_spacing  * scale_factor_;
         glyphs_width_ += advance;
-        space_count_++;
+        ++space_count_;
     }
     glyphs_.emplace_back(std::move(glyph));
 }

test/standalone/font_registration_test.cpp

@@ -57,7 +57,7 @@ SECTION("registration") {
         mapnik::load_map_string(m3,"<Map font-directory=\"test/data/fonts/Noto/\"></Map>");
         REQUIRE( m3.get_font_memory_cache().size() == 0 );
         REQUIRE( m3.load_fonts() );
-        REQUIRE( m3.get_font_memory_cache().size() == 1 );
+        REQUIRE( m3.get_font_memory_cache().size() == 2 );
 
         std::vector<std::string> face_names;
         std::string foo("foo");

test/unit/core/expressions_test.cpp

@@ -183,7 +183,16 @@ TEST_CASE("expressions")
     auto val1 = eval("'♜♞♝♛♚♝♞♜'.replace('♞','♘')"); // ==> expected ♜♘♝♛♚♝♘♜
     TRY_CHECK(val0 == val1);
     TRY_CHECK(val0.to_string() == val1.to_string()); // UTF-8
-    TRY_CHECK(val0.to_unicode() == val1.to_unicode()); // Unicode (UTF-16)
+    TRY_CHECK(val0.to_unicode() == val1.to_unicode()); // Unicode
+    // \u+NNNN \U+NNNNNNNN \xNN\xNN
+    auto val3 = eval(u8"'\u262f\xF0\x9F\x8D\xB7'");
+    auto val4 = eval(u8"'\U0000262f\U0001F377'");
+    // UTF16 surrogate pairs work also ;)
+    // \ud83d\udd7a\ud83c\udffc => \U0001F57A\U0001F3FC works also
+    // TODO: find a way to enter UTF16 pairs
+    TRY_CHECK(val3 == val4);
+    TRY_CHECK(val3.to_string() == val4.to_string()); // UTF-8
+    TRY_CHECK(val3.to_unicode() == val4.to_unicode()); // Unicode
 
     // following test will fail if boost_regex is built without ICU support (unpaired surrogates in output)
     TRY_CHECK(eval("[name].replace('(\\B)|( )',' ') ") == tr.transcode("Q u é b e c"));

test/unit/text/shaping.cpp

@@ -1,25 +1,101 @@
-
 #include "catch.hpp"
 #include <mapnik/text/icu_shaper.hpp>
 #include <mapnik/text/harfbuzz_shaper.hpp>
 #include <mapnik/text/font_library.hpp>
+#include <mapnik/unicode.hpp>
 
-TEST_CASE("shapers compile") {
+namespace {
 
-    mapnik::text_line line(0,0);
+void test_shaping( mapnik::font_set const& fontset, mapnik::face_manager& fm,
-    mapnik::text_itemizer itemizer;
+                   std::vector<std::pair<unsigned, unsigned>> const& expected, char const* str, bool debug = false)
+{
+    mapnik::transcoder tr("utf8");
     std::map<unsigned,double> width_map;
+    mapnik::text_itemizer itemizer;
+    auto props = std::make_unique<mapnik::detail::evaluated_format_properties>();
+    props->fontset = fontset;
+    props->text_size = 32;
+
     double scale_factor = 1;
-    mapnik::font_library fl;
+    auto ustr = tr.transcode(str);
-    mapnik::freetype_engine::font_file_mapping_type font_file_mapping;
+    auto length = ustr.length();
-    mapnik::freetype_engine::font_memory_cache_type font_memory_cache;
+    itemizer.add_text(ustr, props);
-    mapnik::face_manager fm(fl,font_file_mapping,font_memory_cache);
+
+    mapnik::text_line line(0, length);
     mapnik::harfbuzz_shaper::shape_text(line, itemizer,
                                         width_map,
                                         fm,
                                         scale_factor);
-    mapnik::icu_shaper::shape_text(line,itemizer,
+
-                                width_map,
+    std::size_t index = 0;
-                                fm,
+    for (auto const& g : line)
-                                scale_factor);
+    {
+        if (debug)
+        {
+            if (index++ > 0) std::cerr << ",";
+            std::cerr << "{" << g.glyph_index << ", " << g.char_index
+                //<< ", " << g.face->family_name() << ":" << g.face->style_name()
+                      <<  "}";
+        }
+        else
+        {
+            unsigned glyph_index, char_index;
+            CHECK(index < expected.size());
+            std::tie(glyph_index, char_index) = expected[index++];
+            REQUIRE(glyph_index == g.glyph_index);
+            REQUIRE(char_index == g.char_index);
+        }
+    }
+}
+}
+
+TEST_CASE("shaping")
+{
+    mapnik::freetype_engine::register_font("test/data/fonts/NotoSans-Regular.ttc");
+    mapnik::freetype_engine::register_fonts("test/data/fonts/Noto");
+    mapnik::font_set fontset("fontset");
+    for (auto const& name : mapnik::freetype_engine::face_names())
+    {
+        fontset.add_face_name(name);
+    }
+
+    mapnik::font_library fl;
+    mapnik::freetype_engine::font_file_mapping_type font_file_mapping;
+    mapnik::freetype_engine::font_memory_cache_type font_memory_cache;
+    mapnik::face_manager fm(fl, font_file_mapping, font_memory_cache);
+
+    {
+        std::vector<std::pair<unsigned, unsigned>> expected =
+            {{977, 0}, {1094, 3}, {1038, 4}, {1168, 4}, {9, 7}, {3, 8}, {11, 9}, {68, 10}, {69, 11}, {70, 12}, {12, 13}};
+        test_shaping(fontset, fm, expected, u8"སྤུ་ཧྲེང (abc)");
+    }
+
+    {
+        std::vector<std::pair<unsigned, unsigned>> expected =
+            {{977, 0}, {1094, 3}, {1038, 4}, {1168, 4}, {9, 7}, {3, 8}, {11, 9}, {0, 10}, {0, 11}, {0, 12}, {12, 13}};
+        test_shaping(fontset, fm, expected, u8"སྤུ་ཧྲེང (普兰镇)");
+    }
+
+    {
+        std::vector<std::pair<unsigned, unsigned>> expected =
+            {{68, 0}, {69, 1}, {70, 2}, {3, 3}, {11, 4}, {0, 5}, {0, 6}, {0, 7}, {12, 8}};
+        test_shaping(fontset, fm, expected, u8"abc (普兰镇)");
+    }
+
+    {
+        std::vector<std::pair<unsigned, unsigned>> expected =
+            {{68, 0}, {69, 1}, {70, 2}, {3, 3}, {11, 4}, {68, 5}, {69, 6}, {70, 7}, {12, 8}};
+        test_shaping(fontset, fm, expected, "abc (abc)");
+    }
+
+    {
+        //    "ⵃⴰⵢ ⵚⵉⵏⴰⵄⵉ الحي الصناعي"
+        std::vector<std::pair<unsigned, unsigned>> expected =
+            {{0, 0}, {0, 1}, {0, 2}, {3, 3}, {0, 4}, {0, 5}, {0, 6}, {0, 7},
+              {0, 8}, {0, 9}, {3, 10}, {509, 22}, {481, 21}, {438, 20}, {503, 19},
+              {470, 18}, {496, 17}, {43, 16}, {3, 15}, {509, 14}, {454, 13}, {496, 12}, {43, 11}};
+        test_shaping(fontset, fm, expected, u8"ⵃⴰⵢ ⵚⵉⵏⴰⵄⵉ الحي الصناعي");
+    }
+
+
 }