From 89e516b49348775fca512718fb5afa16ecdbb122 Mon Sep 17 00:00:00 2001
From: artemp <artem@mapnik.org>
Date: Mon, 2 Feb 2015 11:20:23 +0100
Subject: [PATCH] unicode string grammar via
 boost/libs/spirit/example/qi/json/json/parser/grammar.hpp update json based
 grammars

---
 .../json/extract_bounding_box_grammar.hpp     |   2 -
 .../extract_bounding_box_grammar_impl.hpp     |  16 +--
 .../json/feature_collection_grammar.hpp       |   2 -
 include/mapnik/json/feature_grammar.hpp       |   3 -
 include/mapnik/json/feature_grammar_impl.hpp  |  16 ---
 include/mapnik/json/feature_parser.hpp        |   2 +-
 include/mapnik/json/generic_json.hpp          | 109 +++++++++++++++++-
 include/mapnik/json/geometry_grammar.hpp      |   3 +-
 include/mapnik/json/geometry_parser.hpp       |   2 +-
 include/mapnik/json/positions_grammar.hpp     |   3 +-
 include/mapnik/json/topojson_grammar.hpp      |   3 +-
 include/mapnik/json/topojson_grammar_impl.hpp |  20 +---
 plugins/input/geojson/geojson_datasource.cpp  |   6 +-
 .../geojson/large_geojson_featureset.cpp      |   2 +-
 .../input/topojson/topojson_datasource.cpp    |   2 +-
 15 files changed, 116 insertions(+), 75 deletions(-)

diff --git a/include/mapnik/json/extract_bounding_box_grammar.hpp b/include/mapnik/json/extract_bounding_box_grammar.hpp
index 6818f906d..e79ed00e1 100644
--- a/include/mapnik/json/extract_bounding_box_grammar.hpp
+++ b/include/mapnik/json/extract_bounding_box_grammar.hpp
@@ -46,8 +46,6 @@ using position = std::tuple<double,double>;
 using boxes = std::vector<std::pair<box2d<double>, std::pair<std::size_t, std::size_t>>>;
 
 namespace qi = boost::spirit::qi;
-namespace standard_wide =  boost::spirit::standard_wide;
-using standard_wide::space_type;
 
 struct calculate_bounding_box_impl
 {
diff --git a/include/mapnik/json/extract_bounding_box_grammar_impl.hpp b/include/mapnik/json/extract_bounding_box_grammar_impl.hpp
index fb68bad35..0624de3f8 100644
--- a/include/mapnik/json/extract_bounding_box_grammar_impl.hpp
+++ b/include/mapnik/json/extract_bounding_box_grammar_impl.hpp
@@ -48,7 +48,6 @@ extract_bounding_box_grammar<Iterator, ErrorHandler>::extract_bounding_box_gramm
     qi::_2_type _2;
     qi::_3_type _3;
     qi::_4_type _4;
-    qi::no_skip_type no_skip;
     qi::omit_type omit;
     qi::_r1_type _r1;
     qi::_r2_type _r2;
@@ -56,7 +55,7 @@ extract_bounding_box_grammar<Iterator, ErrorHandler>::extract_bounding_box_gramm
     qi::_b_type _b;
     qi::eps_type eps;
     qi::raw_type raw;
-    standard_wide::char_type char_;
+    ascii::char_type char_;
     boost::spirit::repository::qi::iter_pos_type iter_pos;
     using qi::fail;
     using qi::on_error;
@@ -109,19 +108,6 @@ extract_bounding_box_grammar<Iterator, ErrorHandler>::extract_bounding_box_gramm
         | lit("false")
         | lit("null")
         ;
-    json.unesc_char.add
-        ("\\\"", '\"') // quotation mark
-        ("\\\\", '\\') // reverse solidus
-        ("\\/", '/')   // solidus
-        ("\\b", '\b')  // backspace
-        ("\\f", '\f')  // formfeed
-        ("\\n", '\n')  // newline
-        ("\\r", '\r')  // carrige return
-        ("\\t", '\t')  // tab
-        ;
-    json.string_ %= lit('"') >> no_skip[*(json.unesc_char | "\\u" >> json.hex4 | (char_ - lit('"')))] >> lit('"')
-        ;
-
     coords.name("Coordinates");
     pos.name("Position");
     ring.name("Ring");
diff --git a/include/mapnik/json/feature_collection_grammar.hpp b/include/mapnik/json/feature_collection_grammar.hpp
index ad4c72c4b..33b82ffbe 100644
--- a/include/mapnik/json/feature_collection_grammar.hpp
+++ b/include/mapnik/json/feature_collection_grammar.hpp
@@ -37,8 +37,6 @@ namespace mapnik { namespace json {
 
 namespace qi = boost::spirit::qi;
 namespace phoenix = boost::phoenix;
-namespace standard_wide =  boost::spirit::standard_wide;
-using standard_wide::space_type;
 
 struct default_feature_callback
 {
diff --git a/include/mapnik/json/feature_grammar.hpp b/include/mapnik/json/feature_grammar.hpp
index a75a84f3a..b00e2b686 100644
--- a/include/mapnik/json/feature_grammar.hpp
+++ b/include/mapnik/json/feature_grammar.hpp
@@ -42,8 +42,6 @@ namespace mapnik { namespace json {
 namespace qi = boost::spirit::qi;
 namespace phoenix = boost::phoenix;
 namespace fusion = boost::fusion;
-namespace standard_wide =  boost::spirit::standard_wide;
-using standard_wide::space_type;
 
 class attribute_value_visitor
 
@@ -99,7 +97,6 @@ struct feature_grammar :
     // start
     // generic JSON
     generic_json<Iterator> json_;
-
     // geoJSON
     qi::rule<Iterator,void(FeatureType&),space_type> feature; // START
     qi::rule<Iterator,space_type> feature_type;
diff --git a/include/mapnik/json/feature_grammar_impl.hpp b/include/mapnik/json/feature_grammar_impl.hpp
index 51239a9a7..ae1532052 100644
--- a/include/mapnik/json/feature_grammar_impl.hpp
+++ b/include/mapnik/json/feature_grammar_impl.hpp
@@ -35,8 +35,6 @@ feature_grammar<Iterator,FeatureType,ErrorHandler>::feature_grammar(mapnik::tran
     qi::lit_type lit;
     qi::long_long_type long_long;
     qi::double_type double_;
-    qi::no_skip_type no_skip;
-    standard_wide::char_type char_;
     qi::_val_type _val;
     qi::_1_type _1;
     qi::_2_type _2;
@@ -77,20 +75,6 @@ feature_grammar<Iterator,FeatureType,ErrorHandler>::feature_grammar(mapnik::tran
         | lit("null")[_val = construct<value_null>()]
         ;
 
-    json_.unesc_char.add
-        ("\\\"", '\"') // quotation mark
-        ("\\\\", '\\') // reverse solidus
-        ("\\/", '/')   // solidus
-        ("\\b", '\b')  // backspace
-        ("\\f", '\f')  // formfeed
-        ("\\n", '\n')  // newline
-        ("\\r", '\r')  // carrige return
-        ("\\t", '\t')  // tab
-        ;
-
-    json_.string_ %= lit('"') >> no_skip[*(json_.unesc_char | "\\u" >> json_.hex4 | (char_ - lit('"')))] >> lit('"')
-        ;
-
     // geojson types
     feature_type = lit("\"type\"")
         >> lit(':')
diff --git a/include/mapnik/json/feature_parser.hpp b/include/mapnik/json/feature_parser.hpp
index c7ef79aee..c8afb6481 100644
--- a/include/mapnik/json/feature_parser.hpp
+++ b/include/mapnik/json/feature_parser.hpp
@@ -39,7 +39,7 @@ inline bool from_geojson(std::string const& json, mapnik::feature_impl & feature
     using iterator_type = std::string::const_iterator;
     static const mapnik::json::feature_grammar<iterator_type,mapnik::feature_impl> g(tr);
     using namespace boost::spirit;
-    standard_wide::space_type space;
+    ascii::space_type space;
     return qi::phrase_parse(json.begin(), json.end(), (g)(boost::phoenix::ref(feature)), space);
 }
 
diff --git a/include/mapnik/json/generic_json.hpp b/include/mapnik/json/generic_json.hpp
index fda147792..e86e9aa40 100644
--- a/include/mapnik/json/generic_json.hpp
+++ b/include/mapnik/json/generic_json.hpp
@@ -37,19 +37,116 @@
 namespace mapnik { namespace json {
 
 namespace qi = boost::spirit::qi;
-namespace standard_wide =  boost::spirit::standard_wide;
-using standard_wide::space_type;
-
+namespace ascii = boost::spirit::ascii;
+namespace phoenix = boost::phoenix;
+using space_type = ascii::space_type;
 using json_value = mapnik::util::variant<value_null,value_bool, value_integer, value_double, std::string>;
+using uchar = std::uint32_t; // a unicode code point
+
+// unicode string grammar via boost/libs/spirit/example/qi/json/json/parser/grammar.hpp
+
+template <typename Iterator>
+struct unicode_string : qi::grammar<Iterator, std::string()>
+{
+    unicode_string();
+    qi::rule<Iterator, void(std::string&)> escape;
+    qi::rule<Iterator, void(std::string&)> char_esc;
+    qi::rule<Iterator, std::string()> double_quoted;
+};
+
+
+struct push_utf8
+{
+    template <typename S, typename C>
+    struct result { typedef void type; };
+
+    void operator()(std::string& utf8, uchar code_point) const
+    {
+        typedef std::back_insert_iterator<std::string> insert_iter;
+        insert_iter out_iter(utf8);
+        boost::utf8_output_iterator<insert_iter> utf8_iter(out_iter);
+        *utf8_iter++ = code_point;
+    }
+};
+
+struct push_esc
+{
+    template <typename S, typename C>
+    struct result { typedef void type; };
+
+    void operator()(std::string& utf8, uchar c) const
+    {
+        switch (c)
+        {
+        case ' ': utf8 += ' ';          break;
+        case '\t': utf8 += '\t';        break;
+        case '0': utf8 += char(0);      break;
+        case 'a': utf8 += 0x7;          break;
+        case 'b': utf8 += 0x8;          break;
+        case 't': utf8 += 0x9;          break;
+        case 'n': utf8 += 0xA;          break;
+        case 'v': utf8 += 0xB;          break;
+        case 'f': utf8 += 0xC;          break;
+        case 'r': utf8 += 0xD;          break;
+        case 'e': utf8 += 0x1B;         break;
+        case '"': utf8 += '"';          break;
+        case '/': utf8 += '/';          break;
+        case '\\': utf8 += '\\';        break;
+        case '_': push_utf8()(utf8, 0xA0);  break;
+        case 'N': push_utf8()(utf8, 0x85);  break;
+        case 'L': push_utf8()(utf8, 0x2028);  break;
+        case 'P': push_utf8()(utf8, 0x2029);  break;
+        }
+    }
+};
+
+template< typename Iterator >
+unicode_string<Iterator>::unicode_string()
+    : unicode_string::base_type(double_quoted)
+{
+    qi::char_type char_;
+    qi::_val_type _val;
+    qi::_r1_type _r1;
+    qi::_1_type _1;
+    qi::lit_type lit;
+    qi::eol_type eol;
+    qi::repeat_type repeat;
+    qi::hex_type hex;
+
+    using boost::spirit::qi::uint_parser;
+    using boost::phoenix::function;
+    using boost::phoenix::ref;
+
+    uint_parser<uchar, 16, 4, 4> hex4;
+    uint_parser<uchar, 16, 8, 8> hex8;
+    function<push_utf8> push_utf8;
+    function<push_esc> push_esc;
+
+    escape =
+        ('x' > hex)                     [push_utf8(_r1, _1)]
+        |   ('u' > hex4)                    [push_utf8(_r1, _1)]
+        |   ('U' > hex8)                    [push_utf8(_r1, _1)]
+        |   char_("0abtnvfre\"/\\N_LP \t")  [push_esc(_r1, _1)]
+        |   eol                             // continue to next line
+        ;
+
+    char_esc =
+        '\\' > escape(_r1)
+        ;
+
+    double_quoted =
+              '"'
+        > *(char_esc(_val) | (~char_('"'))    [_val += _1])
+        > '"'
+        ;
+}
 
 template <typename Iterator>
 struct generic_json
 {
     qi::rule<Iterator,space_type> value;
-    qi::symbols<char const, char const> unesc_char;
-    qi::uint_parser< unsigned, 16, 4, 4 > hex4 ;
     qi::int_parser<mapnik::value_integer,10,1,-1> int__;
-    qi::rule<Iterator,std::string(), space_type> string_;
+    unicode_string<Iterator> string_;
     qi::rule<Iterator,space_type> key_value;
     qi::rule<Iterator,json_value(),space_type> number;
     qi::rule<Iterator,space_type> object;
diff --git a/include/mapnik/json/geometry_grammar.hpp b/include/mapnik/json/geometry_grammar.hpp
index 6d132cfea..f93e8a0ea 100644
--- a/include/mapnik/json/geometry_grammar.hpp
+++ b/include/mapnik/json/geometry_grammar.hpp
@@ -27,6 +27,7 @@
 #include <mapnik/geometry.hpp>  // for geometry_type
 #include <mapnik/vertex.hpp>  // for CommandType
 #include <mapnik/make_unique.hpp>
+#include <mapnik/json/generic_json.hpp>
 #include <mapnik/json/positions_grammar.hpp>
 #include <mapnik/json/geometry_util.hpp>
 #include <mapnik/geometry_container.hpp>
@@ -38,8 +39,6 @@
 namespace mapnik { namespace json {
 
 namespace qi = boost::spirit::qi;
-namespace standard_wide =  boost::spirit::standard_wide;
-using standard_wide::space_type;
 
 template <typename Iterator, typename ErrorHandler = error_handler<Iterator> >
 struct geometry_grammar :
diff --git a/include/mapnik/json/geometry_parser.hpp b/include/mapnik/json/geometry_parser.hpp
index 9d6fabe6b..80e0000e9 100644
--- a/include/mapnik/json/geometry_parser.hpp
+++ b/include/mapnik/json/geometry_parser.hpp
@@ -38,7 +38,7 @@ inline bool from_geojson(std::string const& json, geometry_container & paths)
 {
     using namespace boost::spirit;
     static const geometry_grammar<std::string::const_iterator> g;
-    standard_wide::space_type space;
+    ascii::space_type space;
     std::string::const_iterator start = json.begin();
     std::string::const_iterator end = json.end();
     return qi::phrase_parse(start, end, (g)(boost::phoenix::ref(paths)), space);
diff --git a/include/mapnik/json/positions_grammar.hpp b/include/mapnik/json/positions_grammar.hpp
index 32b1e2a84..a6ec17303 100644
--- a/include/mapnik/json/positions_grammar.hpp
+++ b/include/mapnik/json/positions_grammar.hpp
@@ -25,6 +25,7 @@
 
 // mapnik
 #include <mapnik/util/variant.hpp>
+#include <mapnik/json/generic_json.hpp>
 #include <mapnik/json/error_handler.hpp>
 
 // boost
@@ -46,8 +47,6 @@ using positions = std::vector<position>;
 using coordinates = util::variant<empty, position, positions, std::vector<positions>, std::vector<std::vector<positions> > > ;
 
 namespace qi = boost::spirit::qi;
-namespace standard_wide =  boost::spirit::standard_wide;
-using standard_wide::space_type;
 
 struct set_position_impl
 {
diff --git a/include/mapnik/json/topojson_grammar.hpp b/include/mapnik/json/topojson_grammar.hpp
index 944814f5e..881dcb594 100644
--- a/include/mapnik/json/topojson_grammar.hpp
+++ b/include/mapnik/json/topojson_grammar.hpp
@@ -44,8 +44,7 @@ namespace mapnik { namespace topojson {
 
 namespace qi = boost::spirit::qi;
 namespace fusion = boost::fusion;
-namespace standard_wide = boost::spirit::standard_wide;
-using standard_wide::space_type;
+using space_type = boost::spirit::ascii::space_type;
 
 template <typename Iterator, typename ErrorHandler = json::error_handler<Iterator> >
 struct topojson_grammar : qi::grammar<Iterator, space_type, topology()>
diff --git a/include/mapnik/json/topojson_grammar_impl.hpp b/include/mapnik/json/topojson_grammar_impl.hpp
index bb54680af..8e4a606fd 100644
--- a/include/mapnik/json/topojson_grammar_impl.hpp
+++ b/include/mapnik/json/topojson_grammar_impl.hpp
@@ -27,8 +27,8 @@ namespace mapnik { namespace topojson {
 namespace qi = boost::spirit::qi;
 namespace phoenix = boost::phoenix;
 namespace fusion = boost::fusion;
-namespace standard_wide = boost::spirit::standard_wide;
-using standard_wide::space_type;
+//namespace standard_wide = boost::spirit::standard_wide;
+//using standard_wide::space_type;
 
 template <typename Iterator, typename ErrorHandler>
 topojson_grammar<Iterator, ErrorHandler>::topojson_grammar()
@@ -37,7 +37,6 @@ topojson_grammar<Iterator, ErrorHandler>::topojson_grammar()
     qi::lit_type lit;
     qi::double_type double_;
     qi::int_type int_;
-    qi::no_skip_type no_skip;
     qi::omit_type omit;
     qi::_val_type _val;
     qi::_1_type _1;
@@ -45,7 +44,6 @@ topojson_grammar<Iterator, ErrorHandler>::topojson_grammar()
     qi::_3_type _3;
     qi::_4_type _4;
     qi::_r1_type _r1;
-    standard_wide::char_type char_;
     using qi::fail;
     using qi::on_error;
     using phoenix::push_back;
@@ -75,20 +73,6 @@ topojson_grammar<Iterator, ErrorHandler>::topojson_grammar()
         | lit("null")[_val = construct<value_null>()]
         ;
 
-    json_.unesc_char.add
-        ("\\\"", '\"') // quotation mark
-        ("\\\\", '\\') // reverse solidus
-        ("\\/", '/')   // solidus
-        ("\\b", '\b')  // backspace
-        ("\\f", '\f')  // formfeed
-        ("\\n", '\n')  // newline
-        ("\\r", '\r')  // carrige return
-        ("\\t", '\t')  // tab
-        ;
-
-    json_.string_ %= lit('"') >> no_skip[*(json_.unesc_char | "\\u" >> json_.hex4 | (char_ - lit('"')))] >> lit('"')
-        ;
-
     // topo json
     topology = lit('{') >> lit("\"type\"") >> lit(':') >> lit("\"Topology\"")
                         >> ( (lit(',') >> objects) ^ ( lit(',') >> arcs) ^ (lit(',') >> transform) ^ (lit(',') >> bbox))
diff --git a/plugins/input/geojson/geojson_datasource.cpp b/plugins/input/geojson/geojson_datasource.cpp
index 6b96865f2..21144c3ba 100644
--- a/plugins/input/geojson/geojson_datasource.cpp
+++ b/plugins/input/geojson/geojson_datasource.cpp
@@ -159,7 +159,7 @@ void geojson_datasource::initialise_index(Iterator start, Iterator end)
 {
     mapnik::json::boxes boxes;
     mapnik::json::extract_bounding_box_grammar<Iterator> bbox_grammar;
-    boost::spirit::standard_wide::space_type space;
+    boost::spirit::ascii::space_type space;
     if (!boost::spirit::qi::phrase_parse(start, end, (bbox_grammar)(boost::phoenix::ref(boxes)) , space))
     {
         throw mapnik::datasource_exception("GeoJSON Plugin: could not parse: '" + filename_ + "'");
@@ -194,7 +194,7 @@ void geojson_datasource::initialise_index(Iterator start, Iterator end)
             using namespace boost::spirit;
             static const mapnik::transcoder tr("utf8");
             static const mapnik::json::feature_grammar<chr_iterator_type,mapnik::feature_impl> grammar(tr);
-            standard_wide::space_type space;
+            ascii::space_type space;
             if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space))
             {
                 throw std::runtime_error("Failed to parse geojson feature");
@@ -216,7 +216,7 @@ void geojson_datasource::initialise_index(Iterator start, Iterator end)
 template <typename T>
 void geojson_datasource::parse_geojson(T const& buffer)
 {
-    boost::spirit::standard_wide::space_type space;
+    boost::spirit::ascii::space_type space;
     mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
     std::size_t start_id = 1;
 
diff --git a/plugins/input/geojson/large_geojson_featureset.cpp b/plugins/input/geojson/large_geojson_featureset.cpp
index fa17cef8a..043d3fe5a 100644
--- a/plugins/input/geojson/large_geojson_featureset.cpp
+++ b/plugins/input/geojson/large_geojson_featureset.cpp
@@ -69,7 +69,7 @@ mapnik::feature_ptr large_geojson_featureset::next()
         static const mapnik::transcoder tr("utf8");
         static const mapnik::json::feature_grammar<chr_iterator_type,mapnik::feature_impl> grammar(tr);
         using namespace boost::spirit;
-        standard_wide::space_type space;
+        ascii::space_type space;
         mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx_,1));
         if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space))
         {
diff --git a/plugins/input/topojson/topojson_datasource.cpp b/plugins/input/topojson/topojson_datasource.cpp
index 80baae2be..5c1c8d4af 100644
--- a/plugins/input/topojson/topojson_datasource.cpp
+++ b/plugins/input/topojson/topojson_datasource.cpp
@@ -190,7 +190,7 @@ const mapnik::topojson::topojson_grammar<base_iterator_type> g;
 template <typename T>
 void topojson_datasource::parse_topojson(T const& buffer)
 {
-    boost::spirit::standard_wide::space_type space;
+    boost::spirit::ascii::space_type space;
     bool result = boost::spirit::qi::phrase_parse(buffer.begin(), buffer.end(), g, space, topo_);
     if (!result)
     {