Merge branch 'master' into harfbuzz

Conflicts:
	bindings/python/mapnik/__init__.py

CHANGELOG.md

@@ -8,6 +8,8 @@ For a complete change history, see the git log.
 
 ## Future
 
+- Improved detection of newlines in CSV files - now more robust in the face of mixed newline types (#1497)
+
 - Allow style level compositing operations to work outside of featureset extents across tiled requests (#1477)
 
 - Support for encoding `literal` postgres types as strings 69fb17cd3/#1466

bindings/python/mapnik/__init__.py

@@ -784,6 +784,7 @@ class _TextSymbolizer(TextSymbolizer,_injector):
     def wrap_before(self, wrap_before):
         self.properties.wrap_before = wrap_before
 
+
     @property
     def text_transform(self):
         return self.format.text_transform

include/mapnik/json/geometry_parser.hpp

@@ -40,7 +40,7 @@ template <typename Iterator> struct geometry_grammar;
 MAPNIK_DECL bool from_geojson(std::string const& json, boost::ptr_vector<geometry_type> & paths);
 
 template <typename Iterator>
-class geometry_parser : private boost::noncopyable
+class MAPNIK_DECL geometry_parser : private boost::noncopyable
 {
     typedef Iterator iterator_type;
 public:

include/mapnik/wkt/wkt_factory.hpp

@@ -42,7 +42,7 @@ MAPNIK_DECL bool from_wkt(std::string const& wkt, boost::ptr_vector<geometry_typ
 
 #if BOOST_VERSION >= 104700
 
-class wkt_parser : boost::noncopyable
+class MAPNIK_DECL wkt_parser : boost::noncopyable
 {
     typedef std::string::const_iterator iterator_type;
 public:

plugins/input/csv/csv_datasource.cpp

@@ -172,33 +172,19 @@ void csv_datasource::parse_csv(T & stream,
     // autodetect newlines
     char newline = '\n';
     bool has_newline = false;
-    int newline_count = 0;
-    int carriage_count = 0;
-    for (unsigned idx = 0; idx < file_length_; idx++)
+    for (unsigned lidx = 0; lidx < file_length_ && lidx < 4000; lidx++)
     {
         char c = static_cast<char>(stream.get());
+        if (c == '\r')
+        {
+            newline = '\r';
+            has_newline = true;
+            break;
+        }
         if (c == '\n')
         {
-            ++newline_count;
             has_newline = true;
-        }
-        else if (c == '\r')
-        {
-            ++carriage_count;
-            has_newline = true;
-        }
-        // read at least 2000 bytes before testing
-        if (idx == file_length_-1 || idx > 4000)
-        {
-            if (newline_count > carriage_count)
-            {
-                break;
-            }
-            else if (carriage_count > newline_count)
-            {
-                newline = '\r';
-                break;
-            }
+            break;
         }
     }
 

src/agg/process_debug_symbolizer.cpp

@@ -22,9 +22,7 @@
 
 // mapnik
 #include <mapnik/agg_renderer.hpp>
-#include <mapnik/agg_rasterizer.hpp>
 #include <mapnik/graphics.hpp>
-#include "agg_renderer_primitives.h"
 
 namespace mapnik {
 

src/image_scaling.cpp

@@ -264,7 +264,8 @@ void scale_image_agg(Image & target,
                      double ratio)
 {
     typedef agg::pixfmt_rgba32 pixfmt;
-    typedef agg::renderer_base<pixfmt> renderer_base;
+    typedef agg::pixfmt_rgba32_pre pixfmt_pre;
+    typedef agg::renderer_base<pixfmt_pre> renderer_base;
 
     // define some stuff we'll use soon
     agg::rasterizer_scanline_aa<> ras;
@@ -280,7 +281,7 @@ void scale_image_agg(Image & target,
 
     // initialize destination AGG buffer (with transparency)
     agg::rendering_buffer rbuf_dst((unsigned char*)target.getBytes(), target.width(), target.height(), target.width() * 4);
-    pixfmt pixf_dst(rbuf_dst);
+    pixfmt_pre pixf_dst(rbuf_dst);
     renderer_base rb_dst(pixf_dst);
     rb_dst.clear(agg::rgba(0, 0, 0, 0));
 

src/proj_transform.cpp

@@ -150,7 +150,7 @@ bool proj_transform::backward (double * x, double * y , double * z, int point_co
         }
     }
 
-    do {
+    {
 #if defined(MAPNIK_THREADSAFE) && PJ_VERSION < 480
         mutex::scoped_lock lock(projection::mutex_);
 #endif
@@ -160,7 +160,7 @@ bool proj_transform::backward (double * x, double * y , double * z, int point_co
         {
             return false;
         }
-    } while(false);
+    }
 
     if (is_source_longlat_)
     {

src/warp.cpp

@@ -58,17 +58,19 @@ void reproject_and_scale_raster(raster & target, raster const& source,
     CoordTransform tt(target.data_.width(), target.data_.height(),
                       target.ext_, offset_x, offset_y);
     unsigned i, j;
-    unsigned mesh_nx = ceil(source.data_.width()/double(mesh_size)+1);
-    unsigned mesh_ny = ceil(source.data_.height()/double(mesh_size)+1);
+    unsigned mesh_nx = ceil(source.data_.width()/double(mesh_size) + 1);
+    unsigned mesh_ny = ceil(source.data_.height()/double(mesh_size) + 1);
 
     ImageData<double> xs(mesh_nx, mesh_ny);
     ImageData<double> ys(mesh_nx, mesh_ny);
 
     // Precalculate reprojected mesh
-    for(j=0; j<mesh_ny; j++) {
-        for (i=0; i<mesh_nx; i++) {
-            xs(i,j) = i*mesh_size;
-            ys(i,j) = j*mesh_size;
+    for(j=0; j<mesh_ny; ++j)
+    {
+        for (i=0; i<mesh_nx; ++i)
+        {
+            xs(i,j) = std::min(i*mesh_size,source.data_.width());
+            ys(i,j) = std::min(j*mesh_size,source.data_.height());
             ts.backward(&xs(i,j), &ys(i,j));
         }
     }
@@ -78,6 +80,8 @@ void reproject_and_scale_raster(raster & target, raster const& source,
     typedef agg::pixfmt_rgba32 pixfmt;
     typedef pixfmt::color_type color_type;
     typedef agg::renderer_base<pixfmt> renderer_base;
+    typedef agg::pixfmt_rgba32_pre pixfmt_pre;
+    typedef agg::renderer_base<pixfmt_pre> renderer_base_pre;
 
     agg::rasterizer_scanline_aa<> rasterizer;
     agg::scanline_u8  scanline;
@@ -85,8 +89,8 @@ void reproject_and_scale_raster(raster & target, raster const& source,
                               target.data_.width(),
                               target.data_.height(),
                               target.data_.width()*4);
-    pixfmt pixf(buf);
-    renderer_base rb(pixf);
+    pixfmt_pre pixf_pre(buf);
+    renderer_base_pre rb_pre(pixf_pre);
     rasterizer.clip_box(0, 0, target.data_.width(), target.data_.height());
     agg::rendering_buffer buf_tile(
         (unsigned char*)source.data_.getData(),
@@ -142,8 +146,10 @@ void reproject_and_scale_raster(raster & target, raster const& source,
     }
 
     // Project mesh cells into target interpolating raster inside each one
-    for(j=0; j<mesh_ny-1; j++) {
-        for (i=0; i<mesh_nx-1; i++) {
+    for(j=0; j<mesh_ny-1; j++)
+    {
+        for (i=0; i<mesh_nx-1; i++)
+        {
             double polygon[8] = {xs(i,j), ys(i,j),
                                  xs(i+1,j), ys(i+1,j),
                                  xs(i+1,j+1), ys(i+1,j+1),
@@ -163,7 +169,8 @@ void reproject_and_scale_raster(raster & target, raster const& source,
             unsigned y0 = j * mesh_size;
             unsigned x1 = (i+1) * mesh_size;
             unsigned y1 = (j+1) * mesh_size;
-
+            x1 = std::min(x1, source.data_.width());
+            y1 = std::min(y1, source.data_.height());
             agg::trans_affine tr(polygon, x0, y0, x1, y1);
             if (tr.is_valid())
             {
@@ -176,13 +183,13 @@ void reproject_and_scale_raster(raster & target, raster const& source,
                         <img_accessor_type, interpolator_type>
                         span_gen_type;
                     span_gen_type sg(ia, interpolator);
-                    agg::render_scanlines_aa(rasterizer, scanline, rb,
+                    agg::render_scanlines_aa(rasterizer, scanline, rb_pre,
                                              sa, sg);
                 } else {
                     typedef mapnik::span_image_resample_rgba_affine
                         <img_accessor_type> span_gen_type;
                     span_gen_type sg(ia, interpolator, filter);
-                    agg::render_scanlines_aa(rasterizer, scanline, rb,
+                    agg::render_scanlines_aa(rasterizer, scanline, rb_pre,
                                              sa, sg);
                 }
             }

tests/python_tests/csv_test.py

@@ -206,7 +206,7 @@ if 'csv' in mapnik.DatasourceCache.plugin_names():
         eq_(desc['type'],mapnik.DataType.Vector)
         eq_(desc['encoding'],'utf-8')
 
-    def test_windows_newlines(**kwargs):
+    def test_reading_windows_newlines(**kwargs):
         ds = get_csv_ds('windows_newlines.csv')
         eq_(len(ds.fields()),3)
         feats = ds.all_features()
@@ -222,8 +222,8 @@ if 'csv' in mapnik.DatasourceCache.plugin_names():
         eq_(desc['type'],mapnik.DataType.Vector)
         eq_(desc['encoding'],'utf-8')
 
-    def test_mac_newlines(**kwargs):
-        ds = get_csv_ds('windows_newlines.csv')
+    def test_reading_mac_newlines(**kwargs):
+        ds = get_csv_ds('mac_newlines.csv')
         eq_(len(ds.fields()),3)
         feats = ds.all_features()
         eq_(len(feats),1)
@@ -238,6 +238,42 @@ if 'csv' in mapnik.DatasourceCache.plugin_names():
         eq_(desc['type'],mapnik.DataType.Vector)
         eq_(desc['encoding'],'utf-8')
 
+    def check_newlines(filename):
+        ds = get_csv_ds(filename)
+        eq_(len(ds.fields()),3)
+        feats = ds.all_features()
+        eq_(len(feats),1)
+        fs = ds.featureset()
+        feat = fs.next()
+        eq_(feat['x'],0)
+        eq_(feat['y'],0)
+        eq_(feat['line'],'many\n  lines\n  of text\n  with unix newlines')
+        desc = ds.describe()
+        eq_(desc['geometry_type'],mapnik.DataGeometryType.Point)
+        eq_(desc['name'],'csv')
+        eq_(desc['type'],mapnik.DataType.Vector)
+        eq_(desc['encoding'],'utf-8')
+
+    def test_mixed_mac_unix_newlines(**kwargs):
+        check_newlines('mac_newlines_with_unix_inline.csv')
+
+    def test_mixed_mac_unix_newlines_escaped(**kwargs):
+        check_newlines('mac_newlines_with_unix_inline_escaped.csv')
+
+    # To hard to support this case
+    #def test_mixed_unix_windows_newlines(**kwargs):
+    #    check_newlines('unix_newlines_with_windows_inline.csv')
+
+    # To hard to support this case
+    #def test_mixed_unix_windows_newlines_escaped(**kwargs):
+    #    check_newlines('unix_newlines_with_windows_inline_escaped.csv')
+
+    def test_mixed_windows_unix_newlines(**kwargs):
+        check_newlines('windows_newlines_with_unix_inline.csv')
+
+    def test_mixed_windows_unix_newlines_escaped(**kwargs):
+        check_newlines('windows_newlines_with_unix_inline_escaped.csv')
+
     def test_tabs(**kwargs):
         ds = get_csv_ds('tabs_in_csv.csv')
         eq_(len(ds.fields()),3)