Merge commit '8dca9dbb2b9012caf9d37ae60906756aa6c0e037' into harfbuzz

Conflicts:
	src/agg/process_shield_symbolizer.cpp
	src/grid/process_shield_symbolizer.cpp

.travis.yml

@@ -10,4 +10,4 @@ before_install:
  - sudo apt-get update -qq
  - sudo apt-get install -qq libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-python-dev libboost-regex-dev libboost-system-dev libboost-thread-dev python-nose libicu-dev libpng-dev libjpeg-dev libtiff-dev libz-dev libfreetype6-dev libxml2-dev libproj-dev libpq-dev libgdal-dev libcairomm-1.0-dev python-cairo-dev libsqlite3-dev
 
-script: scons configure JOBS=2 FAST=True CXX="$CXX $CXX_SCONS" WARNING_CXXFLAGS=$WARNING_CXXFLAGS && sudo make install && make test || echo "Overall Test Exit Code: $?"
+script: scons configure JOBS=2 FAST=True CXX="$CXX $CXX_SCONS" WARNING_CXXFLAGS=$WARNING_CXXFLAGS && sudo make install

CHANGELOG.md

@@ -8,6 +8,8 @@ For a complete change history, see the git log.
 
 ## Future
 
+- Faster rendering of rasters by reducing memory allocation of temporary buffers (#1516)
+
 - Added ability to pass a pre-created collision detector to the cairo renderer (#1444)
 
 - Tolerance parameter is now supported for querying datasources at a given point (#503/#1499)

Makefile

@@ -14,6 +14,7 @@ distclean:
 	if test -e ".sconf_temp/"; then rm -r ".sconf_temp/"; fi
 	if test -e ".sconsign.dblite"; then rm ".sconsign.dblite"; fi
 	if test -e "config.cache"; then rm "config.cache"; fi
+	if test -e "config.py"; then mv "config.py" "config.py.backup"; fi
 
 reset: distclean
 
@@ -50,4 +51,4 @@ render:
 		nik2img.py $${FILE} /tmp/$$(basename $${FILE}).png; \
 	done
 
-.PHONY: clean reset uninstall test install demo
+.PHONY: install mapnik clean distclean reset uninstall test demo pep8 grind render

bindings/python/mapnik/__init__.py

@@ -778,11 +778,11 @@ class _TextSymbolizer(TextSymbolizer,_injector):
 
     @property
     def wrap_before(self):
-        return self.properties.wrap_before
+        return self.format.wrap_before
 
     @wrap_before.setter
     def wrap_before(self, wrap_before):
-        self.properties.wrap_before = wrap_before
+        self.format.wrap_before = wrap_before
 
 
     @property

bindings/python/mapnik_image.cpp

@@ -228,7 +228,7 @@ void export_image()
         .def("composite",&composite,
          ( arg("self"),
            arg("image"),
-           arg("mode"),
+           arg("mode")=mapnik::src_over,
            arg("opacity")=1.0f
          ))
         .def("premultiply",&image_32::premultiply)

bindings/python/mapnik_style.cpp

@@ -49,14 +49,16 @@ void set_image_filters(feature_type_style & style, std::string const& filters)
     std::string::const_iterator end = filters.end();
     mapnik::image_filter_grammar<std::string::const_iterator,
                                  std::vector<mapnik::filter::filter_type> > filter_grammar;
+    std::vector<mapnik::filter::filter_type> new_filters;
     bool result = boost::spirit::qi::phrase_parse(itr,end,
                                                   filter_grammar,
                                                   boost::spirit::qi::ascii::space,
-                                                  style.image_filters());
+                                                  new_filters);
     if (!result || itr!=end)
     {
         throw mapnik::value_error("failed to parse image-filters: '" + std::string(itr,end) + "'");
     }
+    style.image_filters().swap(new_filters);
 }
 
 void export_style()

include/mapnik/color.hpp

@@ -71,6 +71,8 @@ public:
 
     std::string to_string() const;
     std::string to_hex_string() const;
+    void premultiply();
+    void demultiply();
 
     color& operator=(color const& rhs)
     {

include/mapnik/css_color_grammar.hpp

@@ -40,6 +40,25 @@
 // stl
 #include <string>
 
+namespace mapnik {
+
+// http://www.w3.org/TR/css3-color/#hsl-color
+inline double hue_to_rgb( double m1, double m2, double h)
+{
+    if (h < 0.0) h = h + 1.0;
+    else if (h > 1) h = h - 1.0;
+
+    if (h * 6 < 1.0)
+        return m1 + (m2 - m1) * h * 6.0;
+    if (h * 2 < 1.0)
+        return m2;
+    if (h * 3 < 2.0)
+        return m1 + (m2 - m1)* (2.0/3.0 - h) * 6.0;
+    return m1;
+}
+
+} // namespace mapnik
+
 // boost
 #include <boost/version.hpp>
 #if BOOST_VERSION >= 104500
@@ -105,21 +124,6 @@ struct alpha_conv_impl
     }
 };
 
-// http://www.w3.org/TR/css3-color/#hsl-color
-inline double hue_to_rgb( double m1, double m2, double h)
-{
-    if (h < 0.0) h = h + 1.0;
-    else if (h > 1) h = h - 1.0;
-
-    if (h * 6 < 1.0)
-        return m1 + (m2 - m1) * h * 6.0;
-    if (h * 2 < 1.0)
-        return m2;
-    if (h * 3 < 2.0)
-        return m1 + (m2 - m1)* (2.0/3.0 - h) * 6.0;
-    return m1;
-}
-
 struct hsl_conv_impl
 {
     template<typename T0,typename T1, typename T2, typename T3>

include/mapnik/feature.hpp

@@ -257,7 +257,7 @@ public:
         return result;
     }
 
-    const raster_ptr& get_raster() const
+    raster_ptr const& get_raster() const
     {
         return raster_;
     }

include/mapnik/raster.hpp

@@ -27,9 +27,13 @@
 #include <mapnik/box2d.hpp>
 #include <mapnik/image_data.hpp>
 
+// boost
+#include <boost/utility.hpp>
+
 namespace mapnik {
-struct raster
+class raster : private boost::noncopyable
 {
+public:
     box2d<double> ext_;
     image_data_32 data_;
     bool premultiplied_alpha_;
@@ -38,12 +42,6 @@ struct raster
           data_(width,height),
           premultiplied_alpha_(premultiplied_alpha)
     {}
-    raster(box2d<double> const& ext,image_data_32 const& data, bool premultiplied_alpha = false)
-        : ext_(ext),
-          data_(data),
-          premultiplied_alpha_(premultiplied_alpha)
-    {}
-
 };
 }
 

include/mapnik/rule.hpp

@@ -38,6 +38,7 @@
 #include <mapnik/feature.hpp>
 #include <mapnik/expression.hpp>
 #include <mapnik/expression_string.hpp>
+#include <mapnik/config.hpp> // MAPNIK_DECL
 
 // boost
 #include <boost/concept_check.hpp>
@@ -127,7 +128,7 @@ typedef boost::variant<point_symbolizer,
                        markers_symbolizer,
                        debug_symbolizer> symbolizer;
 
-class rule
+class MAPNIK_DECL rule
 {
 public:
     typedef std::vector<symbolizer> symbolizers;

include/mapnik/util/geometry_to_wkb.hpp

@@ -39,241 +39,243 @@
 
 namespace mapnik { namespace util {
 
-    std::string to_hex(const char* blob, unsigned size)
+std::string to_hex(const char* blob, unsigned size)
+{
+    std::string buf;
+    buf.reserve(size*2);
+    std::ostringstream s(buf);
+    s.seekp(0);
+    char hex[3];
+    std::memset(hex,0,3);
+    for ( unsigned pos=0; pos < size; ++pos)
     {
-        std::string buf;
-        buf.reserve(size*2);
-        std::ostringstream s(buf);
-        s.seekp(0);
-        char hex[3];
-        std::memset(hex,0,3);
-        for ( unsigned pos=0; pos < size; ++pos)
-        {
-            std::sprintf (hex, "%02x", int(blob[pos]) & 0xff);
-            s << hex;
-        }
-        return s.str();
+        std::sprintf (hex, "%02x", int(blob[pos]) & 0xff);
+        s << hex;
     }
+    return s.str();
+}
 
-    enum wkbByteOrder {
-        wkbXDR=0,
-        wkbNDR=1
-    };
+enum wkbByteOrder {
+    wkbXDR=0,
+    wkbNDR=1
+};
 
 
-    inline void reverse_bytes(char size, char *address)
+inline void reverse_bytes(char size, char *address)
+{
+    char * first = address;
+    char * last = first + size - 1;
+    for(;first < last;++first, --last)
     {
-        char * first = address;
-        char * last = first + size - 1;
-        for(;first < last;++first, --last)
-        {
-            char x = *last;
-            *last = *first;
-            *first = x;
-        }
+        char x = *last;
+        *last = *first;
+        *first = x;
     }
+}
 
-    template <typename S, typename T>
-    inline void write (S & stream, T val, std::size_t size, wkbByteOrder byte_order)
-    {
+template <typename S, typename T>
+inline void write (S & stream, T val, std::size_t size, wkbByteOrder byte_order)
+{
 #ifdef MAPNIK_BIG_ENDIAN
-        bool need_swap =  byte_order ? wkbNDR : wkbXDR;
+    bool need_swap =  byte_order ? wkbNDR : wkbXDR;
 #else
-        bool need_swap =  byte_order ? wkbXDR : wkbNDR;
+    bool need_swap =  byte_order ? wkbXDR : wkbNDR;
 #endif
-        char* buf = reinterpret_cast<char*>(&val);
-        if (need_swap)
-        {
-            reverse_bytes(size,buf);
-        }
-        stream.write(buf,size);
+    char* buf = reinterpret_cast<char*>(&val);
+    if (need_swap)
+    {
+        reverse_bytes(size,buf);
+    }
+    stream.write(buf,size);
+}
+
+struct wkb_buffer
+{
+    wkb_buffer(std::size_t size)
+        : size_(size),
+          data_( (size_!=0) ? static_cast<char*>(::operator new (size_)):0)
+    {}
+
+    ~wkb_buffer()
+    {
+        ::operator delete(data_);
     }
 
-    struct wkb_buffer
+    inline std::size_t size() const
     {
-        wkb_buffer(std::size_t size)
-            : size_(size),
-              data_( (size_!=0) ? static_cast<char*>(::operator new (size_)):0)
-        {}
+        return size_;
+    }
 
-        ~wkb_buffer()
-        {
-            ::operator delete(data_);
-        }
-
-        inline std::size_t size() const
-        {
-            return size_;
-        }
-
-        inline char* buffer()
-        {
-            return data_;
-        }
-
-        std::size_t size_;
-        char * data_;
-    };
-
-    typedef boost::shared_ptr<wkb_buffer> wkb_buffer_ptr;
-
-    wkb_buffer_ptr to_point_wkb( geometry_type const& g, wkbByteOrder byte_order)
+    inline char* buffer()
     {
-        assert(g.size() == 1);
-        std::size_t size = 1 + 4 + 8*2 ; // byteOrder + wkbType + Point
-        wkb_buffer_ptr wkb = boost::make_shared<wkb_buffer>(size);
-        boost::interprocess::bufferstream ss(wkb->buffer(), wkb->size(), std::ios::out | std::ios::binary);
-        ss.write(reinterpret_cast<char*>(&byte_order),1);
-        int type = static_cast<int>(mapnik::Point);
-        write(ss,type,4,byte_order);
-        double x = 0;
-        double y = 0;
-        g.vertex(0,&x,&y);
+        return data_;
+    }
+
+    std::size_t size_;
+    char * data_;
+};
+
+typedef boost::shared_ptr<wkb_buffer> wkb_buffer_ptr;
+
+template<typename GeometryType>
+wkb_buffer_ptr to_point_wkb( GeometryType const& g, wkbByteOrder byte_order)
+{
+    assert(g.size() == 1);
+    std::size_t size = 1 + 4 + 8*2 ; // byteOrder + wkbType + Point
+    wkb_buffer_ptr wkb = boost::make_shared<wkb_buffer>(size);
+    boost::interprocess::bufferstream ss(wkb->buffer(), wkb->size(), std::ios::out | std::ios::binary);
+    ss.write(reinterpret_cast<char*>(&byte_order),1);
+    int type = static_cast<int>(mapnik::Point);
+    write(ss,type,4,byte_order);
+    double x = 0;
+    double y = 0;
+    g.vertex(0,&x,&y);
+    write(ss,x,8,byte_order);
+    write(ss,y,8,byte_order);
+    assert(ss.good());
+    return wkb;
+}
+
+template<typename GeometryType>
+wkb_buffer_ptr to_line_string_wkb( GeometryType const& g, wkbByteOrder byte_order)
+{
+    unsigned num_points = g.size();
+    assert(num_points > 1);
+    std::size_t size = 1 + 4 + 4 + 8*2*num_points ; // byteOrder + wkbType + numPoints + Point*numPoints
+    wkb_buffer_ptr wkb = boost::make_shared<wkb_buffer>(size);
+    boost::interprocess::bufferstream ss(wkb->buffer(), wkb->size(), std::ios::out | std::ios::binary);
+    ss.write(reinterpret_cast<char*>(&byte_order),1);
+    int type = static_cast<int>(mapnik::LineString);
+    write(ss,type,4,byte_order);
+    write(ss,num_points,4,byte_order);
+    double x = 0;
+    double y = 0;
+    for (unsigned i=0; i< num_points; ++i)
+    {
+        g.vertex(i,&x,&y);
         write(ss,x,8,byte_order);
         write(ss,y,8,byte_order);
-        assert(ss.good());
-        return wkb;
     }
+    assert(ss.good());
+    return wkb;
+}
 
-    wkb_buffer_ptr to_line_string_wkb( geometry_type const& g, wkbByteOrder byte_order)
+template<typename GeometryType>
+wkb_buffer_ptr to_polygon_wkb( GeometryType const& g, wkbByteOrder byte_order)
+{
+    unsigned num_points = g.size();
+    assert(num_points > 1);
+
+    typedef std::pair<double,double> point_type;
+    typedef std::vector<point_type> linear_ring;
+    boost::ptr_vector<linear_ring> rings;
+
+    double x = 0;
+    double y = 0;
+    std::size_t size = 1 + 4 + 4 ; // byteOrder + wkbType + numRings
+    for (unsigned i=0; i< num_points; ++i)
     {
-        unsigned num_points = g.size();
-        assert(num_points > 1);
-        std::size_t size = 1 + 4 + 4 + 8*2*num_points ; // byteOrder + wkbType + numPoints + Point*numPoints
-        wkb_buffer_ptr wkb = boost::make_shared<wkb_buffer>(size);
-        boost::interprocess::bufferstream ss(wkb->buffer(), wkb->size(), std::ios::out | std::ios::binary);
-        ss.write(reinterpret_cast<char*>(&byte_order),1);
-        int type = static_cast<int>(mapnik::LineString);
-        write(ss,type,4,byte_order);
-        write(ss,num_points,4,byte_order);
-        double x = 0;
-        double y = 0;
-        for (unsigned i=0; i< num_points; ++i)
+        unsigned command = g.vertex(i,&x,&y);
+        if (command == SEG_MOVETO)
         {
-            g.vertex(i,&x,&y);
+            rings.push_back(new linear_ring); // start new loop
+            size += 4; // num_points
+        }
+        rings.back().push_back(std::make_pair(x,y));
+        size += 2 * 8; // point
+    }
+    unsigned num_rings = rings.size();
+    wkb_buffer_ptr wkb = boost::make_shared<wkb_buffer>(size);
+    boost::interprocess::bufferstream ss(wkb->buffer(), wkb->size(), std::ios::out | std::ios::binary);
+
+    ss.write(reinterpret_cast<char*>(&byte_order),1);
+    int type = static_cast<int>(mapnik::Polygon);
+    write(ss,type,4,byte_order);
+    write(ss,num_rings,4,byte_order);
+
+    BOOST_FOREACH ( linear_ring const& ring, rings)
+    {
+        unsigned num_points = ring.size();
+        write(ss,num_points,4,byte_order);
+        BOOST_FOREACH ( point_type const& pt, ring)
+        {
+            double x = pt.first;
+            double y = pt.second;
             write(ss,x,8,byte_order);
             write(ss,y,8,byte_order);
         }
-        assert(ss.good());
-        return wkb;
     }
 
-    wkb_buffer_ptr to_polygon_wkb( geometry_type const& g, wkbByteOrder byte_order)
+    assert(ss.good());
+    return wkb;
+}
+
+template<typename GeometryType>
+wkb_buffer_ptr to_wkb(GeometryType const& g, wkbByteOrder byte_order )
+{
+    wkb_buffer_ptr wkb;
+
+    switch (g.type())
     {
-        unsigned num_points = g.size();
-        assert(num_points > 1);
+    case mapnik::Point:
+        wkb = to_point_wkb(g, byte_order);
+        break;
+    case mapnik::LineString:
+        wkb = to_line_string_wkb(g, byte_order);
+        break;
+    case mapnik::Polygon:
+        wkb = to_polygon_wkb(g, byte_order);
+        break;
+    default:
+        break;
+    }
+    return wkb;
+}
 
-        typedef std::pair<double,double> point_type;
-        typedef std::vector<point_type> linear_ring;
-        boost::ptr_vector<linear_ring> rings;
+wkb_buffer_ptr to_wkb(geometry_container const& paths, wkbByteOrder byte_order )
+{
+    if (paths.size() == 1)
+    {
+        // single geometry
+        return to_wkb(paths.front(), byte_order);
+    }
 
-        double x = 0;
-        double y = 0;
-        std::size_t size = 1 + 4 + 4 ; // byteOrder + wkbType + numRings
-        for (unsigned i=0; i< num_points; ++i)
+    if (paths.size() > 1)
+    {
+        // multi geometry or geometry collection
+        std::vector<wkb_buffer_ptr> wkb_cont;
+        bool collection = false;
+        int multi_type = 0;
+        size_t multi_size = 1 + 4 + 4;
+        geometry_container::const_iterator itr = paths.begin();
+        geometry_container::const_iterator end = paths.end();
+        for ( ; itr!=end; ++itr)
         {
-            unsigned command = g.vertex(i,&x,&y);
-            if (command == SEG_MOVETO)
-            {
-                rings.push_back(new linear_ring); // start new loop
-                size += 4; // num_points
-            }
-            rings.back().push_back(std::make_pair(x,y));
-            size += 2 * 8; // point
+            wkb_buffer_ptr wkb = to_wkb(*itr,byte_order);
+            multi_size += wkb->size();
+            int type = static_cast<int>(itr->type());
+            if (multi_type > 0 && multi_type != itr->type())
+                collection = true;
+            multi_type = type;
+            wkb_cont.push_back(wkb);
         }
-        unsigned num_rings = rings.size();
-        wkb_buffer_ptr wkb = boost::make_shared<wkb_buffer>(size);
-        boost::interprocess::bufferstream ss(wkb->buffer(), wkb->size(), std::ios::out | std::ios::binary);
 
+        wkb_buffer_ptr multi_wkb = boost::make_shared<wkb_buffer>(multi_size);
+        boost::interprocess::bufferstream ss(multi_wkb->buffer(), multi_wkb->size(), std::ios::out | std::ios::binary);
         ss.write(reinterpret_cast<char*>(&byte_order),1);
-        int type = static_cast<int>(mapnik::Polygon);
-        write(ss,type,4,byte_order);
-        write(ss,num_rings,4,byte_order);
+        multi_type = collection ? 7 : multi_type + 3;
+        write(ss,multi_type, 4, byte_order);
+        write(ss,paths.size(),4,byte_order);
 
-        BOOST_FOREACH ( linear_ring const& ring, rings)
+        BOOST_FOREACH ( wkb_buffer_ptr const& wkb, wkb_cont)
         {
-            unsigned num_points = ring.size();
-            write(ss,num_points,4,byte_order);
-            BOOST_FOREACH ( point_type const& pt, ring)
-            {
-                double x = pt.first;
-                double y = pt.second;
-                write(ss,x,8,byte_order);
-                write(ss,y,8,byte_order);
-            }
+            ss.write(wkb->buffer(),wkb->size());
         }
-
-        assert(ss.good());
-        return wkb;
+        return multi_wkb;
     }
 
-    wkb_buffer_ptr to_wkb(geometry_type const& g, wkbByteOrder byte_order )
-    {
-        wkb_buffer_ptr wkb;
-
-        switch (g.type())
-        {
-        case mapnik::Point:
-            wkb = to_point_wkb(g, byte_order);
-            break;
-        case mapnik::LineString:
-            wkb = to_line_string_wkb(g, byte_order);
-            break;
-        case mapnik::Polygon:
-            wkb = to_polygon_wkb(g, byte_order);
-            break;
-        default:
-            break;
-        }
-        return wkb;
-    }
-
-    wkb_buffer_ptr to_wkb(geometry_container const& paths, wkbByteOrder byte_order )
-    {
-        if (paths.size() == 1)
-        {
-            // single geometry
-            return to_wkb(paths.front(), byte_order);
-        }
-
-        if (paths.size() > 1)
-        {
-            // multi geometry or geometry collection
-            std::vector<wkb_buffer_ptr> wkb_cont;
-            bool collection = false;
-            int multi_type = 0;
-            size_t multi_size = 1 + 4 + 4;
-            geometry_container::const_iterator itr = paths.begin();
-            geometry_container::const_iterator end = paths.end();
-            for ( ; itr!=end; ++itr)
-            {
-                wkb_buffer_ptr wkb = to_wkb(*itr,byte_order);
-                multi_size += wkb->size();
-                int type = static_cast<int>(itr->type());
-                if (multi_type > 0 && multi_type != itr->type())
-                    collection = true;
-                multi_type = type;
-                wkb_cont.push_back(wkb);
-            }
-
-            wkb_buffer_ptr multi_wkb = boost::make_shared<wkb_buffer>(multi_size);
-            boost::interprocess::bufferstream ss(multi_wkb->buffer(), multi_wkb->size(), std::ios::out | std::ios::binary);
-            ss.write(reinterpret_cast<char*>(&byte_order),1);
-            multi_type = collection ? 7 : multi_type + 3;
-            write(ss,multi_type, 4, byte_order);
-            write(ss,paths.size(),4,byte_order);
-
-            BOOST_FOREACH ( wkb_buffer_ptr const& wkb, wkb_cont)
-            {
-                ss.write(wkb->buffer(),wkb->size());
-            }
-            return multi_wkb;
-        }
-
-        return wkb_buffer_ptr();
-    }
-
-    }}
-
+    return wkb_buffer_ptr();
+}
+}}
 
 #endif // MAPNIK_GEOMETRY_TO_WKB_HPP

plugins/input/csv/csv_datasource.cpp

@@ -247,7 +247,8 @@ void csv_datasource::parse_csv(T & stream,
     std::string quo = boost::trim_copy(quote);
     if (quo.empty()) quo = "\"";
 
-    MAPNIK_LOG_DEBUG(csv) << "csv_datasource: csv grammer: sep: '" << sep << "' quo: '" << quo << "' esc: '" << esc;
+    MAPNIK_LOG_DEBUG(csv) << "csv_datasource: csv grammar: sep: '" << sep
+                          << "' quo: '" << quo << "' esc: '" << esc << "'";
 
     boost::escaped_list_separator<char> grammer;
     try

plugins/input/gdal/gdal_featureset.cpp

@@ -223,14 +223,14 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
 
         if (im_width > 0 && im_height > 0)
         {
-            mapnik::image_data_32 image(im_width, im_height);
+            mapnik::raster_ptr raster = boost::make_shared<mapnik::raster>(intersect, im_width, im_height);
+            feature->set_raster(raster);
+            mapnik::image_data_32 & image = raster->data_;
             image.set(0xffffffff);
 
             MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Image Size=(" << im_width << "," << im_height << ")";
             MAPNIK_LOG_DEBUG(gdal) << "gdal_featureset: Reading band=" << band_;
 
-            typedef std::vector<int,int> pallete;
-
             if (band_ > 0) // we are querying a single band
             {
                 if (band_ > nbands_)
@@ -255,7 +255,6 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
                                imageData, image.width(), image.height(),
                                GDT_Float32, 0, 0);
 
-                feature->set_raster(boost::make_shared<mapnik::raster>(intersect,image));
                 if (hasNoData)
                 {
                     feature->put("NODATA",nodata);
@@ -488,8 +487,6 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
                     alpha->RasterIO(GF_Read, x_off, y_off, width, height, image.getBytes() + 3,
                                     image.width(), image.height(), GDT_Byte, 4, 4 * image.width());
                 }
-
-                feature->set_raster(boost::make_shared<mapnik::raster>(intersect, image));
             }
             return feature;
         }

plugins/input/ogr/ogr_datasource.cpp

@@ -203,6 +203,7 @@ void ogr_datasource::bind() const
 
         unsigned num_layers = dataset_->GetLayerCount();
         bool layer_found = false;
+        std::vector<std::string> layer_names;
         for (unsigned i = 0; i < num_layers; ++i )
         {
             OGRLayer* ogr_layer = dataset_->GetLayer(i);
@@ -210,7 +211,7 @@ void ogr_datasource::bind() const
             if (ogr_layer_def != 0)
             {
                 layer_found = true;
-                s << " '" << ogr_layer_def->GetName() << "' ";
+                layer_names.push_back(std::string("'") + ogr_layer_def->GetName() + std::string("'"));
             }
         }
 
@@ -218,6 +219,10 @@ void ogr_datasource::bind() const
         {
             s << "None (no layers were found in dataset)";
         }
+        else
+        {
+            s << boost::algorithm::join(layer_names,", ");
+        }
 
         throw datasource_exception(s.str());
     }

plugins/input/ogr/ogr_featureset.cpp

@@ -86,7 +86,7 @@ feature_ptr ogr_featureset::next()
 {
     ogr_feature_ptr feat (layer_.GetNextFeature());
 
-    if ((*feat) != NULL)
+    while ((*feat) != NULL)
     {
         // ogr feature ids start at 0, so add one to stay
         // consistent with other mapnik datasources that start at 1
@@ -101,6 +101,7 @@ feature_ptr ogr_featureset::next()
         else
         {
             MAPNIK_LOG_DEBUG(ogr) << "ogr_featureset: Feature with null geometry=" << (*feat)->GetFID();
+            continue;
         }
 
         ++count_;

plugins/input/ogr/ogr_index_featureset.cpp

@@ -87,85 +87,87 @@ ogr_index_featureset<filterT>::~ogr_index_featureset() {}
 template <typename filterT>
 feature_ptr ogr_index_featureset<filterT>::next()
 {
-    if (itr_ != ids_.end())
+    while (itr_ != ids_.end())
     {
         int pos = *itr_++;
         layer_.SetNextByIndex (pos);
 
         ogr_feature_ptr feat (layer_.GetNextFeature());
-        if ((*feat) != NULL)
+        if ((*feat) == NULL)
         {
-            // ogr feature ids start at 0, so add one to stay
-            // consistent with other mapnik datasources that start at 1
-            int feature_id = ((*feat)->GetFID() + 1);
-            feature_ptr feature(feature_factory::create(ctx_,feature_id));
-
-            OGRGeometry* geom=(*feat)->GetGeometryRef();
-            if (geom && !geom->IsEmpty())
-            {
-                ogr_converter::convert_geometry (geom, feature);
-            }
-            else
-            {
-                MAPNIK_LOG_DEBUG(ogr) << "ogr_index_featureset: Feature with null geometry=" << (*feat)->GetFID();
-            }
-
-            int fld_count = layerdef_->GetFieldCount();
-            for (int i = 0; i < fld_count; i++)
-            {
-                OGRFieldDefn* fld = layerdef_->GetFieldDefn (i);
-                OGRFieldType type_oid = fld->GetType ();
-                std::string fld_name = fld->GetNameRef ();
-
-                switch (type_oid)
-                {
-                case OFTInteger:
-                {
-                    feature->put(fld_name,(*feat)->GetFieldAsInteger (i));
-                    break;
-                }
-
-                case OFTReal:
-                {
-                    feature->put(fld_name,(*feat)->GetFieldAsDouble (i));
-                    break;
-                }
-
-                case OFTString:
-                case OFTWideString:     // deprecated !
-                {
-                    UnicodeString ustr = tr_->transcode((*feat)->GetFieldAsString (i));
-                    feature->put(fld_name,ustr);
-                    break;
-                }
-
-                case OFTIntegerList:
-                case OFTRealList:
-                case OFTStringList:
-                case OFTWideStringList: // deprecated !
-                {
-                    MAPNIK_LOG_WARN(ogr) << "ogr_index_featureset: Unhandled type_oid=" << type_oid;
-                    break;
-                }
-
-                case OFTBinary:
-                {
-                    MAPNIK_LOG_WARN(ogr) << "ogr_index_featureset: Unhandled type_oid=" << type_oid;
-                    //feature->put(name,feat->GetFieldAsBinary (i, size));
-                    break;
-                }
-
-                case OFTDate:
-                case OFTTime:
-                case OFTDateTime:       // unhandled !
-                {
-                    MAPNIK_LOG_WARN(ogr) << "ogr_index_featureset: Unhandled type_oid=" << type_oid;
-                    break;
-                }
-                }
-            }
-            return feature;
+            continue;
         }
+        // ogr feature ids start at 0, so add one to stay
+        // consistent with other mapnik datasources that start at 1
+        int feature_id = ((*feat)->GetFID() + 1);
+        feature_ptr feature(feature_factory::create(ctx_,feature_id));
+
+        OGRGeometry* geom=(*feat)->GetGeometryRef();
+        if (geom && !geom->IsEmpty())
+        {
+            ogr_converter::convert_geometry (geom, feature);
+        }
+        else
+        {
+            MAPNIK_LOG_DEBUG(ogr) << "ogr_index_featureset: Feature with null geometry=" << (*feat)->GetFID();
+            continue;
+        }
+
+        int fld_count = layerdef_->GetFieldCount();
+        for (int i = 0; i < fld_count; i++)
+        {
+            OGRFieldDefn* fld = layerdef_->GetFieldDefn (i);
+            OGRFieldType type_oid = fld->GetType ();
+            std::string fld_name = fld->GetNameRef ();
+
+            switch (type_oid)
+            {
+            case OFTInteger:
+            {
+                feature->put(fld_name,(*feat)->GetFieldAsInteger (i));
+                break;
+            }
+
+            case OFTReal:
+            {
+                feature->put(fld_name,(*feat)->GetFieldAsDouble (i));
+                break;
+            }
+
+            case OFTString:
+            case OFTWideString:     // deprecated !
+            {
+                UnicodeString ustr = tr_->transcode((*feat)->GetFieldAsString (i));
+                feature->put(fld_name,ustr);
+                break;
+            }
+
+            case OFTIntegerList:
+            case OFTRealList:
+            case OFTStringList:
+            case OFTWideStringList: // deprecated !
+            {
+                MAPNIK_LOG_WARN(ogr) << "ogr_index_featureset: Unhandled type_oid=" << type_oid;
+                break;
+            }
+
+            case OFTBinary:
+            {
+                MAPNIK_LOG_WARN(ogr) << "ogr_index_featureset: Unhandled type_oid=" << type_oid;
+                //feature->put(name,feat->GetFieldAsBinary (i, size));
+                break;
+            }
+
+            case OFTDate:
+            case OFTTime:
+            case OFTDateTime:       // unhandled !
+            {
+                MAPNIK_LOG_WARN(ogr) << "ogr_index_featureset: Unhandled type_oid=" << type_oid;
+                break;
+            }
+            }
+        }
+        return feature;
     }
 
     return feature_ptr();

plugins/input/postgis/postgis_datasource.cpp

@@ -389,6 +389,7 @@ void postgis_datasource::bind() const
                     case 1042:  // bpchar
                     case 1043:  // varchar
                     case 25:    // text
+                    case 705:   // literal
                         desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
                         break;
                     default: // should not get here

plugins/input/raster/raster_featureset.cpp

@@ -111,9 +111,10 @@ feature_ptr raster_featureset<LookupPolicy>::next()
                                                             rem.maxy() + y_off + height);
                         intersect = t.backward(feature_raster_extent);
 
-                        image_data_32 image(width,height);
-                        reader->read(x_off, y_off, image);
-                        feature->set_raster(boost::make_shared<raster>(intersect, image,reader->premultiplied_alpha()));
+                        mapnik::raster_ptr raster = boost::make_shared<mapnik::raster>(intersect, width, height);
+                        reader->read(x_off, y_off, raster->data_);
+                        raster->premultiplied_alpha_ = reader->premultiplied_alpha();
+                        feature->set_raster(raster);
                     }
                 }
             }

plugins/input/sqlite/sqlite_connection.hpp

@@ -163,6 +163,12 @@ public:
         return db_;
     }
 
+    bool load_extension(std::string const& ext_path)
+    {
+        sqlite3_enable_load_extension(db_, 1);
+        int result = sqlite3_load_extension(db_, ext_path.c_str(), 0 , 0);
+        return (result == SQLITE_OK)? true : false;
+    }
 
 private:
 

src/agg/agg_renderer.cpp

@@ -62,32 +62,6 @@
 
 namespace mapnik
 {
-class pattern_source : private boost::noncopyable
-{
-public:
-    pattern_source(image_data_32 const& pattern)
-        : pattern_(pattern) {}
-
-    unsigned int width() const
-    {
-        return pattern_.width();
-    }
-    unsigned int height() const
-    {
-        return pattern_.height();
-    }
-    agg::rgba8 pixel(int x, int y) const
-    {
-        unsigned c = pattern_(x,y);
-        return agg::rgba8(c & 0xff,
-                          (c >> 8) & 0xff,
-                          (c >> 16) & 0xff,
-                          (c >> 24) & 0xff);
-    }
-private:
-    image_data_32 const& pattern_;
-};
-
 
 template <typename T>
 agg_renderer<T>::agg_renderer(Map const& m, T & pixmap, double scale_factor, unsigned offset_x, unsigned offset_y)
@@ -132,11 +106,24 @@ template <typename T>
 void agg_renderer<T>::setup(Map const &m)
 {
     boost::optional<color> const& bg = m.background();
-    if (bg) pixmap_.set_background(*bg);
+    if (bg)
+    {
+        if (bg->alpha() < 255)
+        {
+            mapnik::color bg_color = *bg;
+            bg_color.premultiply();
+            pixmap_.set_background(bg_color);
+        }
+        else
+        {
+            pixmap_.set_background(*bg);
+        }
+    }
 
     boost::optional<std::string> const& image_filename = m.background_image();
     if (image_filename)
     {
+        // NOTE: marker_cache returns premultiplied image, if needed
         boost::optional<mapnik::marker_ptr> bg_marker = mapnik::marker_cache::instance().find(*image_filename,true);
         if (bg_marker && (*bg_marker)->is_bitmap())
         {
@@ -152,17 +139,12 @@ void agg_renderer<T>::setup(Map const &m)
                 {
                     for (unsigned y=0;y<y_steps;++y)
                     {
-                        composite(pixmap_.data(),*bg_image, src_over, 1.0f, x*w, y*h, true);
+                        composite(pixmap_.data(),*bg_image, src_over, 1.0f, x*w, y*h, false);
                     }
                 }
             }
         }
     }
-
-    agg::rendering_buffer buf(pixmap_.raw_data(),width_,height_, width_ * 4);
-    agg::pixfmt_rgba32 pixf(buf);
-    pixf.premultiply();
-
     MAPNIK_LOG_DEBUG(agg_renderer) << "agg_renderer: Scale=" << m.scale();
 }
 
@@ -300,6 +282,7 @@ void agg_renderer<T>::render_marker(pixel_position const& pos, marker const& mar
     typedef agg::pixfmt_custom_blend_rgba<blender_type, agg::rendering_buffer> pixfmt_comp_type;
     typedef agg::renderer_base<pixfmt_comp_type> renderer_base;
     typedef agg::renderer_scanline_aa_solid<renderer_base> renderer_type;
+    typedef agg::pod_bvector<mapnik::svg::path_attributes> svg_attribute_type;
 
     ras_ptr->reset();
     ras_ptr->gamma(agg::gamma_power());
@@ -324,9 +307,9 @@ void agg_renderer<T>::render_marker(pixel_position const& pos, marker const& mar
         vertex_stl_adapter<svg_path_storage> stl_storage((*marker.get_vector_data())->source());
         svg_path_adapter svg_path(stl_storage);
         svg_renderer_agg<svg_path_adapter,
-            agg::pod_bvector<path_attributes>,
+            svg_attribute_type,
             renderer_type,
-            agg::pixfmt_rgba32> svg_renderer(svg_path,
+            pixfmt_comp_type> svg_renderer(svg_path,
                                                    (*marker.get_vector_data())->attributes());
 
         svg_renderer.render(*ras_ptr, sl, renb, mtx, opacity, bbox);

src/agg/process_building_symbolizer.cpp

@@ -30,6 +30,9 @@
 // boost
 #include <boost/scoped_ptr.hpp>
 
+// stl
+#include <deque>
+
 // agg
 #include "agg_basics.h"
 #include "agg_rendering_buffer.h"

src/agg/process_line_pattern_symbolizer.cpp

@@ -20,8 +20,6 @@
  *
  *****************************************************************************/
 
-// boost
-#include <boost/foreach.hpp>
 // mapnik
 #include <mapnik/debug.hpp>
 #include <mapnik/graphics.hpp>
@@ -33,6 +31,7 @@
 #include <mapnik/marker_cache.hpp>
 #include <mapnik/line_pattern_symbolizer.hpp>
 #include <mapnik/vertex_converters.hpp>
+
 // agg
 #include "agg_basics.h"
 #include "agg_rendering_buffer.h"
@@ -47,6 +46,40 @@
 #include "agg_renderer_outline_image.h"
 #include "agg_conv_clip_polyline.h"
 
+// boost
+#include <boost/utility.hpp>
+#include <boost/foreach.hpp>
+
+namespace {
+
+class pattern_source : private boost::noncopyable
+{
+public:
+    pattern_source(mapnik::image_data_32 const& pattern)
+        : pattern_(pattern) {}
+
+    unsigned int width() const
+    {
+        return pattern_.width();
+    }
+    unsigned int height() const
+    {
+        return pattern_.height();
+    }
+    agg::rgba8 pixel(int x, int y) const
+    {
+        unsigned c = pattern_(x,y);
+        return agg::rgba8(c & 0xff,
+                          (c >> 8) & 0xff,
+                          (c >> 16) & 0xff,
+                          (c >> 24) & 0xff);
+    }
+private:
+    mapnik::image_data_32 const& pattern_;
+};
+
+}
+
 namespace mapnik {
 
 template <typename T>

src/build.py

@@ -398,4 +398,5 @@ else:
     env['create_uninstall_target'](env, target1)
     env['create_uninstall_target'](env, target)
 
-Depends(mapnik, env.subst('../deps/agg/libagg.a'))
+if not env['RUNTIME_LINK'] == 'static':
+    Depends(mapnik, env.subst('../deps/agg/libagg.a'))

src/cairo_renderer.cpp

@@ -43,6 +43,7 @@
 #include <mapnik/config.hpp>
 #include <mapnik/vertex_converters.hpp>
 #include <mapnik/marker_helpers.hpp>
+
 // cairo
 #include <cairomm/context.h>
 #include <cairomm/surface.h>
@@ -64,6 +65,9 @@
 #include "agg_path_storage.h"
 #include "agg_ellipse.h"
 
+// stl
+#include <deque>
+
 namespace mapnik
 {
 class cairo_pattern : private boost::noncopyable

src/color.cpp

@@ -25,6 +25,9 @@
 #include <mapnik/color_factory.hpp>
 #include <mapnik/config_error.hpp>
 
+// agg
+#include "agg_color_rgba.h"
+
 // boost
 #include <boost/format.hpp>
 
@@ -78,5 +81,24 @@ std::string color::to_hex_string() const
     }
 }
 
+void color::premultiply()
+{
+    agg::rgba8 pre_c = agg::rgba8(red_,green_,blue_,alpha_);
+    pre_c.premultiply();
+    red_ = pre_c.r;
+    green_ = pre_c.g;
+    blue_ = pre_c.b;
+}
+
+void color::demultiply()
+{
+    // note: this darkens too much: https://github.com/mapnik/mapnik/issues/1519
+    agg::rgba8 pre_c = agg::rgba8(red_,green_,blue_,alpha_);
+    pre_c.demultiply();
+    red_ = pre_c.r;
+    green_ = pre_c.g;
+    blue_ = pre_c.b;
+}
+
 }
 

src/grid/process_building_symbolizer.cpp

@@ -32,6 +32,9 @@
 // boost
 #include <boost/scoped_ptr.hpp>
 
+// stl
+#include <deque>
+
 // agg
 #include "agg_rasterizer_scanline_aa.h"
 #include "agg_renderer_scanline.h"

src/jpeg_reader.cpp

@@ -50,7 +50,7 @@ public:
     ~JpegReader();
     unsigned width() const;
     unsigned height() const;
-    inline bool premultiplied_alpha() const { return true ;}
+    inline bool premultiplied_alpha() const { return true; }
     void read(unsigned x,unsigned y,image_data_32& image);
 private:
     void init();

src/load_map.cpp

@@ -670,7 +670,14 @@ void map_parser::parse_layer(Map & map, xml_node const& node)
                 {
                     std::map<std::string,parameters>::const_iterator base_itr = datasource_templates_.find(*base);
                     if (base_itr!=datasource_templates_.end())
+                    {
                         params = base_itr->second;
+                    }
+                    else
+                    {
+                        MAPNIK_LOG_ERROR(datasource) << "Datasource template '" << *base
+                            << "' not found for layer '" << name << "'";
+                    }
                 }
 
                 xml_node::const_iterator paramIter = child->begin();

src/marker_cache.cpp

@@ -198,11 +198,12 @@ boost::optional<marker_ptr> marker_cache::find(std::string const& uri,
                     BOOST_ASSERT(width > 0 && height > 0);
                     mapnik::image_ptr image(boost::make_shared<mapnik::image_data_32>(width,height));
                     reader->read(0,0,*image);
-                    // ensure images are premultiplied
-                    // TODO - don't need to multiply jpegs
-                    agg::rendering_buffer buffer(image->getBytes(),image->width(),image->height(),image->width() * 4);
-                    agg::pixfmt_rgba32 pixf(buffer);
-                    pixf.premultiply();
+                    if (!reader->premultiplied_alpha())
+                    {
+                        agg::rendering_buffer buffer(image->getBytes(),image->width(),image->height(),image->width() * 4);
+                        agg::pixfmt_rgba32 pixf(buffer);
+                        pixf.premultiply();
+                    }
                     marker_ptr mark(boost::make_shared<marker>(image));
                     result.reset(mark);
                     if (update_cache)

tests/cpp_tests/build.py

@@ -20,7 +20,8 @@ for cpp_test in glob.glob('*_test.cpp'):
         agg_env = Environment(ENV=os.environ)
         agg_env['CXX'] = env['CXX']
         agg_env['CXXFLAGS'] = env['CXXFLAGS']
-        agg_env.AppendUnique(LIBS='agg')
+        if 'agg' in test_env['LIBS']:
+            agg_env.AppendUnique(LIBS='agg')
         agg_env.Append(CPPPATH = '#deps/agg/include')
         agg_env.Append(LIBPATH = '#deps/agg')
         agg_env['CPPPATH'] = ['#deps/agg/include',env['BOOST_INCLUDES']]

tests/cpp_tests/fontset_runtime_test.cpp

@@ -0,0 +1,71 @@
+#include <boost/version.hpp>
+#include <boost/detail/lightweight_test.hpp>
+#include <iostream>
+#include <mapnik/memory_datasource.hpp>
+#include <mapnik/feature.hpp>
+#include <mapnik/feature_factory.hpp>
+#include <mapnik/unicode.hpp>
+#include <mapnik/geometry.hpp>
+#include <mapnik/map.hpp>
+#include <mapnik/params.hpp>
+#include <mapnik/expression.hpp>
+#include <mapnik/layer.hpp>
+#include <mapnik/rule.hpp>
+#include <mapnik/feature_type_style.hpp>
+#include <mapnik/agg_renderer.hpp>
+#include <mapnik/graphics.hpp>
+#include <mapnik/image_util.hpp>
+#include <mapnik/color_factory.hpp>
+#include <mapnik/save_map.hpp>
+
+
+int main( int, char*[] )
+{
+    // create a renderable map with a fontset and a text symbolizer
+    // and do not register any fonts, to ensure the error thrown is reasonable
+    mapnik::context_ptr ctx = boost::make_shared<mapnik::context_type>();
+    ctx->push("name");
+    mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
+    mapnik::transcoder tr("utf-8");
+    UnicodeString ustr = tr.transcode("hello world!");
+    feature->put("name",ustr);
+    mapnik::geometry_type * pt = new mapnik::geometry_type(mapnik::Point);
+    pt->move_to(128,128);
+    feature->add_geometry(pt);
+    mapnik::datasource_ptr memory_ds = boost::make_shared<mapnik::memory_datasource>();
+    mapnik::memory_datasource *cache = dynamic_cast<mapnik::memory_datasource *>(memory_ds.get());
+    cache->push(feature);
+    mapnik::Map m(256,256);
+    mapnik::font_set fontset("fontset");
+    // NOTE: this is a valid font, but will fail because none are registered
+    fontset.add_face_name("DejaVu Sans Book");
+    m.insert_fontset("fontset", fontset);
+    mapnik::layer lyr("layer");
+    lyr.set_datasource(memory_ds);
+    lyr.add_style("style");
+    m.addLayer(lyr);
+    mapnik::feature_type_style the_style;
+    mapnik::rule the_rule;
+    mapnik::text_symbolizer text_sym(mapnik::parse_expression("[name]"),10,mapnik::color(0,0,0));
+    text_sym.set_fontset(fontset);
+    the_rule.append(text_sym);
+    the_style.add_rule(the_rule);
+    m.insert_style("style",the_style );
+    m.zoom_to_box(mapnik::box2d<double>(-256,-256,
+                                256,256));
+    mapnik::image_32 buf(m.width(),m.height());
+    mapnik::agg_renderer<mapnik::image_32> ren(m,buf);
+    try {
+        ren.apply();
+    } catch (std::exception const& ex) {
+        BOOST_TEST_EQ(std::string(ex.what()),std::string("No valid font face could be loaded for font set: 'fontset'"));
+    }
+    if (!::boost::detail::test_errors()) {
+        std::clog << "C++ fontset runtime: \x1b[1;32m✓ \x1b[0m\n";
+#if BOOST_VERSION >= 104600
+        ::boost::detail::report_errors_remind().called_report_errors_function = true;
+#endif
+    } else {
+        return ::boost::report_errors();
+    }
+}

tests/data/csv/long_lat.vrt

@@ -0,0 +1,8 @@
+<OGRVRTDataSource>
+    <OGRVRTLayer name="long_lat">
+        <SrcDataSource>long_lat.csv</SrcDataSource>
+        <GeometryType>wkbPoint</GeometryType>
+        <LayerSRS>WGS84</LayerSRS>
+        <GeometryField encoding="PointFromColumns" x="long" y="lat"/>
+    </OGRVRTLayer>
+</OGRVRTDataSource>

tests/data/good_maps/line_symbolizer_offset.xml

@@ -0,0 +1,18 @@
+<Map minimum-version="2.1.0" background-color="white">
+  <Style name="style">
+    <Rule>
+      <LineSymbolizer offset="2"/>
+      <LineSymbolizer offset="-2"/>
+    </Rule>  
+  </Style>
+  <Layer name="lay">
+    <StyleName>style</StyleName>
+    <Datasource>
+      <Parameter name="type">csv</Parameter>
+      <Parameter name="inline">
+      wkt,name
+      "MULTILINESTRING ((10 10, 20 20, 10 40),(40 40, 30 30, 40 20, 30 10))",line
+      </Parameter>
+    </Datasource>
+  </Layer>
+</Map>

tests/data/good_maps/style_level_comp_op.xml

@@ -0,0 +1,49 @@
+<Map srs="+proj=lonlat +ellps=WGS84 +datum=WGS84 +no_defs +over"
+     background-color="lightsteelblue">
+
+  <Style name="land">
+    <Rule>
+      <PolygonSymbolizer fill="darkslategrey" geometry-transform="translate(+1,+1)" />
+      <PolygonSymbolizer fill="lightcyan"     geometry-transform="translate(-1,-1)" />
+      <PolygonSymbolizer fill="forestgreen" />
+    </Rule>
+  </Style>
+
+  <Style name="markers" comp-op="src-over">
+    <Rule>
+      <MarkersSymbolizer fill="saddlebrown"
+                         fill-opacity="0.5"
+                         stroke-width="0"
+                         width="45 - 5 * [SCALERANK]"
+                         height="45 - 5 * [SCALERANK]"
+                         allow-overlap="true"
+                         ignore-placement="true"
+                         />
+      <MarkersSymbolizer fill="white"
+                         fill-opacity="0.7"
+                         stroke-width="0"
+                         width="5"
+                         height="5"
+                         allow-overlap="true"
+                         ignore-placement="true"
+                         />
+    </Rule>
+  </Style>
+
+  <Layer name="land">
+    <StyleName>land</StyleName>
+    <Datasource>
+      <Parameter name="type">shape</Parameter>
+      <Parameter name="file">../shp/new_zealand/ne_50m_land.shp</Parameter>
+    </Datasource>
+  </Layer>
+
+  <Layer name="cities">
+    <StyleName>markers</StyleName>
+    <Datasource>
+      <Parameter name="type">shape</Parameter>
+      <Parameter name="file">../shp/new_zealand/ne_50m_populated_places_simple.shp</Parameter>
+    </Datasource>
+  </Layer>
+
+</Map>

tests/data/good_maps/style_level_image_filter.xml

@@ -0,0 +1,68 @@
+<Map srs="+proj=lonlat +ellps=WGS84 +datum=WGS84 +no_defs +over"
+     background-color="lightsteelblue">
+
+  <Style name="land">
+    <Rule>
+      <PolygonSymbolizer fill="darkslategrey" geometry-transform="translate(+1,+1)" />
+      <PolygonSymbolizer fill="lightcyan"     geometry-transform="translate(-1,-1)" />
+      <PolygonSymbolizer fill="forestgreen" />
+    </Rule>
+  </Style>
+
+  <Style name="markers" comp-op="hard-light">
+    <Rule>
+      <MarkersSymbolizer fill="saddlebrown"
+                         fill-opacity="0.5"
+                         stroke-width="0"
+                         width="45 - 5 * [SCALERANK]"
+                         height="45 - 5 * [SCALERANK]"
+                         allow-overlap="true"
+                         ignore-placement="true"
+                         />
+      <MarkersSymbolizer fill="white"
+                         fill-opacity="0.7"
+                         stroke-width="0"
+                         width="5"
+                         height="5"
+                         allow-overlap="true"
+                         ignore-placement="true"
+                         />
+    </Rule>
+  </Style>
+
+  <Style name="labels" comp-op="overlay">
+    <Rule>
+      <TextSymbolizer face-name="DejaVu Sans Book"
+                      size="12"
+                      fill="black"
+                      halo-fill="white"
+                      halo-radius="2"
+                      dx="7"
+                      dy="1"
+                      avoid-edges="true"
+                      vertical-alignment="middle"
+                      placement-type="simple"
+                      placements="NE,NW"
+                      allow-overlap="false"
+                      >[NAME]</TextSymbolizer>
+    </Rule>
+  </Style>
+
+  <Layer name="land">
+    <StyleName>land</StyleName>
+    <Datasource>
+      <Parameter name="type">shape</Parameter>
+      <Parameter name="file">../shp/new_zealand/ne_50m_land.shp</Parameter>
+    </Datasource>
+  </Layer>
+
+  <Layer name="cities">
+    <StyleName>markers</StyleName>
+    <StyleName>labels</StyleName>
+    <Datasource>
+      <Parameter name="type">shape</Parameter>
+      <Parameter name="file">../shp/new_zealand/ne_50m_populated_places_simple.shp</Parameter>
+    </Datasource>
+  </Layer>
+
+</Map>

tests/data/shp/new_zealand/ne_50m_land.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.017453292519943295]]

tests/data/shp/new_zealand/ne_50m_populated_places_simple.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.017453292519943295]]

tests/python_tests/compositing_test.py

@@ -2,8 +2,8 @@
 
 from nose.tools import *
 import os,sys
-from utilities import execution_path
-from utilities import Todo
+from utilities import execution_path, run_tests, Todo
+from utilities import get_unique_colors, pixel2channels, side_by_side_image
 import mapnik
 
 def setup():
@@ -18,12 +18,13 @@ def debug_image(image,step=2):
     for x in range(0,image.width(),step):
         for y in range(0,image.height(),step):
             pixel = image.get_pixel(x,y)
-            alpha = (pixel >> 24) & 0xff
-            red = pixel & 0xff
-            green = (pixel >> 8) & 0xff
-            blue = (pixel >> 16) & 0xff
+            red,green,blue,alpha = pixel2channels(pixel)
             print "rgba(%s,%s,%s,%s) at %s,%s" % (red,green,blue,alpha,x,y)
 
+def replace_style(m, name, style):
+    m.remove_style(name)
+    m.append_style(name, style)
+
 # note: it is impossible to know for all pixel colors
 # we can only detect likely cases of non premultiplied colors
 def validate_pixels_are_not_premultiplied(image):
@@ -33,14 +34,11 @@ def validate_pixels_are_not_premultiplied(image):
     for x in range(0,image.width(),2):
         for y in range(0,image.height(),2):
             pixel = image.get_pixel(x,y)
-            alpha = (pixel >> 24) & 0xff
+            red,green,blue,alpha = pixel2channels(pixel)
             if alpha > 0:
                 transparent = False
                 if alpha < 255:
                     fully_opaque = False
-                red = pixel & 0xff
-                green = (pixel >> 8) & 0xff
-                blue = (pixel >> 16) & 0xff
                 color_max = max(red,green,blue)
                 if color_max > alpha:
                     over_alpha = True
@@ -51,12 +49,9 @@ def validate_pixels_are_not_premultiplied2(image):
     for x in range(0,image.width(),2):
         for y in range(0,image.height(),2):
             pixel = image.get_pixel(x,y)
-            alpha = (pixel >> 24) & 0xff
+            red,green,blue,alpha = pixel2channels(pixel)
             #each value of the color channels will never be bigger than that of the alpha channel.
             if alpha > 0:
-                red = pixel & 0xff
-                green = (pixel >> 8) & 0xff
-                blue = (pixel >> 16) & 0xff
                 if red > 0 and red > alpha:
                     print 'red: %s, a: %s' % (red,alpha)
                     looks_not_multiplied = True
@@ -67,12 +62,9 @@ def validate_pixels_are_premultiplied(image):
     for x in range(0,image.width(),2):
         for y in range(0,image.height(),2):
             pixel = image.get_pixel(x,y)
-            alpha = (pixel >> 24) & 0xff
+            red,green,blue,alpha = pixel2channels(pixel)
             if alpha > 0:
                 pixel = image.get_pixel(x,y)
-                red = pixel & 0xff
-                green = (pixel >> 8) & 0xff
-                blue = (pixel >> 16) & 0xff
                 is_valid = ((0 <= red <= alpha) and is_pre(red,alpha)) \
                         and ((0 <= green <= alpha) and is_pre(green,alpha)) \
                         and ((0 <= blue <= alpha) and is_pre(blue,alpha)) \
@@ -96,10 +88,10 @@ def test_compare_images():
         expected = 'images/composited/' + name + '.png'
         valid = validate_pixels_are_premultiplied(a)
         if not valid[0]:
-            print '%s not validly pre-:\n\t%s pixels (%s)' % (name,len(valid[1]),valid[1][0])
+            fails.append('%s not validly premultiplied!:\n\t %s pixels (%s)' % (name,len(valid[1]),valid[1][0]))
         a.demultiply()
         if not validate_pixels_are_not_premultiplied(a):
-            print '%s not validly demultiplied' % (name)
+            fails.append('%s not validly demultiplied' % (name))
         a.save(actual)
         if not os.path.exists(expected):
             print 'generating expected test image: %s' % expected
@@ -110,6 +102,8 @@ def test_compare_images():
             successes.append(name)
         else:
             fails.append('failed comparing actual (%s) and expected(%s)' % (actual,'tests/python_tests/'+ expected))
+            fail_im = side_by_side_image(expected_im, a)
+            fail_im.save('/tmp/mapnik-comp-op-test-' + name + '.fail.png')
     eq_(len(successes),num_ops,'\n'+'\n'.join(fails))
     b.demultiply()
     # b will be slightly modified by pre and then de multiplication rounding errors
@@ -155,6 +149,36 @@ def test_pre_multiply_status_of_map2():
     mapnik.render(m,im)
     eq_(validate_pixels_are_not_premultiplied(im),True)
 
+def test_style_level_comp_op():
+    m = mapnik.Map(256, 256)
+    mapnik.load_map(m, '../data/good_maps/style_level_comp_op.xml')
+    m.zoom_all()
+    successes = []
+    fails = []
+    for name in mapnik.CompositeOp.names:
+        # find_style returns a copy of the style object
+        style_markers = m.find_style("markers")
+        style_markers.comp_op = getattr(mapnik.CompositeOp, name)
+        # replace the original style with the modified one
+        replace_style(m, "markers", style_markers)
+        im = mapnik.Image(m.width, m.height)
+        mapnik.render(m, im)
+        actual = '/tmp/mapnik-style-comp-op-' + name + '.png'
+        expected = 'images/style-comp-op/' + name + '.png'
+        im.save(actual)
+        if not os.path.exists(expected):
+            print 'generating expected test image: %s' % expected
+            im.save(expected)
+        expected_im = mapnik.Image.open(expected)
+        # compare them
+        if im.tostring() == expected_im.tostring():
+            successes.append(name)
+        else:
+            fails.append('failed comparing actual (%s) and expected(%s)' % (actual,'tests/python_tests/'+ expected))
+            fail_im = side_by_side_image(expected_im, im)
+            fail_im.save('/tmp/mapnik-style-comp-op-' + name + '.fail.png')
+    eq_(len(fails), 0, '\n'+'\n'.join(fails))
+
 def test_style_level_opacity():
     m = mapnik.Map(512,512)
     mapnik.load_map(m,'../data/good_maps/style_level_opacity_and_blur.xml')
@@ -167,6 +191,71 @@ def test_style_level_opacity():
     expected_im = mapnik.Image.open(expected)
     eq_(im.tostring(),expected_im.tostring(), 'failed comparing actual (%s) and expected (%s)' % (actual,'tests/python_tests/'+ expected))
 
+def test_rounding_and_color_expectations():
+    m = mapnik.Map(1,1)
+    m.background = mapnik.Color('rgba(255,255,255,.4999999)')
+    im = mapnik.Image(m.width,m.height)
+    mapnik.render(m,im)
+    # ugh 252, see: https://github.com/mapnik/mapnik/issues/1519
+    eq_(get_unique_colors(im),['rgba(252,252,252,127)'])
+    m = mapnik.Map(1,1)
+    m.background = mapnik.Color('rgba(255,255,255,.5)')
+    im = mapnik.Image(m.width,m.height)
+    mapnik.render(m,im)
+    eq_(get_unique_colors(im),['rgba(253,253,253,128)'])
+    im_file = mapnik.Image.open('../data/images/stripes_pattern.png')
+    eq_(get_unique_colors(im_file),['rgba(0,0,0,0)', 'rgba(74,74,74,255)'])
+    # should have no effect
+    im_file.premultiply()
+    eq_(get_unique_colors(im_file),['rgba(0,0,0,0)', 'rgba(74,74,74,255)'])
+    im_file.set_alpha(.5)
+    # should have effect now that image has transparency
+    im_file.premultiply()
+    eq_(get_unique_colors(im_file),['rgba(0,0,0,0)', 'rgba(37,37,37,127)'])
+    # should restore to original nonpremultiplied colors
+    im_file.demultiply()
+    eq_(get_unique_colors(im_file),['rgba(0,0,0,0)', 'rgba(74,74,74,127)'])
+
+
+def test_background_image_and_background_color():
+    m = mapnik.Map(8,8)
+    m.background = mapnik.Color('rgba(255,255,255,.5)')
+    m.background_image = '../data/images/stripes_pattern.png'
+    im = mapnik.Image(m.width,m.height)
+    mapnik.render(m,im)
+    # note: data loss due to rounding as per https://github.com/mapnik/mapnik/issues/1519
+    # means that background will roundtrip to 253 not 255
+    #eq_(get_unique_colors(im),['rgba(255,255,255,128)', 'rgba(74,74,74,255)'])
+    eq_(get_unique_colors(im),['rgba(253,253,253,128)', 'rgba(74,74,74,255)'])
+
+def test_background_image_with_alpha_and_background_color():
+    m = mapnik.Map(10,10)
+    m.background = mapnik.Color('rgba(255,255,255,.5)')
+    m.background_image = '../data/images/yellow_half_trans.png'
+    im = mapnik.Image(m.width,m.height)
+    mapnik.render(m,im)
+    eq_(get_unique_colors(im),['rgba(255,255,85,191)'])
+
+def test_background_image_with_alpha_and_background_color_against_composited_control():
+    m = mapnik.Map(10,10)
+    m.background = mapnik.Color('rgba(255,255,255,.5)')
+    m.background_image = '../data/images/yellow_half_trans.png'
+    im = mapnik.Image(m.width,m.height)
+    mapnik.render(m,im)
+    # create and composite the expected result
+    im1 = mapnik.Image(10,10)
+    im1.background = mapnik.Color('rgba(255,255,255,.5)')
+    im1.premultiply()
+    im2 = mapnik.Image(10,10)
+    im2.background = mapnik.Color('rgba(255,255,0,.5)')
+    im2.premultiply()
+    im1.composite(im2)
+    im1.demultiply()
+    # compare image rendered (compositing in `agg_renderer<T>::setup`)
+    # vs image composited via python bindings
+    raise Todo("looks like we need to investigate PNG rounding when saving")
+    eq_(get_unique_colors(im),get_unique_colors(im1))
+
 if __name__ == "__main__":
     setup()
-    [eval(run)() for run in dir() if 'test_' in run]
+    run_tests(eval(x) for x in dir() if x.startswith("test_"))

tests/python_tests/image_filters_test.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+from nose.tools import *
+from utilities import execution_path, run_tests
+from utilities import side_by_side_image
+import os, mapnik
+import re
+
+def setup():
+    # All of the paths used are relative, if we run the tests
+    # from another directory we need to chdir()
+    os.chdir(execution_path('.'))
+
+def replace_style(m, name, style):
+    m.remove_style(name)
+    m.append_style(name, style)
+
+def test_append():
+    s = mapnik.Style()
+    eq_(s.image_filters,'')
+    s.image_filters = 'gray'
+    eq_(s.image_filters,'gray')
+    s.image_filters = 'sharpen'
+    eq_(s.image_filters,'sharpen')
+
+def test_style_level_image_filter():
+    m = mapnik.Map(256, 256)
+    mapnik.load_map(m, '../data/good_maps/style_level_image_filter.xml')
+    m.zoom_all()
+    successes = []
+    fails = []
+    for name in ("", "agg-stack-blur(2,2)", "blur",
+                 "edge-detect", "emboss", "gray", "invert",
+                 "sharpen", "sobel", "x-gradient", "y-gradient"):
+        if name == "":
+            filename = "none"
+        else:
+            filename = re.sub(r"[^-_a-z.0-9]", "", name)
+        # find_style returns a copy of the style object
+        style_markers = m.find_style("markers")
+        style_markers.image_filters = name
+        style_labels = m.find_style("labels")
+        style_labels.image_filters = name
+        # replace the original style with the modified one
+        replace_style(m, "markers", style_markers)
+        replace_style(m, "labels", style_labels)
+        im = mapnik.Image(m.width, m.height)
+        mapnik.render(m, im)
+        actual = '/tmp/mapnik-style-image-filter-' + filename + '.png'
+        expected = 'images/style-image-filter/' + filename + '.png'
+        im.save(actual)
+        if not os.path.exists(expected):
+            print 'generating expected test image: %s' % expected
+            im.save(expected)
+        expected_im = mapnik.Image.open(expected)
+        # compare them
+        if im.tostring() == expected_im.tostring():
+            successes.append(name)
+        else:
+            fails.append('failed comparing actual (%s) and expected(%s)' % (actual,'tests/python_tests/'+ expected))
+            fail_im = side_by_side_image(expected_im, im)
+            fail_im.save('/tmp/mapnik-style-image-filter-' + filename + '.fail.png')
+    eq_(len(fails), 0, '\n'+'\n'.join(fails))
+
+if __name__ == "__main__":
+    setup()
+    run_tests(eval(x) for x in dir() if x.startswith("test_"))

tests/python_tests/raster_symbolizer_test.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 from nose.tools import *
-from utilities import execution_path, contains_word
+from utilities import execution_path, contains_word, get_unique_colors
 
 import os, mapnik
 
@@ -103,22 +103,6 @@ def test_load_save_map():
         if not 'Could not create datasource' in str(e):
             raise RuntimeError(str(e))
 
-def pixel2rgba(pixel):
-    alpha = (pixel >> 24) & 0xff
-    red = pixel & 0xff
-    green = (pixel >> 8) & 0xff
-    blue = (pixel >> 16) & 0xff
-    return 'rgba(%s,%s,%s,%s)' % (red,green,blue,alpha)
-
-def get_unique_colors(im):
-    pixels = []
-    for x in range(im.width()):
-        for y in range(im.height()):
-            pixel = im.get_pixel(x,y)
-            if pixel not in pixels:
-                 pixels.append(pixel)
-    return map(pixel2rgba,pixels)
-
 def test_raster_with_alpha_blends_correctly_with_background():
     WIDTH = 500
     HEIGHT = 500
@@ -151,7 +135,7 @@ def test_raster_with_alpha_blends_correctly_with_background():
         mapnik.render(map, mim)
         imdata = mim.tostring()
         # All white is expected
-        eq_(contains_word('\xff\xff\xff\xff', imdata),True,'Image expected to contain true white, instead found %s' % get_unique_colors(mim))
+        eq_(get_unique_colors(mim),['rgba(254,254,254,255)'])
 
 def test_raster_warping():
     lyrSrs = "+init=epsg:32630"

tests/python_tests/render_test.py

@@ -140,7 +140,7 @@ def test_render_points():
         p = mapnik.Projection(projs[projdescr])
         m.zoom_to_box(p.forward(mapnik.Box2d(ul_lonlat,lr_lonlat)))
         # Render to SVG so that it can be checked how many points are there with string comparison
-        svg_file = os.path.join(tempfile.gettempdir(),'%s.svg')
+        svg_file = os.path.join(tempfile.gettempdir(), 'mapnik-render-points-%s.svg' % projdescr)
         mapnik.render_to_file(m, svg_file)
         num_points_present = len(ds.all_features())
         svg = open(svg_file,'r').read()

tests/python_tests/sqlite_test.py

@@ -334,6 +334,32 @@ if 'sqlite' in mapnik.DatasourceCache.plugin_names():
             pass
         eq_(feature,None)
 
+    # https://github.com/mapnik/mapnik/issues/1537
+    # this works because key_field is manually set
+    def test_db_with_one_text_column():
+        # form up an in-memory test db
+        wkb = '010100000000000000000000000000000000000000'
+        ds = mapnik.SQLite(file=':memory:',
+            table='test1',
+            initdb='''
+                create table test1 (alias TEXT,geometry BLOB);
+                insert into test1 values ("test",x'%s');
+                ''' % wkb,
+            extent='-180,-60,180,60',
+            use_spatial_index=False,
+            key_field='alias'
+        )
+        eq_(len(ds.fields()),1)
+        eq_(ds.fields(),['alias'])
+        eq_(ds.field_types(),['str'])
+        fs = ds.all_features()
+        eq_(len(fs),1)
+        feat = fs[0]
+        #eq_(feat.id(),1)
+        eq_(feat['alias'],'test')
+        eq_(len(feat.geometries()),1)
+        eq_(feat.geometries()[0].to_wkt(),'Point(0.0 0.0)')
+
 if __name__ == "__main__":
     setup()
     [eval(run)() for run in dir() if 'test_' in run]

tests/python_tests/utilities.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
 from nose.plugins.errorclass import ErrorClass, ErrorClassPlugin
 
-import os, sys, inspect
+import os, sys, inspect, traceback
+import mapnik
 
 def execution_path(filename):
     return os.path.join(os.path.dirname(sys._getframe(1).f_code.co_filename), filename)
@@ -35,3 +37,51 @@ def contains_word(word, bytestring_):
     assert len(bytestring_)%n == 0, "len(bytestring_) not multiple of len(word)"
     chunks = [bytestring_[i:i+n] for i in xrange(0, len(bytestring_), n)]
     return word in chunks
+
+def pixel2channels(pixel):
+    alpha = (pixel >> 24) & 0xff
+    red = pixel & 0xff
+    green = (pixel >> 8) & 0xff
+    blue = (pixel >> 16) & 0xff
+    return red,green,blue,alpha
+
+def pixel2rgba(pixel):
+    return 'rgba(%s,%s,%s,%s)' % pixel2channels(pixel)
+
+def get_unique_colors(im):
+    pixels = []
+    for x in range(im.width()):
+        for y in range(im.height()):
+            pixel = im.get_pixel(x,y)
+            if pixel not in pixels:
+                 pixels.append(pixel)
+    pixels = sorted(pixels)
+    return map(pixel2rgba,pixels)
+
+def run_tests(iterable):
+    failed = 0
+    for test in iterable:
+        try:
+            test()
+            sys.stderr.write("\x1b[32m✓ \x1b[m" + test.__name__ + "\x1b[m\n")
+        except:
+            exc_type, exc_value, exc_tb = sys.exc_info()
+            failed += 1
+            sys.stderr.write("\x1b[31m✘ \x1b[m" + test.__name__ + "\x1b[m\n")
+            for mline in traceback.format_exception_only(exc_type, exc_value):
+                for line in mline.rstrip().split("\n"):
+                    sys.stderr.write("  \x1b[31m" + line + "\x1b[m\n")
+            sys.stderr.write("  Traceback:\n")
+            for mline in traceback.format_tb(exc_tb):
+                for line in mline.rstrip().split("\n"):
+                    sys.stderr.write("  " + line + "\n")
+        sys.stderr.flush()
+    return failed
+
+def side_by_side_image(left_im, right_im):
+    width = left_im.width() + 1 + right_im.width()
+    height = max(left_im.height(), right_im.height())
+    im = mapnik.Image(width, height)
+    im.blend(0, 0, left_im, 1.0)
+    im.blend(left_im.width() + 1, 0, right_im, 1.0)
+    return im

tests/visual_tests/test_python.py

@@ -71,7 +71,7 @@ m.append_style('Style', style)
 
 
 layer = mapnik.Layer('Layer')
-layer.datasource = mapnik.Shapefile(file=os.path.join(dirname,"data/points.shp"))
+layer.datasource = mapnik.Osm(file=os.path.join(dirname,"data/points.osm"))
 layer.styles.append('Style')
 m.layers.append(layer)
 

utils/svg2png/svg2png.cpp

@@ -142,7 +142,7 @@ int main (int argc,char** argv)
                 continue;
             }
 
-            typedef agg::pixfmt_rgba32_plain pixfmt;
+            typedef agg::pixfmt_rgba32_pre pixfmt;
             typedef agg::renderer_base<pixfmt> renderer_base;
             typedef agg::renderer_scanline_aa_solid<renderer_base> renderer_solid;
             agg::rasterizer_scanline_aa<> ras_ptr;
@@ -173,12 +173,13 @@ int main (int argc,char** argv)
             mapnik::svg::svg_renderer_agg<mapnik::svg::svg_path_adapter,
                 agg::pod_bvector<mapnik::svg::path_attributes>,
                 renderer_solid,
-                agg::pixfmt_rgba32_plain > svg_renderer_this(svg_path,
+                agg::pixfmt_rgba32_pre > svg_renderer_this(svg_path,
                                                              (*marker.get_vector_data())->attributes());
 
             svg_renderer_this.render(ras_ptr, sl, renb, mtx, opacity, bbox);
 
             boost::algorithm::ireplace_last(svg_name,".svg",".png");
+            im.demultiply();
             mapnik::save_to_file<mapnik::image_data_32>(im.data(),svg_name,"png");
             if (auto_open)
             {