Merge pull request #1337 from rjw57/rjw57-python-plugin

Request for comments: python: a new plugin to use arbitrary Python as a data source

AUTHORS.md

@@ -57,5 +57,6 @@ Mapnik is written by Artem Pavlenko with contributions from:
 * Andreas Volz
 * Lennard voor den Dag
 * Shaun Walbridge
+* Rich Wareham
 * Nick Whitelegg
 * Leslie Wu

SConstruct

@@ -110,6 +110,7 @@ PLUGINS = { # plugins with external dependencies
             'raster':  {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
             'geojson': {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
             'kismet':  {'default':False,'path':None,'inc':None,'lib':None,'lang':'C++'},
+            'python':  {'default':True,'path':None,'inc':None,'lib':None,'lang':'C++'},
             }
 
 

bindings/python/mapnik/__init__.py

@@ -39,6 +39,7 @@ Several things happen when you do:
 
 """
 
+import itertools
 import os
 import sys
 import warnings
@@ -594,6 +595,75 @@ def Geos(**keywords):
     keywords['type'] = 'geos'
     return CreateDatasource(keywords)
 
+def Python(**keywords):
+    """Create a Python Datasource.
+
+    >>> from mapnik import Python, PythonDatasource
+    >>> datasource = Python('PythonDataSource')
+    >>> lyr = Layer('Python datasource')
+    >>> lyr.datasource = datasource
+    """
+    keywords['type'] = 'python'
+    return CreateDatasource(keywords)
+
+class PythonDatasource(object):
+    """A base class for a Python data source.
+
+    Optional arguments:
+      envelope -- a mapnik.Box2d (minx, miny, maxx, maxy) envelope of the data source, default (-180,-90,180,90)
+      geometry_type -- one of the DataGeometryType enumeration values, default Point
+      data_type -- one of the DataType enumerations, default Vector
+    """
+    def __init__(self, envelope=None, geometry_type=None, data_type=None):
+        self.envelope = envelope or Box2d(-180, -90, 180, 90)
+        self.geometry_type = geometry_type or DataGeometryType.Point
+        self.data_type = data_type or DataType.Vector
+
+    def features(self, query):
+        """Return an iterable which yields instances of Feature for features within the passed query.
+        
+        Required arguments:
+          query -- a Query instance specifying the region for which features should be returned
+        """
+        return None
+
+    def features_at_point(self, point):
+        """Rarely uses. Return an iterable which yields instances of Feature for the specified point."""
+        return None
+
+    @classmethod
+    def wkb_features(cls, keys, features):
+        """A convenience function to wrap an iterator yielding pairs of WKB format geometry and dictionaries of
+        key-value pairs into mapnik features. Return this from PythonDatasource.features() passing it a sequence of keys
+        to appear in the output and an iterator yielding features.
+
+        For example. One might have a features() method in a derived class like the following:
+
+        def features(self, query):
+            # ... create WKB features feat1 and feat2
+
+            return mapnik.PythonDatasource.wkb_features(
+                keys = ( 'name', 'author' ),
+                features = [
+                    (feat1, { 'name': 'feat1', 'author': 'alice' }),
+                    (feat2, { 'name': 'feat2', 'author': 'bob' }),
+                ]
+            )
+
+        """
+        ctx = Context()
+        [ctx.push(x) for x in keys]
+
+        def make_it(feat, idx):
+            f = Feature(ctx, idx)
+            geom, attrs = feat
+            f.add_geometries_from_wkb(geom)
+            for k, v in attrs.iteritems():
+                f[k] = v
+            return f
+
+        return itertools.imap(make_it, features, itertools.count(1))
+
 class _TextSymbolizer(TextSymbolizer,_injector):
     @property
     def text_size(self):

plugins/input/python/README.md

@@ -0,0 +1,241 @@
+# Python plugin
+
+This plugin allows you to write data sources in the Python programming language.
+This is useful if you want to rapidly prototype a plugin, perform some custom
+manipulation on data or if you want to bind mapnik to a datasource which is most
+conveniently accessed through Python.
+
+The plugin may be used from the existing mapnik Python bindings or it can embed
+the Python interpreter directly allowing it to be used from C++, XML or even
+JavaScript.
+
+## Rationale
+
+Mapnik already has excellent Python bindings but they only directly support
+calling *into* mapnik *from* Python. This forces mapnik and its input plugins to
+be the lowest layer of the stack. The role of this plugin is to allow mapnik to
+call *into* Python itself. This allows mapnik to sit as rendering middleware
+between a custom Python frontend and a custom Python datasource. This increases
+the utility of mapnik as a component in a larger system.
+
+There already exists MemoryDatasource which can be used to dynamically create
+geometry in Python. It suffers from the problem that it does not allow
+generating only the geometry which is seen by a particular query. Similarly the
+entire geometry must exist in memory before rendering can progress. By using a
+custom iterator object or by using generator expressions this plugin allows
+geometry to be created on demand and to be destroyed after use. This can have a
+great impact on memory efficiency. Since geometry is generated on-demand as
+rendering progresses there can be arbitrarily complex 'cleverness' optimising
+the geometry generated for a particular query. Obvious examples of this would
+be generating only geometry within the query bounding box and generating
+geometry with an appropriate level of detail for the output resolution.
+
+## Initialization
+
+Only the `factory` parameter is required. This is of the form
+`[module:]callable`. If `module` is present then `module` will be imported and
+its attribute named `callable` will be used as a factory callable. If `module`
+is omitted, then `__main__` is used. Any other parameter aside from `factory` or
+`type` will be passed directly to the callable as keyword arguments. Note that
+these will always be passed as strings even if the parameter can be parsed as an
+integer of floating point value.
+
+The callable should return an object with the following required attributes:
+
+* `envelope` - a 4-tuple giving the (minx, miny, maxx, maxy) extent of the
+  datasource;
+
+* `data_type` - a `mapnik.DataType` instance giving the type of data stored in
+  this datasource. This will usually be one of `mapnik.DataType.Vector` or
+  `mapnik.DataType.Raster`.
+
+The following attributes are optional:
+
+* `geometry_type` - if the dataset is a vector dataset, this is an instance of
+  `mapnik.DataGeometryType` giving the type of geometry returned by the
+   datasource.
+
+The following methods must be present:
+
+* `features(query)` - takes a single argument which is an instance of
+  `mapnik.Query` and returns an iterable of `mapnik.Feature` instances for that
+  query.
+
+* `features_at_point(point)` - almost never used. Takes a single argument which
+  is an instance of `mapnik.Point` (I think) and returns an iterable of
+  features associated with that point.
+
+## Convenience classes
+
+The standard `mapnik` module provides a convenience class called
+`mapnik.PythonDatasource` which has default implementations for the required
+methods and accepts the geometry type, data type and envelope as constructor
+arguments. It also provides some convenience class methods which take care of
+constructing features for you:
+
+* `mapnik.PythonDatasource.wkb_features` - constructs features from
+  well-known-binary (WKB) format geometry. Takes two keyword arguments: `keys`
+  which is a sequence of keys associated with each feature and `features` which
+  is a sequence of pairs. The first element in each pair is the WKB
+  representation of the feature and the second element is a dictionary mapping
+  keys to values.
+
+# Caveats
+
+* If used directly from C++, `Py_Initialize()` must have been called before the
+  plugin is loaded to initialise the interpreter correctly.
+
+* When inside the interpreter the global interpreter lock is held each time a
+  feature is fetched and so multi-threaded rendering performance may suffer. You
+  can mitigate this by making sure that the feature iterator yields its value as
+  quickly as possible, potentially from an in-memory buffer filled fom another
+  process over IPC.
+
+# Examples
+
+In XML:
+
+```xml
+<?xml version="1.0" encoding="utf-8"?>
+<Map srs="+init=epsg:4326" background-color="white">
+    <Style name="style">
+        <Rule>
+            <PointSymbolizer />
+            <TextSymbolizer name="[label]" face_name="DejaVu Sans Book" size="10" dx="5" dy="5"/>
+        </Rule>
+    </Style>
+    <Layer name="test" srs="+init=epsg:4326">
+        <StyleName>style</StyleName>
+        <Datasource>
+            <Parameter name="type">python</Parameter>
+            <Parameter name="factory">test:TestDatasource</Parameter>
+        </Datasource>
+    </Layer>
+</Map>
+```
+
+In Python using the shapely geometry library:
+
+```python
+import mapnik
+from shapely.geometry import *
+
+class TestDatasource(mapnik.PythonDatasource):
+    def __init__(self):
+        super(TestDatasource, self).__init__()
+
+    def features(self, query):
+        return mapnik.PythonDatasource.wkb_features(
+            keys = ('label',), 
+            features = (
+                ( Point(5,6).wkb, { 'label': 'foo-bar'} ), 
+                ( Point(100,60).wkb, { 'label': 'buzz-quux'} ), 
+            )
+        )
+
+if __name__ == '__main__':
+    m = mapnik.Map(1280,1024)
+    m.background = mapnik.Color('white')
+    s = mapnik.Style()
+    r = mapnik.Rule()
+    r.symbols.append(mapnik.PointSymbolizer())
+    t = mapnik.TextSymbolizer(mapnik.Expression("[label]"),"DejaVu Sans Book",10,mapnik.Color('black'))
+    t.displacement = (5,5)
+    r.symbols.append(t)
+    s.rules.append(r)
+    m.append_style('point_style',s)
+    ds = mapnik.Python(factory='TestDatasource')
+    layer = mapnik.Layer('python')
+    layer.datasource = ds
+    layer.styles.append('point_style')
+    m.layers.append(layer)
+    m.zoom_all()
+    mapnik.render_to_file(m,'map.png', 'png')
+```
+
+A more complex Python example which makes use of iterators to generate geometry
+dynamically:
+
+```python
+"""A more complex example which renders an infinite series of concentric
+circles centred on a point.
+
+The circles are represented by a Python iterator which will yield only the
+circles which intersect the query's bounding box. The advantage of this
+approach over a MemoryDatasource is that a) only those circles which intersect
+the viewport are actually generated and b) only the memory for the largest
+circle need be available since each circle is created on demand and destroyed
+when finished with.
+"""
+import math
+import mapnik
+from shapely.geometry import *
+
+def box2d_to_shapely(box):
+    import shapely.geometry
+    return shapely.geometry.box(box.minx, box.miny, box.maxx, box.maxy)
+
+class ConcentricCircles(object):
+    def __init__(self, centre, bounds, step=1):
+        self.centre = centre
+        self.bounds = bounds
+        self.step = step
+
+    class Iterator(object):
+        def __init__(self, container):
+            self.container = container
+
+            centre = self.container.centre
+            bounds = self.container.bounds
+            step = self.container.step
+
+            if centre.within(bounds):
+                self.radius = 0
+            else:
+                self.radius = math.ceil(centre.distance(bounds) / float(step)) * step
+
+        def next(self):
+            circle = self.container.centre.buffer(self.radius)
+            self.radius += self.container.step
+
+            # has the circle grown so large that the boundary is entirely within it?
+            if circle.contains(self.container.bounds):
+                raise StopIteration()
+
+            return ( circle.wkb, { } )
+
+    def __iter__(self):
+        return ConcentricCircles.Iterator(self)
+
+class TestDatasource(mapnik.PythonDatasource):
+    def __init__(self):
+        super(TestDatasource, self).__init__(geometry_type=mapnik.DataGeometryType.Polygon)
+
+    def features(self, query):
+        # Get the query bounding-box as a shapely bounding box
+        bounding_box = box2d_to_shapely(query.bbox)
+        centre = Point(-20, 0)
+
+        return mapnik.PythonDatasource.wkb_features(
+            keys = (),
+            features = ConcentricCircles(centre, bounding_box, 0.5)
+        )
+
+if __name__ == '__main__':
+    m = mapnik.Map(640, 320)
+
+    m.background = mapnik.Color('white')
+    s = mapnik.Style()
+    r = mapnik.Rule()
+    r.symbols.append(mapnik.LineSymbolizer())
+    s.rules.append(r)
+    m.append_style('point_style',s)
+    ds = mapnik.Python(factory='TestDatasource')
+    layer = mapnik.Layer('python')
+    layer.datasource = ds
+    layer.styles.append('point_style')
+    m.layers.append(layer)
+    box = mapnik.Box2d(-60, -60, 0, -30)
+    m.zoom_to_box(box)
+    mapnik.render_to_file(m,'map.png', 'png')
+```

plugins/input/python/build.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python
+
+# Mapnik uses the build tool SCons.
+
+# This python file is run to compile a plugin
+# It must be called from the main 'SConstruct' file like:
+
+# SConscript('path/to/this/file.py')
+
+# see docs at: http://www.scons.org/wiki/SConscript()
+
+import os
+
+# Give this plugin a name
+# here this happens to be the same as the directory
+PLUGIN_NAME = 'python'
+
+# Here we pull from the SCons environment exported from the main instance
+Import ('plugin_base')
+Import ('env')
+
+# the below install details are also pulled from the
+# main SConstruct file where configuration happens
+
+# plugins can go anywhere, and be registered in custom locations by Mapnik
+# but the standard location is '/usr/local/lib/mapnik/input'
+install_dest = env['MAPNIK_INPUT_PLUGINS_DEST']
+
+# clone the environment here
+# so that if we modify the env it in this file
+# those changes to not pollute other builds later on...
+plugin_env = plugin_base.Clone()
+
+# Add the cpp files that need to be compiled
+plugin_sources = Split(
+  """
+  %(PLUGIN_NAME)s_datasource.cpp
+  %(PLUGIN_NAME)s_featureset.cpp      
+  """ % locals()
+        )
+
+# Add any external libraries this plugin should
+# directly link to
+libraries = [ '' ] # eg 'libfoo'
+
+libraries.append('mapnik')
+libraries.append('boost_system%s' % env['BOOST_APPEND'])
+# link libicuuc, but ICU_LIB_NAME is used custom builds of icu can
+# have different library names like osx which offers /usr/lib/libicucore.dylib
+libraries.append(env['ICU_LIB_NAME'])
+
+libraries.append(env['BOOST_PYTHON_LIB'])
+
+# TODO - do solaris/fedora need direct linking too?
+if env['PLATFORM'] == 'Darwin':
+    if not env['PYTHON_DYNAMIC_LOOKUP']:
+        libraries.append('png')
+        if env['JPEG']:
+            libraries.append('jpeg')
+        libraries.append(env['ICU_LIB_NAME'])
+        libraries.append('boost_regex%s' % env['BOOST_APPEND'])
+        if env['THREADING'] == 'multi':
+            libraries.append('boost_thread%s' % env['BOOST_APPEND'])
+
+    ##### Python linking on OS X is tricky ### 
+    # Confounding problems are:
+    # 1) likelyhood of multiple python installs of the same major.minor version
+    #  because apple supplies python built-in and many users may have installed 
+    #  further versions using macports
+    # 2) boost python directly links to a python version
+    # 3) the below will directly link _mapnik.so to a python version
+    # 4) _mapnik.so must link to the same python lib as boost_python.dylib otherwise
+    # python will Abort with a Version Mismatch error.
+    # See http://trac.mapnik.org/ticket/453 for the seeds of a better approach
+    # for now we offer control over method of direct linking...
+    # The default below is to link against the python dylib in the form of
+    #/path/to/Python.framework/Python instead of -lpython
+    
+    # http://developer.apple.com/mac/library/DOCUMENTATION/Darwin/Reference/ManPages/man1/ld.1.html
+    
+    if env['PYTHON_DYNAMIC_LOOKUP']:
+        python_link_flag = '-undefined dynamic_lookup'
+    elif env['FRAMEWORK_PYTHON']:
+        if env['FRAMEWORK_SEARCH_PATH']:
+            # if the user has supplied a custom root path to search for 
+            # a given Python framework, then use that to direct the linker
+            python_link_flag = '-F%s -framework Python -Z' % env['FRAMEWORK_SEARCH_PATH']
+        else:
+            # otherwise be as explicit as possible for linking to the same Framework
+            # as the executable we are building with (or is pointed to by the PYTHON variable)
+            # otherwise we may accidentally link against either:
+            # /System/Library/Frameworks/Python.framework/Python/Versions/
+            # or
+            # /Library/Frameworks/Python.framework/Python/Versions/
+            # See: http://trac.mapnik.org/ticket/380
+            link_prefix = env['PYTHON_SYS_PREFIX']
+            if '.framework' in link_prefix:
+                python_link_flag = '-F%s -framework Python -Z' % os.path.dirname(link_prefix.split('.')[0])
+            elif '/System' in link_prefix:
+                python_link_flag = '-F/System/Library/Frameworks/ -framework Python -Z'
+            else:
+                # should we fall back to -lpython here?
+                python_link_flag = '-F/ -framework Python'
+            
+    # if we are not linking to a framework then use the *nix standard approach
+    else:
+        # TODO - do we need to pass -L/?
+        python_link_flag = '-lpython%s' % env['PYTHON_VERSION']
+
+elif env['PLATFORM'] == 'SunOS':
+    # make sure to explicitly link mapnik.so against
+    # libmapnik in its installed location
+    python_link_flag = '-R%s' % env['MAPNIK_LIB_BASE']
+else:
+    # all other platforms we don't directly link python
+    python_link_flag = ''
+
+if env['CUSTOM_LDFLAGS']:
+    linkflags = '%s %s' % (env['CUSTOM_LDFLAGS'], python_link_flag)
+else:
+    linkflags = python_link_flag
+
+plugin_env.Append(CPPPATH = env['PYTHON_INCLUDES'])
+    
+TARGET = plugin_env.SharedLibrary(
+              # the name of the target to build, eg 'sqlite.input'
+              '../%s' % PLUGIN_NAME,
+              # prefix - normally none used
+              SHLIBPREFIX='',
+              # extension, mapnik expects '.input'
+              SHLIBSUFFIX='.input',
+              # list of source files to compile
+              source=plugin_sources,
+              # libraries to link to
+              LIBS=libraries,
+              # any custom linkflags, eg. LDFLAGS
+              # in this case CUSTOM_LDFLAGS comes
+              # from Mapnik's main SConstruct file
+              # and can be removed here if you do
+              # not need it
+              LINKFLAGS=env.get('CUSTOM_LDFLAGS')
+              )
+
+# if 'uninstall' is not passed on the command line
+# then we actually create the install targets that
+# scons will install if 'install' is passed as an arg
+if 'uninstall' not in COMMAND_LINE_TARGETS:
+    env.Install(install_dest, TARGET)
+    env.Alias('install', install_dest)

plugins/input/python/examples/concentric_circles.py

@@ -0,0 +1,83 @@
+"""A more complex example which renders an infinite series of concentric
+circles centred on a point.
+
+The circles are represented by a Python iterator which will yield only the
+circles which intersect the query's bounding box. The advantage of this
+approach over a MemoryDatasource is that a) only those circles which intersect
+the viewport are actually generated and b) only the memory for the largest
+circle need be available since each circle is created on demand and destroyed
+when finished with.
+"""
+import math
+import mapnik
+from shapely.geometry import *
+
+def box2d_to_shapely(box):
+    import shapely.geometry
+    return shapely.geometry.box(box.minx, box.miny, box.maxx, box.maxy)
+
+class ConcentricCircles(object):
+    def __init__(self, centre, bounds, step=1):
+        self.centre = centre
+        self.bounds = bounds
+        self.step = step
+
+    class Iterator(object):
+        def __init__(self, container):
+            self.container = container
+
+            centre = self.container.centre
+            bounds = self.container.bounds
+            step = self.container.step
+
+            if centre.within(bounds):
+                self.radius = 0
+            else:
+                self.radius = math.ceil(centre.distance(bounds) / float(step)) * step
+
+        def next(self):
+            circle = self.container.centre.buffer(self.radius)
+            self.radius += self.container.step
+
+            # has the circle grown so large that the boundary is entirely within it?
+            if circle.contains(self.container.bounds):
+                raise StopIteration()
+
+            return ( circle.wkb, { } )
+
+    def __iter__(self):
+        return ConcentricCircles.Iterator(self)
+
+class TestDatasource(mapnik.PythonDatasource):
+    def __init__(self):
+        super(TestDatasource, self).__init__(
+                geometry_type=mapnik.DataGeometryType.Polygon
+        )
+
+    def features(self, query):
+        # Get the query bounding-box as a shapely bounding box
+        bounding_box = box2d_to_shapely(query.bbox)
+        centre = Point(-20, 0)
+
+        return mapnik.PythonDatasource.wkb_features(
+            keys = (),
+            features = ConcentricCircles(centre, bounding_box, 0.5)
+        )
+
+if __name__ == '__main__':
+    m = mapnik.Map(640, 320)
+
+    m.background = mapnik.Color('white')
+    s = mapnik.Style()
+    r = mapnik.Rule()
+    r.symbols.append(mapnik.LineSymbolizer())
+    s.rules.append(r)
+    m.append_style('point_style',s)
+    ds = mapnik.Python(factory='TestDatasource')
+    layer = mapnik.Layer('python')
+    layer.datasource = ds
+    layer.styles.append('point_style')
+    m.layers.append(layer)
+    box = mapnik.Box2d(-60, -60, 0, -30)
+    m.zoom_to_box(box)
+    mapnik.render_to_file(m,'map.png', 'png')

plugins/input/python/examples/simple_points.py

@@ -0,0 +1,34 @@
+import mapnik
+from shapely.geometry import *
+
+class TestDatasource(mapnik.PythonDatasource):
+    def __init__(self):
+        super(TestDatasource, self).__init__()
+
+    def features(self, query):
+        return mapnik.PythonDatasource.wkb_features(
+            keys = ('label',), 
+            features = (
+                ( Point(5,6).wkb, { 'label': 'foo-bar'} ), 
+                ( Point(100,60).wkb, { 'label': 'buzz-quux'} ), 
+            )
+        )
+
+if __name__ == '__main__':
+    m = mapnik.Map(1280,1024)
+    m.background = mapnik.Color('white')
+    s = mapnik.Style()
+    r = mapnik.Rule()
+    r.symbols.append(mapnik.PointSymbolizer())
+    t = mapnik.TextSymbolizer(mapnik.Expression("[label]"),"DejaVu Sans Book",10,mapnik.Color('black'))
+    t.displacement = (5,5)
+    r.symbols.append(t)
+    s.rules.append(r)
+    m.append_style('point_style',s)
+    ds = mapnik.Python(factory='TestDatasource')
+    layer = mapnik.Layer('python')
+    layer.datasource = ds
+    layer.styles.append('point_style')
+    m.layers.append(layer)
+    m.zoom_all()
+    mapnik.render_to_file(m,'map.png', 'png')

plugins/input/python/examples/simple_xml.py

@@ -0,0 +1,8 @@
+import mapnik
+stylesheet = 'simple_xml.xml'
+image = 'simple_xml.png'
+m = mapnik.Map(600, 300)
+mapnik.load_map(m, stylesheet)
+m.zoom_all() 
+mapnik.render_to_file(m, image)
+print "rendered image to '%s'" % image

plugins/input/python/examples/simple_xml.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Map srs="+init=epsg:4326" background-color="white">
+    <Style name="style">
+        <Rule>
+            <PointSymbolizer />
+            <TextSymbolizer name="[label]" face-name="DejaVu Sans Book" size="10" dx="5" dy="5"/>
+        </Rule>
+    </Style>
+    <Layer name="test" srs="+init=epsg:4326">
+        <StyleName>style</StyleName>
+        <Datasource>
+            <Parameter name="type">python</Parameter>
+            <Parameter name="factory">test:TestDatasource</Parameter>
+        </Datasource>
+    </Layer>
+</Map>

plugins/input/python/python_datasource.cpp

@@ -0,0 +1,208 @@
+// file plugin
+#include "python_datasource.hpp"
+#include "python_featureset.hpp"
+
+// stl
+#include <string>
+#include <vector>
+
+// boost
+#include <boost/foreach.hpp>
+#include <boost/make_shared.hpp>
+#include <boost/python.hpp>
+#include <boost/python/stl_iterator.hpp>
+#include <boost/algorithm/string.hpp>
+
+#include "python_utils.hpp"
+
+using mapnik::datasource;
+using mapnik::parameters;
+
+DATASOURCE_PLUGIN(python_datasource)
+
+python_datasource::python_datasource(parameters const& params, bool bind)
+  : datasource(params),
+    desc_(*params_.get<std::string>("type"), *params_.get<std::string>("encoding","utf-8")),
+    factory_(*params_.get<std::string>("factory", ""))
+{
+    // extract any remaining parameters as keyword args for the factory
+    BOOST_FOREACH(const mapnik::parameters::value_type& kv, params_)
+    {
+        if((kv.first != "type") && (kv.first != "factory"))
+        {
+            kwargs_.insert(std::make_pair(kv.first, *params_.get<std::string>(kv.first)));
+        }
+    }
+
+    if (bind)
+    {
+        this->bind();
+    }
+}
+
+python_datasource::~python_datasource() { }
+
+// This name must match the plugin filename, eg 'python.input'
+const char* python_datasource::name_="python";
+
+const char* python_datasource::name()
+{
+    return name_;
+}
+
+mapnik::layer_descriptor python_datasource::get_descriptor() const
+{
+    if (!is_bound_) bind();
+
+    return desc_;
+}
+
+// The following methods call into the Python interpreter and hence require, unfortunately, that the GIL be held.
+
+void python_datasource::bind() const
+{
+    using namespace boost;
+    using namespace boost::python;
+
+    if (is_bound_) return;
+
+    // if no factory callable is defined, bind is a nop
+    if (factory_.empty()) return;
+
+    // split factory at ':' to parse out module and callable
+    std::vector<std::string> factory_split;
+    split(factory_split, factory_, is_any_of(":"));
+    if ((factory_split.size() < 1) || (factory_split.size() > 2))
+    {
+        // FIMXE: is this appropriate error reporting?
+        std::cerr << "python: factory string must be of the form '[module:]callable' when parsing \""
+                  << factory_ << '"' << std::endl;
+        return;
+    }
+
+    // extract the module and the callable
+    str module_name("__main__"), callable_name;
+    if (factory_split.size() == 1)
+    {
+        callable_name = str(factory_split[0]);
+    }
+    else
+    {
+        module_name = str(factory_split[0]);
+        callable_name = str(factory_split[1]);
+    }
+
+    {
+        ensure_gil lock;
+
+        // import the main module from Python (in case we're embedding the
+        // interpreter directly) and also import the callable.
+        object main_module = import("__main__");
+        object callable_module = import(module_name);
+        object callable = callable_module.attr(callable_name);
+
+        // prepare the arguments
+        dict kwargs;
+        typedef std::map<std::string, std::string>::value_type kv_type;
+        BOOST_FOREACH(const kv_type& kv, kwargs_)
+        {
+            kwargs[str(kv.first)] = str(kv.second);
+        }
+
+        // get our wrapped data source
+        datasource_ = callable(*boost::python::make_tuple(), **kwargs);
+    }
+
+    is_bound_ = true;
+}
+
+mapnik::datasource::datasource_t python_datasource::type() const
+{
+    using namespace boost::python;
+
+    typedef boost::optional<mapnik::datasource::geometry_t> return_type;
+
+    if (!is_bound_) bind();
+
+    ensure_gil lock;
+
+    object data_type = datasource_.attr("data_type");
+    long data_type_integer = extract<long>(data_type);
+    return mapnik::datasource::datasource_t(data_type_integer);
+}
+
+mapnik::box2d<double> python_datasource::envelope() const
+{
+    using namespace boost::python;
+
+    if (!is_bound_) bind();
+
+    ensure_gil lock;
+    return extract<mapnik::box2d<double> >(datasource_.attr("envelope"));
+}
+
+boost::optional<mapnik::datasource::geometry_t> python_datasource::get_geometry_type() const
+{
+    using namespace boost::python;
+
+    typedef boost::optional<mapnik::datasource::geometry_t> return_type;
+
+    if (!is_bound_) bind();
+
+    ensure_gil lock;
+
+    // if the datasource object has no geometry_type attribute, return a 'none' value
+    if (!PyObject_HasAttrString(datasource_.ptr(), "geometry_type"))
+        return return_type();
+
+    object py_geometry_type = datasource_.attr("geometry_type");
+
+    // if the attribute value is 'None', return a 'none' value
+    if (py_geometry_type.ptr() == object().ptr())
+        return return_type();
+
+    long geom_type_integer = extract<long>(py_geometry_type);
+    return mapnik::datasource::geometry_t(geom_type_integer);
+}
+
+mapnik::featureset_ptr python_datasource::features(mapnik::query const& q) const
+{
+    using namespace boost::python;
+
+    if (!is_bound_) bind();
+
+    // if the query box intersects our world extent then query for features
+    if (envelope().intersects(q.get_bbox()))
+    {
+        ensure_gil lock;
+
+        object features(datasource_.attr("features")(q));
+
+        // if 'None' was returned, return an empty feature set
+        if(features.ptr() == object().ptr())
+            return mapnik::featureset_ptr();
+
+        return boost::make_shared<python_featureset>(features);
+    }
+
+    // otherwise return an empty featureset pointer
+    return mapnik::featureset_ptr();
+}
+
+mapnik::featureset_ptr python_datasource::features_at_point(mapnik::coord2d const& pt) const
+{
+    using namespace boost::python;
+
+    if (!is_bound_) bind();
+
+    ensure_gil lock;
+
+    object features(datasource_.attr("features_at_point")(pt));
+
+    // if we returned none, return an empty set
+    if(features.ptr() == object().ptr())
+        return mapnik::featureset_ptr();
+
+    // otherwise, return a feature set which can iterate over the iterator
+    return boost::make_shared<python_featureset>(features);
+}

plugins/input/python/python_datasource.hpp

@@ -0,0 +1,56 @@
+#ifndef PYTHON_DATASOURCE_HPP
+#define PYTHON_DATASOURCE_HPP
+
+// mapnik
+#include <mapnik/datasource.hpp>
+
+// boost
+#include <boost/python.hpp>
+
+class python_datasource : public mapnik::datasource
+{
+public:
+    // constructor
+    // arguments must not change
+    python_datasource(mapnik::parameters const& params, bool bind=true);
+
+    // destructor
+    virtual ~python_datasource ();
+
+    // mandatory: type of the plugin, used to match at runtime
+    mapnik::datasource::datasource_t type() const;
+
+    // mandatory: name of the plugin
+    static const char* name();
+
+    // mandatory: function to query features by box2d
+    // this is called when rendering, specifically in feature_style_processor.hpp
+    mapnik::featureset_ptr features(mapnik::query const& q) const;
+
+    // mandatory: function to query features by point (coord2d)
+    // not used by rendering, but available to calling applications
+    mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const;
+
+    // mandatory: return the box2d of the datasource
+    // called during rendering to determine if the layer should be processed
+    mapnik::box2d<double> envelope() const;
+
+    // mandatory: optionally return the overal geometry type of the datasource
+    boost::optional<mapnik::datasource::geometry_t> get_geometry_type() const;
+
+    // mandatory: return the layer descriptor
+    mapnik::layer_descriptor get_descriptor() const;
+
+    // mandatory: will bind the datasource given params
+    void bind() const;
+
+private:
+    static const char* name_;
+    mutable mapnik::layer_descriptor desc_;
+    const std::string factory_;
+    std::map<std::string, std::string> kwargs_;
+    mutable boost::python::object datasource_;
+};
+
+
+#endif // PYTHON_DATASOURCE_HPP

plugins/input/python/python_featureset.cpp

@@ -0,0 +1,30 @@
+// boost
+#include <boost/python.hpp>
+
+#include "python_featureset.hpp"
+#include "python_utils.hpp"
+
+python_featureset::python_featureset(boost::python::object iterator)
+{
+    ensure_gil lock;
+    begin_ = boost::python::stl_input_iterator<mapnik::feature_ptr>(iterator);
+}
+
+python_featureset::~python_featureset()
+{
+    ensure_gil lock;
+    begin_ = end_;
+}
+
+mapnik::feature_ptr python_featureset::next()
+{
+    // checking to see if we've reached the end does not require the GIL.
+    if(begin_ == end_)
+        return mapnik::feature_ptr();
+
+    // getting the next feature might call into the interpreter and so the GIL must be held.
+    ensure_gil lock;
+
+    return *(begin_++);
+}
+

plugins/input/python/python_featureset.hpp

@@ -0,0 +1,30 @@
+#ifndef PYTHON_FEATURESET_HPP
+#define PYTHON_FEATURESET_HPP
+
+// boost
+#include <boost/python.hpp>
+#include <boost/python/stl_iterator.hpp>
+
+// mapnik
+#include <mapnik/datasource.hpp>
+
+// extend the mapnik::Featureset defined in include/mapnik/datasource.hpp
+class python_featureset : public mapnik::Featureset
+{
+public:
+    // this constructor can have any arguments you need
+    python_featureset(boost::python::object iterator);
+
+    // desctructor
+    virtual ~python_featureset();
+
+    // mandatory: you must expose a next() method, called when rendering
+    mapnik::feature_ptr next();
+
+private:
+    typedef boost::python::stl_input_iterator<mapnik::feature_ptr> feature_iter;
+
+    feature_iter begin_, end_;
+};
+
+#endif // PYTHON_FEATURESET_HPP

plugins/input/python/python_utils.hpp

@@ -0,0 +1,16 @@
+#ifndef PYTHON_UTILS_HPP
+#define PYTHON_UTILS_HPP
+
+#include <boost/python.hpp>
+
+// Use RAII to acquire and release the GIL as needed.
+class ensure_gil
+{
+    public:
+        ensure_gil() : gil_state_(PyGILState_Ensure()) {}
+        ~ensure_gil() { PyGILState_Release( gil_state_ ); }
+    protected:
+        PyGILState_STATE gil_state_;
+};
+
+#endif // PYTHON_UTILS_HPP

tests/data/good_maps/python_circle_datasource.xml

@@ -0,0 +1,50 @@
+<!DOCTYPE Map>
+<Map background-color="#b5d0d0" srs="+init=epsg:4326" minimum-version="0.7.2">
+    <Style name="1">
+        <Rule>
+            <LineSymbolizer stroke="rgb(80%,0%,0%)" />
+        </Rule>
+    </Style>
+
+    <Style name="2">
+        <Rule>
+            <LineSymbolizer stroke="rgb(0%,80%,0%)" />
+        </Rule>
+    </Style>
+
+    <Style name="3">
+        <Rule>
+            <LineSymbolizer stroke="rgb(0%,0%,80%)" />
+        </Rule>
+    </Style>
+
+    <Layer name="circles1" srs="+init=epsg:4326">
+        <StyleName>1</StyleName>
+        <Datasource>
+            <Parameter name="type">python</Parameter>
+            <Parameter name="factory">python_plugin_test:CirclesDatasource</Parameter>
+        </Datasource>
+    </Layer>
+
+    <Layer name="circles2" srs="+init=epsg:4326">
+        <StyleName>2</StyleName>
+        <Datasource>
+            <Parameter name="type">python</Parameter>
+            <Parameter name="factory">python_plugin_test:CirclesDatasource</Parameter>
+            <Parameter name="centre_x">-20</Parameter>
+            <Parameter name="centre_y">50</Parameter>
+        </Datasource>
+    </Layer>
+
+    <Layer name="circles3" srs="+init=epsg:4326">
+        <StyleName>3</StyleName>
+        <Datasource>
+            <Parameter name="type">python</Parameter>
+            <Parameter name="factory">python_plugin_test:CirclesDatasource</Parameter>
+            <Parameter name="centre_x">60</Parameter>
+            <Parameter name="centre_y">140</Parameter>
+            <Parameter name="step">20</Parameter>
+        </Datasource>
+    </Layer>
+
+</Map>

tests/data/good_maps/python_point_datasource.xml

@@ -0,0 +1,18 @@
+<!DOCTYPE Map>
+<Map background-color="#b5d0d0" srs="+init=epsg:4326" minimum-version="0.7.2">
+    <Style name="1">
+        <Rule>
+            <TextSymbolizer size="10" dy="-5" face-name="DejaVu Sans Book" halo-radius="1">[label]</TextSymbolizer>
+            <PointSymbolizer/>
+        </Rule>
+    </Style>
+
+    <Layer name="point" srs="+init=epsg:4326">
+        <StyleName>1</StyleName>
+        <Datasource>
+            <Parameter name="type">python</Parameter>
+            <Parameter name="factory">python_plugin_test:PointDatasource</Parameter>
+        </Datasource>
+    </Layer>
+
+</Map>

tests/python_tests/python_plugin_test.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import math
+import mapnik
+import sys
+from utilities import execution_path
+from nose.tools import *
+
+try:
+    from shapely.geometry import Point
+    have_shapely = True
+except ImportError:
+    print('Shapely is required for python data source test.')
+    have_shapely = False
+
+def setup():
+    # All of the paths used are relative, if we run the tests
+    # from another directory we need to chdir()
+    os.chdir(execution_path('.'))
+
+class PointDatasource(mapnik.PythonDatasource):
+    def __init__(self):
+        super(PointDatasource, self).__init__(
+                envelope = mapnik.Box2d(0,-10,100,110)
+        )
+
+    def features(self, query):
+        return mapnik.PythonDatasource.wkb_features(
+            keys = ('label',), 
+            features = (
+                ( Point(5,6).wkb, { 'label': 'foo-bar'} ), 
+                ( Point(60,50).wkb, { 'label': 'buzz-quux'} ), 
+            )
+        )
+
+def box2d_to_shapely(box):
+    import shapely.geometry
+    return shapely.geometry.box(box.minx, box.miny, box.maxx, box.maxy)
+
+class ConcentricCircles(object):
+    def __init__(self, centre, bounds, step=1):
+        self.centre = centre
+        self.bounds = bounds
+        self.step = step
+
+    class Iterator(object):
+        def __init__(self, container):
+            self.container = container
+
+            centre = self.container.centre
+            bounds = self.container.bounds
+            step = self.container.step
+
+            if centre.within(bounds):
+                self.radius = step
+            else:
+                self.radius = math.ceil(centre.distance(bounds) / float(step)) * step
+
+        def next(self):
+            circle = self.container.centre.buffer(self.radius)
+            self.radius += self.container.step
+
+            # has the circle grown so large that the boundary is entirely within it?
+            if circle.contains(self.container.bounds):
+                raise StopIteration()
+
+            return ( circle.wkb, { } )
+
+    def __iter__(self):
+        return ConcentricCircles.Iterator(self)
+
+class CirclesDatasource(mapnik.PythonDatasource):
+    def __init__(self, centre_x=-20, centre_y=0, step=10):
+        super(CirclesDatasource, self).__init__(
+                geometry_type=mapnik.DataGeometryType.Polygon
+        )
+
+        # note that the plugin loader will set all arguments to strings and will not try to parse them
+        centre_x = int(centre_x)
+        centre_y = int(centre_y)
+        step = int(step)
+
+        self.centre_x = centre_x
+        self.centre_y = centre_y
+        self.step = step
+
+    def features(self, query):
+        # Get the query bounding-box as a shapely bounding box
+        bounding_box = box2d_to_shapely(query.bbox)
+        centre = Point(self.centre_x, self.centre_y)
+
+        return mapnik.PythonDatasource.wkb_features(
+            keys = (),
+            features = ConcentricCircles(centre, bounding_box, self.step)
+        )
+
+if 'python' in mapnik.DatasourceCache.instance().plugin_names() and have_shapely:
+    # make sure we can load from ourself as a module
+    sys.path.append(execution_path('.'))
+
+    def test_python_point_init():
+        ds = mapnik.Python(factory='python_plugin_test:PointDatasource')
+        e = ds.envelope()
+
+        assert_almost_equal(e.minx, 0, places=7)
+        assert_almost_equal(e.miny, -10, places=7)
+        assert_almost_equal(e.maxx, 100, places=7)
+        assert_almost_equal(e.maxy, 110, places=7)
+
+    def test_python_circle_init():
+        ds = mapnik.Python(factory='python_plugin_test:CirclesDatasource')
+        e = ds.envelope()
+
+        assert_almost_equal(e.minx, -180, places=7)
+        assert_almost_equal(e.miny, -90, places=7)
+        assert_almost_equal(e.maxx, 180, places=7)
+        assert_almost_equal(e.maxy, 90, places=7)
+
+    def test_python_circle_init_with_args():
+        ds = mapnik.Python(factory='python_plugin_test:CirclesDatasource', centre_x=40, centre_y=7)
+        e = ds.envelope()
+
+        assert_almost_equal(e.minx, -180, places=7)
+        assert_almost_equal(e.miny, -90, places=7)
+        assert_almost_equal(e.maxx, 180, places=7)
+        assert_almost_equal(e.maxy, 90, places=7)
+
+    def test_python_point_rendering():
+        m = mapnik.Map(512,512)
+        mapnik.load_map(m,'../data/good_maps/python_point_datasource.xml')
+        m.zoom_all()
+        im = mapnik.Image(512,512)
+        mapnik.render(m,im)
+        actual = '/tmp/mapnik-python-point-render1.png'
+        expected = 'images/support/mapnik-python-point-render1.png'
+        im.save(actual)
+        expected_im = mapnik.Image.open(expected)
+        eq_(im.tostring(),expected_im.tostring(),
+                'failed comparing actual (%s) and expected (%s)' % (actual,'tests/python_tests/'+ expected))
+
+    def test_python_circle_rendering():
+        m = mapnik.Map(512,512)
+        mapnik.load_map(m,'../data/good_maps/python_circle_datasource.xml')
+        m.zoom_all()
+        im = mapnik.Image(512,512)
+        mapnik.render(m,im)
+        actual = '/tmp/mapnik-python-circle-render1.png'
+        expected = 'images/support/mapnik-python-circle-render1.png'
+        im.save(actual)
+        expected_im = mapnik.Image.open(expected)
+        eq_(im.tostring(),expected_im.tostring(),
+                'failed comparing actual (%s) and expected (%s)' % (actual,'tests/python_tests/'+ expected))
+
+if __name__ == "__main__":
+    setup()
+    [eval(run)() for run in dir() if 'test_' in run]