steps toward a generic spatial indexing api

bindings/python/mapnik_datasource.cpp

@@ -174,6 +174,8 @@ void export_datasource()
         .def("encoding",&encoding) //todo expose as property
         .def("name",&name)
         .def("features_at_point",&datasource::features_at_point)
+        .def("index_on",&datasource::index_on)
+        .def("create_index",&datasource::create_index)
         .def("params",&datasource::params,return_value_policy<copy_const_reference>(), 
              "The configuration parameters of the data source. "  
              "These vary depending on the type of data source.")

include/mapnik/datasource.hpp

@@ -90,6 +90,11 @@ public:
     {
         return params_;
     }
+
+    void set_param(parameter const& p)
+    {
+        params_[p.first] = p.second;
+    }
         
     /*!
      * @brief Get the type of the datasource
@@ -106,6 +111,9 @@ public:
     virtual featureset_ptr features_at_point(coord2d const& pt) const=0;
     virtual box2d<double> envelope() const=0;
     virtual layer_descriptor get_descriptor() const=0;
+    virtual bool index_on(std::string const& field) const=0;
+    virtual bool create_index(std::string const& field, mapnik::parameters const& params) const=0;
+    virtual std::string geometry_field() const=0;
     virtual ~datasource() {};
 protected:
     parameters params_;

include/mapnik/memory_datasource.hpp

@@ -44,6 +44,9 @@ public:
     featureset_ptr features_at_point(coord2d const& pt) const;
     box2d<double> envelope() const;
     layer_descriptor get_descriptor() const;
+    bool index_on(std::string const& field) const { return false; }
+    bool create_index(std::string const& field, mapnik::parameters const& params) const { return false; }
+    std::string geometry_field() const { return ""; }
     size_t size() const;
 private:
     std::vector<feature_ptr> features_;

include/mapnik/sql_utils.hpp

@@ -27,17 +27,16 @@
 #include <boost/algorithm/string.hpp>
 #include <boost/scoped_array.hpp>
 
-namespace mapnik
-{
+namespace mapnik { namespace sql_utils {
 
-inline std::string unquote_sql(const std::string& sql)
+inline std::string unquote_double(const std::string& sql)
 {
     std::string table_name = boost::algorithm::to_lower_copy(sql);  
     boost::algorithm::trim_if(table_name,boost::algorithm::is_any_of("\""));
     return table_name;
 }
 
-inline std::string unquote_sql2(const std::string& sql)
+inline std::string unquote(const std::string& sql)
 {
     std::string table_name = boost::algorithm::to_lower_copy(sql);  
     boost::algorithm::trim_if(table_name,boost::algorithm::is_any_of("\"\'"));
@@ -65,7 +64,6 @@ inline std::string table_from_sql(const std::string& sql)
     std::string::size_type idx = table_name.rfind(" from ");
     if (idx!=std::string::npos)
     {
-      
         idx = table_name.find_first_not_of(" ",idx+5);
         if (idx != std::string::npos)
         {
@@ -191,6 +189,7 @@ inline std::string numeric2string(const char* buf)
    }
    return ss.str();
 }
+}
 
 }
 

plugins/input/csv/csv_datasource.hpp

@@ -9,37 +9,40 @@
 
 class csv_datasource : public mapnik::datasource 
 {
-   public:
-      csv_datasource(mapnik::parameters const& params, bool bind=true);
-      virtual ~csv_datasource ();
-      int type() const;
-      static std::string name();
-      mapnik::featureset_ptr features(mapnik::query const& q) const;
-      mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const;
-      mapnik::box2d<double> envelope() const;
-      mapnik::layer_descriptor get_descriptor() const;
-      void bind() const;
-      template <typename T>
-      void parse_csv(T& stream,
-         std::string const& escape,
-         std::string const& separator,
-         std::string const& quote) const;
-   private:
-      mutable mapnik::layer_descriptor desc_;
-      mutable mapnik::box2d<double> extent_;
-      mutable std::string filename_;
-      mutable std::string inline_string_;
-      mutable unsigned file_length_;
-      mutable int row_limit_;
-      mutable std::vector<mapnik::feature_ptr> features_;
-      mutable std::string escape_;
-      mutable std::string separator_;
-      mutable std::string quote_;
-      mutable std::vector<std::string> headers_;
-      mutable std::string manual_headers_;
-      mutable bool strict_;
-      mutable bool quiet_;
-      mutable double filesize_max_;
+public:
+    csv_datasource(mapnik::parameters const& params, bool bind=true);
+    virtual ~csv_datasource ();
+    int type() const;
+    static std::string name();
+    mapnik::featureset_ptr features(mapnik::query const& q) const;
+    mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const;
+    mapnik::box2d<double> envelope() const;
+    mapnik::layer_descriptor get_descriptor() const;
+    bool index_on(std::string field) const { return false };
+    bool create_index(std::string const& field, mapnik::parameters const& params) const { return false; }
+    std::string geometry_field() const { return ""; }
+    void bind() const;
+    template <typename T>
+    void parse_csv(T& stream,
+       std::string const& escape,
+       std::string const& separator,
+       std::string const& quote) const;
+private:
+    mutable mapnik::layer_descriptor desc_;
+    mutable mapnik::box2d<double> extent_;
+    mutable std::string filename_;
+    mutable std::string inline_string_;
+    mutable unsigned file_length_;
+    mutable int row_limit_;
+    mutable std::vector<mapnik::feature_ptr> features_;
+    mutable std::string escape_;
+    mutable std::string separator_;
+    mutable std::string quote_;
+    mutable std::vector<std::string> headers_;
+    mutable std::string manual_headers_;
+    mutable bool strict_;
+    mutable bool quiet_;
+    mutable double filesize_max_;
 };
 
 

plugins/input/postgis/postgis_datasource.cpp

@@ -126,7 +126,7 @@ void postgis_datasource::bind() const
 
             if(geometry_table_.empty())
             {
-                geometry_table_ = mapnik::table_from_sql(table_);
+                geometry_table_ = mapnik::sql_utils::table_from_sql(table_);
             }
             std::string::size_type idx = geometry_table_.find_last_of('.');
             if (idx!=std::string::npos)
@@ -150,13 +150,13 @@ void postgis_datasource::bind() const
             {
                 std::ostringstream s;
                 s << "SELECT f_geometry_column, srid FROM ";
-                s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << mapnik::unquote_sql(geometry_table_) <<"'";
+                s << GEOMETRY_COLUMNS <<" WHERE f_table_name='" << mapnik::sql_utils::unquote_double(geometry_table_) <<"'";
              
                 if (schema_.length() > 0) 
-                    s << " AND f_table_schema='" << mapnik::unquote_sql(schema_) << "'";
+                    s << " AND f_table_schema='" << mapnik::sql_utils::unquote_double(schema_) << "'";
             
                 if (geometry_field_.length() > 0)
-                    s << " AND f_geometry_column='" << mapnik::unquote_sql(geometry_field_) << "'";
+                    s << " AND f_geometry_column='" << mapnik::sql_utils::unquote_double(geometry_field_) << "'";
 
                 /*
                 if (show_queries_)
@@ -357,6 +357,7 @@ std::string postgis_datasource::sql_bbox(box2d<double> const& env) const
     std::ostringstream b;
     if (srid_ > 0)
         b << "SetSRID(";
+    //ST_MakeBox2D(ST_Point(),ST_Point())
     b << "'BOX3D(";
     b << std::setprecision(16);
     b << env.minx() << " " << env.miny() << ",";
@@ -502,14 +503,14 @@ featureset_ptr postgis_datasource::features(const query& q) const
                 s << "AsBinary(\"" << geometryColumn_ << "\") AS geom";
 
             if (!key_field_.empty()) 
-                mapnik::quote_attr(s,key_field_);
+                mapnik::sql_utils::quote_attr(s,key_field_);
 
             std::set<std::string> const& props=q.property_names();
             std::set<std::string>::const_iterator pos=props.begin();
             std::set<std::string>::const_iterator end=props.end();
             while (pos != end)
             {
-                mapnik::quote_attr(s,*pos);
+                mapnik::sql_utils::quote_attr(s,*pos);
                 ++pos;
             }       
 
@@ -578,14 +579,14 @@ featureset_ptr postgis_datasource::features_at_point(coord2d const& pt) const
                 s << "AsBinary(\"" << geometryColumn_ << "\") AS geom";
             
             if (!key_field_.empty())
-                mapnik::quote_attr(s,key_field_);
+                mapnik::sql_utils::quote_attr(s,key_field_);
 
             std::vector<attribute_descriptor>::const_iterator itr = desc_.get_descriptors().begin();
             std::vector<attribute_descriptor>::const_iterator end = desc_.get_descriptors().end();
             unsigned size=0;
             while (itr != end)
             {
-                mapnik::quote_attr(s,itr->get_name());
+                mapnik::sql_utils::quote_attr(s,itr->get_name());
                 ++itr;
                 ++size;
             }
@@ -644,11 +645,11 @@ box2d<double> postgis_datasource::envelope() const
 
                 if (schema_.length() > 0)
                 {
-                    s << mapnik::unquote_sql(schema_) << "','";
+                    s << mapnik::sql_utils::unquote_double(schema_) << "','";
                 }
 
-                s << mapnik::unquote_sql(geometry_table_) << "','"
-                  << mapnik::unquote_sql(geometryColumn_) << "') as ext) as tmp";
+                s << mapnik::sql_utils::unquote_double(geometry_table_) << "','"
+                  << mapnik::sql_utils::unquote_double(geometryColumn_) << "') as ext) as tmp";
             }
             else
             {

plugins/input/postgis/postgis_featureset.cpp

@@ -162,7 +162,7 @@ feature_ptr postgis_featureset::next()
                 }
                 else if (oid == 1700) // numeric
                 {
-                    std::string str = mapnik::numeric2string(buf);
+                    std::string str = mapnik::sql_utils::numeric2string(buf);
                     try 
                     {
                         double val = boost::lexical_cast<double>(str);

plugins/input/shape/shape_datasource.hpp

@@ -51,6 +51,9 @@ public:
     featureset_ptr features_at_point(coord2d const& pt) const;
     box2d<double> envelope() const;
     layer_descriptor get_descriptor() const;
+    bool index_on(std::string field) const { return false };
+    bool create_index(std::string const& field, mapnik::parameters const& params) const { return false; }
+    std::string geometry_field() const { return ""; }
     void bind() const;   
 private:
     shape_datasource(const shape_datasource&);

plugins/input/sqlite/sqlite_connection.hpp

@@ -0,0 +1,161 @@
+/*****************************************************************************
+ * 
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2011 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef MAPNIK_SQLITE_CONNECTION_HPP
+#define MAPNIK_SQLITE_CONNECTION_HPP
+
+// stl
+#include <string.h>
+
+// mapnik
+#include <mapnik/datasource.hpp>
+#include <mapnik/sql_utils.hpp>
+
+
+// boost
+#include <boost/shared_ptr.hpp>
+#include <boost/make_shared.hpp>
+#include <boost/algorithm/string.hpp>
+
+// sqlite
+extern "C" {
+  #include <sqlite3.h>
+}
+
+#include "sqlite_resultset.hpp"
+
+
+//==============================================================================
+
+class sqlite_connection
+{
+public:
+
+    sqlite_connection (const std::string& file)
+        : db_(0),
+          file_(file)
+    {
+#if SQLITE_VERSION_NUMBER >= 3005000
+        int mode = SQLITE_OPEN_READWRITE;
+        #if SQLITE_VERSION_NUMBER >= 3006018
+            // shared cache flag not available until >= 3.6.18
+            mode |= SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE;
+        #endif
+        const int rc = sqlite3_open_v2 (file_.c_str(), &db_, mode, 0);
+#else
+        #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..."
+        const int rc = sqlite3_open (file_.c_str(), &db_);
+#endif
+        if (rc != SQLITE_OK)
+        {
+            std::ostringstream s;
+            s << "Sqlite Plugin: " << sqlite3_errmsg (db_);
+
+            throw mapnik::datasource_exception (s.str());
+        }
+        
+        sqlite3_busy_timeout(db_,5000);
+    }
+
+    sqlite_connection (const std::string& file, int flags)
+        : db_(0),
+          file_(file)
+    {
+#if SQLITE_VERSION_NUMBER >= 3005000
+        const int rc = sqlite3_open_v2 (file_.c_str(), &db_, flags, 0);
+#else
+        #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.5.x which may make rendering slow..."
+        const int rc = sqlite3_open (file_.c_str(), &db_);
+#endif
+        if (rc != SQLITE_OK)
+        {
+            std::ostringstream s;
+            s << "Sqlite Plugin: " << sqlite3_errmsg (db_);
+
+            throw mapnik::datasource_exception (s.str());
+        }
+    }
+
+    virtual ~sqlite_connection ()
+    {
+        if (db_)
+        {
+            sqlite3_close (db_);
+        }
+    }
+
+    void throw_sqlite_error(const std::string& sql)
+    {
+        std::ostringstream s;
+        s << "Sqlite Plugin: ";
+        if (db_)
+            s << "'" << sqlite3_errmsg(db_) << "'";
+        else
+            s << "unknown error, lost connection";
+        s << " (" << file_ << ")"
+          << "\nFull sql was: '" 
+          <<  sql << "'";
+
+        throw mapnik::datasource_exception (s.str());
+    }
+    
+    boost::shared_ptr<sqlite_resultset> execute_query(const std::string& sql)
+    {
+        sqlite3_stmt* stmt = 0;
+
+        const int rc = sqlite3_prepare_v2 (db_, sql.c_str(), -1, &stmt, 0);
+        if (rc != SQLITE_OK)
+        {
+            throw_sqlite_error(sql);
+        }
+
+        return boost::make_shared<sqlite_resultset>(stmt);
+    }
+  
+    void execute(const std::string& sql)
+    {
+        const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
+        if (rc != SQLITE_OK)
+        {
+            throw_sqlite_error(sql);
+        }
+    }
+
+    int execute_with_code(const std::string& sql)
+    {
+        const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
+        return rc;
+    }
+   
+    sqlite3* operator*()
+    {
+        return db_;
+    }
+
+    
+private:
+
+    sqlite3* db_;
+    std::string file_;
+};
+
+#endif // MAPNIK_SQLITE_CONNECTION_HPP

plugins/input/sqlite/sqlite_datasource.cpp

@@ -22,22 +22,19 @@
 
 #include "sqlite_datasource.hpp"
 #include "sqlite_featureset.hpp"
+#include "sqlite_resultset.hpp"
+#include "sqlite_utils.hpp"
 
 // mapnik
 #include <mapnik/ptree_helpers.hpp>
 #include <mapnik/sql_utils.hpp>
-#include <mapnik/feature_factory.hpp> // to enable extent fallback hack
 
 // boost
 #include <boost/algorithm/string.hpp>
-#include <boost/lexical_cast.hpp>
-#include <boost/format.hpp>
 #include <boost/tokenizer.hpp>
 #include <boost/filesystem/operations.hpp>
 #include <boost/make_shared.hpp>
-
-using boost::lexical_cast;
-using boost::bad_lexical_cast;
+#include <boost/scoped_array.hpp>
 
 using mapnik::box2d;
 using mapnik::coord2d;
@@ -68,9 +65,9 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind)
     desc_(*params_.get<std::string>("type"), *params_.get<std::string>("encoding", "utf-8")),
     format_(mapnik::wkbAuto)
 {
-    // TODO
-    // - change param from 'file' to 'dbname'
-    // - ensure that the supplied key_field is a valid "integer primary key"
+    /* TODO
+      - throw if no primary key but spatial index is present?
+    */
     
     boost::optional<std::string> file = params_.get<std::string>("file");
     if (! file) throw datasource_exception("Sqlite Plugin: missing <file> parameter");
@@ -86,41 +83,58 @@ sqlite_datasource::sqlite_datasource(parameters const& params, bool bind)
     }
 }
 
+bool sqlite_datasource::index_on(std::string const& field) const
+{
+    if (! is_bound_) bind();
+
+    if (field == geometry_field_)
+    {
+        if (use_spatial_index_ && !index_table_.empty())
+        {
+            std::string index_db = sqlite_utils::index_for_db(dataset_name_);
+            if (boost::filesystem::exists(index_db))
+            {
+                dataset_->execute("attach database '" + index_db + "' as " + index_table_);
+            }
+            
+            return sqlite_utils::has_rtree(index_table_,dataset_);
+        }
+    }
+    // else check or attribute index?
+    return false;
+}
+
+bool sqlite_datasource::create_index(std::string const& field, mapnik::parameters const& params) const
+{
+    if (! is_bound_) bind();
+    if (field == geometry_field_)
+    {
+        if (!key_field_.empty() && !index_table_.empty())
+        {
+            std::string index_db = sqlite_utils::index_for_db(dataset_name_);
+            std::ostringstream query;
+            query << "SELECT " 
+                  << geometry_field_
+                  << "," << key_field_
+                  << " FROM ("
+                  << geometry_table_ << ")";
+            boost::shared_ptr<sqlite_resultset> rs = dataset_->execute_query(query.str());
+            sqlite_utils::create_rtree(index_db,index_table_,rs);
+            use_spatial_index_ = sqlite_utils::has_rtree(index_table_,dataset_);
+            return use_spatial_index_;
+        }
+    }
+    // else create attribute index
+    return false;
+
+}
+
 void sqlite_datasource::bind() const
 {
     if (is_bound_) return;
     
     boost::optional<std::string> file = params_.get<std::string>("file");
     if (! file) throw datasource_exception("Sqlite Plugin: missing <file> parameter");
-    
-    // Be careful, automatically using rowid as key field of the table can lead
-    // to misleading results http://www.gaia-gis.it/spatialite-2.4.0-5/SpatialIndex-Update.pdf
-    // TODO - We should try to avoid this and eventually force to specify a valid table primary key
-
-    boost::optional<std::string> key_field_name = params_.get<std::string>("key_field");
-    if (key_field_name)
-    {
-        key_field_ = *key_field_name;
-    }
-    
-    boost::optional<std::string> wkb = params_.get<std::string>("wkb_format");
-    if (wkb)
-    {
-        if (*wkb == "spatialite")
-            format_ = mapnik::wkbSpatiaLite;  
-        else if (*wkb == "generic")
-            format_ = mapnik::wkbGeneric;
-        else
-            format_ = mapnik::wkbAuto;
-    }
-
-    multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries", false);
-    use_spatial_index_ = *params_.get<mapnik::boolean>("use_spatial_index", true);
-    has_spatial_index_ = false;
-    using_subquery_ = false;
-
-    boost::optional<std::string> ext  = params_.get<std::string>("extent");
-    if (ext) extent_initialized_ = extent_.from_string(*ext);
 
     boost::optional<std::string> base = params_.get<std::string>("base");
     if (base)
@@ -128,6 +142,34 @@ void sqlite_datasource::bind() const
     else
         dataset_name_ = *file;
 
+    if ((dataset_name_.compare(":memory:") != 0) && (!boost::filesystem::exists(dataset_name_)))
+    {
+        throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist");
+    }    
+
+    multiple_geometries_ = *params_.get<mapnik::boolean>("multiple_geometries", false);
+    use_spatial_index_ = *params_.get<mapnik::boolean>("use_spatial_index", true);
+
+    boost::optional<std::string> ext  = params_.get<std::string>("extent");
+    if (ext) extent_initialized_ = extent_.from_string(*ext);
+        
+    boost::optional<std::string> wkb = params_.get<std::string>("wkb_format");
+    if (wkb)
+    {
+        if (*wkb == "spatialite")
+        {
+            format_ = mapnik::wkbSpatiaLite;  
+        }
+        else if (*wkb == "generic")
+        {
+            format_ = mapnik::wkbGeneric;
+        }
+        else
+        {
+            format_ = mapnik::wkbAuto;
+        }
+    }
+
     // Populate init_statements_
     //   1. Build attach database statements from the "attachdb" parameter
     //   2. Add explicit init statements from "initdb" parameter
@@ -147,10 +189,21 @@ void sqlite_datasource::bind() const
         init_statements_.push_back(*initdb);
     }
 
-    if (!boost::filesystem::exists(dataset_name_))
-        throw datasource_exception("Sqlite Plugin: " + dataset_name_ + " does not exist");
-          
-    dataset_ = new sqlite_connection(dataset_name_);
+    if (geometry_table_.empty())
+    {
+        geometry_table_ = mapnik::sql_utils::table_from_sql(table_);
+    }
+
+    // if 'table_' is a subquery then we try to deduce names
+    // and types from the first row returned from that query
+    using_subquery_ = false;
+    if (table_ != geometry_table_)
+    {
+        using_subquery_ = true;
+    }
+
+    // now actually create the connection and start executing setup sql
+    dataset_ = boost::make_shared<sqlite_connection>(dataset_name_);
 
     // Execute init_statements_
     for (std::vector<std::string>::const_iterator iter = init_statements_.begin();
@@ -162,193 +215,38 @@ void sqlite_datasource::bind() const
         dataset_->execute(*iter);
     }
     
-    if (geometry_table_.empty())
-    {
-        geometry_table_ = mapnik::table_from_sql(table_);
-    }
-
-    // should we deduce column names and types using PRAGMA?
-    bool use_pragma_table_info = true;
-    
-    if (table_ != geometry_table_)
-    {
-        // if 'table_' is a subquery then we try to deduce names
-        // and types from the first row returned from that query
-        using_subquery_ = true;
-        use_pragma_table_info = false;
-
-        if (key_field_.empty())
-        {
-          key_field_ = "rowid";
-        }
-    }
-
-    if (! use_pragma_table_info)
+    bool found_types_via_subquery = false;
+    if (using_subquery_)
     {
         std::ostringstream s;
         s << "SELECT " << fields_ << " FROM (" << table_ << ") LIMIT 1";
-
-        boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
-        if (rs->is_valid() && rs->step_next())
-        {
-            for (int i = 0; i < rs->column_count(); ++i)
-            {
-                const int type_oid = rs->column_type(i);
-                const char* fld_name = rs->column_name(i);
-                switch (type_oid)
-                {
-                case SQLITE_INTEGER:
-                    desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
-                    break;
-                     
-                case SQLITE_FLOAT:
-                    desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double));
-                    break;
-                     
-                case SQLITE_TEXT:
-                    desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
-                    break;
-                     
-                case SQLITE_NULL:
-                    // sqlite reports based on value, not actual column type unless
-                    // PRAGMA table_info is used so here we assume the column is a string
-                    // which is a lesser evil than altogether dropping the column
-                    desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
-
-                case SQLITE_BLOB:
-                    if (geometry_field_.empty()
-                        && (boost::algorithm::icontains(fld_name, "geom") ||
-                            boost::algorithm::icontains(fld_name, "point") ||
-                            boost::algorithm::icontains(fld_name, "linestring") ||
-                            boost::algorithm::icontains(fld_name, "polygon")))
-                    {
-                        geometry_field_ = std::string(fld_name);
-                    }
-                    break;
-                     
-                default:
-#ifdef MAPNIK_DEBUG
-                    std::clog << "Sqlite Plugin: unknown type_oid=" << type_oid << std::endl;
-#endif
-                    break;
-                }
-            }
-        }
-        else
-        {
-            // if we do not have at least a row and
-            // we cannot determine the right columns types and names 
-            // as all column_type are SQLITE_NULL
-            // so we fallback to using PRAGMA table_info
-            use_pragma_table_info = true;
-        }
+        found_types_via_subquery = sqlite_utils::detect_types_from_subquery(s.str(),geometry_field_,desc_,dataset_);
     }
 
-    if (key_field_.empty())
+    // TODO - consider removing this
+    if (key_field_ == "rowid")
     {
-        use_pragma_table_info = true;
-    }
-    else
-    {
-        // TODO - we can't trust so much the rowid here
-        if (key_field_ == "rowid")
-        {
-            desc_.add_descriptor(attribute_descriptor("rowid", mapnik::Integer));
-        }
+        desc_.add_descriptor(attribute_descriptor("rowid", mapnik::Integer));
     }
     
-    if (use_pragma_table_info)
+    bool found_table = sqlite_utils::table_info(key_field_,
+                                            found_types_via_subquery,
+                                            geometry_field_,
+                                            geometry_table_,
+                                            desc_,
+                                            dataset_);
+
+    if (! found_table)
     {
         std::ostringstream s;
-        s << "PRAGMA table_info(" << geometry_table_ << ")";
+        s << "Sqlite Plugin: could not query table '" << geometry_table_ << "' ";
+        if (using_subquery_) s << " from subquery '" << table_ << "' ";
+        s << "using 'PRAGMA table_info(" << geometry_table_  << ")' ";
 
-        boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
-        bool found_table = false;
-        bool found_pk = false;
-        while (rs->is_valid() && rs->step_next())
-        {
-            found_table = true;
+        std::string sq_err = std::string(sqlite3_errmsg(*(*dataset_)));
+        if (sq_err != "unknown error") s << ": " << sq_err;
 
-            // TODO - support unicode strings
-            const char* fld_name = rs->column_text(1);
-            std::string fld_type(rs->column_text(2));
-            int fld_pk = rs->column_integer(5);
-            boost::algorithm::to_lower(fld_type);
-
-            // TODO - how to handle primary keys on multiple columns ?
-            if (key_field_.empty() && ! found_pk && fld_pk != 0)
-            {
-                key_field_ = fld_name;
-                found_pk = true;
-            }
-
-            // see 2.1 "Column Affinity" at http://www.sqlite.org/datatype3.html
-            // TODO - refactor this somehow ?
-            if (geometry_field_.empty()
-                && ((boost::algorithm::contains(fld_type, "geom") ||
-                     boost::algorithm::contains(fld_type, "point") ||
-                     boost::algorithm::contains(fld_type, "linestring") ||
-                     boost::algorithm::contains(fld_type, "polygon"))
-                ||
-                    (boost::algorithm::icontains(fld_name, "geom") ||
-                     boost::algorithm::icontains(fld_name, "point") ||
-                     boost::algorithm::icontains(fld_name, "linestring") ||
-                     boost::algorithm::icontains(fld_name, "polygon")))
-               )
-            {
-                geometry_field_ = std::string(fld_name);
-            }
-            else if (boost::algorithm::contains(fld_type, "int"))
-            {
-                desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Integer));
-            }
-            else if (boost::algorithm::contains(fld_type, "text") ||
-                     boost::algorithm::contains(fld_type, "char") ||
-                     boost::algorithm::contains(fld_type, "clob"))
-            {
-                desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
-            }
-            else if (boost::algorithm::contains(fld_type, "real") ||
-                     boost::algorithm::contains(fld_type, "float") ||
-                     boost::algorithm::contains(fld_type, "double"))
-            {
-                desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::Double));
-            }
-            else if (boost::algorithm::contains(fld_type, "blob"))
-            {
-                if (! geometry_field_.empty())
-                {
-                    desc_.add_descriptor(attribute_descriptor(fld_name, mapnik::String));
-                }
-            }
-#ifdef MAPNIK_DEBUG
-            else
-            {
-                // "Column Affinity" says default to "Numeric" but for now we pass..
-                //desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
-
-                // TODO - this should not fail when we specify geometry_field in XML file
-
-                std::clog << "Sqlite Plugin: column '"
-                          << std::string(fld_name)
-                          << "' unhandled due to unknown type: "
-                          << fld_type << std::endl;
-            }
-#endif
-        }
-
-        if (! found_table)
-        {
-            std::ostringstream s;
-            s << "Sqlite Plugin: could not query table '" << geometry_table_ << "' ";
-            if (using_subquery_) s << " from subquery '" << table_ << "' ";
-            s << "using 'PRAGMA table_info(" << geometry_table_  << ")' ";
-
-            std::string sq_err = std::string(sqlite3_errmsg(*(*dataset_)));
-            if (sq_err != "unknown error") s << ": " << sq_err;
-
-            throw datasource_exception(s.str());
-        }
+        throw datasource_exception(s.str());
     }
 
     if (geometry_field_.empty())
@@ -360,251 +258,84 @@ void sqlite_datasource::bind() const
     {
         // Generate implicit index_table name - need to do this after
         // we have discovered meta-data or else we don't know the column name
-        index_table_ = "\"idx_" + mapnik::unquote_sql2(geometry_table_) + "_" + geometry_field_ + "\"";
+        index_table_ = sqlite_utils::index_for_table(geometry_table_,geometry_field_);
     }
-    
+
+    std::string index_db = sqlite_utils::index_for_db(dataset_name_);
+ 
+    has_spatial_index_ = false;
     if (use_spatial_index_)
     {
-        std::ostringstream s;
-        s << "SELECT pkid,xmin,xmax,ymin,ymax FROM " << index_table_;
-        s << " LIMIT 0";
-
-        if (dataset_->execute_with_code(s.str()) == SQLITE_OK)
+        if (boost::filesystem::exists(index_db))
         {
-            has_spatial_index_ = true;
-        }
-#ifdef MAPNIK_DEBUG
-        else
-        {
-            std::clog << "SQLite Plugin: rtree index lookup did not succeed: '" << sqlite3_errmsg(*(*dataset_)) << "'\n";
-        }
-#endif
-    }
-
-    if (! metadata_.empty() && ! extent_initialized_)
-    {
-        std::ostringstream s;
-        s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata_;
-        s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table_ << "')";
-
-        boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
-        if (rs->is_valid() && rs->step_next())
-        {
-            double xmin = rs->column_double (0);
-            double ymin = rs->column_double (1);
-            double xmax = rs->column_double (2);
-            double ymax = rs->column_double (3);
-
-            extent_.init (xmin, ymin, xmax, ymax);
-            extent_initialized_ = true;
-        }
-    }
-    
-    if (! extent_initialized_ && has_spatial_index_)
-    {
-        std::ostringstream s;
-        s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM " 
-        << index_table_;
-
-        boost::scoped_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
-        if (rs->is_valid() && rs->step_next())
-        {
-            if (! rs->column_isnull(0))
-            {
-                try 
-                {
-                    double xmin = lexical_cast<double>(rs->column_double(0));
-                    double ymin = lexical_cast<double>(rs->column_double(1));
-                    double xmax = lexical_cast<double>(rs->column_double(2));
-                    double ymax = lexical_cast<double>(rs->column_double(3));
-                    extent_.init(xmin, ymin, xmax, ymax);
-                    extent_initialized_ = true;
-                }
-                catch (bad_lexical_cast& ex)
-                {
-                    std::clog << boost::format("SQLite Plugin: warning: could not determine extent from query: %s\nError was: '%s'\n") % s.str() % ex.what() << std::endl;
-                }
-            }
-        }    
-    }
-
-#ifdef MAPNIK_DEBUG
-    if (! has_spatial_index_ && use_spatial_index_ && using_subquery_
-        && key_field_ == "rowid" && ! boost::algorithm::icontains(table_, "rowid"))
-    {
-       // this is an impossible situation because rowid will be null via a subquery
-       std::clog << "Sqlite Plugin: WARNING: spatial index usage will fail because rowid "
-                 << "is not present in your subquery. You have 4 options: "
-                 << "1) Add rowid into your select statement, "
-                 << "2) alias your primary key as rowid, "
-                 << "3) supply a 'key_field' value that references the primary key of your spatial table, or "
-                 << "4) avoid using a spatial index by setting 'use_spatial_index'=false" << std::endl;
-    }
-#endif
-    
-    // final fallback to gather extent
-    if (! extent_initialized_ || ! has_spatial_index_)
-    {
-        std::ostringstream s;
-        s << "SELECT " << geometry_field_ << "," << key_field_
-          << " FROM " << geometry_table_;
-
-        if (row_limit_ > 0)
-        {
-            s << " LIMIT " << row_limit_;
-        }
-
-        if (row_offset_ > 0)
-        {
-            s << " OFFSET " << row_offset_;
-        }
-
-#ifdef MAPNIK_DEBUG
-        std::clog << "Sqlite Plugin: " << s.str() << std::endl;
-#endif
-
-        boost::shared_ptr<sqlite_resultset> rs(dataset_->execute_query(s.str()));
-        
-        // spatial index sql
-        std::ostringstream spatial_index_sql;
-        spatial_index_sql << "create virtual table " << index_table_
-                          << " using rtree(pkid, xmin, xmax, ymin, ymax)";
-
-        std::ostringstream spatial_index_insert_sql;
-        spatial_index_insert_sql << "insert into " << index_table_
-                                 << " values (?,?,?,?,?)";
-
-        sqlite3_stmt* stmt = 0;
-        if (use_spatial_index_)
-        {
-            dataset_->execute(spatial_index_sql.str());
-
-            const int rc = sqlite3_prepare_v2 (*(*dataset_),
-                                               spatial_index_insert_sql.str().c_str(),
-                                               -1,
-                                               &stmt,
-                                               0);
-            if (rc != SQLITE_OK)
-            {
-                std::ostringstream index_error;
-                index_error << "Sqlite Plugin: auto-index table creation failed: '"
-                            << sqlite3_errmsg(*(*dataset_)) << "' query was: "
-                            << spatial_index_insert_sql;
-
-                throw datasource_exception(index_error.str());
-            }
-        }
-
-        bool first = true;
-        while (rs->is_valid() && rs->step_next())
-        {
-            int size;
-            const char* data = (const char*) rs->column_blob(0, size);
-            if (data)
-            {
-                // create a tmp feature to be able to parse geometry
-                // ideally we would not have to do this.
-                // see: http://trac.mapnik.org/ticket/745
-                mapnik::feature_ptr feature(mapnik::feature_factory::create(0));
-                mapnik::geometry_utils::from_wkb(feature->paths(), data, size, multiple_geometries_, format_);
-                mapnik::box2d<double> const& bbox = feature->envelope();
-
-                if (bbox.valid())
-                {
-                    extent_initialized_ = true;
-                    if (first)
-                    {
-                        first = false;
-                        extent_ = bbox;
-                    }
-                    else
-                    {
-                        extent_.expand_to_include(bbox);
-                    }
-
-                    // index creation
-                    if (use_spatial_index_)
-                    {
-                        const int type_oid = rs->column_type(1);
-                        if (type_oid != SQLITE_INTEGER)
-                        {
-                            std::ostringstream type_error;
-                            type_error << "Sqlite Plugin: invalid type for key field '"
-                                       << key_field_ << "' when creating index '" << index_table_
-                                       << "' type was: " << type_oid << "";
-
-                            throw datasource_exception(type_error.str());
-                        }
-    
-                        const sqlite_int64 pkid = rs->column_integer64(1);
-                        if (sqlite3_bind_int64(stmt, 1, pkid) != SQLITE_OK)
-                        {
-                            throw datasource_exception("invalid value for for key field while generating index");
-                        }
-
-                        if ((sqlite3_bind_double(stmt, 2, bbox.minx()) != SQLITE_OK) ||
-                            (sqlite3_bind_double(stmt, 3, bbox.maxx()) != SQLITE_OK) ||
-                            (sqlite3_bind_double(stmt, 4, bbox.miny()) != SQLITE_OK) ||
-                            (sqlite3_bind_double(stmt, 5, bbox.maxy()) != SQLITE_OK))
-                        {
-                            throw datasource_exception("invalid value for for extent while generating index");
-                        }
-
-                        const int res = sqlite3_step(stmt);
-                        if (res != SQLITE_DONE)
-                        {
-                            std::ostringstream s;
-                            s << "SQLite Plugin: inserting bbox into rtree index failed: "
-                                 << "error code " << sqlite3_errcode(*(*dataset_)) << ": '"
-                                 << sqlite3_errmsg(*(*dataset_)) << "' query was: "
-                                 << spatial_index_insert_sql;
-
-                            throw datasource_exception(s.str());
-                        }
-
-                        sqlite3_reset(stmt);
-                    }
-                }
-                else
-                {
-                    std::ostringstream index_error;
-                    index_error << "SQLite Plugin: encountered invalid bbox at '"
-                                << key_field_ << "' == " << rs->column_integer64(1);
-
-                    throw datasource_exception(index_error.str());
-                }
-            }
+            dataset_->execute("attach database '" + index_db + "' as " + index_table_);
         }
         
-        const int res = sqlite3_finalize(stmt);
-        if (res != SQLITE_OK)
-        {
-            throw datasource_exception("auto-indexing failed: set use_spatial_index=false to disable auto-indexing and avoid this error");
-        }
+        has_spatial_index_ = sqlite_utils::has_rtree(index_table_,dataset_);
     }
 
     if (! extent_initialized_)
     {
-        std::ostringstream s;
-        s << "Sqlite Plugin: extent could not be determined for table '" 
-          << geometry_table_ << "' and geometry field '" << geometry_field_ << "'"
-          << " because an rtree spatial index is missing or empty."
-          << " - either set the table 'extent' or create an rtree spatial index";
 
-        throw datasource_exception(s.str());
-    }
+        // TODO - clean this up - reducing arguments
+        if (!sqlite_utils::detect_extent(dataset_,
+                                         has_spatial_index_,
+                                         extent_,
+                                         index_table_,
+                                         metadata_,
+                                         geometry_field_,
+                                         geometry_table_,
+                                         key_field_,
+                                         table_))
+        {
+            std::ostringstream s;
+            s << "Sqlite Plugin: extent could not be determined for table '" 
+              << geometry_table_ << "' and geometry field '" << geometry_field_ << "'"
+              << " because an rtree spatial index is missing or empty."
+              << " - either set the table 'extent' or create an rtree spatial index";
     
+            throw datasource_exception(s.str());
+        }
+    }
     is_bound_ = true;
 }
 
 sqlite_datasource::~sqlite_datasource()
 {
-    if (is_bound_) 
-    {
-        delete dataset_;
-    }
 }
 
+#if (BOOST_FILESYSTEM_VERSION <= 2)
+namespace boost {
+namespace filesystem {
+  path read_symlink(const path& p)
+  {
+    path symlink_path;
+
+#ifdef BOOST_POSIX_API
+    for (std::size_t path_max = 64;; path_max *= 2)// loop 'til buffer is large enough
+    {
+      boost::scoped_array<char> buf(new char[path_max]);
+      ssize_t result;
+      if ((result=::readlink(p.string().c_str(), buf.get(), path_max))== -1)
+      {
+        throw std::runtime_error("could not read symlink");
+      }
+      else
+      {
+        if(result != static_cast<ssize_t>(path_max))
+        {
+          symlink_path.assign(buf.get(), buf.get() + result);
+          break;
+        }
+      }
+    }
+#endif
+    return symlink_path;
+  }
+}
+}
+#endif
+
 void sqlite_datasource::parse_attachdb(std::string const& attachdb) const
 {
     boost::char_separator<char> sep(",");
@@ -638,23 +369,15 @@ void sqlite_datasource::parse_attachdb(std::string const& attachdb) const
                 boost::filesystem::path absolute_path(dataset_name_);
 
                 // support symlinks
-            #if (BOOST_FILESYSTEM_VERSION == 3)
                 if (boost::filesystem::is_symlink(absolute_path))
                 {
                     absolute_path = boost::filesystem::read_symlink(absolute_path);
                 }
 
+            #if (BOOST_FILESYSTEM_VERSION == 3)
                 filename = boost::filesystem::absolute(absolute_path.parent_path() / filename).string();
-
             #else
-                if (boost::filesystem::is_symlink(absolute_path))
-                {
-                    //cannot figure out how to resolve on in v2 so just print a warning
-                    std::clog << "###Warning: '" << absolute_path.string() << "' is a symlink which is not supported in attachdb\n";
-                }
-
                 filename = boost::filesystem::complete(absolute_path.branch_path() / filename).normalize().string();
-
             #endif
             }
         }
@@ -698,7 +421,9 @@ featureset_ptr sqlite_datasource::features(query const& q) const
 
         std::ostringstream s;
         
-        s << "SELECT " << geometry_field_ << "," << key_field_;
+        s << "SELECT " << geometry_field_;
+        if (!key_field_.empty())
+            s << "," << key_field_;
         std::set<std::string> const& props = q.property_names();
         std::set<std::string>::const_iterator pos = props.begin();
         std::set<std::string>::const_iterator end = props.end();
@@ -712,15 +437,8 @@ featureset_ptr sqlite_datasource::features(query const& q) const
         
         std::string query (table_);
         
-        // TODO
-        // throw if select * and key_field == rowid?
-        // or add schema support so sqlite throws
-
-        if (has_spatial_index_)
+        if (! key_field_.empty() && has_spatial_index_)
         {
-            // Use rtree to limit record id's to a given bbox
-            // then a btree to pull the records for those ids.
-
             std::ostringstream spatial_sql;
             spatial_sql << std::setprecision(16);
             spatial_sql << " WHERE " << key_field_ << " IN (SELECT pkid FROM " << index_table_;
@@ -775,7 +493,9 @@ featureset_ptr sqlite_datasource::features_at_point(coord2d const& pt) const
         mapnik::box2d<double> const e(pt.x, pt.y, pt.x, pt.y);
 
         std::ostringstream s;
-        s << "SELECT " << geometry_field_ << "," << key_field_;
+        s << "SELECT " << geometry_field_;
+        if (!key_field_.empty())
+            s << "," << key_field_;
         std::vector<attribute_descriptor>::const_iterator itr = desc_.get_descriptors().begin();
         std::vector<attribute_descriptor>::const_iterator end = desc_.get_descriptors().end();
         while (itr != end)
@@ -793,7 +513,7 @@ featureset_ptr sqlite_datasource::features_at_point(coord2d const& pt) const
         
         std::string query(table_);
         
-        if (has_spatial_index_)
+        if (! key_field_.empty() && has_spatial_index_)
         {
             std::ostringstream spatial_sql;
             spatial_sql << std::setprecision(16);

plugins/input/sqlite/sqlite_datasource.hpp

@@ -20,8 +20,8 @@
  *
  *****************************************************************************/
 
-#ifndef SQLITE_DATASOURCE_HPP
-#define SQLITE_DATASOURCE_HPP
+#ifndef MAPNIK_SQLITE_DATASOURCE_HPP
+#define MAPNIK_SQLITE_DATASOURCE_HPP
 
 // mapnik
 #include <mapnik/datasource.hpp>
@@ -34,7 +34,7 @@
 #include <boost/scoped_ptr.hpp>
 
 // sqlite
-#include "sqlite_types.hpp"
+#include "sqlite_connection.hpp"
 
 
 class sqlite_datasource : public mapnik::datasource 
@@ -48,6 +48,9 @@ public:
     mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const;
     mapnik::box2d<double> envelope() const;
     mapnik::layer_descriptor get_descriptor() const;
+    bool index_on(std::string const& field) const;
+    bool create_index(std::string const& field, mapnik::parameters const& params) const;
+    std::string geometry_field() const { return geometry_field_; }
     void bind() const;
 
 private:
@@ -55,7 +58,7 @@ private:
     mutable bool extent_initialized_;
     int type_;
     mutable std::string dataset_name_;
-    mutable sqlite_connection* dataset_;
+    mutable boost::shared_ptr<sqlite_connection> dataset_;
     std::string table_;
     std::string fields_;
     std::string metadata_;
@@ -78,4 +81,4 @@ private:
     void parse_attachdb(std::string const& attachdb) const;
 };
 
-#endif // SQLITE_DATASOURCE_HPP
+#endif // MAPNIK_SQLITE_DATASOURCE_HPP

plugins/input/sqlite/sqlite_featureset.cpp

@@ -33,6 +33,7 @@
 
 // ogr
 #include "sqlite_featureset.hpp"
+#include "sqlite_utils.hpp"
 
 using mapnik::query;
 using mapnik::box2d;

plugins/input/sqlite/sqlite_featureset.hpp

@@ -20,8 +20,8 @@
  *
  *****************************************************************************/
 
-#ifndef SQLITE_FEATURESET_HPP
-#define SQLITE_FEATURESET_HPP
+#ifndef MAPNIK_SQLITE_FEATURESET_HPP
+#define MAPNIK_SQLITE_FEATURESET_HPP
 
 // mapnik
 #include <mapnik/datasource.hpp>
@@ -33,7 +33,7 @@
 #include <boost/shared_ptr.hpp>
 
 // sqlite
-#include "sqlite_types.hpp"
+#include "sqlite_resultset.hpp"
   
 
 class sqlite_featureset : public mapnik::Featureset
@@ -55,4 +55,4 @@ private:
     bool using_subquery_;
 };
 
-#endif // SQLITE_FEATURESET_HPP
+#endif // MAPNIK_SQLITE_FEATURESET_HPP

plugins/input/sqlite/sqlite_prepared.hpp

@@ -0,0 +1,132 @@
+/*****************************************************************************
+ * 
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2011 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef MAPNIK_SQLITE_PREPARED_HPP
+#define MAPNIK_SQLITE_PREPARED_HPP
+
+// mapnik
+#include <mapnik/datasource.hpp>
+#include <mapnik/box2d.hpp>
+
+// boost
+#include <boost/shared_ptr.hpp>
+#include <boost/utility.hpp>
+
+// stl
+#include <string.h>
+
+#include "sqlite_connection.hpp"
+
+// sqlite
+extern "C" {
+  #include <sqlite3.h>
+}
+
+class prepared_index_statement : boost::noncopyable 
+{
+
+public:
+    prepared_index_statement(boost::shared_ptr<sqlite_connection> ds, std::string const& sql)
+        : ds_(ds),
+          stmt_(0)
+    {
+
+        const int rc = sqlite3_prepare_v2(*(*ds_),
+                                           sql.c_str(),
+                                           -1,
+                                           &stmt_,
+                                           0);
+
+        if (rc != SQLITE_OK)
+        {
+            std::ostringstream index_error;
+            index_error << "Sqlite Plugin: auto-index table creation failed: '"
+                        << sqlite3_errmsg(*(*ds_)) << "' query was: "
+                        << sql;
+
+            throw mapnik::datasource_exception(index_error.str());
+        }
+    }
+     
+    ~prepared_index_statement()
+    {
+        if (stmt_)
+        {
+            int res = sqlite3_finalize(stmt_);
+            if (res != SQLITE_OK)
+            {
+                if (*(*ds_))
+                {
+                    std::cerr << "ERR:" << sqlite3_errmsg(*(*ds_)) << "\n";
+                }
+                else
+                {
+                    std::cerr << "SQLite Plugin: " << res << "\n";
+                }
+            }
+        }
+    }
+    
+    void bind(sqlite_int64 const pkid)
+    {
+        if (sqlite3_bind_int64(stmt_, 1, pkid) != SQLITE_OK)
+        {
+            throw mapnik::datasource_exception("SQLite Plugin: invalid value for for key field while generating index");
+        }
+        
+    }
+
+    void bind(mapnik::box2d<double> const& bbox)
+    {
+        if ((sqlite3_bind_double(stmt_, 2, bbox.minx()) != SQLITE_OK) ||
+            (sqlite3_bind_double(stmt_, 3, bbox.maxx()) != SQLITE_OK) ||
+            (sqlite3_bind_double(stmt_, 4, bbox.miny()) != SQLITE_OK) ||
+            (sqlite3_bind_double(stmt_, 5, bbox.maxy()) != SQLITE_OK))
+        {
+            throw mapnik::datasource_exception("SQLite Plugin: invalid value for for extent while generating index");
+        }
+    }
+    
+    bool step_next ()
+    {
+        const int status = sqlite3_step(stmt_);
+        if (status != SQLITE_ROW && status != SQLITE_DONE)
+        {
+            std::ostringstream s;
+            s << "SQLite Plugin: inserting bbox into rtree index failed";
+            std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_)));
+            if (msg != "unknown error")
+            {
+                s << ": " << msg;
+            }
+            throw mapnik::datasource_exception(s.str());
+        }
+        sqlite3_reset(stmt_);
+        return status == SQLITE_ROW;
+    }
+
+private:
+    boost::shared_ptr<sqlite_connection> ds_;
+    sqlite3_stmt * stmt_;
+};
+
+#endif // MAPNIK_SQLITE_PREPARED_HPP

plugins/input/sqlite/sqlite_resultset.hpp

@@ -0,0 +1,143 @@
+/*****************************************************************************
+ * 
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2011 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef MAPNIK_SQLITE_RESULTSET_HPP
+#define MAPNIK_SQLITE_RESULTSET_HPP
+
+// mapnik
+#include <mapnik/datasource.hpp>
+
+// stl
+#include <string.h>
+
+// sqlite
+extern "C" {
+  #include <sqlite3.h>
+}
+
+
+
+//==============================================================================
+
+class sqlite_resultset
+{
+public:
+
+    sqlite_resultset (sqlite3_stmt* stmt)
+        : stmt_(stmt)
+    {
+    }
+
+    ~sqlite_resultset ()
+    {
+        if (stmt_)
+        {
+            sqlite3_finalize (stmt_);
+        }
+    }
+
+    bool is_valid ()
+    {
+        return stmt_ != 0;
+    }
+
+    bool step_next ()
+    {
+        const int status = sqlite3_step (stmt_);
+        if (status != SQLITE_ROW && status != SQLITE_DONE)
+        {
+            std::ostringstream s;
+            s << "SQLite Plugin: retrieving next row failed";
+            std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_)));
+            if (msg != "unknown error")
+            {
+                s << ": " << msg;
+            }
+
+            throw mapnik::datasource_exception(s.str());
+        }
+        return status == SQLITE_ROW;
+    }
+
+    int column_count ()
+    {
+        return sqlite3_column_count (stmt_);
+    }
+
+    int column_type (int col)
+    {
+        return sqlite3_column_type (stmt_, col);
+    }
+    
+    const char* column_name (int col)
+    {
+        return sqlite3_column_name (stmt_, col);
+    }
+
+    bool column_isnull (int col)
+    {
+        return sqlite3_column_type (stmt_, col) == SQLITE_NULL;
+    }
+
+    int column_integer (int col)
+    {
+        return sqlite3_column_int (stmt_, col);
+    }
+
+    int column_integer64 (int col)
+    {
+        return sqlite3_column_int64 (stmt_, col);
+    }
+
+    double column_double (int col)
+    {
+        return sqlite3_column_double (stmt_, col);
+    }
+
+    const char* column_text (int col, int& len)
+    {
+        len = sqlite3_column_bytes (stmt_, col);
+        return (const char*) sqlite3_column_text (stmt_, col);
+    }
+
+    const char* column_text (int col)
+    {
+        return (const char*) sqlite3_column_text (stmt_, col);
+    }
+
+    const void* column_blob (int col, int& bytes)
+    {
+        bytes = sqlite3_column_bytes (stmt_, col);
+        return (const char*) sqlite3_column_blob (stmt_, col);
+    }
+
+    sqlite3_stmt* get_statement()
+    {
+        return stmt_;
+    }
+
+private:
+
+    sqlite3_stmt* stmt_;
+};
+
+#endif // MAPNIK_SQLITE_RESULTSET_HPP

plugins/input/sqlite/sqlite_types.hpp

@@ -1,250 +0,0 @@
-/*****************************************************************************
- * 
- * This file is part of Mapnik (c++ mapping toolkit)
- *
- * Copyright (C) 2011 Artem Pavlenko
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
- *
- *****************************************************************************/
-
-#ifndef SQLITE_TYPES_HPP
-#define SQLITE_TYPES_HPP
-
-// stl
-#include <cstring>
-
-// mapnik
-#include <mapnik/datasource.hpp>
-
-// boost
-#include <boost/shared_ptr.hpp>
-#include <boost/algorithm/string.hpp>
-
-// sqlite
-extern "C" {
-  #include <sqlite3.h>
-}
-
-
-class sqlite_utils
-{
-public:
-    static void dequote(std::string& z)
-    {
-        boost::algorithm::trim_if(z, boost::algorithm::is_any_of("[]'\"`"));
-    }
-};
-
-
-class sqlite_resultset
-{
-public:
-    sqlite_resultset(sqlite3_stmt* stmt)
-        : stmt_(stmt)
-    {
-    }
-
-    virtual ~sqlite_resultset()
-    {
-        if (stmt_)
-        {
-            sqlite3_finalize (stmt_);
-        }
-    }
-
-    bool is_valid()
-    {
-        return stmt_ != 0;
-    }
-
-    bool step_next()
-    {
-        const int status = sqlite3_step(stmt_);
-        if (status != SQLITE_ROW && status != SQLITE_DONE)
-        {
-            std::ostringstream s;
-            s << "SQLite Plugin: retrieving next row failed";
-
-            std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_)));
-            if (msg != "unknown error")
-            {
-                s << ": " << msg;
-            }
-
-            throw mapnik::datasource_exception(s.str());
-        }
-        return status == SQLITE_ROW;
-    }
-
-    int column_count()
-    {
-        return sqlite3_column_count(stmt_);
-    }
-
-    int column_type(int col)
-    {
-        return sqlite3_column_type(stmt_, col);
-    }
-    
-    const char* column_name(int col)
-    {
-        return sqlite3_column_name(stmt_, col);
-    }
-
-    bool column_isnull(int col)
-    {
-        return sqlite3_column_type(stmt_, col) == SQLITE_NULL;
-    }
-
-    int column_integer(int col)
-    {
-        return sqlite3_column_int(stmt_, col);
-    }
-
-    int column_integer64(int col)
-    {
-        return sqlite3_column_int64(stmt_, col);
-    }
-
-    double column_double(int col)
-    {
-        return sqlite3_column_double(stmt_, col);
-    }
-
-    const char* column_text(int col, int& len)
-    {
-        len = sqlite3_column_bytes(stmt_, col);
-        return (const char*) sqlite3_column_text(stmt_, col);
-    }
-
-    const char* column_text(int col)
-    {
-        return (const char*) sqlite3_column_text(stmt_, col);
-    }
-
-    const void* column_blob(int col, int& bytes)
-    {
-        bytes = sqlite3_column_bytes(stmt_, col);
-        return (const char*) sqlite3_column_blob(stmt_, col);
-    }
-
-    sqlite3_stmt* get_statement()
-    {
-        return stmt_;
-    }
-
-private:
-
-    sqlite3_stmt* stmt_;
-};
-
-
-class sqlite_connection
-{
-public:
-
-    sqlite_connection(const std::string& file)
-        : db_(0)
-    {
-        // sqlite3_open_v2 is available earlier but 
-        // shared cache not available until >= 3.6.18
-    #if SQLITE_VERSION_NUMBER >= 3006018
-        const int rc = sqlite3_enable_shared_cache(1);
-        if (rc != SQLITE_OK)
-        {
-            throw mapnik::datasource_exception(sqlite3_errmsg (db_));
-        }
-        
-        int mode = SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE;
-        if (sqlite3_open_v2(file.c_str(), &db_, mode, NULL))
-    #else
-        #warning "Mapnik's sqlite plugin is compiling against a version of sqlite older than 3.6.18 which may make rendering slow..."
-        if (sqlite3_open(file.c_str(), &db_))
-    #endif
-        {
-            std::ostringstream s;
-            s << "Sqlite Plugin: " << sqlite3_errmsg(db_);
-
-            throw mapnik::datasource_exception(s.str());
-        }
-    }
-
-    virtual ~sqlite_connection()
-    {
-        if (db_)
-        {
-            sqlite3_close(db_);
-        }
-    }
-
-    void throw_sqlite_error(const std::string& sql)
-    {
-        std::ostringstream s;
-        s << "Sqlite Plugin: ";
-
-        if (db_)
-        {
-            s << "'" << sqlite3_errmsg(db_) << "'";
-        }
-        else
-        {
-            s << "unknown error, lost connection";
-        }
-
-        s << "\nFull sql was: '" <<  sql << "'\n";
-
-        throw mapnik::datasource_exception (s.str());
-    }
-    
-    sqlite_resultset* execute_query(const std::string& sql)
-    {
-        sqlite3_stmt* stmt = 0;
-
-        const int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, 0);
-        if (rc != SQLITE_OK)
-        {
-            throw_sqlite_error(sql);
-        }
-
-        return new sqlite_resultset (stmt);
-    }
-  
-    void execute(const std::string& sql)
-    {
-        const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
-        if (rc != SQLITE_OK)
-        {
-            throw_sqlite_error(sql);
-        }
-    }
-
-    int execute_with_code(const std::string& sql)
-    {
-        const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0);
-        return rc;
-    }
-   
-    sqlite3* operator*()
-    {
-        return db_;
-    }
-
-private:
-
-    sqlite3* db_;
-};
-
-#endif // SQLITE_TYPES_HPP

plugins/input/sqlite/sqlite_utils.hpp

@@ -0,0 +1,440 @@
+/*****************************************************************************
+ * 
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2011 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef MAPNIK_SQLITE_UTILS_HPP
+#define MAPNIK_SQLITE_UTILS_HPP
+
+// stl
+#include <string.h>
+
+// mapnik
+#include <mapnik/datasource.hpp>
+#include <mapnik/geometry.hpp>
+#include <mapnik/sql_utils.hpp>
+
+
+// boost
+#include <boost/shared_ptr.hpp>
+#include <boost/make_shared.hpp>
+#include <boost/algorithm/string.hpp>
+#include <boost/lexical_cast.hpp>
+
+// sqlite
+extern "C" {
+  #include <sqlite3.h>
+}
+
+#include "sqlite_resultset.hpp"
+#include "sqlite_prepared.hpp"
+#include "sqlite_connection.hpp"
+
+
+//==============================================================================
+
+class sqlite_utils
+{
+public:
+
+    static void dequote(std::string & z)
+    {
+        boost::algorithm::trim_if(z,boost::algorithm::is_any_of("[]'\"`"));
+    }
+    
+    static std::string index_for_table(std::string const& table, std::string const& field)
+    {
+        return "\"idx_" + mapnik::sql_utils::unquote(table) + "_" + field + "\"";
+    }
+
+    static std::string index_for_db(std::string const& file)
+    {
+        //std::size_t idx = file.find_last_of(".");
+        //if(idx != std::string::npos) {
+        //    return file.substr(0,idx) + ".index" + file.substr(idx);
+        //}
+        //else
+        //{
+            return file + ".index";
+        //}
+    }    
+    
+    static void query_extent(boost::shared_ptr<sqlite_resultset> rs,
+                             mapnik::box2d<double>& extent)
+    {
+
+        bool first = true;
+        while (rs->is_valid() && rs->step_next())
+        {
+            int size;
+            const char* data = (const char*) rs->column_blob(0, size);
+            if (data)
+            {
+                boost::ptr_vector<mapnik::geometry_type> paths;
+                mapnik::geometry_utils::from_wkb(paths, data, size, false, mapnik::wkbAuto);
+                for (unsigned i=0; i<paths.size(); ++i)
+                {
+                    mapnik::box2d<double> const& bbox = paths[i].envelope();
+    
+                    if (bbox.valid())
+                    {
+                        if (first)
+                        {
+                            first = false;
+                            extent = bbox;
+                        }
+                        else
+                        {
+                            extent.expand_to_include(bbox);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    static void create_rtree(std::string const& index_db,
+                                     std::string const& index_table,
+                                     boost::shared_ptr<sqlite_resultset> rs)
+    {
+        /* TODO
+          - speedups
+          - return early/recover from failure
+          - allow either in db or in separate
+        */
+        
+        if (!rs->is_valid())
+            return;
+        
+#if SQLITE_VERSION_NUMBER >= 3005000
+        int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX;
+#else
+        int flags;
+#endif
+
+        boost::shared_ptr<sqlite_connection> ds = boost::make_shared<sqlite_connection>(index_db,flags);
+    
+        // first drop the index if it already exists
+        std::ostringstream spatial_index_drop_sql;
+        spatial_index_drop_sql << "DROP TABLE IF EXISTS " << index_table;
+        ds->execute(spatial_index_drop_sql.str());
+
+        ds->execute("PRAGMA synchronous=OFF");
+        ds->execute("BEGIN TRANSACTION");
+
+        // create the spatial index
+        std::ostringstream create_idx;
+        create_idx << "create virtual table " 
+                   << index_table
+                   << " using rtree(pkid, xmin, xmax, ymin, ymax)";
+
+        // insert for prepared statement
+        std::ostringstream insert_idx;
+        insert_idx << "insert into "
+                   << index_table
+                   << " values (?,?,?,?,?)";
+
+        ds->execute(create_idx.str());
+        prepared_index_statement ps(ds,insert_idx.str());
+
+        while (rs->is_valid() && rs->step_next())
+        {
+            int size;
+            const char* data = (const char*) rs->column_blob(0, size);
+            if (data)
+            {
+                boost::ptr_vector<mapnik::geometry_type> paths;
+                // TODO - contraint fails if multiple_geometries = true
+                bool multiple_geometries = false;
+                mapnik::geometry_utils::from_wkb(paths, data, size, multiple_geometries, mapnik::wkbAuto);
+                for (unsigned i=0; i<paths.size(); ++i)
+                {
+                    mapnik::box2d<double> const& bbox = paths[i].envelope();
+                    if (bbox.valid())
+                    {
+                        ps.bind(bbox);
+    
+                        const int type_oid = rs->column_type(1);
+                        if (type_oid != SQLITE_INTEGER)
+                        {
+                            std::ostringstream error_msg;
+                            error_msg << "Sqlite Plugin: invalid type for key field '"
+                                      << rs->column_name(1) << "' when creating index '" << index_table
+                                      << "' type was: " << type_oid << "";
+                            throw mapnik::datasource_exception(error_msg.str());
+                        }
+    
+                        const sqlite_int64 pkid = rs->column_integer64(1);
+                        ps.bind(pkid);
+                    }
+                    else
+                    {
+                        std::ostringstream error_msg;
+                        error_msg << "SQLite Plugin: encountered invalid bbox at '"
+                                    << rs->column_name(1) << "' == " << rs->column_integer64(1);
+                        throw mapnik::datasource_exception(error_msg.str());
+                    }
+
+                    ps.step_next();
+                }
+            }
+        }
+        ds->execute("COMMIT");
+    }
+
+    static bool detect_extent(boost::shared_ptr<sqlite_connection> ds,
+                              bool has_spatial_index,
+                              mapnik::box2d<double> & extent,
+                              std::string const& index_table,
+                              std::string const& metadata,
+                              std::string const& geometry_field,
+                              std::string const& geometry_table,
+                              std::string const& key_field,
+                              std::string const& table
+                              )
+    {
+        if (has_spatial_index)
+        {
+            std::ostringstream s;
+            s << "SELECT MIN(xmin), MIN(ymin), MAX(xmax), MAX(ymax) FROM " 
+            << index_table;
+    
+            boost::shared_ptr<sqlite_resultset> rs(ds->execute_query(s.str()));
+            if (rs->is_valid() && rs->step_next())
+            {
+                if (! rs->column_isnull(0))
+                {
+                    try 
+                    {
+                        double xmin = boost::lexical_cast<double>(rs->column_double(0));
+                        double ymin = boost::lexical_cast<double>(rs->column_double(1));
+                        double xmax = boost::lexical_cast<double>(rs->column_double(2));
+                        double ymax = boost::lexical_cast<double>(rs->column_double(3));
+                        extent.init(xmin, ymin, xmax, ymax);
+                        return true;
+                    }
+                    catch (boost::bad_lexical_cast& ex)
+                    {
+                        std::ostringstream ss;
+                        ss << "SQLite Plugin: warning: could not determine extent from query: "
+                           << "'" << s.str() << "' \n problem was: " << ex.what() << std::endl;
+                        std::clog << ss.str();
+                    }
+                }
+            }
+        }
+        else if (! metadata.empty())
+        {
+            std::ostringstream s;
+            s << "SELECT xmin, ymin, xmax, ymax FROM " << metadata;
+            s << " WHERE LOWER(f_table_name) = LOWER('" << geometry_table << "')";
+            boost::shared_ptr<sqlite_resultset> rs(ds->execute_query(s.str()));
+            if (rs->is_valid() && rs->step_next())
+            {
+                double xmin = rs->column_double (0);
+                double ymin = rs->column_double (1);
+                double xmax = rs->column_double (2);
+                double ymax = rs->column_double (3);
+                extent.init (xmin, ymin, xmax, ymax);
+                return true;
+            }
+        }
+        else if (! key_field.empty())
+        {
+            std::ostringstream s;
+            s << "SELECT " << geometry_field << "," << key_field
+              << " FROM (" << table << ")";
+            boost::shared_ptr<sqlite_resultset> rs(ds->execute_query(s.str()));
+            sqlite_utils::query_extent(rs,extent);
+            return true;
+        }
+        return false;
+    }
+
+    static bool has_rtree(std::string const& index_table,boost::shared_ptr<sqlite_connection> ds)
+    {
+        try
+        {
+            std::ostringstream s;
+            s << "SELECT pkid,xmin,xmax,ymin,ymax FROM " << index_table << " LIMIT 1";
+            boost::shared_ptr<sqlite_resultset> rs = ds->execute_query(s.str());
+            if (rs->is_valid() && rs->step_next())
+            {
+                return true;
+            }
+        }
+        catch (...)
+        {
+            return false;
+        }
+        return false;
+    }
+
+    static bool detect_types_from_subquery(std::string const& query,
+                                    std::string & geometry_field,
+                                    mapnik::layer_descriptor & desc,
+                                    boost::shared_ptr<sqlite_connection> ds)
+    {
+        bool found = false;
+        boost::shared_ptr<sqlite_resultset> rs(ds->execute_query(query));
+        if (rs->is_valid() && rs->step_next())
+        {
+            for (int i = 0; i < rs->column_count(); ++i)
+            {
+                found = true;
+                const int type_oid = rs->column_type(i);
+                const char* fld_name = rs->column_name(i);
+                switch (type_oid)
+                {
+                case SQLITE_INTEGER:
+                    desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::Integer));
+                    break;
+                     
+                case SQLITE_FLOAT:
+                    desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::Double));
+                    break;
+                     
+                case SQLITE_TEXT:
+                    desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::String));
+                    break;
+                     
+                case SQLITE_NULL:
+                    // sqlite reports based on value, not actual column type unless
+                    // PRAGMA table_info is used so here we assume the column is a string
+                    // which is a lesser evil than altogether dropping the column
+                    desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::String));
+
+                case SQLITE_BLOB:
+                    if (geometry_field.empty()
+                        && (boost::algorithm::icontains(fld_name, "geom") ||
+                            boost::algorithm::icontains(fld_name, "point") ||
+                            boost::algorithm::icontains(fld_name, "linestring") ||
+                            boost::algorithm::icontains(fld_name, "polygon")))
+                    {
+                        geometry_field = std::string(fld_name);
+                    }
+                    break;
+                     
+                default:
+#ifdef MAPNIK_DEBUG
+                    std::clog << "Sqlite Plugin: unknown type_oid=" << type_oid << std::endl;
+#endif
+                    break;
+                }
+            }
+        }
+        
+        return found;
+    }
+
+    static bool table_info(std::string & key_field,
+                    bool detected_types,
+                    std::string & field,
+                    std::string & table,
+                    mapnik::layer_descriptor & desc,
+                    boost::shared_ptr<sqlite_connection> ds)
+    {
+
+        // http://www.sqlite.org/pragma.html#pragma_table_info
+        // use pragma table_info to detect primary key
+        // and to detect types if no subquery is used or 
+        // if the subquery-based type detection failed
+        std::ostringstream s;
+        s << "PRAGMA table_info(" << table << ")";
+        boost::shared_ptr<sqlite_resultset> rs(ds->execute_query(s.str()));
+        bool found_table = false;
+        bool found_pk = false;
+        while (rs->is_valid() && rs->step_next())
+        {
+            found_table = true;
+            const char* fld_name = rs->column_text(1);
+            std::string fld_type(rs->column_text(2));
+            int fld_pk = rs->column_integer(5);
+            boost::algorithm::to_lower(fld_type);
+    
+            // TODO - how to handle primary keys on multiple columns ?
+            if (key_field.empty() && ! found_pk && fld_pk != 0)
+            {
+                key_field = fld_name;
+                found_pk = true;
+            }
+            if (! detected_types)
+            {
+                // see 2.1 "Column Affinity" at http://www.sqlite.org/datatype3.html
+                // TODO - refactor this somehow ?
+                if (field.empty()
+                    && ((boost::algorithm::contains(fld_type, "geom") ||
+                         boost::algorithm::contains(fld_type, "point") ||
+                         boost::algorithm::contains(fld_type, "linestring") ||
+                         boost::algorithm::contains(fld_type, "polygon"))
+                    ||
+                        (boost::algorithm::icontains(fld_name, "geom") ||
+                         boost::algorithm::icontains(fld_name, "point") ||
+                         boost::algorithm::icontains(fld_name, "linestring") ||
+                         boost::algorithm::icontains(fld_name, "polygon")))
+                   )
+                {
+                    field = std::string(fld_name);
+                }
+                else if (boost::algorithm::contains(fld_type, "int"))
+                {
+                    desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::Integer));
+                }
+                else if (boost::algorithm::contains(fld_type, "text") ||
+                         boost::algorithm::contains(fld_type, "char") ||
+                         boost::algorithm::contains(fld_type, "clob"))
+                {
+                    desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::String));
+                }
+                else if (boost::algorithm::contains(fld_type, "real") ||
+                         boost::algorithm::contains(fld_type, "float") ||
+                         boost::algorithm::contains(fld_type, "double"))
+                {
+                    desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::Double));
+                }
+                else if (boost::algorithm::contains(fld_type, "blob"))
+                {
+                    if (! field.empty())
+                    {
+                        desc.add_descriptor(mapnik::attribute_descriptor(fld_name, mapnik::String));
+                    }
+                }
+        #ifdef MAPNIK_DEBUG
+                else
+                {
+                    // "Column Affinity" says default to "Numeric" but for now we pass..
+                    //desc_.add_descriptor(attribute_descriptor(fld_name,mapnik::Double));
+        
+                    // TODO - this should not fail when we specify geometry_field in XML file
+        
+                    std::clog << "Sqlite Plugin: column '"
+                              << std::string(fld_name)
+                              << "' unhandled due to unknown type: "
+                              << fld_type << std::endl;
+                }
+        #endif
+            }
+        }
+        
+        return found_table;
+    }
+};
+
+#endif // MAPNIK_SQLITE_UTILS_HPP

src/agg/process_text_symbolizer.cpp

@@ -35,6 +35,7 @@ void agg_renderer<T>::process(text_symbolizer const& sym,
 {
 
 
+    // Use a boost::ptr_vector here instread of std::vector?
     std::vector<geometry_type*> geometries_to_process;
     unsigned num_geom = feature.num_geometries();
     for (unsigned i=0; i<num_geom; ++i)

tests/python_tests/sqlite_test.py

@@ -164,7 +164,7 @@ if 'sqlite' in mapnik2.DatasourceCache.instance().plugin_names():
         eq_(feature['lat'],17.078)
         
         ds = mapnik2.SQLite(file='../data/sqlite/world.sqlite', 
-            table='(select OGC_FID,GEOMETRY as geom from world_merc)',
+            table='(select OGC_FID,GEOMETRY from world_merc)',
             )
         fs = ds.featureset()
         feature = fs.next()