+ add 'tiled' policy to avoid loading very large rasters into memory

if (width*height > 1024*1024) : load raster in 1024x1024 chunks
  else : use single_file_policy

  TODO: Expose chunk size as raster_datasource parameter

plugins/input/raster/raster_datasource.cpp

@@ -29,11 +29,11 @@
 
 #include "raster_featureset.hpp"
 #include "raster_info.hpp"
-
 #include "raster_datasource.hpp"
 
 using mapnik::datasource;
 using mapnik::parameters;
+using mapnik::ImageReader;
 
 DATASOURCE_PLUGIN(raster_datasource)
 
@@ -73,6 +73,21 @@ raster_datasource::raster_datasource(const parameters& params)
       extent_.init(*lox,*loy,*hix,*hiy);
    }
    else throw datasource_exception("<lox> <loy> <hix> <hiy> are required");
+
+   try
+   {         
+      std::auto_ptr<ImageReader> reader(mapnik::get_image_reader(filename_, format_));
+      if (reader.get())
+      {
+         width_ = reader->width();
+         height_ = reader->height();
+         std::cout << "RASTER SIZE("<<width_ << "," << height_ << ")\n"; 
+      }
+   }
+   catch (...)
+   {
+      std::cerr << "Exception caught\n";
+   }
 }
 
 raster_datasource::~raster_datasource() {}
@@ -100,12 +115,28 @@ layer_descriptor raster_datasource::get_descriptor() const
 
 featureset_ptr raster_datasource::features(query const& q) const
 {
-    raster_info info(filename_,format_,extent_);
+   mapnik::CoordTransform t(width_,height_,extent_,0,0);
-    single_file_policy policy(info); //todo: handle different policies!
+   mapnik::Envelope<double> intersect=extent_.intersect(q.get_bbox());
-    return featureset_ptr(new raster_featureset<single_file_policy>(policy,q));
+   mapnik::Envelope<double> ext=t.forward(intersect);
+   
+   unsigned width = int(ext.width()+0.5);
+   unsigned height = int(ext.height() + 0.5);
+   std::cout << width << " " << height << "\n";
+   if (width * height > 1024*1024)
+   {
+      std::cout << "TILED policy\n";
+      tiled_file_policy policy(filename_,format_, 1024, extent_,q.get_bbox(), width_,height_);
+      return featureset_ptr(new raster_featureset<tiled_file_policy>(policy,extent_,q));
+   }
+   else
+   {
+      std::cout << "SINGLE FILE\n";
+      raster_info info(filename_,format_,extent_,width_,height_);
+      single_file_policy policy(info);
+      return featureset_ptr(new raster_featureset<single_file_policy>(policy,extent_,q));
+   }
 }
 
-
 featureset_ptr raster_datasource::features_at_point(coord2d const&) const
 {
     return featureset_ptr();

plugins/input/raster/raster_datasource.hpp

@@ -30,26 +30,28 @@
 
 class raster_datasource : public mapnik::datasource
 {
-   private:
+private:
-      std::string                  filename_;
+   std::string                  filename_;
-      std::string                  format_;
+   std::string                  format_;
-      mapnik::Envelope<double>     extent_;
+   mapnik::Envelope<double>     extent_;
-      mapnik::layer_descriptor     desc_;
+   mapnik::layer_descriptor     desc_;
-      static std::string           name_;  
+   unsigned                     width_;
-   public:
+   unsigned                     height_;
-      raster_datasource(const mapnik::parameters& params);
+   static std::string           name_;  
-      virtual ~raster_datasource();
+public:
-      int type() const;
+   raster_datasource(const mapnik::parameters& params);
-      static std::string name();
+   virtual ~raster_datasource();
-      mapnik::featureset_ptr features(const mapnik::query& q) const;
+   int type() const;
-      mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const;
+   static std::string name();
-      mapnik::Envelope<double> envelope() const;
+   mapnik::featureset_ptr features(const mapnik::query& q) const;
-      mapnik::layer_descriptor get_descriptor() const;
+   mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt) const;
-   private:
+   mapnik::Envelope<double> envelope() const;
-      //no copying
+   mapnik::layer_descriptor get_descriptor() const;
-      raster_datasource(const raster_datasource&);
+private:
-      raster_datasource& operator=(const raster_datasource&);
+   //no copying
-      //
+   raster_datasource(const raster_datasource&);
+   raster_datasource& operator=(const raster_datasource&);
+   //
 };
 
 #endif //RASTER_DATASOURCE_HPP

plugins/input/raster/raster_featureset.cpp

@@ -35,11 +35,13 @@ using mapnik::raster;
 
 template <typename LookupPolicy>
 raster_featureset<LookupPolicy>::raster_featureset(LookupPolicy const& policy,
+                                                   Envelope<double> const& extent,
                                                    query const& q)
    : policy_(policy),
      id_(1),
-     extent_(q.get_bbox()),
+     extent_(extent),
-     curIter_(policy_.query(extent_)),
+     bbox_(q.get_bbox()),
+     curIter_(policy_.begin()),
      endIter_(policy_.end()) 
 {}
 
@@ -55,8 +57,11 @@ feature_ptr raster_featureset<LookupPolicy>::next()
       try
       {         
          std::auto_ptr<ImageReader> reader(mapnik::get_image_reader(curIter_->file(),curIter_->format()));
-         
+
-         std::cout << "READER = " << curIter_->format() << " " << curIter_->file() << "\n";
+#ifdef MAPNIK_DEBUG         
+         std::cout << "READER = " << curIter_->format() << " " << curIter_->file() 
+                   << " size(" << curIter_->width() << "," << curIter_->height() << ")\n";
+#endif
          if (reader.get())
          {
             int image_width=reader->width();
@@ -64,19 +69,20 @@ feature_ptr raster_featureset<LookupPolicy>::next()
             
             if (image_width>0 && image_height>0)
             {
-               CoordTransform t(image_width,image_height,curIter_->envelope(),0,0);
+               CoordTransform t(image_width,image_height,extent_,0,0);
-               Envelope<double> intersect=extent_.intersect(curIter_->envelope());
+               Envelope<double> intersect=bbox_.intersect(curIter_->envelope());
                Envelope<double> ext=t.forward(intersect);
-                    
+               ImageData32 image(int(ext.width()+0.5),int(ext.height()+0.5));
-               ImageData32 image((int)(ext.width()+0.5),(int)(ext.height()+0.5));
+               reader->read(int(ext.minx()+0.5),int(ext.miny()+0.5),image);
-               reader->read((int)(ext.minx()+0.5),(int)(ext.miny()+0.5),image);
                feature->set_raster(mapnik::raster_ptr(new raster(intersect,image)));
             }
          }
       }
       catch (...)
       {
+         std::cerr << "Exception caught\n";
       }
+      
       ++curIter_;
       return feature;
    }
@@ -84,3 +90,4 @@ feature_ptr raster_featureset<LookupPolicy>::next()
 }
 
 template class raster_featureset<single_file_policy>;
+template class raster_featureset<tiled_file_policy>;

plugins/input/raster/raster_featureset.hpp

@@ -19,14 +19,16 @@
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  *
  *****************************************************************************/
-#ifndef RASTER_FEATURESET_HH
+#ifndef RASTER_FEATURESET_HPP
-#define RASTER_FEATURESET_HH
+#define RASTER_FEATURESET_HPP
 
 #include <vector>
 
 #include "raster_datasource.hpp"
 #include "raster_info.hpp"
 
+#include <boost/utility.hpp>
+
 class single_file_policy
 {
     raster_info info_;
@@ -93,19 +95,79 @@ public:
     }
 };
 
+class tiled_file_policy
+{
+public:
+   
+   typedef std::vector<raster_info>::const_iterator const_iterator;
+   
+   tiled_file_policy(std::string const& file, std::string const& format, unsigned tile_size, 
+                     Envelope<double> extent, Envelope<double> bbox,unsigned width, unsigned height)
+   {     
+      
+      double lox = extent.minx();
+      double loy = extent.miny();
+   
+      int max_x = int(ceil(double(width)/double(tile_size)));
+      int max_y = int(ceil(double(height)/double(tile_size)));
+
+      double pixel_x = extent.width()/double(width);
+      double pixel_y = extent.height()/double(height);
+      std::cout << "PIXEL SIZE("<< pixel_x << "," << pixel_y << ")\n";
+      
+      Envelope<double> e = bbox.intersect(extent);
+      
+      for (int x = 0 ; x < max_x ; ++x)
+      {
+         for (int y = 0 ; y < max_y ; ++y)
+         {
+            double x0 = lox + x*tile_size*pixel_x;
+            double y0 = loy + y*tile_size*pixel_y;
+            double x1 = x0 + tile_size*pixel_x;
+            double y1 = y0 + tile_size*pixel_y;
+            
+            if (e.intersects(Envelope<double>(x0,y0,x1,y1)))
+            {
+               Envelope<double> tile_box = e.intersect(Envelope<double>(x0,y0,x1,y1));            
+               raster_info info(file,format,tile_box,tile_size,tile_size);
+               infos_.push_back(info);
+            }
+         }
+      }
+      std::cout << "INFO SIZE=" << infos_.size() << " " << file << "\n";
+      
+   }
+   
+   const_iterator begin()
+   {
+      return infos_.begin();
+   }
+      
+   const_iterator end()
+   {
+      return infos_.end();
+   }
+   
+private:
+
+   std::vector<raster_info> infos_;
+};
+
+
 template <typename LookupPolicy>
 class raster_featureset : public mapnik::Featureset
 {
-      typedef typename LookupPolicy::const_iterator iterator_type;
+   typedef typename LookupPolicy::const_iterator iterator_type;
-      LookupPolicy policy_;
+   LookupPolicy policy_;
-      size_t id_;
+   size_t id_;
-      mapnik::Envelope<double> extent_;
+   mapnik::Envelope<double> extent_;
-      iterator_type curIter_;
+   mapnik::Envelope<double> bbox_;
-      iterator_type endIter_;
+   iterator_type curIter_;
+   iterator_type endIter_;
 public:
-      raster_featureset(LookupPolicy const& policy,mapnik::query const& q);
+   raster_featureset(LookupPolicy const& policy,Envelope<double> const& exttent, mapnik::query const& q);
-      virtual ~raster_featureset();
+   virtual ~raster_featureset();
-      mapnik::feature_ptr next();
+   mapnik::feature_ptr next();
 };
 
-#endif //RASTER_FEATURESET_HH
+#endif //RASTER_FEATURESET_HPP

plugins/input/raster/raster_info.cpp

@@ -23,24 +23,28 @@
 
 #include "raster_info.hpp"
 
-raster_info::raster_info(const std::string& file,const std::string& format,const mapnik::Envelope<double>& extent,int srid)
+raster_info::raster_info(std::string const& file, std::string const& format,
+                         mapnik::Envelope<double> const& extent, unsigned width, unsigned height)
     :file_(file),
      format_(format),
      extent_(extent),
-     srid_(srid) {}
+     width_(width),
+     height_(height) {}
 
 raster_info::raster_info(const raster_info& rhs)
     :file_(rhs.file_),
      format_(rhs.format_),
      extent_(rhs.extent_),
-     srid_(rhs.srid_) {}
+     width_(rhs.width_),
+     height_(rhs.height_) {}
 
 void raster_info::swap(raster_info& other) throw()
 {
     file_=other.file_;
     format_=other.format_;
     extent_=other.extent_;
-    srid_=other.srid_;
+    width_=other.width_;
+    height_=other.height_;
 }
 
 
@@ -52,23 +56,4 @@ raster_info& raster_info::operator=(const raster_info& rhs)
 }
 
 
-const Envelope<double>& raster_info::envelope() const
-{
-    return extent_;
-}
 
-
-const std::string& raster_info::file() const
-{
-    return file_;
-}
-
-const std::string& raster_info::format() const
-{
-    return format_;
-}
-
-const int raster_info::srid() const
-{
-    return srid_;
-}

plugins/input/raster/raster_info.hpp

@@ -21,8 +21,8 @@
  *****************************************************************************/
 //$Id: raster_info.hh 17 2005-03-08 23:58:43Z pavlenko $
 
-#ifndef RASTER_INFO
+#ifndef RASTER_INFO_HPP
-#define RASTER_INFO
+#define RASTER_INFO_HPP
 
 #include "raster_datasource.hpp"
 #include <string>
@@ -31,19 +31,23 @@ using mapnik::Envelope;
 
 class raster_info
 {
-    std::string file_;
+   std::string file_;
-    std::string format_;
+   std::string format_;
-    Envelope<double> extent_;
+   Envelope<double> extent_;
-    int srid_;
+   unsigned width_;
-    public:
+   unsigned height_;
-    raster_info(const std::string& file,const std::string& format,const Envelope<double>& extent,int srid=-1);
+public:
-    raster_info(const raster_info& rhs);
+   raster_info(const std::string& file,const std::string& format, const Envelope<double>& extent, unsigned width, unsigned height);
-    raster_info& operator=(const raster_info& rhs);
+   raster_info(const raster_info& rhs);
-    const Envelope<double>& envelope() const;
+   raster_info& operator=(const raster_info& rhs);
-    const std::string& file() const;
+   inline Envelope<double> const& envelope() const {return extent_;}
-    const std::string& format() const;
+   inline std::string const& file() const { return file_;}
-    const int srid() const;
+   inline std::string const& format() const {return format_;}
+   inline unsigned width() const { return width_;}
+   inline unsigned height() const { return height_;}
+   
 private:
-    void swap(raster_info& other) throw();
+   void swap(raster_info& other) throw();
 };
-#endif                                            //RASTER_INFO
+
+#endif  //RASTER_INFO_HPP