OSM plugin: Fixed warnings, added URL support with cURL, removed multiple parsing

plugins/input/osm/SConscript

@@ -30,10 +30,13 @@ osm_src = Split(
 	osm.cpp
 	osm_datasource.cpp
 	osm_featureset.cpp 
+	dataset_deliverer.cpp
+	basiccurl.cpp
 	"""
         )
 
 libraries = [ 'xml2' ]
+libraries.append('curl')
 if env['PLATFORM'] == 'Darwin':
     libraries.append('mapnik')
     libraries.append('icuuc')

plugins/input/osm/basiccurl.cpp

@@ -0,0 +1,45 @@
+#include "basiccurl.h"
+
+CURL_LOAD_DATA *grab_http_response(const char *url)
+{
+	CURL_LOAD_DATA *data;
+
+	CURL *curl =  curl_easy_init(); 
+
+	if(curl)
+	{
+		data = do_grab(curl,url);
+		curl_easy_cleanup(curl);
+		return data;
+	}
+	return NULL;
+}
+
+CURL_LOAD_DATA *do_grab(CURL *curl,const char *url)
+{
+	CURLcode res;
+	CURL_LOAD_DATA *data = (CURL_LOAD_DATA *)malloc(sizeof(CURL_LOAD_DATA));
+	data->data = NULL;
+	data->nbytes = 0;
+	
+	curl_easy_setopt(curl,CURLOPT_URL,url);
+	curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION,response_callback);
+	curl_easy_setopt(curl,CURLOPT_WRITEDATA,data);
+
+	res=curl_easy_perform(curl);
+
+	return data;
+}
+
+size_t response_callback(void *ptr,size_t size,size_t nmemb, void *d)
+{
+	size_t rsize=size*nmemb;
+	CURL_LOAD_DATA *data=(CURL_LOAD_DATA *)d;
+//	fprintf(stderr,"rsize is %d\n", rsize);
+	data->data=(char *)realloc(data->data,(data->nbytes+rsize)
+										*sizeof(char));
+	memcpy(&(data->data[data->nbytes]),ptr,rsize);
+	data->nbytes += rsize;
+//	fprintf(stderr,"data->nbytes is %d\n", data->nbytes);
+	return rsize;
+}

plugins/input/osm/basiccurl.h

@@ -0,0 +1,18 @@
+#ifndef BASICCURL_H
+#define BASICCURL_H
+
+#include <curl/curl.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct
+{
+	char *data;
+	int nbytes;
+} CURL_LOAD_DATA;
+
+CURL_LOAD_DATA *grab_http_response(const char *url);
+CURL_LOAD_DATA *do_grab(CURL *curl,const char *url);
+size_t response_callback(void *ptr,size_t size,size_t nmemb, void *data);
+
+#endif

plugins/input/osm/dataset_deliverer.cpp

@@ -0,0 +1,33 @@
+#include "dataset_deliverer.h"
+#include "basiccurl.h"
+#include <sstream>
+
+osm_dataset * dataset_deliverer::dataset=NULL;
+
+osm_dataset* dataset_deliverer::load_from_file(const string& file,
+												const string& parser)
+{
+	// Only actually load from file if we haven't done so already
+	if(dataset == NULL)
+	{
+		dataset = new osm_dataset;
+		if(dataset->load(file.c_str(),parser)==false)
+				return NULL;
+		atexit(dataset_deliverer::release);
+
+	}
+	return dataset;
+}
+
+osm_dataset* dataset_deliverer::load_from_url
+	(const string& url,const string& bbox,const string& parser)
+{
+	if(dataset==NULL)
+	{
+		dataset = new osm_dataset;
+		if(dataset->load_from_url(url.c_str(),bbox,parser)==false)
+			return NULL;
+		atexit(dataset_deliverer::release);
+	}
+	return dataset;
+}

plugins/input/osm/dataset_deliverer.h

@@ -0,0 +1,22 @@
+#include "osm.h"
+#include <string>
+#include <iostream>
+
+using  namespace std;
+
+class dataset_deliverer 
+{
+private:
+	static osm_dataset* dataset; 
+
+public:
+	static osm_dataset *load_from_file(const string&,const string&);
+	static osm_dataset *load_from_url
+		(const string&,const string&,const string&);
+
+	static void release()
+	{
+		delete dataset;	
+	}
+};
+

plugins/input/osm/osm.cpp

@@ -5,6 +5,7 @@
 #include <fstream>
 #include <sstream>
 #include <string>
+#include "basiccurl.h"
 
 
 #include <iostream>
@@ -21,11 +22,40 @@ bool osm_dataset::load(const char* filename,const std::string& parser)
 	return false;	
 }
 
+bool osm_dataset::load_from_url(const std::string& url, 
+								const std::string& bbox,
+								const std::string& parser)
+{
+	if(parser=="libxml2")
+	{
+		cerr<<"osm_dataset::load_from_url: url=" << url << 
+			" bbox="<<bbox<<endl;
+		std::ostringstream str;
+		// use curl to grab the data
+		// fetch all the data we want - probably from osmxpai
+
+        str << url << "?bbox=" << bbox;
+		cerr << "FULL URL : " << str.str() << endl; 
+        CURL_LOAD_DATA *resp = grab_http_response(str.str().c_str()); 
+		if(resp!=NULL)
+		{
+			char *blx = new char[resp->nbytes+1];
+			memcpy(blx,resp->data,resp->nbytes);
+			blx[resp->nbytes] = '\0';
+			cerr<< " CURL RESPONSE: " << blx << endl;
+			delete[] blx;
+			bool success= osmparser::parse(this,resp->data,resp->nbytes);
+			return success;
+		}
+	}
+	return false;
+}
+
 osm_dataset::~osm_dataset()
 {
-	for(int count=0; count<ways.size(); count++)
+	for(unsigned int count=0; count<ways.size(); count++)
 		delete ways[count];
-	for(int count=0; count<nodes.size(); count++)
+	for(unsigned int count=0; count<nodes.size(); count++)
 		delete nodes[count];
 }
 
@@ -33,11 +63,11 @@ std::string osm_dataset::to_string()
 {
 	std::string result;
 
-	for(int count=0; count<nodes.size(); count++)
+	for(unsigned int count=0; count<nodes.size(); count++)
 	{
 		result += nodes[count]->to_string();
 	}
-	for(int count=0; count<ways.size(); count++)
+	for(unsigned int count=0; count<ways.size(); count++)
 	{
 		result += ways[count]->to_string();
 	}
@@ -47,7 +77,7 @@ std::string osm_dataset::to_string()
 bounds osm_dataset::get_bounds()
 {
 	bounds b (-180,-90,180,90);
-	for(int count=0; count<nodes.size();count++)
+	for(unsigned int count=0; count<nodes.size();count++)
 	{
 		if(nodes[count]->lon > b.w)
 			b.w=nodes[count]->lon;
@@ -102,7 +132,7 @@ osm_item *osm_dataset::next_item()
 std::set<std::string> osm_dataset::get_keys()
 {
 	std::set<std::string> keys;
-	for(int count=0; count<nodes.size(); count++)
+	for(unsigned int count=0; count<nodes.size(); count++)
 	{
 		for(std::map<std::string,std::string>::iterator i=
 			nodes[count]->keyvals.begin(); i!=nodes[count]->keyvals.end(); i++)
@@ -110,7 +140,7 @@ std::set<std::string> osm_dataset::get_keys()
 			keys.insert(i->first);
 		}
 	}
-	for(int count=0; count<ways.size(); count++)
+	for(unsigned int count=0; count<ways.size(); count++)
 	{
 		for(std::map<std::string,std::string>::iterator i=
 			ways[count]->keyvals.begin(); i!=ways[count]->keyvals.end(); i++)
@@ -149,7 +179,7 @@ std::string osm_way::to_string()
 	std::ostringstream strm;
 	strm << "Way: " << osm_item::to_string() << "Nodes in way:";
 
-	for(int count=0; count<nodes.size(); count++)
+	for(unsigned int count=0; count<nodes.size(); count++)
 	{
 		if(nodes[count]!=NULL)
 		{
@@ -163,7 +193,7 @@ std::string osm_way::to_string()
 bounds osm_way::get_bounds()
 {
 	bounds b (-180,-90,180,90);
-	for(int count=0; count<nodes.size();count++)
+	for(unsigned int count=0; count<nodes.size();count++)
 	{
 		if(nodes[count]->lon > b.w)
 			b.w=nodes[count]->lon;
@@ -179,7 +209,7 @@ bounds osm_way::get_bounds()
 
 bool osm_way::is_polygon()
 {
-	for(int count=0; count<ptypes.ptypes.size(); count++)
+	for(unsigned int count=0; count<ptypes.ptypes.size(); count++)
 	{
 		if(keyvals.find(ptypes.ptypes[count].first) != keyvals.end() &&
 		   keyvals[ptypes.ptypes[count].first] == ptypes.ptypes[count].second)

plugins/input/osm/osm.h

@@ -31,6 +31,8 @@ public:
         ptypes.push_back(std::pair<std::string,std::string>("natural","water"));
         ptypes.push_back(std::pair<std::string,std::string>("natural","heath"));
         ptypes.push_back(std::pair<std::string,std::string>("natural","marsh"));
+        ptypes.push_back(std::pair<std::string,std::string>("military",
+				"danger_area"));
         ptypes.push_back(std::pair<std::string,std::string>
                 ("landuse","forest"));
         ptypes.push_back(std::pair<std::string,std::string>
@@ -43,6 +45,7 @@ struct osm_item
     long id;
     std::map<std::string,std::string> keyvals;    
     virtual std::string to_string();
+	virtual ~osm_item() { }
 };
 
 
@@ -78,6 +81,8 @@ public:
         { node_i=nodes.begin(); way_i=ways.begin();
          next_item_mode=Node; load(name); }
     bool load(const char* name,const std::string& parser="libxml2");
+    bool load_from_url(const std::string&,const std::string&,
+                const std::string& parser="libxml2");
     ~osm_dataset();
     void add_node(osm_node* n) { nodes.push_back(n); }
     void add_way(osm_way* w) { ways.push_back(w); }

plugins/input/osm/osm_datasource.cpp

@@ -27,7 +27,9 @@
 #include <mapnik/query.hpp>
 #include "osm_datasource.hpp"
 #include "osm_featureset.hpp"
+#include "dataset_deliverer.h"
 #include "osmtagtypes.h"
+#include "osmparser.h"
 #include <set>
 
 DATASOURCE_PLUGIN(osm_datasource)
@@ -47,16 +49,38 @@ osm_datasource::osm_datasource(const parameters &params)
      type_(datasource::Vector),
      desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding","utf-8")) 
 {
-	osm_data_ = new osm_dataset;
+    osm_data_ = NULL;
     std::string osm_filename= *params.get<std::string>("file","");
     std::string parser = *params.get<std::string>("parser","libxml2");
+    std::string url = *params.get<std::string>("url","");
+    std::string bbox = *params.get<std::string>("bbox","");
+
+    bool do_process=false;
+
     // load the data
-	if (osm_data_->load(osm_filename.c_str(),parser)==false)
+    // if we supplied a filename, load from file
+    if (url!="" && bbox!="")
     {
-		//throw datasource_exception("Error loading OSM data");
-		return ;
+        // otherwise if we supplied a url and a bounding box, load from the url
+        if((osm_data_=dataset_deliverer::load_from_url
+            (url,bbox,parser))==NULL)    
+        {
+            throw datasource_exception("Error loading from URL");
+        }
+        do_process=true;
+    }
+    else if(osm_filename!="")
+    {
+        if ((osm_data_=
+            dataset_deliverer::load_from_file(osm_filename,parser))==NULL)
+        {
+            throw datasource_exception("Error loading from file");
+        }    
+        do_process=true;
     }
 
+    if(do_process==true)
+    {
         osm_tag_types tagtypes;
         tagtypes.add_type("maxspeed",mapnik::Integer);
         tagtypes.add_type("z_order",mapnik::Integer);
@@ -73,9 +97,15 @@ osm_datasource::osm_datasource(const parameters &params)
         // Get the bounds of the data and set extent_ accordingly
         bounds b = osm_data_->get_bounds();
         extent_ =  Envelope<double>(b.w,b.s,b.e,b.n);
+    }
 }
 
-osm_datasource::~osm_datasource() { delete osm_data_; }
+
+osm_datasource::~osm_datasource() 
+{ 
+    // Do not do as is now static variable and cleaned up atexit
+    //delete osm_data_; 
+}
 
 
 int osm_datasource::type() const

plugins/input/osm/osm_featureset.cpp

@@ -45,8 +45,8 @@ osm_featureset<filterT>::osm_featureset(const filterT& filter,
      query_ext_(),
      tr_(new transcoder(encoding)),
      count_(0),
-	attribute_names_ (attribute_names),
-	dataset_ (dataset)
+	dataset_ (dataset),
+	attribute_names_ (attribute_names)
 {
 	dataset_->rewind();
 }
@@ -101,7 +101,7 @@ feature_ptr osm_featureset<filterT>::next()
 								static_cast<osm_way*>(cur_item)->
 										nodes[0]->lat);
 				
-					for(int count=1; count<static_cast<osm_way*>(cur_item)
+					for(unsigned int count=1; count<static_cast<osm_way*>(cur_item)
 								->nodes.size(); count++)
 					{
 						geom->line_to(static_cast<osm_way*>(cur_item)

plugins/input/osm/osmparser.cpp

@@ -36,8 +36,6 @@ void osmparser::processNode(xmlTextReaderPtr reader)
 
 void osmparser::startElement(xmlTextReaderPtr reader, const xmlChar *name)
 {
-	double lat, lon;
-	int from, to;
 	std::string tags; 
 	xmlChar *xid, *xlat, *xlon, *xk, *xv;
 
@@ -45,7 +43,6 @@ void osmparser::startElement(xmlTextReaderPtr reader, const xmlChar *name)
 		{
 			curID = 0;
 			in_node = true;
-			int count=0;
 			osm_node *node=new osm_node;
 			xlat=xmlTextReaderGetAttribute(reader,BAD_CAST "lat");
 			xlon=xmlTextReaderGetAttribute(reader,BAD_CAST "lon");
@@ -118,6 +115,26 @@ bool osmparser::parse(osm_dataset *ds, const char* filename)
 {
 	components=ds;
 	xmlTextReaderPtr reader = xmlNewTextReaderFilename(filename);
+	int ret=do_parse(reader);
+	xmlFreeTextReader(reader);
+	return (ret==0) ?  true:false;
+}
+
+bool osmparser::parse(osm_dataset *ds,char* data, int nbytes)
+{
+	// from cocoasamurai.blogspot.com/2008/10/getting-some-xml-love-with-
+	// libxml2.html, converted from Objective-C to straight C
+
+	components=ds;
+	xmlTextReaderPtr reader = xmlReaderForMemory(data,nbytes,NULL,NULL,0);
+	int ret=do_parse(reader);
+	xmlFreeTextReader(reader);
+	return (ret==0) ? true:false;
+}
+
+	
+int osmparser::do_parse(xmlTextReaderPtr reader)
+{
 	int ret=-1;
 	if(reader!=NULL)
 	{
@@ -127,7 +144,6 @@ bool osmparser::parse(osm_dataset *ds, const char* filename)
 			processNode(reader);
 			ret=xmlTextReaderRead(reader);
 		}
-		xmlFreeTextReader(reader);
 	}
-	return (ret==0) ?  true:false;
+	return ret;
 }

plugins/input/osm/osmparser.h

@@ -17,10 +17,13 @@ private:
     static std::string error;
     static std::map<long,osm_node*> tmp_node_store;
 
+	static int do_parse(xmlTextReaderPtr);
+
 public:
     static void processNode(xmlTextReaderPtr reader);
     static void startElement(xmlTextReaderPtr reader, const xmlChar *name);
     static void endElement(const xmlChar* name);
     static bool parse(osm_dataset *ds, const char* filename);
+    static bool parse(osm_dataset *ds, char* data,int nbytes);
 };