sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

OSM plugin: Fixed warnings, added URL support with cURL, removed multiple parsing

Browse source
This commit is contained in:
Nick Whitelegg 2009-02-15 13:51:07 +00:00
parent d1b365098f
commit e68d934eff
11 changed files with 252 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
plugins/input/osm/SConscript
View file

@ -30,10 +30,13 @@ osm_src = Split(
osm.cpp
osm_datasource.cpp
osm_featureset.cpp
dataset_deliverer.cpp
basiccurl.cpp
"""
)
libraries = [ 'xml2' ]
libraries.append('curl')
if env['PLATFORM'] == 'Darwin':
libraries.append('mapnik')
libraries.append('icuuc')

45
plugins/input/osm/basiccurl.cpp Executable file
View file

@ -0,0 +1,45 @@
#include "basiccurl.h"
CURL_LOAD_DATA *grab_http_response(const char *url)
{
CURL_LOAD_DATA *data;
CURL *curl = curl_easy_init();
if(curl)
{
data = do_grab(curl,url);
curl_easy_cleanup(curl);
return data;
}
return NULL;
}
CURL_LOAD_DATA *do_grab(CURL *curl,const char *url)
{
CURLcode res;
CURL_LOAD_DATA *data = (CURL_LOAD_DATA *)malloc(sizeof(CURL_LOAD_DATA));
data->data = NULL;
data->nbytes = 0;
curl_easy_setopt(curl,CURLOPT_URL,url);
curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION,response_callback);
curl_easy_setopt(curl,CURLOPT_WRITEDATA,data);
res=curl_easy_perform(curl);
return data;
}
size_t response_callback(void *ptr,size_t size,size_t nmemb, void *d)
{
size_t rsize=size*nmemb;
CURL_LOAD_DATA *data=(CURL_LOAD_DATA *)d;
// fprintf(stderr,"rsize is %d\n", rsize);
data->data=(char *)realloc(data->data,(data->nbytes+rsize)
*sizeof(char));
memcpy(&(data->data[data->nbytes]),ptr,rsize);
data->nbytes += rsize;
// fprintf(stderr,"data->nbytes is %d\n", data->nbytes);
return rsize;
}

18
plugins/input/osm/basiccurl.h Executable file
View file

@ -0,0 +1,18 @@
#ifndef BASICCURL_H
#define BASICCURL_H
#include <curl/curl.h>
#include <stdlib.h>
#include <string.h>
typedef struct
{
char *data;
int nbytes;
} CURL_LOAD_DATA;
CURL_LOAD_DATA *grab_http_response(const char *url);
CURL_LOAD_DATA *do_grab(CURL *curl,const char *url);
size_t response_callback(void *ptr,size_t size,size_t nmemb, void *data);
#endif

33
plugins/input/osm/dataset_deliverer.cpp Normal file
View file

@ -0,0 +1,33 @@
#include "dataset_deliverer.h"
#include "basiccurl.h"
#include <sstream>
osm_dataset * dataset_deliverer::dataset=NULL;
osm_dataset* dataset_deliverer::load_from_file(const string& file,
const string& parser)
{
// Only actually load from file if we haven't done so already
if(dataset == NULL)
{
dataset = new osm_dataset;
if(dataset->load(file.c_str(),parser)==false)
return NULL;
atexit(dataset_deliverer::release);
}
return dataset;
}
osm_dataset* dataset_deliverer::load_from_url
(const string& url,const string& bbox,const string& parser)
{
if(dataset==NULL)
{
dataset = new osm_dataset;
if(dataset->load_from_url(url.c_str(),bbox,parser)==false)
return NULL;
atexit(dataset_deliverer::release);
}
return dataset;
}

22
plugins/input/osm/dataset_deliverer.h Normal file
View file

@ -0,0 +1,22 @@
#include "osm.h"
#include <string>
#include <iostream>
using namespace std;
class dataset_deliverer
{
private:
static osm_dataset* dataset;
public:
static osm_dataset *load_from_file(const string&,const string&);
static osm_dataset *load_from_url
(const string&,const string&,const string&);
static void release()
{
delete dataset;
}
};

50
plugins/input/osm/osm.cpp
View file

@ -5,6 +5,7 @@
#include <fstream>
#include <sstream>
#include <string>
#include "basiccurl.h"
#include <iostream>
@ -21,11 +22,40 @@ bool osm_dataset::load(const char* filename,const std::string& parser)
return false;
}
bool osm_dataset::load_from_url(const std::string& url,
const std::string& bbox,
const std::string& parser)
{
if(parser=="libxml2")
{
cerr<<"osm_dataset::load_from_url: url=" << url <<
" bbox="<<bbox<<endl;
std::ostringstream str;
// use curl to grab the data
// fetch all the data we want - probably from osmxpai
str << url << "?bbox=" << bbox;
cerr << "FULL URL : " << str.str() << endl;
CURL_LOAD_DATA *resp = grab_http_response(str.str().c_str());
if(resp!=NULL)
{
char *blx = new char[resp->nbytes+1];
memcpy(blx,resp->data,resp->nbytes);
blx[resp->nbytes] = '\0';
cerr<< " CURL RESPONSE: " << blx << endl;
delete[] blx;
bool success= osmparser::parse(this,resp->data,resp->nbytes);
return success;
}
}
return false;
}
osm_dataset::~osm_dataset()
{
for(int count=0; count<ways.size(); count++)
for(unsigned int count=0; count<ways.size(); count++)
delete ways[count];
for(int count=0; count<nodes.size(); count++)
for(unsigned int count=0; count<nodes.size(); count++)
delete nodes[count];
}
@ -33,11 +63,11 @@ std::string osm_dataset::to_string()
{
std::string result;
for(int count=0; count<nodes.size(); count++)
for(unsigned int count=0; count<nodes.size(); count++)
{
result += nodes[count]->to_string();
}
for(int count=0; count<ways.size(); count++)
for(unsigned int count=0; count<ways.size(); count++)
{
result += ways[count]->to_string();
}
@ -47,7 +77,7 @@ std::string osm_dataset::to_string()
bounds osm_dataset::get_bounds()
{
bounds b (-180,-90,180,90);
for(int count=0; count<nodes.size();count++)
for(unsigned int count=0; count<nodes.size();count++)
{
if(nodes[count]->lon > b.w)
b.w=nodes[count]->lon;
@ -102,7 +132,7 @@ osm_item *osm_dataset::next_item()
std::set<std::string> osm_dataset::get_keys()
{
std::set<std::string> keys;
for(int count=0; count<nodes.size(); count++)
for(unsigned int count=0; count<nodes.size(); count++)
{
for(std::map<std::string,std::string>::iterator i=
nodes[count]->keyvals.begin(); i!=nodes[count]->keyvals.end(); i++)
@ -110,7 +140,7 @@ std::set<std::string> osm_dataset::get_keys()
keys.insert(i->first);
}
}
for(int count=0; count<ways.size(); count++)
for(unsigned int count=0; count<ways.size(); count++)
{
for(std::map<std::string,std::string>::iterator i=
ways[count]->keyvals.begin(); i!=ways[count]->keyvals.end(); i++)
@ -149,7 +179,7 @@ std::string osm_way::to_string()
std::ostringstream strm;
strm << "Way: " << osm_item::to_string() << "Nodes in way:";
for(int count=0; count<nodes.size(); count++)
for(unsigned int count=0; count<nodes.size(); count++)
{
if(nodes[count]!=NULL)
{
@ -163,7 +193,7 @@ std::string osm_way::to_string()
bounds osm_way::get_bounds()
{
bounds b (-180,-90,180,90);
for(int count=0; count<nodes.size();count++)
for(unsigned int count=0; count<nodes.size();count++)
{
if(nodes[count]->lon > b.w)
b.w=nodes[count]->lon;
@ -179,7 +209,7 @@ bounds osm_way::get_bounds()
bool osm_way::is_polygon()
{
for(int count=0; count<ptypes.ptypes.size(); count++)
for(unsigned int count=0; count<ptypes.ptypes.size(); count++)
{
if(keyvals.find(ptypes.ptypes[count].first) != keyvals.end() &&
keyvals[ptypes.ptypes[count].first] == ptypes.ptypes[count].second)

5
plugins/input/osm/osm.h
View file

@ -31,6 +31,8 @@ public:
ptypes.push_back(std::pair<std::string,std::string>("natural","water"));
ptypes.push_back(std::pair<std::string,std::string>("natural","heath"));
ptypes.push_back(std::pair<std::string,std::string>("natural","marsh"));
ptypes.push_back(std::pair<std::string,std::string>("military",
"danger_area"));
ptypes.push_back(std::pair<std::string,std::string>
("landuse","forest"));
ptypes.push_back(std::pair<std::string,std::string>
@ -43,6 +45,7 @@ struct osm_item
long id;
std::map<std::string,std::string> keyvals;
virtual std::string to_string();
virtual ~osm_item() { }
};
@ -78,6 +81,8 @@ public:
{ node_i=nodes.begin(); way_i=ways.begin();
next_item_mode=Node; load(name); }
bool load(const char* name,const std::string& parser="libxml2");
bool load_from_url(const std::string&,const std::string&,
const std::string& parser="libxml2");
~osm_dataset();
void add_node(osm_node* n) { nodes.push_back(n); }
void add_way(osm_way* w) { ways.push_back(w); }

88
plugins/input/osm/osm_datasource.cpp
View file

@ -27,7 +27,9 @@
#include <mapnik/query.hpp>
#include "osm_datasource.hpp"
#include "osm_featureset.hpp"
#include "dataset_deliverer.h"
#include "osmtagtypes.h"
#include "osmparser.h"
#include <set>
DATASOURCE_PLUGIN(osm_datasource)
@ -45,37 +47,65 @@ const std::string osm_datasource::name_ = "osm";
osm_datasource::osm_datasource(const parameters &params)
: datasource (params),
type_(datasource::Vector),
desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding","utf-8"))
desc_(*params.get<std::string>("type"), *params.get<std::string>("encoding","utf-8"))
{
osm_data_ = new osm_dataset;
std::string osm_filename= *params.get<std::string>("file","");
std::string parser = *params.get<std::string>("parser","libxml2");
// load the data
if (osm_data_->load(osm_filename.c_str(),parser)==false)
{
//throw datasource_exception("Error loading OSM data");
return ;
}
osm_data_ = NULL;
std::string osm_filename= *params.get<std::string>("file","");
std::string parser = *params.get<std::string>("parser","libxml2");
std::string url = *params.get<std::string>("url","");
std::string bbox = *params.get<std::string>("bbox","");
osm_tag_types tagtypes;
tagtypes.add_type("maxspeed",mapnik::Integer);
tagtypes.add_type("z_order",mapnik::Integer);
bool do_process=false;
osm_data_->rewind();
// Need code to get the attributes of all the data
std::set<std::string> keys= osm_data_->get_keys();
// load the data
// if we supplied a filename, load from file
if (url!="" && bbox!="")
{
// otherwise if we supplied a url and a bounding box, load from the url
if((osm_data_=dataset_deliverer::load_from_url
(url,bbox,parser))==NULL)
{
throw datasource_exception("Error loading from URL");
}
do_process=true;
}
else if(osm_filename!="")
{
if ((osm_data_=
dataset_deliverer::load_from_file(osm_filename,parser))==NULL)
{
throw datasource_exception("Error loading from file");
}
do_process=true;
}
// Add the attributes to the datasource descriptor - assume they're
// all of type String
for(std::set<std::string>::iterator i=keys.begin(); i!=keys.end(); i++)
desc_.add_descriptor(attribute_descriptor(*i,tagtypes.get_type(*i)));
if(do_process==true)
{
osm_tag_types tagtypes;
tagtypes.add_type("maxspeed",mapnik::Integer);
tagtypes.add_type("z_order",mapnik::Integer);
// Get the bounds of the data and set extent_ accordingly
bounds b = osm_data_->get_bounds();
extent_ = Envelope<double>(b.w,b.s,b.e,b.n);
osm_data_->rewind();
// Need code to get the attributes of all the data
std::set<std::string> keys= osm_data_->get_keys();
// Add the attributes to the datasource descriptor - assume they're
// all of type String
for(std::set<std::string>::iterator i=keys.begin(); i!=keys.end(); i++)
desc_.add_descriptor(attribute_descriptor(*i,tagtypes.get_type(*i)));
// Get the bounds of the data and set extent_ accordingly
bounds b = osm_data_->get_bounds();
extent_ = Envelope<double>(b.w,b.s,b.e,b.n);
}
}
osm_datasource::~osm_datasource() { delete osm_data_; }
osm_datasource::~osm_datasource()
{
// Do not do as is now static variable and cleaned up atexit
//delete osm_data_;
}
int osm_datasource::type() const
@ -91,9 +121,9 @@ layer_descriptor osm_datasource::get_descriptor() const
featureset_ptr osm_datasource::features(const query& q) const
{
filter_in_box filter(q.get_bbox());
// so we need to filter osm features by bbox here...
return featureset_ptr
// so we need to filter osm features by bbox here...
return featureset_ptr
(new osm_featureset<filter_in_box>(filter,
osm_data_,
q.property_names(),
@ -105,7 +135,7 @@ featureset_ptr osm_datasource::features_at_point(coord2d const& pt) const
filter_at_point filter(pt);
// collect all attribute names
std::vector<attribute_descriptor> const& desc_vector =
desc_.get_descriptors();
desc_.get_descriptors();
std::vector<attribute_descriptor>::const_iterator itr = desc_vector.begin();
std::vector<attribute_descriptor>::const_iterator end = desc_vector.end();
std::set<std::string> names;
@ -116,7 +146,7 @@ featureset_ptr osm_datasource::features_at_point(coord2d const& pt) const
++itr;
}
return featureset_ptr
return featureset_ptr
(new osm_featureset<filter_at_point>(filter,
osm_data_,
names,

6
plugins/input/osm/osm_featureset.cpp
View file

@ -45,8 +45,8 @@ osm_featureset<filterT>::osm_featureset(const filterT& filter,
query_ext_(),
tr_(new transcoder(encoding)),
count_(0),
attribute_names_ (attribute_names),
dataset_ (dataset)
dataset_ (dataset),
attribute_names_ (attribute_names)
{
dataset_->rewind();
}
@ -101,7 +101,7 @@ feature_ptr osm_featureset<filterT>::next()
static_cast<osm_way*>(cur_item)->
nodes[0]->lat);
for(int count=1; count<static_cast<osm_way*>(cur_item)
for(unsigned int count=1; count<static_cast<osm_way*>(cur_item)
->nodes.size(); count++)
{
geom->line_to(static_cast<osm_way*>(cur_item)

26
plugins/input/osm/osmparser.cpp
View file

@ -36,8 +36,6 @@ void osmparser::processNode(xmlTextReaderPtr reader)
void osmparser::startElement(xmlTextReaderPtr reader, const xmlChar *name)
{
double lat, lon;
int from, to;
std::string tags;
xmlChar *xid, *xlat, *xlon, *xk, *xv;
@ -45,7 +43,6 @@ void osmparser::startElement(xmlTextReaderPtr reader, const xmlChar *name)
{
curID = 0;
in_node = true;
int count=0;
osm_node *node=new osm_node;
xlat=xmlTextReaderGetAttribute(reader,BAD_CAST "lat");
xlon=xmlTextReaderGetAttribute(reader,BAD_CAST "lon");
@ -118,6 +115,26 @@ bool osmparser::parse(osm_dataset *ds, const char* filename)
{
components=ds;
xmlTextReaderPtr reader = xmlNewTextReaderFilename(filename);
int ret=do_parse(reader);
xmlFreeTextReader(reader);
return (ret==0) ? true:false;
}
bool osmparser::parse(osm_dataset *ds,char* data, int nbytes)
{
// from cocoasamurai.blogspot.com/2008/10/getting-some-xml-love-with-
// libxml2.html, converted from Objective-C to straight C
components=ds;
xmlTextReaderPtr reader = xmlReaderForMemory(data,nbytes,NULL,NULL,0);
int ret=do_parse(reader);
xmlFreeTextReader(reader);
return (ret==0) ? true:false;
}
int osmparser::do_parse(xmlTextReaderPtr reader)
{
int ret=-1;
if(reader!=NULL)
{
@ -127,7 +144,6 @@ bool osmparser::parse(osm_dataset *ds, const char* filename)
processNode(reader);
ret=xmlTextReaderRead(reader);
}
xmlFreeTextReader(reader);
}
return (ret==0) ? true:false;
return ret;
}

3
plugins/input/osm/osmparser.h
View file

@ -17,10 +17,13 @@ private:
static std::string error;
static std::map<long,osm_node*> tmp_node_store;
static int do_parse(xmlTextReaderPtr);
public:
static void processNode(xmlTextReaderPtr reader);
static void startElement(xmlTextReaderPtr reader, const xmlChar *name);
static void endElement(const xmlChar* name);
static bool parse(osm_dataset *ds, const char* filename);
static bool parse(osm_dataset *ds, char* data,int nbytes);
};