2009-02-20 16:58:33 +01:00
|
|
|
/*****************************************************************************
|
|
|
|
*
|
|
|
|
* This file is part of Mapnik (c++ mapping toolkit)
|
|
|
|
*
|
|
|
|
* Copyright (C) 2009 Artem Pavlenko
|
|
|
|
*
|
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
|
|
* License along with this library; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
*
|
|
|
|
*****************************************************************************/
|
|
|
|
//$Id$
|
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
#include "sqlite.hpp"
|
|
|
|
|
2009-02-20 16:58:33 +01:00
|
|
|
#include <mapnik/datasource.hpp>
|
|
|
|
#include <mapnik/wkb.hpp>
|
|
|
|
|
|
|
|
#include "connection_manager.hpp"
|
|
|
|
#include "cursorresultset.hpp"
|
|
|
|
// boost
|
|
|
|
#include <boost/cstdint.hpp>
|
|
|
|
#include <boost/optional.hpp>
|
|
|
|
#include <boost/scoped_ptr.hpp>
|
|
|
|
#include <boost/shared_ptr.hpp>
|
|
|
|
#include <boost/lexical_cast.hpp>
|
|
|
|
#include <boost/format.hpp>
|
2009-02-22 22:36:34 +01:00
|
|
|
#include <boost/algorithm/string.hpp>
|
2009-02-20 16:58:33 +01:00
|
|
|
#include <boost/program_options.hpp>
|
|
|
|
|
|
|
|
//stl
|
|
|
|
#include <iostream>
|
|
|
|
#include <fstream>
|
|
|
|
|
|
|
|
namespace mapnik {
|
|
|
|
|
2009-03-12 00:40:44 +01:00
|
|
|
std::string numeric2string(const char* buf)
|
|
|
|
{
|
|
|
|
int16_t ndigits = int2net(buf);
|
|
|
|
int16_t weight = int2net(buf+2);
|
|
|
|
int16_t sign = int2net(buf+4);
|
|
|
|
int16_t dscale = int2net(buf+6);
|
|
|
|
|
|
|
|
boost::scoped_array<int16_t> digits(new int16_t[ndigits]);
|
|
|
|
for (int n=0; n < ndigits ;++n)
|
|
|
|
{
|
|
|
|
digits[n] = int2net(buf+8+n*2);
|
|
|
|
}
|
|
|
|
|
|
|
|
std::ostringstream ss;
|
|
|
|
|
|
|
|
if (sign == 0x4000) ss << "-";
|
|
|
|
|
|
|
|
int i = std::max(weight,int16_t(0));
|
|
|
|
int d = 0;
|
|
|
|
while ( i >= 0)
|
|
|
|
{
|
|
|
|
if (i <= weight && d < ndigits)
|
|
|
|
ss << digits[d++];
|
|
|
|
else
|
|
|
|
ss << '0';
|
|
|
|
i--;
|
|
|
|
}
|
|
|
|
if (dscale > 0)
|
|
|
|
{
|
|
|
|
ss << '.';
|
|
|
|
while ( i >= -dscale)
|
|
|
|
{
|
|
|
|
if (i <= weight && d < ndigits)
|
|
|
|
ss << digits[d++];
|
|
|
|
i--;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ss.str();
|
|
|
|
}
|
|
|
|
|
2009-02-20 16:58:33 +01:00
|
|
|
struct blob_to_hex
|
|
|
|
{
|
|
|
|
std::string operator() (const char* blob, unsigned size)
|
|
|
|
{
|
|
|
|
std::string buf;
|
|
|
|
buf.reserve(size*2);
|
|
|
|
std::ostringstream s(buf);
|
|
|
|
s.seekp(0);
|
|
|
|
char hex[3];
|
|
|
|
std::memset(hex,0,3);
|
|
|
|
for ( unsigned pos=0; pos < size; ++pos)
|
|
|
|
{
|
|
|
|
std::sprintf (hex, "%02X", int(blob[pos]) & 0xff);
|
|
|
|
s << hex;
|
|
|
|
}
|
|
|
|
return s.str();
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
bool valid_envelope(mapnik::Envelope<double> const& e)
|
|
|
|
{
|
2009-03-04 23:27:22 +01:00
|
|
|
return (e.minx() <= e.maxx() && e.miny() <= e.maxy()) ;
|
2009-02-20 16:58:33 +01:00
|
|
|
}
|
|
|
|
|
2009-02-22 22:36:34 +01:00
|
|
|
std::string table_from_sql(std::string const& sql)
|
|
|
|
{
|
|
|
|
std::string table_name = boost::algorithm::to_lower_copy(sql);
|
|
|
|
boost::algorithm::replace_all(table_name,"\n"," ");
|
2009-02-20 16:58:33 +01:00
|
|
|
|
2009-02-22 22:36:34 +01:00
|
|
|
std::string::size_type idx = table_name.rfind("from");
|
|
|
|
if (idx!=std::string::npos)
|
|
|
|
{
|
2009-02-21 00:04:30 +01:00
|
|
|
|
2009-02-22 22:36:34 +01:00
|
|
|
idx=table_name.find_first_not_of(" ",idx+4);
|
|
|
|
if (idx != std::string::npos)
|
2009-02-21 00:04:30 +01:00
|
|
|
{
|
2009-02-22 22:36:34 +01:00
|
|
|
table_name=table_name.substr(idx);
|
2009-02-21 00:04:30 +01:00
|
|
|
}
|
2009-02-22 23:34:31 +01:00
|
|
|
idx=table_name.find_first_of(" ),");
|
2009-02-22 22:36:34 +01:00
|
|
|
if (idx != std::string::npos)
|
2009-02-21 00:04:30 +01:00
|
|
|
{
|
2009-02-22 22:36:34 +01:00
|
|
|
table_name = table_name.substr(0,idx);
|
2009-02-21 00:04:30 +01:00
|
|
|
}
|
|
|
|
}
|
2009-02-22 22:36:34 +01:00
|
|
|
return table_name;
|
2009-02-21 00:04:30 +01:00
|
|
|
}
|
2009-02-22 22:36:34 +01:00
|
|
|
|
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
template <typename Connection>
|
2009-02-22 22:36:34 +01:00
|
|
|
void pgsql2sqlite(Connection conn,
|
|
|
|
std::string const& query,
|
|
|
|
std::string const& output_table_name,
|
2009-02-22 23:34:31 +01:00
|
|
|
std::string const& output_filename)
|
2009-02-21 12:22:17 +01:00
|
|
|
{
|
2009-02-21 00:04:30 +01:00
|
|
|
namespace sqlite = mapnik::sqlite;
|
|
|
|
sqlite::database db(output_filename);
|
|
|
|
|
2009-02-22 22:36:34 +01:00
|
|
|
boost::shared_ptr<ResultSet> rs = conn->executeQuery("select * from (" + query + ") as query limit 0;");
|
2009-02-21 00:04:30 +01:00
|
|
|
int count = rs->getNumFields();
|
2009-02-22 22:36:34 +01:00
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
std::ostringstream select_sql;
|
|
|
|
|
|
|
|
select_sql << "select ";
|
2009-02-22 22:36:34 +01:00
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
for (int i=0; i<count; ++i)
|
|
|
|
{
|
|
|
|
if (i!=0) select_sql << ",";
|
|
|
|
select_sql << "\"" << rs->getFieldName(i) << "\"";
|
|
|
|
}
|
|
|
|
|
2009-02-22 22:36:34 +01:00
|
|
|
select_sql << " from (" << query << ") as query";
|
2009-02-21 00:04:30 +01:00
|
|
|
|
2009-02-22 22:36:34 +01:00
|
|
|
std::string table_name = table_from_sql(query);
|
|
|
|
|
2009-03-12 00:40:44 +01:00
|
|
|
std::string schema_name="";
|
|
|
|
std::string::size_type idx=table_name.find_last_of('.');
|
|
|
|
if (idx!=std::string::npos)
|
|
|
|
{
|
|
|
|
schema_name=table_name.substr(0,idx);
|
|
|
|
table_name=table_name.substr(idx+1);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
table_name=table_name.substr(0);
|
|
|
|
}
|
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
std::ostringstream geom_col_sql;
|
|
|
|
geom_col_sql << "select f_geometry_column,srid,type from geometry_columns ";
|
|
|
|
geom_col_sql << "where f_table_name='" << table_name << "'";
|
2009-03-12 00:40:44 +01:00
|
|
|
if (schema_name.length() > 0)
|
|
|
|
{
|
|
|
|
geom_col_sql <<" and f_table_schema='"<< schema_name <<"'";
|
|
|
|
}
|
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
rs = conn->executeQuery(geom_col_sql.str());
|
|
|
|
|
|
|
|
int srid = -1;
|
|
|
|
std::string geom_col = "UNKNOWN";
|
|
|
|
std::string geom_type = "UNKNOWN";
|
2009-02-22 22:36:34 +01:00
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
if ( rs->next())
|
|
|
|
{
|
|
|
|
try
|
|
|
|
{
|
|
|
|
srid = boost::lexical_cast<int>(rs->getValue("srid"));
|
|
|
|
}
|
|
|
|
catch (boost::bad_lexical_cast &ex)
|
|
|
|
{
|
|
|
|
std::clog << ex.what() << std::endl;
|
|
|
|
}
|
|
|
|
geom_col = rs->getValue("f_geometry_column");
|
|
|
|
geom_type = rs->getValue("type");
|
|
|
|
}
|
|
|
|
|
|
|
|
// add AsBinary(<geometry_column>) modifier
|
|
|
|
std::string select_sql_str = select_sql.str();
|
2009-02-22 23:34:31 +01:00
|
|
|
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","AsBinary(" + geom_col+") as " + geom_col);
|
2009-02-22 22:36:34 +01:00
|
|
|
|
|
|
|
#ifdef MAPNIK_DEBUG
|
2009-02-21 00:04:30 +01:00
|
|
|
std::cout << select_sql_str << "\n";
|
2009-02-22 22:36:34 +01:00
|
|
|
#endif
|
2009-02-22 23:34:31 +01:00
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
std::ostringstream cursor_sql;
|
|
|
|
std::string cursor_name("my_cursor");
|
|
|
|
|
|
|
|
cursor_sql << "DECLARE " << cursor_name << " BINARY INSENSITIVE NO SCROLL CURSOR WITH HOLD FOR " << select_sql_str << " FOR READ ONLY";
|
|
|
|
conn->execute(cursor_sql.str());
|
|
|
|
|
|
|
|
boost::shared_ptr<CursorResultSet> cursor(new CursorResultSet(conn,cursor_name,10000));
|
|
|
|
|
|
|
|
unsigned num_fields = cursor->getNumFields();
|
2009-03-12 00:40:44 +01:00
|
|
|
|
|
|
|
std::string feature_id = "fid";
|
2009-02-21 00:04:30 +01:00
|
|
|
|
|
|
|
std::ostringstream create_sql;
|
2009-03-12 00:40:44 +01:00
|
|
|
create_sql << "create table if not exists " << output_table_name << " (" << feature_id << " INTEGER PRIMARY KEY AUTOINCREMENT,";
|
2009-02-26 16:35:07 +01:00
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
int geometry_oid = -1;
|
2009-02-21 01:23:41 +01:00
|
|
|
|
2009-02-22 22:36:34 +01:00
|
|
|
std::string output_table_insert_sql = "insert into " + output_table_name + " values (?";
|
2009-02-21 01:23:41 +01:00
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
for ( unsigned pos = 0; pos < num_fields ; ++pos)
|
|
|
|
{
|
2009-02-21 01:23:41 +01:00
|
|
|
if (pos > 0)
|
|
|
|
{
|
|
|
|
create_sql << ",";
|
|
|
|
}
|
|
|
|
output_table_insert_sql +=",?";
|
2009-03-12 00:40:44 +01:00
|
|
|
int oid = cursor->getTypeOID(pos);
|
2009-02-21 01:23:41 +01:00
|
|
|
if (geom_col == cursor->getFieldName(pos))
|
|
|
|
{
|
2009-03-12 00:40:44 +01:00
|
|
|
geometry_oid = oid;
|
|
|
|
create_sql << "'" << cursor->getFieldName(pos) << "' BLOB";
|
2009-02-21 01:23:41 +01:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2009-03-12 00:40:44 +01:00
|
|
|
create_sql << "'" << cursor->getFieldName(pos);
|
|
|
|
switch (oid)
|
|
|
|
{
|
|
|
|
case 700:
|
|
|
|
case 701:
|
|
|
|
create_sql << "' REAL";
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
create_sql << "' TEXT";
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2009-02-21 01:23:41 +01:00
|
|
|
}
|
2009-02-21 00:04:30 +01:00
|
|
|
}
|
2009-03-12 00:40:44 +01:00
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
create_sql << ");";
|
2009-02-21 01:23:41 +01:00
|
|
|
output_table_insert_sql +=")";
|
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
std::cout << "client_encoding=" << conn->client_encoding() << "\n";
|
|
|
|
std::cout << "geometry_column=" << geom_col << "(" << geom_type
|
|
|
|
<< ") srid=" << srid << " oid=" << geometry_oid << "\n";
|
|
|
|
|
|
|
|
|
|
|
|
db.execute("begin;");
|
2009-02-21 01:23:41 +01:00
|
|
|
// output table sql
|
2009-02-21 00:04:30 +01:00
|
|
|
db.execute(create_sql.str());
|
2009-02-21 01:23:41 +01:00
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
// spatial index sql
|
2009-02-22 22:36:34 +01:00
|
|
|
std::string spatial_index_sql = "create virtual table idx_" + output_table_name
|
2009-02-21 01:23:41 +01:00
|
|
|
+ "_" + geom_col + " using rtree(pkid, xmin, xmax, ymin, ymax)";
|
|
|
|
|
2009-02-21 00:04:30 +01:00
|
|
|
db.execute(spatial_index_sql);
|
2009-02-20 16:58:33 +01:00
|
|
|
|
2009-02-21 12:22:17 +01:00
|
|
|
//blob_to_hex hex;
|
2009-02-21 00:04:30 +01:00
|
|
|
int pkid = 0;
|
2009-02-22 22:36:34 +01:00
|
|
|
|
|
|
|
std::string spatial_index_insert_sql = "insert into idx_" + output_table_name + "_"
|
|
|
|
+ geom_col + " values (?,?,?,?,?)" ;
|
|
|
|
|
2009-02-21 01:23:41 +01:00
|
|
|
sqlite::prepared_statement spatial_index(db,spatial_index_insert_sql);
|
2009-02-22 22:36:34 +01:00
|
|
|
|
|
|
|
#ifdef MAPNIK_DEBUG
|
2009-02-21 01:23:41 +01:00
|
|
|
std::cout << output_table_insert_sql << "\n";
|
2009-02-22 22:36:34 +01:00
|
|
|
#endif
|
|
|
|
|
2009-02-21 01:23:41 +01:00
|
|
|
sqlite::prepared_statement output_table(db,output_table_insert_sql);
|
2009-02-21 00:04:30 +01:00
|
|
|
|
|
|
|
while (cursor->next())
|
|
|
|
{
|
|
|
|
++pkid;
|
2009-02-21 12:22:17 +01:00
|
|
|
|
2009-02-21 01:23:41 +01:00
|
|
|
sqlite::record_type output_rec;
|
|
|
|
output_rec.push_back(sqlite::value_type(pkid));
|
2009-02-21 00:04:30 +01:00
|
|
|
bool empty_geom = true;
|
2009-02-21 12:22:17 +01:00
|
|
|
const char * buf = 0;
|
2009-02-20 16:58:33 +01:00
|
|
|
for (unsigned pos=0 ; pos < num_fields; ++pos)
|
|
|
|
{
|
|
|
|
if (! cursor->isNull(pos))
|
|
|
|
{
|
|
|
|
int size=cursor->getFieldLength(pos);
|
|
|
|
int oid = cursor->getTypeOID(pos);
|
2009-02-21 12:22:17 +01:00
|
|
|
buf=cursor->getValue(pos);
|
2009-02-20 16:58:33 +01:00
|
|
|
|
|
|
|
switch (oid)
|
|
|
|
{
|
|
|
|
case 25:
|
|
|
|
case 1042:
|
|
|
|
case 1043:
|
|
|
|
{
|
|
|
|
std::string text(buf);
|
|
|
|
boost::algorithm::replace_all(text,"'","''");
|
2009-02-21 01:23:41 +01:00
|
|
|
output_rec.push_back(sqlite::value_type(text));
|
2009-02-20 16:58:33 +01:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
case 23:
|
2009-03-12 00:40:44 +01:00
|
|
|
output_rec.push_back(sqlite::value_type(int4net(buf)));
|
|
|
|
break;
|
|
|
|
case 21:
|
|
|
|
output_rec.push_back(sqlite::value_type(int2net(buf)));
|
|
|
|
break;
|
|
|
|
case 700:
|
|
|
|
{
|
|
|
|
float val;
|
|
|
|
float4net(val,buf);
|
|
|
|
output_rec.push_back(sqlite::value_type(val));
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case 701:
|
|
|
|
{
|
|
|
|
double val;
|
|
|
|
float8net(val,buf);
|
|
|
|
output_rec.push_back(sqlite::value_type(val));
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case 1700:
|
|
|
|
{
|
|
|
|
std::string str = numeric2string(buf);
|
|
|
|
try
|
|
|
|
{
|
|
|
|
double val = boost::lexical_cast<double>(str);
|
|
|
|
output_rec.push_back(sqlite::value_type(val));
|
|
|
|
}
|
|
|
|
catch (boost::bad_lexical_cast & ex)
|
|
|
|
{
|
|
|
|
std::clog << ex.what() << "\n";
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2009-02-20 16:58:33 +01:00
|
|
|
default:
|
|
|
|
{
|
|
|
|
if (oid == geometry_oid)
|
|
|
|
{
|
|
|
|
mapnik::Feature feat(pkid);
|
|
|
|
geometry_utils::from_wkb(feat,buf,size,false,wkbGeneric);
|
|
|
|
if (feat.num_geometries() > 0)
|
|
|
|
{
|
|
|
|
geometry2d const& geom=feat.get_geometry(0);
|
|
|
|
Envelope<double> bbox = geom.envelope();
|
|
|
|
if (valid_envelope(bbox))
|
|
|
|
{
|
2009-02-21 01:23:41 +01:00
|
|
|
sqlite::record_type rec;
|
|
|
|
|
|
|
|
rec.push_back(sqlite::value_type(pkid));
|
|
|
|
rec.push_back(sqlite::value_type(bbox.minx()));
|
|
|
|
rec.push_back(sqlite::value_type(bbox.maxx()));
|
|
|
|
rec.push_back(sqlite::value_type(bbox.miny()));
|
|
|
|
rec.push_back(sqlite::value_type(bbox.maxy()));
|
|
|
|
|
|
|
|
spatial_index.insert_record(rec);
|
|
|
|
empty_geom = false;
|
2009-02-20 16:58:33 +01:00
|
|
|
}
|
|
|
|
}
|
2009-02-21 12:22:17 +01:00
|
|
|
|
|
|
|
//output_rec.push_back(sqlite::value_type("X'" + hex(buf,size) + "'"));
|
|
|
|
output_rec.push_back(sqlite::blob(buf,size));
|
2009-02-20 16:58:33 +01:00
|
|
|
}
|
|
|
|
else
|
2009-02-21 01:23:41 +01:00
|
|
|
{
|
|
|
|
output_rec.push_back(sqlite::null_type());
|
2009-02-20 16:58:33 +01:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2009-02-21 01:23:41 +01:00
|
|
|
output_rec.push_back(sqlite::null_type());
|
2009-02-20 16:58:33 +01:00
|
|
|
}
|
|
|
|
}
|
2009-02-21 12:22:17 +01:00
|
|
|
|
2009-02-21 01:23:41 +01:00
|
|
|
if (!empty_geom) output_table.insert_record(output_rec);
|
|
|
|
|
2009-02-20 16:58:33 +01:00
|
|
|
if (pkid % 1000 == 0)
|
|
|
|
{
|
|
|
|
std::cout << "\r processing " << pkid << " features";
|
|
|
|
std::cout.flush();
|
|
|
|
}
|
2009-02-21 01:23:41 +01:00
|
|
|
|
|
|
|
if (pkid % 100000 == 0)
|
2009-02-20 16:58:33 +01:00
|
|
|
{
|
2009-02-21 12:22:17 +01:00
|
|
|
db.execute("commit;begin;");
|
2009-02-20 16:58:33 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
// commit
|
2009-02-21 01:23:41 +01:00
|
|
|
db.execute("commit;");
|
2009-02-20 16:58:33 +01:00
|
|
|
std::cout << "\r processed " << pkid << " features";
|
|
|
|
std::cout << "\n Done!" << std::endl;
|
|
|
|
}
|
|
|
|
}
|