sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

+ add direct sqlite3 support (work in progress..)

Browse source
This commit is contained in:
Artem Pavlenko 2009-02-20 15:58:33 +00:00
parent 0bcf0004be
commit 0e2a4cddc5
5 changed files with 583 additions and 224 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
utils/pgsql2sqlite/Jamroot
View file

@ -2,17 +2,20 @@
lib program_options : : <name>boost_program_options-xgcc40-mt <search>/usr/local/lib ;
lib pq : : <name>pq <search>/opt/postgresql/lib ;
lib sqlite : : <name>sqlite3 <search>/usr/local/lib ;
exe pgsql2sqlite :
main.cpp
sqlite.cpp
../../src/wkb.cpp
../../src/envelope.cpp
.//program_options
.//pq
.//sqlite
:
<include>/usr/local/include/boost-1_38
<include>/opt/mapnik/include
<include>../../plugins/input/postgis
<include>/opt/postgresql/include
#<define>MAPNIK_DEBUG
<define>MAPNIK_DEBUG
;

248
utils/pgsql2sqlite/main.cpp
View file

@ -21,11 +21,14 @@
*****************************************************************************/
//$Id$
#include "pgsql2sqlite.hpp"
#include "sqlite.hpp"
#include <mapnik/datasource.hpp>
#include <mapnik/wkb.hpp>
#include "connection_manager.hpp"
#include "cursorresultset.hpp"
//#include "cursorresultset.hpp"
// boost
#include <boost/cstdint.hpp>
#include <boost/optional.hpp>
@ -33,236 +36,33 @@
#include <boost/shared_ptr.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/format.hpp>
#include <boost/algorithm/string/replace.hpp>
//#include <boost/algorithm/string/replace.hpp>
#include <boost/program_options.hpp>
//stl
#include <iostream>
#include <fstream>
struct blob_to_hex
{
std::string operator() (const char* blob, unsigned size)
{
std::string buf;
buf.reserve(size*2);
std::ostringstream s(buf);
s.seekp(0);
char hex[3];
std::memset(hex,0,3);
for ( unsigned pos=0; pos < size; ++pos)
{
std::sprintf (hex, "%02X", int(blob[pos]) & 0xff);
s << hex;
}
return s.str();
}
};
bool valid_envelope(mapnik::Envelope<double> const& e)
{
return (e.minx() < e.maxx() && e.miny() < e.maxy()) ;
}
template <typename Connection, typename OUT>
void pgsql2sqlite(Connection conn, std::string const& table_name, OUT & out, unsigned tolerance)
{
using namespace mapnik;
boost::shared_ptr<ResultSet> rs = conn->executeQuery("select * from " + table_name + " limit 0;");
int count = rs->getNumFields();
std::ostringstream select_sql;
select_sql << "select ";
for (int i=0; i<count; ++i)
{
if (i!=0) select_sql << ",";
select_sql << "\"" << rs->getFieldName(i) << "\"";
}
select_sql << " from " << table_name ;
std::ostringstream geom_col_sql;
geom_col_sql << "select f_geometry_column,srid,type from geometry_columns ";
geom_col_sql << "where f_table_name='" << table_name << "'";
rs = conn->executeQuery(geom_col_sql.str());
int srid = -1;
std::string geom_col = "UNKNOWN";
std::string geom_type = "UNKNOWN";
if ( rs->next())
{
try
{
srid = boost::lexical_cast<int>(rs->getValue("srid"));
}
catch (boost::bad_lexical_cast &ex)
{
std::clog << ex.what() << std::endl;
}
geom_col = rs->getValue("f_geometry_column");
geom_type = rs->getValue("type");
}
// add AsBinary(<geometry_column>) modifier
std::string select_sql_str = select_sql.str();
if (tolerance > 0)
{
std::string from = "\"" + geom_col + "\"";
std::string to = (boost::format("AsBinary(Simplify(%1%,%2%)) as %1%") % geom_col % tolerance).str();
boost::algorithm::replace_all(select_sql_str,from ,to);
}
else
{
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","AsBinary(" + geom_col+") as " + geom_col);
}
std::cout << select_sql_str << "\n";
std::ostringstream cursor_sql;
std::string cursor_name("my_cursor");
cursor_sql << "DECLARE " << cursor_name << " BINARY INSENSITIVE NO SCROLL CURSOR WITH HOLD FOR " << select_sql_str << " FOR READ ONLY";
conn->execute(cursor_sql.str());
boost::shared_ptr<CursorResultSet> cursor(new CursorResultSet(conn,cursor_name,10000));
unsigned num_fields = cursor->getNumFields();
std::ostringstream create_sql;
create_sql << "create table " << table_name << "(PK_UID INTEGER PRIMARY KEY AUTOINCREMENT,";
int geometry_oid = -1;
for ( unsigned pos = 0; pos < num_fields ; ++pos)
{
if (pos > 0) create_sql << ",";
if (geom_col == cursor->getFieldName(pos))
{
geometry_oid = cursor->getTypeOID(pos);
create_sql << "'" << cursor->getFieldName(pos) << "' BLOB";
}
else
{
create_sql << "'" << cursor->getFieldName(pos) << "' TEXT";
}
}
create_sql << ");";
std::cout << "client_encoding=" << conn->client_encoding() << "\n";
std::cout << "geometry_column=" << geom_col << "(" << geom_type
<< ") srid=" << srid << " oid=" << geometry_oid << "\n";
// begin
out << "begin;\n";
out << create_sql.str() << "\n";
// spatial index sql
out << "create virtual table idx_"<< table_name << "_" << geom_col << " using rtree(pkid, xmin, xmax, ymin, ymax);\n";
blob_to_hex hex;
int pkid = 0;
while (cursor->next())
{
++pkid;
std::ostringstream insert_sql;
insert_sql << "insert into " << table_name << " values(" << pkid;
bool empty_geom = true;
for (unsigned pos=0 ; pos < num_fields; ++pos)
{
insert_sql << ",";
if (! cursor->isNull(pos))
{
int size=cursor->getFieldLength(pos);
int oid = cursor->getTypeOID(pos);
const char* buf=cursor->getValue(pos);
switch (oid)
{
case 25:
case 1042:
case 1043:
{
std::string text(buf);
boost::algorithm::replace_all(text,"'","''");
insert_sql << "'"<< text << "'";
break;
}
case 23:
insert_sql << int4net(buf);
break;
default:
{
if (oid == geometry_oid)
{
mapnik::Feature feat(pkid);
geometry_utils::from_wkb(feat,buf,size,false,wkbGeneric);
if (feat.num_geometries() > 0)
{
geometry2d const& geom=feat.get_geometry(0);
Envelope<double> bbox = geom.envelope();
if (valid_envelope(bbox))
{
out << "insert into idx_" << table_name << "_" << geom_col << " values (" ;
out << pkid << "," << bbox.minx() << "," << bbox.maxx();
out << "," << bbox.miny() << "," << bbox.maxy() << ");\n";
empty_geom = false;
}
}
insert_sql << "X'" << hex(buf,size) << "'";
}
else
{
insert_sql << "NULL";
}
break;
}
}
}
else
{
insert_sql << "NULL";
}
}
insert_sql << ");";
if (!empty_geom) out << insert_sql.str() << "\n";
if (pkid % 1000 == 0)
{
std::cout << "\r processing " << pkid << " features";
std::cout.flush();
}
if (pkid % 100000 == 0)
{
out << "commit;\n";
out << "begin;\n";
}
}
// commit
out << "commit;\n";
std::cout << "\r processed " << pkid << " features";
std::cout << "\n Done!" << std::endl;
}
int main ( int argc, char** argv)
{
namespace sqlite = mapnik::sqlite;
sqlite::database db("/tmp/testing.sqlite");
db.execute("create table test(id integer, name text)");
sqlite::prepared_statement stmt(db,"insert into test values(?,?)");
for (unsigned i=0;i<1000;++i)
{
sqlite::record_type rec;
rec.push_back(sqlite::value_type(int(i)));
rec.push_back(sqlite::value_type("testing ...."));
stmt.insert_record(rec);
}
namespace po = boost::program_options;
po::options_description desc("Postgresql/PostGIS to SQLite3 converter\n Options");
desc.add_options()
@ -322,10 +122,12 @@ int main ( int argc, char** argv)
std::string output_file = vm["file"].as<std::string>();
std::ofstream file(output_file.c_str());
if (file)
{
pgsql2sqlite(conn,table_name,file,tolerance);
mapnik::pgsql2sqlite(conn,table_name,file,tolerance);
}
file.close();
}
catch (mapnik::datasource_exception & ex)

262
utils/pgsql2sqlite/pgsql2sqlite.hpp Normal file
View file

@ -0,0 +1,262 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2009 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
//$Id$
#include <mapnik/datasource.hpp>
#include <mapnik/wkb.hpp>
#include "connection_manager.hpp"
#include "cursorresultset.hpp"
// boost
#include <boost/cstdint.hpp>
#include <boost/optional.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/format.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/program_options.hpp>
//stl
#include <iostream>
#include <fstream>
namespace mapnik {
struct blob_to_hex
{
std::string operator() (const char* blob, unsigned size)
{
std::string buf;
buf.reserve(size*2);
std::ostringstream s(buf);
s.seekp(0);
char hex[3];
std::memset(hex,0,3);
for ( unsigned pos=0; pos < size; ++pos)
{
std::sprintf (hex, "%02X", int(blob[pos]) & 0xff);
s << hex;
}
return s.str();
}
};
bool valid_envelope(mapnik::Envelope<double> const& e)
{
return (e.minx() < e.maxx() && e.miny() < e.maxy()) ;
}
template <typename Connection, typename OUT>
void pgsql2sqlite(Connection conn, std::string const& table_name, OUT & out, unsigned tolerance)
{
using namespace mapnik;
boost::shared_ptr<ResultSet> rs = conn->executeQuery("select * from " + table_name + " limit 0;");
int count = rs->getNumFields();
std::ostringstream select_sql;
select_sql << "select ";
for (int i=0; i<count; ++i)
{
if (i!=0) select_sql << ",";
select_sql << "\"" << rs->getFieldName(i) << "\"";
}
select_sql << " from " << table_name ;
std::ostringstream geom_col_sql;
geom_col_sql << "select f_geometry_column,srid,type from geometry_columns ";
geom_col_sql << "where f_table_name='" << table_name << "'";
rs = conn->executeQuery(geom_col_sql.str());
int srid = -1;
std::string geom_col = "UNKNOWN";
std::string geom_type = "UNKNOWN";
if ( rs->next())
{
try
{
srid = boost::lexical_cast<int>(rs->getValue("srid"));
}
catch (boost::bad_lexical_cast &ex)
{
std::clog << ex.what() << std::endl;
}
geom_col = rs->getValue("f_geometry_column");
geom_type = rs->getValue("type");
}
// add AsBinary(<geometry_column>) modifier
std::string select_sql_str = select_sql.str();
if (tolerance > 0)
{
std::string from = "\"" + geom_col + "\"";
std::string to = (boost::format("AsBinary(Simplify(%1%,%2%)) as %1%") % geom_col % tolerance).str();
boost::algorithm::replace_all(select_sql_str,from ,to);
}
else
{
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","AsBinary(" + geom_col+") as " + geom_col);
}
std::cout << select_sql_str << "\n";
std::ostringstream cursor_sql;
std::string cursor_name("my_cursor");
cursor_sql << "DECLARE " << cursor_name << " BINARY INSENSITIVE NO SCROLL CURSOR WITH HOLD FOR " << select_sql_str << " FOR READ ONLY";
conn->execute(cursor_sql.str());
boost::shared_ptr<CursorResultSet> cursor(new CursorResultSet(conn,cursor_name,10000));
unsigned num_fields = cursor->getNumFields();
std::ostringstream create_sql;
create_sql << "create table " << table_name << "(PK_UID INTEGER PRIMARY KEY AUTOINCREMENT,";
int geometry_oid = -1;
for ( unsigned pos = 0; pos < num_fields ; ++pos)
{
if (pos > 0) create_sql << ",";
if (geom_col == cursor->getFieldName(pos))
{
geometry_oid = cursor->getTypeOID(pos);
create_sql << "'" << cursor->getFieldName(pos) << "' BLOB";
}
else
{
create_sql << "'" << cursor->getFieldName(pos) << "' TEXT";
}
}
create_sql << ");";
std::cout << "client_encoding=" << conn->client_encoding() << "\n";
std::cout << "geometry_column=" << geom_col << "(" << geom_type
<< ") srid=" << srid << " oid=" << geometry_oid << "\n";
// begin
out << "begin;\n";
out << create_sql.str() << "\n";
// spatial index sql
out << "create virtual table idx_"<< table_name << "_" << geom_col << " using rtree(pkid, xmin, xmax, ymin, ymax);\n";
blob_to_hex hex;
int pkid = 0;
while (cursor->next())
{
++pkid;
std::ostringstream insert_sql;
insert_sql << "insert into " << table_name << " values(" << pkid;
bool empty_geom = true;
for (unsigned pos=0 ; pos < num_fields; ++pos)
{
insert_sql << ",";
if (! cursor->isNull(pos))
{
int size=cursor->getFieldLength(pos);
int oid = cursor->getTypeOID(pos);
const char* buf=cursor->getValue(pos);
switch (oid)
{
case 25:
case 1042:
case 1043:
{
std::string text(buf);
boost::algorithm::replace_all(text,"'","''");
insert_sql << "'"<< text << "'";
break;
}
case 23:
insert_sql << int4net(buf);
break;
default:
{
if (oid == geometry_oid)
{
mapnik::Feature feat(pkid);
geometry_utils::from_wkb(feat,buf,size,false,wkbGeneric);
if (feat.num_geometries() > 0)
{
geometry2d const& geom=feat.get_geometry(0);
Envelope<double> bbox = geom.envelope();
if (valid_envelope(bbox))
{
out << "insert into idx_" << table_name << "_" << geom_col << " values (" ;
out << pkid << "," << bbox.minx() << "," << bbox.maxx();
out << "," << bbox.miny() << "," << bbox.maxy() << ");\n";
empty_geom = false;
}
}
insert_sql << "X'" << hex(buf,size) << "'";
}
else
{
insert_sql << "NULL";
}
break;
}
}
}
else
{
insert_sql << "NULL";
}
}
insert_sql << ");";
if (!empty_geom) out << insert_sql.str() << "\n";
if (pkid % 1000 == 0)
{
std::cout << "\r processing " << pkid << " features";
std::cout.flush();
}
if (pkid % 100000 == 0)
{
out << "commit;\n";
out << "begin;\n";
}
}
// commit
out << "commit;\n";
std::cout << "\r processed " << pkid << " features";
std::cout << "\n Done!" << std::endl;
}
}

121
utils/pgsql2sqlite/sqlite.cpp Normal file
View file

@ -0,0 +1,121 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2009 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
//$Id$
#include "sqlite.hpp"
//#include <iostream>
namespace mapnik { namespace sqlite {
database::database(std::string const& name)
{
sqlite3 *db;
int res = sqlite3_open(name.c_str(), &db);
if (res)
{
sqlite3_close(db);
throw;
}
db_ = sqlite_db(db,database_closer());
#ifdef MAPNIK_DEBUG
std::cerr << "Open database " << name << "\n";
#endif
}
database::~database() {}
bool database::execute(std::string const& sql)
{
char * err_msg;
int res = sqlite3_exec(db_.get(),sql.c_str(),0,0,&err_msg);
if (res != SQLITE_OK)
{
std::cerr << "SQL"<< sql << " ERR:" << err_msg << "\n";
sqlite3_free(err_msg);
return false;
}
return true;
}
//char * err_msg;
// create spatial index
/*
{
std::string sql("create virtual table test using rtree(id,minx,maxx,miny,maxy);");
res = sqlite3_exec(db_.get(),sql.c_str(),0,0,&err_msg);
if (res != SQLITE_OK)
{
std::cerr << "SQL"<< sql << " ERR:" << err_msg << "\n";
sqlite3_free(err_msg);
}
}
*/
// prepare statement
//std::string sql="insert into test values(?,?,?,?,?);";
//const char * tail;
//res = sqlite3_prepare_v2(db_.get(), sql.c_str(),-1, &stmt_,&tail);
//if (res != SQLITE_OK)
//{
// std::cerr << "ERR:"<< res << "\n";
//sqlite3_close(db);
//throw;
// }
// begin transaction
//sqlite3_exec(db_.get(),"BEGIN;",0,0,&err_msg);
/*
void sqlite_loader::insert_record(int id, double minx,double maxx,double miny, double maxy)
{
if (sqlite3_bind_int(stmt_, 1, id ) != SQLITE_OK)
std::cerr << "cannot bind " << id << "\n";
if (sqlite3_bind_double(stmt_, 2,minx ) != SQLITE_OK)
std::cerr << "cannot bind " << minx << "\n";
int res = sqlite3_bind_double(stmt_, 3,maxx );
res = sqlite3_bind_double(stmt_, 4,miny );
res = sqlite3_bind_double(stmt_, 5,maxy );
sqlite3_step(stmt_);
sqlite3_reset(stmt_);
}
*/
//{
//char * err_msg;
// commit transaction
//sqlite3_exec(db_.get(),"COMMIT;",0,0,&err_msg);
//std::cerr << "finalize\n";
//int res = sqlite3_finalize(stmt_);
//if (res != SQLITE_OK)
//{
// std::cerr << "ERR:" << res << "\n";
//}
//}
}
}

171
utils/pgsql2sqlite/sqlite.hpp Normal file
View file

@ -0,0 +1,171 @@
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2009 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
//$Id$
// boost
#include <boost/shared_ptr.hpp>
#include <boost/utility.hpp>
#include <boost/variant.hpp>
//sqlite3
#include <sqlite3.h>
//stl
#ifdef MAPNIK_DEBUG
#include <iostream>
#endif
#include <string>
#include <vector>
namespace mapnik { namespace sqlite {
class database : private boost::noncopyable
{
friend class prepared_statement;
struct database_closer
{
void operator () (sqlite3 * db)
{
#ifdef MAPNIK_DEBUG
std::cerr << "close database " << db << "\n";
#endif
sqlite3_close(db);
}
};
typedef boost::shared_ptr<sqlite3> sqlite_db;
sqlite_db db_;
public:
database(std::string const& name);
~database();
bool execute(std::string const& sql);
};
typedef boost::variant<int,double,std::string> value_type;
typedef std::vector<value_type> record_type;
class prepared_statement : boost::noncopyable
{
struct binder : public boost::static_visitor<bool>
{
binder(sqlite3_stmt * stmt, unsigned index)
: stmt_(stmt), index_(index) {}
bool operator() (int val)
{
if (sqlite3_bind_int(stmt_, index_ , val ) != SQLITE_OK)
{
std::cerr << "cannot bind " << val << "\n";
return false;
}
return true;
}
bool operator() (double val)
{
if (sqlite3_bind_double(stmt_, index_ , val ) != SQLITE_OK)
{
std::cerr << "cannot bind " << val << "\n";
return false;
}
return true;
}
bool operator() (std::string const& val)
{
if (sqlite3_bind_text(stmt_, index_, val.c_str(), val.length(), 0) != SQLITE_OK)
{
std::cerr << "cannot bind " << val << "\n";
return false;
}
return true;
}
sqlite3_stmt * stmt_;
unsigned index_;
};
public:
prepared_statement(database & db, std::string const& sql)
: db_(db.db_.get()), stmt_(0)
{
const char * tail;
char * err_msg;
int res = sqlite3_prepare_v2(db_, sql.c_str(),-1, &stmt_,&tail);
if (res != SQLITE_OK)
{
std::cerr << "ERR:"<< res << "\n";
throw;
}
// begin transaction
res = sqlite3_exec(db_,"BEGIN;",0,0,&err_msg);
if (res != SQLITE_OK)
{
std::cerr << "ERR:" << err_msg << "\n";
sqlite3_free(err_msg);
}
}
~prepared_statement()
{
char * err_msg;
//commit transaction
#ifdef MAPNIK_DEBUG
std::cerr << "COMMIT\n";
#endif
sqlite3_exec(db_,"COMMIT;",0,0,&err_msg);
int res = sqlite3_finalize(stmt_);
if (res != SQLITE_OK)
{
std::cerr << "ERR:" << err_msg << "\n";
sqlite3_free(err_msg);
}
}
bool insert_record(record_type const& rec) const
{
record_type::const_iterator itr = rec.begin();
record_type::const_iterator end = rec.end();
int count = 1;
for (; itr!=end;++itr)
{
binder op(stmt_,count++);
boost::apply_visitor(op,*itr);
}
sqlite3_step(stmt_);
sqlite3_reset(stmt_);
return true;
}
private:
sqlite3 * db_;
sqlite3_stmt * stmt_;
};
//
}
}