+ change (table,t) option to specify output table
+ add support for SQL select statement (query,q) + if output table is not specified, use table name extracted from input query
This commit is contained in:
Artem Pavlenko
parent
76a4800231
commit
4868eb805e
2 changed files with 52 additions and 213 deletions
|
|
@ -57,9 +57,11 @@ int main ( int argc, char** argv)
|
|||
("user,u",po::value<std::string>(),"Connect to the database as the specified user.")
|
||||
("dbname,d",po::value<std::string>(),"postgresql database name")
|
||||
("password,P",po::value<std::string>(),"Connect to the database with the specified password.")
|
||||
("table,t",po::value<std::string>(),"Name of the table to export")
|
||||
("query,q",po::value<std::string>(),"Name of the table/or query to pass to postmaster")
|
||||
("table,t",po::value<std::string>(),"Name of the table to create")
|
||||
("simplify,s",po::value<unsigned>(),"Use this option to reduce the complexity\nand weight of a geometry using the Douglas-Peucker algorithm.")
|
||||
("file,f",po::value<std::string>(),"Use this option to specify the name of the file to create.")
|
||||
|
||||
;
|
||||
|
||||
po::positional_options_description p;
|
||||
|
|
@ -72,7 +74,7 @@ int main ( int argc, char** argv)
|
|||
po::store(po::command_line_parser(argc,argv).options(desc).positional(p).run(),vm);
|
||||
po::notify(vm);
|
||||
|
||||
if (vm.count("help") || !vm.count("file") || !vm.count("table"))
|
||||
if (vm.count("help") || !vm.count("file") || !vm.count("query"))
|
||||
{
|
||||
std::cout << desc << "\n";
|
||||
return EXIT_SUCCESS;
|
||||
|
|
@ -102,20 +104,12 @@ int main ( int argc, char** argv)
|
|||
try
|
||||
{
|
||||
boost::shared_ptr<Connection> conn(creator());
|
||||
|
||||
std::string table_name = vm["table"].as<std::string>();
|
||||
|
||||
std::string query = vm["query"].as<std::string>();
|
||||
std::string output_table_name = vm.count("table") ? vm["table"].as<std::string>() : mapnik::table_from_sql(query);
|
||||
std::string output_file = vm["file"].as<std::string>();
|
||||
|
||||
/*
|
||||
std::ofstream file(output_file.c_str());
|
||||
if (file)
|
||||
{
|
||||
mapnik::pgsql2sqlite(conn,table_name,file,tolerance);
|
||||
}
|
||||
|
||||
file.close();
|
||||
*/
|
||||
mapnik::pgsql2sqlite(conn,table_name,output_file,tolerance);
|
||||
mapnik::pgsql2sqlite(conn,query,output_table_name,output_file,tolerance);
|
||||
}
|
||||
catch (mapnik::datasource_exception & ex)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@
|
|||
#include <boost/shared_ptr.hpp>
|
||||
#include <boost/lexical_cast.hpp>
|
||||
#include <boost/format.hpp>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
#include <boost/program_options.hpp>
|
||||
|
||||
//stl
|
||||
|
|
@ -67,221 +67,58 @@ namespace mapnik {
|
|||
{
|
||||
return (e.minx() < e.maxx() && e.miny() < e.maxy()) ;
|
||||
}
|
||||
|
||||
template <typename Connection, typename OUT>
|
||||
void pgsql_dump(Connection conn, std::string const& table_name, OUT & out, unsigned tolerance)
|
||||
|
||||
std::string table_from_sql(std::string const& sql)
|
||||
{
|
||||
boost::shared_ptr<ResultSet> rs = conn->executeQuery("select * from " + table_name + " limit 0;");
|
||||
int count = rs->getNumFields();
|
||||
|
||||
std::ostringstream select_sql;
|
||||
|
||||
select_sql << "select ";
|
||||
|
||||
for (int i=0; i<count; ++i)
|
||||
{
|
||||
if (i!=0) select_sql << ",";
|
||||
select_sql << "\"" << rs->getFieldName(i) << "\"";
|
||||
}
|
||||
|
||||
select_sql << " from " << table_name ;
|
||||
|
||||
std::ostringstream geom_col_sql;
|
||||
geom_col_sql << "select f_geometry_column,srid,type from geometry_columns ";
|
||||
geom_col_sql << "where f_table_name='" << table_name << "'";
|
||||
|
||||
rs = conn->executeQuery(geom_col_sql.str());
|
||||
|
||||
int srid = -1;
|
||||
std::string geom_col = "UNKNOWN";
|
||||
std::string geom_type = "UNKNOWN";
|
||||
|
||||
if ( rs->next())
|
||||
{
|
||||
try
|
||||
{
|
||||
srid = boost::lexical_cast<int>(rs->getValue("srid"));
|
||||
}
|
||||
catch (boost::bad_lexical_cast &ex)
|
||||
{
|
||||
std::clog << ex.what() << std::endl;
|
||||
}
|
||||
geom_col = rs->getValue("f_geometry_column");
|
||||
geom_type = rs->getValue("type");
|
||||
}
|
||||
|
||||
// add AsBinary(<geometry_column>) modifier
|
||||
std::string select_sql_str = select_sql.str();
|
||||
if (tolerance > 0)
|
||||
{
|
||||
std::string from = "\"" + geom_col + "\"";
|
||||
std::string to = (boost::format("AsBinary(Simplify(%1%,%2%)) as %1%") % geom_col % tolerance).str();
|
||||
boost::algorithm::replace_all(select_sql_str,from ,to);
|
||||
}
|
||||
else
|
||||
{
|
||||
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","AsBinary(" + geom_col+") as " + geom_col);
|
||||
}
|
||||
|
||||
std::cout << select_sql_str << "\n";
|
||||
|
||||
std::ostringstream cursor_sql;
|
||||
std::string cursor_name("my_cursor");
|
||||
|
||||
cursor_sql << "DECLARE " << cursor_name << " BINARY INSENSITIVE NO SCROLL CURSOR WITH HOLD FOR " << select_sql_str << " FOR READ ONLY";
|
||||
conn->execute(cursor_sql.str());
|
||||
|
||||
boost::shared_ptr<CursorResultSet> cursor(new CursorResultSet(conn,cursor_name,10000));
|
||||
|
||||
unsigned num_fields = cursor->getNumFields();
|
||||
|
||||
std::ostringstream create_sql;
|
||||
create_sql << "create table " << table_name << "(PK_UID INTEGER PRIMARY KEY AUTOINCREMENT,";
|
||||
|
||||
int geometry_oid = -1;
|
||||
|
||||
for ( unsigned pos = 0; pos < num_fields ; ++pos)
|
||||
{
|
||||
if (pos > 0) create_sql << ",";
|
||||
if (geom_col == cursor->getFieldName(pos))
|
||||
{
|
||||
geometry_oid = cursor->getTypeOID(pos);
|
||||
create_sql << "'" << cursor->getFieldName(pos) << "' BLOB";
|
||||
}
|
||||
else
|
||||
{
|
||||
create_sql << "'" << cursor->getFieldName(pos) << "' TEXT";
|
||||
}
|
||||
}
|
||||
|
||||
create_sql << ");";
|
||||
|
||||
|
||||
std::cout << "client_encoding=" << conn->client_encoding() << "\n";
|
||||
std::cout << "geometry_column=" << geom_col << "(" << geom_type
|
||||
<< ") srid=" << srid << " oid=" << geometry_oid << "\n";
|
||||
|
||||
|
||||
// begin
|
||||
out << "begin;\n";
|
||||
|
||||
out << create_sql.str() << "\n";
|
||||
|
||||
// spatial index sql
|
||||
out << "create virtual table idx_"<< table_name << "_" << geom_col << " using rtree(pkid, xmin, xmax, ymin, ymax);\n";
|
||||
|
||||
blob_to_hex hex;
|
||||
int pkid = 0;
|
||||
while (cursor->next())
|
||||
{
|
||||
++pkid;
|
||||
std::string table_name = boost::algorithm::to_lower_copy(sql);
|
||||
boost::algorithm::replace_all(table_name,"\n"," ");
|
||||
|
||||
std::ostringstream insert_sql;
|
||||
insert_sql << "insert into " << table_name << " values(" << pkid;
|
||||
std::string::size_type idx = table_name.rfind("from");
|
||||
if (idx!=std::string::npos)
|
||||
{
|
||||
|
||||
bool empty_geom = true;
|
||||
|
||||
for (unsigned pos=0 ; pos < num_fields; ++pos)
|
||||
idx=table_name.find_first_not_of(" ",idx+4);
|
||||
if (idx != std::string::npos)
|
||||
{
|
||||
insert_sql << ",";
|
||||
if (! cursor->isNull(pos))
|
||||
{
|
||||
int size=cursor->getFieldLength(pos);
|
||||
int oid = cursor->getTypeOID(pos);
|
||||
const char* buf=cursor->getValue(pos);
|
||||
|
||||
switch (oid)
|
||||
{
|
||||
case 25:
|
||||
case 1042:
|
||||
case 1043:
|
||||
{
|
||||
std::string text(buf);
|
||||
boost::algorithm::replace_all(text,"'","''");
|
||||
insert_sql << "'"<< text << "'";
|
||||
break;
|
||||
}
|
||||
case 23:
|
||||
insert_sql << int4net(buf);
|
||||
break;
|
||||
default:
|
||||
{
|
||||
if (oid == geometry_oid)
|
||||
{
|
||||
mapnik::Feature feat(pkid);
|
||||
geometry_utils::from_wkb(feat,buf,size,false,wkbGeneric);
|
||||
if (feat.num_geometries() > 0)
|
||||
{
|
||||
geometry2d const& geom=feat.get_geometry(0);
|
||||
Envelope<double> bbox = geom.envelope();
|
||||
if (valid_envelope(bbox))
|
||||
{
|
||||
out << "insert into idx_" << table_name << "_" << geom_col << " values (" ;
|
||||
out << pkid << "," << bbox.minx() << "," << bbox.maxx();
|
||||
out << "," << bbox.miny() << "," << bbox.maxy() << ");\n";
|
||||
empty_geom = false;
|
||||
}
|
||||
}
|
||||
|
||||
insert_sql << "X'" << hex(buf,size) << "'";
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
insert_sql << "NULL";
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
insert_sql << "NULL";
|
||||
}
|
||||
table_name=table_name.substr(idx);
|
||||
}
|
||||
insert_sql << ");";
|
||||
|
||||
if (!empty_geom) out << insert_sql.str() << "\n";
|
||||
|
||||
if (pkid % 1000 == 0)
|
||||
idx=table_name.find_first_of(" )");
|
||||
if (idx != std::string::npos)
|
||||
{
|
||||
std::cout << "\r processing " << pkid << " features";
|
||||
std::cout.flush();
|
||||
}
|
||||
if (pkid % 100000 == 0)
|
||||
{
|
||||
out << "commit;\n";
|
||||
out << "begin;\n";
|
||||
table_name = table_name.substr(0,idx);
|
||||
}
|
||||
}
|
||||
// commit
|
||||
out << "commit;\n";
|
||||
std::cout << "\r processed " << pkid << " features";
|
||||
std::cout << "\n Done!" << std::endl;
|
||||
return table_name;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
template <typename Connection>
|
||||
void pgsql2sqlite(Connection conn, std::string const& table_name, std::string const& output_filename , unsigned tolerance)
|
||||
void pgsql2sqlite(Connection conn,
|
||||
std::string const& query,
|
||||
std::string const& output_table_name,
|
||||
std::string const& output_filename ,
|
||||
unsigned tolerance)
|
||||
{
|
||||
namespace sqlite = mapnik::sqlite;
|
||||
sqlite::database db(output_filename);
|
||||
|
||||
boost::shared_ptr<ResultSet> rs = conn->executeQuery("select * from " + table_name + " limit 0;");
|
||||
boost::shared_ptr<ResultSet> rs = conn->executeQuery("select * from (" + query + ") as query limit 0;");
|
||||
int count = rs->getNumFields();
|
||||
|
||||
|
||||
std::ostringstream select_sql;
|
||||
|
||||
select_sql << "select ";
|
||||
|
||||
|
||||
for (int i=0; i<count; ++i)
|
||||
{
|
||||
if (i!=0) select_sql << ",";
|
||||
select_sql << "\"" << rs->getFieldName(i) << "\"";
|
||||
}
|
||||
|
||||
select_sql << " from " << table_name ;
|
||||
select_sql << " from (" << query << ") as query";
|
||||
|
||||
std::string table_name = table_from_sql(query);
|
||||
|
||||
std::ostringstream geom_col_sql;
|
||||
geom_col_sql << "select f_geometry_column,srid,type from geometry_columns ";
|
||||
geom_col_sql << "where f_table_name='" << table_name << "'";
|
||||
|
|
@ -291,7 +128,7 @@ namespace mapnik {
|
|||
int srid = -1;
|
||||
std::string geom_col = "UNKNOWN";
|
||||
std::string geom_type = "UNKNOWN";
|
||||
|
||||
|
||||
if ( rs->next())
|
||||
{
|
||||
try
|
||||
|
|
@ -318,9 +155,11 @@ namespace mapnik {
|
|||
{
|
||||
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","AsBinary(" + geom_col+") as " + geom_col);
|
||||
}
|
||||
|
||||
|
||||
#ifdef MAPNIK_DEBUG
|
||||
std::cout << select_sql_str << "\n";
|
||||
|
||||
#endif
|
||||
|
||||
std::ostringstream cursor_sql;
|
||||
std::string cursor_name("my_cursor");
|
||||
|
||||
|
|
@ -332,11 +171,11 @@ namespace mapnik {
|
|||
unsigned num_fields = cursor->getNumFields();
|
||||
|
||||
std::ostringstream create_sql;
|
||||
create_sql << "create table " << table_name << "(PK_UID INTEGER PRIMARY KEY AUTOINCREMENT,";
|
||||
create_sql << "create table " << output_table_name << "(PK_UID INTEGER PRIMARY KEY AUTOINCREMENT,";
|
||||
|
||||
int geometry_oid = -1;
|
||||
|
||||
std::string output_table_insert_sql = "insert into " + table_name + " values (?";
|
||||
std::string output_table_insert_sql = "insert into " + output_table_name + " values (?";
|
||||
|
||||
for ( unsigned pos = 0; pos < num_fields ; ++pos)
|
||||
{
|
||||
|
|
@ -370,17 +209,23 @@ namespace mapnik {
|
|||
db.execute(create_sql.str());
|
||||
|
||||
// spatial index sql
|
||||
std::string spatial_index_sql = "create virtual table idx_" + table_name
|
||||
std::string spatial_index_sql = "create virtual table idx_" + output_table_name
|
||||
+ "_" + geom_col + " using rtree(pkid, xmin, xmax, ymin, ymax)";
|
||||
|
||||
db.execute(spatial_index_sql);
|
||||
|
||||
//blob_to_hex hex;
|
||||
int pkid = 0;
|
||||
|
||||
std::string spatial_index_insert_sql = "insert into idx_" + table_name + "_" + geom_col + " values (?,?,?,?,?)" ;
|
||||
|
||||
std::string spatial_index_insert_sql = "insert into idx_" + output_table_name + "_"
|
||||
+ geom_col + " values (?,?,?,?,?)" ;
|
||||
|
||||
sqlite::prepared_statement spatial_index(db,spatial_index_insert_sql);
|
||||
|
||||
#ifdef MAPNIK_DEBUG
|
||||
std::cout << output_table_insert_sql << "\n";
|
||||
#endif
|
||||
|
||||
sqlite::prepared_statement output_table(db,output_table_insert_sql);
|
||||
|
||||
while (cursor->next())
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue