sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
mapnik/utils/pgsql2sqlite/pgsql2sqlite.hpp

460 lines
14 KiB
C++
Raw Blame History

/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2015 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
#include "sqlite.hpp"
// mapnik
#include <mapnik/datasource.hpp>
#include <mapnik/wkb.hpp>
#include <mapnik/feature.hpp>
#include <mapnik/global.hpp>
#include <mapnik/sql_utils.hpp>
#include <mapnik/util/conversions.hpp>
#include <mapnik/geometry_is_empty.hpp>
#include <mapnik/geometry_envelope.hpp>
#include "connection_manager.hpp"
#include "cursorresultset.hpp"
#pragma GCC diagnostic push
#include <mapnik/warning_ignore.hpp>
#include <boost/algorithm/string.hpp>
#pragma GCC diagnostic pop
//st
#include <cstdint>
#include <iostream>
#include <fstream>
#include <memory>
static std::string numeric2string(const char* buf)
{
std::int16_t ndigits = int2net(buf);
std::int16_t weight = int2net(buf+2);
std::int16_t sign = int2net(buf+4);
std::int16_t dscale = int2net(buf+6);
const std::unique_ptr<std::int16_t[]> digits(new std::int16_t[ndigits]);
for (int n=0; n < ndigits ;++n)
{
digits[n] = int2net(buf+8+n*2);
}
std::ostringstream ss;
if (sign == 0x4000) ss << "-";
int i = std::max(weight,std::int16_t(0));
int d = 0;
// Each numeric "digit" is actually a value between 0000 and 9999 stored in a 16 bit field.
// For example, the number 1234567809990001 is stored as four digits: [1234] [5678] [999] [1].
// Note that the last two digits show that the leading 0's are lost when the number is split.
// We must be careful to re-insert these 0's when building the string.
while ( i >= 0)
{
if (i <= weight && d < ndigits)
{
// All digits after the first must be padded to make the field 4 characters long
if (d != 0)
{
#ifdef _WINDOWS
int dig = digits[d];
if (dig < 10)
{
ss << "000"; // 0000 - 0009
}
else if (dig < 100)
{
ss << "00"; // 0010 - 0099
}
else
{
ss << "0"; // 0100 - 0999;
}
#else
switch(digits[d])
{
case 0 ... 9:
ss << "000"; // 0000 - 0009
break;
case 10 ... 99:
ss << "00"; // 0010 - 0099
break;
case 100 ... 999:
ss << "0"; // 0100 - 0999
break;
}
#endif
}
ss << digits[d++];
}
else
{
if (d == 0)
ss << "0";
else
ss << "0000";
}
i--;
}
if (dscale > 0)
{
ss << '.';
// dscale counts the number of decimal digits following the point, not the numeric digits
while (dscale > 0)
{
int value;
if (i <= weight && d < ndigits)
value = digits[d++];
else
value = 0;
// Output up to 4 decimal digits for this value
if (dscale > 0) {
ss << (value / 1000);
value %= 1000;
dscale--;
}
if (dscale > 0) {
ss << (value / 100);
value %= 100;
dscale--;
}
if (dscale > 0) {
ss << (value / 10);
value %= 10;
dscale--;
}
if (dscale > 0) {
ss << value;
dscale--;
}
i--;
}
}
return ss.str();
}
namespace mapnik {
template <typename Connection>
void pgsql2sqlite(Connection conn,
std::string const& query,
std::string const& output_table_name,
std::string const& output_filename,
bool output_twkb)
{
namespace sqlite = mapnik::sqlite;
sqlite::database db(output_filename);
std::shared_ptr<ResultSet> rs = conn->executeQuery("select * from (" + query + ") as query limit 0;");
int count = rs->getNumFields();
std::ostringstream select_sql;
select_sql << "select ";
for (int i=0; i<count; ++i)
{
if (i!=0) select_sql << ",";
select_sql << "\"" << rs->getFieldName(i) << "\"";
}
select_sql << " from (" << query << ") as query";
std::string table_name = mapnik::sql_utils::table_from_sql(query);
std::string schema_name="";
std::string::size_type idx=table_name.find_last_of('.');
if (idx!=std::string::npos)
{
schema_name=table_name.substr(0,idx);
table_name=table_name.substr(idx+1);
}
else
{
table_name=table_name.substr(0);
}
std::ostringstream geom_col_sql;
geom_col_sql << "select f_geometry_column,srid,type from geometry_columns ";
geom_col_sql << "where f_table_name='" << table_name << "'";
if (schema_name.length() > 0)
{
geom_col_sql <<" and f_table_schema='"<< schema_name <<"'";
}
rs = conn->executeQuery(geom_col_sql.str());
int srid = -1;
std::string geom_col = "UNKNOWN";
std::string geom_type = "UNKNOWN";
if ( rs->next())
{
if (!mapnik::util::string2int(rs->getValue("srid"),srid))
{
std::clog << "could not convert srid to integer\n";
}
geom_col = rs->getValue("f_geometry_column");
geom_type = rs->getValue("type");
}
std::string select_sql_str = select_sql.str();
if (output_twkb)
{
// add AsTWKB(<geometry_column>, 2) modifier
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","ST_AsTWKB(" + geom_col+", 2) as " + geom_col);
}
else
{
// add AsBinary(<geometry_column>) modifier
boost::algorithm::replace_all(select_sql_str, "\"" + geom_col + "\"","ST_AsBinary(" + geom_col+") as " + geom_col);
}
//#ifdef MAPNIK_DEBUG
#if 1
std::cout << select_sql_str << "\n";
#endif
std::ostringstream cursor_sql;
std::string cursor_name("my_cursor");
cursor_sql << "DECLARE " << cursor_name << " BINARY INSENSITIVE NO SCROLL CURSOR WITH HOLD FOR " << select_sql_str << " FOR READ ONLY";
conn->execute(cursor_sql.str());
std::shared_ptr<CursorResultSet> cursor(new CursorResultSet(conn,cursor_name,10000));
unsigned num_fields = cursor->getNumFields();
if (num_fields == 0) return;
std::string feature_id = "fid";
std::ostringstream create_sql;
create_sql << "create table if not exists " << output_table_name << " (" << feature_id << " INTEGER PRIMARY KEY AUTOINCREMENT,";
int geometry_oid = -1;
std::string output_table_insert_sql = "insert into " + output_table_name + " values (?";
context_ptr ctx = std::make_shared<context_type>();
for ( unsigned pos = 0; pos < num_fields ; ++pos)
{
const char* field_name = cursor->getFieldName(pos);
ctx->push(field_name);
if (pos > 0)
{
create_sql << ",";
}
output_table_insert_sql +=",?";
int oid = cursor->getTypeOID(pos);
if (geom_col == cursor->getFieldName(pos))
{
geometry_oid = oid;
create_sql << "'" << cursor->getFieldName(pos) << "' BLOB";
}
else
{
create_sql << "'" << cursor->getFieldName(pos);
switch (oid)
{
case 20:
case 21:
case 23:
create_sql << "' INTEGER";
break;
case 700:
case 701:
create_sql << "' REAL";
break;
default:
create_sql << "' TEXT";
break;
}
}
}
create_sql << ");";
output_table_insert_sql +=")";
std::cout << "client_encoding=" << conn->client_encoding() << "\n";
std::cout << "geometry_column=" << geom_col << "(" << geom_type
<< ") srid=" << srid << " oid=" << geometry_oid << "\n";
db.execute("begin;");
// output table sql
db.execute(create_sql.str());
// spatial index sql
std::string spatial_index_sql = "create virtual table idx_" + output_table_name
+ "_" + geom_col + " using rtree(pkid, xmin, xmax, ymin, ymax)";
db.execute(spatial_index_sql);
//blob_to_hex hex;
int pkid = 0;
std::string spatial_index_insert_sql = "insert into idx_" + output_table_name + "_"
+ geom_col + " values (?,?,?,?,?)" ;
sqlite::prepared_statement spatial_index(db,spatial_index_insert_sql);
#ifdef MAPNIK_DEBUG
std::cout << output_table_insert_sql << "\n";
#endif
sqlite::prepared_statement output_table(db,output_table_insert_sql);
while (cursor->next())
{
++pkid;
sqlite::record_type output_rec;
output_rec.push_back(sqlite::value_type(pkid));
bool empty_geom = true;
const char * buf = 0;
for (unsigned pos=0 ; pos < num_fields; ++pos)
{
if (! cursor->isNull(pos))
{
int size=cursor->getFieldLength(pos);
int oid = cursor->getTypeOID(pos);
buf=cursor->getValue(pos);
switch (oid)
{
case 25:
case 1042:
case 1043:
{
std::string text(buf);
boost::algorithm::replace_all(text,"'","''");
output_rec.push_back(sqlite::value_type(text));
break;
}
case 23:
output_rec.push_back(sqlite::value_type(int4net(buf)));
break;
case 21:
output_rec.push_back(sqlite::value_type(int2net(buf)));
break;
case 700:
{
float val;
float4net(val,buf);
output_rec.push_back(sqlite::value_type(val));
break;
}
case 701:
{
double val;
float8net(val,buf);
output_rec.push_back(sqlite::value_type(val));
break;
}
case 1700:
{
std::string str = numeric2string(buf);
double val;
if (mapnik::util::string2double(str,val))
{
output_rec.push_back(sqlite::value_type(val));
}
break;
}
default:
{
if (oid == geometry_oid)
{
mapnik::feature_impl feat(ctx,pkid);
mapnik::geometry::geometry<double> geom;
if (output_twkb)
{
geom = geometry_utils::from_twkb(buf, size);
}
else
{
geom = geometry_utils::from_wkb(buf, size, wkbGeneric);
}
if (!mapnik::geometry::is_empty(geom))
{
box2d<double> bbox = mapnik::geometry::envelope(geom);
if (bbox.valid())
{
sqlite::record_type rec;
rec.push_back(sqlite::value_type(pkid));
rec.push_back(sqlite::value_type(bbox.minx()));
rec.push_back(sqlite::value_type(bbox.maxx()));
rec.push_back(sqlite::value_type(bbox.miny()));
rec.push_back(sqlite::value_type(bbox.maxy()));
spatial_index.insert_record(rec);
empty_geom = false;
}
}
output_rec.push_back(sqlite::blob(buf,size));
}
else
{
output_rec.push_back(sqlite::null_type());
}
break;
}
}
}
else
{
output_rec.push_back(sqlite::null_type());
}
}
if (!empty_geom) output_table.insert_record(output_rec);
if (pkid % 1000 == 0)
{
std::cout << "\r processing " << pkid << " features";
std::cout.flush();
}
if (pkid % 100000 == 0)
{
db.execute("commit;begin;");
}
}
// commit
db.execute("commit;");
std::cout << "\r processed " << pkid << " features";
db.execute("VACUUM;");
std::cout << "\r vacumming";
std::cout << "\n Done!" << std::endl;
}
}
Reference in a new issue View git blame Copy permalink