sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1671 from mapnik/bigint-feature-id

Browse source 
Bigint feature ids and grid_rendering buffer - closes #1662
This commit is contained in:
Dane Springmeyer 2012-12-21 19:56:49 -08:00
commit 0bc5b08470
33 changed files with 197 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bindings/python/mapnik_feature.cpp
View file

@ -212,7 +212,7 @@ void export_feature()
;
class_<Feature,boost::shared_ptr<Feature>,
boost::noncopyable>("Feature",init<context_ptr,int>("Default ctor."))
boost::noncopyable>("Feature",init<context_ptr,mapnik::value_integer>("Default ctor."))
.def("id",&Feature::id)
.def("__str__",&Feature::to_string)
.def("add_geometries_from_wkb", &feature_add_geometries_from_wkb)

8
include/mapnik/feature.hpp
View file

@ -106,7 +106,7 @@ public:
typedef std::vector<value_type> cont_type;
typedef feature_kv_iterator iterator;
feature_impl(context_ptr const& ctx, int id)
feature_impl(context_ptr const& ctx, mapnik::value_integer id)
: id_(id),
ctx_(ctx),
data_(ctx_->mapping_.size()),
@ -114,9 +114,9 @@ public:
raster_()
{}
inline int id() const { return id_;}
inline mapnik::value_integer id() const { return id_;}
inline void set_id(int id) { id_ = id;}
inline void set_id(mapnik::value_integer id) { id_ = id;}
template <typename T>
void put(context_type::key_type const& key, T const& val)
@ -304,7 +304,7 @@ public:
}
private:
int id_;
mapnik::value_integer id_;
context_ptr ctx_;
cont_type data_;
boost::ptr_vector<geometry_type> geom_cont_;

3
include/mapnik/feature_factory.hpp
View file

@ -25,6 +25,7 @@
// mapnik
#include <mapnik/feature.hpp>
#include <mapnik/value.hpp>
// boost
#include <boost/make_shared.hpp>
@ -34,7 +35,7 @@ namespace mapnik
{
struct feature_factory
{
static boost::shared_ptr<Feature> create (context_ptr const& ctx, int fid)
static boost::shared_ptr<Feature> create (context_ptr const& ctx, mapnik::value_integer fid)
{
//return boost::allocate_shared<Feature>(boost::pool_allocator<Feature>(),fid);
//return boost::allocate_shared<Feature>(boost::fast_pool_allocator<Feature>(),fid);

2
include/mapnik/grid/grid.hpp
View file

@ -257,7 +257,7 @@ public:
};
typedef MAPNIK_DECL hit_grid<int> grid;
typedef MAPNIK_DECL hit_grid<mapnik::value_integer> grid;
}
#endif //MAPNIK_GRID_HPP

20
include/mapnik/grid/grid_pixel.hpp
View file

@ -171,6 +171,11 @@ struct gray32
typedef agg::int32 value_type;
typedef agg::int64u calc_type;
typedef agg::int64 long_type;
// NOTE: don't touch this enum since enums cannot be
// 64 bit and we need to ensure that alpha = base_mask
// in grid_pixfmt.hpp#blend_hiline#l256
// otherwise code will get invoked that breaks
// with 32 bit or 64 bit ints (blender_gray::blend_pix)
enum base_scale_e
{
base_shift = 16,
@ -186,8 +191,8 @@ struct gray32
gray32() {}
//--------------------------------------------------------------------
gray32(unsigned v_, unsigned a_=base_mask) :
v(agg::int32(v_)), a(agg::int32(a_)) {}
gray32(value_type v_, unsigned a_=base_mask) :
v(v_), a(value_type(a_)) {}
//--------------------------------------------------------------------
gray32(const self_type& c, unsigned a_) :
@ -308,9 +313,14 @@ struct gray64
typedef agg::int64 value_type;
typedef agg::int64u calc_type;
typedef agg::int64 long_type;
// NOTE: don't touch this enum since enums cannot be
// 64 bit and we need to ensure that alpha = base_mask
// in grid_pixfmt.hpp#blend_hiline#l256
// otherwise code will get invoked that breaks
// with 32 bit or 64 bit ints (blender_gray::blend_pix)
enum base_scale_e
{
base_shift = 32,
base_shift = 16,
base_scale = 1 << base_shift,
base_mask = base_scale - 1
};
@ -323,8 +333,8 @@ struct gray64
gray64() {}
//--------------------------------------------------------------------
gray64(value_type v_, value_type a_=base_mask) :
v(v_), a(a_) {}
gray64(value_type v_, unsigned a_=base_mask) :
v(v_), a(value_type(a_)) {}
//--------------------------------------------------------------------
gray64(const self_type& c, unsigned a_) :

13
include/mapnik/grid/grid_pixfmt.hpp
View file

@ -258,7 +258,16 @@ public:
const color_type& c,
agg::int8u cover)
{
if (c.a)
value_type* p = (value_type*)
m_rbuf->row_ptr(x, y, len) + x * Step + Offset;
do
{
*p = c.v;
p += Step;
}
while(--len);
// We ignore alpha since grid_renderer is a binary renderer for now
/*if (c.a)
{
value_type* p = (value_type*)
m_rbuf->row_ptr(x, y, len) + x * Step + Offset;
@ -282,7 +291,7 @@ public:
}
while(--len);
}
}
}*/
}

2
include/mapnik/grid/grid_renderer_base.hpp
View file

@ -29,7 +29,7 @@
namespace mapnik {
typedef agg::renderer_base<mapnik::pixfmt_gray32> grid_renderer_base_type;
typedef agg::renderer_base<mapnik::pixfmt_gray64> grid_renderer_base_type;
}

2
include/mapnik/grid/grid_view.hpp
View file

@ -201,7 +201,7 @@ private:
feature_type const& features_;
};
typedef hit_grid_view<mapnik::ImageData<int> > grid_view;
typedef hit_grid_view<mapnik::ImageData<mapnik::value_integer> > grid_view;
}

50
plugins/input/csv/csv_datasource.cpp
View file

@ -398,7 +398,7 @@ void csv_datasource::parse_csv(T & stream,
throw mapnik::datasource_exception("CSV Plugin: could not detect column headers with the name of wkt, geojson, x/y, or latitude/longitude - this is required for reading geometry data");
}
int feature_count(0);
mapnik::value_integer feature_count(0);
bool extent_initialized = false;
std::size_t num_headers = headers_.size();
@ -652,7 +652,7 @@ void csv_datasource::parse_csv(T & stream,
}
}
// now, add attributes, skipping any WKT or JSON fiels
// now, add attributes, skipping any WKT or JSON fields
if ((has_wkt_field) && (i == wkt_idx)) continue;
if ((has_json_field) && (i == json_idx)) continue;
/* First we detect likely strings, then try parsing likely numbers,
@ -664,27 +664,34 @@ void csv_datasource::parse_csv(T & stream,
to assume are numbers)
*/
bool matched = false;
bool has_dot = value.find(".") != std::string::npos;
if (value.empty() ||
(value_length > 20) ||
(value_length > 1 && !has_dot && value[0] == '0'))
{
matched = true;
feature->put(fld_name,tr.transcode(value.c_str()));
if (feature_count == 1)
{
desc_.add_descriptor(mapnik::attribute_descriptor(fld_name,mapnik::String));
}
}
else if ((value[0] >= '0' && value[0] <= '9') || value[0] == '-')
else if ((value[0] >= '0' && value[0] <= '9') ||
value[0] == '-' ||
value[0] == '+' ||
value[0] == '.')
{
double float_val = 0.0;
std::string::const_iterator str_beg = value.begin();
std::string::const_iterator str_end = value.end();
bool r = qi::phrase_parse(str_beg,str_end,qi::double_,ascii::space,float_val);
if (r && (str_beg == str_end))
bool has_e = value.find("e") != std::string::npos;
if (has_dot || has_e)
{
if (has_dot)
double float_val = 0.0;
std::string::const_iterator str_beg = value.begin();
std::string::const_iterator str_end = value.end();
if (qi::phrase_parse(str_beg,str_end,qi::double_,ascii::space,float_val)
&& (str_beg == str_end))
{
matched = true;
feature->put(fld_name,float_val);
if (feature_count == 1)
{
@ -693,9 +700,17 @@ void csv_datasource::parse_csv(T & stream,
fld_name,mapnik::Double));
}
}
else
}
else
{
mapnik::value_integer int_val = 0;
std::string::const_iterator str_beg = value.begin();
std::string::const_iterator str_end = value.end();
if (qi::phrase_parse(str_beg,str_end,qi::long_long,ascii::space,int_val)
&& (str_beg == str_end))
{
feature->put(fld_name,static_cast<mapnik::value_integer>(float_val));
matched = true;
feature->put(fld_name,int_val);
if (feature_count == 1)
{
desc_.add_descriptor(
@ -704,19 +719,8 @@ void csv_datasource::parse_csv(T & stream,
}
}
}
else
{
// fallback to normal string
feature->put(fld_name,tr.transcode(value.c_str()));
if (feature_count == 1)
{
desc_.add_descriptor(
mapnik::attribute_descriptor(
fld_name,mapnik::String));
}
}
}
else
if (!matched)
{
// fallback to normal string
feature->put(fld_name,tr.transcode(value.c_str()));

6
plugins/input/geos/geos_featureset.cpp
View file

@ -44,14 +44,14 @@ using mapnik::feature_factory;
geos_featureset::geos_featureset(GEOSGeometry* geometry,
GEOSGeometry* extent,
int identifier,
mapnik::value_integer feature_id,
std::string const& field,
std::string const& field_name,
std::string const& encoding)
: geometry_(geometry),
tr_(new transcoder(encoding)),
extent_(extent),
identifier_(identifier),
feature_id_(feature_id),
field_(field),
field_name_(field_name),
already_rendered_(false),
@ -108,7 +108,7 @@ feature_ptr geos_featureset::next()
geos_wkb_ptr wkb(geometry_);
if (wkb.is_valid())
{
feature_ptr feature(feature_factory::create(ctx_,identifier_));
feature_ptr feature(feature_factory::create(ctx_,feature_id_));
if (geometry_utils::from_wkb(feature->paths(),
wkb.data(),

4
plugins/input/geos/geos_featureset.hpp
View file

@ -41,7 +41,7 @@ class geos_featureset : public mapnik::Featureset
public:
geos_featureset(GEOSGeometry* geometry,
GEOSGeometry* extent,
int identifier,
mapnik::value_integer feature_id,
std::string const& field,
std::string const& field_name,
std::string const& encoding);
@ -52,7 +52,7 @@ private:
GEOSGeometry* geometry_;
boost::scoped_ptr<mapnik::transcoder> tr_;
geos_feature_ptr extent_;
int identifier_;
mapnik::value_integer feature_id_;
std::string field_;
std::string field_name_;
bool already_rendered_;

2
plugins/input/kismet/kismet_featureset.hpp
View file

@ -51,7 +51,7 @@ public:
private:
std::list<kismet_network_data> const& knd_list_;
boost::scoped_ptr<mapnik::transcoder> tr_;
int feature_id_;
mapnik::value_integer feature_id_;
std::list<kismet_network_data>::const_iterator knd_list_it;
mapnik::projection source_;
mapnik::context_ptr ctx_;

2
plugins/input/occi/occi_featureset.hpp
View file

@ -70,7 +70,7 @@ private:
occi_connection_ptr conn_;
oracle::occi::ResultSet* rs_;
boost::scoped_ptr<mapnik::transcoder> tr_;
mutable int feature_id_;
mapnik::value_integer feature_id_;
mapnik::context_ptr ctx_;
bool use_wkb_;
std::vector<char> buffer_;

2
plugins/input/ogr/ogr_featureset.cpp
View file

@ -88,7 +88,7 @@ feature_ptr ogr_featureset::next()
{
// ogr feature ids start at 0, so add one to stay
// consistent with other mapnik datasources that start at 1
const int feature_id = (poFeature->GetFID() + 1);
mapnik::value_integer feature_id = (poFeature->GetFID() + 1);
feature_ptr feature(feature_factory::create(ctx_,feature_id));
OGRGeometry* geom = poFeature->GetGeometryRef();

2
plugins/input/ogr/ogr_index_featureset.cpp
View file

@ -99,7 +99,7 @@ feature_ptr ogr_index_featureset<filterT>::next()
// ogr feature ids start at 0, so add one to stay
// consistent with other mapnik datasources that start at 1
int feature_id = (poFeature->GetFID() + 1);
mapnik::value_integer feature_id = (poFeature->GetFID() + 1);
feature_ptr feature(feature_factory::create(ctx_,feature_id));
OGRGeometry* geom=poFeature->GetGeometryRef();

3
plugins/input/osm/osm.h
View file

@ -23,6 +23,7 @@
#ifndef OSM_H
#define OSM_H
#include <mapnik/value.hpp>
#include <vector>
#include <string>
#include <map>
@ -65,7 +66,7 @@ public:
struct osm_item
{
long id;
mapnik::value_integer id;
std::map<std::string, std::string> keyvals;
virtual std::string to_string();
virtual ~osm_item() {}

2
plugins/input/osm/osmparser.cpp
View file

@ -6,7 +6,7 @@
#include <cassert>
osm_item* osmparser::cur_item=NULL;
long osmparser::curID=0;
mapnik::value_integer osmparser::curID=0;
bool osmparser::in_node=false, osmparser::in_way=false;
osm_dataset* osmparser::components=NULL;
std::string osmparser::error="";

3
plugins/input/osm/osmparser.h
View file

@ -23,6 +23,7 @@
#ifndef OSMPARSER_H
#define OSMPARSER_H
#include <mapnik/value.hpp>
#include <libxml/xmlreader.h>
#include <cstdio>
#include <cstdlib>
@ -41,7 +42,7 @@ public:
private:
static osm_item *cur_item;
static long curID;
static mapnik::value_integer curID;
static bool in_node, in_way;
static osm_dataset* components;
static std::string error;

2
plugins/input/postgis/postgis_featureset.hpp
View file

@ -56,7 +56,7 @@ private:
context_ptr ctx_;
boost::scoped_ptr<mapnik::transcoder> tr_;
unsigned totalGeomSize_;
int feature_id_;
mapnik::value_integer feature_id_;
bool key_field_;
};

2
plugins/input/raster/raster_featureset.hpp
View file

@ -315,7 +315,7 @@ public:
private:
LookupPolicy policy_;
int feature_id_;
mapnik::value_integer feature_id_;
mapnik::context_ptr ctx_;
mapnik::box2d<double> extent_;
mapnik::box2d<double> bbox_;

4
plugins/input/sqlite/sqlite_featureset.cpp
View file

@ -73,7 +73,7 @@ feature_ptr sqlite_featureset::next()
return feature_ptr();
}
feature_ptr feature = feature_factory::create(ctx_,rs_->column_integer(1));
feature_ptr feature = feature_factory::create(ctx_,rs_->column_integer64(1));
if (!geometry_utils::from_wkb(feature->paths(), data, size, format_))
continue;
@ -104,7 +104,7 @@ feature_ptr sqlite_featureset::next()
{
case SQLITE_INTEGER:
{
feature->put<mapnik::value_integer>(fld_name_str, rs_->column_integer(i));
feature->put<mapnik::value_integer>(fld_name_str, rs_->column_integer64(i));
break;
}

2
plugins/input/sqlite/sqlite_featureset.hpp
View file

@ -53,7 +53,7 @@ private:
boost::shared_ptr<sqlite_resultset> rs_;
mapnik::context_ptr ctx_;
boost::scoped_ptr<mapnik::transcoder> tr_;
mapnik::box2d<double> const& bbox_;
mapnik::box2d<double> bbox_;
mapnik::wkbFormat format_;
bool spatial_index_;
bool using_subquery_;

2
plugins/input/sqlite/sqlite_utils.hpp
View file

@ -640,7 +640,7 @@ public:
found_table = true;
const char* fld_name = rs->column_text(1);
std::string fld_type(rs->column_text(2));
int fld_pk = rs->column_integer(5);
sqlite_int64 fld_pk = rs->column_integer64(5);
boost::algorithm::to_lower(fld_type);
// TODO - how to handle primary keys on multiple columns ?

2
plugins/input/templates/helloworld/hello_featureset.hpp
View file

@ -23,7 +23,7 @@ public:
private:
// members are up to you, but these are recommended
mapnik::box2d<double> const& box_;
mutable int feature_id_;
mapnik::value_integer feature_id_;
boost::scoped_ptr<mapnik::transcoder> tr_;
mapnik::context_ptr ctx_;
};

2
src/grid/grid.cpp
View file

@ -138,6 +138,6 @@ void hit_grid<T>::add_feature(mapnik::feature_impl & feature)
}
template class hit_grid<int>;
template class hit_grid<mapnik::value_integer>;
}

3
tests/data/csv/64bit_int.csv Normal file
View file

@ -0,0 +1,3 @@
x,y,bigint
0,0,2147483648
0,0,9223372036854775807
1 x y bigint
2 0 0 2147483648
3 0 0 9223372036854775807

9
tests/data/csv/number_types.csv Normal file
View file

@ -0,0 +1,9 @@
x,y,floats
0,0,.0
0,0,+.0
0,0,1e-06
0,0,-1e-06
0,0,0.000001
0,0,1.234e+16
0,0,1.234e16
0,0,"1.234e16"
1 x y floats
2 0 0 .0
3 0 0 +.0
4 0 0 1e-06
5 0 0 -1e-06
6 0 0 0.000001
7 0 0 1.234e+16
8 0 0 1.234e16
9 0 0 1.234e16

BIN
tests/data/sqlite/64bit_int.sqlite Normal file
View file

Binary file not shown.

41
tests/python_tests/csv_test.py
View file

@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
import glob
import sys
from nose.tools import *
from utilities import execution_path
@ -529,6 +530,46 @@ if 'csv' in mapnik.DatasourceCache.plugin_names():
eq_(desc['geometry_type'],mapnik.DataGeometryType.Point)
eq_(len(ds.all_features()),1)
def test_that_64bit_int_fields_work(**kwargs):
ds = get_csv_ds('64bit_int.csv')
eq_(len(ds.fields()),3)
eq_(ds.fields(),['x','y','bigint'])
eq_(ds.field_types(),['int','int','int'])
fs = ds.featureset()
feat = fs.next()
eq_(feat['bigint'],2147483648)
feat = fs.next()
eq_(feat['bigint'],sys.maxint)
eq_(feat['bigint'],9223372036854775807)
eq_(feat['bigint'],0x7FFFFFFFFFFFFFFF)
desc = ds.describe()
eq_(desc['geometry_type'],mapnik.DataGeometryType.Point)
eq_(len(ds.all_features()),2)
def test_various_number_types(**kwargs):
ds = get_csv_ds('number_types.csv')
eq_(len(ds.fields()),3)
eq_(ds.fields(),['x','y','floats'])
eq_(ds.field_types(),['int','int','float'])
fs = ds.featureset()
feat = fs.next()
eq_(feat['floats'],.0)
feat = fs.next()
eq_(feat['floats'],+.0)
feat = fs.next()
eq_(feat['floats'],1e-06)
feat = fs.next()
eq_(feat['floats'],-1e-06)
feat = fs.next()
eq_(feat['floats'],0.000001)
feat = fs.next()
eq_(feat['floats'],1.234e+16)
feat = fs.next()
eq_(feat['floats'],1.234e+16)
desc = ds.describe()
eq_(desc['geometry_type'],mapnik.DataGeometryType.Point)
eq_(len(ds.all_features()),8)
if __name__ == "__main__":
setup()
[eval(run)(visual=True) for run in dir() if 'test_' in run]

8
tests/python_tests/filter_test.py
View file

@ -239,7 +239,9 @@ null_equality = [
[.1,False,float],
[False,False,int], # TODO - should become bool
[True,False,int], # TODO - should become bool
[None,True,None]
[None,True,None],
[2147483648,False,int],
[922337203685477580,False,int]
]
def test_expressions_with_null_equality():
@ -285,7 +287,9 @@ truthyness = [
[.1,True,float],
[False,False,int], # TODO - should become bool
[True,True,int], # TODO - should become bool
[None,False,None]
[None,False,None],
[2147483648,True,int],
[922337203685477580,True,int]
]
def test_expressions_for_thruthyness():

23
tests/python_tests/postgis_test.py
View file

@ -132,6 +132,12 @@ CREATE TABLE test7(gid serial PRIMARY KEY, geom geometry);
INSERT INTO test7(gid, geom) values (1, GeomFromEWKT('SRID=4326;GEOMETRYCOLLECTION(MULTILINESTRING((10 10,20 20,10 40),(40 40,30 30,40 20,30 10)),LINESTRING EMPTY)'));
'''
insert_table_8 = '''
CREATE TABLE test8(gid serial PRIMARY KEY,int_field bigint, geom geometry);
INSERT INTO test8(gid, int_field, geom) values (1, 2147483648, ST_MakePoint(1,1));
INSERT INTO test8(gid, int_field, geom) values (2, 922337203685477580, ST_MakePoint(1,1));
'''
def postgis_setup():
call('dropdb %s' % MAPNIK_TEST_DBNAME,silent=True)
@ -146,6 +152,7 @@ def postgis_setup():
call("""psql -q %s -c '%s'""" % (MAPNIK_TEST_DBNAME,insert_table_5b),silent=False)
call('''psql -q %s -c "%s"''' % (MAPNIK_TEST_DBNAME,insert_table_6),silent=False)
call('''psql -q %s -c "%s"''' % (MAPNIK_TEST_DBNAME,insert_table_7),silent=False)
call('''psql -q %s -c "%s"''' % (MAPNIK_TEST_DBNAME,insert_table_8),silent=False)
def postgis_takedown():
pass
@ -472,7 +479,21 @@ if 'postgis' in mapnik.DatasourceCache.plugin_names() \
t.start()
t.join()
def test_that_64bit_int_fields_work():
ds = mapnik.PostGIS(dbname=MAPNIK_TEST_DBNAME,
table='test8')
eq_(len(ds.fields()),2)
eq_(ds.fields(),['gid','int_field'])
eq_(ds.field_types(),['int','int'])
fs = ds.featureset()
feat = fs.next()
eq_(feat.id(),1)
eq_(feat['gid'],1)
eq_(feat['int_field'],2147483648)
feat = fs.next()
eq_(feat.id(),2)
eq_(feat['gid'],2)
eq_(feat['int_field'],922337203685477580)
atexit.register(postgis_takedown)

16
tests/python_tests/render_grid_test.py
View file

@ -268,16 +268,24 @@ def test_32bit_int_id():
eq_(grid.get_pixel(128,128),int32)
utf1 = grid.encode('utf',resolution=4)
eq_(utf1['keys'],[str(int32)])
# this will fail because it is used internally to mark alpha
#max_neg = -(int32+1)
# so we use max neg-1
max_neg = -(int32)
grid = gen_grid_for_id(max_neg)
eq_(grid.get_pixel(128,128),max_neg)
utf1 = grid.encode('utf',resolution=4)
eq_(utf1['keys'],[str(max_neg)])
def test_64bit_int_id():
int64 = 0x7FFFFFFFFFFFFFFF
grid = gen_grid_for_id(int64)
eq_(grid.get_pixel(128,128),int64)
utf1 = grid.encode('utf',resolution=4)
eq_(utf1['keys'],[str(int64)])
max_neg = -(int64)
grid = gen_grid_for_id(max_neg)
eq_(grid.get_pixel(128,128),max_neg)
utf1 = grid.encode('utf',resolution=4)
eq_(utf1['keys'],[str(max_neg)])
def test_id_zero():
grid = gen_grid_for_id(0)
eq_(grid.get_pixel(128,128),0)

19
tests/python_tests/sqlite_test.py
View file

@ -360,6 +360,25 @@ if 'sqlite' in mapnik.DatasourceCache.plugin_names():
eq_(len(feat.geometries()),1)
eq_(feat.geometries()[0].to_wkt(),'Point(0 0)')
def test_that_64bit_int_fields_work():
ds = mapnik.SQLite(file='../data/sqlite/64bit_int.sqlite',
table='int_table',
use_spatial_index=False
)
eq_(len(ds.fields()),3)
eq_(ds.fields(),['OGC_FID','id','bigint'])
eq_(ds.field_types(),['int','int','int'])
fs = ds.featureset()
feat = fs.next()
eq_(feat.id(),1)
eq_(feat['OGC_FID'],1)
eq_(feat['bigint'],2147483648)
feat = fs.next()
eq_(feat.id(),2)
eq_(feat['OGC_FID'],2)
eq_(feat['bigint'],922337203685477580)
if __name__ == "__main__":
setup()
[eval(run)() for run in dir() if 'test_' in run]