diff --git a/SConstruct b/SConstruct index b2ff67d63..4e794ef30 100644 --- a/SConstruct +++ b/SConstruct @@ -162,6 +162,7 @@ BOOST_LIBSHEADERS = [ # ['system', 'boost/system/system_error.hpp', True], # uncomment this on Darwin + boost_1_35 ['filesystem', 'boost/filesystem/operations.hpp', True], ['regex', 'boost/regex.hpp', True], + ['iostreams','boost/iostreams/device/mapped_file.hpp',True], ['program_options', 'boost/program_options.hpp', False] ] diff --git a/plugins/input/shape/SConscript b/plugins/input/shape/SConscript index 70c46345f..68f924d28 100644 --- a/plugins/input/shape/SConscript +++ b/plugins/input/shape/SConscript @@ -33,11 +33,17 @@ shape_src = Split( shapefile.cpp shape_index_featureset.cpp shape_io.cpp - shp_index.cpp """ ) - -libraries = [] +thread_suffix = '-mt' +if env['PLATFORM'] == 'FreeBSD': + thread_suffix = '' + +if env['THREADING'] == 'multi': + libraries = ['boost_iostreams%s%s' % (env['BOOST_APPEND'],thread_suffix) ] +else: + libraries = ['boost_iostreams%s' % (env['BOOST_APPEND']) ] + if env['PLATFORM'] == 'Darwin': libraries.append('mapnik') libraries.append('iconv') diff --git a/plugins/input/shape/dbffile.cpp b/plugins/input/shape/dbffile.cpp index 63aae938e..98e126099 100644 --- a/plugins/input/shape/dbffile.cpp +++ b/plugins/input/shape/dbffile.cpp @@ -35,13 +35,13 @@ dbf_file::dbf_file() record_length_(0), record_(0) {} -dbf_file::dbf_file(const char* file_name) +dbf_file::dbf_file(std::string const& file_name) :num_records_(0), num_fields_(0), record_length_(0), + file_(file_name), record_(0) { - file_.open(file_name); if (file_.is_open()) { read_header(); @@ -52,16 +52,6 @@ dbf_file::dbf_file(const char* file_name) dbf_file::~dbf_file() { ::operator delete(record_); - file_.close(); -} - - -bool dbf_file::open(const std::string& file_name) -{ - file_.open(file_name.c_str(),std::ios::in|std::ios::binary); - if (file_.is_open()) - read_header(); - return file_?true:false; } diff --git a/plugins/input/shape/dbffile.hpp b/plugins/input/shape/dbffile.hpp index e460ac2d1..9d7014cc4 100644 --- a/plugins/input/shape/dbffile.hpp +++ b/plugins/input/shape/dbffile.hpp @@ -23,12 +23,16 @@ #ifndef DBFFILE_HPP #define DBFFILE_HPP +#include +// boost +#include +#include +#include +// stl #include #include -#include #include -#include using mapnik::transcoder; using mapnik::Feature; @@ -44,37 +48,36 @@ struct field_descriptor int offset_; }; +using namespace boost::iostreams; + class dbf_file { -private: - - int num_records_; - int num_fields_; - int record_length_; - std::vector fields_; - std::ifstream file_; - char* record_; + private: + int num_records_; + int num_fields_; + int record_length_; + std::vector fields_; + stream file_; + char* record_; public: - dbf_file(); - dbf_file(const char* file_name); - dbf_file(const std::string& file_name); - ~dbf_file(); - bool open(const std::string& file_name); - bool is_open(); - void close(); - int num_records() const; - int num_fields() const; - field_descriptor const& descriptor(int col) const; - void move_to(int index); - std::string string_value(int col) const; - void add_attribute(int col, transcoder const& tr, Feature const& f) const throw(); -private: - dbf_file(const dbf_file&); - dbf_file& operator=(const dbf_file&); - void read_header(); - int read_short(); - int read_int(); - void skip(int bytes); + dbf_file(); + dbf_file(const std::string& file_name); + ~dbf_file(); + bool is_open(); + void close(); + int num_records() const; + int num_fields() const; + field_descriptor const& descriptor(int col) const; + void move_to(int index); + std::string string_value(int col) const; + void add_attribute(int col, transcoder const& tr, Feature const& f) const throw(); + private: + dbf_file(const dbf_file&); + dbf_file& operator=(const dbf_file&); + void read_header(); + int read_short(); + int read_int(); + void skip(int bytes); }; #endif //DBFFILE_HPP diff --git a/plugins/input/shape/shape_index_featureset.cpp b/plugins/input/shape/shape_index_featureset.cpp index 36146eea7..033706d58 100644 --- a/plugins/input/shape/shape_index_featureset.cpp +++ b/plugins/input/shape/shape_index_featureset.cpp @@ -23,8 +23,15 @@ //$Id: shape_index_featureset.cc 36 2005-04-05 14:32:18Z pavlenko $ #include +// boost +#include +#include +#include + #include "shape_index_featureset.hpp" +using namespace boost::iostreams; + template shape_index_featureset::shape_index_featureset(const filterT& filter, const std::string& shape_file, @@ -38,17 +45,17 @@ shape_index_featureset::shape_index_featureset(const filterT& filter, { shape_.shp().skip(100); - std::string indexname(shape_file + ".index"); - std::ifstream file(indexname.c_str(),std::ios::in|std::ios::binary); + stream file(shape_file + ".index"); if (file) { - shp_index::query(filter,file,ids_); - file.close(); + shp_index >::query(filter,file,ids_); + file.close(); } - -#ifdef MAPNIK_DEBUG + std::sort(ids_.begin(),ids_.end()); + +//#ifdef MAPNIK_DEBUG std::clog<< "query size=" << ids_.size() << "\n"; -#endif +//#endif itr_ = ids_.begin(); @@ -60,7 +67,7 @@ shape_index_featureset::shape_index_featureset(const filterT& filter, { if (shape_.dbf().descriptor(i).name_ == *pos) { - attr_ids_.push_back(i); + attr_ids_.insert(i); break; } } @@ -170,7 +177,7 @@ feature_ptr shape_index_featureset::next() if (attr_ids_.size()) { shape_.dbf().move_to(shape_.id_); - std::vector::const_iterator pos=attr_ids_.begin(); + std::set::const_iterator pos=attr_ids_.begin(); while (pos!=attr_ids_.end()) { try diff --git a/plugins/input/shape/shape_index_featureset.hpp b/plugins/input/shape/shape_index_featureset.hpp index e0cee5d66..b7da287c5 100644 --- a/plugins/input/shape/shape_index_featureset.hpp +++ b/plugins/input/shape/shape_index_featureset.hpp @@ -35,9 +35,9 @@ class shape_index_featureset : public Featureset int shape_type_; shape_io shape_; boost::scoped_ptr tr_; - std::set ids_; - std::set::iterator itr_; - std::vector attr_ids_; + std::vector ids_; + std::vector::iterator itr_; + std::set attr_ids_; mutable Envelope feature_ext_; mutable int total_geom_size; mutable int count_; diff --git a/plugins/input/shape/shape_io.cpp b/plugins/input/shape/shape_io.cpp index f3227cf7d..7f3f31c52 100644 --- a/plugins/input/shape/shape_io.cpp +++ b/plugins/input/shape/shape_io.cpp @@ -32,11 +32,12 @@ const std::string shape_io::DBF = ".dbf"; shape_io::shape_io(const std::string& shape_name) : type_(shape_null), + shp_(shape_name + SHP), + dbf_(shape_name + DBF), reclength_(0), id_(0) { - bool ok = (shp_.open(shape_name + SHP) && - dbf_.open(shape_name + DBF)); + bool ok = (shp_.is_open() && dbf_.is_open()); if (!ok) { throw datasource_exception("cannot read shape file"); @@ -49,7 +50,6 @@ shape_io::~shape_io() dbf_.close(); } - void shape_io::move_to (int pos) { shp_.seek(pos); @@ -57,6 +57,12 @@ void shape_io::move_to (int pos) reclength_ = shp_.read_xdr_integer(); type_ = shp_.read_ndr_integer(); + if (shp_.is_eof()) { + id_ = 0; + reclength_ = 0; + type_ = shape_null; + } + if (type_ != shape_point && type_ != shape_pointm && type_ != shape_pointz) { shp_.read_envelope(cur_extent_); diff --git a/plugins/input/shape/shape_io.hpp b/plugins/input/shape/shape_io.hpp index d293628aa..409b991bd 100644 --- a/plugins/input/shape/shape_io.hpp +++ b/plugins/input/shape/shape_io.hpp @@ -31,36 +31,36 @@ using mapnik::geometry2d; struct shape_io { - static const std::string SHP; - static const std::string SHX; - static const std::string DBF; + static const std::string SHP; + static const std::string SHX; + static const std::string DBF; + unsigned type_; + shape_file shp_; + shape_file shx_; + dbf_file dbf_; + + unsigned reclength_; + unsigned id_; + Envelope cur_extent_; - shape_file shp_; - shape_file shx_; - dbf_file dbf_; - unsigned type_; - unsigned reclength_; - unsigned id_; - Envelope cur_extent_; - -public: - enum shapeType - { - shape_null = 0, - shape_point = 1, - shape_polyline = 3, - shape_polygon = 5, - shape_multipoint = 8, - shape_pointz = 11, - shape_polylinez = 13, - shape_polygonz = 15, - shape_multipointz = 18, - shape_pointm = 21, - shape_polylinem = 23, - shape_polygonm = 25, - shape_multipointm = 28, - shape_multipatch = 31 - }; + public: + enum shapeType + { + shape_null = 0, + shape_point = 1, + shape_polyline = 3, + shape_polygon = 5, + shape_multipoint = 8, + shape_pointz = 11, + shape_polylinez = 13, + shape_polygonz = 15, + shape_multipointz = 18, + shape_pointm = 21, + shape_polylinem = 23, + shape_polygonm = 25, + shape_multipointm = 28, + shape_multipatch = 31 + }; shape_io(const std::string& shape_name); ~shape_io(); diff --git a/plugins/input/shape/shapefile.cpp b/plugins/input/shape/shapefile.cpp index e1e3ecfa6..a0838dfc7 100644 --- a/plugins/input/shape/shapefile.cpp +++ b/plugins/input/shape/shapefile.cpp @@ -25,25 +25,9 @@ shape_file::shape_file() {} shape_file::shape_file(const std::string& file_name) -{ - //file_.rdbuf()->pubsetbuf(buff_,buffer_size); - file_.open(file_name.c_str(),std::ios::in|std::ios::binary); -} - -shape_file::~shape_file() -{ - if (file_ && file_.is_open()) - file_.close(); -} - - -bool shape_file::open(const std::string& file_name) -{ - //file_.rdbuf()->pubsetbuf(buff_,buffer_size); - file_.open(file_name.c_str(),std::ios::in | std::ios::binary); - return file_?true:false; -} + : file_(file_name) {} +shape_file::~shape_file() {} bool shape_file::is_open() { diff --git a/plugins/input/shape/shapefile.hpp b/plugins/input/shape/shapefile.hpp index 8dbc9a11c..e139917d6 100644 --- a/plugins/input/shape/shapefile.hpp +++ b/plugins/input/shape/shapefile.hpp @@ -20,64 +20,66 @@ * *****************************************************************************/ -//$Id: shapefile.hh 33 2005-04-04 13:01:03Z pavlenko $ +//$Id: shapefile.hpp 33 2005-04-04 13:01:03Z pavlenko $ #ifndef SHAPEFILE_HPP #define SHAPEFILE_HPP -#include - #include +// boost +#include +#include +#include +#include using mapnik::Envelope; struct shape_record { - char* data; - size_t size; - size_t pos; - explicit shape_record(size_t size) - : data(static_cast(::operator new(sizeof(char)*size))), - size(size), - pos(0) {} - - char* rawdata() - { - return &data[0]; - } - - void skip(unsigned n) - { - pos+=n; - } - - int read_ndr_integer() - { - int val=(data[pos] & 0xff) | + const char* data; + size_t size; + mutable size_t pos; + explicit shape_record(size_t size) + : size(size), + pos(0) {} + + void set_data(const char * data_) + { + data = data_; + } + + void skip(unsigned n) + { + pos+=n; + } + + int read_ndr_integer() + { + int val=(data[pos] & 0xff) | (data[pos+1] & 0xff) << 8 | (data[pos+2] & 0xff) << 16 | (data[pos+3] & 0xff) << 24; - pos+=4; - return val; - } - - int read_xdr_integer() - { - int val=(data[pos] & 0xff) << 24 | + pos+=4; + return val; + } + + int read_xdr_integer() + { + int val=(data[pos] & 0xff) << 24 | (data[pos+1] & 0xff) << 16 | (data[pos+2] & 0xff) << 8 | (data[pos+3] & 0xff); - pos+=4; - return val; - } - - double read_double() - { - double val; + pos+=4; + return val; + } + + double read_double() + { + double val; #ifndef WORDS_BIGENDIAN - std::memcpy(&val,&data[pos],8); + std::memcpy(&val,&data[pos],8); #else - long long bits = ((long long)data[pos] & 0xff) | + long long bits = ((long long)data[pos] & 0xff) | ((long long)data[pos+1] & 0xff) << 8 | ((long long)data[pos+2] & 0xff) << 16 | ((long long)data[pos+3] & 0xff) << 24 | @@ -85,64 +87,62 @@ struct shape_record ((long long)data[pos+5] & 0xff) << 40 | ((long long)data[pos+6] & 0xff) << 48 | ((long long)data[pos+7] & 0xff) << 56 ; - std::memcpy(&val,&bits,8); + std::memcpy(&val,&bits,8); #endif - pos+=8; - return val; - } - long remains() - { - return (size-pos); - } - ~shape_record() - { - ::operator delete(data); - } + pos+=8; + return val; + } + long remains() + { + return (size-pos); + } + ~shape_record() {} + }; -class shape_file +using namespace boost::iostreams; + +class shape_file : boost::noncopyable { - std::ifstream file_; - //static const int buffer_size = 16; - //char buff_[buffer_size]; -public: - shape_file(); - shape_file(const std::string& file_name); - ~shape_file(); - bool open(const std::string& file_name); - bool is_open(); - void close(); - - inline void read_record(shape_record& rec) - { - file_.read(rec.rawdata(),rec.size); - } - - inline int read_xdr_integer() - { - char b[4]; - file_.read(b, 4); - return b[3] & 0xffu | (b[2] & 0xffu) << 8 | + stream file_; + public: + shape_file(); + shape_file(const std::string& file_name); + ~shape_file(); + bool is_open(); + void close(); + + inline void read_record(shape_record& rec) + { + rec.set_data(file_->data() + file_.tellg()); + file_.seekg(rec.size,std::ios::cur); + } + + inline int read_xdr_integer() + { + char b[4]; + file_.read(b, 4); + return b[3] & 0xffu | (b[2] & 0xffu) << 8 | (b[1] & 0xffu) << 16 | (b[0] & 0xffu) << 24; - } - - inline int read_ndr_integer() - { - char b[4]; - file_.read(b,4); - return b[0]&0xffu | (b[1]&0xffu) << 8 | + } + + inline int read_ndr_integer() + { + char b[4]; + file_.read(b,4); + return b[0]&0xffu | (b[1]&0xffu) << 8 | (b[2]&0xffu) << 16 | (b[3]&0xffu) << 24; - } - - inline double read_double() - { - double val; + } + + inline double read_double() + { + double val; #ifndef WORDS_BIGENDIAN - file_.read(reinterpret_cast(&val),8); + file_.read(reinterpret_cast(&val),8); #else - char b[8]; - file_.read(b,8); - long long bits = ((long long)b[0] & 0xff) | + char b[8]; + file_.read(b,8); + long long bits = ((long long)b[0] & 0xff) | ((long long)b[1] & 0xff) << 8 | ((long long)b[2] & 0xff) << 16 | ((long long)b[3] & 0xff) << 24 | @@ -150,52 +150,48 @@ public: ((long long)b[5] & 0xff) << 40 | ((long long)b[6] & 0xff) << 48 | ((long long)b[7] & 0xff) << 56 ; - memcpy(&val,&bits,8); + memcpy(&val,&bits,8); #endif - return val; - } - - inline void read_envelope(Envelope& envelope) - { + return val; + } + + inline void read_envelope(Envelope& envelope) + { #ifndef WORDS_BIGENDIAN - file_.read(reinterpret_cast(&envelope),sizeof(envelope)); + file_.read(reinterpret_cast(&envelope),sizeof(envelope)); #else - double minx=read_double(); - double miny=read_double(); - double maxx=read_double(); - double maxy=read_double(); - envelope.init(minx,miny,maxx,maxy); + double minx=read_double(); + double miny=read_double(); + double maxx=read_double(); + double maxy=read_double(); + envelope.init(minx,miny,maxx,maxy); #endif - } - - inline void skip(std::streampos bytes) - { - file_.seekg(bytes,std::ios::cur); - } - - inline void rewind() - { - seek(100); - } - - inline void seek(std::streampos pos) - { - file_.seekg(pos,std::ios::beg); - } - - inline std::streampos pos() - { - return file_.tellg(); - } - - inline bool is_eof() - { - return file_.eof(); - } - -private: - shape_file(const shape_file&); - shape_file& operator=(const shape_file&); + } + + inline void skip(std::streampos bytes) + { + file_.seekg(bytes,std::ios::cur); + } + + inline void rewind() + { + seek(100); + } + + inline void seek(std::streampos pos) + { + file_.seekg(pos,std::ios::beg); + } + + inline std::streampos pos() + { + return file_.tellg(); + } + + inline bool is_eof() + { + return file_.eof(); + } }; #endif //SHAPEFILE_HPP diff --git a/plugins/input/shape/shp_index.cpp b/plugins/input/shape/shp_index.cpp deleted file mode 100644 index 1d26307b1..000000000 --- a/plugins/input/shape/shp_index.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/***************************************************************************** - * - * This file is part of Mapnik (c++ mapping toolkit) - * - * Copyright (C) 2006 Artem Pavlenko - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - * - *****************************************************************************/ -#include -#include "shp_index.hpp" - -template -void shp_index::query(const filterT& filter,std::ifstream& file,std::set& pos) -{ - file.seekg(16,std::ios::beg); - query_node(filter,file,pos); -} - - -template -void shp_index::query_node(const filterT& filter,std::ifstream& file,std::set& ids) -{ - int offset=read_ndr_integer(file); - - Envelope node_ext; - read_envelope(file,node_ext); - - int num_shapes=read_ndr_integer(file); - - if (!filter.pass(node_ext)) - { - file.seekg(offset+num_shapes*4+4,std::ios::cur); - return; - } - - for (int i=0;i -int shp_index::read_ndr_integer(std::ifstream& file) -{ - char b[4]; - file.read(b,4); - return (b[0]&0xff) | (b[1]&0xff)<<8 | (b[2]&0xff)<<16 | (b[3]&0xff)<<24; -} - - -template -void shp_index::read_envelope(std::ifstream& file,Envelope& envelope) -{ - file.read(reinterpret_cast(&envelope),sizeof(envelope)); -} - -template class shp_index; -template class shp_index; diff --git a/plugins/input/shape/shp_index.hpp b/plugins/input/shape/shp_index.hpp index c4411a07b..a9dbc5c59 100644 --- a/plugins/input/shape/shp_index.hpp +++ b/plugins/input/shape/shp_index.hpp @@ -25,7 +25,7 @@ // st #include -#include +#include // mapnik #include #include @@ -33,19 +33,73 @@ using mapnik::Envelope; using mapnik::query; -template +template class shp_index { public: - static void query(const filterT& filter,std::ifstream& file,std::set& pos); + static void query(const filterT& filter, IStream& file,std::vector& pos); private: shp_index(); ~shp_index(); shp_index(const shp_index&); shp_index& operator=(const shp_index&); - static int read_ndr_integer(std::ifstream& in); - static void read_envelope(std::ifstream& in,Envelope &envelope); - static void query_node(const filterT& filter,std::ifstream& file,std::set& pos); + static int read_ndr_integer(IStream & in); + static void read_envelope(IStream & in,Envelope &envelope); + static void query_node(const filterT& filter,IStream & in,std::vector& pos); }; +template +void shp_index::query(const filterT& filter,IStream & file,std::vector& pos) +{ + file.seekg(16,std::ios::beg); + query_node(filter,file,pos); +} + +template +void shp_index::query_node(const filterT& filter,IStream & file,std::vector& ids) +{ + int offset=read_ndr_integer(file); + + Envelope node_ext; + read_envelope(file,node_ext); + + int num_shapes=read_ndr_integer(file); + + if (!filter.pass(node_ext)) + { + file.seekg(offset+num_shapes*4+4,std::ios::cur); + return; + } + + for (int i=0;i +int shp_index::read_ndr_integer(IStream & file) +{ + char b[4]; + file.read(b,4); + return (b[0]&0xff) | (b[1]&0xff)<<8 | (b[2]&0xff)<<16 | (b[3]&0xff)<<24; +} + + +template +void shp_index::read_envelope(IStream & file,Envelope& envelope) +{ + file.read(reinterpret_cast(&envelope),sizeof(envelope)); +} + + #endif //SHP_INDEX_HH diff --git a/utils/shapeindex/SConscript b/utils/shapeindex/SConscript index 30d6844fe..6e86b2cc3 100644 --- a/utils/shapeindex/SConscript +++ b/utils/shapeindex/SConscript @@ -42,9 +42,13 @@ source = Split( headers = ['#plugins/input/shape'] + env['CPPPATH'] boost_program_options = 'boost_program_options%s' % env['BOOST_APPEND'] +boost_iostreams = 'boost_iostreams%s' % env['BOOST_APPEND'] if env['THREADING'] == 'multi': boost_program_options = '%s%s' % (boost_program_options,thread_suffix) -shapeindex = env.Program('shapeindex', source, CPPPATH=headers, LIBS=boost_program_options) + boost_iostreams = '%s%s' % (boost_iostreams,thread_suffix) +libraries = [boost_program_options,boost_iostreams] + +shapeindex = env.Program('shapeindex', source, CPPPATH=headers, LIBS=libraries) env.Install(install_prefix + '/bin', shapeindex) env.Alias('install', install_prefix + '/bin') diff --git a/utils/shapeindex/shapeindex.cpp b/utils/shapeindex/shapeindex.cpp index a87e14950..17eb3b79d 100644 --- a/utils/shapeindex/shapeindex.cpp +++ b/utils/shapeindex/shapeindex.cpp @@ -31,7 +31,8 @@ #include #include #include "quadtree.hpp" -#include "shape.hpp" +#include "shapefile.hpp" +#include "shape_io.hpp" const int MAXDEPTH = 64; const int DEFAULT_DEPTH = 8; @@ -45,6 +46,7 @@ int main (int argc,char** argv) namespace po = boost::program_options; using std::string; using std::vector; + using std::clog; bool verbose=false; unsigned int depth=DEFAULT_DEPTH; @@ -112,14 +114,17 @@ int main (int argc,char** argv) while (itr != shape_files.end()) { std::clog<<"processing "<<*itr << std::endl; - shape_file shp; + //shape_file shp; std::string shapename(*itr++); - if (!shp.open(shapename+".shp")) { + shape_file shp(shapename+".shp"); + + if (!shp.is_open()) { std::clog<<"error : cannot open "<< (shapename+".shp") <<"\n"; continue; } - - shp.read_xdr_integer(); //file_code == 9994 + + int code = shp.read_xdr_integer(); //file_code == 9994 + std::clog << code << "\n"; shp.skip(5*4); int file_length=shp.read_xdr_integer();