sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added Unicode support based on ICU

Browse source
This commit is contained in:
Artem Pavlenko 2008-02-18 21:40:34 +00:00
parent 1ea7bffd4c
commit 308e315432
20 changed files with 218 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
SConstruct
View file

@ -39,8 +39,8 @@ opts.Add(PathOption('BOOST_LIBS', 'Search path for boost library files', '/usr/'
opts.Add('BOOST_TOOLKIT','Specify boost toolkit e.g. gcc41.','',False)
opts.Add(('FREETYPE_CONFIG', 'The path to the freetype-config executable.', 'freetype-config'))
opts.Add(('XML2_CONFIG', 'The path to the xml2-config executable.', 'xml2-config'))
opts.Add(PathOption('FRIBIDI_INCLUDES', 'Search path for fribidi include files', '/usr/include'))
opts.Add(PathOption('FRIBIDI_LIBS','Search path for fribidi include files','/usr/' + LIBDIR_SCHEMA))
opts.Add(PathOption('ICU_INCLUDES', 'Search path for ICU include files', '/usr/include'))
opts.Add(PathOption('ICU_LIBS','Search path for ICU include files','/usr/' + LIBDIR_SCHEMA))
opts.Add(PathOption('PNG_INCLUDES', 'Search path for libpng include files', '/usr/include'))
opts.Add(PathOption('PNG_LIBS','Search path for libpng include files','/usr/' + LIBDIR_SCHEMA))
opts.Add(PathOption('JPEG_INCLUDES', 'Search path for libjpeg include files', '/usr/include'))
@ -58,7 +58,6 @@ opts.Add(ListOption('INPUT_PLUGINS','Input drivers to include','all',['postgis',
opts.Add(ListOption('BINDINGS','Language bindings to build','all',['python']))
opts.Add(BoolOption('DEBUG', 'Compile a debug version of mapnik', 'False'))
opts.Add('DESTDIR', 'The root directory to install into. Useful mainly for binary package building', '/')
opts.Add(BoolOption('BIDI', 'BIDI support', 'False'))
opts.Add(EnumOption('THREADING','Set threading support','multi', ['multi','single']))
opts.Add(EnumOption('XMLPARSER','Set xml parser ','tinyxml', ['tinyxml','spirit','libxml2']))
@ -124,14 +123,8 @@ for prereq in ('BOOST', 'PNG', 'JPEG', 'TIFF', 'PGSQL', 'PROJ', 'GDAL',):
uniq_add(env, 'LIBPATH', lib_path)
env.ParseConfig(env['FREETYPE_CONFIG'] + ' --libs --cflags')
if env['BIDI']:
env.Append(CXXFLAGS = '-DUSE_FRIBIDI')
if env['FRIBIDI_INCLUDES'] not in env['CPPPATH']:
env['CPPPATH'].append(env['FRIBIDI_INCLUDES'])
if env['FRIBIDI_LIBS'] not in env['LIBPATH']:
env['LIBPATH'].append(env['FRIBIDI_LIBS'])
env['LIBS'].append('fribidi')
#env.ParseConfig('pkg-config --libs --cflags cairomm-1.0')
if env['XMLPARSER'] == 'tinyxml':
env.Append(CXXFLAGS = '-DBOOST_PROPERTY_TREE_XML_PARSER_TINYXML -DTIXML_USE_STL')
@ -147,16 +140,15 @@ C_LIBSHEADERS = [
['z', 'zlib.h', True],
['jpeg', ['stdio.h', 'jpeglib.h'], True],
['proj', 'proj_api.h', True],
['iconv', 'iconv.h', False],
['pq', 'libpq-fe.h', False]
]
CXX_LIBSHEADERS = [
['icuuc','unicode/unistr.h',True],
['icudata','unicode/utypes.h' , True],
['gdal', 'gdal_priv.h',False]
]
if env['BIDI'] : C_LIBSHEADERS.append(['fribidi','fribidi/fribidi.h',True])
BOOST_LIBSHEADERS = [
# ['system', 'boost/system/system_error.hpp', True], # uncomment this on Darwin + boost_1_35
['filesystem', 'boost/filesystem/operations.hpp', True],

2
bindings/python/SConscript
View file

@ -41,6 +41,8 @@ else :
libraries.append('boost_python%s' % env['BOOST_APPEND'])
if env['PLATFORM'] == 'Darwin':
libraries.append('icuuc')
libraries.append('icudata')
if env['THREADING'] == 'multi':
libraries.append('boost_regex%s%s' % (env['BOOST_APPEND'],thread_suffix))
else :

8
bindings/python/mapnik/ogcserver/common.py
View file

@ -283,11 +283,7 @@ class WMSBaseServiceHandler(BaseServiceHandler):
m = self._buildMap(params)
im = Image(params['width'], params['height'])
render(m, im)
im = fromstring('RGBA', (params['width'], params['height']), rawdata(im))
fh = StringIO()
im.save(fh, PIL_TYPE_MAPPING[params['format']], quality=100)
fh.seek(0)
return Response(params['format'], fh.read())
return Response(params['format'], im.tostring(params['format'])
def GetFeatureInfo(self, params, querymethodname='query_point'):
m = self._buildMap(params)
@ -460,4 +456,4 @@ class XMLFeatureInfo:
self.currentfeature.append(attribute)
def __str__(self):
return '<?xml version="1.0"?>\n' + ElementTree.tostring(self.rootelement)
return '<?xml version="1.0"?>\n' + ElementTree.tostring(self.rootelement)

4
bindings/python/mapnik_feature.cpp
View file

@ -43,9 +43,9 @@ namespace boost { namespace python {
return ::PyFloat_FromDouble(val);
}
PyObject * operator() (std::wstring const& s) const
PyObject * operator() (UnicodeString const& s) const
{
return ::PyUnicode_FromWideChar(s.data(),implicit_cast<ssize_t>(s.size()));
return ::PyUnicode_FromWideChar((wchar_t*)s.getBuffer(),implicit_cast<ssize_t>(s.length()));
}
};

2
include/mapnik/expression.hpp
View file

@ -51,7 +51,7 @@ namespace mapnik {
literal(double val)
: expression<FeatureT>(),
value_(val) {}
literal(std::wstring const& val)
literal(UnicodeString const& val)
: expression<FeatureT>(),
value_(val) {}
literal(literal const& other)

21
include/mapnik/filter_parser.hpp
View file

@ -91,16 +91,17 @@ namespace mapnik
template <typename Iter>
void operator() (Iter start,Iter end) const
{
std::wstring str(start,end);
std::wstring quote = L"\\";
std::wstring::size_type idx;
idx = str.find(quote);
while (idx != string::npos)
{
str.erase(idx,1);
idx = str.find(quote);
}
exprs_.push(shared_ptr<expression<FeatureT> >(new literal<FeatureT>(str)));
//std::wstring str(start,end);
//std::wstring quote = L"\\";
//std::wstring::size_type idx;
//idx = str.find(quote);
//while (idx != string::npos)
///{
// str.erase(idx,1);
// idx = str.find(quote);
//}
UnicodeString str(start,end-start);
exprs_.push(shared_ptr<expression<FeatureT> >(new literal<FeatureT>(str)));
}
stack<shared_ptr<expression<FeatureT> > >& exprs_;
};

85
include/mapnik/font_engine_freetype.hpp
View file

@ -52,6 +52,10 @@ extern "C"
#include <map>
#include <iostream>
// icu
#include <unicode/ubidi.h>
#include <unicode/ushape.h>
namespace mapnik
{
class font_face : boost::noncopyable
@ -84,7 +88,12 @@ namespace mapnik
{
return face_;
}
unsigned get_char(unsigned c) const
{
return FT_Get_Char_Index(face_, c);
}
bool set_pixel_sizes(unsigned size)
{
if (! FT_Set_Pixel_Sizes( face_, 0, size ))
@ -311,19 +320,75 @@ namespace mapnik
{
unsigned width = 0;
unsigned height = 0;
UErrorCode err = U_ZERO_ERROR;
UnicodeString const& ustr = info.get_string();
const UChar * text = ustr.getBuffer();
UBiDi * bidi = ubidi_openSized(ustr.length(),0,&err);
std::wstring const& text = info.get_string();
for (std::wstring::const_iterator i=text.begin();i!=text.end();++i)
if (U_SUCCESS(err))
{
dimension_t char_dim = character_dimensions(*i);
info.add_info(*i, char_dim.first, char_dim.second);
width += char_dim.first;
height = char_dim.second > height ? char_dim.second : height;
ubidi_setPara(bidi,text,ustr.length(), UBIDI_DEFAULT_LTR,0,&err);
if (U_SUCCESS(err))
{
int32_t count = ubidi_countRuns(bidi,&err);
int32_t logicalStart;
int32_t length;
for (int32_t i=0; i< count;++i)
{
if (UBIDI_LTR == ubidi_getVisualRun(bidi,i,&logicalStart,&length))
{
do {
UChar ch = text[logicalStart++];
dimension_t char_dim = character_dimensions(ch);
info.add_info(ch, char_dim.first, char_dim.second);
width += char_dim.first;
height = char_dim.second > height ? char_dim.second : height;
} while (--length > 0);
}
else
{
logicalStart += length;
int32_t j=0,i=length;
UnicodeString arabic;
UChar * buf = arabic.getBuffer(length);
do {
UChar ch = text[--logicalStart];
buf[j++] = ch;
} while (--i > 0);
arabic.releaseBuffer(length);
if ( *arabic.getBuffer() >= 0x0600 && *arabic.getBuffer() <= 0x06ff)
{
UnicodeString shaped;
u_shapeArabic(arabic.getBuffer(),arabic.length(),shaped.getBuffer(arabic.length()),arabic.length(),
U_SHAPE_LETTERS_SHAPE|U_SHAPE_LENGTH_FIXED_SPACES_NEAR|
U_SHAPE_TEXT_DIRECTION_VISUAL_LTR
,&err);
shaped.releaseBuffer(arabic.length());
if (U_SUCCESS(err))
{
for (int j=0;j<shaped.length();++j)
{
dimension_t char_dim = character_dimensions(shaped[j]);
info.add_info(shaped[j], char_dim.first, char_dim.second);
width += char_dim.first;
height = char_dim.second > height ? char_dim.second : height;
}
}
}
}
}
}
ubidi_close(bidi);
}
info.set_dimensions(width, height);
}

9
include/mapnik/label_collision_detector.hpp
View file

@ -28,6 +28,7 @@
#include <mapnik/quad_tree.hpp>
// stl
#include <vector>
#include <unicode/unistr.h>
namespace mapnik
{
@ -139,10 +140,10 @@ namespace mapnik
struct label
{
label(Envelope<double> const& b) : box(b) {}
label(Envelope<double> const& b, std::wstring const& t) : box(b), text(t) {}
label(Envelope<double> const& b, UnicodeString const& t) : box(b), text(t) {}
Envelope<double> box;
std::wstring text;
UnicodeString text;
};
typedef quad_tree< label > tree_t;
@ -171,7 +172,7 @@ namespace mapnik
return true;
}
bool has_placement(Envelope<double> const& box, std::wstring const& text, double distance)
bool has_placement(Envelope<double> const& box, UnicodeString const& text, double distance)
{
Envelope<double> bigger_box(box.minx() - distance, box.miny() - distance, box.maxx() + distance, box.maxy() + distance);
@ -195,7 +196,7 @@ namespace mapnik
tree_.insert(label(box), box);
}
void insert(Envelope<double> const& box, std::wstring const& text)
void insert(Envelope<double> const& box, UnicodeString const& text)
{
tree_.insert(label(box, text), box);
}

7
include/mapnik/text_path.hpp
View file

@ -27,6 +27,7 @@
#define __TEXT_PATH_H__
#include <boost/utility.hpp>
#include <unicode/unistr.h>
namespace mapnik
{
@ -51,11 +52,11 @@ namespace mapnik
protected:
typedef boost::ptr_vector<character_info> characters_t;
characters_t characters_;
std::wstring const& text_;
UnicodeString const& text_;
double width_;
double height_;
public:
string_info(std::wstring const& text)
string_info(UnicodeString const& text)
: text_(text),
width_(0),
height_(0) {}
@ -91,7 +92,7 @@ namespace mapnik
return std::pair<double, double>(width_, height_);
}
std::wstring const& get_string() const
UnicodeString const& get_string() const
{
return text_;
}

10
include/mapnik/unicode.hpp
View file

@ -24,9 +24,8 @@
#ifndef UNICODE_HPP
#define UNICODE_HPP
extern "C" {
#include <iconv.h>
}
#include <unicode/unistr.h>
#include <unicode/ucnv.h>
#include <boost/utility.hpp>
@ -35,10 +34,11 @@ namespace mapnik {
{
public:
explicit transcoder (std::string const& encoding);
std::wstring transcode(std::string const& input) const;
UnicodeString transcode(const char* data) const;
~transcoder();
private:
iconv_t desc_;
bool ok_;
UConverter * conv_;
};
}

102
include/mapnik/value.hpp
View file

@ -33,10 +33,12 @@
#include <string>
#include <sstream>
#include <iomanip>
// uci
#include <unicode/unistr.h>
namespace mapnik {
typedef boost::variant<int,double,std::wstring> value_base;
typedef boost::variant<int,double,UnicodeString> value_base;
namespace impl {
struct equals
@ -64,8 +66,8 @@ namespace mapnik {
return lhs == rhs;
}
bool operator() (std::wstring const& lhs,
std::wstring const& rhs) const
bool operator() (UnicodeString const& lhs,
UnicodeString const& rhs) const
{
return lhs == rhs;
}
@ -96,7 +98,7 @@ namespace mapnik {
return lhs > rhs;
}
bool operator() (std::wstring const& lhs, std::wstring const& rhs) const
bool operator() (UnicodeString const& lhs, UnicodeString const& rhs) const
{
return lhs > rhs;
}
@ -127,7 +129,7 @@ namespace mapnik {
return lhs >= rhs;
}
bool operator() (std::wstring const& lhs, std::wstring const& rhs ) const
bool operator() (UnicodeString const& lhs, UnicodeString const& rhs ) const
{
return lhs >= rhs;
}
@ -158,8 +160,8 @@ namespace mapnik {
return lhs < rhs;
}
bool operator()( std::wstring const& lhs,
std::wstring const& rhs ) const
bool operator()( UnicodeString const& lhs,
UnicodeString const& rhs ) const
{
return lhs < rhs;
}
@ -191,8 +193,8 @@ namespace mapnik {
}
template <typename T>
bool operator()( std::wstring const& lhs,
std::wstring const& rhs ) const
bool operator()( UnicodeString const& lhs,
UnicodeString const& rhs ) const
{
return lhs <= rhs;
}
@ -213,8 +215,8 @@ namespace mapnik {
return lhs + rhs ;
}
value_type operator() (std::wstring const& lhs ,
std::wstring const& rhs ) const
value_type operator() (UnicodeString const& lhs ,
UnicodeString const& rhs ) const
{
return lhs + rhs;
}
@ -245,8 +247,8 @@ namespace mapnik {
return lhs - rhs ;
}
value_type operator() (std::wstring const& lhs,
std::wstring const& ) const
value_type operator() (UnicodeString const& lhs,
UnicodeString const& ) const
{
return lhs;
}
@ -277,8 +279,8 @@ namespace mapnik {
return lhs * rhs;
}
value_type operator() (std::wstring const& lhs,
std::wstring const& ) const
value_type operator() (UnicodeString const& lhs,
UnicodeString const& ) const
{
return lhs;
}
@ -310,8 +312,8 @@ namespace mapnik {
return lhs / rhs;
}
value_type operator() (std::wstring const& lhs,
std::wstring const&) const
value_type operator() (UnicodeString const& lhs,
UnicodeString const&) const
{
return lhs;
}
@ -338,27 +340,28 @@ namespace mapnik {
return ss.str();
}
// specializations
std::string operator() (std::wstring const& val) const
std::string operator() (UnicodeString const& val) const
{
std::stringstream ss;
std::wstring::const_iterator pos = val.begin();
ss << std::hex ;
for (;pos!=val.end();++pos)
{
wchar_t c = *pos;
if (c < 0x80)
{
ss << char(c);
}
else
{
ss << "\\x";
unsigned c0 = (c >> 8) & 0xff;
if (c0) ss << c0;
ss << (c & 0xff);
}
}
return ss.str();
//std::stringstream ss;
//std::wstring::const_iterator pos = val.begin();
//ss << std::hex ;
//for (;pos!=val.end();++pos)
//{
// wchar_t c = *pos;
// if (c < 0x80)
// {
// ss << char(c);
// }
// else
// {
// ss << "\\x";
// unsigned c0 = (c >> 8) & 0xff;
// if (c0) ss << c0;
// ss << (c & 0xff);
// }
//}
//return ss.str();
return "TODO";
}
std::string operator() (double val) const
@ -369,37 +372,38 @@ namespace mapnik {
}
};
struct to_unicode : public boost::static_visitor<std::wstring>
struct to_unicode : public boost::static_visitor<UnicodeString>
{
template <typename T>
std::wstring operator() (T val) const
UnicodeString operator() (T val) const
{
std::basic_ostringstream<wchar_t> out;
std::basic_ostringstream<char> out;
out << val;
return out.str();
return UnicodeString(out.str().c_str());
}
// specializations
std::wstring const& operator() (std::wstring const& val) const
UnicodeString const& operator() (UnicodeString const& val) const
{
return val;
}
std::wstring operator() (double val) const
UnicodeString operator() (double val) const
{
std::basic_ostringstream<wchar_t> out;
std::basic_ostringstream<char> out;
out << std::setprecision(16) << val;
return out.str();
return UnicodeString(out.str().c_str());
}
};
struct to_expression_string : public boost::static_visitor<std::string>
{
std::string operator() (std::wstring const& val) const
std::string operator() (UnicodeString const& val) const
{
/*
std::stringstream ss;
std::wstring::const_iterator pos = val.begin();
UnicodeString::const_iterator pos = val.begin();
ss << std::hex ;
for (;pos!=val.end();++pos)
{
@ -417,6 +421,8 @@ namespace mapnik {
}
}
return "\'" + ss.str() + "\'";
*/
return "TODO";
}
template <typename T>
@ -496,7 +502,7 @@ namespace mapnik {
return boost::apply_visitor(impl::to_string(),base_);
}
std::wstring to_unicode() const
UnicodeString to_unicode() const
{
return boost::apply_visitor(impl::to_unicode(),base_);
}

4
plugins/input/gdal/SConscript
View file

@ -34,8 +34,8 @@ gdal_src = Split(
libraries = ['gdal' ]
if env['PLATFORM'] == 'Darwin':
libraries.append('mapnik')
libraries.append('iconv')
if env['BIDI'] : libraries.append('fribidi')
libraries.append('icuuc')
libraries.append('icudata')
gdal_inputdriver = env.SharedLibrary('gdal', source=gdal_src, SHLIBPREFIX='', SHLIBSUFFIX='.input', LIBS=libraries)

4
plugins/input/postgis/SConscript
View file

@ -35,8 +35,8 @@ libraries = ['pq']
if env['PLATFORM'] == 'Darwin':
libraries.append('mapnik')
libraries.append('iconv')
if env['BIDI'] : libraries.append('fribidi')
libraries.append('icuuc')
libraries.append('icudata')
if env['THREADING'] == 'multi':
libraries.append('boost_thread%s-mt' % env['BOOST_APPEND'])

9
plugins/input/postgis/postgisfs.cpp
View file

@ -132,10 +132,11 @@ feature_ptr postgis_featureset::next()
}
else if (oid==25 || oid==1042 || oid==1043) // text or bpchar or varchar
{
std::string str(buf);
trim(str);
std::wstring wstr = tr_->transcode(str);
boost::put(*feature,name,wstr);
//std::string str(buf);
//trim(str);
//std::wstring wstr = tr_->transcode(str);
UnicodeString ustr = tr_->transcode(buf);
boost::put(*feature,name,ustr);
}
else if (oid == 1700) // numeric
{

2
plugins/input/raster/SConscript
View file

@ -35,6 +35,8 @@ raster_src = Split(
libraries = []
if env['PLATFORM'] == 'Darwin':
libraries.append('mapnik')
libraries.append('icuuc')
libraries.append('icudata')
raster_inputdriver = env.SharedLibrary('raster', source=raster_src, SHLIBPREFIX='', SHLIBSUFFIX='.input', LIBS=libraries)

7
plugins/input/shape/SConscript
View file

@ -1,4 +1,4 @@
#
# This file is part of Mapnik (c++ mapping toolkit)
#
# Copyright (C) 2006 Artem Pavlenko, Jean-Francois Doyon
@ -46,9 +46,8 @@ else:
if env['PLATFORM'] == 'Darwin':
libraries.append('mapnik')
libraries.append('iconv')
if env['BIDI'] : libraries.append('fribidi')
libraries.append('icuuc')
libraries.append('icudata')
shape_inputdriver = env.SharedLibrary('shape', SHLIBSUFFIX='.input', source=shape_src, SHLIBPREFIX='', LIBS = libraries)

7
plugins/input/shape/dbffile.cpp
View file

@ -113,8 +113,9 @@ void dbf_file::add_attribute(int col, mapnik::transcoder const& tr, Feature cons
if (col>=0 && col<num_fields_)
{
std::string name=fields_[col].name_;
std::string str=boost::trim_copy(std::string(record_+fields_[col].offset_,fields_[col].length_));
std::string str(record_+fields_[col].offset_,fields_[col].length_);
boost::trim(str);
switch (fields_[col].type_)
{
case 'C':
@ -122,7 +123,7 @@ void dbf_file::add_attribute(int col, mapnik::transcoder const& tr, Feature cons
case 'M':
case 'L':
{
f[name] = tr.transcode(str);
f[name] = tr.transcode(str.c_str());
break;
}
case 'N':

2
src/SConscript
View file

@ -78,6 +78,8 @@ source = Split(
"""
)
#source.append("cairo_renderer.cpp")
if env['XMLPARSER'] == 'tinyxml':
source += Split(
"""

4
src/agg_renderer.cpp
View file

@ -456,7 +456,7 @@ namespace mapnik
proj_transform const& prj_trans)
{
typedef coord_transform2<CoordTransform,geometry2d> path_type;
std::wstring text = feature[sym.get_name()].to_unicode();
UnicodeString text = feature[sym.get_name()].to_unicode();
boost::shared_ptr<ImageData32> const& data = sym.get_image();
if (text.length() > 0 && data)
{
@ -666,7 +666,7 @@ namespace mapnik
{
typedef coord_transform2<CoordTransform,geometry2d> path_type;
std::wstring text = feature[sym.get_name()].to_unicode();
UnicodeString text = feature[sym.get_name()].to_unicode();
if ( text.length() > 0 )
{
Color const& fill = sym.get_fill();

38
src/unicode.cpp
View file

@ -76,7 +76,7 @@ namespace mapnik {
}
#endif
/*
inline std::wstring to_unicode(std::string const& text)
{
std::wstring out;
@ -159,22 +159,39 @@ namespace mapnik {
return out;
}
*/
transcoder::transcoder (std::string const& encoding)
: ok_(false),
conv_(0)
{
#ifdef MAPNIK_DEBUG
std::cerr << "ENCODING = " << encoding << "\n";
#endif
#ifndef WORDS_BIGENDIAN
desc_ = iconv_open("UCS-4LE",encoding.c_str());
#else
desc_ = iconv_open("UCS-4BE",encoding.c_str());
#endif
//#ifndef WORDS_BIGENDIAN
// desc_ = iconv_open("UCS-4LE",encoding.c_str());
//#else
// desc_ = iconv_open("UCS-4BE",encoding.c_str());
//#endif
UErrorCode err = U_ZERO_ERROR;
conv_ = ucnv_open(encoding.c_str(),&err);
if (U_SUCCESS(err)) ok_ = true;
// TODO
}
std::wstring transcoder::transcode(std::string const& input) const
UnicodeString transcoder::transcode(const char* data) const
{
UErrorCode err = U_ZERO_ERROR;
UnicodeString ustr(data,-1,conv_,err);
if (ustr.isBogus())
{
ustr.remove();
}
return ustr;
/*
if (desc_ == iconv_t(-1)) return to_unicode(input);
size_t inleft = input.size();
std::wstring output(inleft,0);
@ -196,10 +213,13 @@ namespace mapnik {
}
#endif
return output;
*/
}
transcoder::~transcoder()
{
if (desc_ != iconv_t(-1)) iconv_close(desc_);
// if (desc_ != iconv_t(-1)) iconv_close(desc_);
if (conv_)
ucnv_close(conv_);
}
}