sij/mapnik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
mapnik/test/unit/datasource/csv.cpp

1153 lines
49 KiB
C++
Raw Normal View History

move csv_test to unit/datasource
2015-10-22 19:31:57 +02:00
/*****************************************************************************
*
* This file is part of Mapnik (c++ mapping toolkit)
*
* Copyright (C) 2015 Artem Pavlenko
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*****************************************************************************/
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
#include "catch.hpp"
#include <mapnik/map.hpp>
#include <mapnik/datasource.hpp>
#include <mapnik/datasource_cache.hpp>
#include <mapnik/geometry.hpp>
#include <mapnik/geometry_types.hpp>
#include <mapnik/geometry_type.hpp>
#include <mapnik/expression.hpp>
#include <mapnik/expression_evaluator.hpp>
#include <mapnik/debug.hpp>
ensure tests pass with no plugins built
2015-07-01 02:04:48 +02:00
#include <mapnik/util/fs.hpp>
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
#include <boost/filesystem.hpp>
#include <boost/range/iterator_range_core.hpp>
#include <boost/format.hpp>
Need optional I/O header on clang++, apparently.
2015-05-18 23:34:34 +02:00
#include <boost/optional/optional_io.hpp>
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
#include <iostream>
namespace bfs = boost::filesystem;
namespace {
formatting
2015-08-19 12:04:45 +02:00
void add_csv_files(bfs::path dir, std::vector<bfs::path> &csv_files)
{
for (auto const &entry : boost::make_iterator_range(
bfs::directory_iterator(dir), bfs::directory_iterator()))
{
auto path = entry.path();
if (path.extension().native() == ".csv")
{
csv_files.emplace_back(path);
}
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
}
}
formatting
2015-08-19 12:04:45 +02:00
mapnik::datasource_ptr get_csv_ds(std::string const &file_name, bool strict = true)
{
mapnik::parameters params;
params["type"] = std::string("csv");
params["file"] = file_name;
params["strict"] = mapnik::value_bool(strict);
auto ds = mapnik::datasource_cache::instance().create(params);
// require a non-null pointer returned
REQUIRE(ds != nullptr);
return ds;
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
}
void require_field_names(std::vector<mapnik::attribute_descriptor> const &fields,
formatting
2015-08-19 12:04:45 +02:00
std::initializer_list<std::string> const &names)
{
REQUIRE(fields.size() == names.size());
auto itr_a = fields.begin();
auto const end_a = fields.end();
auto itr_b = names.begin();
for (; itr_a != end_a; ++itr_a, ++itr_b)
{
CHECK(itr_a->get_name() == *itr_b);
}
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
}
void require_field_types(std::vector<mapnik::attribute_descriptor> const &fields,
std::initializer_list<mapnik::eAttributeType> const &types) {
formatting
2015-08-19 12:04:45 +02:00
REQUIRE(fields.size() == types.size());
auto itr_a = fields.begin();
auto const end_a = fields.end();
auto itr_b = types.begin();
for (; itr_a != end_a; ++itr_a, ++itr_b) {
CHECK(itr_a->get_type() == *itr_b);
}
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
}
mapnik::featureset_ptr all_features(mapnik::datasource_ptr ds) {
formatting
2015-08-19 12:04:45 +02:00
auto fields = ds->get_descriptor().get_descriptors();
mapnik::query query(ds->envelope());
for (auto const &field : fields) {
query.add_property_name(field.get_name());
}
return ds->features(query);
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
}
std::size_t count_features(mapnik::featureset_ptr features) {
formatting
2015-08-19 12:04:45 +02:00
std::size_t count = 0;
while (features->next()) {
++count;
}
return count;
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
}
using attr = std::tuple<std::string, mapnik::value>;
void require_attributes(mapnik::feature_ptr feature,
std::initializer_list<attr> const &attrs) {
formatting
2015-08-19 12:04:45 +02:00
REQUIRE(bool(feature));
for (auto const &kv : attrs) {
REQUIRE(feature->has_key(std::get<0>(kv)));
CHECK(feature->get(std::get<0>(kv)) == std::get<1>(kv));
}
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
}
namespace detail {
struct feature_count {
formatting
2015-08-19 12:04:45 +02:00
template <typename T>
std::size_t operator()(T const &geom) const {
return mapnik::util::apply_visitor(*this, geom);
}
std::size_t operator()(mapnik::geometry::geometry_empty const &) const {
return 0;
}
template <typename T>
std::size_t operator()(mapnik::geometry::point<T> const &) const {
return 1;
}
template <typename T>
std::size_t operator()(mapnik::geometry::line_string<T> const &) const {
return 1;
}
template <typename T>
std::size_t operator()(mapnik::geometry::polygon<T> const &) const {
return 1;
}
template <typename T>
std::size_t operator()(mapnik::geometry::multi_point<T> const &mp) const {
return mp.size();
}
template <typename T>
std::size_t operator()(mapnik::geometry::multi_line_string<T> const &mls) const {
return mls.size();
}
template <typename T>
std::size_t operator()(mapnik::geometry::multi_polygon<T> const &mp) const {
return mp.size();
}
template <typename T>
std::size_t operator()(mapnik::geometry::geometry_collection<T> const &col) const {
std::size_t sum = 0;
for (auto const &geom : col) {
sum += operator()(geom);
}
return sum;
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
}
};
} // namespace detail
template <typename T>
std::size_t feature_count(mapnik::geometry::geometry<T> const &g) {
formatting
2015-08-19 12:04:45 +02:00
return detail::feature_count()(g);
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
}
void require_geometry(mapnik::feature_ptr feature,
std::size_t num_parts,
mapnik::geometry::geometry_types type) {
formatting
2015-08-19 12:04:45 +02:00
REQUIRE(bool(feature));
CHECK(mapnik::geometry::geometry_type(feature->get_geometry()) == type);
CHECK(feature_count(feature->get_geometry()) == num_parts);
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
}
add disk index tests
2015-10-23 14:11:08 +02:00
int create_disk_index(std::string const& filename, bool silent = true)
{
std::string cmd;
if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
{
cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
}
cmd += "mapnik-index " + filename;
if (silent)
{
#ifndef _WINDOWS
cmd += " 2>/dev/null";
#else
cmd += " 2> nul";
#endif
}
return std::system(cmd.c_str());
}
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
} // anonymous namespace
fix odd inf loop in catch.hpp
2015-07-01 03:47:36 +02:00
static const std::string csv_plugin("./plugins/input/csv.input");
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
fix odd inf loop in catch.hpp
2015-07-01 03:47:36 +02:00
const bool registered = mapnik::datasource_cache::instance().register_datasources(csv_plugin);
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
TEST_CASE("csv") {
formatting
2015-08-19 12:04:45 +02:00
if (mapnik::util::exists(csv_plugin))
{
REQUIRE(registered);
// make the tests silent since we intentially test error conditions that are noisy
auto const severity = mapnik::logger::instance().get_severity();
mapnik::logger::instance().set_severity(mapnik::logger::none);
// check the CSV datasource is loaded
const std::vector<std::string> plugin_names =
mapnik::datasource_cache::instance().plugin_names();
const bool have_csv_plugin =
std::find(plugin_names.begin(), plugin_names.end(), "csv") != plugin_names.end();
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("broken files")
{
for (auto create_index : { false, true })
{
if (have_csv_plugin)
formatting
2015-08-19 12:04:45 +02:00
{
add disk index tests
2015-10-23 14:11:08 +02:00
std::vector<bfs::path> broken;
add_csv_files("test/data/csv/fails", broken);
add_csv_files("test/data/csv/warns", broken);
broken.emplace_back("test/data/csv/fails/does_not_exist.csv");
for (auto const& path : broken)
{
bool require_fail = true;
if (create_index)
{
int ret = create_disk_index(path.native());
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
require_fail = (path.native() == "test/data/csv/warns/feature_id_counting.csv") ? false : true;
if (!require_fail)
{
REQUIRE(mapnik::util::exists(path.native() + ".index"));
}
}
INFO(path);
if (require_fail)
{
REQUIRE_THROWS(get_csv_ds(path.native()));
}
else
{
CHECK(bool(get_csv_ds(path.native())));
clean up -> remove *.index
2015-10-23 14:17:18 +02:00
}
if (mapnik::util::exists(path.native() + ".index"))
{
CHECK(mapnik::util::remove(path.native() + ".index"));
add disk index tests
2015-10-23 14:11:08 +02:00
}
}
formatting
2015-08-19 12:04:45 +02:00
}
}
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("good files")
{
if (have_csv_plugin)
{
formatting
2015-08-19 12:04:45 +02:00
std::vector<bfs::path> good;
add_csv_files("test/data/csv", good);
add_csv_files("test/data/csv/warns", good);
for (auto const& path : good)
{
add disk index tests
2015-10-23 14:11:08 +02:00
// cleanup in the case of a failed previous run
if (mapnik::util::exists(path.native() + ".index"))
{
boost::filesystem::remove(path.native() + ".index");
}
for (auto create_index : { false, true })
{
if (create_index)
{
int ret = create_disk_index(path.native());
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
if (path.native() != "test/data/csv/more_headers_than_column_values.csv") // mapnik-index won't create *.index for 0 features
{
CHECK(mapnik::util::exists(path.native() + ".index"));
}
}
auto ds = get_csv_ds(path.native(), false);
// require a non-null pointer returned
REQUIRE(bool(ds));
if (mapnik::util::exists(path.native() + ".index"))
{
CHECK(mapnik::util::remove(path.native() + ".index"));
}
}
formatting
2015-08-19 12:04:45 +02:00
}
}
} // END SECTION
SECTION("lon/lat detection")
{
add disk index tests
2015-10-23 14:11:08 +02:00
for (auto create_index : { false, true })
formatting
2015-08-19 12:04:45 +02:00
{
add disk index tests
2015-10-23 14:11:08 +02:00
for (auto const& lon_name : {std::string("lon"), std::string("lng")})
{
std::string filename = (boost::format("test/data/csv/%1%_lat.csv") % lon_name).str();
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {lon_name, "lat"});
require_field_types(fields, {mapnik::Integer, mapnik::Integer});
CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
mapnik::query query(ds->envelope());
for (auto const &field : fields)
{
query.add_property_name(field.get_name());
}
auto features = ds->features(query);
auto feature = features->next();
require_attributes(feature, {
attr { lon_name, mapnik::value_integer(0) },
attr { "lat", mapnik::value_integer(0) }
});
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
}
} // END SECTION
formatting
2015-08-19 12:04:45 +02:00
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("type detection")
{
for (auto create_index : { false, true })
{
std::string filename = "test/data/csv/nypd.csv";
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
formatting
2015-08-19 12:04:45 +02:00
{
add disk index tests
2015-10-23 14:11:08 +02:00
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
formatting
2015-08-19 12:04:45 +02:00
}
add disk index tests
2015-10-23 14:11:08 +02:00
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"Precinct", "Phone", "Address", "City", "geo_longitude", "geo_latitude", "geo_accuracy"});
require_field_types(fields, {mapnik::String, mapnik::String, mapnik::String, mapnik::String, mapnik::Double, mapnik::Double, mapnik::String});
CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
CHECK(count_features(all_features(ds)) == 2);
formatting
2015-08-19 12:04:45 +02:00
add disk index tests
2015-10-23 14:11:08 +02:00
auto feature = all_features(ds)->next();
formatting
2015-08-19 12:04:45 +02:00
require_attributes(feature, {
add disk index tests
2015-10-23 14:11:08 +02:00
attr { "City", mapnik::value_unicode_string("New York, NY") }
, attr { "geo_accuracy", mapnik::value_unicode_string("house") }
, attr { "Phone", mapnik::value_unicode_string("(212) 334-0711") }
, attr { "Address", mapnik::value_unicode_string("19 Elizabeth Street") }
, attr { "Precinct", mapnik::value_unicode_string("5th Precinct") }
, attr { "geo_longitude", mapnik::value_integer(-70) }
, attr { "geo_latitude", mapnik::value_integer(40) }
formatting
2015-08-19 12:04:45 +02:00
});
add disk index tests
2015-10-23 14:11:08 +02:00
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
formatting
2015-08-19 12:04:45 +02:00
}
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("skipping blank rows")
{
for (auto create_index : { false, true })
{
std::string filename = "test/data/csv/blank_rows.csv";
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "name"});
require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
CHECK(count_features(all_features(ds)) == 2);
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
formatting
2015-08-19 12:04:45 +02:00
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("empty rows")
{
for (auto create_index : { false, true })
{
std::string filename = "test/data/csv/empty_rows.csv";
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
formatting
2015-08-19 12:04:45 +02:00
add disk index tests
2015-10-23 14:11:08 +02:00
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "text", "date", "integer", "boolean", "float", "time", "datetime", "empty_column"});
require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::String,
mapnik::Integer, mapnik::Boolean, mapnik::Double, mapnik::String, mapnik::String, mapnik::String});
CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
CHECK(count_features(all_features(ds)) == 4);
formatting
2015-08-19 12:04:45 +02:00
add disk index tests
2015-10-23 14:11:08 +02:00
auto featureset = all_features(ds);
auto feature = featureset->next();
require_attributes(feature, {
attr { "x", mapnik::value_integer(0) }
, attr { "empty_column", mapnik::value_unicode_string("") }
, attr { "text", mapnik::value_unicode_string("a b") }
, attr { "float", mapnik::value_double(1.0) }
, attr { "datetime", mapnik::value_unicode_string("1971-01-01T04:14:00") }
, attr { "y", mapnik::value_integer(0) }
, attr { "boolean", mapnik::value_bool(true) }
, attr { "time", mapnik::value_unicode_string("04:14:00") }
, attr { "date", mapnik::value_unicode_string("1971-01-01") }
, attr { "integer", mapnik::value_integer(40) }
});
while (bool(feature = featureset->next())) {
CHECK(feature->size() == 10);
CHECK(feature->get("empty_column") == mapnik::value_unicode_string(""));
}
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
formatting
2015-08-19 12:04:45 +02:00
}
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("slashes")
{
for (auto create_index : { false, true })
{
std::string filename = "test/data/csv/has_attributes_with_slashes.csv";
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "name"});
// NOTE: y column is integer, even though a double value is used below in the test?
require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
auto featureset = all_features(ds);
require_attributes(featureset->next(), {
attr{"x", 0}
, attr{"y", 0}
, attr{"name", mapnik::value_unicode_string("a/a") } });
require_attributes(featureset->next(), {
attr{"x", 1}
, attr{"y", 4}
, attr{"name", mapnik::value_unicode_string("b/b") } });
require_attributes(featureset->next(), {
attr{"x", 10}
, attr{"y", 2.5}
, attr{"name", mapnik::value_unicode_string("c/c") } });
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
formatting
2015-08-19 12:04:45 +02:00
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("wkt field")
{
for (auto create_index : { false, true })
{
std::string filename = "test/data/csv/wkt.csv";
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
using mapnik::geometry::geometry_types;
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"type"});
require_field_types(fields, {mapnik::String});
auto featureset = all_features(ds);
require_geometry(featureset->next(), 1, geometry_types::Point);
require_geometry(featureset->next(), 1, geometry_types::LineString);
require_geometry(featureset->next(), 1, geometry_types::Polygon);
require_geometry(featureset->next(), 1, geometry_types::Polygon);
require_geometry(featureset->next(), 4, geometry_types::MultiPoint);
require_geometry(featureset->next(), 2, geometry_types::MultiLineString);
require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
formatting
2015-08-19 12:04:45 +02:00
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("handling of missing header")
{
for (auto create_index : { false, true })
{
std::string filename = "test/data/csv/missing_header.csv";
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
// TODO: does this mean 'missing_header.csv' should be in the warnings
// subdirectory, since it doesn't work in strict mode?
auto ds = get_csv_ds(filename, false);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"one", "two", "x", "y", "_4", "aftermissing"});
auto feature = all_features(ds)->next();
REQUIRE(feature);
REQUIRE(feature->has_key("_4"));
CHECK(feature->get("_4") == mapnik::value_unicode_string("missing"));
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
formatting
2015-08-19 12:04:45 +02:00
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("handling of headers that are numbers")
{
for (auto create_index : { false, true })
{
std::string filename = "test/data/csv/numbers_for_headers.csv";
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "1990", "1991", "1992"});
auto feature = all_features(ds)->next();
require_attributes(feature, {
attr{"x", 0}
, attr{"y", 0}
, attr{"1990", 1}
, attr{"1991", 2}
, attr{"1992", 3}
});
auto expression = mapnik::parse_expression("[1991]=2");
REQUIRE(bool(expression));
auto value = mapnik::util::apply_visitor(
mapnik::evaluate<mapnik::feature_impl, mapnik::value_type, mapnik::attributes>(
*feature, mapnik::attributes()), *expression);
CHECK(value == true);
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
formatting
2015-08-19 12:04:45 +02:00
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("quoted numbers")
{
formatting
2015-08-19 12:04:45 +02:00
using ustring = mapnik::value_unicode_string;
add disk index tests
2015-10-23 14:11:08 +02:00
for (auto create_index : { false, true })
{
std::string filename = "test/data/csv/quoted_numbers.csv";
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "label"});
auto featureset = all_features(ds);
require_attributes(featureset->next(), {
attr{"x", 0}, attr{"y", 0}, attr{"label", ustring("0,0") } });
require_attributes(featureset->next(), {
attr{"x", 5}, attr{"y", 5}, attr{"label", ustring("5,5") } });
require_attributes(featureset->next(), {
attr{"x", 0}, attr{"y", 5}, attr{"label", ustring("0,5") } });
require_attributes(featureset->next(), {
attr{"x", 5}, attr{"y", 0}, attr{"label", ustring("5,0") } });
require_attributes(featureset->next(), {
attr{"x", 2.5}, attr{"y", 2.5}, attr{"label", ustring("2.5,2.5") } });
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
formatting
2015-08-19 12:04:45 +02:00
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("reading newlines")
{
for (auto create_index : { false, true })
{
for (auto const& platform : {std::string("windows"), std::string("mac")})
{
std::string filename = (boost::format("test/data/csv/%1%_newlines.csv") % platform).str();
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "z"});
require_attributes(all_features(ds)->next(), {
attr{"x", 1}, attr{"y", 10}, attr{"z", 9999.9999} });
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
formatting
2015-08-19 12:04:45 +02:00
}
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("mixed newlines")
{
formatting
2015-08-19 12:04:45 +02:00
using ustring = mapnik::value_unicode_string;
add disk index tests
2015-10-23 14:11:08 +02:00
for (auto create_index : { false, true })
{
for (auto const& filename : {
std::string("test/data/csv/mac_newlines_with_unix_inline.csv")
, std::string("test/data/csv/mac_newlines_with_unix_inline_escaped.csv")
, std::string("test/data/csv/windows_newlines_with_unix_inline.csv")
, std::string("test/data/csv/windows_newlines_with_unix_inline_escaped.csv")
})
{
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "line"});
require_attributes(all_features(ds)->next(), {
attr{"x", 0}, attr{"y", 0}
, attr{"line", ustring("many\n lines\n of text\n with unix newlines")} });
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
}
} // END SECTION
formatting
2015-08-19 12:04:45 +02:00
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("tabs")
{
for (auto create_index : { false, true })
{
std::string filename = "test/data/csv/tabs_in_csv.csv";
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
formatting
2015-08-19 12:04:45 +02:00
auto fields = ds->get_descriptor().get_descriptors();
add disk index tests
2015-10-23 14:11:08 +02:00
require_field_names(fields, {"x", "y", "z"});
formatting
2015-08-19 12:04:45 +02:00
require_attributes(all_features(ds)->next(), {
add disk index tests
2015-10-23 14:11:08 +02:00
attr{"x", -122}, attr{"y", 48}, attr{"z", 0} });
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
formatting
2015-08-19 12:04:45 +02:00
}
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("separators")
{
formatting
2015-08-19 12:04:45 +02:00
using ustring = mapnik::value_unicode_string;
add disk index tests
2015-10-23 14:11:08 +02:00
for (auto const& filename : {
formatting
2015-08-19 12:04:45 +02:00
std::string("test/data/csv/pipe_delimiters.csv")
, std::string("test/data/csv/semicolon_delimiters.csv")
add disk index tests
2015-10-23 14:11:08 +02:00
})
{
for (auto create_index : { false, true })
{
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "z"});
require_attributes(all_features(ds)->next(), {
attr{"x", 0}, attr{"y", 0}, attr{"z", ustring("hello")} });
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
formatting
2015-08-19 12:04:45 +02:00
}
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("null and bool keywords are empty strings")
{
formatting
2015-08-19 12:04:45 +02:00
using ustring = mapnik::value_unicode_string;
add disk index tests
2015-10-23 14:11:08 +02:00
std::string filename = "test/data/csv/nulls_and_booleans_as_strings.csv";
for (auto create_index : { false, true })
{
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "null", "boolean"});
require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::Boolean});
formatting
2015-08-19 12:04:45 +02:00
add disk index tests
2015-10-23 14:11:08 +02:00
auto featureset = all_features(ds);
require_attributes(featureset->next(), {
attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("null")}, attr{"boolean", true}});
require_attributes(featureset->next(), {
attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("")}, attr{"boolean", false}});
formatting
2015-08-19 12:04:45 +02:00
add disk index tests
2015-10-23 14:11:08 +02:00
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
formatting
2015-08-19 12:04:45 +02:00
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("nonexistent query fields throw")
{
std::string filename = "test/data/csv/lon_lat.csv";
for (auto create_index : { false, true })
{
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"lon", "lat"});
require_field_types(fields, {mapnik::Integer, mapnik::Integer});
formatting
2015-08-19 12:04:45 +02:00
add disk index tests
2015-10-23 14:11:08 +02:00
mapnik::query query(ds->envelope());
for (auto const &field : fields)
{
query.add_property_name(field.get_name());
}
// also add an invalid one, triggering throw
query.add_property_name("bogus");
REQUIRE_THROWS(ds->features(query));
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
formatting
2015-08-19 12:04:45 +02:00
}
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("leading zeros mean strings")
{
formatting
2015-08-19 12:04:45 +02:00
using ustring = mapnik::value_unicode_string;
add disk index tests
2015-10-23 14:11:08 +02:00
std::string filename = "test/data/csv/leading_zeros.csv";
for (auto create_index : { false, true })
{
// cleanup in the case of a failed previous run
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
if (create_index)
{
int ret = create_disk_index(filename);
int ret_posix = (ret >> 8) & 0x000000ff;
INFO(ret);
INFO(ret_posix);
CHECK(mapnik::util::exists(filename + ".index"));
}
auto ds = get_csv_ds(filename);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "fips"});
require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
auto featureset = all_features(ds);
require_attributes(featureset->next(), {
attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("001")}});
require_attributes(featureset->next(), {
attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("003")}});
require_attributes(featureset->next(), {
attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("005")}});
if (mapnik::util::exists(filename + ".index"))
{
boost::filesystem::remove(filename + ".index");
}
}
formatting
2015-08-19 12:04:45 +02:00
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("advanced geometry detection")
{
formatting
2015-08-19 12:04:45 +02:00
using row = std::pair<std::string, mapnik::datasource_geometry_t>;
for (row r : {
row{"point", mapnik::datasource_geometry_t::Point}
, row{"poly", mapnik::datasource_geometry_t::Polygon}
, row{"multi_poly", mapnik::datasource_geometry_t::Polygon}
, row{"line", mapnik::datasource_geometry_t::LineString}
}) {
std::string file_name = (boost::format("test/data/csv/%1%_wkt.csv") % r.first).str();
auto ds = get_csv_ds(file_name);
CHECK(ds->get_geometry_type() == r.second);
}
} // END SECTION
add disk index tests
2015-10-23 14:11:08 +02:00
SECTION("creation of CSV from in-memory strings")
{
formatting
2015-08-19 12:04:45 +02:00
using ustring = mapnik::value_unicode_string;
for (auto const &name : {std::string("Winthrop, WA"), std::string(u8"Qu\u00e9bec")}) {
std::string csv_string =
(boost::format(
"wkt,Name\n"
"\"POINT (120.15 48.47)\",\"%1%\"\n"
) % name).str();
mapnik::parameters params;
params["type"] = std::string("csv");
params["inline"] = csv_string;
auto ds = mapnik::datasource_cache::instance().create(params);
REQUIRE(bool(ds));
auto feature = all_features(ds)->next();
REQUIRE(bool(feature));
REQUIRE(feature->has_key("Name"));
CHECK(feature->get("Name") == ustring(name.c_str()));
}
} // END SECTION
CSV - add more tests
2015-10-22 16:11:18 +02:00
SECTION("creation of CSV from in-memory strings with bogus headers")
{
mapnik::parameters params;
params["type"] = std::string("csv");
// should throw
params["inline"] = "latitude, longtitude, Name\n" // misspellt (!)
"120.15,48.47,Winhrop";
REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
// should throw
params["strict"] = true;
params["inline"] = "latitude, longitude\n" // -- missing header
"120.15,48.47,Winhrop";
REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
// should not throw
params["strict"] = false;
params["inline"] = "latitude, longitude,Name\n"
"0,0,Unknown, extra bogus field\n"
"120.15,48.47,Winhrop\n";
auto ds = mapnik::datasource_cache::instance().create(params);
REQUIRE(bool(ds));
REQUIRE(ds->envelope() == mapnik::box2d<double>(48.47,120.15,48.47,120.15));
auto feature = all_features(ds)->next();
REQUIRE(bool(feature));
REQUIRE(feature->has_key("Name"));
// should throw
params["strict"] = false;
params["inline"] = "x, Name\n" // -- missing required *geometry* header
"120.15,Winhrop";
REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
} // END SECTION
formatting
2015-08-19 12:04:45 +02:00
SECTION("geojson quoting") {
using mapnik::geometry::geometry_types;
for (auto const &file : {
std::string("test/data/csv/geojson_double_quote_escape.csv")
add initial 'qoute' auto-detection + restore csv_test's
2015-10-05 17:56:33 +02:00
, std::string("test/data/csv/geojson_single_quote.csv")
, std::string("test/data/csv/geojson_2x_double_quote_filebakery_style.csv")
formatting
2015-08-19 12:04:45 +02:00
}) {
auto ds = get_csv_ds(file);
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"type"});
require_field_types(fields, {mapnik::String});
auto featureset = all_features(ds);
require_geometry(featureset->next(), 1, geometry_types::Point);
require_geometry(featureset->next(), 1, geometry_types::LineString);
require_geometry(featureset->next(), 1, geometry_types::Polygon);
require_geometry(featureset->next(), 1, geometry_types::Polygon);
require_geometry(featureset->next(), 4, geometry_types::MultiPoint);
require_geometry(featureset->next(), 2, geometry_types::MultiLineString);
require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
}
} // END SECTION
SECTION("fewer headers than rows throws") {
REQUIRE_THROWS(get_csv_ds("test/data/csv/more_column_values_than_headers.csv"));
} // END SECTION
SECTION("feature ID only incremented for valid rows") {
auto ds = get_csv_ds("test/data/csv/warns/feature_id_counting.csv", false);
auto fs = all_features(ds);
// first
auto feature = fs->next();
REQUIRE(bool(feature));
CHECK(feature->id() == 1);
// second, should have skipped bogus one
feature = fs->next();
REQUIRE(bool(feature));
CHECK(feature->id() == 2);
feature = fs->next();
CHECK(!feature);
} // END SECTION
SECTION("dynamically defining headers") {
using ustring = mapnik::value_unicode_string;
using row = std::pair<std::string, std::size_t>;
for (auto const &r : {
row{"test/data/csv/fails/needs_headers_two_lines.csv", 2},
row{"test/data/csv/fails/needs_headers_one_line.csv", 1},
row{"test/data/csv/fails/needs_headers_one_line_no_newline.csv", 1}})
{
mapnik::parameters params;
params["type"] = std::string("csv");
params["file"] = r.first;
params["headers"] = "x,y,name";
auto ds = mapnik::datasource_cache::instance().create(params);
REQUIRE(bool(ds));
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "name"});
require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
require_attributes(all_features(ds)->next(), {
attr{"x", 0}, attr{"y", 0}, attr{"name", ustring("data_name")} });
REQUIRE(count_features(all_features(ds)) == r.second);
add missing geometry type check
2015-08-27 13:02:17 +02:00
CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
formatting
2015-08-19 12:04:45 +02:00
}
} // END SECTION
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wlong-long"
SECTION("64bit int fields work") {
auto ds = get_csv_ds("test/data/csv/64bit_int.csv");
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "bigint"});
require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Integer});
auto fs = all_features(ds);
auto feature = fs->next();
require_attributes(feature, {
attr{"x", 0}, attr{"y", 0}, attr{"bigint", 2147483648} });
feature = fs->next();
require_attributes(feature, {
attr{"x", 0}, attr{"y", 0}, attr{"bigint", 9223372036854775807ll} });
require_attributes(feature, {
attr{"x", 0}, attr{"y", 0}, attr{"bigint", 0x7FFFFFFFFFFFFFFFll} });
} // END SECTION
#pragma GCC diagnostic pop
SECTION("various number types") {
auto ds = get_csv_ds("test/data/csv/number_types.csv");
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {"x", "y", "floats"});
require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Double});
auto fs = all_features(ds);
for (double d : { .0, +.0, 1e-06, -1e-06, 0.000001, 1.234e+16, 1.234e+16 }) {
auto feature = fs->next();
REQUIRE(bool(feature));
CHECK(feature->get("floats").get<mapnik::value_double>() == Approx(d));
}
} // END SECTION
SECTION("manually supplied extent") {
std::string csv_string("wkt,Name\n");
mapnik::parameters params;
params["type"] = std::string("csv");
params["inline"] = csv_string;
params["extent"] = "-180,-90,180,90";
auto ds = mapnik::datasource_cache::instance().create(params);
ensure tests pass with no plugins built
2015-07-01 02:04:48 +02:00
REQUIRE(bool(ds));
formatting
2015-08-19 12:04:45 +02:00
auto box = ds->envelope();
CHECK(box.minx() == -180);
CHECK(box.miny() == -90);
CHECK(box.maxx() == 180);
CHECK(box.maxy() == 90);
} // END SECTION
SECTION("inline geojson") {
std::string csv_string = "geojson\n'{\"coordinates\":[-92.22568,38.59553],\"type\":\"Point\"}'";
mapnik::parameters params;
params["type"] = std::string("csv");
params["inline"] = csv_string;
make quote single char
2015-10-02 13:20:54 +02:00
params["quote"] = "'";
formatting
2015-08-19 12:04:45 +02:00
auto ds = mapnik::datasource_cache::instance().create(params);
REQUIRE(bool(ds));
auto fields = ds->get_descriptor().get_descriptors();
require_field_names(fields, {});
// TODO: this originally had the following comment:
// - re-enable after https://github.com/mapnik/mapnik/issues/2319 is fixed
// but that seems to have been merged and tested separately?
auto fs = all_features(ds);
auto feat = fs->next();
CHECK(feature_count(feat->get_geometry()) == 1);
} // END SECTION
mapnik::logger::instance().set_severity(severity);
}
Ported CSV tests from Python.
2015-05-18 23:14:37 +02:00
} // END TEST CASE
Reference in a new issue Copy permalink