From 468bc5257b88c757df3a989f42802795d987d9c8 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 18 May 2015 14:45:30 +0100 Subject: [PATCH 1/9] Re-order libs to satify correct link order for ICU symbols from mapnik-json library. --- test/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/build.py b/test/build.py index 91e5ee418..8fd005a67 100644 --- a/test/build.py +++ b/test/build.py @@ -11,9 +11,9 @@ if not env['CPP_TESTS']: os.unlink(cpp_test_bin) else: test_env['LIBS'] = [env['MAPNIK_NAME']] - test_env.AppendUnique(LIBS=copy(env['LIBMAPNIK_LIBS'])) test_env.AppendUnique(LIBS='mapnik-wkt') test_env.AppendUnique(LIBS='mapnik-json') + test_env.AppendUnique(LIBS=copy(env['LIBMAPNIK_LIBS'])) if env['RUNTIME_LINK'] == 'static' and env['PLATFORM'] == 'Linux': test_env.AppendUnique(LIBS='dl') test_env.AppendUnique(CXXFLAGS='-g') From 8e62007e4f297875c1311436efad34a8f914a6f9 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 18 May 2015 14:46:01 +0100 Subject: [PATCH 2/9] Added port of agg_rasterizer_integer_overflow_test.py --- .../agg_rasterizer_integer_overflow_test.cpp | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 test/standalone/agg_rasterizer_integer_overflow_test.cpp diff --git a/test/standalone/agg_rasterizer_integer_overflow_test.cpp b/test/standalone/agg_rasterizer_integer_overflow_test.cpp new file mode 100644 index 000000000..0341f3138 --- /dev/null +++ b/test/standalone/agg_rasterizer_integer_overflow_test.cpp @@ -0,0 +1,70 @@ +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +// geojson box of the world +const std::string geojson("{ \"type\": \"Feature\", \"properties\": { }, \"geometry\": { \"type\": \"Polygon\", \"coordinates\": [ [ [ -17963313.143242701888084, -6300857.11560364998877 ], [ -17963313.143242701888084, 13071343.332991421222687 ], [ 7396658.353099936619401, 13071343.332991421222687 ], [ 7396658.353099936619401, -6300857.11560364998877 ], [ -17963313.143242701888084, -6300857.11560364998877 ] ] ] } }"); + +TEST_CASE("agg_rasterizer_integer_overflow") { + +SECTION("coordinates_do_not_overflow_and_polygon_is_rendered") { + try { + auto expected_color = mapnik::color("white"); + + mapnik::Map m(256,256); + m.set_background(mapnik::color("black")); + + mapnik::feature_type_style s; + { + mapnik::rule r; + mapnik::polygon_symbolizer sym; + mapnik::put(sym, mapnik::keys::fill, expected_color); + mapnik::put(sym, mapnik::keys::clip, false); + r.append(std::move(sym)); + s.add_rule(std::move(r)); + } + m.insert_style("style",std::move(s)); + + mapnik::layer lyr("Layer"); + lyr.styles().emplace_back("style"); + { + auto ds = std::make_shared(mapnik::parameters()); + auto context = std::make_shared(); + auto f = std::make_shared(context, 0); + REQUIRE(mapnik::json::from_geojson(geojson, *f)); + ds->push(f); + lyr.set_datasource(ds); + } + m.add_layer(std::move(lyr)); + + // 17/20864/45265.png + m.zoom_to_box(mapnik::box2d(-13658379.710221574,6197514.253362091,-13657768.213995293,6198125.749588372)); + + // works 15/5216/11316.png + //m.zoom_to_box(mapnik::box2d(-13658379.710221574,6195679.764683247,-13655933.72531645,6198125.749588372)); + + mapnik::image_rgba8 im(256, 256); + { + mapnik::agg_renderer ren(m, im); + ren.apply(); + } + + REQUIRE(im(128,128) == expected_color.rgba()); + + } catch (std::exception const &e) { + std::clog << e.what() << std::endl; + REQUIRE(false); + } +} // SECTION +} // TEST_CASE From 818ede5b93aefae9271392f82c655ba399d93e2e Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 18 May 2015 15:10:55 +0100 Subject: [PATCH 3/9] Ported box2d_test.py --- test/unit/core/box2d_test.cpp | 163 ++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 test/unit/core/box2d_test.cpp diff --git a/test/unit/core/box2d_test.cpp b/test/unit/core/box2d_test.cpp new file mode 100644 index 000000000..c51980794 --- /dev/null +++ b/test/unit/core/box2d_test.cpp @@ -0,0 +1,163 @@ +#include "catch.hpp" + +#include +#include +#include + +TEST_CASE("box2d") { +SECTION("coord init") { + auto c = mapnik::coord2d(100, 100); + + REQUIRE(c.x == 100); + REQUIRE(c.y == 100); +} + +SECTION("coord multiplication") { + auto c = mapnik::coord2d(100, 100); + c *= 2; + + REQUIRE(c.x == 200); + REQUIRE(c.y == 200); +} + +SECTION("envelope init") { + auto e = mapnik::box2d(100, 100, 200, 200); + + REQUIRE(e.contains(100, 100)); + REQUIRE(e.contains(100, 200)); + REQUIRE(e.contains(200, 200)); + REQUIRE(e.contains(200, 100)); + + REQUIRE(e.contains(e.center())); + + REQUIRE(!e.contains(99.9, 99.9)); + REQUIRE(!e.contains(99.9, 200.1)); + REQUIRE(!e.contains(200.1, 200.1)); + REQUIRE(!e.contains(200.1, 99.9)); + + REQUIRE(e.width() == 100); + REQUIRE(e.height() == 100); + + REQUIRE(e.minx() == 100); + REQUIRE(e.miny() == 100); + + REQUIRE(e.maxx() == 200); + REQUIRE(e.maxy() == 200); + + REQUIRE(e[0] == 100); + REQUIRE(e[1] == 100); + REQUIRE(e[2] == 200); + REQUIRE(e[3] == 200); + REQUIRE(e[0] == e[-4]); + REQUIRE(e[1] == e[-3]); + REQUIRE(e[2] == e[-2]); + REQUIRE(e[3] == e[-1]); + + auto c = e.center(); + + REQUIRE(c.x == 150); + REQUIRE(c.y == 150); +} + +SECTION("envelope static init") { + auto e = mapnik::box2d(100, 100, 200, 200); + + mapnik::box2d e1, e2, e3; + REQUIRE(e1.from_string("100 100 200 200")); + REQUIRE(e2.from_string("100,100,200,200")); + REQUIRE(e3.from_string("100 , 100 , 200 , 200")); + + REQUIRE(e == e1); + REQUIRE(e == e2); + REQUIRE(e == e3); +} + +SECTION("envelope multiplication") { + // no width then no impact of multiplication + { + auto a = mapnik::box2d(100, 100, 100, 100); + a *= 5; + + REQUIRE(a.minx() == 100); + REQUIRE(a.miny() == 100); + REQUIRE(a.maxx() == 100); + REQUIRE(a.maxy() == 100); + } + + { + auto a = mapnik::box2d(100.0, 100.0, 100.0, 100.0); + a *= 5; + + REQUIRE(a.minx() == 100); + REQUIRE(a.miny() == 100); + REQUIRE(a.maxx() == 100); + REQUIRE(a.maxy() == 100); + } + + { + auto a = mapnik::box2d(100.0, 100.0, 100.001, 100.001); + a *= 5; + + REQUIRE(std::abs(a.minx() - 99.9980) < 0.001); + REQUIRE(std::abs(a.miny() - 99.9980) < 0.001); + REQUIRE(std::abs(a.maxx() - 100.0030) < 0.001); + REQUIRE(std::abs(a.maxy() - 100.0030) < 0.001); + } + + { + auto e = mapnik::box2d(100, 100, 200, 200); + e *= 2; + + REQUIRE(e.minx() == 50); + REQUIRE(e.miny() == 50); + REQUIRE(e.maxx() == 250); + REQUIRE(e.maxy() == 250); + + REQUIRE(e.contains(50, 50)); + REQUIRE(e.contains(50, 250)); + REQUIRE(e.contains(250, 250)); + REQUIRE(e.contains(250, 50)); + + REQUIRE(!e.contains(49.9, 49.9)); + REQUIRE(!e.contains(49.9, 250.1)); + REQUIRE(!e.contains(250.1, 250.1)); + REQUIRE(!e.contains(250.1, 49.9)); + + REQUIRE(e.contains(e.center())); + + REQUIRE(e.width() == 200); + REQUIRE(e.height() == 200); + + REQUIRE(e.minx() == 50); + REQUIRE(e.miny() == 50); + + REQUIRE(e.maxx() == 250); + REQUIRE(e.maxy() == 250); + + auto c = e.center(); + + REQUIRE(c.x == 150); + REQUIRE(c.y == 150); + } +} + +SECTION("envelope clipping") { + auto e1 = mapnik::box2d(-180,-90,180,90); + auto e2 = mapnik::box2d(-120,40,-110,48); + e1.clip(e2); + REQUIRE(e1 == e2); + + // madagascar in merc + e1 = mapnik::box2d(4772116.5490, -2744395.0631, 5765186.4203, -1609458.0673); + e2 = mapnik::box2d(5124338.3753, -2240522.1727, 5207501.8621, -2130452.8520); + e1.clip(e2); + REQUIRE(e1 == e2); + + // nz in lon/lat + e1 = mapnik::box2d(163.8062, -47.1897, 179.3628, -33.9069); + e2 = mapnik::box2d(173.7378, -39.6395, 174.4849, -38.9252); + e1.clip(e2); + REQUIRE(e1 == e2); +} + +} // TEST_CASE From a16b6156eda3a308a919f827465e394d7cf60062 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 18 May 2015 22:14:37 +0100 Subject: [PATCH 4/9] Ported CSV tests from Python. --- test/standalone/csv_test.cpp | 715 +++++++++++++++++++++++++++++++++++ 1 file changed, 715 insertions(+) create mode 100644 test/standalone/csv_test.cpp diff --git a/test/standalone/csv_test.cpp b/test/standalone/csv_test.cpp new file mode 100644 index 000000000..d078ae83b --- /dev/null +++ b/test/standalone/csv_test.cpp @@ -0,0 +1,715 @@ +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +namespace bfs = boost::filesystem; + +namespace { +void add_csv_files(bfs::path dir, std::vector &csv_files) { + for (auto const &entry : boost::make_iterator_range( + bfs::directory_iterator(dir), bfs::directory_iterator())) { + auto path = entry.path(); + if (path.extension().native() == ".csv") { + csv_files.emplace_back(path); + } + } +} + +bool operator==(mapnik::attribute_descriptor const &a, + mapnik::attribute_descriptor const &b) { + return ((a.get_name() == b.get_name()) && + (a.get_type() == b.get_type()) && + (a.is_primary_key() == b.is_primary_key()) & + (a.get_size() == b.get_size()) && + (a.get_precision() == b.get_precision())); +} + +mapnik::datasource_ptr get_csv_ds(std::string const &file_name, bool strict = true) { + mapnik::parameters params; + params["type"] = std::string("csv"); + params["file"] = file_name; + params["strict"] = mapnik::value_bool(strict); + auto ds = mapnik::datasource_cache::instance().create(params); + // require a non-null pointer returned + REQUIRE(bool(ds)); + return ds; +} + +void require_field_names(std::vector const &fields, + std::initializer_list const &names) { + REQUIRE(fields.size() == names.size()); + auto itr_a = fields.begin(); + auto const end_a = fields.end(); + auto itr_b = names.begin(); + for (; itr_a != end_a; ++itr_a, ++itr_b) { + CHECK(itr_a->get_name() == *itr_b); + } +} + +void require_field_types(std::vector const &fields, + std::initializer_list const &types) { + REQUIRE(fields.size() == types.size()); + auto itr_a = fields.begin(); + auto const end_a = fields.end(); + auto itr_b = types.begin(); + for (; itr_a != end_a; ++itr_a, ++itr_b) { + CHECK(itr_a->get_type() == *itr_b); + } +} + +mapnik::featureset_ptr all_features(mapnik::datasource_ptr ds) { + auto fields = ds->get_descriptor().get_descriptors(); + mapnik::query query(ds->envelope()); + for (auto const &field : fields) { + query.add_property_name(field.get_name()); + } + return ds->features(query); +} + +std::size_t count_features(mapnik::featureset_ptr features) { + std::size_t count = 0; + while (features->next()) { + ++count; + } + return count; +} + +using attr = std::tuple; +void require_attributes(mapnik::feature_ptr feature, + std::initializer_list const &attrs) { + REQUIRE(bool(feature)); + for (auto const &kv : attrs) { + REQUIRE(feature->has_key(std::get<0>(kv))); + CHECK(feature->get(std::get<0>(kv)) == std::get<1>(kv)); + } +} + +namespace detail { +struct feature_count { + template + std::size_t operator()(T const &geom) const { + return mapnik::util::apply_visitor(*this, geom); + } + + std::size_t operator()(mapnik::geometry::geometry_empty const &) const { + return 0; + } + + template + std::size_t operator()(mapnik::geometry::point const &) const { + return 1; + } + + template + std::size_t operator()(mapnik::geometry::line_string const &) const { + return 1; + } + + template + std::size_t operator()(mapnik::geometry::polygon const &) const { + return 1; + } + + template + std::size_t operator()(mapnik::geometry::multi_point const &mp) const { + return mp.size(); + } + + template + std::size_t operator()(mapnik::geometry::multi_line_string const &mls) const { + return mls.size(); + } + + template + std::size_t operator()(mapnik::geometry::multi_polygon const &mp) const { + return mp.size(); + } + + template + std::size_t operator()(mapnik::geometry::geometry_collection const &col) const { + std::size_t sum = 0; + for (auto const &geom : col) { + sum += operator()(geom); + } + return sum; + } +}; +} // namespace detail + +template +std::size_t feature_count(mapnik::geometry::geometry const &g) { + return detail::feature_count()(g); +} + +void require_geometry(mapnik::feature_ptr feature, + std::size_t num_parts, + mapnik::geometry::geometry_types type) { + REQUIRE(bool(feature)); + CHECK(mapnik::geometry::geometry_type(feature->get_geometry()) == type); + CHECK(feature_count(feature->get_geometry()) == num_parts); +} +} // anonymous namespace + +const bool registered = mapnik::datasource_cache::instance().register_datasources("./plugins/input/"); + +TEST_CASE("csv") { + REQUIRE(registered); + + // make the tests silent since we intentially test error conditions that are noisy + auto const severity = mapnik::logger::instance().get_severity(); + mapnik::logger::instance().set_severity(mapnik::logger::none); + + // check the CSV datasource is loaded + const std::vector plugin_names = + mapnik::datasource_cache::instance().plugin_names(); + const bool have_csv_plugin = + std::find(plugin_names.begin(), plugin_names.end(), "csv") != plugin_names.end(); + + SECTION("broken files") { + if (have_csv_plugin) { + std::vector broken; + add_csv_files("test/data/csv/fails", broken); + add_csv_files("test/data/csv/warns", broken); + broken.emplace_back("test/data/csv/fails/does_not_exist.csv"); + + for (auto const &path : broken) { + bool threw = false; + + try { + auto ds = get_csv_ds(path.native()); + + } catch (const std::exception &e) { + threw = true; + } + + REQUIRE(threw); + } + } + } // END SECTION + + SECTION("good files") { + if (have_csv_plugin) { + std::vector good; + add_csv_files("test/data/csv", good); + add_csv_files("test/data/csv/warns", good); + + for (auto const &path : good) { + bool threw = false; + + try { + auto ds = get_csv_ds(path.native(), false); + // require a non-null pointer returned + REQUIRE(bool(ds)); + + } catch (const std::exception &e) { + threw = true; + } + + REQUIRE(!threw); + } + } + } // END SECTION + + SECTION("lon/lat detection") { + for (auto const &lon_name : {std::string("lon"), std::string("lng")}) { + auto ds = get_csv_ds((boost::format("test/data/csv/%1%_lat.csv") % lon_name).str()); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {lon_name, "lat"}); + require_field_types(fields, {mapnik::Integer, mapnik::Integer}); + + CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point); + + mapnik::query query(ds->envelope()); + for (auto const &field : fields) { + query.add_property_name(field.get_name()); + } + auto features = ds->features(query); + auto feature = features->next(); + + require_attributes(feature, { + attr { lon_name, mapnik::value_integer(0) }, + attr { "lat", mapnik::value_integer(0) } + }); + } + } // END SECTION + + SECTION("type detection") { + auto ds = get_csv_ds("test/data/csv/nypd.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"Precinct", "Phone", "Address", "City", "geo_longitude", "geo_latitude", "geo_accuracy"}); + require_field_types(fields, {mapnik::String, mapnik::String, mapnik::String, mapnik::String, mapnik::Double, mapnik::Double, mapnik::String}); + + CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point); + CHECK(count_features(all_features(ds)) == 2); + + auto feature = all_features(ds)->next(); + require_attributes(feature, { + attr { "City", mapnik::value_unicode_string("New York, NY") } + , attr { "geo_accuracy", mapnik::value_unicode_string("house") } + , attr { "Phone", mapnik::value_unicode_string("(212) 334-0711") } + , attr { "Address", mapnik::value_unicode_string("19 Elizabeth Street") } + , attr { "Precinct", mapnik::value_unicode_string("5th Precinct") } + , attr { "geo_longitude", mapnik::value_integer(-70) } + , attr { "geo_latitude", mapnik::value_integer(40) } + }); + } // END SECTION + + SECTION("skipping blank rows") { + auto ds = get_csv_ds("test/data/csv/blank_rows.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "name"}); + require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String}); + + CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point); + CHECK(count_features(all_features(ds)) == 2); + } // END SECTION + + SECTION("empty rows") { + auto ds = get_csv_ds("test/data/csv/empty_rows.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "text", "date", "integer", "boolean", "float", "time", "datetime", "empty_column"}); + require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::String, mapnik::Integer, mapnik::Boolean, mapnik::Double, mapnik::String, mapnik::String, mapnik::String}); + + CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point); + CHECK(count_features(all_features(ds)) == 4); + + auto featureset = all_features(ds); + auto feature = featureset->next(); + require_attributes(feature, { + attr { "x", mapnik::value_integer(0) } + , attr { "empty_column", mapnik::value_unicode_string("") } + , attr { "text", mapnik::value_unicode_string("a b") } + , attr { "float", mapnik::value_double(1.0) } + , attr { "datetime", mapnik::value_unicode_string("1971-01-01T04:14:00") } + , attr { "y", mapnik::value_integer(0) } + , attr { "boolean", mapnik::value_bool(true) } + , attr { "time", mapnik::value_unicode_string("04:14:00") } + , attr { "date", mapnik::value_unicode_string("1971-01-01") } + , attr { "integer", mapnik::value_integer(40) } + }); + + while (bool(feature = featureset->next())) { + CHECK(feature->size() == 10); + CHECK(feature->get("empty_column") == mapnik::value_unicode_string("")); + } + } // END SECTION + + SECTION("slashes") { + auto ds = get_csv_ds("test/data/csv/has_attributes_with_slashes.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "name"}); + // NOTE: y column is integer, even though a double value is used below in the test? + require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String}); + + auto featureset = all_features(ds); + require_attributes(featureset->next(), { + attr{"x", 0} + , attr{"y", 0} + , attr{"name", mapnik::value_unicode_string("a/a") } }); + require_attributes(featureset->next(), { + attr{"x", 1} + , attr{"y", 4} + , attr{"name", mapnik::value_unicode_string("b/b") } }); + require_attributes(featureset->next(), { + attr{"x", 10} + , attr{"y", 2.5} + , attr{"name", mapnik::value_unicode_string("c/c") } }); + } // END SECTION + + SECTION("wkt field") { + using mapnik::geometry::geometry_types; + + auto ds = get_csv_ds("test/data/csv/wkt.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"type"}); + require_field_types(fields, {mapnik::String}); + + auto featureset = all_features(ds); + require_geometry(featureset->next(), 1, geometry_types::Point); + require_geometry(featureset->next(), 1, geometry_types::LineString); + require_geometry(featureset->next(), 1, geometry_types::Polygon); + require_geometry(featureset->next(), 1, geometry_types::Polygon); + require_geometry(featureset->next(), 4, geometry_types::MultiPoint); + require_geometry(featureset->next(), 2, geometry_types::MultiLineString); + require_geometry(featureset->next(), 2, geometry_types::MultiPolygon); + require_geometry(featureset->next(), 2, geometry_types::MultiPolygon); + } // END SECTION + + SECTION("handling of missing header") { + // TODO: does this mean 'missing_header.csv' should be in the warnings + // subdirectory, since it doesn't work in strict mode? + auto ds = get_csv_ds("test/data/csv/missing_header.csv", false); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"one", "two", "x", "y", "_4", "aftermissing"}); + auto feature = all_features(ds)->next(); + REQUIRE(feature); + REQUIRE(feature->has_key("_4")); + CHECK(feature->get("_4") == mapnik::value_unicode_string("missing")); + } // END SECTION + + SECTION("handling of headers that are numbers") { + auto ds = get_csv_ds("test/data/csv/numbers_for_headers.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "1990", "1991", "1992"}); + auto feature = all_features(ds)->next(); + require_attributes(feature, { + attr{"x", 0} + , attr{"y", 0} + , attr{"1990", 1} + , attr{"1991", 2} + , attr{"1992", 3} + }); + auto expression = mapnik::parse_expression("[1991]=2"); + REQUIRE(bool(expression)); + auto value = mapnik::util::apply_visitor( + mapnik::evaluate( + *feature, mapnik::attributes()), *expression); + CHECK(value == true); + } // END SECTION + + SECTION("quoted numbers") { + using ustring = mapnik::value_unicode_string; + + auto ds = get_csv_ds("test/data/csv/quoted_numbers.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "label"}); + auto featureset = all_features(ds); + + require_attributes(featureset->next(), { + attr{"x", 0}, attr{"y", 0}, attr{"label", ustring("0,0") } }); + require_attributes(featureset->next(), { + attr{"x", 5}, attr{"y", 5}, attr{"label", ustring("5,5") } }); + require_attributes(featureset->next(), { + attr{"x", 0}, attr{"y", 5}, attr{"label", ustring("0,5") } }); + require_attributes(featureset->next(), { + attr{"x", 5}, attr{"y", 0}, attr{"label", ustring("5,0") } }); + require_attributes(featureset->next(), { + attr{"x", 2.5}, attr{"y", 2.5}, attr{"label", ustring("2.5,2.5") } }); + + } // END SECTION + + SECTION("reading newlines") { + for (auto const &platform : {std::string("windows"), std::string("mac")}) { + std::string file_name = (boost::format("test/data/csv/%1%_newlines.csv") % platform).str(); + auto ds = get_csv_ds(file_name); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "z"}); + require_attributes(all_features(ds)->next(), { + attr{"x", 1}, attr{"y", 10}, attr{"z", 9999.9999} }); + } + } // END SECTION + + SECTION("mixed newlines") { + using ustring = mapnik::value_unicode_string; + + for (auto const &file : { + std::string("test/data/csv/mac_newlines_with_unix_inline.csv") + , std::string("test/data/csv/mac_newlines_with_unix_inline_escaped.csv") + , std::string("test/data/csv/windows_newlines_with_unix_inline.csv") + , std::string("test/data/csv/windows_newlines_with_unix_inline_escaped.csv") + }) { + auto ds = get_csv_ds(file); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "line"}); + require_attributes(all_features(ds)->next(), { + attr{"x", 0}, attr{"y", 0} + , attr{"line", ustring("many\n lines\n of text\n with unix newlines")} }); + } + } // END SECTION + + SECTION("tabs") { + auto ds = get_csv_ds("test/data/csv/tabs_in_csv.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "z"}); + require_attributes(all_features(ds)->next(), { + attr{"x", -122}, attr{"y", 48}, attr{"z", 0} }); + } // END SECTION + + SECTION("separators") { + using ustring = mapnik::value_unicode_string; + + for (auto const &file : { + std::string("test/data/csv/pipe_delimiters.csv") + , std::string("test/data/csv/semicolon_delimiters.csv") + }) { + auto ds = get_csv_ds(file); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "z"}); + require_attributes(all_features(ds)->next(), { + attr{"x", 0}, attr{"y", 0}, attr{"z", ustring("hello")} }); + } + } // END SECTION + + SECTION("null and bool keywords are empty strings") { + using ustring = mapnik::value_unicode_string; + + auto ds = get_csv_ds("test/data/csv/nulls_and_booleans_as_strings.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "null", "boolean"}); + require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::Boolean}); + + auto featureset = all_features(ds); + require_attributes(featureset->next(), { + attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("null")}, attr{"boolean", true}}); + require_attributes(featureset->next(), { + attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("")}, attr{"boolean", false}}); + } // END SECTION + + SECTION("nonexistent query fields throw") { + auto ds = get_csv_ds("test/data/csv/lon_lat.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"lon", "lat"}); + require_field_types(fields, {mapnik::Integer, mapnik::Integer}); + + mapnik::query query(ds->envelope()); + for (auto const &field : fields) { + query.add_property_name(field.get_name()); + } + // also add an invalid one, triggering throw + query.add_property_name("bogus"); + + bool threw = false; + try { + ds->features(query); + + } catch (std::exception const &) { + threw = true; + } + + CHECK(threw); + } // END SECTION + + SECTION("leading zeros mean strings") { + using ustring = mapnik::value_unicode_string; + + auto ds = get_csv_ds("test/data/csv/leading_zeros.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "fips"}); + require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String}); + + auto featureset = all_features(ds); + require_attributes(featureset->next(), { + attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("001")}}); + require_attributes(featureset->next(), { + attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("003")}}); + require_attributes(featureset->next(), { + attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("005")}}); + } // END SECTION + + SECTION("advanced geometry detection") { + using row = std::pair; + + for (row r : { + row{"point", mapnik::datasource_geometry_t::Point} + , row{"poly", mapnik::datasource_geometry_t::Polygon} + , row{"multi_poly", mapnik::datasource_geometry_t::Polygon} + , row{"line", mapnik::datasource_geometry_t::LineString} + }) { + std::string file_name = (boost::format("test/data/csv/%1%_wkt.csv") % r.first).str(); + auto ds = get_csv_ds(file_name); + CHECK(ds->get_geometry_type() == r.second); + } + } // END SECTION + + SECTION("creation of CSV from in-memory strings") { + using ustring = mapnik::value_unicode_string; + + for (auto const &name : {std::string("Winthrop, WA"), std::string(u8"Qu\u00e9bec")}) { + std::string csv_string = + (boost::format( + "wkt,Name\n" + "\"POINT (120.15 48.47)\",\"%1%\"\n" + ) % name).str(); + + mapnik::parameters params; + params["type"] = std::string("csv"); + params["inline"] = csv_string; + auto ds = mapnik::datasource_cache::instance().create(params); + REQUIRE(bool(ds)); + + auto feature = all_features(ds)->next(); + REQUIRE(bool(feature)); + REQUIRE(feature->has_key("Name")); + CHECK(feature->get("Name") == ustring(name.c_str())); + } + } // END SECTION + + SECTION("geojson quoting") { + using mapnik::geometry::geometry_types; + + for (auto const &file : { + std::string("test/data/csv/geojson_double_quote_escape.csv") + , std::string("test/data/csv/geojson_single_quote.csv") + , std::string("test/data/csv/geojson_2x_double_quote_filebakery_style.csv") + }) { + auto ds = get_csv_ds(file); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"type"}); + require_field_types(fields, {mapnik::String}); + + auto featureset = all_features(ds); + require_geometry(featureset->next(), 1, geometry_types::Point); + require_geometry(featureset->next(), 1, geometry_types::LineString); + require_geometry(featureset->next(), 1, geometry_types::Polygon); + require_geometry(featureset->next(), 1, geometry_types::Polygon); + require_geometry(featureset->next(), 4, geometry_types::MultiPoint); + require_geometry(featureset->next(), 2, geometry_types::MultiLineString); + require_geometry(featureset->next(), 2, geometry_types::MultiPolygon); + require_geometry(featureset->next(), 2, geometry_types::MultiPolygon); + } + } // END SECTION + + SECTION("blank undelimited rows are still parsed") { + using ustring = mapnik::value_unicode_string; + + // TODO: does this mean this CSV file should be in the warnings + // subdirectory, since it doesn't work in strict mode? + auto ds = get_csv_ds("test/data/csv/more_headers_than_column_values.csv", false); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "one", "two", "three"}); + require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::String, mapnik::String}); + + require_attributes(all_features(ds)->next(), { + attr{"x", 0}, attr{"y", 0}, attr{"one", ustring("")}, attr{"two", ustring("")}, attr{"three", ustring("")} }); + } // END SECTION + + SECTION("fewer headers than rows throws") { + bool threw = false; + try { + get_csv_ds("test/data/csv/more_column_values_than_headers.csv"); + } catch (std::exception const &) { + threw = true; + } + CHECK(threw); + } // END SECTION + + SECTION("feature ID only incremented for valid rows") { + auto ds = get_csv_ds("test/data/csv/warns/feature_id_counting.csv", false); + auto fs = all_features(ds); + + // first + auto feature = fs->next(); + REQUIRE(bool(feature)); + CHECK(feature->id() == 1); + + // second, should have skipped bogus one + feature = fs->next(); + REQUIRE(bool(feature)); + CHECK(feature->id() == 2); + + feature = fs->next(); + CHECK(!feature); + } // END SECTION + + SECTION("dynamically defining headers") { + using ustring = mapnik::value_unicode_string; + using row = std::pair; + + for (auto const &r : { + row{"test/data/csv/fails/needs_headers_two_lines.csv", 2} + , row{"test/data/csv/fails/needs_headers_one_line.csv", 1} + , row{"test/data/csv/fails/needs_headers_one_line_no_newline.csv", 1} + }) { + mapnik::parameters params; + params["type"] = std::string("csv"); + params["file"] = r.first; + params["headers"] = "x,y,name"; + auto ds = mapnik::datasource_cache::instance().create(params); + REQUIRE(bool(ds)); + + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "name"}); + require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String}); + require_attributes(all_features(ds)->next(), { + attr{"x", 0}, attr{"y", 0}, attr{"name", ustring("data_name")} }); + REQUIRE(count_features(all_features(ds)) == r.second); + } + } // END SECTION + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wlong-long" + SECTION("64bit int fields work") { + auto ds = get_csv_ds("test/data/csv/64bit_int.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "bigint"}); + require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Integer}); + + auto fs = all_features(ds); + auto feature = fs->next(); + require_attributes(feature, { + attr{"x", 0}, attr{"y", 0}, attr{"bigint", 2147483648} }); + + feature = fs->next(); + require_attributes(feature, { + attr{"x", 0}, attr{"y", 0}, attr{"bigint", 9223372036854775807ll} }); + require_attributes(feature, { + attr{"x", 0}, attr{"y", 0}, attr{"bigint", 0x7FFFFFFFFFFFFFFFll} }); + } // END SECTION +#pragma GCC diagnostic pop + + SECTION("various number types") { + auto ds = get_csv_ds("test/data/csv/number_types.csv"); + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {"x", "y", "floats"}); + require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Double}); + + auto fs = all_features(ds); + for (double d : { .0, +.0, 1e-06, -1e-06, 0.000001, 1.234e+16, 1.234e+16 }) { + auto feature = fs->next(); + REQUIRE(bool(feature)); + CHECK(feature->get("floats") == d); + } + } // END SECTION + + SECTION("manually supplied extent") { + std::string csv_string("wkt,Name\n"); + mapnik::parameters params; + params["type"] = std::string("csv"); + params["inline"] = csv_string; + params["extent"] = "-180,-90,180,90"; + auto ds = mapnik::datasource_cache::instance().create(params); + REQUIRE(bool(ds)); + + auto box = ds->envelope(); + CHECK(box.minx() == -180); + CHECK(box.miny() == -90); + CHECK(box.maxx() == 180); + CHECK(box.maxy() == 90); + } // END SECTION + + SECTION("inline geojson") { + std::string csv_string = "geojson\n'{\"coordinates\":[-92.22568,38.59553],\"type\":\"Point\"}'"; + mapnik::parameters params; + params["type"] = std::string("csv"); + params["inline"] = csv_string; + auto ds = mapnik::datasource_cache::instance().create(params); + REQUIRE(bool(ds)); + + auto fields = ds->get_descriptor().get_descriptors(); + require_field_names(fields, {}); + + // TODO: this originally had the following comment: + // - re-enable after https://github.com/mapnik/mapnik/issues/2319 is fixed + // but that seems to have been merged and tested separately? + auto fs = all_features(ds); + auto feat = fs->next(); + CHECK(feature_count(feat->get_geometry()) == 1); + } // END SECTION + + mapnik::logger::instance().set_severity(severity); +} // END TEST CASE From 2bb50371ca182ca1a25dea560e1fe0066b3e0697 Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 18 May 2015 22:34:34 +0100 Subject: [PATCH 5/9] Need optional I/O header on clang++, apparently. --- test/standalone/csv_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/standalone/csv_test.cpp b/test/standalone/csv_test.cpp index d078ae83b..86b94e247 100644 --- a/test/standalone/csv_test.cpp +++ b/test/standalone/csv_test.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include From 26f233c505099872833cc54e29b17c51fc457fdd Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 18 May 2015 22:37:13 +0100 Subject: [PATCH 6/9] Test failure on thrown exception is already handled by the test framework. --- .../agg_rasterizer_integer_overflow_test.cpp | 82 +++++++++---------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/test/standalone/agg_rasterizer_integer_overflow_test.cpp b/test/standalone/agg_rasterizer_integer_overflow_test.cpp index 0341f3138..101596b83 100644 --- a/test/standalone/agg_rasterizer_integer_overflow_test.cpp +++ b/test/standalone/agg_rasterizer_integer_overflow_test.cpp @@ -19,52 +19,46 @@ const std::string geojson("{ \"type\": \"Feature\", \"properties\": { }, \"geome TEST_CASE("agg_rasterizer_integer_overflow") { SECTION("coordinates_do_not_overflow_and_polygon_is_rendered") { - try { - auto expected_color = mapnik::color("white"); + auto expected_color = mapnik::color("white"); - mapnik::Map m(256,256); - m.set_background(mapnik::color("black")); + mapnik::Map m(256,256); + m.set_background(mapnik::color("black")); - mapnik::feature_type_style s; - { - mapnik::rule r; - mapnik::polygon_symbolizer sym; - mapnik::put(sym, mapnik::keys::fill, expected_color); - mapnik::put(sym, mapnik::keys::clip, false); - r.append(std::move(sym)); - s.add_rule(std::move(r)); - } - m.insert_style("style",std::move(s)); - - mapnik::layer lyr("Layer"); - lyr.styles().emplace_back("style"); - { - auto ds = std::make_shared(mapnik::parameters()); - auto context = std::make_shared(); - auto f = std::make_shared(context, 0); - REQUIRE(mapnik::json::from_geojson(geojson, *f)); - ds->push(f); - lyr.set_datasource(ds); - } - m.add_layer(std::move(lyr)); - - // 17/20864/45265.png - m.zoom_to_box(mapnik::box2d(-13658379.710221574,6197514.253362091,-13657768.213995293,6198125.749588372)); - - // works 15/5216/11316.png - //m.zoom_to_box(mapnik::box2d(-13658379.710221574,6195679.764683247,-13655933.72531645,6198125.749588372)); - - mapnik::image_rgba8 im(256, 256); - { - mapnik::agg_renderer ren(m, im); - ren.apply(); - } - - REQUIRE(im(128,128) == expected_color.rgba()); - - } catch (std::exception const &e) { - std::clog << e.what() << std::endl; - REQUIRE(false); + mapnik::feature_type_style s; + { + mapnik::rule r; + mapnik::polygon_symbolizer sym; + mapnik::put(sym, mapnik::keys::fill, expected_color); + mapnik::put(sym, mapnik::keys::clip, false); + r.append(std::move(sym)); + s.add_rule(std::move(r)); } + m.insert_style("style",std::move(s)); + + mapnik::layer lyr("Layer"); + lyr.styles().emplace_back("style"); + { + auto ds = std::make_shared(mapnik::parameters()); + auto context = std::make_shared(); + auto f = std::make_shared(context, 0); + REQUIRE(mapnik::json::from_geojson(geojson, *f)); + ds->push(f); + lyr.set_datasource(ds); + } + m.add_layer(std::move(lyr)); + + // 17/20864/45265.png + m.zoom_to_box(mapnik::box2d(-13658379.710221574,6197514.253362091,-13657768.213995293,6198125.749588372)); + + // works 15/5216/11316.png + //m.zoom_to_box(mapnik::box2d(-13658379.710221574,6195679.764683247,-13655933.72531645,6198125.749588372)); + + mapnik::image_rgba8 im(256, 256); + { + mapnik::agg_renderer ren(m, im); + ren.apply(); + } + + REQUIRE(im(128,128) == expected_color.rgba()); } // SECTION } // TEST_CASE From 369887428e82d923a8baee77cdae57adcdc5c33b Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 18 May 2015 22:40:34 +0100 Subject: [PATCH 7/9] Use REQUIRE_THROWS instead of manually checking with try/catch. --- test/standalone/csv_test.cpp | 44 +++++------------------------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/test/standalone/csv_test.cpp b/test/standalone/csv_test.cpp index 86b94e247..2a164083c 100644 --- a/test/standalone/csv_test.cpp +++ b/test/standalone/csv_test.cpp @@ -189,16 +189,7 @@ TEST_CASE("csv") { broken.emplace_back("test/data/csv/fails/does_not_exist.csv"); for (auto const &path : broken) { - bool threw = false; - - try { - auto ds = get_csv_ds(path.native()); - - } catch (const std::exception &e) { - threw = true; - } - - REQUIRE(threw); + REQUIRE_THROWS(get_csv_ds(path.native())); } } } // END SECTION @@ -210,18 +201,9 @@ TEST_CASE("csv") { add_csv_files("test/data/csv/warns", good); for (auto const &path : good) { - bool threw = false; - - try { - auto ds = get_csv_ds(path.native(), false); - // require a non-null pointer returned - REQUIRE(bool(ds)); - - } catch (const std::exception &e) { - threw = true; - } - - REQUIRE(!threw); + auto ds = get_csv_ds(path.native(), false); + // require a non-null pointer returned + REQUIRE(bool(ds)); } } } // END SECTION @@ -484,15 +466,7 @@ TEST_CASE("csv") { // also add an invalid one, triggering throw query.add_property_name("bogus"); - bool threw = false; - try { - ds->features(query); - - } catch (std::exception const &) { - threw = true; - } - - CHECK(threw); + REQUIRE_THROWS(ds->features(query)); } // END SECTION SECTION("leading zeros mean strings") { @@ -590,13 +564,7 @@ TEST_CASE("csv") { } // END SECTION SECTION("fewer headers than rows throws") { - bool threw = false; - try { - get_csv_ds("test/data/csv/more_column_values_than_headers.csv"); - } catch (std::exception const &) { - threw = true; - } - CHECK(threw); + REQUIRE_THROWS(get_csv_ds("test/data/csv/more_column_values_than_headers.csv")); } // END SECTION SECTION("feature ID only incremented for valid rows") { From 782a9495436d5c7dd6b00f52d943ac720624a38b Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Mon, 18 May 2015 22:45:52 +0100 Subject: [PATCH 8/9] Use approx when testing floating point approximate equality. --- test/standalone/csv_test.cpp | 2 +- test/unit/core/box2d_test.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/standalone/csv_test.cpp b/test/standalone/csv_test.cpp index 2a164083c..242bb1074 100644 --- a/test/standalone/csv_test.cpp +++ b/test/standalone/csv_test.cpp @@ -641,7 +641,7 @@ TEST_CASE("csv") { for (double d : { .0, +.0, 1e-06, -1e-06, 0.000001, 1.234e+16, 1.234e+16 }) { auto feature = fs->next(); REQUIRE(bool(feature)); - CHECK(feature->get("floats") == d); + CHECK(feature->get("floats").get() == Approx(d)); } } // END SECTION diff --git a/test/unit/core/box2d_test.cpp b/test/unit/core/box2d_test.cpp index c51980794..0ed32f851 100644 --- a/test/unit/core/box2d_test.cpp +++ b/test/unit/core/box2d_test.cpp @@ -98,10 +98,10 @@ SECTION("envelope multiplication") { auto a = mapnik::box2d(100.0, 100.0, 100.001, 100.001); a *= 5; - REQUIRE(std::abs(a.minx() - 99.9980) < 0.001); - REQUIRE(std::abs(a.miny() - 99.9980) < 0.001); - REQUIRE(std::abs(a.maxx() - 100.0030) < 0.001); - REQUIRE(std::abs(a.maxy() - 100.0030) < 0.001); + REQUIRE(a.minx() == Approx( 99.9980)); + REQUIRE(a.miny() == Approx( 99.9980)); + REQUIRE(a.maxx() == Approx(100.0030)); + REQUIRE(a.maxy() == Approx(100.0030)); } { From 0db207d07659e41221dc9aaaba8b0639b49d253e Mon Sep 17 00:00:00 2001 From: Matt Amos Date: Tue, 19 May 2015 00:33:46 +0100 Subject: [PATCH 9/9] Bump submodule hash for new test data file. --- test/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/data b/test/data index 6802b2795..419f95563 160000 --- a/test/data +++ b/test/data @@ -1 +1 @@ -Subproject commit 6802b27950218d7f620ffb7e73c5213aa003ea3c +Subproject commit 419f955634dec99fe1417b6e7092d608803e0f90