From 5dee576df26b4aeafd182c41b510c4aa67b14b29 Mon Sep 17 00:00:00 2001 From: Dane Springmeyer Date: Tue, 21 Feb 2012 22:37:56 -0500 Subject: [PATCH] hextree: avoid memory corruption in create_palette_rek() by not skipping <3 pixel nodes and avoid potential divide by zero in assign_node_colors() (led to hang on osx) - closes #1087 --- include/mapnik/hextree.hpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/include/mapnik/hextree.hpp b/include/mapnik/hextree.hpp index 8a94293dd..780c42c56 100644 --- a/include/mapnik/hextree.hpp +++ b/include/mapnik/hextree.hpp @@ -394,8 +394,16 @@ private: // clip extreme alfa values void create_palette_rek(std::vector & palette, node * itr) const { - // actually, ignore ones with < 3 pixels - if (itr->count >= 3) + /* + NOTE: previous code did: + + // actually, ignore ones with < 3 pixels + if (itr->count >= 3) + + But this could lead to memory corruption + */ + + if (itr->count > 0) { unsigned count = itr->count; byte a = byte(itr->alphas/float(count)); @@ -475,8 +483,10 @@ private: } tries = 0; // ignore leaves and also nodes with small mean error and not excessive number of pixels - if (((cur_node->reduce_cost / cur_node->pixel_count + 1) * std::log(double(cur_node->pixel_count))) > 15 - && (cur_node->children_count > 0)) + if (cur_node->pixel_count > 0 && + (cur_node->children_count > 0) && + (((cur_node->reduce_cost / cur_node->pixel_count + 1) * std::log(double(cur_node->pixel_count))) > 15) + ) { colors_--; cur_node->count = 0;