diff --git a/tests/visual_tests/compare.py b/tests/visual_tests/compare.py
new file mode 100644
index 000000000..18dfffc74
--- /dev/null
+++ b/tests/visual_tests/compare.py
@@ -0,0 +1,43 @@
+import math, operator
+import Image
+import sys
+
+COMPUTE_THRESHOLD = 16
+
+errors = []
+
+# returns true if pixels are not identical
+def compare_pixels(pixel1, pixel2):
+    r_diff = abs(pixel1[0] - pixel2[0])
+    g_diff = abs(pixel1[1] - pixel2[1])
+    b_diff = abs(pixel1[2] - pixel2[2])
+    if(r_diff > COMPUTE_THRESHOLD or g_diff > COMPUTE_THRESHOLD or b_diff > COMPUTE_THRESHOLD):
+        return True
+    else:
+        return False
+
+# compare tow images and return number of different pixels
+def compare(fn1, fn2):
+    global errors
+    im1 = Image.open(fn1)
+    im2 = Image.open(fn2)
+    diff = 0
+    pixels = im1.size[0] * im1.size[1]
+    im1 = im1.getdata()
+    im2 = im2.getdata()
+    for i in range(3, pixels - 1, 3):
+        if(compare_pixels(im1[i], im2[i])):
+            diff = diff + 1
+    if diff != 0:
+        errors.append((fn1, diff))
+    return diff
+
+def summary():
+    global errors
+    if len(errors) != 0:
+        print "-"*80
+        print "Summary:"
+        for error in errors:
+            print "%s failed: %d different pixels" % error
+        print "-"*80
+        sys.exit(1)
diff --git a/tests/visual_tests/test.py b/tests/visual_tests/test.py
index d9b11e957..8a3cf78f0 100755
--- a/tests/visual_tests/test.py
+++ b/tests/visual_tests/test.py
@@ -5,8 +5,7 @@ import mapnik
 import cairo
 import sys
 import os.path
-import math, operator
-import Image
+from compare import compare, summary
 
 dirname = os.path.dirname(sys.argv[0])
 
@@ -17,38 +16,14 @@ filenames_one_width = ["simple-E", "simple-NE", "simple-NW", "simple-N",
     "formating-1", "formating-2", "formating-3", "formating-4",
     "shieldsymbolizer-1", "expressionformat"]
 
-COMPUTE_THRESHOLD = 0
-
-# returns true if pixels are not identical
-def compare_pixels(pixel1, pixel2):
-    r_diff = abs(pixel1[0] - pixel2[0])
-    g_diff = abs(pixel1[1] - pixel2[1])
-    b_diff = abs(pixel1[2] - pixel2[2])
-    if(r_diff > COMPUTE_THRESHOLD or g_diff > COMPUTE_THRESHOLD or b_diff > COMPUTE_THRESHOLD):
-        return True
-    else:
-        return False
-
-# compare tow images and return number of different pixels
-def compare(im1, im2):
-    diff = 0
-    pixels = im1.size[0] * im1.size[1]
-    im1 = im1.getdata()
-    im2 = im2.getdata()
-    for i in range(3, pixels - 1, 3):
-        if(compare_pixels(im1[i], im2[i])):
-            diff = diff + 1
-    return diff
-
 def render(filename, width):
     m = mapnik.Map(width, 100)
     mapnik.load_map(m, os.path.join(dirname, "%s.xml" % filename), False)
     bbox = mapnik.Box2d(-0.05, -0.01, 0.95, 0.01)
     m.zoom_to_box(bbox)
-    mapnik.render_to_file(m, '%s-%d-agg.png' % (filename, width))
-    im1 = Image.open('%s-%d-reference.png' % (filename, width))
-    im2 = Image.open('%s-%d-agg.png' % (filename, width))
-    diff = compare(im1, im2)
+    basefn = '%s-%d' % (filename, width)
+    mapnik.render_to_file(m, basefn+'-agg.png')
+    diff = compare(basefn + '-agg.png', basefn + '-reference.png')
     if diff == 0:
         rms = 'ok'
     else:
@@ -70,3 +45,5 @@ for filename in filenames_one_width:
     m = render(filename, 500)
     mapnik.save_map(m, "%s-out.xml" % filename)
 
+summary()
+