Use SHA1 of image file for check_graphic

lib/iris/tests/__init__.py

@@ -34,13 +34,16 @@
 from six.moves import (filter, input, map, range, zip)  # noqa
 import six
 
+import codecs
 import collections
 import contextlib
 import difflib
 import filecmp
 import functools
 import gzip
+import hashlib
 import inspect
+import json
 import io
 import logging
 import os
@@ -620,16 +623,21 @@ def _unique_id(self):
 
         """
         # Obtain a consistent ID for the current test.
-
         # NB. unittest.TestCase.id() returns different values depending on
         # whether the test has been run explicitly, or via test discovery.
         # For example:
         #   python tests/test_plot.py => '__main__.TestContourf.test_tx'
         #   ird -t => 'iris.tests.test_plot.TestContourf.test_tx'
-        bits = self.id().split('.')[-3:]
+        bits = self.id().split('.')
         if bits[0] == '__main__':
-            file_name = os.path.basename(sys.modules['__main__'].__file__)
+            floc = sys.modules['__main__'].__file__
+            path, file_name = os.path.split(os.path.abspath(floc))
             bits[0] = os.path.splitext(file_name)[0]
+            folder, location = os.path.split(path)
+            bits = [location] + bits
+            while location != 'iris':
+                folder, location = os.path.split(folder)
+                bits = [location] + bits
         test_id = '.'.join(bits)
 
         # Derive the sequential assertion ID within the test
@@ -652,21 +660,22 @@ def _ensure_folder(self, path):
             logger.warning('Creating folder: %s', dir_path)
             os.makedirs(dir_path)
 
-    def check_graphic(self, tol=_DEFAULT_IMAGE_TOLERANCE):
-        """Checks the CRC matches for the current matplotlib.pyplot figure, and closes the figure."""
+    def check_graphic(self, tol=None):
+        """
+        Checks the CRC matches for the current matplotlib.pyplot figure,
+        and closes the figure.
+
+        """
 
         unique_id = self._unique_id()
 
         figure = plt.gcf()
+
+        repo_fname = os.path.join(os.path.dirname(__file__), 'results', 'imagerepo.json')
+        with open(repo_fname, 'rb') as fi:
+            repo = json.load(codecs.getreader('utf-8')(fi))
 
         try:
-            expected_fname = os.path.join(os.path.dirname(__file__),
-                                          'results', 'visual_tests',
-                                          unique_id + '.png')
-
-            if not os.path.isdir(os.path.dirname(expected_fname)):
-                os.makedirs(os.path.dirname(expected_fname))
-
             #: The path where the images generated by the tests should go.
             image_output_directory = os.path.join(os.path.dirname(__file__),
                                                   'result_image_comparison')
@@ -695,18 +704,38 @@ def check_graphic(self, tol=_DEFAULT_IMAGE_TOLERANCE):
 
             figure.savefig(result_fname)
 
-            if not os.path.exists(expected_fname):
-                warnings.warn('Created image for test %s' % unique_id)
-                shutil.copy2(result_fname, expected_fname)
+            # XXX: Deal with a new test result i.e. it's not in the repo
 
-            err = mcompare.compare_images(expected_fname, result_fname, tol=tol)
+            # hash the created image using sha1
+            with open(result_fname, 'rb') as res_file:
+                sha1 = hashlib.sha1(res_file.read())
+            if unique_id not in repo:
+                wmsg = 'Created image {} for test {}.'
+                warnings.warn(wmsg.format(result_fname, unique_id))
+            else:
+                uris = repo[unique_id]
+                # Cherry-pick the registered expected hashes from the
+                # test case uri/s.
+                expected = [os.path.splitext(os.path.basename(uri))[0]
+                            for uri in uris]
+
+                if sha1.hexdigest() not in expected:
+                    emsg = 'Actual SHA1 {} not in expected {} for test {}.'
+                    emsg = emsg.format(sha1.hexdigest(), expected, unique_id)
+                    if _DISPLAY_FIGURES:
+                        print('Image comparison would have failed. '
+                              'Message: %s' % emsg)
+                    else:
+                        raise ValueError('Image comparison failed.'
+                                         ' Message: %s' % emsg)
 
+                else:
+                    # There is no difference between the actual and expected
+                    # result, so remove the actual result file.
+                    os.remove(result_fname)
+
             if _DISPLAY_FIGURES:
-                if err:
-                    print('Image comparison would have failed. Message: %s' % err)
                 plt.show()
-            else:
-                assert not err, 'Image comparison failed. Message: %s' % err
 
         finally:
             plt.close()