Merge pull request #235 from siu/deterministicCreateManifestEntry

Deterministic create manifest entry

Author: frerich

clcache.py

@@ -192,7 +192,7 @@ class ManifestRepository(object):
     # invalidation, such that a manifest that was stored using the old format is not
     # interpreted using the new format. Instead the old file will not be touched
     # again due to a new manifest hash and is cleaned away after some time.
-    MANIFEST_FILE_FORMAT_VERSION = 5
+    MANIFEST_FILE_FORMAT_VERSION = 6
 
     def __init__(self, manifestsRootDir):
         self._manifestsRootDir = manifestsRootDir
@@ -1398,11 +1398,13 @@ def processCacheHit(cache, objectFile, cachekey):
 
 
 def createManifestEntry(manifestHash, includePaths):
-    includesWithHash = {path:getFileHash(path) for path in includePaths}
-    includesContentHash = ManifestRepository.getIncludesContentHashForHashes(includesWithHash.values())
+    sortedIncludePaths = sorted(set(includePaths))
+    includeHashes = [getFileHash(path) for path in sortedIncludePaths]
+
+    safeIncludes = [collapseBasedirToPlaceholder(path) for path in sortedIncludePaths]
+    includesContentHash = ManifestRepository.getIncludesContentHashForHashes(includeHashes)
     cachekey = CompilerArtifactsRepository.computeKeyDirect(manifestHash, includesContentHash)
 
-    safeIncludes = [collapseBasedirToPlaceholder(path) for path in includesWithHash.keys()]
     return ManifestEntry(safeIncludes, includesContentHash, cachekey)
 
 

unittests.py

@@ -13,6 +13,7 @@
 import multiprocessing
 import os
 import unittest
+import tempfile
 
 import clcache
 from clcache import (
@@ -939,6 +940,45 @@ def testTouchEntry(self):
         self.assertEqual(TestManifest.entry2, manifest.entries()[0])
 
 
+class TestCreateManifestEntry(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.tempDir = tempfile.TemporaryDirectory()
+        for i in range(10):
+            sampleName = 'sample{}.h'.format(i)
+            filePath = os.path.join(cls.tempDir.name, '{}.h'.format(sampleName))
+            with open(filePath, 'w') as f:
+                f.write('#define {}'.format(sampleName))
+
+        cls.includePaths = list(sorted(clcache.filesBeneath(cls.tempDir.name)))
+        cls.manifestHash = 'ffffffffffffffffffffffffffffffff'
+        cls.expectedManifestEntry = clcache.createManifestEntry(TestCreateManifestEntry.manifestHash,
+                                                                TestCreateManifestEntry.includePaths)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.tempDir.cleanup()
+
+    def assertManifestEntryIsCorrect(self, entry):
+        self.assertEqual(entry.includesContentHash, TestCreateManifestEntry.expectedManifestEntry.includesContentHash)
+        self.assertEqual(entry.objectHash, TestCreateManifestEntry.expectedManifestEntry.objectHash)
+        self.assertEqual(entry.includeFiles, TestCreateManifestEntry.expectedManifestEntry.includeFiles)
+
+    def testIsConsistentWithSameInput(self):
+        entry = clcache.createManifestEntry(TestCreateManifestEntry.manifestHash, TestCreateManifestEntry.includePaths)
+        self.assertManifestEntryIsCorrect(entry)
+
+    def testIsConsistentWithReverseList(self):
+        reversedIncludePaths = list(reversed(TestCreateManifestEntry.includePaths))
+        entry = clcache.createManifestEntry(TestCreateManifestEntry.manifestHash, reversedIncludePaths)
+        self.assertManifestEntryIsCorrect(entry)
+
+    def testIsConsistentWithDuplicateEntries(self):
+        includePathsWithDuplicates = TestCreateManifestEntry.includePaths + TestCreateManifestEntry.includePaths
+        entry = clcache.createManifestEntry(TestCreateManifestEntry.manifestHash, includePathsWithDuplicates)
+        self.assertManifestEntryIsCorrect(entry)
+
+
 if __name__ == '__main__':
     unittest.TestCase.longMessage = True
     unittest.main()