From e5f1e106f8af2252a904c8320745f3bb3b68363c Mon Sep 17 00:00:00 2001 From: Andreas Kleber Date: Mon, 23 May 2016 07:38:37 +0200 Subject: [PATCH 1/4] adds lockfile based locking strategy --- clcache.py | 115 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 108 insertions(+), 7 deletions(-) diff --git a/clcache.py b/clcache.py index d1a3754b..388643f5 100644 --- a/clcache.py +++ b/clcache.py @@ -51,6 +51,9 @@ import sys import multiprocessing import re +import time +import msvcrt +import random VERSION = "3.0.3-dev" @@ -84,10 +87,9 @@ def __str__(self): return repr(self.message) -class ObjectCacheLock: +class ObjectCacheLockMutex: """ Implements a lock for the object cache which can be used in 'with' statements. """ - INFINITE = 0xFFFFFFFF def __init__(self, mutexName, timeoutMs): mutexName = 'Local\\' + mutexName @@ -126,6 +128,80 @@ def release(self): self._acquired = False +class ObjectCacheLockFile: + """ Implements a lock file lock for the object cache which + can be used in 'with' statements. + Inspired by + https://github.com/harlowja/fasteners + https://github.com/benediktschmitt/py-filelock""" + + def __init__(self, lockfileName, timeoutMs): + self._lockfileName = lockfileName + self._lockfile = None + self._timeoutMs = timeoutMs + + def __enter__(self): + self.acquire() + + def __exit__(self, typ, value, traceback): + self.release() + + def __del__(self): + self.release() + + def _acquire(self): + try: + lockfile = open(self._lockfileName, 'a') + except OSError: + lockfile = None + else: + try: + msvcrt.locking(lockfile.fileno(), msvcrt.LK_NBLCK, 1) + except (IOError, OSError): + lockfile.close() + else: + self._lockfile = lockfile + return None + + def _release(self): + if self._lockfile is not None: + lockfile = self._lockfile + self._lockfile = None + msvcrt.locking(lockfile.fileno(), msvcrt.LK_UNLCK, 1) + lockfile.close() + + #The following might fail because another instance already has locked the file. + #This is no problem because the existence of the file does not provide the + #locking but win32 api base file locking mechanism. + try: + os.remove(self._lockfileName) + except OSError: + pass + + return None + + def is_locked(self): + return self._lockfile is not None + + def acquire(self): + start_time = time.time() + while True: + if not self.is_locked(): + self._acquire() + + if self.is_locked(): + break + elif self._timeoutMs >= 0 and time.time() - start_time > self._timeoutMs/1000: + raise ObjectCacheLockException("Timeout waiting for file lock") + else: + poll_intervall = random.uniform(0.01, 0.1) + time.sleep(poll_intervall) + + def release(self): + if self.is_locked(): + self._release() + + class ObjectCache: def __init__(self): try: @@ -142,7 +218,13 @@ def __init__(self): os.makedirs(self.objectsDir) lockName = self.cacheDirectory().replace(':', '-').replace('\\', '-') timeout_ms = int(os.environ.get('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000)) - self.lock = ObjectCacheLock(lockName, timeout_ms) + + cfg = Configuration(self) + if cfg.lockingStrategy() == "File": + lockfileName = os.path.join(self.cacheDirectory(), "cache.lock") + self.lock = ObjectCacheLockFile(lockfileName, timeout_ms) + else: + self.lock = ObjectCacheLockMutex(lockName, timeout_ms) def cacheDirectory(self): return self.dir @@ -342,7 +424,8 @@ def __contains__(self, key): class Configuration: - _defaultValues = {"MaximumCacheSize": 1073741824} # 1 GiB + _defaultValues = {"MaximumCacheSize": 1073741824, # 1 GiB + "LockingStrategy": "Mutex"} def __init__(self, objectCache): self._objectCache = objectCache @@ -358,6 +441,9 @@ def maximumCacheSize(self): def setMaximumCacheSize(self, size): self._cfg["MaximumCacheSize"] = size + def lockingStrategy(self): + return self._cfg["LockingStrategy"] + def save(self): self._cfg.save() @@ -888,6 +974,7 @@ def printStatistics(cache): current cache dir : {} cache size : {:,} bytes maximum cache size : {:,} bytes + locking strategy : {} cache entries : {} cache hits : {} cache misses @@ -905,6 +992,7 @@ def printStatistics(cache): cache.cacheDirectory(), stats.currentCacheSize(), cfg.maximumCacheSize(), + cfg.lockingStrategy(), stats.numCacheEntries(), stats.numCacheHits(), stats.numCacheMisses(), @@ -1165,7 +1253,11 @@ def processCompileRequest(cache, compiler, args): def processDirect(cache, outputFile, compiler, cmdLine, sourceFile): manifestHash = ObjectCache.getManifestHash(compiler, cmdLine, sourceFile) - with cache.lock: + postProcessing = None + + try: + cache.lock.acquire() + manifest = cache.getManifest(manifestHash) baseDir = os.environ.get('CLCACHE_BASEDIR') if baseDir and not baseDir.endswith(os.path.sep): @@ -1196,9 +1288,18 @@ def processDirect(cache, outputFile, compiler, cmdLine, sourceFile): stripIncludes = True postProcessing = lambda compilerResult: postprocessNoManifestMiss(cache, outputFile, manifestHash, baseDir, compiler, origCmdLine, sourceFile, compilerResult, stripIncludes) + except ObjectCacheLockException: + printTraceStatement("Timeout waiting for lock") + + finally: + cache.lock.release() + compilerResult = invokeRealCompiler(compiler, cmdLine, captureOutput=True) - compilerResult = postProcessing(compilerResult) - printTraceStatement("Finished. Exit code %d" % compilerResult[0]) + + if postProcessing is not None: + compilerResult = postProcessing(compilerResult) + printTraceStatement("Finished. Exit code %d" % compilerResult[0]) + return compilerResult From a23081a47bf0b7c85256e92b1d16d2c8ee3d16c3 Mon Sep 17 00:00:00 2001 From: Andreas Kleber Date: Wed, 25 May 2016 09:19:52 +0200 Subject: [PATCH 2/4] implements review feedback * simplifies code * locking strategy as env variable * documentation --- README.asciidoc | 12 +++++++++ clcache.py | 72 +++++++++++++++++-------------------------------- 2 files changed, 37 insertions(+), 47 deletions(-) diff --git a/README.asciidoc b/README.asciidoc index 2c8d8233..1e311219 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -90,6 +90,18 @@ CLCACHE_OBJECT_CACHE_TIMEOUT_MS:: used by the clcache script. You may override this variable if you are getting ObjectCacheLockExceptions with return code 258 (which is the WAIT_TIMEOUT return code). +CLCACHE_LOCKFILE:: + Enables lockfile based access management to the cache. If this variable is set, + clcache will use a lockfile to ensure that only one process can access the + cache and statistics at a time. + The default of clcache is to use a windows kernel mutex to manage concurrent + access to the cache which only works for processes running on the same host. + If the cache is shared between different hosts this lockfile based access + management can be used. + The exact requirements for lockfiles to work are not available yet, but + they work localy on at least Windows 7 with NTFS and for network shares with + SMB 2.1 (comes with Windows 7). + How clcache works ~~~~~~~~~~~~~~~~~ diff --git a/clcache.py b/clcache.py index 388643f5..14ae4d2e 100644 --- a/clcache.py +++ b/clcache.py @@ -138,7 +138,11 @@ class ObjectCacheLockFile: def __init__(self, lockfileName, timeoutMs): self._lockfileName = lockfileName self._lockfile = None - self._timeoutMs = timeoutMs + + if timeoutMs < 0: + raise ObjectCacheLockException("Timeout needs to be a positive value") + else: + self._timeoutMs = timeoutMs def __enter__(self): self.acquire() @@ -153,49 +157,37 @@ def _acquire(self): try: lockfile = open(self._lockfileName, 'a') except OSError: - lockfile = None - else: - try: - msvcrt.locking(lockfile.fileno(), msvcrt.LK_NBLCK, 1) - except (IOError, OSError): - lockfile.close() - else: - self._lockfile = lockfile - return None + return - def _release(self): - if self._lockfile is not None: - lockfile = self._lockfile - self._lockfile = None - msvcrt.locking(lockfile.fileno(), msvcrt.LK_UNLCK, 1) + try: + msvcrt.locking(lockfile.fileno(), msvcrt.LK_NBLCK, 1) + except (IOError, OSError): lockfile.close() + else: + self._lockfile = lockfile - #The following might fail because another instance already has locked the file. - #This is no problem because the existence of the file does not provide the - #locking but win32 api base file locking mechanism. - try: - os.remove(self._lockfileName) - except OSError: - pass - - return None + def _release(self): + lockfile = self._lockfile + self._lockfile = None + msvcrt.locking(lockfile.fileno(), msvcrt.LK_UNLCK, 1) + lockfile.close() def is_locked(self): return self._lockfile is not None def acquire(self): - start_time = time.time() + startTime = time.time() while True: if not self.is_locked(): self._acquire() if self.is_locked(): break - elif self._timeoutMs >= 0 and time.time() - start_time > self._timeoutMs/1000: + elif time.time() - startTime > self._timeoutMs/1000: raise ObjectCacheLockException("Timeout waiting for file lock") else: - poll_intervall = random.uniform(0.01, 0.1) - time.sleep(poll_intervall) + pollDelay = random.uniform(0.01, 0.1) + time.sleep(pollDelay) def release(self): if self.is_locked(): @@ -219,8 +211,7 @@ def __init__(self): lockName = self.cacheDirectory().replace(':', '-').replace('\\', '-') timeout_ms = int(os.environ.get('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000)) - cfg = Configuration(self) - if cfg.lockingStrategy() == "File": + if "CLCACHE_LOCKFILE" in os.environ: lockfileName = os.path.join(self.cacheDirectory(), "cache.lock") self.lock = ObjectCacheLockFile(lockfileName, timeout_ms) else: @@ -424,8 +415,7 @@ def __contains__(self, key): class Configuration: - _defaultValues = {"MaximumCacheSize": 1073741824, # 1 GiB - "LockingStrategy": "Mutex"} + _defaultValues = {"MaximumCacheSize": 1073741824} # 1 GiB def __init__(self, objectCache): self._objectCache = objectCache @@ -441,9 +431,6 @@ def maximumCacheSize(self): def setMaximumCacheSize(self, size): self._cfg["MaximumCacheSize"] = size - def lockingStrategy(self): - return self._cfg["LockingStrategy"] - def save(self): self._cfg.save() @@ -974,7 +961,6 @@ def printStatistics(cache): current cache dir : {} cache size : {:,} bytes maximum cache size : {:,} bytes - locking strategy : {} cache entries : {} cache hits : {} cache misses @@ -992,7 +978,6 @@ def printStatistics(cache): cache.cacheDirectory(), stats.currentCacheSize(), cfg.maximumCacheSize(), - cfg.lockingStrategy(), stats.numCacheEntries(), stats.numCacheHits(), stats.numCacheMisses(), @@ -1253,10 +1238,8 @@ def processCompileRequest(cache, compiler, args): def processDirect(cache, outputFile, compiler, cmdLine, sourceFile): manifestHash = ObjectCache.getManifestHash(compiler, cmdLine, sourceFile) - postProcessing = None - try: - cache.lock.acquire() + with cache.lock: manifest = cache.getManifest(manifestHash) baseDir = os.environ.get('CLCACHE_BASEDIR') @@ -1288,17 +1271,12 @@ def processDirect(cache, outputFile, compiler, cmdLine, sourceFile): stripIncludes = True postProcessing = lambda compilerResult: postprocessNoManifestMiss(cache, outputFile, manifestHash, baseDir, compiler, origCmdLine, sourceFile, compilerResult, stripIncludes) - except ObjectCacheLockException: - printTraceStatement("Timeout waiting for lock") - finally: - cache.lock.release() compilerResult = invokeRealCompiler(compiler, cmdLine, captureOutput=True) - if postProcessing is not None: - compilerResult = postProcessing(compilerResult) - printTraceStatement("Finished. Exit code %d" % compilerResult[0]) + compilerResult = postProcessing(compilerResult) + printTraceStatement("Finished. Exit code %d" % compilerResult[0]) return compilerResult From 0b6a6970641499e6c1bcfb283ac69009b1f6f9e3 Mon Sep 17 00:00:00 2001 From: Andreas Kleber Date: Mon, 6 Jun 2016 14:16:27 +0200 Subject: [PATCH 3/4] fixes pollDelay for smb share --- clcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clcache.py b/clcache.py index 14ae4d2e..a7452a56 100644 --- a/clcache.py +++ b/clcache.py @@ -186,7 +186,7 @@ def acquire(self): elif time.time() - startTime > self._timeoutMs/1000: raise ObjectCacheLockException("Timeout waiting for file lock") else: - pollDelay = random.uniform(0.01, 0.1) + pollDelay = random.uniform(0.1, 1.0) time.sleep(pollDelay) def release(self): From 49a0d9541afb87d9948bf840e539ff279e2d784f Mon Sep 17 00:00:00 2001 From: Andreas Kleber Date: Tue, 7 Jun 2016 11:21:45 +0200 Subject: [PATCH 4/4] removes whitespace --- clcache.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/clcache.py b/clcache.py index 6bb0be31..a2067374 100644 --- a/clcache.py +++ b/clcache.py @@ -1286,9 +1286,7 @@ def processCompileRequest(cache, compiler, args): def processDirect(cache, outputFile, compiler, cmdLine, sourceFile): manifestHash = ObjectCache.getManifestHash(compiler, cmdLine, sourceFile) - with cache.lock: - manifest = cache.getManifest(manifestHash) baseDir = os.environ.get('CLCACHE_BASEDIR') if baseDir and not baseDir.endswith(os.path.sep): @@ -1319,13 +1317,9 @@ def processDirect(cache, outputFile, compiler, cmdLine, sourceFile): stripIncludes = True postProcessing = lambda compilerResult: postprocessNoManifestMiss(cache, outputFile, manifestHash, baseDir, compiler, origCmdLine, sourceFile, compilerResult, stripIncludes) - - compilerResult = invokeRealCompiler(compiler, cmdLine, captureOutput=True) - compilerResult = postProcessing(compilerResult) printTraceStatement("Finished. Exit code %d" % compilerResult[0]) - return compilerResult