diff --git a/README.asciidoc b/README.asciidoc index 2c8d8233..1e311219 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -90,6 +90,18 @@ CLCACHE_OBJECT_CACHE_TIMEOUT_MS:: used by the clcache script. You may override this variable if you are getting ObjectCacheLockExceptions with return code 258 (which is the WAIT_TIMEOUT return code). +CLCACHE_LOCKFILE:: + Enables lockfile based access management to the cache. If this variable is set, + clcache will use a lockfile to ensure that only one process can access the + cache and statistics at a time. + The default of clcache is to use a windows kernel mutex to manage concurrent + access to the cache which only works for processes running on the same host. + If the cache is shared between different hosts this lockfile based access + management can be used. + The exact requirements for lockfiles to work are not available yet, but + they work localy on at least Windows 7 with NTFS and for network shares with + SMB 2.1 (comes with Windows 7). + How clcache works ~~~~~~~~~~~~~~~~~ diff --git a/clcache.py b/clcache.py index 4187a030..a2067374 100644 --- a/clcache.py +++ b/clcache.py @@ -51,6 +51,9 @@ import sys import multiprocessing import re +import time +import msvcrt +import random VERSION = "3.0.3-dev" @@ -84,10 +87,9 @@ def __str__(self): return repr(self.message) -class ObjectCacheLock: +class ObjectCacheLockMutex: """ Implements a lock for the object cache which can be used in 'with' statements. """ - INFINITE = 0xFFFFFFFF def __init__(self, mutexName, timeoutMs): mutexName = 'Local\\' + mutexName @@ -126,6 +128,72 @@ def release(self): self._acquired = False +class ObjectCacheLockFile: + """ Implements a lock file lock for the object cache which + can be used in 'with' statements. + Inspired by + https://github.com/harlowja/fasteners + https://github.com/benediktschmitt/py-filelock""" + + def __init__(self, lockfileName, timeoutMs): + self._lockfileName = lockfileName + self._lockfile = None + + if timeoutMs < 0: + raise ObjectCacheLockException("Timeout needs to be a positive value") + else: + self._timeoutMs = timeoutMs + + def __enter__(self): + self.acquire() + + def __exit__(self, typ, value, traceback): + self.release() + + def __del__(self): + self.release() + + def _acquire(self): + try: + lockfile = open(self._lockfileName, 'a') + except OSError: + return + + try: + msvcrt.locking(lockfile.fileno(), msvcrt.LK_NBLCK, 1) + except (IOError, OSError): + lockfile.close() + else: + self._lockfile = lockfile + + def _release(self): + lockfile = self._lockfile + self._lockfile = None + msvcrt.locking(lockfile.fileno(), msvcrt.LK_UNLCK, 1) + lockfile.close() + + def is_locked(self): + return self._lockfile is not None + + def acquire(self): + startTime = time.time() + while True: + if not self.is_locked(): + self._acquire() + + if self.is_locked(): + break + elif time.time() - startTime > self._timeoutMs/1000: + raise ObjectCacheLockException("Timeout waiting for file lock") + else: + pollDelay = random.uniform(0.1, 1.0) + time.sleep(pollDelay) + + def release(self): + if self.is_locked(): + self._release() + + class ObjectCache: def __init__(self): try: @@ -142,7 +210,12 @@ def __init__(self): os.makedirs(self.objectsDir) lockName = self.cacheDirectory().replace(':', '-').replace('\\', '-') timeout_ms = int(os.environ.get('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000)) - self.lock = ObjectCacheLock(lockName, timeout_ms) + + if "CLCACHE_LOCKFILE" in os.environ: + lockfileName = os.path.join(self.cacheDirectory(), "cache.lock") + self.lock = ObjectCacheLockFile(lockfileName, timeout_ms) + else: + self.lock = ObjectCacheLockMutex(lockName, timeout_ms) def cacheDirectory(self): return self.dir