diff --git a/README.asciidoc b/README.asciidoc index 82775d6b..4932a3f2 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -46,8 +46,13 @@ Options Sets the maximum size of the cache in bytes. The default value is 1073741824 (1 GiB). -Environment Variables -~~~~~~~~~~~~~~~~~~~~~ +Configuration +~~~~~~~~~~~~~ + +Following values are read from Environment variables, and if absent, from either (in that order): + - (current-working-dir)\clcache.conf + - %HOME%\.clcache\clcache.conf + - %ALLUSERSPROFILE%\.clcache\clcache.conf CLCACHE_DIR:: If set, points to the directory within which all the cached object files diff --git a/clcache/__main__.py b/clcache/__main__.py index daa77fe0..dbd5563f 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -24,7 +24,7 @@ import sys import threading from tempfile import TemporaryFile -from typing import Any, List, Tuple, Iterator +from typing import Any, Dict, List, Tuple, Iterator VERSION = "4.1.0-dev" @@ -308,6 +308,50 @@ def getIncludesContentHashForHashes(listOfHashes): return HashAlgorithm(','.join(listOfHashes).encode()).hexdigest() +class GlobalSettings: + """ Implements a common place to obain settings from. """ + @staticmethod + def getValue(settingName, defaultValue=None): + value = os.environ.get(settingName, None) + if value is None: # compare to None to allow empty values + value = GlobalSettings._getFromCache(settingName) + return value if value is not None else defaultValue + + # serves as a cache to only read the config file once + _cache = {} # type: Dict[str, str] + + @staticmethod + def _getFromCache(settingName): + if not GlobalSettings._cache: + GlobalSettings._readFromFile() + return GlobalSettings._cache.get(settingName, None) + + @staticmethod + def _readFromFile(): + GlobalSettings._cache['dummy'] = 'dummy' # so that _readFromFile is only called once + + # prefer config in current directory + filename = os.path.join(os.getcwd(), "clcache.conf") + + # ..or in home directory.. + if not os.path.exists(filename): + filename = os.path.join(os.path.expanduser("~"), ".clcache", "clcache.conf") + + # or in "sysconfdir" (%ALLUSERSPROFILE%) + if not os.path.exists(filename): + dirname = os.environ.get('ALLUSERSPROFILE', None) + filename = os.path.join(dirname if dirname else "C:\\Users", ".clcache", "clcache.conf") + try: + with open(filename) as f: + for line in f.readlines(): + kv = line.split("=") + if len(kv) != 2 or kv[0].startswith("#"): + continue + GlobalSettings._cache[kv[0].strip()] = kv[1].split("#")[0].strip() + except IOError: + pass # only ignore file access errors (including not-existing path) + + class CacheLock: """ Implements a lock for the object cache which can be used in 'with' statements. """ @@ -359,7 +403,7 @@ def release(self): @staticmethod def forPath(path): - timeoutMs = int(os.environ.get('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000)) + timeoutMs = int(GlobalSettings.getValue('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000)) lockName = path.replace(':', '-').replace('\\', '-') return CacheLock(lockName, timeoutMs) @@ -505,10 +549,8 @@ class CacheFileStrategy: def __init__(self, cacheDirectory=None): self.dir = cacheDirectory if not self.dir: - try: - self.dir = os.environ["CLCACHE_DIR"] - except KeyError: - self.dir = os.path.join(os.path.expanduser("~"), "clcache") + self.dir = GlobalSettings.getValue("CLCACHE_DIR", + os.path.join(os.path.expanduser("~"), "clcache")) manifestsRootDir = os.path.join(self.dir, "manifests") ensureDirectoryExists(manifestsRootDir) @@ -593,9 +635,10 @@ def clean(self, stats, maximumSize): class Cache: def __init__(self, cacheDirectory=None): - if os.environ.get("CLCACHE_MEMCACHED"): + memcached = GlobalSettings.getValue("CLCACHE_MEMCACHED") + if memcached and memcached not in ['0', 'false', 'False']: from .storage import CacheFileWithMemcacheFallbackStrategy - self.strategy = CacheFileWithMemcacheFallbackStrategy(os.environ.get("CLCACHE_MEMCACHED"), + self.strategy = CacheFileWithMemcacheFallbackStrategy(memcached, cacheDirectory=cacheDirectory) else: self.strategy = CacheFileStrategy(cacheDirectory=cacheDirectory) @@ -900,7 +943,8 @@ def getCompilerHash(compilerBinary): def getFileHashes(filePaths): - if 'CLCACHE_SERVER' in os.environ: + server = GlobalSettings.getValue('CLCACHE_SERVER') + if server and server not in ['0', 'false', 'False']: pipeName = r'\\.\pipe\clcache_srv' while True: try: @@ -939,7 +983,7 @@ def getStringHash(dataString): def expandBasedirPlaceholder(path): - baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR')) + baseDir = normalizeBaseDir(GlobalSettings.getValue('CLCACHE_BASEDIR')) if path.startswith(BASEDIR_REPLACEMENT): if not baseDir: raise LogicException('No CLCACHE_BASEDIR set, but found relative path ' + path) @@ -949,7 +993,7 @@ def expandBasedirPlaceholder(path): def collapseBasedirToPlaceholder(path): - baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR')) + baseDir = normalizeBaseDir(GlobalSettings.getValue('CLCACHE_BASEDIR')) if baseDir is None: return path else: @@ -971,8 +1015,8 @@ def ensureDirectoryExists(path): def copyOrLink(srcFilePath, dstFilePath): ensureDirectoryExists(os.path.dirname(os.path.abspath(dstFilePath))) - - if "CLCACHE_HARDLINK" in os.environ: + hardlink = GlobalSettings.getValue("CLCACHE_HARDLINK") + if hardlink and hardlink not in ['0', 'false', 'False']: ret = windll.kernel32.CreateHardLinkW(str(dstFilePath), str(srcFilePath), None) if ret != 0: # Touch the time stamp of the new link so that the build system @@ -998,11 +1042,10 @@ def myExecutablePath(): def findCompilerBinary(): - if "CLCACHE_CL" in os.environ: - path = os.environ["CLCACHE_CL"] + path = GlobalSettings.getValue("CLCACHE_CL") + if path: if os.path.basename(path) == path: path = which(path) - return path if os.path.exists(path) else None frozenByPy2Exe = hasattr(sys, "frozen") @@ -1020,7 +1063,8 @@ def findCompilerBinary(): def printTraceStatement(msg: str) -> None: - if "CLCACHE_LOG" in os.environ: + clcachelog = GlobalSettings.getValue("CLCACHE_LOG") + if clcachelog and clcachelog not in ['0', 'false', 'False']: scriptDir = os.path.realpath(os.path.dirname(sys.argv[0])) with OUTPUT_LOCK: print(os.path.join(scriptDir, "clcache.py") + " " + msg) @@ -1570,8 +1614,8 @@ def main(): printTraceStatement("Found real compiler binary at '{0!s}'".format(compiler)) printTraceStatement("Arguments we care about: '{}'".format(sys.argv)) - - if "CLCACHE_DISABLE" in os.environ: + enabled = GlobalSettings.getValue("CLCACHE_DISABLE") + if enabled and enabled not in ['0', 'false', 'False']: return invokeRealCompiler(compiler, sys.argv[1:])[0] try: return processCompileRequest(cache, compiler, sys.argv) @@ -1670,8 +1714,8 @@ def processSingleSource(compiler, cmdLine, sourceFile, objectFile, environment): try: assert objectFile is not None cache = Cache() - - if 'CLCACHE_NODIRECT' in os.environ: + nodirect = GlobalSettings.getValue('CLCACHE_NODIRECT') + if nodirect and nodirect not in ['0', 'false', 'False']: return processNoDirect(cache, objectFile, compiler, cmdLine, environment) else: return processDirect(cache, objectFile, compiler, cmdLine, sourceFile) @@ -1770,7 +1814,8 @@ def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, ex if __name__ == '__main__': - if 'CLCACHE_PROFILE' in os.environ: + CLCACHE_PROFILE_ENABLED = GlobalSettings.getValue('CLCACHE_PROFILE') + if CLCACHE_PROFILE_ENABLED and CLCACHE_PROFILE_ENABLED not in ['0', 'false', 'False']: INVOCATION_HASH = getStringHash(','.join(sys.argv)) cProfile.run('main()', filename='clcache-{}.prof'.format(INVOCATION_HASH)) else: diff --git a/tests/test_integration.py b/tests/test_integration.py index ced806ec..109662e9 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -47,6 +47,28 @@ def cd(targetDirectory): os.chdir(oldDirectory) +def executeStatsCommand(customEnv=None): + cmd = CLCACHE_CMD + ["-s"] + if customEnv: + out = subprocess.check_output(cmd, env=customEnv) + else: + out = subprocess.check_output(cmd) + return extractStatsOutput(out.decode("ascii").strip()) + + +def extractStatsOutput(outputLines): + stats = dict() + print(outputLines) + for line in outputLines.splitlines(): + kv = line.split(":", 1) + if len(kv) != 2 or not kv[1]: + continue + stats[kv[0].strip()] = kv[1].strip() + # special case to avoid duplication: Update 'Disc cache at X:\\blah\\ccache' => 'X:\\blah\\ccache' + stats["current cache dir"] = stats["current cache dir"].split("cache at")[1].strip() + return stats + + class TestCommandLineArguments(unittest.TestCase): def testValidMaxSize(self): with tempfile.TemporaryDirectory() as tempDir: @@ -74,6 +96,87 @@ def testPrintStatistics(self): 0, "Command must be able to print statistics") + +class TestGlobalSettings(unittest.TestCase): + def testSettingsDefault(self): + with tempfile.TemporaryDirectory() as tempDir: + customEnv = dict(os.environ, HOME=tempDir) + stats = executeStatsCommand(customEnv) + print(stats) + self.assertEqual(stats["current cache dir"], os.path.join(tempDir, "clcache")) + + def testSettingsEnvironmentVariables(self): + with tempfile.TemporaryDirectory() as tempDir: + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + stats = executeStatsCommand(customEnv) + print(stats) + self.assertEqual(stats["current cache dir"], os.path.join(tempDir)) + + def testSettingsLocalConfigFile(self): + with tempfile.TemporaryDirectory() as tempDir: + with cd(tempDir): + confFileName = os.path.join(tempDir, "clcache.conf") + clcacheDir = os.path.join(tempDir, "clcache") + self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir) + stats = executeStatsCommand() + self.assertEqual(stats["current cache dir"], clcacheDir) + + def testConfigFileInHomeDir(self): + with tempfile.TemporaryDirectory() as tempDir: + confFileName = os.path.join(tempDir, ".clcache", "clcache.conf") + clcacheDir = os.path.join(tempDir, "clcache") + self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir) + customEnv = dict(os.environ, HOME=tempDir) + stats = executeStatsCommand(customEnv) + self.assertEqual(stats["current cache dir"], clcacheDir) + + def testHomeDirOverridenByEnvironment(self): + with tempfile.TemporaryDirectory() as tempDir: + confFileName = os.path.join(tempDir, ".clcache", "clcache.conf") + clcacheDir = os.path.join(tempDir, "clcache") + self._createConfFile(confFileName, CLCACHE_DIR="this should be ignored") + customEnv = dict(os.environ, HOME=tempDir, CLCACHE_DIR=clcacheDir) + stats = executeStatsCommand(customEnv) + self.assertEqual(stats["current cache dir"], clcacheDir) + + def testSettingsConfigFileInProfiles(self): + with tempfile.TemporaryDirectory() as tempDir: + confFileName = os.path.join(tempDir, ".clcache", "clcache.conf") + clcacheDir = os.path.join(tempDir, "clcache") + self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir) + customEnv = dict(os.environ, HOME="blah", ALLUSERSPROFILE=tempDir) + stats = executeStatsCommand(customEnv) + self.assertEqual(stats["current cache dir"], clcacheDir) + + def testConfProfilesOverridenByEnvironment(self): + with tempfile.TemporaryDirectory() as tempDir: + confFileName = os.path.join(tempDir, ".clcache", "clcache.conf") + clcacheDir = os.path.join(tempDir, "clcache") + self._createConfFile(confFileName, CLCACHE_DIR="should be ignored") + customEnv = dict(os.environ, HOME="blah", ALLUSERSPROFILE=tempDir, CLCACHE_DIR=clcacheDir) + stats = executeStatsCommand(customEnv) + self.assertEqual(stats["current cache dir"], clcacheDir) + + def testProfilesOverridenByHomeDir(self): + with tempfile.TemporaryDirectory() as tempDir: + clcacheDir = os.path.join(tempDir, "clcache") + homeDir = os.path.join(tempDir, "home") + self._createConfFile(os.path.join(homeDir, ".clcache", "clcache.conf"), CLCACHE_DIR=clcacheDir) + profilesDir = os.path.join(tempDir, "allusersprofile") + self._createConfFile(os.path.join(profilesDir, ".clcache", "clcache.conf"), CLCACHE_DIR="ignored") + customEnv = dict(os.environ, HOME=homeDir, ALLUSERSPROFILE=profilesDir) + stats = executeStatsCommand(customEnv) + self.assertEqual(stats["current cache dir"], clcacheDir) + + def _createConfFile(self, filename, **settings): + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(filename, "w") as f: + for k, v in settings.items(): + f.write("{0} = {1}\n\r".format(k, v)) + + class TestDistutils(unittest.TestCase): @pytest.mark.skipif(not MONKEY_LOADED, reason="Monkeypatch not loaded") @pytest.mark.skipif(CLCACHE_MEMCACHED, reason="Fails with memcached")