From 40b5736934a23f835946bfe41f9a1e6d3970981b Mon Sep 17 00:00:00 2001 From: Lukasz Forynski Date: Tue, 4 Sep 2018 22:20:28 +0100 Subject: [PATCH] Add support for loading config from a clcache.conf file This is useful and the GCC version of ccache also implements a similar concept. It might be useful in situations where some default settings could be specified or when environment is not passed correctly down to the compiler chain. This change introduces support for clcache.conf file to allow clcache to read settings from there. The precedence is as follows: 1. If present, Environment variable(s) are used, or 2. [current_working_dir]\clcache.conf is loaded, or, if not found 3. [%HOME% or ~\].clcache\clcache.conf is be loaded, or if not found 4. [%ALLUSERSPROFILE% or C:\Users\].clcache\clcache.conf is loaded. In each case, once a clcache.conf file is found, no other conf file is considered and values only from this file are used. It is also loaded only once (and all its values then cached) - all to avoid unnecessary performance penalties. --- README.asciidoc | 9 +++- clcache/__main__.py | 89 ++++++++++++++++++++++++-------- tests/test_integration.py | 103 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 24 deletions(-) diff --git a/README.asciidoc b/README.asciidoc index 82775d6b..4932a3f2 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -46,8 +46,13 @@ Options Sets the maximum size of the cache in bytes. The default value is 1073741824 (1 GiB). -Environment Variables -~~~~~~~~~~~~~~~~~~~~~ +Configuration +~~~~~~~~~~~~~ + +Following values are read from Environment variables, and if absent, from either (in that order): + - (current-working-dir)\clcache.conf + - %HOME%\.clcache\clcache.conf + - %ALLUSERSPROFILE%\.clcache\clcache.conf CLCACHE_DIR:: If set, points to the directory within which all the cached object files diff --git a/clcache/__main__.py b/clcache/__main__.py index daa77fe0..dbd5563f 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -24,7 +24,7 @@ import sys import threading from tempfile import TemporaryFile -from typing import Any, List, Tuple, Iterator +from typing import Any, Dict, List, Tuple, Iterator VERSION = "4.1.0-dev" @@ -308,6 +308,50 @@ def getIncludesContentHashForHashes(listOfHashes): return HashAlgorithm(','.join(listOfHashes).encode()).hexdigest() +class GlobalSettings: + """ Implements a common place to obain settings from. """ + @staticmethod + def getValue(settingName, defaultValue=None): + value = os.environ.get(settingName, None) + if value is None: # compare to None to allow empty values + value = GlobalSettings._getFromCache(settingName) + return value if value is not None else defaultValue + + # serves as a cache to only read the config file once + _cache = {} # type: Dict[str, str] + + @staticmethod + def _getFromCache(settingName): + if not GlobalSettings._cache: + GlobalSettings._readFromFile() + return GlobalSettings._cache.get(settingName, None) + + @staticmethod + def _readFromFile(): + GlobalSettings._cache['dummy'] = 'dummy' # so that _readFromFile is only called once + + # prefer config in current directory + filename = os.path.join(os.getcwd(), "clcache.conf") + + # ..or in home directory.. + if not os.path.exists(filename): + filename = os.path.join(os.path.expanduser("~"), ".clcache", "clcache.conf") + + # or in "sysconfdir" (%ALLUSERSPROFILE%) + if not os.path.exists(filename): + dirname = os.environ.get('ALLUSERSPROFILE', None) + filename = os.path.join(dirname if dirname else "C:\\Users", ".clcache", "clcache.conf") + try: + with open(filename) as f: + for line in f.readlines(): + kv = line.split("=") + if len(kv) != 2 or kv[0].startswith("#"): + continue + GlobalSettings._cache[kv[0].strip()] = kv[1].split("#")[0].strip() + except IOError: + pass # only ignore file access errors (including not-existing path) + + class CacheLock: """ Implements a lock for the object cache which can be used in 'with' statements. """ @@ -359,7 +403,7 @@ def release(self): @staticmethod def forPath(path): - timeoutMs = int(os.environ.get('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000)) + timeoutMs = int(GlobalSettings.getValue('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000)) lockName = path.replace(':', '-').replace('\\', '-') return CacheLock(lockName, timeoutMs) @@ -505,10 +549,8 @@ class CacheFileStrategy: def __init__(self, cacheDirectory=None): self.dir = cacheDirectory if not self.dir: - try: - self.dir = os.environ["CLCACHE_DIR"] - except KeyError: - self.dir = os.path.join(os.path.expanduser("~"), "clcache") + self.dir = GlobalSettings.getValue("CLCACHE_DIR", + os.path.join(os.path.expanduser("~"), "clcache")) manifestsRootDir = os.path.join(self.dir, "manifests") ensureDirectoryExists(manifestsRootDir) @@ -593,9 +635,10 @@ def clean(self, stats, maximumSize): class Cache: def __init__(self, cacheDirectory=None): - if os.environ.get("CLCACHE_MEMCACHED"): + memcached = GlobalSettings.getValue("CLCACHE_MEMCACHED") + if memcached and memcached not in ['0', 'false', 'False']: from .storage import CacheFileWithMemcacheFallbackStrategy - self.strategy = CacheFileWithMemcacheFallbackStrategy(os.environ.get("CLCACHE_MEMCACHED"), + self.strategy = CacheFileWithMemcacheFallbackStrategy(memcached, cacheDirectory=cacheDirectory) else: self.strategy = CacheFileStrategy(cacheDirectory=cacheDirectory) @@ -900,7 +943,8 @@ def getCompilerHash(compilerBinary): def getFileHashes(filePaths): - if 'CLCACHE_SERVER' in os.environ: + server = GlobalSettings.getValue('CLCACHE_SERVER') + if server and server not in ['0', 'false', 'False']: pipeName = r'\\.\pipe\clcache_srv' while True: try: @@ -939,7 +983,7 @@ def getStringHash(dataString): def expandBasedirPlaceholder(path): - baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR')) + baseDir = normalizeBaseDir(GlobalSettings.getValue('CLCACHE_BASEDIR')) if path.startswith(BASEDIR_REPLACEMENT): if not baseDir: raise LogicException('No CLCACHE_BASEDIR set, but found relative path ' + path) @@ -949,7 +993,7 @@ def expandBasedirPlaceholder(path): def collapseBasedirToPlaceholder(path): - baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR')) + baseDir = normalizeBaseDir(GlobalSettings.getValue('CLCACHE_BASEDIR')) if baseDir is None: return path else: @@ -971,8 +1015,8 @@ def ensureDirectoryExists(path): def copyOrLink(srcFilePath, dstFilePath): ensureDirectoryExists(os.path.dirname(os.path.abspath(dstFilePath))) - - if "CLCACHE_HARDLINK" in os.environ: + hardlink = GlobalSettings.getValue("CLCACHE_HARDLINK") + if hardlink and hardlink not in ['0', 'false', 'False']: ret = windll.kernel32.CreateHardLinkW(str(dstFilePath), str(srcFilePath), None) if ret != 0: # Touch the time stamp of the new link so that the build system @@ -998,11 +1042,10 @@ def myExecutablePath(): def findCompilerBinary(): - if "CLCACHE_CL" in os.environ: - path = os.environ["CLCACHE_CL"] + path = GlobalSettings.getValue("CLCACHE_CL") + if path: if os.path.basename(path) == path: path = which(path) - return path if os.path.exists(path) else None frozenByPy2Exe = hasattr(sys, "frozen") @@ -1020,7 +1063,8 @@ def findCompilerBinary(): def printTraceStatement(msg: str) -> None: - if "CLCACHE_LOG" in os.environ: + clcachelog = GlobalSettings.getValue("CLCACHE_LOG") + if clcachelog and clcachelog not in ['0', 'false', 'False']: scriptDir = os.path.realpath(os.path.dirname(sys.argv[0])) with OUTPUT_LOCK: print(os.path.join(scriptDir, "clcache.py") + " " + msg) @@ -1570,8 +1614,8 @@ def main(): printTraceStatement("Found real compiler binary at '{0!s}'".format(compiler)) printTraceStatement("Arguments we care about: '{}'".format(sys.argv)) - - if "CLCACHE_DISABLE" in os.environ: + enabled = GlobalSettings.getValue("CLCACHE_DISABLE") + if enabled and enabled not in ['0', 'false', 'False']: return invokeRealCompiler(compiler, sys.argv[1:])[0] try: return processCompileRequest(cache, compiler, sys.argv) @@ -1670,8 +1714,8 @@ def processSingleSource(compiler, cmdLine, sourceFile, objectFile, environment): try: assert objectFile is not None cache = Cache() - - if 'CLCACHE_NODIRECT' in os.environ: + nodirect = GlobalSettings.getValue('CLCACHE_NODIRECT') + if nodirect and nodirect not in ['0', 'false', 'False']: return processNoDirect(cache, objectFile, compiler, cmdLine, environment) else: return processDirect(cache, objectFile, compiler, cmdLine, sourceFile) @@ -1770,7 +1814,8 @@ def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, ex if __name__ == '__main__': - if 'CLCACHE_PROFILE' in os.environ: + CLCACHE_PROFILE_ENABLED = GlobalSettings.getValue('CLCACHE_PROFILE') + if CLCACHE_PROFILE_ENABLED and CLCACHE_PROFILE_ENABLED not in ['0', 'false', 'False']: INVOCATION_HASH = getStringHash(','.join(sys.argv)) cProfile.run('main()', filename='clcache-{}.prof'.format(INVOCATION_HASH)) else: diff --git a/tests/test_integration.py b/tests/test_integration.py index ced806ec..109662e9 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -47,6 +47,28 @@ def cd(targetDirectory): os.chdir(oldDirectory) +def executeStatsCommand(customEnv=None): + cmd = CLCACHE_CMD + ["-s"] + if customEnv: + out = subprocess.check_output(cmd, env=customEnv) + else: + out = subprocess.check_output(cmd) + return extractStatsOutput(out.decode("ascii").strip()) + + +def extractStatsOutput(outputLines): + stats = dict() + print(outputLines) + for line in outputLines.splitlines(): + kv = line.split(":", 1) + if len(kv) != 2 or not kv[1]: + continue + stats[kv[0].strip()] = kv[1].strip() + # special case to avoid duplication: Update 'Disc cache at X:\\blah\\ccache' => 'X:\\blah\\ccache' + stats["current cache dir"] = stats["current cache dir"].split("cache at")[1].strip() + return stats + + class TestCommandLineArguments(unittest.TestCase): def testValidMaxSize(self): with tempfile.TemporaryDirectory() as tempDir: @@ -74,6 +96,87 @@ def testPrintStatistics(self): 0, "Command must be able to print statistics") + +class TestGlobalSettings(unittest.TestCase): + def testSettingsDefault(self): + with tempfile.TemporaryDirectory() as tempDir: + customEnv = dict(os.environ, HOME=tempDir) + stats = executeStatsCommand(customEnv) + print(stats) + self.assertEqual(stats["current cache dir"], os.path.join(tempDir, "clcache")) + + def testSettingsEnvironmentVariables(self): + with tempfile.TemporaryDirectory() as tempDir: + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + stats = executeStatsCommand(customEnv) + print(stats) + self.assertEqual(stats["current cache dir"], os.path.join(tempDir)) + + def testSettingsLocalConfigFile(self): + with tempfile.TemporaryDirectory() as tempDir: + with cd(tempDir): + confFileName = os.path.join(tempDir, "clcache.conf") + clcacheDir = os.path.join(tempDir, "clcache") + self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir) + stats = executeStatsCommand() + self.assertEqual(stats["current cache dir"], clcacheDir) + + def testConfigFileInHomeDir(self): + with tempfile.TemporaryDirectory() as tempDir: + confFileName = os.path.join(tempDir, ".clcache", "clcache.conf") + clcacheDir = os.path.join(tempDir, "clcache") + self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir) + customEnv = dict(os.environ, HOME=tempDir) + stats = executeStatsCommand(customEnv) + self.assertEqual(stats["current cache dir"], clcacheDir) + + def testHomeDirOverridenByEnvironment(self): + with tempfile.TemporaryDirectory() as tempDir: + confFileName = os.path.join(tempDir, ".clcache", "clcache.conf") + clcacheDir = os.path.join(tempDir, "clcache") + self._createConfFile(confFileName, CLCACHE_DIR="this should be ignored") + customEnv = dict(os.environ, HOME=tempDir, CLCACHE_DIR=clcacheDir) + stats = executeStatsCommand(customEnv) + self.assertEqual(stats["current cache dir"], clcacheDir) + + def testSettingsConfigFileInProfiles(self): + with tempfile.TemporaryDirectory() as tempDir: + confFileName = os.path.join(tempDir, ".clcache", "clcache.conf") + clcacheDir = os.path.join(tempDir, "clcache") + self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir) + customEnv = dict(os.environ, HOME="blah", ALLUSERSPROFILE=tempDir) + stats = executeStatsCommand(customEnv) + self.assertEqual(stats["current cache dir"], clcacheDir) + + def testConfProfilesOverridenByEnvironment(self): + with tempfile.TemporaryDirectory() as tempDir: + confFileName = os.path.join(tempDir, ".clcache", "clcache.conf") + clcacheDir = os.path.join(tempDir, "clcache") + self._createConfFile(confFileName, CLCACHE_DIR="should be ignored") + customEnv = dict(os.environ, HOME="blah", ALLUSERSPROFILE=tempDir, CLCACHE_DIR=clcacheDir) + stats = executeStatsCommand(customEnv) + self.assertEqual(stats["current cache dir"], clcacheDir) + + def testProfilesOverridenByHomeDir(self): + with tempfile.TemporaryDirectory() as tempDir: + clcacheDir = os.path.join(tempDir, "clcache") + homeDir = os.path.join(tempDir, "home") + self._createConfFile(os.path.join(homeDir, ".clcache", "clcache.conf"), CLCACHE_DIR=clcacheDir) + profilesDir = os.path.join(tempDir, "allusersprofile") + self._createConfFile(os.path.join(profilesDir, ".clcache", "clcache.conf"), CLCACHE_DIR="ignored") + customEnv = dict(os.environ, HOME=homeDir, ALLUSERSPROFILE=profilesDir) + stats = executeStatsCommand(customEnv) + self.assertEqual(stats["current cache dir"], clcacheDir) + + def _createConfFile(self, filename, **settings): + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(filename, "w") as f: + for k, v in settings.items(): + f.write("{0} = {1}\n\r".format(k, v)) + + class TestDistutils(unittest.TestCase): @pytest.mark.skipif(not MONKEY_LOADED, reason="Monkeypatch not loaded") @pytest.mark.skipif(CLCACHE_MEMCACHED, reason="Fails with memcached")