Add support for loading config from a clcache.conf file

README.asciidoc

@@ -46,8 +46,13 @@ Options
     Sets the maximum size of the cache in bytes.
     The default value is 1073741824 (1 GiB).
 
-Environment Variables
-~~~~~~~~~~~~~~~~~~~~~
+Configuration
+~~~~~~~~~~~~~
+
+Following values are read from Environment variables, and if absent, from either (in that order):
+  - (current-working-dir)\clcache.conf
+  - %HOME%\.clcache\clcache.conf
+  - %ALLUSERSPROFILE%\.clcache\clcache.conf
 
 CLCACHE_DIR::
     If set, points to the directory within which all the cached object files

clcache/__main__.py

@@ -24,7 +24,7 @@
 import sys
 import threading
 from tempfile import TemporaryFile
-from typing import Any, List, Tuple, Iterator
+from typing import Any, Dict, List, Tuple, Iterator
 
 VERSION = "4.1.0-dev"
 
@@ -308,6 +308,50 @@ def getIncludesContentHashForHashes(listOfHashes):
         return HashAlgorithm(','.join(listOfHashes).encode()).hexdigest()
 
 
+class GlobalSettings:
+    """ Implements a common place to obain settings from. """
+    @staticmethod
+    def getValue(settingName, defaultValue=None):
+        value = os.environ.get(settingName, None)
+        if value is None: # compare to None to allow empty values
+            value = GlobalSettings._getFromCache(settingName)
+        return value if value is not None else defaultValue
+
+    # serves as a cache to only read the config file once
+    _cache = {} # type: Dict[str, str]
+
+    @staticmethod
+    def _getFromCache(settingName):
+        if not GlobalSettings._cache:
+            GlobalSettings._readFromFile()
+        return GlobalSettings._cache.get(settingName, None)
+
+    @staticmethod
+    def _readFromFile():
+        GlobalSettings._cache['dummy'] = 'dummy' # so that _readFromFile is only called once
+
+        # prefer config in current directory
+        filename = os.path.join(os.getcwd(), "clcache.conf")
+
+        # ..or in home directory..
+        if not os.path.exists(filename):
+            filename = os.path.join(os.path.expanduser("~"), ".clcache", "clcache.conf")
+
+        # or in "sysconfdir" (%ALLUSERSPROFILE%)
+        if not os.path.exists(filename):
+            dirname = os.environ.get('ALLUSERSPROFILE', None)
+            filename = os.path.join(dirname if dirname else "C:\\Users", ".clcache", "clcache.conf")
+        try:
+            with open(filename) as f:
+                for line in f.readlines():
+                    kv = line.split("=")
+                    if len(kv) != 2 or kv[0].startswith("#"):
+                        continue
+                    GlobalSettings._cache[kv[0].strip()] = kv[1].split("#")[0].strip()
+        except IOError:
+            pass # only ignore file access errors (including not-existing path)
+
+
 class CacheLock:
     """ Implements a lock for the object cache which
     can be used in 'with' statements. """
@@ -359,7 +403,7 @@ def release(self):
 
     @staticmethod
     def forPath(path):
-        timeoutMs = int(os.environ.get('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000))
+        timeoutMs = int(GlobalSettings.getValue('CLCACHE_OBJECT_CACHE_TIMEOUT_MS', 10 * 1000))
         lockName = path.replace(':', '-').replace('\\', '-')
         return CacheLock(lockName, timeoutMs)
 
@@ -505,10 +549,8 @@ class CacheFileStrategy:
     def __init__(self, cacheDirectory=None):
         self.dir = cacheDirectory
         if not self.dir:
-            try:
-                self.dir = os.environ["CLCACHE_DIR"]
-            except KeyError:
-                self.dir = os.path.join(os.path.expanduser("~"), "clcache")
+            self.dir = GlobalSettings.getValue("CLCACHE_DIR",
+                                               os.path.join(os.path.expanduser("~"), "clcache"))
 
         manifestsRootDir = os.path.join(self.dir, "manifests")
         ensureDirectoryExists(manifestsRootDir)
@@ -593,9 +635,10 @@ def clean(self, stats, maximumSize):
 
 class Cache:
     def __init__(self, cacheDirectory=None):
-        if os.environ.get("CLCACHE_MEMCACHED"):
+        memcached = GlobalSettings.getValue("CLCACHE_MEMCACHED")
+        if memcached and memcached not in ['0', 'false', 'False']:
             from .storage import CacheFileWithMemcacheFallbackStrategy
-            self.strategy = CacheFileWithMemcacheFallbackStrategy(os.environ.get("CLCACHE_MEMCACHED"),
+            self.strategy = CacheFileWithMemcacheFallbackStrategy(memcached,
                                                                   cacheDirectory=cacheDirectory)
         else:
             self.strategy = CacheFileStrategy(cacheDirectory=cacheDirectory)
@@ -900,7 +943,8 @@ def getCompilerHash(compilerBinary):
 
 
 def getFileHashes(filePaths):
-    if 'CLCACHE_SERVER' in os.environ:
+    server = GlobalSettings.getValue('CLCACHE_SERVER')
+    if server and server not in ['0', 'false', 'False']:
         pipeName = r'\\.\pipe\clcache_srv'
         while True:
             try:
@@ -939,7 +983,7 @@ def getStringHash(dataString):
 
 
 def expandBasedirPlaceholder(path):
-    baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR'))
+    baseDir = normalizeBaseDir(GlobalSettings.getValue('CLCACHE_BASEDIR'))
     if path.startswith(BASEDIR_REPLACEMENT):
         if not baseDir:
             raise LogicException('No CLCACHE_BASEDIR set, but found relative path ' + path)
@@ -949,7 +993,7 @@ def expandBasedirPlaceholder(path):
 
 
 def collapseBasedirToPlaceholder(path):
-    baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR'))
+    baseDir = normalizeBaseDir(GlobalSettings.getValue('CLCACHE_BASEDIR'))
     if baseDir is None:
         return path
     else:
@@ -971,8 +1015,8 @@ def ensureDirectoryExists(path):
 
 def copyOrLink(srcFilePath, dstFilePath):
     ensureDirectoryExists(os.path.dirname(os.path.abspath(dstFilePath)))
-
-    if "CLCACHE_HARDLINK" in os.environ:
+    hardlink = GlobalSettings.getValue("CLCACHE_HARDLINK")
+    if hardlink and hardlink not in ['0', 'false', 'False']:
         ret = windll.kernel32.CreateHardLinkW(str(dstFilePath), str(srcFilePath), None)
         if ret != 0:
             # Touch the time stamp of the new link so that the build system
@@ -998,11 +1042,10 @@ def myExecutablePath():
 
 
 def findCompilerBinary():
-    if "CLCACHE_CL" in os.environ:
-        path = os.environ["CLCACHE_CL"]
+    path = GlobalSettings.getValue("CLCACHE_CL")
+    if path:
         if os.path.basename(path) == path:
             path = which(path)
-
         return path if os.path.exists(path) else None
 
     frozenByPy2Exe = hasattr(sys, "frozen")
@@ -1020,7 +1063,8 @@ def findCompilerBinary():
 
 
 def printTraceStatement(msg: str) -> None:
-    if "CLCACHE_LOG" in os.environ:
+    clcachelog = GlobalSettings.getValue("CLCACHE_LOG")
+    if clcachelog and clcachelog not in ['0', 'false', 'False']:
         scriptDir = os.path.realpath(os.path.dirname(sys.argv[0]))
         with OUTPUT_LOCK:
             print(os.path.join(scriptDir, "clcache.py") + " " + msg)
@@ -1570,8 +1614,8 @@ def main():
 
     printTraceStatement("Found real compiler binary at '{0!s}'".format(compiler))
     printTraceStatement("Arguments we care about: '{}'".format(sys.argv))
-
-    if "CLCACHE_DISABLE" in os.environ:
+    enabled = GlobalSettings.getValue("CLCACHE_DISABLE")
+    if enabled and enabled not in ['0', 'false', 'False']:
         return invokeRealCompiler(compiler, sys.argv[1:])[0]
     try:
         return processCompileRequest(cache, compiler, sys.argv)
@@ -1670,8 +1714,8 @@ def processSingleSource(compiler, cmdLine, sourceFile, objectFile, environment):
     try:
         assert objectFile is not None
         cache = Cache()
-
-        if 'CLCACHE_NODIRECT' in os.environ:
+        nodirect = GlobalSettings.getValue('CLCACHE_NODIRECT')
+        if nodirect and nodirect not in ['0', 'false', 'False']:
             return processNoDirect(cache, objectFile, compiler, cmdLine, environment)
         else:
             return processDirect(cache, objectFile, compiler, cmdLine, sourceFile)
@@ -1770,7 +1814,8 @@ def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, ex
 
 
 if __name__ == '__main__':
-    if 'CLCACHE_PROFILE' in os.environ:
+    CLCACHE_PROFILE_ENABLED = GlobalSettings.getValue('CLCACHE_PROFILE')
+    if CLCACHE_PROFILE_ENABLED and CLCACHE_PROFILE_ENABLED not in ['0', 'false', 'False']:
         INVOCATION_HASH = getStringHash(','.join(sys.argv))
         cProfile.run('main()', filename='clcache-{}.prof'.format(INVOCATION_HASH))
     else:

tests/test_integration.py

@@ -47,6 +47,28 @@ def cd(targetDirectory):
         os.chdir(oldDirectory)
 
 
+def executeStatsCommand(customEnv=None):
+    cmd = CLCACHE_CMD +  ["-s"]
+    if customEnv:
+        out = subprocess.check_output(cmd, env=customEnv)
+    else:
+        out = subprocess.check_output(cmd)
+    return extractStatsOutput(out.decode("ascii").strip())
+
+
+def extractStatsOutput(outputLines):
+    stats = dict()
+    print(outputLines)
+    for line in outputLines.splitlines():
+        kv = line.split(":", 1)
+        if len(kv) != 2 or not kv[1]:
+            continue
+        stats[kv[0].strip()] = kv[1].strip()
+    # special case to avoid duplication: Update 'Disc cache at X:\\blah\\ccache' => 'X:\\blah\\ccache'
+    stats["current cache dir"] = stats["current cache dir"].split("cache at")[1].strip()
+    return stats
+
+
 class TestCommandLineArguments(unittest.TestCase):
     def testValidMaxSize(self):
         with tempfile.TemporaryDirectory() as tempDir:
@@ -74,6 +96,87 @@ def testPrintStatistics(self):
                 0,
                 "Command must be able to print statistics")
 
+
+class TestGlobalSettings(unittest.TestCase):
+    def testSettingsDefault(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            customEnv = dict(os.environ, HOME=tempDir)
+            stats = executeStatsCommand(customEnv)
+            print(stats)
+            self.assertEqual(stats["current cache dir"], os.path.join(tempDir, "clcache"))
+
+    def testSettingsEnvironmentVariables(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            customEnv = dict(os.environ, CLCACHE_DIR=tempDir)
+            stats = executeStatsCommand(customEnv)
+            print(stats)
+            self.assertEqual(stats["current cache dir"], os.path.join(tempDir))
+
+    def testSettingsLocalConfigFile(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            with cd(tempDir):
+                confFileName = os.path.join(tempDir, "clcache.conf")
+                clcacheDir = os.path.join(tempDir, "clcache")
+                self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir)
+                stats = executeStatsCommand()
+                self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testConfigFileInHomeDir(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir)
+            customEnv = dict(os.environ, HOME=tempDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testHomeDirOverridenByEnvironment(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR="this should be ignored")
+            customEnv = dict(os.environ, HOME=tempDir, CLCACHE_DIR=clcacheDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testSettingsConfigFileInProfiles(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR=clcacheDir)
+            customEnv = dict(os.environ, HOME="blah", ALLUSERSPROFILE=tempDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testConfProfilesOverridenByEnvironment(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            confFileName = os.path.join(tempDir, ".clcache", "clcache.conf")
+            clcacheDir = os.path.join(tempDir, "clcache")
+            self._createConfFile(confFileName, CLCACHE_DIR="should be ignored")
+            customEnv = dict(os.environ, HOME="blah", ALLUSERSPROFILE=tempDir, CLCACHE_DIR=clcacheDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def testProfilesOverridenByHomeDir(self):
+        with tempfile.TemporaryDirectory() as tempDir:
+            clcacheDir = os.path.join(tempDir, "clcache")
+            homeDir = os.path.join(tempDir, "home")
+            self._createConfFile(os.path.join(homeDir, ".clcache", "clcache.conf"), CLCACHE_DIR=clcacheDir)
+            profilesDir = os.path.join(tempDir, "allusersprofile")
+            self._createConfFile(os.path.join(profilesDir, ".clcache", "clcache.conf"), CLCACHE_DIR="ignored")
+            customEnv = dict(os.environ, HOME=homeDir, ALLUSERSPROFILE=profilesDir)
+            stats = executeStatsCommand(customEnv)
+            self.assertEqual(stats["current cache dir"], clcacheDir)
+
+    def _createConfFile(self, filename, **settings):
+        dirname = os.path.dirname(filename)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        with open(filename, "w") as f:
+            for k, v in settings.items():
+                f.write("{0} = {1}\n\r".format(k, v))
+
+
 class TestDistutils(unittest.TestCase):
     @pytest.mark.skipif(not MONKEY_LOADED, reason="Monkeypatch not loaded")
     @pytest.mark.skipif(CLCACHE_MEMCACHED, reason="Fails with memcached")