Merge pull request #2 from izmmisha/add-tracker-support

Add tracker support

Author: izmmisha

appveyor.yml

@@ -8,7 +8,6 @@ environment:
     # AppVeyor installed Python versions
     # http://www.appveyor.com/docs/installed-software#python
 
-    - PYTHON_INSTALL: "C:\\Python34"
     - PYTHON_INSTALL: "C:\\Python35"
     - PYTHON_INSTALL: "C:\\Python36"
 
@@ -27,16 +26,16 @@ install:
   - set PATH=%PYTHON_INSTALL%;%PATH%
 
   # Prepend Python scripts to PATH (e.g. pip, py.test, pylint)
-  - set PATH=%PYTHON_INSTALL%\\Scripts;%PATH%
+  - set PATH=%PYTHON_INSTALL%\Scripts;%PATH%
 
+  - echo "PATH is %PATH%"
   # Check Python version
   - python --version
 
   # Upgrade the Python build tools
-  - pip install -U pip setuptools wheel
+  - python -m pip install --upgrade pip setuptools wheel
 
   # Check pip version
-  - python -m pip install --upgrade pip setuptools wheel
   - pip --version
 
   # Install pytest
@@ -56,7 +55,8 @@ install:
   - pip install .
 
   # Install tool for uploading coverage data to codecov.io
-  - pip install codecov
+  - choco install codecov
+  - where coverage
   - coverage --version
 
   # Install dependencies for clcachesrv such that we can lint the source code
@@ -96,7 +96,7 @@ test_script:
       $wc = New-Object 'System.Net.WebClient'
       $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\unittests.xml))
 
-      & codecov --no-color -X gcov -F unittests -e PYTHON_INSTALL
+      & codecov -f coverage.xml
 
       if ($testsExitCode -ne 0) {exit $testsExitCode}
 
@@ -113,7 +113,7 @@ test_script:
       $wc = New-Object 'System.Net.WebClient'
       $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\integrationtests.xml))
 
-      & codecov --no-color -X gcov -F integrationtests_memcached -e PYTHON_INSTALL
+      & codecov -f coverage.xml
 
       if ($testsExitCode -ne 0) {exit $testsExitCode}
 

clcache/__main__.py

@@ -42,7 +42,7 @@
 except ImportError:
     WALK = os.walk
     try:
-        LIST = os.scandir # pylint: disable=no-name-in-module
+        LIST = os.scandir # type: ignore # pylint: disable=no-name-in-module
     except AttributeError:
         LIST = os.listdir
 
@@ -94,7 +94,7 @@ def filesBeneath(baseDir):
 
 
 def childDirectories(path, absolute=True):
-    supportsScandir = (LIST != os.listdir)
+    supportsScandir = (LIST != os.listdir) # pylint: disable=comparison-with-callable
     for entry in LIST(path):
         if supportsScandir:
             if entry.is_dir():
@@ -116,6 +116,44 @@ def normalizeBaseDir(baseDir):
         return None
 
 
+class SuspendTracker():
+    fileTracker = None
+    def __init__(self):
+        if not SuspendTracker.fileTracker:
+            if windll.kernel32.GetModuleHandleW("FileTracker.dll"):
+                SuspendTracker.fileTracker = windll.FileTracker
+            elif windll.kernel32.GetModuleHandleW("FileTracker32.dll"):
+                SuspendTracker.fileTracker = windll.FileTracker32
+
+    def __enter__(self):
+        SuspendTracker.suspend()
+
+    def __exit__(self, typ, value, traceback):
+        SuspendTracker.resume()
+
+    @staticmethod
+    def suspend():
+        if SuspendTracker.fileTracker:
+            SuspendTracker.fileTracker.SuspendTracking()
+
+    @staticmethod
+    def resume():
+        if SuspendTracker.fileTracker:
+            SuspendTracker.fileTracker.ResumeTracking()
+
+def isTrackerEnabled():
+    return 'TRACKER_ENABLED' in os.environ
+
+def untrackable(func):
+    if not isTrackerEnabled():
+        return func
+
+    def untrackedFunc(*args, **kwargs):
+        with SuspendTracker():
+            return func(*args, **kwargs)
+
+    return untrackedFunc
+
 @contextlib.contextmanager
 def atomicWrite(fileName):
     tempFileName = fileName + '.new'
@@ -163,7 +201,7 @@ def __str__(self):
         return repr(self.message)
 
 
-class Manifest(object):
+class Manifest:
     def __init__(self, entries=None):
         if entries is None:
             entries = []
@@ -182,7 +220,7 @@ def touchEntry(self, objectHash):
         self._entries.insert(0, self._entries.pop(entryIndex))
 
 
-class ManifestSection(object):
+class ManifestSection:
     def __init__(self, manifestSectionDir):
         self.manifestSectionDir = manifestSectionDir
         self.lock = CacheLock.forPath(self.manifestSectionDir)
@@ -193,6 +231,7 @@ def manifestPath(self, manifestHash):
     def manifestFiles(self):
         return filesBeneath(self.manifestSectionDir)
 
+    @untrackable
     def setManifest(self, manifestHash, manifest):
         manifestPath = self.manifestPath(manifestHash)
         printTraceStatement("Writing manifest with manifestHash = {} to {}".format(manifestHash, manifestPath))
@@ -203,6 +242,7 @@ def setManifest(self, manifestHash, manifest):
             jsonobject = {'entries': entries}
             json.dump(jsonobject, outFile, sort_keys=True, indent=2)
 
+    @untrackable
     def getManifest(self, manifestHash):
         fileName = self.manifestPath(manifestHash)
         if not os.path.exists(fileName):
@@ -231,7 +271,7 @@ def allSectionsLocked(repository):
             section.lock.release()
 
 
-class ManifestRepository(object):
+class ManifestRepository:
     # Bump this counter whenever the current manifest file format changes.
     # E.g. changing the file format from {'oldkey': ...} to {'newkey': ...} requires
     # invalidation, such that a manifest that was stored using the old format is not
@@ -308,7 +348,7 @@ def getIncludesContentHashForHashes(listOfHashes):
         return HashAlgorithm(','.join(listOfHashes).encode()).hexdigest()
 
 
-class CacheLock(object):
+class CacheLock:
     """ Implements a lock for the object cache which
     can be used in 'with' statements. """
     INFINITE = 0xFFFFFFFF
@@ -364,7 +404,7 @@ def forPath(path):
         return CacheLock(lockName, timeoutMs)
 
 
-class CompilerArtifactsSection(object):
+class CompilerArtifactsSection:
     OBJECT_FILE = 'object'
     STDOUT_FILE = 'output.txt'
     STDERR_FILE = 'stderr.txt'
@@ -413,7 +453,7 @@ def getEntry(self, key):
             )
 
 
-class CompilerArtifactsRepository(object):
+class CompilerArtifactsRepository:
     def __init__(self, compilerArtifactsRootDir):
         self._compilerArtifactsRootDir = compilerArtifactsRootDir
 
@@ -501,7 +541,7 @@ def _normalizedCommandLine(cmdline):
         return [arg for arg in cmdline
                 if not (arg[0] in "/-" and arg[1:].startswith(argsToStrip))]
 
-class CacheFileStrategy(object):
+class CacheFileStrategy:
     def __init__(self, cacheDirectory=None):
         self.dir = cacheDirectory
         if not self.dir:
@@ -591,7 +631,7 @@ def clean(self, stats, maximumSize):
         stats.setNumCacheEntries(currentCompilerArtifactsCount)
 
 
-class Cache(object):
+class Cache:
     def __init__(self, cacheDirectory=None):
         if os.environ.get("CLCACHE_MEMCACHED"):
             from .storage import CacheFileWithMemcacheFallbackStrategy
@@ -644,7 +684,7 @@ def getManifest(self, manifestHash):
         return self.strategy.getManifest(manifestHash)
 
 
-class PersistentJSONDict(object):
+class PersistentJSONDict:
     def __init__(self, fileName):
         self._dirty = False
         self._dict = {}
@@ -676,7 +716,7 @@ def __eq__(self, other):
         return type(self) is type(other) and self.__dict__ == other.__dict__
 
 
-class Configuration(object):
+class Configuration:
     _defaultValues = {"MaximumCacheSize": 1073741824} # 1 GiB
 
     def __init__(self, configurationFile):
@@ -701,7 +741,7 @@ def setMaximumCacheSize(self, size):
         self._cfg["MaximumCacheSize"] = size
 
 
-class Statistics(object):
+class Statistics:
     CALLS_WITH_INVALID_ARGUMENT = "CallsWithInvalidArgument"
     CALLS_WITHOUT_SOURCE_FILE = "CallsWithoutSourceFile"
     CALLS_WITH_MULTIPLE_SOURCE_FILES = "CallsWithMultipleSourceFiles"
@@ -741,13 +781,15 @@ def __init__(self, statsFile):
         self._stats = None
         self.lock = CacheLock.forPath(self._statsFile)
 
+    @untrackable
     def __enter__(self):
         self._stats = PersistentJSONDict(self._statsFile)
         for k in Statistics.RESETTABLE_KEYS | Statistics.NON_RESETTABLE_KEYS:
             if k not in self._stats:
                 self._stats[k] = 0
         return self
 
+    @untrackable
     def __exit__(self, typ, value, traceback):
         # Does not write to disc when unchanged
         self._stats.save()
@@ -1026,7 +1068,7 @@ def printTraceStatement(msg: str) -> None:
             print(os.path.join(scriptDir, "clcache.py") + " " + msg)
 
 
-class CommandLineTokenizer(object):
+class CommandLineTokenizer:
     def __init__(self, content):
         self.argv = []
         self._content = content
@@ -1154,7 +1196,7 @@ def extendCommandLineFromEnvironment(cmdLine, environment):
     return cmdLine, remainingEnvironment
 
 
-class Argument(object):
+class Argument:
     def __init__(self, name):
         self.name = name
 
@@ -1192,7 +1234,7 @@ class ArgumentT4(Argument):
     pass
 
 
-class CommandLineAnalyzer(object):
+class CommandLineAnalyzer:
 
     @staticmethod
     def _getParameterizedArgumentType(cmdLineArgument):
@@ -1645,7 +1687,13 @@ def scheduleJobs(cache: Any, compiler: str, cmdLine: List[str], environment: Any
 
     exitCode = 0
     cleanupRequired = False
-    with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor:
+
+    def poolExecutor(*args, **kwargs) -> concurrent.futures.Executor:
+        if isTrackerEnabled():
+            return concurrent.futures.ProcessPoolExecutor(*args, **kwargs)
+        return concurrent.futures.ThreadPoolExecutor(*args, **kwargs)
+
+    with poolExecutor(max_workers=min(jobCount(cmdLine), len(objectFiles))) as executor:
         jobs = []
         for (srcFile, srcLanguage), objFile in zip(sourceFiles, objectFiles):
             jobCmdLine = baseCmdLine + [srcLanguage + srcFile]

clcache/server/__main__.py

@@ -10,7 +10,7 @@
 
 import pyuv
 
-class HashCache(object):
+class HashCache:
     def __init__(self, loop, excludePatterns, disableWatching):
         self._loop = loop
         self._watchedDirectories = {}
@@ -66,7 +66,7 @@ def isExcluded(self, dirname):
         return excluded
 
 
-class Connection(object):
+class Connection:
     def __init__(self, pipe, cache, onCloseCallback):
         self._readBuffer = b''
         self._pipe = pipe
@@ -92,7 +92,7 @@ def _onWriteDone(self, pipe, error):
         self._onCloseCallback(self)
 
 
-class PipeServer(object):
+class PipeServer:
     def __init__(self, loop, address, cache):
         self._pipeServer = pyuv.Pipe(loop)
         self._pipeServer.bind(address)

clcache/storage.py

@@ -8,15 +8,15 @@
     CACHE_COMPILER_OUTPUT_STORAGE_CODEC
 
 
-class CacheDummyLock(object):
+class CacheDummyLock:
     def __enter__(self):
         pass
 
     def __exit__(self, typ, value, traceback):
         pass
 
 
-class CacheMemcacheStrategy(object):
+class CacheMemcacheStrategy:
     def __init__(self, server, cacheDirectory=None, manifestPrefix='manifests_', objectPrefix='objects_'):
         self.fileStrategy = CacheFileStrategy(cacheDirectory=cacheDirectory)
         # XX Memcache Strategy should be independent
@@ -156,7 +156,7 @@ def clean(self, stats, maximumSize):
                                 maximumSize)
 
 
-class CacheFileWithMemcacheFallbackStrategy(object):
+class CacheFileWithMemcacheFallbackStrategy:
     def __init__(self, server, cacheDirectory=None, manifestPrefix='manifests_', objectPrefix='objects_'):
         self.localCache = CacheFileStrategy(cacheDirectory=cacheDirectory)
         self.remoteCache = CacheMemcacheStrategy(server, cacheDirectory=cacheDirectory,

pyinstaller/clcache_main.py

@@ -1,2 +1,4 @@
+import multiprocessing
 from clcache.__main__ import main
+multiprocessing.freeze_support()
 main()

setup.py

@@ -22,6 +22,7 @@
     entry_points={
           'console_scripts': [
               'clcache = clcache.__main__:main',
+              'cl.cache = clcache.__main__:main',
               'clcache-server = clcache.server.__main__:main',
           ]
     },

tests/integrationtests/msbuild/another.cpp

@@ -0,0 +1 @@
+int somefunc() { return 1; }

tests/integrationtests/msbuild/test.vcxproj

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+  </PropertyGroup>
+
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <DebugInformationFormat>OldStyle</DebugInformationFormat>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="another.cpp">
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <ClCompile Include="../minimal.cpp" />
+    <ClCompile Include="../fibonacci.cpp" />
+  </ItemGroup>
+</Project>