From dea8d67a0d35478c7d34f0ffaf0e03dac2b7d44b Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Mon, 15 Dec 2025 15:30:48 -0800 Subject: [PATCH] [Dependency Scanning] Query Swift overlay dependencies in parallel In 'resolveSwiftOverlayDependencies', instead of calling into 'resolveImportedModuleDependencies' for every newly-discovered Swift overlay dependency, gather all collecected overlay dependencies under one umbrella dummy query module and execute a single call into 'resolveImportedModuleDependencies' which will cover all of them. With this change, we not only lookup all the various Swift module overlays' imports in parallel, but also all of their aggregate respective Clang dependencies are queried in one shot as well. --- .../DependencyScan/ModuleDependencyScanner.h | 1 - lib/AST/ModuleDependencies.cpp | 12 --- .../ModuleDependencyScanner.cpp | 98 ++++++++++++------- 3 files changed, 62 insertions(+), 49 deletions(-) diff --git a/include/swift/DependencyScan/ModuleDependencyScanner.h b/include/swift/DependencyScan/ModuleDependencyScanner.h index 01cf52734b8ca..164170c0aaccc 100644 --- a/include/swift/DependencyScan/ModuleDependencyScanner.h +++ b/include/swift/DependencyScan/ModuleDependencyScanner.h @@ -309,7 +309,6 @@ class ModuleDependencyScanner { /// Identify all cross-import overlay module dependencies of the /// source module under scan and apply an action for each. void resolveCrossImportOverlayDependencies( - StringRef mainModuleName, llvm::function_ref action); /// Perform Bridging Header Chaining. diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index da4e33804df5b..a1ab8180eb78a 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -717,18 +717,6 @@ ModuleDependenciesCache::findDependency( auto known = map.find(moduleName); if (known != map.end()) optionalDep = &(known->second); - - // During a scan, only produce the cached source module info for the current - // module under scan. - if (optionalDep.has_value()) { - auto dep = optionalDep.value(); - if (dep->getAsSwiftSourceModule() && - moduleName != mainScanModuleName && - moduleName != "MainModuleCrossImportOverlays") { - return std::nullopt; - } - } - return optionalDep; } diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp index 83c60ed7e0177..cde074b37d588 100644 --- a/lib/DependencyScan/ModuleDependencyScanner.cpp +++ b/lib/DependencyScan/ModuleDependencyScanner.cpp @@ -713,11 +713,12 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(ModuleDecl *mainModule) { /// from any single source file via direct or indirect imports, then /// the cross-import overlay module is not required for compilation. static void discoverCrossImportOverlayFiles( - StringRef mainModuleName, ModuleDependenciesCache &cache, - ASTContext &scanASTContext, llvm::SetVector &newOverlays, + ModuleDependenciesCache &cache, ASTContext &scanASTContext, + llvm::SetVector &newOverlays, std::set> &overlayFiles) { - auto mainModuleInfo = cache.findKnownDependency(ModuleDependencyID{ - mainModuleName.str(), ModuleDependencyKind::SwiftSource}); + auto mainModuleID = ModuleDependencyID{cache.getMainModuleName().str(), + ModuleDependencyKind::SwiftSource}; + auto mainModuleInfo = cache.findKnownDependency(mainModuleID); llvm::StringMap perSourceFileDependencies; const ModuleDependencyIDSet mainModuleDirectSwiftDepsSet{ @@ -805,12 +806,12 @@ static void discoverCrossImportOverlayFiles( // direct imports from a given file, determine the available and required // cross-import overlays. auto discoverCrossImportOverlayFilesForModuleSet = - [&mainModuleName, &cache, &scanASTContext, &newOverlays, + [&mainModuleID, &cache, &scanASTContext, &newOverlays, &overlayFiles](const ModuleDependencyIDSet &inputDependencies) { for (auto moduleID : inputDependencies) { auto moduleName = moduleID.ModuleName; // Do not look for overlays of main module under scan - if (moduleName == mainModuleName) + if (moduleName == mainModuleID.ModuleName) continue; auto dependencies = @@ -824,13 +825,13 @@ static void discoverCrossImportOverlayFiles( for (const auto &dependencyId : inputDependencies) { auto moduleName = dependencyId.ModuleName; // Do not look for overlays of main module under scan - if (moduleName == mainModuleName) + if (moduleName == mainModuleID.ModuleName) continue; // check if any explicitly imported modules can serve as a // secondary module, and add the overlay names to the // dependencies list. for (auto overlayName : overlayMap[moduleName]) { - if (overlayName.str() != mainModuleName && + if (overlayName.str() != mainModuleID.ModuleName && std::find_if(inputDependencies.begin(), inputDependencies.end(), [&](ModuleDependencyID Id) { @@ -874,7 +875,6 @@ ModuleDependencyScanner::performDependencyScan(ModuleDependencyID rootModuleID) // overlays with explicit imports. if (ScanCompilerInvocation.getLangOptions().EnableCrossImportOverlays) resolveCrossImportOverlayDependencies( - rootModuleID.ModuleName, [&](ModuleDependencyID id) { allModules.insert(id); }); if (ScanCompilerInvocation.getSearchPathOptions().BridgingHeaderChaining) { @@ -1266,8 +1266,16 @@ void ModuleDependencyScanner::resolveHeaderDependencies( void ModuleDependencyScanner::resolveSwiftOverlayDependencies( ArrayRef allSwiftModules, ModuleDependencyIDSetVector &allDiscoveredDependencies) { + std::string batchOverlayQueryModuleName = + "_" + DependencyCache.getMainModuleName().str() + "-OverlayDependencies"; + ModuleDependencyIDSetVector discoveredSwiftOverlays; for (const auto &moduleID : allSwiftModules) { + // Do not re-consider the supplied dummy module's Swift overlays, + // if it is included in this list then we have already done so. + if (moduleID.ModuleName == batchOverlayQueryModuleName) + continue; + auto moduleDependencyInfo = DependencyCache.findKnownDependency(moduleID); if (!moduleDependencyInfo.getSwiftOverlayDependencies().empty()) { allDiscoveredDependencies.insert( @@ -1282,17 +1290,34 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependencies( } } + if (discoveredSwiftOverlays.empty()) + return; + + auto batchOverlayQueryModuleID = ModuleDependencyID{ + batchOverlayQueryModuleName, ModuleDependencyKind::SwiftSource}; + auto batchOverlayQueryModuleInfo = + ModuleDependencyInfo::forSwiftSourceModule(); // For each additional Swift overlay dependency, ensure we perform a full scan // in case it itself has unresolved module dependencies. - for (const auto &overlayDepID : discoveredSwiftOverlays) { - ModuleDependencyIDSetVector allNewModules = - resolveImportedModuleDependencies(overlayDepID); - allDiscoveredDependencies.insert(allNewModules.begin(), - allNewModules.end()); - } + llvm::for_each(discoveredSwiftOverlays, [&](ModuleDependencyID modID) { + batchOverlayQueryModuleInfo.addModuleImport(modID.ModuleName, false, + AccessLevel::Public); + }); + // Record the dummy main module's direct dependencies. The dummy query module + // only directly depend on these newly discovered overlay modules. + if (DependencyCache.findDependency(batchOverlayQueryModuleID)) + DependencyCache.updateDependency(batchOverlayQueryModuleID, + batchOverlayQueryModuleInfo); + else + DependencyCache.recordDependency(batchOverlayQueryModuleName, + batchOverlayQueryModuleInfo); + + ModuleDependencyIDSetVector allNewModules = + resolveImportedModuleDependencies(batchOverlayQueryModuleID); + // Remove the dummy module + allNewModules.remove(batchOverlayQueryModuleID); - allDiscoveredDependencies.insert(discoveredSwiftOverlays.begin(), - discoveredSwiftOverlays.end()); + allDiscoveredDependencies.insert(allNewModules.begin(), allNewModules.end()); } void ModuleDependencyScanner::resolveSwiftImportsForModule( @@ -1647,13 +1672,12 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependenciesForModule( } void ModuleDependencyScanner::resolveCrossImportOverlayDependencies( - StringRef mainModuleName, llvm::function_ref action) { // Modules explicitly imported. Only these can be secondary module. llvm::SetVector newOverlays; std::set> overlayFiles; - discoverCrossImportOverlayFiles(mainModuleName, DependencyCache, - ScanASTContext, newOverlays, overlayFiles); + discoverCrossImportOverlayFiles(DependencyCache, ScanASTContext, newOverlays, + overlayFiles); // No new cross-import overlays are found, return. if (newOverlays.empty()) @@ -1661,49 +1685,51 @@ void ModuleDependencyScanner::resolveCrossImportOverlayDependencies( // Construct a dummy main to resolve the newly discovered cross import // overlays. - StringRef dummyMainName = "MainModuleCrossImportOverlays"; - auto dummyMainID = ModuleDependencyID{dummyMainName.str(), + std::string batchCrossImportQueryModuleName = + "_" + DependencyCache.getMainModuleName().str() + "-CrossImportOverlays"; + auto queryModuleID = ModuleDependencyID{batchCrossImportQueryModuleName, ModuleDependencyKind::SwiftSource}; - auto actualMainID = ModuleDependencyID{mainModuleName.str(), + auto mainModuleID = ModuleDependencyID{DependencyCache.getMainModuleName().str(), ModuleDependencyKind::SwiftSource}; - auto dummyMainDependencies = ModuleDependencyInfo::forSwiftSourceModule(); + auto queryModuleInfo = ModuleDependencyInfo::forSwiftSourceModule(); llvm::for_each( newOverlays, [&](Identifier modName) { - dummyMainDependencies.addModuleImport( + queryModuleInfo.addModuleImport( modName.str(), /* isExported */ false, // TODO: What is the right access level for a cross-import overlay? AccessLevel::Public); }); - // Record the dummy main module's direct dependencies. The dummy main module + // Record the dummy query module's direct dependencies. The dummy query module // only directly depend on these newly discovered overlay modules. - if (DependencyCache.findDependency(dummyMainID)) - DependencyCache.updateDependency(dummyMainID, dummyMainDependencies); + if (DependencyCache.findDependency(queryModuleID)) + DependencyCache.updateDependency(queryModuleID, queryModuleInfo); else - DependencyCache.recordDependency(dummyMainName, dummyMainDependencies); + DependencyCache.recordDependency(batchCrossImportQueryModuleName, + queryModuleInfo); ModuleDependencyIDSetVector allModules = - resolveImportedModuleDependencies(dummyMainID); + resolveImportedModuleDependencies(queryModuleID); // Update main module's dependencies to include these new overlays. DependencyCache.setCrossImportOverlayDependencies( - actualMainID, DependencyCache.getAllDependencies(dummyMainID)); + mainModuleID, DependencyCache.getAllDependencies(queryModuleID)); // Update the command-line on the main module to disable implicit // cross-import overlay search. - auto mainDep = DependencyCache.findKnownDependency(actualMainID); - std::vector cmdCopy = mainDep.getCommandline(); + auto mainModuleInfo = DependencyCache.findKnownDependency(mainModuleID); + std::vector cmdCopy = mainModuleInfo.getCommandline(); cmdCopy.push_back("-disable-cross-import-overlay-search"); for (auto &entry : overlayFiles) { - mainDep.addAuxiliaryFile(entry.second); + mainModuleInfo.addAuxiliaryFile(entry.second); cmdCopy.push_back("-swift-module-cross-import"); cmdCopy.push_back(entry.first); auto overlayPath = remapPath(entry.second); cmdCopy.push_back(overlayPath); } - mainDep.updateCommandLine(cmdCopy); - DependencyCache.updateDependency(actualMainID, mainDep); + mainModuleInfo.updateCommandLine(cmdCopy); + DependencyCache.updateDependency(mainModuleID, mainModuleInfo); // Report any discovered modules to the clients, which include all overlays // and their dependencies.