Skip to content

[Dependency Scanning] Query Swift overlay dependencies in parallel #86077

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
artemcm wants to merge 1 commit into
base: main
Choose a base branch
from
+62 −49
Open

[Dependency Scanning] Query Swift overlay dependencies in parallel #86077

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion include/swift/DependencyScan/ModuleDependencyScanner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,6 @@ class ModuleDependencyScanner {
/// Identify all cross-import overlay module dependencies of the
/// source module under scan and apply an action for each.
void resolveCrossImportOverlayDependencies(
StringRef mainModuleName,
llvm::function_ref<void(ModuleDependencyID)> action);

/// Perform Bridging Header Chaining.
Expand Down
12 changes: 0 additions & 12 deletions lib/AST/ModuleDependencies.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -717,18 +717,6 @@ ModuleDependenciesCache::findDependency(
auto known = map.find(moduleName);
if (known != map.end())
optionalDep = &(known->second);

// During a scan, only produce the cached source module info for the current
// module under scan.
if (optionalDep.has_value()) {
auto dep = optionalDep.value();
if (dep->getAsSwiftSourceModule() &&
moduleName != mainScanModuleName &&
moduleName != "MainModuleCrossImportOverlays") {
return std::nullopt;
}
}

return optionalDep;
}

Expand Down
98 changes: 62 additions & 36 deletions lib/DependencyScan/ModuleDependencyScanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -713,11 +713,12 @@ ModuleDependencyScanner::getMainModuleDependencyInfo(ModuleDecl *mainModule) {
/// from any single source file via direct or indirect imports, then
/// the cross-import overlay module is not required for compilation.
static void discoverCrossImportOverlayFiles(
StringRef mainModuleName, ModuleDependenciesCache &cache,
ASTContext &scanASTContext, llvm::SetVector<Identifier> &newOverlays,
ModuleDependenciesCache &cache, ASTContext &scanASTContext,
llvm::SetVector<Identifier> &newOverlays,
std::set<std::pair<std::string, std::string>> &overlayFiles) {
auto mainModuleInfo = cache.findKnownDependency(ModuleDependencyID{
mainModuleName.str(), ModuleDependencyKind::SwiftSource});
auto mainModuleID = ModuleDependencyID{cache.getMainModuleName().str(),
ModuleDependencyKind::SwiftSource};
auto mainModuleInfo = cache.findKnownDependency(mainModuleID);

llvm::StringMap<ModuleDependencyIDSet> perSourceFileDependencies;
const ModuleDependencyIDSet mainModuleDirectSwiftDepsSet{
Expand Down Expand Up @@ -805,12 +806,12 @@ static void discoverCrossImportOverlayFiles(
// direct imports from a given file, determine the available and required
// cross-import overlays.
auto discoverCrossImportOverlayFilesForModuleSet =
[&mainModuleName, &cache, &scanASTContext, &newOverlays,
[&mainModuleID, &cache, &scanASTContext, &newOverlays,
&overlayFiles](const ModuleDependencyIDSet &inputDependencies) {
for (auto moduleID : inputDependencies) {
auto moduleName = moduleID.ModuleName;
// Do not look for overlays of main module under scan
if (moduleName == mainModuleName)
if (moduleName == mainModuleID.ModuleName)
continue;

auto dependencies =
Expand All @@ -824,13 +825,13 @@ static void discoverCrossImportOverlayFiles(
for (const auto &dependencyId : inputDependencies) {
auto moduleName = dependencyId.ModuleName;
// Do not look for overlays of main module under scan
if (moduleName == mainModuleName)
if (moduleName == mainModuleID.ModuleName)
continue;
// check if any explicitly imported modules can serve as a
// secondary module, and add the overlay names to the
// dependencies list.
for (auto overlayName : overlayMap[moduleName]) {
if (overlayName.str() != mainModuleName &&
if (overlayName.str() != mainModuleID.ModuleName &&
std::find_if(inputDependencies.begin(),
inputDependencies.end(),
[&](ModuleDependencyID Id) {
Expand Down Expand Up @@ -874,7 +875,6 @@ ModuleDependencyScanner::performDependencyScan(ModuleDependencyID rootModuleID)
// overlays with explicit imports.
if (ScanCompilerInvocation.getLangOptions().EnableCrossImportOverlays)
resolveCrossImportOverlayDependencies(
rootModuleID.ModuleName,
[&](ModuleDependencyID id) { allModules.insert(id); });

if (ScanCompilerInvocation.getSearchPathOptions().BridgingHeaderChaining) {
Expand Down Expand Up @@ -1266,8 +1266,16 @@ void ModuleDependencyScanner::resolveHeaderDependencies(
void ModuleDependencyScanner::resolveSwiftOverlayDependencies(
ArrayRef<ModuleDependencyID> allSwiftModules,
ModuleDependencyIDSetVector &allDiscoveredDependencies) {
std::string batchOverlayQueryModuleName =
"_" + DependencyCache.getMainModuleName().str() + "-OverlayDependencies";

ModuleDependencyIDSetVector discoveredSwiftOverlays;
for (const auto &moduleID : allSwiftModules) {
// Do not re-consider the supplied dummy module's Swift overlays,
// if it is included in this list then we have already done so.
if (moduleID.ModuleName == batchOverlayQueryModuleName)
continue;

auto moduleDependencyInfo = DependencyCache.findKnownDependency(moduleID);
if (!moduleDependencyInfo.getSwiftOverlayDependencies().empty()) {
allDiscoveredDependencies.insert(
Expand All @@ -1282,17 +1290,34 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependencies(
}
}

if (discoveredSwiftOverlays.empty())
return;

auto batchOverlayQueryModuleID = ModuleDependencyID{
batchOverlayQueryModuleName, ModuleDependencyKind::SwiftSource};
auto batchOverlayQueryModuleInfo =
ModuleDependencyInfo::forSwiftSourceModule();
// For each additional Swift overlay dependency, ensure we perform a full scan
// in case it itself has unresolved module dependencies.
for (const auto &overlayDepID : discoveredSwiftOverlays) {
ModuleDependencyIDSetVector allNewModules =
resolveImportedModuleDependencies(overlayDepID);
allDiscoveredDependencies.insert(allNewModules.begin(),
allNewModules.end());
}
llvm::for_each(discoveredSwiftOverlays, [&](ModuleDependencyID modID) {
batchOverlayQueryModuleInfo.addModuleImport(modID.ModuleName, false,
AccessLevel::Public);
});
// Record the dummy main module's direct dependencies. The dummy query module
// only directly depend on these newly discovered overlay modules.
if (DependencyCache.findDependency(batchOverlayQueryModuleID))
DependencyCache.updateDependency(batchOverlayQueryModuleID,
batchOverlayQueryModuleInfo);
else
DependencyCache.recordDependency(batchOverlayQueryModuleName,
batchOverlayQueryModuleInfo);

ModuleDependencyIDSetVector allNewModules =
resolveImportedModuleDependencies(batchOverlayQueryModuleID);
// Remove the dummy module
allNewModules.remove(batchOverlayQueryModuleID);

allDiscoveredDependencies.insert(discoveredSwiftOverlays.begin(),
discoveredSwiftOverlays.end());
allDiscoveredDependencies.insert(allNewModules.begin(), allNewModules.end());
}

void ModuleDependencyScanner::resolveSwiftImportsForModule(
Expand Down Expand Up @@ -1647,63 +1672,64 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependenciesForModule(
}

void ModuleDependencyScanner::resolveCrossImportOverlayDependencies(
StringRef mainModuleName,
llvm::function_ref<void(ModuleDependencyID)> action) {
// Modules explicitly imported. Only these can be secondary module.
llvm::SetVector<Identifier> newOverlays;
std::set<std::pair<std::string, std::string>> overlayFiles;
discoverCrossImportOverlayFiles(mainModuleName, DependencyCache,
ScanASTContext, newOverlays, overlayFiles);
discoverCrossImportOverlayFiles(DependencyCache, ScanASTContext, newOverlays,
overlayFiles);

// No new cross-import overlays are found, return.
if (newOverlays.empty())
return;

// Construct a dummy main to resolve the newly discovered cross import
// overlays.
StringRef dummyMainName = "MainModuleCrossImportOverlays";
auto dummyMainID = ModuleDependencyID{dummyMainName.str(),
std::string batchCrossImportQueryModuleName =
"_" + DependencyCache.getMainModuleName().str() + "-CrossImportOverlays";
auto queryModuleID = ModuleDependencyID{batchCrossImportQueryModuleName,
ModuleDependencyKind::SwiftSource};
auto actualMainID = ModuleDependencyID{mainModuleName.str(),
auto mainModuleID = ModuleDependencyID{DependencyCache.getMainModuleName().str(),
ModuleDependencyKind::SwiftSource};
auto dummyMainDependencies = ModuleDependencyInfo::forSwiftSourceModule();
auto queryModuleInfo = ModuleDependencyInfo::forSwiftSourceModule();
llvm::for_each(
newOverlays, [&](Identifier modName) {
dummyMainDependencies.addModuleImport(
queryModuleInfo.addModuleImport(
modName.str(),
/* isExported */ false,
// TODO: What is the right access level for a cross-import overlay?
AccessLevel::Public);
});

// Record the dummy main module's direct dependencies. The dummy main module
// Record the dummy query module's direct dependencies. The dummy query module
// only directly depend on these newly discovered overlay modules.
if (DependencyCache.findDependency(dummyMainID))
DependencyCache.updateDependency(dummyMainID, dummyMainDependencies);
if (DependencyCache.findDependency(queryModuleID))
DependencyCache.updateDependency(queryModuleID, queryModuleInfo);
else
DependencyCache.recordDependency(dummyMainName, dummyMainDependencies);
DependencyCache.recordDependency(batchCrossImportQueryModuleName,
queryModuleInfo);

ModuleDependencyIDSetVector allModules =
resolveImportedModuleDependencies(dummyMainID);
resolveImportedModuleDependencies(queryModuleID);

// Update main module's dependencies to include these new overlays.
DependencyCache.setCrossImportOverlayDependencies(
actualMainID, DependencyCache.getAllDependencies(dummyMainID));
mainModuleID, DependencyCache.getAllDependencies(queryModuleID));

// Update the command-line on the main module to disable implicit
// cross-import overlay search.
auto mainDep = DependencyCache.findKnownDependency(actualMainID);
std::vector<std::string> cmdCopy = mainDep.getCommandline();
auto mainModuleInfo = DependencyCache.findKnownDependency(mainModuleID);
std::vector<std::string> cmdCopy = mainModuleInfo.getCommandline();
cmdCopy.push_back("-disable-cross-import-overlay-search");
for (auto &entry : overlayFiles) {
mainDep.addAuxiliaryFile(entry.second);
mainModuleInfo.addAuxiliaryFile(entry.second);
cmdCopy.push_back("-swift-module-cross-import");
cmdCopy.push_back(entry.first);
auto overlayPath = remapPath(entry.second);
cmdCopy.push_back(overlayPath);
}
mainDep.updateCommandLine(cmdCopy);
DependencyCache.updateDependency(actualMainID, mainDep);
mainModuleInfo.updateCommandLine(cmdCopy);
DependencyCache.updateDependency(mainModuleID, mainModuleInfo);

// Report any discovered modules to the clients, which include all overlays
// and their dependencies.
Expand Down