diff --git a/build-local.ps1 b/build-local.ps1 index ebc312aa11..4a660a0a86 100644 --- a/build-local.ps1 +++ b/build-local.ps1 @@ -40,7 +40,7 @@ $env:OCTOVERSION_MajorMinorPatch= $numericVersion $env:OCTOVERSION_PreReleaseTagWithDash = "-$sanitizedBranch" $env:OCTOVERSION_FullSemVer = "$numericVersion-$sanitizedBranch" -./build.ps1 -BuildVerbosity Minimal -Verbosity Normal -PackInParallel --Append-Timestamp -SetOctopusServerVersion -TargetFramework "$Framework" -TargetRuntime "$Runtime" +./build.ps1 -BuildVerbosity Minimal -Verbosity Normal --Append-Timestamp -SetOctopusServerVersion -TargetFramework "$Framework" -TargetRuntime "$Runtime" Write-Host " ######################################################################################## diff --git a/build-local.sh b/build-local.sh index 3a587d61c5..67c5846e6e 100755 --- a/build-local.sh +++ b/build-local.sh @@ -92,6 +92,6 @@ export OCTOVERSION_MajorMinorPatch="$numericVersion" export OCTOVERSION_PreReleaseTagWithDash="-$sanitizedBranch" export OCTOVERSION_FullSemVer="$numericVersion-$sanitizedBranch" -./build.sh -BuildVerbosity Minimal -Verbosity Minimal -PackInParallel -AppendTimestamp -SetOctopusServerVersion -TargetFramework "$target_framework" -TargetRuntime "$target_runtime" +./build.sh -BuildVerbosity Minimal -Verbosity Minimal -AppendTimestamp -SetOctopusServerVersion -TargetFramework "$target_framework" -TargetRuntime "$target_runtime" echo -e "$FinishMessage" \ No newline at end of file diff --git a/build/Build.cs b/build/Build.cs index 7f0d212035..f9512bac65 100644 --- a/build/Build.cs +++ b/build/Build.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.IO.Compression; @@ -33,8 +34,6 @@ partial class Build : NukeBuild [Required] readonly Solution Solution = SolutionModelTasks.ParseSolution(SourceDirectory / "Calamari.sln"); - [Parameter("Run packing step in parallel")] readonly bool PackInParallel; - [Parameter] readonly DotNetVerbosity BuildVerbosity = DotNetVerbosity.minimal; [Parameter] readonly bool SignBinaries; @@ -76,7 +75,8 @@ partial class Build : NukeBuild static readonly List CalamariProjectsToSkipConsolidation = new() { "Calamari.CloudAccounts", "Calamari.Common", "Calamari.ConsolidateCalamariPackages" }; - List SignDirectoriesTasks = new(); + CalamariPackageMetadata[] PackagesToPublish = new CalamariPackageMetadata[0]; + List CalamariProjects = new(); List ProjectCompressionTasks = new(); public Build() @@ -221,15 +221,14 @@ public Build() nugetVersion, FixedRuntimes.Cloud); - // Create the self-contained Calamari packages for each runtime ID defined in Calamari.csproj foreach (var rid in GetRuntimeIdentifiers(Solution.GetProject(RootProjectName)!)!) DoPublish(RootProjectName, Frameworks.Net60, nugetVersion, rid); }); - Target PublishCalamariFlavourProjects => + Target GetCalamariFlavourProjectsToPublish => d => d.DependsOn(Compile) - .Executes(async () => + .Executes(() => { var flavours = GetCalamariFlavours(); var migratedCalamariFlavoursTests = flavours.Select(f => $"{f}.Tests"); @@ -246,90 +245,175 @@ public Build() .Concat(calamariScriptingProjectAndTest) .ToList(); - await PublishCalamariProjects(calamariProjects); + CalamariProjects = calamariProjects; + + // All cross-platform Target Frameworks contain dots, all NetFx Target Frameworks don't + // eg: net40, net452, net48 vs netcoreapp3.1, net5.0, net6.0 + bool IsCrossPlatform(string targetFramework) => targetFramework.Contains('.'); + + var calamariPackages = + calamariProjects.SelectMany(project => project.GetTargetFrameworks()!, (p, f) => new + { + Project = p, + Framework = f, + CrossPlatform = IsCrossPlatform(f) + }) + .ToList(); + + // for NetFx target frameworks, we use "netfx" as the architecture, and ignore defined runtime identifiers + var netFxPackages = + calamariPackages.Where(p => !p.CrossPlatform) + .Select(packageToBuild => new CalamariPackageMetadata() + { + Project = packageToBuild.Project, + Framework = packageToBuild.Framework, + Architecture = null, + IsCrossPlatform = packageToBuild.CrossPlatform + }); + + // for cross-platform frameworks, we combine each runtime identifier with each target framework + var crossPlatformPackages = + calamariPackages.Where(p => p.CrossPlatform) + .Where(p => string.IsNullOrWhiteSpace(TargetFramework) || p.Framework == TargetFramework) + .SelectMany(packageToBuild => GetRuntimeIdentifiers(packageToBuild.Project) ?? Enumerable.Empty(), + (packageToBuild, runtimeIdentifier) => new CalamariPackageMetadata() + { + Project = packageToBuild.Project, + Framework = packageToBuild.Framework, + Architecture = runtimeIdentifier, + IsCrossPlatform = packageToBuild.CrossPlatform + }) + .Distinct(t => new { t.Project?.Name, t.Architecture, t.Framework }); + + PackagesToPublish = crossPlatformPackages.Concat(netFxPackages).ToArray(); }); - async Task PublishCalamariProjects(List projects) - { - // All cross-platform Target Frameworks contain dots, all NetFx Target Frameworks don't - // eg: net40, net452, net48 vs netcoreapp3.1, net5.0, net6.0 - bool IsCrossPlatform(string targetFramework) => targetFramework.Contains('.'); - - var calamariPackages = - projects.SelectMany(project => project.GetTargetFrameworks()!, (p, f) => new - { - Project = p, - Framework = f, - CrossPlatform = IsCrossPlatform(f) - }) - .ToList(); - - // for NetFx target frameworks, we use "netfx" as the architecture, and ignore defined runtime identifiers - var netFxPackages = - calamariPackages.Where(p => !p.CrossPlatform) - .Select(packageToBuild => new CalamariPackageMetadata() - { - Project = packageToBuild.Project, - Framework = packageToBuild.Framework, - Architecture = null, - IsCrossPlatform = packageToBuild.CrossPlatform - }); - - // for cross-platform frameworks, we combine each runtime identifier with each target framework - var crossPlatformPackages = - calamariPackages.Where(p => p.CrossPlatform) - .Where(p => string.IsNullOrWhiteSpace(TargetFramework) || p.Framework == TargetFramework) - .SelectMany(packageToBuild => GetRuntimeIdentifiers(packageToBuild.Project) ?? Enumerable.Empty(), - (packageToBuild, runtimeIdentifier) => new CalamariPackageMetadata() - { - Project = packageToBuild.Project, - Framework = packageToBuild.Framework, - Architecture = runtimeIdentifier, - IsCrossPlatform = packageToBuild.CrossPlatform - }) - .Distinct(t => new { t.Project?.Name, t.Architecture, t.Framework }); - - var packagesToPublish = crossPlatformPackages.Concat(netFxPackages).ToArray(); - - packagesToPublish.ForEach(PublishPackage); - await Task.WhenAll(SignDirectoriesTasks); - - StageLegacyCalamariAssemblies(packagesToPublish); - - projects.ForEach(CompressCalamariProject); - await Task.WhenAll(ProjectCompressionTasks); - } + Target RestoreCalamariProjects => + d => + d.DependsOn(GetCalamariFlavourProjectsToPublish) + .Executes(() => + { + PackagesToPublish + .Select(p => p.Architecture) + .Distinct() + .ForEach(rid => DotNetRestore(s => + s.SetProjectFile(Solution) + .SetProperty("DisableImplicitNuGetFallbackFolder", true) + .SetRuntime(rid))); + }); - void PublishPackage(CalamariPackageMetadata calamariPackageMetadata) + Target BuildCalamariProjects => + d => + d.DependsOn(RestoreCalamariProjects) + .Executes(async () => + { + var globalSemaphore = new SemaphoreSlim(3); + var semaphores = new ConcurrentDictionary(); + + var buildTasks = PackagesToPublish.Select(async calamariPackageMetadata => + { + if (!OperatingSystem.IsWindows() && !calamariPackageMetadata.IsCrossPlatform) + { + Log.Warning($"Not Building {calamariPackageMetadata.Framework}: can only publish netfx on a Windows OS"); + return; + } + + var projectName = calamariPackageMetadata.Project?.Name ?? throw new Exception("Could not find project name"); + var projectSemaphore = semaphores.GetOrAdd(projectName, _ => new SemaphoreSlim(1, 1)); + + // for NetFx target frameworks, we use "netfx" as the architecture, and ignore defined runtime identifiers, here we'll just block on all netfx + var architectureSemaphore = semaphores.GetOrAdd(calamariPackageMetadata.Architecture ?? "Unknown Architecture", _ => new SemaphoreSlim(1, 1)); + + await globalSemaphore.WaitAsync(); + await projectSemaphore.WaitAsync(); + await architectureSemaphore.WaitAsync(); + try + { + Log.Information($"Building {calamariPackageMetadata.Project?.Name} for framework '{calamariPackageMetadata.Framework}' and arch '{calamariPackageMetadata.Architecture}'"); + + await Task.Run(() => + DotNetBuild(s => + s.SetProjectFile(calamariPackageMetadata.Project) + .SetConfiguration(Configuration) + .SetFramework(calamariPackageMetadata.Framework) + .EnableNoRestore() + .SetRuntime(calamariPackageMetadata.Architecture) + .EnableSelfContained())); + } + finally + { + projectSemaphore.Release(); + architectureSemaphore.Release(); + globalSemaphore.Release(); + } + }); + + await Task.WhenAll(buildTasks); + }); + + Target PublishCalamariProjects => + d => + d.DependsOn(BuildCalamariProjects) + .Executes(async () => + { + var semaphore = new SemaphoreSlim(4); + var outputPaths = new ConcurrentBag(); + + var publishTasks = PackagesToPublish.Select(async package => + { + await semaphore.WaitAsync(); + try + { + var outputPath = await PublishPackageAsync(package); + outputPaths.Add(outputPath); + } + finally + { + semaphore.Release(); + } + }); + + await Task.WhenAll(publishTasks); + + // Sign and compress tasks + var signTasks = outputPaths + .Where(output => output != null && !output.ToString().Contains("Tests")) + .Select(output => Task.Run(() => SignDirectory(output!))) + .ToList(); + + await Task.WhenAll(signTasks); + StageLegacyCalamariAssemblies(PackagesToPublish); + CalamariProjects.ForEach(CompressCalamariProject); + await Task.WhenAll(ProjectCompressionTasks); + }); + + async Task PublishPackageAsync(CalamariPackageMetadata calamariPackageMetadata) { if (!OperatingSystem.IsWindows() && !calamariPackageMetadata.IsCrossPlatform) { - Log.Warning($"Not building {calamariPackageMetadata.Framework}: can only build netfx on a Windows OS"); - return; + Log.Warning($"Not publishing {calamariPackageMetadata.Framework}: can only publish netfx on a Windows OS"); + return null; } - Log.Information($"Building {calamariPackageMetadata.Project?.Name} for framework '{calamariPackageMetadata.Framework}' and arch '{calamariPackageMetadata.Architecture}'"); + Log.Information($"Publishing {calamariPackageMetadata.Project?.Name} for framework '{calamariPackageMetadata.Framework}' and arch '{calamariPackageMetadata.Architecture}'"); var project = calamariPackageMetadata.Project; var outputDirectory = PublishDirectory / project?.Name / (calamariPackageMetadata.IsCrossPlatform ? calamariPackageMetadata.Architecture : "netfx"); - DotNetPublish(s => - s.SetConfiguration(Configuration) - .SetProject(project) - .SetFramework(calamariPackageMetadata.Framework) - .SetRuntime(calamariPackageMetadata.Architecture) - //explicitly mark all of these as self-contained (because they use a specific runtime) - .EnableSelfContained() - .SetOutput(outputDirectory) - ); - - if (!project.Name.Contains("Tests")) - { - var signDirectoryTask = Task.Run(() => SignDirectory(outputDirectory)); - SignDirectoriesTasks.Add(signDirectoryTask); - } + await Task.Run(() => + DotNetPublish(s => + s.SetConfiguration(Configuration) + .SetProject(project) + .SetFramework(calamariPackageMetadata.Framework) + .SetRuntime(calamariPackageMetadata.Architecture) + .EnableNoBuild() + .EnableNoRestore() + .EnableSelfContained() + .SetOutput(outputDirectory))); File.Copy(RootDirectory / "global.json", outputDirectory / "global.json"); + + return outputDirectory; } static void StageLegacyCalamariAssemblies(CalamariPackageMetadata[] packagesToPublish) @@ -397,7 +481,7 @@ void CompressCalamariProject(Project project) Target PackLegacyCalamari => d => d.DependsOn(Publish) - .DependsOn(PublishCalamariFlavourProjects) + .DependsOn(PublishCalamariProjects) .Executes(() => { if (!OperatingSystem.IsWindows()) @@ -412,7 +496,7 @@ void CompressCalamariProject(Project project) Target PackBinaries => d => d.DependsOn(Publish) - .DependsOn(PublishCalamariFlavourProjects) + .DependsOn(PublishCalamariProjects) .Executes(async () => { var nugetVersion = NugetVersion.Value; @@ -464,7 +548,7 @@ void CompressCalamariProject(Project project) Target PackTests => d => d.DependsOn(Publish) - .DependsOn(PublishCalamariFlavourProjects) + .DependsOn(PublishCalamariProjects) .Executes(async () => { var nugetVersion = NugetVersion.Value; @@ -481,8 +565,7 @@ void CompressCalamariProject(Project project) foreach (var rid in GetRuntimeIdentifiers(Solution.GetProject("Calamari.Tests")!)!) actions.Add(() => { - var publishedLocation = - DoPublish("Calamari.Tests", Frameworks.Net60, nugetVersion, rid); + var publishedLocation = DoPublish("Calamari.Tests", Frameworks.Net60, nugetVersion, rid); var zipName = $"Calamari.Tests.{rid}.{nugetVersion}.zip"; File.Copy(RootDirectory / "global.json", publishedLocation / "global.json"); CompressionTasks.Compress(publishedLocation, ArtifactsDirectory / zipName); @@ -615,16 +698,8 @@ void CompressCalamariProject(Project project) async Task RunPackActions(List actions) { - if (PackInParallel) - { var tasks = actions.Select(Task.Run).ToList(); await Task.WhenAll(tasks); - } - else - { - foreach (var action in actions) - action(); - } } AbsolutePath DoPublish(string project, string framework, string version, string? runtimeId = null) @@ -646,12 +721,15 @@ AbsolutePath DoPublish(string project, string framework, string version, string? .SetVerbosity(BuildVerbosity) .SetRuntime(runtimeId) .SetVersion(version) - .EnableSelfContained()); + .EnableSelfContained() + ); if (WillSignBinaries) + { Signing.SignAndTimestampBinaries(publishedTo, AzureKeyVaultUrl, AzureKeyVaultAppId, AzureKeyVaultAppSecret, AzureKeyVaultTenantId, AzureKeyVaultCertificateName, SigningCertificatePath, SigningCertificatePassword); + } return publishedTo; }