From 2c1186ed43345519b28590c121172447325de5e6 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Wed, 12 Aug 2026 16:04:25 -0700 Subject: [PATCH] [xharness] Build simulator variations in parallel Use a parent MSBuild invocation with BuildInParallel and /m so MSBuild controls concurrency across cloned simulator variation projects. Keep restores and simulator execution serialized, and preserve per-variation build results. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../TestTasks/AggregatedRunSimulatorTask.cs | 30 ++- .../xharness/Jenkins/TestTasks/MSBuildTask.cs | 196 ++++++++++++++++++ tests/xharness/Jenkins/TestTasks/RunTest.cs | 29 ++- .../xharness/Jenkins/TestTasks/RunTestTask.cs | 4 + 4 files changed, 254 insertions(+), 5 deletions(-) diff --git a/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs b/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs index 19c02906f162..888f456e72da 100644 --- a/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs +++ b/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.DotNet.XHarness.iOS.Shared; +using Microsoft.DotNet.XHarness.iOS.Shared.Logging; #nullable enable @@ -44,9 +45,24 @@ protected override async Task ExecuteAsync () // First build everything. This is required for the run simulator // task to properly configure the simulator. buildTimer.Start (); - await Task.WhenAll (Tasks.Select ((v) => v.BuildAsync ()).Distinct ()); + var prepareBuildTasks = Tasks.Select (PrepareBuildAsync).ToArray (); + var preparedBuildTasks = await Task.WhenAll (prepareBuildTasks); + var tasksToBuild = preparedBuildTasks.Where (v => v.Build == true).Select (v => v.Task).ToArray (); + if (tasksToBuild.Length > 0) { + using var buildLog = Logs.Create ($"parallel-build-{Xharness.Harness.Helpers.Timestamp}.txt", LogType.BuildLog.ToString ()); + await MSBuildTask.BuildInParallelAsync (tasksToBuild.Select (v => (MSBuildTask) v.BuildTask).ToArray (), buildLog, Jenkins.MainLog, Jenkins.Harness.DryRun); + foreach (var task in tasksToBuild) + task.CompleteBuild (); + } buildTimer.Stop (); + if (Jenkins.Harness.DryRun) { + foreach (var task in Tasks.Where (v => !v.Ignored && !v.Failed)) + task.ExecutionResult = TestExecutingResult.BuildSucceeded | TestExecutingResult.Finished; + ExecutionResult = TestExecutingResult.Succeeded; + return; + } + var executingTasks = Tasks.Where ((v) => !v.Ignored && !v.Failed); if (!executingTasks.Any ()) { ExecutionResult = TestExecutingResult.Failed; @@ -104,5 +120,17 @@ protected override async Task ExecuteAsync () ExecutionResult = Tasks.Any ((v) => v.Failed) ? TestExecutingResult.Failed : TestExecutingResult.Succeeded; } } + + static async Task<(RunSimulatorTask Task, bool? Build)> PrepareBuildAsync (RunSimulatorTask task) + { + try { + return (task, await task.PrepareBuildAsync ()); + } catch (Exception e) { + task.BuildTask.ExecutionResult = TestExecutingResult.HarnessException | TestExecutingResult.Finished; + task.BuildTask.FailureMessage = $"Harness exception while preparing '{task.TestName}' for build: {e}"; + task.CompleteBuild (); + return (task, false); + } + } } } diff --git a/tests/xharness/Jenkins/TestTasks/MSBuildTask.cs b/tests/xharness/Jenkins/TestTasks/MSBuildTask.cs index eb1edff9fcc4..1264dbbe0494 100644 --- a/tests/xharness/Jenkins/TestTasks/MSBuildTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MSBuildTask.cs @@ -1,9 +1,17 @@ +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Threading.Tasks; +using System.Xml; using Microsoft.DotNet.XHarness.Common.Execution; +using Microsoft.DotNet.XHarness.Common.Logging; +using Microsoft.DotNet.XHarness.iOS.Shared; using Microsoft.DotNet.XHarness.iOS.Shared.Logging; +using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; +using Xamarin; +using Xamarin.Utils; namespace Xharness.Jenkins.TestTasks { public class MSBuildTask : BuildProjectTask { @@ -60,6 +68,194 @@ protected override async Task ExecuteAsync () BuildLog.Dispose (); } + public override IEnumerable AggregatedLogs + => BuildLog is null ? base.AggregatedLogs : base.AggregatedLogs.Union (new [] { BuildLog }); + + public static async Task BuildInParallelAsync (IReadOnlyList tasks, IFileBackedLog buildLog, ILog mainLog, bool dryRun) + { + if (tasks.Count == 0) + return; + + var buildDirectory = Cache.CreateTemporaryDirectory ("parallel-msbuild"); + var projectFile = Path.Combine (buildDirectory, "build.proj"); + var buildProjects = new List<(MSBuildTask Task, string Project, string SuccessMarker, string FailureMarker, string RestoreFailureMarker)> (); + + for (var i = 0; i < tasks.Count; i++) { + var task = tasks [i]; + var wrapperProject = Path.Combine (buildDirectory, $"build-{i}.proj"); + var successMarker = Path.Combine (buildDirectory, $"build-{i}.success"); + var failureMarker = Path.Combine (buildDirectory, $"build-{i}.failure"); + var restoreFailureMarker = Path.Combine (buildDirectory, $"restore-{i}.failure"); + WriteBuildProject (wrapperProject, successMarker, failureMarker, restoreFailureMarker, task); + buildProjects.Add ((task, wrapperProject, successMarker, failureMarker, restoreFailureMarker)); + } + WriteRootBuildProject (projectFile, buildProjects.Select (v => v.Project)); + + var binlogPath = Path.ChangeExtension (buildLog.FullPath, ".binlog"); + var restoreBinlogPath = Path.ChangeExtension (buildLog.FullPath, ".restore.binlog"); + var firstTask = tasks [0]; + + mainLog.WriteLine ($"Building {tasks.Count} projects in parallel"); + ProcessExecutionResult? processResult = null; + if (!dryRun) { + var timeout = TimeSpan.FromMinutes (60); + using (await firstTask.ResourceManager.DesktopResource.AcquireExclusiveAsync ()) { + ProcessExecutionResult restoreResult; + using (await firstTask.ResourceManager.NugetResource.AcquireExclusiveAsync ()) { + mainLog.WriteLine ($"Restoring {tasks.Count} projects serially"); + restoreResult = await RunMSBuildAsync (firstTask, projectFile, "Restore", restoreBinlogPath, false, buildLog, timeout); + } + processResult = restoreResult.TimedOut + ? restoreResult + : await RunMSBuildAsync (firstTask, projectFile, "Build", binlogPath, true, buildLog, timeout); + } + } + + var failedTasks = new List (); + foreach (var buildProject in buildProjects) { + var succeeded = dryRun || File.Exists (buildProject.SuccessMarker); + var failed = File.Exists (buildProject.FailureMarker) || File.Exists (buildProject.RestoreFailureMarker); + var task = buildProject.Task; + task.BuildLog = buildLog; + task.KnownFailure = null; + task.ExecutionResult = (succeeded ? TestExecutingResult.Succeeded : failed ? TestExecutingResult.Failed : processResult?.TimedOut == true ? TestExecutingResult.TimedOut : TestExecutingResult.Failed) | TestExecutingResult.Finished; + if (!succeeded) { + task.FailureMessage = failed || processResult?.TimedOut != true ? "Project failed in the parallel build." : "Parallel build timed out."; + failedTasks.Add (task); + } + } + + if (failedTasks.Count == 1 && firstTask.Jenkins.ErrorKnowledgeBase.IsKnownBuildIssue (buildLog, out var knownFailure)) { + failedTasks [0].KnownFailure = knownFailure; + } + mainLog.WriteLine ($"Built {(dryRun ? tasks.Count : buildProjects.Count (v => File.Exists (v.SuccessMarker)))} of {tasks.Count} projects in parallel"); + } + + static async Task RunMSBuildAsync (MSBuildTask task, string projectFile, string target, string binlogPath, bool buildInParallel, ILog buildLog, TimeSpan timeout) + { + using var process = new Process (); + process.StartInfo.FileName = task.ToolName; + var arguments = new List { + "msbuild", + $"/t:{target}", + "/verbosity:diagnostic", + $"/bl:{binlogPath}", + projectFile, + }; + if (buildInParallel) + arguments.Insert (1, "/m"); + process.StartInfo.Arguments = StringUtils.FormatArguments (arguments); + process.StartInfo.WorkingDirectory = Path.GetDirectoryName (projectFile); + task.SetEnvironmentVariables (process); + return await task.ProcessManager.RunAsync (process, buildLog, timeout); + } + + static void WriteBuildProject (string path, string successMarker, string failureMarker, string restoreFailureMarker, MSBuildTask task) + { + var properties = new List { + $"RootTestsDirectory={EscapePropertyValue (HarnessConfiguration.RootDirectory)}", + }; + if (task.SpecifyPlatform) + properties.Add ($"Platform={EscapePropertyValue (task.ProjectPlatform ?? "")}"); + if (task.SpecifyConfiguration) + properties.Add ($"Configuration={EscapePropertyValue (task.ProjectConfiguration ?? "")}"); + if (task.Constants.Count > 0) + properties.Add ($"DefineConstants={EscapePropertyValue (string.Join (";", task.Constants))}"); + + var settings = new XmlWriterSettings { Indent = true }; + using var writer = XmlWriter.Create (path, settings); + writer.WriteStartElement ("Project"); + writer.WriteStartElement ("PropertyGroup"); + writer.WriteElementString ("ProjectToBuild", task.ProjectFile); + writer.WriteElementString ("BuildProperties", string.Join (";", properties)); + writer.WriteElementString ("SuccessMarker", successMarker); + writer.WriteElementString ("FailureMarker", failureMarker); + writer.WriteElementString ("RestoreFailureMarker", restoreFailureMarker); + writer.WriteEndElement (); + writer.WriteStartElement ("Target"); + writer.WriteAttributeString ("Name", "Restore"); + writer.WriteStartElement ("MSBuild"); + writer.WriteAttributeString ("Projects", "$(ProjectToBuild)"); + writer.WriteAttributeString ("Targets", "Restore"); + writer.WriteAttributeString ("Properties", "$(BuildProperties)"); + writer.WriteEndElement (); + writer.WriteStartElement ("OnError"); + writer.WriteAttributeString ("ExecuteTargets", "RestoreFailed"); + writer.WriteEndElement (); + writer.WriteEndElement (); + writer.WriteStartElement ("Target"); + writer.WriteAttributeString ("Name", "RestoreFailed"); + writer.WriteStartElement ("WriteLinesToFile"); + writer.WriteAttributeString ("File", "$(RestoreFailureMarker)"); + writer.WriteAttributeString ("Lines", "failure"); + writer.WriteAttributeString ("Overwrite", "true"); + writer.WriteEndElement (); + writer.WriteEndElement (); + writer.WriteStartElement ("Target"); + writer.WriteAttributeString ("Name", "Build"); + writer.WriteStartElement ("MSBuild"); + writer.WriteAttributeString ("Projects", "$(ProjectToBuild)"); + writer.WriteAttributeString ("Targets", "Build"); + writer.WriteAttributeString ("Properties", "$(BuildProperties)"); + writer.WriteEndElement (); + writer.WriteStartElement ("WriteLinesToFile"); + writer.WriteAttributeString ("File", "$(SuccessMarker)"); + writer.WriteAttributeString ("Lines", "success"); + writer.WriteAttributeString ("Overwrite", "true"); + writer.WriteEndElement (); + writer.WriteStartElement ("OnError"); + writer.WriteAttributeString ("ExecuteTargets", "BuildFailed"); + writer.WriteEndElement (); + writer.WriteEndElement (); + writer.WriteStartElement ("Target"); + writer.WriteAttributeString ("Name", "BuildFailed"); + writer.WriteStartElement ("WriteLinesToFile"); + writer.WriteAttributeString ("File", "$(FailureMarker)"); + writer.WriteAttributeString ("Lines", "failure"); + writer.WriteAttributeString ("Overwrite", "true"); + writer.WriteEndElement (); + writer.WriteEndElement (); + writer.WriteEndElement (); + } + + static void WriteRootBuildProject (string path, IEnumerable projects) + { + var settings = new XmlWriterSettings { Indent = true }; + using var writer = XmlWriter.Create (path, settings); + writer.WriteStartElement ("Project"); + writer.WriteAttributeString ("DefaultTargets", "Build"); + writer.WriteStartElement ("ItemGroup"); + foreach (var project in projects) { + writer.WriteStartElement ("ProjectsToBuild"); + writer.WriteAttributeString ("Include", project); + writer.WriteEndElement (); + } + writer.WriteEndElement (); + writer.WriteStartElement ("Target"); + writer.WriteAttributeString ("Name", "Restore"); + writer.WriteStartElement ("MSBuild"); + writer.WriteAttributeString ("Projects", "@(ProjectsToBuild)"); + writer.WriteAttributeString ("Targets", "Restore"); + writer.WriteAttributeString ("BuildInParallel", "false"); + writer.WriteAttributeString ("StopOnFirstFailure", "false"); + writer.WriteAttributeString ("ContinueOnError", "WarnAndContinue"); + writer.WriteEndElement (); + writer.WriteEndElement (); + writer.WriteStartElement ("Target"); + writer.WriteAttributeString ("Name", "Build"); + writer.WriteStartElement ("MSBuild"); + writer.WriteAttributeString ("Projects", "@(ProjectsToBuild)"); + writer.WriteAttributeString ("Targets", "Build"); + writer.WriteAttributeString ("BuildInParallel", "true"); + writer.WriteAttributeString ("StopOnFirstFailure", "false"); + writer.WriteEndElement (); + writer.WriteEndElement (); + writer.WriteEndElement (); + } + + static string EscapePropertyValue (string value) + => value.Replace ("%", "%25").Replace (";", "%3B"); + public override Task CleanAsync () => MSBuild.CleanAsync ( projectPlatform: ProjectPlatform!, diff --git a/tests/xharness/Jenkins/TestTasks/RunTest.cs b/tests/xharness/Jenkins/TestTasks/RunTest.cs index 46af2d0be5fd..136ae5fde223 100644 --- a/tests/xharness/Jenkins/TestTasks/RunTest.cs +++ b/tests/xharness/Jenkins/TestTasks/RunTest.cs @@ -49,17 +49,24 @@ public RunTest (RunTestTask testTask, public IEnumerable BuildAggregatedLogs => BuildTask.AggregatedLogs; public TestExecutingResult BuildResult => BuildTask.ExecutionResult; - public async Task BuildAsync () + public async Task PrepareBuildAsync () { if (testTask.Finished) - return true; + return null; await testTask.VerifyBuildAsync (); if (testTask.Finished) - return BuildTask.Succeeded; + return false; testTask.ExecutionResult = TestExecutingResult.Building; - await BuildTask.RunAsync (); + if (BuildTask.InitialTask is not null) + await BuildTask.InitialTask; + + return true; + } + + public bool CompleteBuild () + { if (!BuildTask.Succeeded) { if (BuildTask.TimedOut) { testTask.ExecutionResult = TestExecutingResult.TimedOut; @@ -91,6 +98,20 @@ public async Task BuildAsync () return BuildTask.Succeeded; } + public async Task BuildAsync () + { + if (testTask.Finished) + return true; + + await testTask.VerifyBuildAsync (); + if (testTask.Finished) + return BuildTask.Succeeded; + + testTask.ExecutionResult = TestExecutingResult.Building; + await BuildTask.RunAsync (); + return CompleteBuild (); + } + public async Task ExecuteAsync () { if (testTask.Finished) diff --git a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs index 5d8500feaf25..e3e2af029edb 100644 --- a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs @@ -76,6 +76,10 @@ public override TestExecutingResult ExecutionResult { public Task BuildAsync () => runTest.BuildAsync (); + public Task PrepareBuildAsync () => runTest.PrepareBuildAsync (); + + public bool CompleteBuild () => runTest.CompleteBuild (); + protected override Task ExecuteAsync () => runTest.ExecuteAsync (); public abstract Task RunTestAsync ();