From 727782e4b0da13ac45dd9fa914cb1cf2507dfd73 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 15:54:21 +0200 Subject: [PATCH 01/95] Add iOSInnerLoopParser, wire Startup.cs, fix Reporter.cs - Create iOSInnerLoopParser.cs: binlog parser for iOS inner loop build timings, extracting iOS-specific tasks (AOTCompile, Codesign, MTouch, etc.) and targets (_AOTCompile, _CodesignAppBundle, _CreateAppBundle, etc.) plus shared tasks (Csc, XamlC, LinkAssembliesNoShrink) - Wire into Startup.cs: add iOSInnerLoop to MetricType enum and map it to iOSInnerLoopParser in the parser switch expression - Fix Reporter.cs: guard against null/empty PERFLAB_BUILDTIMESTAMP to prevent ArgumentNullException on DateTime.Parse(null) when the env var is unset (falls back to DateTime.Now) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/tools/Reporting/Reporting/Reporter.cs | 5 +- .../ScenarioMeasurement/Startup/Startup.cs | 2 + .../Util/Parsers/iOSInnerLoopParser.cs | 224 ++++++++++++++++++ 3 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 src/tools/ScenarioMeasurement/Util/Parsers/iOSInnerLoopParser.cs diff --git a/src/tools/Reporting/Reporting/Reporter.cs b/src/tools/Reporting/Reporting/Reporter.cs index a405e6e5096..1d5b57e9827 100644 --- a/src/tools/Reporting/Reporting/Reporter.cs +++ b/src/tools/Reporting/Reporting/Reporter.cs @@ -110,6 +110,9 @@ private void InitializeFromEnvironment(IEnvironment environment) private static Build ParseBuildInfo(IEnvironment environment) { + var buildTimestampStr = environment.GetEnvironmentVariable("PERFLAB_BUILDTIMESTAMP"); + var buildTimestamp = !string.IsNullOrEmpty(buildTimestampStr) ? DateTime.Parse(buildTimestampStr) : DateTime.Now; + var build = new Build() { Repo = environment.GetEnvironmentVariable("PERFLAB_REPO"), @@ -118,7 +121,7 @@ private static Build ParseBuildInfo(IEnvironment environment) Locale = environment.GetEnvironmentVariable("PERFLAB_LOCALE"), GitHash = environment.GetEnvironmentVariable("PERFLAB_HASH"), BuildName = environment.GetEnvironmentVariable("PERFLAB_BUILDNUM"), - TimeStamp = DateTime.Parse(environment.GetEnvironmentVariable("PERFLAB_BUILDTIMESTAMP")), + TimeStamp = buildTimestamp, }; diff --git a/src/tools/ScenarioMeasurement/Startup/Startup.cs b/src/tools/ScenarioMeasurement/Startup/Startup.cs index 3c8a180561b..1ea81840d63 100644 --- a/src/tools/ScenarioMeasurement/Startup/Startup.cs +++ b/src/tools/ScenarioMeasurement/Startup/Startup.cs @@ -27,6 +27,7 @@ enum MetricType WinUIBlazor, TimeToMain2, BuildTime, + iOSInnerLoop, } public class InnerLoopMarkerEventSource : EventSource @@ -291,6 +292,7 @@ static void checkArg(string arg, string name) MetricType.WinUIBlazor => new WinUIBlazorParser(), MetricType.TimeToMain2 => new TimeToMain2Parser(AddTestProcessEnvironmentVariable), MetricType.BuildTime => new BuildTimeParser(), + MetricType.iOSInnerLoop => new iOSInnerLoopParser(), _ => throw new ArgumentOutOfRangeException(), }; diff --git a/src/tools/ScenarioMeasurement/Util/Parsers/iOSInnerLoopParser.cs b/src/tools/ScenarioMeasurement/Util/Parsers/iOSInnerLoopParser.cs new file mode 100644 index 00000000000..d91ff5e31d1 --- /dev/null +++ b/src/tools/ScenarioMeasurement/Util/Parsers/iOSInnerLoopParser.cs @@ -0,0 +1,224 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Build.Logging.StructuredLogger; +using StructuredLogViewer; +using Microsoft.Diagnostics.Tracing; +using Reporting; + +namespace ScenarioMeasurement; + +/// +/// Parses iOS inner loop (build+deploy) target and task durations from a binary log file. +/// +public class iOSInnerLoopParser : IParser +{ + public void EnableKernelProvider(ITraceSession kernel) { throw new NotImplementedException(); } + public void EnableUserProviders(ITraceSession user) { throw new NotImplementedException(); } + + public IEnumerable Parse(string binlogFile, string processName, IList pids, string commandLine) + { + var buildDeployTimes = new List(); + + // Build tasks (shared) + var cscTimes = new List(); + var xamlCTimes = new List(); + var linkAssembliesNoShrinkTimes = new List(); + var filterAssembliesTimes = new List(); + var resolveSdksTimes = new List(); + var processAssembliesTimes = new List(); + var generateNativeApplicationConfigSourcesTimes = new List(); + + // iOS-specific build tasks + var aotCompileTimes = new List(); + var monoAOTCompilerTimes = new List(); + var codesignTimes = new List(); + var compileNativeFilesTimes = new List(); + var linkNativeCodeTimes = new List(); + var generateBundleNameTimes = new List(); + var createAssetPackTimes = new List(); + var computeCodesignInputsTimes = new List(); + var detectSigningIdentityTimes = new List(); + var compileAppManifestTimes = new List(); + var compileEntitlementsTimes = new List(); + var createBindingResourcePackageTimes = new List(); + var mTouchTimes = new List(); + var acToolTimes = new List(); + var ibToolTimes = new List(); + var dSymUtilTimes = new List(); + + // Build targets (shared) + var coreCompileTargetTimes = new List(); + var xamlCTargetTimes = new List(); + + // iOS-specific build targets + var aotCompileTargetTimes = new List(); + var codesignAppBundleTargetTimes = new List(); + var compileToNativeTargetTimes = new List(); + var createAppBundleTargetTimes = new List(); + var copyResourcesToBundleTargetTimes = new List(); + var generateBundleNameTargetTimes = new List(); + + if (File.Exists(binlogFile)) + { + var build = BinaryLog.ReadBuild(binlogFile); + BuildAnalyzer.AnalyzeBuild(build); + + foreach (var task in build.FindChildrenRecursive()) + { + var name = task.Name; + var s = task.Duration.TotalMilliseconds / 1000.0; + + // Shared build tasks + if (name.Equals("Csc", StringComparison.OrdinalIgnoreCase)) + cscTimes.Add(s); + else if (name.Equals("XamlCTask", StringComparison.OrdinalIgnoreCase)) + xamlCTimes.Add(s); + else if (name.Equals("LinkAssembliesNoShrink", StringComparison.OrdinalIgnoreCase)) + linkAssembliesNoShrinkTimes.Add(s); + else if (name.Equals("FilterAssemblies", StringComparison.OrdinalIgnoreCase)) + filterAssembliesTimes.Add(s); + else if (name.Equals("ResolveSdks", StringComparison.OrdinalIgnoreCase)) + resolveSdksTimes.Add(s); + else if (name.Equals("ProcessAssemblies", StringComparison.OrdinalIgnoreCase)) + processAssembliesTimes.Add(s); + else if (name.Equals("GenerateNativeApplicationConfigSources", StringComparison.OrdinalIgnoreCase)) + generateNativeApplicationConfigSourcesTimes.Add(s); + // iOS-specific build tasks + else if (name.Equals("AOTCompile", StringComparison.OrdinalIgnoreCase)) + aotCompileTimes.Add(s); + else if (name.Equals("MonoAOTCompiler", StringComparison.OrdinalIgnoreCase)) + monoAOTCompilerTimes.Add(s); + else if (name.Equals("Codesign", StringComparison.OrdinalIgnoreCase)) + codesignTimes.Add(s); + else if (name.Equals("CompileNativeFiles", StringComparison.OrdinalIgnoreCase)) + compileNativeFilesTimes.Add(s); + else if (name.Equals("LinkNativeCode", StringComparison.OrdinalIgnoreCase)) + linkNativeCodeTimes.Add(s); + else if (name.Equals("GenerateBundleName", StringComparison.OrdinalIgnoreCase)) + generateBundleNameTimes.Add(s); + else if (name.Equals("CreateAssetPack", StringComparison.OrdinalIgnoreCase)) + createAssetPackTimes.Add(s); + else if (name.Equals("ComputeCodesignInputs", StringComparison.OrdinalIgnoreCase)) + computeCodesignInputsTimes.Add(s); + else if (name.Equals("DetectSigningIdentity", StringComparison.OrdinalIgnoreCase)) + detectSigningIdentityTimes.Add(s); + else if (name.Equals("CompileAppManifest", StringComparison.OrdinalIgnoreCase)) + compileAppManifestTimes.Add(s); + else if (name.Equals("CompileEntitlements", StringComparison.OrdinalIgnoreCase)) + compileEntitlementsTimes.Add(s); + else if (name.Equals("CreateBindingResourcePackage", StringComparison.OrdinalIgnoreCase)) + createBindingResourcePackageTimes.Add(s); + else if (name.Equals("MTouch", StringComparison.OrdinalIgnoreCase)) + mTouchTimes.Add(s); + else if (name.Equals("ACTool", StringComparison.OrdinalIgnoreCase)) + acToolTimes.Add(s); + else if (name.Equals("IBTool", StringComparison.OrdinalIgnoreCase)) + ibToolTimes.Add(s); + else if (name.Equals("DSymUtil", StringComparison.OrdinalIgnoreCase)) + dSymUtilTimes.Add(s); + } + + foreach (var target in build.FindChildrenRecursive()) + { + var name = target.Name; + var s = target.Duration.TotalMilliseconds / 1000.0; + + // Shared build targets + if (name.Equals("CoreCompile", StringComparison.Ordinal)) + coreCompileTargetTimes.Add(s); + else if (name.Equals("XamlC", StringComparison.Ordinal)) + xamlCTargetTimes.Add(s); + // iOS-specific build targets + else if (name.Equals("_AOTCompile", StringComparison.Ordinal)) + aotCompileTargetTimes.Add(s); + else if (name.Equals("_CodesignAppBundle", StringComparison.Ordinal)) + codesignAppBundleTargetTimes.Add(s); + else if (name.Equals("_CompileToNative", StringComparison.Ordinal)) + compileToNativeTargetTimes.Add(s); + else if (name.Equals("_CreateAppBundle", StringComparison.Ordinal)) + createAppBundleTargetTimes.Add(s); + else if (name.Equals("_CopyResourcesToBundle", StringComparison.Ordinal)) + copyResourcesToBundleTargetTimes.Add(s); + else if (name.Equals("_GenerateBundleName", StringComparison.Ordinal)) + generateBundleNameTargetTimes.Add(s); + } + + buildDeployTimes.Add(build.Duration.TotalMilliseconds / 1000.0); + } + + // Overall duration + if (buildDeployTimes.Count > 0) + yield return new Counter { Name = "Build Time", MetricName = "s", DefaultCounter = true, TopCounter = true, Results = buildDeployTimes.ToArray() }; + + // Shared build task counters + if (cscTimes.Count > 0) + yield return new Counter { Name = "Csc Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = cscTimes.ToArray() }; + if (xamlCTimes.Count > 0) + yield return new Counter { Name = "XamlC Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = xamlCTimes.ToArray() }; + if (linkAssembliesNoShrinkTimes.Count > 0) + yield return new Counter { Name = "LinkAssembliesNoShrink Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = linkAssembliesNoShrinkTimes.ToArray() }; + if (filterAssembliesTimes.Count > 0) + yield return new Counter { Name = "FilterAssemblies Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = filterAssembliesTimes.ToArray() }; + if (resolveSdksTimes.Count > 0) + yield return new Counter { Name = "ResolveSdks Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = resolveSdksTimes.ToArray() }; + if (processAssembliesTimes.Count > 0) + yield return new Counter { Name = "ProcessAssemblies Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = processAssembliesTimes.ToArray() }; + if (generateNativeApplicationConfigSourcesTimes.Count > 0) + yield return new Counter { Name = "GenerateNativeApplicationConfigSources Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateNativeApplicationConfigSourcesTimes.ToArray() }; + + // iOS-specific build task counters + if (aotCompileTimes.Count > 0) + yield return new Counter { Name = "AOTCompile Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = aotCompileTimes.ToArray() }; + if (monoAOTCompilerTimes.Count > 0) + yield return new Counter { Name = "MonoAOTCompiler Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = monoAOTCompilerTimes.ToArray() }; + if (codesignTimes.Count > 0) + yield return new Counter { Name = "Codesign Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = codesignTimes.ToArray() }; + if (compileNativeFilesTimes.Count > 0) + yield return new Counter { Name = "CompileNativeFiles Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileNativeFilesTimes.ToArray() }; + if (linkNativeCodeTimes.Count > 0) + yield return new Counter { Name = "LinkNativeCode Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = linkNativeCodeTimes.ToArray() }; + if (generateBundleNameTimes.Count > 0) + yield return new Counter { Name = "GenerateBundleName Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateBundleNameTimes.ToArray() }; + if (createAssetPackTimes.Count > 0) + yield return new Counter { Name = "CreateAssetPack Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = createAssetPackTimes.ToArray() }; + if (computeCodesignInputsTimes.Count > 0) + yield return new Counter { Name = "ComputeCodesignInputs Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = computeCodesignInputsTimes.ToArray() }; + if (detectSigningIdentityTimes.Count > 0) + yield return new Counter { Name = "DetectSigningIdentity Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = detectSigningIdentityTimes.ToArray() }; + if (compileAppManifestTimes.Count > 0) + yield return new Counter { Name = "CompileAppManifest Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileAppManifestTimes.ToArray() }; + if (compileEntitlementsTimes.Count > 0) + yield return new Counter { Name = "CompileEntitlements Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileEntitlementsTimes.ToArray() }; + if (createBindingResourcePackageTimes.Count > 0) + yield return new Counter { Name = "CreateBindingResourcePackage Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = createBindingResourcePackageTimes.ToArray() }; + if (mTouchTimes.Count > 0) + yield return new Counter { Name = "MTouch Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = mTouchTimes.ToArray() }; + if (acToolTimes.Count > 0) + yield return new Counter { Name = "ACTool Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = acToolTimes.ToArray() }; + if (ibToolTimes.Count > 0) + yield return new Counter { Name = "IBTool Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = ibToolTimes.ToArray() }; + if (dSymUtilTimes.Count > 0) + yield return new Counter { Name = "DSymUtil Task Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = dSymUtilTimes.ToArray() }; + + // Shared build target counters + if (coreCompileTargetTimes.Count > 0) + yield return new Counter { Name = "CoreCompile Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = coreCompileTargetTimes.ToArray() }; + if (xamlCTargetTimes.Count > 0) + yield return new Counter { Name = "XamlC Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = xamlCTargetTimes.ToArray() }; + + // iOS-specific build target counters + if (aotCompileTargetTimes.Count > 0) + yield return new Counter { Name = "_AOTCompile Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = aotCompileTargetTimes.ToArray() }; + if (codesignAppBundleTargetTimes.Count > 0) + yield return new Counter { Name = "_CodesignAppBundle Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = codesignAppBundleTargetTimes.ToArray() }; + if (compileToNativeTargetTimes.Count > 0) + yield return new Counter { Name = "_CompileToNative Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = compileToNativeTargetTimes.ToArray() }; + if (createAppBundleTargetTimes.Count > 0) + yield return new Counter { Name = "_CreateAppBundle Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = createAppBundleTargetTimes.ToArray() }; + if (copyResourcesToBundleTargetTimes.Count > 0) + yield return new Counter { Name = "_CopyResourcesToBundle Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = copyResourcesToBundleTargetTimes.ToArray() }; + if (generateBundleNameTargetTimes.Count > 0) + yield return new Counter { Name = "_GenerateBundleName Target Time", MetricName = "s", DefaultCounter = false, TopCounter = true, Results = generateBundleNameTargetTimes.ToArray() }; + } +} From ee23cf0cd165cc7aa84a0c8fbef9210a4f1bd996 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 16:02:13 +0200 Subject: [PATCH 02/95] Add shared Python infrastructure for iOS inner loop - const.py: Add IOSINNERLOOP constant and SCENARIO_NAMES mapping - ioshelper.py: New module with iOSHelper class for simulator and physical device management (boot, install, launch, terminate, uninstall, find bundle) - runner.py: Add iosinnerloop subparser, attribute assignment, and full execution branch (first build+deploy+launch, incremental loop with source toggling, binlog parsing, report aggregation, and Helix upload) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/const.py | 4 +- src/scenarios/shared/ioshelper.py | 167 +++++++++++++++++ src/scenarios/shared/runner.py | 294 +++++++++++++++++++++++++++++- 3 files changed, 463 insertions(+), 2 deletions(-) create mode 100644 src/scenarios/shared/ioshelper.py diff --git a/src/scenarios/shared/const.py b/src/scenarios/shared/const.py index 074fa7fdf89..b43387e366c 100644 --- a/src/scenarios/shared/const.py +++ b/src/scenarios/shared/const.py @@ -19,6 +19,7 @@ ANDROIDINSTRUMENTATION = "androidinstrumentation" DEVICEPOWERCONSUMPTION = "devicepowerconsumption" BUILDTIME = "buildtime" +IOSINNERLOOP = "iosinnerloop" SCENARIO_NAMES = {STARTUP: 'Startup', SDK: 'SDK', @@ -27,7 +28,8 @@ INNERLOOP: 'Innerloop', INNERLOOPMSBUILD: 'InnerLoopMsBuild', DOTNETWATCH: 'DotnetWatch', - BUILDTIME: 'BuildTime'} + BUILDTIME: 'BuildTime', + IOSINNERLOOP: 'iOSInnerLoop'} BINDIR = 'bin' PUBDIR = 'pub' diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py new file mode 100644 index 00000000000..4b7e520262a --- /dev/null +++ b/src/scenarios/shared/ioshelper.py @@ -0,0 +1,167 @@ +import os +import time +import glob +import subprocess +from performance.common import RunCommand +from logging import getLogger + + +class iOSHelper: + def __init__(self): + self.bundle_id = None + self.device_id = None + self.app_bundle_path = None + + def setup_simulator(self, bundle_id, app_bundle_path, device_id='booted'): + """Boot the iOS simulator and install the app bundle.""" + self.bundle_id = bundle_id + self.device_id = device_id + self.app_bundle_path = app_bundle_path + + if device_id != 'booted': + getLogger().info("Booting iOS simulator: %s", device_id) + result = subprocess.run( + ['xcrun', 'simctl', 'boot', device_id], + capture_output=True, text=True + ) + if result.returncode != 0: + if 'already booted' in result.stderr.lower(): + getLogger().info("Simulator %s is already booted.", device_id) + else: + raise subprocess.CalledProcessError(result.returncode, result.args, result.stdout, result.stderr) + else: + getLogger().info("Using already-booted simulator (device_id='booted')") + + # Install app + getLogger().info("Installing app bundle: %s", app_bundle_path) + RunCommand(['xcrun', 'simctl', 'install', device_id, app_bundle_path], verbose=True).run() + getLogger().info("Completed install.") + + def setup_physical_device(self, bundle_id, app_bundle_path, device_id): + """Set up a physical iOS device for testing. + + Installs the app bundle on the connected physical device using devicectl. + """ + self.bundle_id = bundle_id + self.device_id = device_id + self.app_bundle_path = app_bundle_path + + getLogger().info("Installing app bundle on physical device %s: %s", device_id, app_bundle_path) + RunCommand(['xcrun', 'devicectl', 'device', 'install', 'app', + '--device', device_id, app_bundle_path], verbose=True).run() + getLogger().info("Completed install on physical device.") + + def install_app(self, app_bundle_path): + """Install the app bundle and return install time in milliseconds.""" + getLogger().info("Installing app bundle: %s", app_bundle_path) + start = time.time() + RunCommand(['xcrun', 'simctl', 'install', self.device_id, app_bundle_path], verbose=True).run() + elapsed_ms = (time.time() - start) * 1000 + getLogger().info("Install completed in %.1f ms", elapsed_ms) + return elapsed_ms + + def install_app_physical(self, app_bundle_path): + """Install the app bundle on a physical device and return install time in milliseconds.""" + getLogger().info("Installing app bundle on physical device: %s", app_bundle_path) + start = time.time() + RunCommand(['xcrun', 'devicectl', 'device', 'install', 'app', + '--device', self.device_id, app_bundle_path], verbose=True).run() + elapsed_ms = (time.time() - start) * 1000 + getLogger().info("Install completed in %.1f ms", elapsed_ms) + return elapsed_ms + + def measure_cold_startup(self, bundle_id): + """Measure app cold startup time in milliseconds. + + Terminates any running instance, waits briefly, then launches the app. + Returns wall-clock time for the launch command in milliseconds as int. + """ + # Terminate any running instance (ignore errors) + getLogger().info("Terminating app for cold startup: %s", bundle_id) + try: + RunCommand(['xcrun', 'simctl', 'terminate', self.device_id, bundle_id], verbose=True).run() + except subprocess.CalledProcessError: + getLogger().debug("Terminate returned error (app may not be running), ignoring.") + + time.sleep(0.5) + + # Launch and measure wall-clock time + getLogger().info("Launching app: %s", bundle_id) + start = time.time() + RunCommand(['xcrun', 'simctl', 'launch', self.device_id, bundle_id], verbose=True).run() + elapsed_ms = (time.time() - start) * 1000 + getLogger().info("Cold startup time: %d ms", int(elapsed_ms)) + return int(elapsed_ms) + + def measure_cold_startup_physical(self, bundle_id): + """Measure app cold startup time on a physical device in milliseconds. + + Terminates any running instance, waits briefly, then launches the app via devicectl. + Returns wall-clock time for the launch command in milliseconds as int. + """ + getLogger().info("Terminating app for cold startup on physical device: %s", bundle_id) + try: + RunCommand(['xcrun', 'devicectl', 'device', 'process', 'terminate', + '--device', self.device_id, '--bundle-id', bundle_id], verbose=True).run() + except subprocess.CalledProcessError: + getLogger().debug("Terminate returned error (app may not be running), ignoring.") + + time.sleep(0.5) + + getLogger().info("Launching app on physical device: %s", bundle_id) + start = time.time() + RunCommand(['xcrun', 'devicectl', 'device', 'process', 'launch', + '--device', self.device_id, bundle_id], verbose=True).run() + elapsed_ms = (time.time() - start) * 1000 + getLogger().info("Cold startup time: %d ms", int(elapsed_ms)) + return int(elapsed_ms) + + def uninstall_app(self, bundle_id): + """Uninstall the app from the simulator.""" + getLogger().info("Uninstalling app: %s", bundle_id) + RunCommand(['xcrun', 'simctl', 'uninstall', self.device_id, bundle_id], verbose=True).run() + + def terminate_app(self, bundle_id): + """Terminate the app on the simulator (ignore errors).""" + getLogger().info("Terminating app: %s", bundle_id) + try: + RunCommand(['xcrun', 'simctl', 'terminate', self.device_id, bundle_id], verbose=True).run() + except subprocess.CalledProcessError: + getLogger().debug("Terminate returned error (app may not be running), ignoring.") + + def close_simulator(self, skip_uninstall=False): + """Clean up the simulator session. + + Terminates and uninstalls the app unless skip_uninstall is True. + Does NOT shutdown the simulator. + """ + if not skip_uninstall: + getLogger().info("Stopping app for uninstall") + self.terminate_app(self.bundle_id) + self.uninstall_app(self.bundle_id) + + def find_app_bundle(self, build_output_dir, app_name, configuration='Debug'): + """Find the .app bundle in the build output directory. + + Searches for the typical path pattern: + bin//net*/iossimulator-*/.app + Also searches for physical device builds: + bin//net*/ios-arm64/.app + + Returns the absolute path to the .app bundle. + Raises FileNotFoundError if no bundle is found. + """ + # Try simulator path first, then physical device path + for rid_pattern in ['iossimulator-*', 'ios-arm64']: + pattern = os.path.join(build_output_dir, 'bin', configuration, 'net*', rid_pattern, f'{app_name}.app') + matches = glob.glob(pattern) + if matches: + if len(matches) > 1: + getLogger().warning("Found multiple app bundles matching pattern %s: %s. Using first match.", pattern, matches) + app_path = os.path.abspath(matches[0]) + getLogger().info("Found app bundle: %s", app_path) + return app_path + + raise FileNotFoundError( + f"Could not find .app bundle in {build_output_dir}/bin/{configuration}/net*/(iossimulator-*|ios-arm64)/{app_name}.app" + ) diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 165f0e882ba..a0a96a5a938 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -18,6 +18,7 @@ from typing import Optional from shared.androidhelper import AndroidHelper from shared.androidinstrumentation import AndroidInstrumentationHelper +from shared.ioshelper import iOSHelper from shared.devicepowerconsumption import DevicePowerConsumptionHelper from shared.crossgen import CrossgenArguments from shared.startup import StartupWrapper @@ -174,6 +175,19 @@ def parseargs(self): buildtimeparser.add_argument('--binlog-path', help='Location of binlog', dest='binlogpath') self.add_common_arguments(buildtimeparser) + iosinnerloopparser = subparsers.add_parser(const.IOSINNERLOOP, + description='measure first and incremental build+deploy time via binlogs (iOS)') + iosinnerloopparser.add_argument('--csproj-path', help='Path to .csproj file to build', dest='csprojpath') + iosinnerloopparser.add_argument('--edit-src', help='Modified source file paths, semicolon-separated', dest='editsrc') + iosinnerloopparser.add_argument('--edit-dest', help='Destination paths for modified files, semicolon-separated', dest='editdest') + iosinnerloopparser.add_argument('--framework', '-f', help='Target framework (e.g., net11.0-ios)', dest='framework') + iosinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', default='Debug') + iosinnerloopparser.add_argument('--msbuild-args', help='Additional MSBuild arguments', dest='msbuildargs', default='') + iosinnerloopparser.add_argument('--bundle-id', help='iOS bundle identifier', dest='bundleid') + iosinnerloopparser.add_argument('--device-id', help='iOS Simulator device ID', dest='deviceid', default='booted') + iosinnerloopparser.add_argument('--inner-loop-iterations', help='Number of incremental build+deploy+startup iterations (1+)', type=int, default=10, dest='innerloopiterations') + self.add_common_arguments(iosinnerloopparser) + args = parser.parse_args() if not args.testtype: @@ -196,6 +210,17 @@ def parseargs(self): if self.testtype == const.BUILDTIME: self.binlogpath = args.binlogpath + + if self.testtype == const.IOSINNERLOOP: + self.csprojpath = args.csprojpath + self.editsrcs = args.editsrc.split(';') if args.editsrc else [] + self.editdests = args.editdest.split(';') if args.editdest else [] + self.framework = args.framework + self.configuration = args.configuration + self.msbuildargs = args.msbuildargs or os.environ.get('PERFLAB_MSBUILD_ARGS', '') + self.bundleid = args.bundleid + self.deviceid = args.deviceid + self.innerloopiterations = args.innerloopiterations if self.testtype == const.DEVICESTARTUP: self.packagepath = args.packagepath @@ -974,4 +999,271 @@ def run(self): if not (self.binlogpath and os.path.exists(os.path.join(const.TRACEDIR, self.binlogpath))): raise Exception("For build time measurements a valid binlog path must be provided.") self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.BUILDTIME, tracename=self.binlogpath, scenarioname=self.scenarioname) - startup.parsetraces(self.traits) \ No newline at end of file + startup.parsetraces(self.traits) + + elif self.testtype == const.IOSINNERLOOP: + import hashlib + import subprocess + from shutil import copytree + from performance.common import runninginlab + from performance.constants import UPLOAD_CONTAINER, UPLOAD_STORAGE_URI, UPLOAD_QUEUE + from shared.util import helixuploaddir + import upload + + def merge_build_and_startup(build_report_path, startup_results, final_report_path): + """Load the build metrics report, append a startup time counter, write to final path.""" + with open(build_report_path, 'r') as f: + report = json.load(f) + startup_counter = { + "name": "Time to Main", + "topCounter": True, + "defaultCounter": False, + "higherIsBetter": False, + "metricName": "ms", + "results": startup_results + } + # Report structure: { "tests": [ { "counters": [...] } ] } + report["tests"][0]["counters"].append(startup_counter) + with open(final_report_path, 'w') as f: + json.dump(report, f, indent=2) + getLogger().info("Merged report written to: %s" % final_report_path) + + def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, + bundleid, app_bundle, + scenarioprefix, startup, traits, iosHelper): + """Run one incremental build+deploy+startup iteration. + + edit_pairs is a list of (dest_path, original_content, modified_content) tuples. + Returns (startup_ms, counters_list, binlog_path, test_metadata). + """ + import subprocess + + getLogger().info("=== Incremental iteration %d/%d ===" % (iteration, num_iterations)) + + # Toggle source files + for dest, original, modified in edit_pairs: + if iteration % 2 == 1: + with open(dest, 'w') as f: + f.write(modified) + content_hash = hashlib.md5(modified.encode()).hexdigest()[:8] + getLogger().info("Applied modified source: %s (hash=%s, len=%d)" % (dest, content_hash, len(modified))) + else: + with open(dest, 'w') as f: + f.write(original) + content_hash = hashlib.md5(original.encode()).hexdigest()[:8] + getLogger().info("Restored original source: %s (hash=%s, len=%d)" % (dest, content_hash, len(original))) + + # Incremental build with per-iteration binlog + iter_binlog_name = 'incremental-build-and-deploy-%d.binlog' % iteration + iter_binlog = os.path.join(const.TRACEDIR, iter_binlog_name) + incremental_cmd = base_cmd + [f'-bl:{iter_binlog}'] + getLogger().info("Incremental build: %s" % ' '.join(incremental_cmd)) + subprocess.run(incremental_cmd, check=True) + + # Install app on simulator + iosHelper.install_app(app_bundle) + + # Measure startup + ms = iosHelper.measure_cold_startup(bundleid) + getLogger().info("Incremental iteration %d/%d: build+deploy done, startup: %d ms" % (iteration, num_iterations, ms)) + + # Parse this iteration's binlog → temp build report + iter_report_name = 'incremental-build-report-%d.json' % iteration + iter_report = os.path.join(const.TRACEDIR, iter_report_name) + startup.reportjson = iter_report + traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.IOSINNERLOOP, + tracename=iter_binlog_name, + scenarioname=scenarioprefix + " - Incremental Build and Deploy", + upload_to_perflab_container=False) + startup.parsetraces(traits) + + # Extract build counters and test metadata from temp report + with open(iter_report, 'r') as f: + iter_data = json.load(f) + test_obj = iter_data["tests"][0] + counters = test_obj["counters"] + # Return test metadata (without counters) for building the final report + test_metadata = test_obj.copy() + test_metadata["counters"] = [] + + # Clean up temp report (leave binlog for later cleanup) + if os.path.exists(iter_report): + os.remove(iter_report) + getLogger().info("Removed temp report: %s" % iter_report) + + return ms, counters, iter_binlog, test_metadata + + # --- Validate inputs --- + if not self.csprojpath: + raise Exception("For iOS inner loop measurements, --csproj-path must be provided.") + if not self.bundleid: + raise Exception("For iOS inner loop measurements, --bundle-id must be provided.") + scenarioprefix = self.scenarioname or "MAUI iOS Build and Deploy" + + os.makedirs(const.TRACEDIR, exist_ok=True) + first_binlog = os.path.join(const.TRACEDIR, 'first-build-and-deploy.binlog') + + # Build the base MSBuild command (no -t:Install for iOS — plain dotnet build) + base_cmd = ['dotnet', 'build', self.csprojpath] + if self.configuration: + base_cmd.extend(['-c', self.configuration]) + if self.framework: + base_cmd.extend(['-f', self.framework]) + if self.msbuildargs: + for arg in re.split(r'[;\s]+', self.msbuildargs): + if arg.strip(): + base_cmd.append(arg.strip()) + + # Determine the project directory from csprojpath + project_dir = os.path.dirname(os.path.abspath(self.csprojpath)) + exename = self.traits.exename + + # --- First build + deploy --- + first_cmd = base_cmd + [f'-bl:{first_binlog}'] + getLogger().info("First build: %s" % ' '.join(first_cmd)) + subprocess.run(first_cmd, check=True) + + # --- Simulator setup and first deploy --- + iosHelper = iOSHelper() + try: + app_bundle = iosHelper.find_app_bundle(project_dir, exename, self.configuration) + iosHelper.setup_simulator(self.bundleid, app_bundle, self.deviceid) + + # --- First startup measurement --- + first_startup_ms = iosHelper.measure_cold_startup(self.bundleid) + getLogger().info("First deploy startup: %d ms" % first_startup_ms) + + # --- Parse first build report --- + startup = StartupWrapper() + first_build_report = os.path.join(const.TRACEDIR, 'first-build-and-deploy-perf-lab-report.json') + startup.reportjson = first_build_report + saved_upload = self.traits.upload_to_perflab_container + self.traits.add_traits(overwrite=True, apptorun="app", startupmetric=const.IOSINNERLOOP, + tracename='first-build-and-deploy.binlog', + scenarioname=scenarioprefix + " - First Build and Deploy", + upload_to_perflab_container=False) + startup.parsetraces(self.traits) + + # Merge first build metrics + startup → first e2e report + first_e2e_report = os.path.join(const.TRACEDIR, 'first-debug-e2e-perf-lab-report.json') + merge_build_and_startup(first_build_report, [first_startup_ms], first_e2e_report) + + # --- Incremental loop --- + num_iterations = self.innerloopiterations + getLogger().info("Starting incremental loop: %d iterations" % num_iterations) + + # Build list of (dest, original_content, modified_content) tuples for toggling + edit_pairs = [] + if self.editsrcs and self.editdests: + if len(self.editsrcs) != len(self.editdests): + raise Exception("--edit-src and --edit-dest must have the same number of semicolon-separated paths") + for src, dest in zip(self.editsrcs, self.editdests): + original = None + modified = None + with open(dest, 'r') as f: + original = f.read() + with open(src, 'r') as f: + modified = f.read() + edit_pairs.append((dest, original, modified)) + getLogger().info("Edit pair: %s <-> %s" % (src, dest)) + else: + raise Exception("No edit-src/edit-dest specified; incremental builds require file pairs to toggle") + + incremental_startup_results = [] + aggregated_counters = {} # counter_name -> aggregated counter dict + report_template = None # test metadata from first parsed report + intermediate_files = [] # files to clean up + + for iteration in range(1, num_iterations + 1): + ms, counters, iter_binlog, test_metadata = run_incremental_iteration( + iteration, num_iterations, base_cmd, + edit_pairs, + self.bundleid, app_bundle, scenarioprefix, startup, self.traits, + iosHelper) + + incremental_startup_results.append(ms) + intermediate_files.append(iter_binlog) + + # Save test metadata from the first iteration + if report_template is None: + report_template = test_metadata + + for counter in counters: + name = counter["name"] + if name not in aggregated_counters: + aggregated_counters[name] = { + "name": name, + "topCounter": counter.get("topCounter", False), + "defaultCounter": counter.get("defaultCounter", False), + "higherIsBetter": counter.get("higherIsBetter", False), + "metricName": counter.get("metricName", "ms"), + "results": [] + } + aggregated_counters[name]["results"].extend(counter.get("results", [])) + + # --- Aggregate incremental results --- + incremental_e2e_report = os.path.join(const.TRACEDIR, 'incremental-debug-e2e-perf-lab-report.json') + final_counters = list(aggregated_counters.values()) + final_counters.append({ + "name": "Time to Main", + "topCounter": True, + "defaultCounter": False, + "higherIsBetter": False, + "metricName": "ms", + "results": incremental_startup_results + }) + if report_template is not None: + report_template["counters"] = final_counters + final_report_data = {"tests": [report_template]} + else: + # Fallback: should not happen if at least 1 iteration ran + final_report_data = {"tests": [{"counters": final_counters}]} + with open(incremental_e2e_report, 'w') as f: + json.dump(final_report_data, f, indent=2) + getLogger().info("Final incremental E2E report written to: %s" % incremental_e2e_report) + + # --- Persist Reports for Local Runs --- + # Save both first and incremental E2E reports to a results directory so they survive cleanup. + # This is especially important for local runs where post.py cleans up the traces directory. + runtime_flavor = os.environ.get('RUNTIME_FLAVOR', 'unknown') + results_dir = os.path.join(os.getcwd(), 'results', runtime_flavor) + try: + os.makedirs(results_dir, exist_ok=True) + from shutil import copy2 + copy2(first_e2e_report, os.path.join(results_dir, os.path.basename(first_e2e_report))) + getLogger().info("Persisted first E2E report to: %s" % os.path.join(results_dir, os.path.basename(first_e2e_report))) + copy2(incremental_e2e_report, os.path.join(results_dir, os.path.basename(incremental_e2e_report))) + getLogger().info("Persisted incremental E2E report to: %s" % os.path.join(results_dir, os.path.basename(incremental_e2e_report))) + except Exception as e: + getLogger().warning("Failed to persist reports: %s" % str(e)) + + # --- Cleanup and upload --- + # Clean up intermediates from TRACEDIR + for f_path in intermediate_files + [first_build_report]: + if f_path.endswith('.binlog'): + getLogger().info("Keeping binlog for upload: %s" % f_path) + continue + if os.path.exists(f_path): + os.remove(f_path) + getLogger().info("Removed intermediate: %s" % f_path) + + # Wipe helix upload traces dir so copytree repopulates it cleanly + if runninginlab(): + traces_upload = os.path.join(helixuploaddir() or '', 'traces') + if os.path.exists(traces_upload): + rmtree(traces_upload) + + # Final upload + self.traits.add_traits(overwrite=True, upload_to_perflab_container=saved_upload) + helix_upload_dir = helixuploaddir() + if runninginlab() and helix_upload_dir is not None: + copytree(const.TRACEDIR, os.path.join(helix_upload_dir, 'traces'), dirs_exist_ok=True) + if self.traits.upload_to_perflab_container: + for report_path in [first_e2e_report, incremental_e2e_report]: + upload_code = upload.upload(report_path, UPLOAD_CONTAINER, UPLOAD_QUEUE, UPLOAD_STORAGE_URI) + getLogger().info("Upload code for %s: %s" % (os.path.basename(report_path), upload_code)) + if upload_code != 0: + sys.exit(upload_code) + + finally: + iosHelper.close_simulator(skip_uninstall=True) \ No newline at end of file From 664f06153e36e6b7fe38b180ee9055ba07806108 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 16:07:47 +0200 Subject: [PATCH 03/95] Add mauiiosinnerloop scenario directory (pre/test/post.py) pre.py: Install maui-ios workload, create MAUI template (no-restore for Helix), strip non-iOS TFMs with flexible regex, inject MSBuild properties (AllowMissingPrunePackageData, UseSharedCompilation), copy merged NuGet.config for Helix-side restore, create modified source files for incremental edit loop, check Xcode compatibility. test.py: Thin entrypoint that builds TestTraits and invokes Runner. post.py: Uninstall app from simulator, shut down dotnet build server, clean directories. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiiosinnerloop/post.py | 26 ++ src/scenarios/mauiiosinnerloop/pre.py | 371 +++++++++++++++++++++++++ src/scenarios/mauiiosinnerloop/test.py | 14 + 3 files changed, 411 insertions(+) create mode 100644 src/scenarios/mauiiosinnerloop/post.py create mode 100644 src/scenarios/mauiiosinnerloop/pre.py create mode 100644 src/scenarios/mauiiosinnerloop/test.py diff --git a/src/scenarios/mauiiosinnerloop/post.py b/src/scenarios/mauiiosinnerloop/post.py new file mode 100644 index 00000000000..d6e4d460992 --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/post.py @@ -0,0 +1,26 @@ +''' +post cleanup script +''' + +import subprocess +import sys +import traceback +from performance.logger import setup_loggers, getLogger +from shared.postcommands import clean_directories +from test import EXENAME + +setup_loggers(True) +logger = getLogger(__name__) + +try: + bundle_id = f'com.companyname.{EXENAME.lower()}' + logger.info(f"Uninstalling {bundle_id} from simulator") + subprocess.run(['xcrun', 'simctl', 'uninstall', 'booted', bundle_id], check=False) + + logger.info("Shutting down dotnet build servers") + subprocess.run(['dotnet', 'build-server', 'shutdown'], check=False) + + clean_directories() +except Exception as e: + logger.error(f"Post cleanup failed: {e}\n{traceback.format_exc()}") + sys.exit(1) diff --git a/src/scenarios/mauiiosinnerloop/pre.py b/src/scenarios/mauiiosinnerloop/pre.py new file mode 100644 index 00000000000..1614988de14 --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/pre.py @@ -0,0 +1,371 @@ +''' +pre-command: Set up a MAUI iOS app for deploy measurement. +Creates the template (without restore) and prepares the modified file for incremental deploy. +NuGet packages are restored on the Helix machine, not shipped in the payload. +''' +import glob +import json +import os +import re +import shutil +import subprocess +import sys +from performance.common import get_repo_root_path +from performance.logger import setup_loggers, getLogger +from shared import const +from shared.mauisharedpython import extract_latest_dotnet_feed_from_nuget_config, MauiNuGetConfigContext +from shared.precommands import PreCommands +from test import EXENAME + +def install_maui_ios_workload(precommands: PreCommands): + ''' + Install the maui-ios workload (not the full 'maui' workload). + The full 'maui' workload includes Android/Windows components that aren't + needed for this scenario. Since this scenario only needs iOS, 'maui-ios' + is sufficient and much smaller. + ''' + logger.info("########## Installing maui-ios workload ##########") + + if precommands.has_workload: + logger.info("Skipping maui-ios installation due to --has-workload=true") + return + + feed = extract_latest_dotnet_feed_from_nuget_config( + path=os.path.join(get_repo_root_path(), "NuGet.config") + ) + logger.info(f"Installing the latest maui-ios workload from feed {feed}") + + workload = "microsoft.net.sdk.ios" + try: + packages = precommands.get_packages_for_sdk_from_feed(workload, feed) + except Exception as e: + logger.warning(f"Failed to get packages for {workload} from latest feed: {e}") + logger.info("Trying second latest feed as fallback") + fallback_feed = extract_latest_dotnet_feed_from_nuget_config( + path=os.path.join(get_repo_root_path(), "NuGet.config"), + offset=1 + ) + logger.info(f"Using fallback feed: {fallback_feed}") + packages = precommands.get_packages_for_sdk_from_feed(workload, fallback_feed) + + # Filter to manifest packages only + pattern = r'Microsoft\.NET\.Sdk\..*\.Manifest\-\d+\.\d+\.\d+(\-(preview|rc|alpha)\.\d+)?$' + packages = [pkg for pkg in packages if re.match(pattern, pkg['id'])] + logger.info(f"After manifest pattern filtering, found {len(packages)} packages for {workload}") + + # Extract SDK and .NET versions from package IDs + for package in packages: + match = re.search(r'Manifest-(.+)$', package["id"]) + if not match: + raise Exception(f"Unable to find .NET SDK version in package ID: {package['id']}") + sdk_version = match.group(1) + package['sdk_version'] = sdk_version + + ver_match = re.search(r'^\d+\.\d+', sdk_version) + if not ver_match: + raise Exception(f"Unable to find .NET version in SDK version '{sdk_version}'") + package['dotnet_version'] = ver_match.group(0) + + # Keep only packages targeting the highest .NET version + dotnet_versions = [float(pkg['dotnet_version']) for pkg in packages] + highest = max(dotnet_versions) + packages = [pkg for pkg in packages if float(pkg['dotnet_version']) == highest] + logger.info(f"After .NET version filtering for {workload}: {len(packages)} packages (highest={highest})") + + # Prefer non-preview packages + preview_pattern = r'\-(preview|rc|alpha)\.\d+$' + non_preview = [pkg for pkg in packages if not re.search(preview_pattern, pkg['id'])] + if non_preview: + packages = non_preview + + # Sort by SDK version descending and take the latest + packages.sort(key=lambda x: x['sdk_version'], reverse=True) + if not packages: + raise Exception(f"No packages available for {workload} after filtering") + + latest = packages[0] + logger.info(f"Latest package: ID={latest['id']}, Version={latest['latestVersion']}, SDK={latest['sdk_version']}") + + # Create rollback file with only the iOS workload + rollback_value = f"{latest['latestVersion']}/{latest['sdk_version']}" + rollback_dict = {workload: rollback_value} + logger.info(f"Rollback dictionary: {rollback_dict}") + with open("rollback_maui.json", "w", encoding="utf-8") as f: + f.write(json.dumps(rollback_dict, indent=4)) + logger.info("Created rollback_maui.json file") + + # Install maui-ios (not 'maui') — only installs iOS components + precommands.install_workload('maui-ios', ['--from-rollback-file', 'rollback_maui.json']) + logger.info("########## Finished installing maui-ios workload ##########") + +def check_xcode_compatibility(framework: str): + ''' + Best-effort check that the active Xcode version matches the iOS SDK's + _RecommendedXcodeVersion. Logs a warning on mismatch — does not fail. + The caller (pipeline or run-local.sh) handles the actual Xcode selection. + ''' + try: + result = subprocess.run( + ['xcodebuild', '-version'], + capture_output=True, text=True, timeout=10 + ) + if result.returncode != 0: + logger.warning("Could not detect Xcode version (xcodebuild -version failed)") + return + + # Parse "Xcode 26.4" → "26.4" + xcode_line = result.stdout.strip().split('\n')[0] + xcode_match = re.search(r'Xcode\s+(\d+\.\d+)', xcode_line) + if not xcode_match: + logger.warning(f"Could not parse Xcode version from: {xcode_line}") + return + active_xcode = xcode_match.group(1) + + # Extract TFM prefix: "net11.0-ios" → "net11.0" + tfm_prefix = framework.split('-')[0] if '-' in framework else framework + + # Find the iOS SDK Versions.props file + dotnet_path = shutil.which('dotnet') + if not dotnet_path: + logger.warning("Could not find dotnet in PATH — skipping Xcode version check") + return + dotnet_dir = os.path.dirname(os.path.realpath(dotnet_path)) + packs_dir = os.path.join(dotnet_dir, 'packs') + + # Search for Microsoft.iOS.Sdk._*/*/targets/Microsoft.iOS.Sdk.Versions.props + search_pattern = os.path.join( + packs_dir, + f'Microsoft.iOS.Sdk.{tfm_prefix}_*', + '*', 'targets', 'Microsoft.iOS.Sdk.Versions.props' + ) + props_files = sorted(glob.glob(search_pattern)) + if not props_files: + logger.warning( + f"Could not find Microsoft.iOS.Sdk.Versions.props for {tfm_prefix} " + f"in {packs_dir} — skipping Xcode version check" + ) + return + + # Use the last (highest version) match + versions_props = props_files[-1] + logger.info(f"Found iOS SDK Versions.props: {versions_props}") + + # Parse _RecommendedXcodeVersion + with open(versions_props, 'r') as f: + content = f.read() + rec_match = re.search(r'<_RecommendedXcodeVersion>([^<]+)', content) + if not rec_match: + logger.warning(f"Could not find _RecommendedXcodeVersion in {versions_props}") + return + required_xcode = rec_match.group(1) + + active_major_minor = '.'.join(active_xcode.split('.')[:2]) + required_major_minor = '.'.join(required_xcode.split('.')[:2]) + + if active_major_minor != required_major_minor: + logger.warning( + f"Xcode version MISMATCH: " + f"active Xcode is {active_xcode} but iOS SDK requires {required_xcode}. " + f"The build may fail with _ValidateXcodeVersion error. " + f"Set XCODE_PATH or DEVELOPER_DIR to a compatible Xcode installation." + ) + else: + logger.info( + f"Xcode version OK: active={active_xcode}, " + f"required={required_xcode} (major.minor match: {active_major_minor})" + ) + except Exception as e: + logger.warning(f"Xcode compatibility check failed (non-fatal): {e}") + +def strip_non_ios_tfms(csproj_path: str, framework: str): + ''' + Strip non-iOS TargetFrameworks from the generated .csproj. + The MAUI template (since .NET 10+) generates multiple conditional + elements for android, ios, maccatalyst, and windows. + We replace all of them with a single unconditional + containing only the iOS TFM we want to build. + Uses ]*> to match both unconditional and + Condition="..." variants of the element. + ''' + with open(csproj_path, 'r') as f: + content = f.read() + + logger.info(f"Stripping non-iOS TFMs from {csproj_path}, keeping: {framework}") + + # Remove all existing ... lines + # (both unconditional and conditional variants). + stripped = re.sub( + r'\s*]*>[^<]*', + '', + content + ) + + # Also handle singular if present + stripped = re.sub( + r'\s*]*>[^<]*', + '', + stripped + ) + + # Insert a single unconditional with the iOS TFM + # into the first + stripped = stripped.replace( + '', + f'\n {framework}', + 1 # only the first PropertyGroup + ) + + with open(csproj_path, 'w') as f: + f.write(stripped) + + logger.info(f"Stripped non-iOS TFMs. csproj now targets: {framework}") + +setup_loggers(True) +logger = getLogger(__name__) +logger.info("Starting pre-command for MAUI iOS deploy measurement") + +precommands = PreCommands() + +with MauiNuGetConfigContext(precommands.framework): + install_maui_ios_workload(precommands) + check_xcode_compatibility(precommands.framework) + precommands.print_dotnet_info() + + # Create template without restoring packages — packages will be restored + # on the Helix machine to avoid shipping ~1-2GB in the workitem payload. + precommands.new(template='maui', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=True) + + # Copy the merged NuGet.config into the app directory. This file contains + # MAUI NuGet feed URLs added by MauiNuGetConfigContext. The Helix machine + # needs these feeds during restore, and we must copy before the context + # manager restores the original NuGet.config. + repo_root = os.path.normpath(os.path.join(sys.path[0], '..', '..', '..')) + repo_nuget_config = os.path.join(repo_root, 'NuGet.config') + app_nuget_config = os.path.join(const.APPDIR, 'NuGet.config') + shutil.copy2(repo_nuget_config, app_nuget_config) + logger.info(f"Copied merged NuGet.config from {repo_nuget_config} to {app_nuget_config}") + + # Strip non-iOS TFMs from the csproj. The MAUI template generates + # multi-TFM projects with conditional elements for android, ios, + # maccatalyst, and windows. We only need iOS. + csproj_path = os.path.join(const.APPDIR, f'{EXENAME}.csproj') + strip_non_ios_tfms(csproj_path, precommands.framework) + + # Inject properties into the csproj so they apply to every command that + # targets this project (restore, build, install). + with open(csproj_path, 'r') as f: + csproj_content = f.read() + + logger.info(f"Csproj content after TFM stripping:\n{csproj_content}") + + injected_props = { + # Preview SDKs may lack prune-package-data files, causing NETSDK1226. + 'AllowMissingPrunePackageData': 'true', + # The perf repo globally disables the Roslyn compiler server to avoid + # BenchmarkDotNet file-locking issues. Re-enable it here to match real + # MAUI developer inner loop experience. + 'UseSharedCompilation': 'true', + } + csproj_modified = csproj_content + if '' not in csproj_modified: + raise Exception( + f"Cannot inject properties into {csproj_path}: " + f"no found in the generated template." + ) + for prop_name, prop_value in injected_props.items(): + if prop_name not in csproj_modified: + csproj_modified = csproj_modified.replace( + '', + f' <{prop_name}>{prop_value}\n ', + 1 # only the first PropertyGroup + ) + + with open(csproj_path, 'w') as f: + f.write(csproj_modified) + + logger.info(f"Updated {csproj_path} with injected properties") + logger.info(f"Final .csproj content:\n{csproj_modified}") + + # Create modified source files in src/ for the incremental deploy simulation. + # The runner toggles between original and modified versions each iteration, + # exercising both the C# compiler (Csc) and XAML compiler (XamlC) paths. + src_dir = os.path.join(sys.path[0], const.SRCDIR) + os.makedirs(src_dir, exist_ok=True) + + # --- Modified MainPage.xaml.cs: add a debug line in the constructor --- + # The template may place MainPage in either the root or Pages/ subdirectory. + cs_candidates = [ + os.path.join(const.APPDIR, 'Pages', 'MainPage.xaml.cs'), + os.path.join(const.APPDIR, 'MainPage.xaml.cs'), + ] + cs_original = None + for candidate in cs_candidates: + if os.path.exists(candidate): + cs_original = candidate + break + if cs_original is None: + raise Exception( + "Could not find MainPage.xaml.cs in template — " + f"searched: {cs_candidates}" + ) + + cs_modified = os.path.join(src_dir, 'MainPage.xaml.cs') + with open(cs_original, 'r') as f: + cs_content = f.read() + + cs_modified_content = cs_content.replace( + 'InitializeComponent();', + 'InitializeComponent();\n\t\tSystem.Diagnostics.Debug.WriteLine("incremental-touch");' + ) + if cs_modified_content == cs_content: + raise Exception( + "Could not find 'InitializeComponent();' in %s — template may have changed" % cs_original + ) + + with open(cs_modified, 'w') as f: + f.write(cs_modified_content) + logger.info(f"Modified MainPage.xaml.cs written to {cs_modified}") + + # --- Modified MainPage.xaml: change a label's text --- + # Look in the same directory where we found the .cs file + xaml_original = os.path.join(os.path.dirname(cs_original), 'MainPage.xaml') + if not os.path.exists(xaml_original): + raise Exception(f"Could not find MainPage.xaml at {xaml_original}") + + xaml_modified = os.path.join(src_dir, 'MainPage.xaml') + with open(xaml_original, 'r') as f: + xaml_content = f.read() + + # Use a flexible match — look for any Text="..." attribute on a Label + # to handle template variations. Prefer a known string first. + xaml_modified_content = xaml_content.replace( + 'Text="Hello, World!"', + 'Text="Hello, World! (updated)"' + ) + if xaml_modified_content == xaml_content: + # Fallback: try the .NET 10+ template's "Task Categories" text + xaml_modified_content = xaml_content.replace( + 'Text="Task Categories"', + 'Text="Task Categories (updated)"' + ) + if xaml_modified_content == xaml_content: + # Last resort: replace the first Text="..." attribute we find + xaml_modified_content = re.sub( + r'Text="([^"]*)"', + r'Text="\1 (updated)"', + xaml_content, + count=1 + ) + if xaml_modified_content == xaml_content: + raise Exception( + "Could not find any Text=\"...\" attribute in %s — template may have changed" % xaml_original + ) + + with open(xaml_modified, 'w') as f: + f.write(xaml_modified_content) + logger.info(f"Modified MainPage.xaml written to {xaml_modified}") diff --git a/src/scenarios/mauiiosinnerloop/test.py b/src/scenarios/mauiiosinnerloop/test.py new file mode 100644 index 00000000000..2c493cb33f6 --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/test.py @@ -0,0 +1,14 @@ +''' +MAUI iOS Inner Loop (Debug End-2-End) Time Measurement +Orchestrates first build-deploy-startup → file edit → incremental build-deploy-startup → parse binlogs and startup times. +''' +import os +from shared.runner import TestTraits, Runner + +EXENAME = 'MauiiOSInnerLoop' + +if __name__ == "__main__": + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() From e20e8a4d3425b7cb36de421201ffbd82c5f710e6 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 16:12:15 +0200 Subject: [PATCH 04/95] Add setup_helix.py for iOS Helix bootstrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create the Helix machine setup script for MAUI iOS inner loop measurements. This script runs on the macOS Helix machine before test.py and handles: 1. DOTNET_ROOT/PATH configuration from the correlation payload SDK 2. Xcode selection — auto-detects highest versioned Xcode_*.app, matching the pattern used by maui_scenarios_ios.proj PreparePayloadWorkItem 3. iOS simulator runtime validation via xcrun simctl 4. Simulator device boot with graceful already-booted handling 5. maui-ios workload install using rollback file from pre.py, with --ignore-failed-sources for dead NuGet feeds 6. NuGet package restore with --ignore-failed-sources /p:NuGetAudit=false 7. Spotlight indexing disabled via mdutil to prevent file-lock errors Follows the same structure as the Android inner loop setup_helix.py: context dict pattern, step-by-step functions, structured logging to HELIX_WORKITEM_UPLOAD_ROOT for post-mortem debugging. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiiosinnerloop/setup_helix.py | 369 ++++++++++++++++++ 1 file changed, 369 insertions(+) create mode 100644 src/scenarios/mauiiosinnerloop/setup_helix.py diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py new file mode 100644 index 00000000000..ac87235d40d --- /dev/null +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +"""setup_helix.py — Helix machine setup for MAUI iOS inner loop (macOS). + +Runs on the Helix machine BEFORE test.py. Bootstraps the macOS environment +for iOS builds: + 1. Configure DOTNET_ROOT and PATH from the correlation payload SDK. + 2. Select the correct Xcode version (highest versioned Xcode_*.app). + 3. Validate iOS simulator runtime availability. + 4. Boot the target iOS simulator device. + 5. Install the maui-ios workload. + 6. Restore NuGet packages for the app project. + 7. Disable Spotlight indexing on the workitem directory. +""" + +import os +import subprocess +import sys +from datetime import datetime + +# --- Logging --- +# Follows the same logging pattern as the Android inner loop setup_helix.py: +# structured log file written to HELIX_WORKITEM_UPLOAD_ROOT for post-mortem +# debugging, with key messages also printed to stdout for Helix console output. +_logfile = None + + +def log(msg, tee=False): + """Write *msg* with a timestamp to the log file.""" + line = f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {msg}" + if _logfile: + _logfile.write(line + "\n") + _logfile.flush() + if tee: + print(line, flush=True) + + +def log_raw(msg, tee=False): + """Write *msg* verbatim (no timestamp) to the log file.""" + if _logfile: + _logfile.write(msg + "\n") + _logfile.flush() + if tee: + print(msg, flush=True) + + +def run_cmd(args, check=True, **kwargs): + """Run a command, logging stdout/stderr. Returns CompletedProcess.""" + log(f"Running: {args}") + result = subprocess.run( + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + **kwargs, + ) + if result.stdout: + for line in result.stdout.splitlines(): + log_raw(line) + if check and result.returncode != 0: + raise subprocess.CalledProcessError(result.returncode, args, result.stdout) + return result + + +def _dump_log(): + """Print the full log file to stdout so it appears in Helix console output.""" + if not _logfile: + return + _logfile.flush() + try: + with open(_logfile.name, "r") as f: + print(f.read()) + except Exception: + pass + + +# --- Setup Steps --- + +def print_diagnostics(): + """Log environment variables useful for debugging Helix failures.""" + log_raw("=== DIAGNOSTICS ===", tee=True) + for var in ["DOTNET_ROOT", "PATH", "DEVELOPER_DIR", "HELIX_WORKITEM_ROOT", + "HELIX_CORRELATION_PAYLOAD"]: + log_raw(f" {var}={os.environ.get(var, '')}") + log_raw(f" macOS version:", tee=True) + run_cmd(["sw_vers"], check=False) + + +def setup_dotnet(correlation_payload): + """Set DOTNET_ROOT and PATH to point at the SDK in the correlation payload. + + Returns the path to the dotnet executable. + """ + dotnet_root = os.path.join(correlation_payload, "dotnet") + dotnet_exe = os.path.join(dotnet_root, "dotnet") + + os.environ["DOTNET_ROOT"] = dotnet_root + os.environ["PATH"] = dotnet_root + ":" + os.environ.get("PATH", "") + + log(f"DOTNET_ROOT={dotnet_root}", tee=True) + run_cmd([dotnet_exe, "--version"], check=False) + return dotnet_exe + + +def select_xcode(): + """Select the highest versioned Xcode_*.app installation. + + Follows the same pattern as maui_scenarios_ios.proj PreparePayloadWorkItem: + find /Applications -maxdepth 1 -type d -name 'Xcode_*.app' | sort ... | tail -1 + + This avoids runner-image symlink aliases that don't work with the iOS SDK. + If XCODE_PATH env var is already set, uses that instead of auto-detecting. + """ + log_raw("=== XCODE SELECTION ===", tee=True) + + xcode_path = os.environ.get("XCODE_PATH", "") + if not xcode_path: + # Auto-detect: find highest versioned Xcode_*.app + result = run_cmd( + ["find", "/Applications", "-maxdepth", "1", "-type", "d", + "-name", "Xcode_*.app"], + check=False, + ) + candidates = [line.strip() for line in (result.stdout or "").splitlines() + if line.strip()] + if not candidates: + log("WARNING: No Xcode_*.app found in /Applications. " + "Falling back to system default Xcode.", tee=True) + run_cmd(["xcode-select", "-p"], check=False) + run_cmd(["xcodebuild", "-version"], check=False) + return + + # Sort by version number (Xcode_16.2.app → key on "16.2") + candidates.sort(key=lambda p: p.rsplit("_", 1)[-1].replace(".app", "")) + xcode_path = candidates[-1] + + log(f"Selected Xcode: {xcode_path}", tee=True) + + if not os.path.isdir(os.path.join(xcode_path, "Contents", "Developer")): + log(f"WARNING: {xcode_path} does not look like a valid Xcode installation " + "(missing Contents/Developer)", tee=True) + return + + # Use sudo xcode-select -s to switch the system Xcode (same as .proj pattern) + result = run_cmd( + ["sudo", "xcode-select", "-s", xcode_path], + check=False, + ) + if result.returncode != 0: + log(f"WARNING: xcode-select -s failed (exit {result.returncode}). " + "Falling back to DEVELOPER_DIR.", tee=True) + os.environ["DEVELOPER_DIR"] = os.path.join(xcode_path, "Contents", "Developer") + else: + log(f"Xcode switched to: {xcode_path}") + + # Log the active Xcode version for diagnostics + run_cmd(["xcodebuild", "-version"], check=False) + + +def validate_simulator_runtimes(): + """Check that iOS simulator runtimes are available on this machine.""" + log_raw("=== SIMULATOR RUNTIME VALIDATION ===", tee=True) + result = run_cmd(["xcrun", "simctl", "list", "runtimes"], check=False) + if result.returncode != 0: + log("WARNING: 'xcrun simctl list runtimes' failed. " + "Simulator may not work.", tee=True) + return + + # Check that at least one iOS runtime is listed + ios_runtimes = [line for line in (result.stdout or "").splitlines() + if "iOS" in line] + if ios_runtimes: + log(f"Found {len(ios_runtimes)} iOS runtime(s):", tee=True) + for rt in ios_runtimes: + log(f" {rt.strip()}") + else: + log("WARNING: No iOS simulator runtimes found. " + "Simulator-based testing will fail.", tee=True) + + +def boot_simulator(device_name): + """Boot the target iOS simulator device. + + Handles the case where the device is already booted (exit code 149 + from simctl boot = "Unable to boot device in current state: Booted"). + """ + log_raw("=== SIMULATOR BOOT ===", tee=True) + log(f"Booting simulator device: '{device_name}'", tee=True) + + result = run_cmd( + ["xcrun", "simctl", "boot", device_name], + check=False, + ) + + if result.returncode == 0: + log(f"Simulator '{device_name}' booted successfully.") + elif "Booted" in (result.stdout or "") or result.returncode == 149: + # Already booted — not an error + log(f"Simulator '{device_name}' is already booted (OK).") + else: + log(f"WARNING: Failed to boot simulator '{device_name}' " + f"(exit code {result.returncode}). " + "Available devices:", tee=True) + run_cmd(["xcrun", "simctl", "list", "devices", "available"], check=False) + + # Log booted devices for confirmation + log("Currently booted devices:") + run_cmd(["xcrun", "simctl", "list", "devices", "booted"], check=False) + + +def install_workload(ctx): + """Install the maui-ios workload using the shipped SDK. + + Uses the rollback file created by pre.py to pin to the exact workload + version. Falls back to a plain install if no rollback file is present. + Always uses --ignore-failed-sources because dead NuGet feeds are common + in CI. + """ + log_raw("=== WORKLOAD INSTALL ===", tee=True) + + rollback_file = os.path.join(ctx["workitem_root"], "rollback_maui.json") + nuget_config = ctx["nuget_config"] + + install_args = [ + ctx["dotnet_exe"], "workload", "install", "maui-ios", + ] + + if os.path.isfile(rollback_file): + log(f"Using rollback file: {rollback_file}") + install_args.extend(["--from-rollback-file", rollback_file]) + else: + log("No rollback_maui.json found — installing latest maui-ios workload") + + if os.path.isfile(nuget_config): + install_args.extend(["--configfile", nuget_config]) + + # Dead NuGet feeds are common in CI — always tolerate failures + install_args.append("--ignore-failed-sources") + + result = run_cmd(install_args, check=False) + if result.returncode != 0: + log(f"WORKLOAD INSTALL FAILED (exit code {result.returncode})", tee=True) + _dump_log() + sys.exit(1) + + log("maui-ios workload installed successfully") + + +def restore_packages(ctx): + """Restore NuGet packages for the app project. + + Uses --ignore-failed-sources and /p:NuGetAudit=false to handle dead + feeds and avoid audit warnings that slow down restore. + """ + log_raw("=== NUGET RESTORE ===", tee=True) + + csproj = ctx["csproj"] + if not os.path.isfile(csproj): + log(f"ERROR: Project file not found at {csproj}", tee=True) + _dump_log() + sys.exit(2) + + restore_args = [ + ctx["dotnet_exe"], "restore", csproj, + "--ignore-failed-sources", + "/p:NuGetAudit=false", + ] + + nuget_config = ctx["nuget_config"] + if os.path.isfile(nuget_config): + restore_args.extend(["--configfile", nuget_config]) + + framework = ctx.get("framework") + if framework: + restore_args.append(f"/p:TargetFrameworks={framework}") + + msbuild_args = ctx.get("msbuild_args") + if msbuild_args: + restore_args.extend(msbuild_args.split()) + + result = run_cmd(restore_args, check=False) + if result.returncode != 0: + log(f"RESTORE FAILED (exit code {result.returncode})", tee=True) + _dump_log() + sys.exit(2) + + log("NuGet restore succeeded") + + +def disable_spotlight(workitem_root): + """Disable Spotlight indexing on the workitem directory. + + Spotlight's mds_stores process can hold file locks during builds, + causing intermittent build failures. This is a well-known issue on + macOS CI machines. + """ + log_raw("=== DISABLE SPOTLIGHT ===", tee=True) + result = run_cmd( + ["sudo", "mdutil", "-i", "off", workitem_root], + check=False, + ) + if result.returncode != 0: + # Non-fatal — Spotlight interference is intermittent + log(f"WARNING: mdutil -i off failed (exit {result.returncode}). " + "Spotlight may interfere with builds.", tee=True) + else: + log(f"Spotlight indexing disabled for {workitem_root}") + + +# --- Main --- + +def main(): + global _logfile + + # Open log file in HELIX_WORKITEM_UPLOAD_ROOT for post-mortem debugging + upload_root = os.environ.get("HELIX_WORKITEM_UPLOAD_ROOT") + if upload_root: + os.makedirs(upload_root, exist_ok=True) + _logfile = open(os.path.join(upload_root, "setup_helix.log"), "a") + + workitem_root = os.environ.get("HELIX_WORKITEM_ROOT", ".") + correlation_payload = os.environ.get("HELIX_CORRELATION_PAYLOAD", ".") + + # The simulator device name can be overridden via env var; default to + # "iPhone 16" which is available on current macOS Helix images. + device_name = os.environ.get("IOS_SIMULATOR_DEVICE", "iPhone 16") + + # Framework and MSBuild args are passed as command-line arguments when + # available (from the .proj PreCommands), or fall back to env vars. + framework = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("PERFLAB_Framework", "") + msbuild_args = sys.argv[2] if len(sys.argv) > 2 else "" + + ctx = { + "framework": framework, + "msbuild_args": msbuild_args, + "workitem_root": workitem_root, + "correlation_payload": correlation_payload, + "nuget_config": os.path.join(workitem_root, "app", "NuGet.config"), + "csproj": os.path.join(workitem_root, "app", "MauiiOSInnerLoop.csproj"), + } + + log_raw("=== iOS HELIX SETUP START ===", tee=True) + + # Step 1: Configure the .NET SDK from the correlation payload + ctx["dotnet_exe"] = setup_dotnet(correlation_payload) + print_diagnostics() + + # Step 2: Select the correct Xcode version + select_xcode() + + # Step 3: Validate simulator runtimes are available + validate_simulator_runtimes() + + # Step 4: Boot the target simulator device + boot_simulator(device_name) + + # Step 5: Install the maui-ios workload + # Must happen BEFORE restore because restore needs workload packs + install_workload(ctx) + + # Step 6: Restore NuGet packages + restore_packages(ctx) + + # Step 7: Disable Spotlight indexing to prevent file-lock errors + disable_spotlight(workitem_root) + + log_raw("=== iOS HELIX SETUP SUCCEEDED ===", tee=True) + _dump_log() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From cb6315ea2ab8b57139d5cf6c18492f8bd39a6881 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 16:20:38 +0200 Subject: [PATCH 05/95] Add maui_scenarios_ios_innerloop.proj Helix workitem definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Define the Helix .proj file for iOS inner loop measurements, modeled after the Android inner loop .proj and existing maui_scenarios_ios.proj patterns. Key design decisions: - Build on Helix machine (not build agent) because deploy requires a connected device/simulator. PreparePayloadWorkItem only creates the template and modified source files via pre.py. - Workload packs stripped from correlation payload (RemoveDotnetFromCorrelation Staging) and reinstalled on Helix machine by setup_helix.py. - Environment variables set via shell 'export' in PreCommands (not in Python) because os.environ changes don't persist across process boundaries. - No XHarness — iOS inner loop uses xcrun simctl directly. - Simulator-only for now; physical device support (ios-arm64, code signing) is structured as a future TODO pending runner.py device support. - 01:30 timeout to accommodate iOS build + workload install + NuGet restore. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../maui_scenarios_ios_innerloop.proj | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 eng/performance/maui_scenarios_ios_innerloop.proj diff --git a/eng/performance/maui_scenarios_ios_innerloop.proj b/eng/performance/maui_scenarios_ios_innerloop.proj new file mode 100644 index 00000000000..6f37e9e2b6f --- /dev/null +++ b/eng/performance/maui_scenarios_ios_innerloop.proj @@ -0,0 +1,93 @@ + + + + + + + + + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'mono'">/p:UseMonoRuntime=true + <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false + + + iossimulator-arm64 + <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(iOSRid) + + $(RuntimeFlavor)_Default + 3 + + + + + + + + + + + + 01:30 + + + + + + mauiiosinnerloop + $(ScenariosDir)%(ScenarioDirectoryName) + + + + + + + + XCODE_PATH=`find /Applications -maxdepth 1 -type d -name 'Xcode_*.app' | sort -t_ -k2 -V | tail -1` && echo "Selected Xcode: $XCODE_PATH" && sudo xcode-select -s "$XCODE_PATH" && $(Python) pre.py default -f $(PERFLAB_Framework)-ios + %(PreparePayloadWorkItem.PayloadDirectory) + + + + + + <_MacEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$PATH + + + + + + $(_MacEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" + $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id com.companyname.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) + $(Python) post.py + output.log + + + + + + + + From 7c9e0c6e16bd3d94281b8468552d76a72542c14b Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 16:23:36 +0200 Subject: [PATCH 06/95] Wire iOS inner loop into CI pipeline YAML and routing - sdk-perf-jobs.yml: Add Mono Debug job entry for maui_scenarios_ios_innerloop on osx-x64-ios-arm64 (Mac.iPhone.17.Perf queue) - run-performance-job.yml: Add maui_scenarios_ios_innerloop to the in() check so --runtime-flavor is forwarded to run_performance_job.py - run_performance_job.py: Add maui_scenarios_ios_innerloop to get_run_configurations() (CodegenType, RuntimeType, BuildConfig) and to the binlog copy block for PreparePayloadWorkItems artifacts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-perf-jobs.yml | 19 +++++++++++++++++++ .../templates/run-performance-job.yml | 2 +- scripts/run_performance_job.py | 11 ++++++++++- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 4c3a62d1a72..834892cc975 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -565,6 +565,25 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS Inner Loop (Mono - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios_innerloop.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_InnerLoop + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + # Maui scenario benchmarks - ${{ if false }}: - template: /eng/pipelines/templates/build-machine-matrix.yml diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 4fe69d8c312..68b3b16df4f 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -190,7 +190,7 @@ jobs: - '--is-scenario' - ${{ if ne(length(parameters.runEnvVars), 0) }}: - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android'), ne(parameters.runtimeFlavor, '')) }}: + - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android', 'maui_scenarios_ios_innerloop'), ne(parameters.runtimeFlavor, '')) }}: - '--runtime-flavor ${{ parameters.runtimeFlavor }}' - ${{ if ne(parameters.osVersion, '') }}: - '--os-version ${{ parameters.osVersion }}' diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index e7be2b04caf..e531d4158e7 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -587,6 +587,15 @@ def get_run_configurations( if build_config is not None and build_config != DEFAULT_BUILD_CONFIG: configurations["BuildConfig"] = build_config + # .NET MAUI iOS inner loop (build+deploy) scenarios + if run_kind == "maui_scenarios_ios_innerloop": + if not runtime_flavor in ("mono", "coreclr"): + raise Exception("Runtime flavor must be specified for maui_scenarios_ios_innerloop") + configurations["CodegenType"] = str(codegen_type) + configurations["RuntimeType"] = str(runtime_flavor) + if build_config is not None and build_config != DEFAULT_BUILD_CONFIG: + configurations["BuildConfig"] = build_config + return configurations def get_work_item_command(os_group: str, target_csproj: str, architecture: str, perf_lab_framework: str, internal: bool, wasm: bool, bdn_artifacts_dir: str, wasm_coreclr: bool = False, only_sanity_check: bool = False): @@ -1190,7 +1199,7 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_ verbose=True).run() # Search for additional binlogs generated by the maui scenarios prepare payload work items to copy to the artifacts log dir - if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios"]: + if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios", "maui_scenarios_ios_innerloop"]: for binlog_path in glob(os.path.join(payload_dir, "scenarios_out", "**", "*.binlog"), recursive=True): shutil.copy(binlog_path, ci_artifacts_log_dir) From 0d8677252c71ef15eca93fc6c904db6bd930ef3f Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 16:31:13 +0200 Subject: [PATCH 07/95] Add physical device support for iOS inner loop - ioshelper.py: Add detect_connected_device() with auto-detection via xcrun devicectl (JSON + fallback text parsing), uninstall_app_physical, terminate_app_physical, close_physical_device, and cleanup() dispatch - runner.py: Add --device-type arg (simulator/device) to iosinnerloop subparser, auto-infer from RuntimeIdentifier, auto-detect device UDID, branch setup/install/startup/cleanup for physical vs simulator - setup_helix.py: Detect device type from IOS_RID env var, skip simulator boot for physical device, add detect_physical_device() for Helix - post.py: Handle physical device uninstall via devicectl with UDID auto-detection fallback - maui_scenarios_ios_innerloop.proj: Add physical device HelixWorkItem (conditioned on iOSRid=ios-arm64), pass IOS_RID to Pre/PostCommands, add --device-type arg to both simulator and device workitems Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../maui_scenarios_ios_innerloop.proj | 29 ++-- src/scenarios/mauiiosinnerloop/post.py | 26 +++- src/scenarios/mauiiosinnerloop/setup_helix.py | 84 +++++++++++- src/scenarios/shared/ioshelper.py | 128 ++++++++++++++++++ src/scenarios/shared/runner.py | 51 +++++-- 5 files changed, 289 insertions(+), 29 deletions(-) diff --git a/eng/performance/maui_scenarios_ios_innerloop.proj b/eng/performance/maui_scenarios_ios_innerloop.proj index 6f37e9e2b6f..fee7cbf197c 100644 --- a/eng/performance/maui_scenarios_ios_innerloop.proj +++ b/eng/performance/maui_scenarios_ios_innerloop.proj @@ -73,20 +73,29 @@ which has both physical iPhones and Xcode simulator runtimes. --> - $(_MacEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" - $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id com.companyname.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) - $(Python) post.py + $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" + $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id com.companyname.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --device-type simulator --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) + export IOS_RID=$(iOSRid);$(Python) post.py output.log - + + + + $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" + $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id com.companyname.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --device-type device --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) + export IOS_RID=$(iOSRid);$(Python) post.py + output.log + + diff --git a/src/scenarios/mauiiosinnerloop/post.py b/src/scenarios/mauiiosinnerloop/post.py index d6e4d460992..4b9b648dbf5 100644 --- a/src/scenarios/mauiiosinnerloop/post.py +++ b/src/scenarios/mauiiosinnerloop/post.py @@ -2,6 +2,7 @@ post cleanup script ''' +import os import subprocess import sys import traceback @@ -14,8 +15,29 @@ try: bundle_id = f'com.companyname.{EXENAME.lower()}' - logger.info(f"Uninstalling {bundle_id} from simulator") - subprocess.run(['xcrun', 'simctl', 'uninstall', 'booted', bundle_id], check=False) + + # Determine device type from IOS_RID env var (set by .proj PreCommands) + ios_rid = os.environ.get('IOS_RID', 'iossimulator-arm64') + is_physical_device = (ios_rid == 'ios-arm64') + + if is_physical_device: + device_udid = os.environ.get('IOS_DEVICE_UDID', '').strip() + if not device_udid: + # Auto-detect since env var may not have been exported by PreCommands + try: + from shared.ioshelper import iOSHelper + device_udid = iOSHelper.detect_connected_device() + except Exception: + device_udid = None + if device_udid: + logger.info(f"Uninstalling {bundle_id} from physical device {device_udid}") + subprocess.run(['xcrun', 'devicectl', 'device', 'uninstall', 'app', + '--device', device_udid, bundle_id], check=False) + else: + logger.warning("No IOS_DEVICE_UDID available — skipping physical device uninstall") + else: + logger.info(f"Uninstalling {bundle_id} from simulator") + subprocess.run(['xcrun', 'simctl', 'uninstall', 'booted', bundle_id], check=False) logger.info("Shutting down dotnet build servers") subprocess.run(['dotnet', 'build-server', 'shutdown'], check=False) diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py index ac87235d40d..2c391c64c6f 100644 --- a/src/scenarios/mauiiosinnerloop/setup_helix.py +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -303,6 +303,60 @@ def disable_spotlight(workitem_root): log(f"Spotlight indexing disabled for {workitem_root}") +def detect_physical_device(): + """Detect whether a physical iOS device is connected and return its UDID. + + Checks IOS_DEVICE_UDID env var first, then uses 'xcrun devicectl list devices'. + Returns the UDID string, or None if no device is found. + """ + log_raw("=== PHYSICAL DEVICE DETECTION ===", tee=True) + + udid = os.environ.get("IOS_DEVICE_UDID", "").strip() + if udid: + log(f"Using IOS_DEVICE_UDID from environment: {udid}", tee=True) + return udid + + # Auto-detect via devicectl + result = run_cmd( + ["xcrun", "devicectl", "list", "devices"], + check=False, + ) + if result.returncode != 0: + log("WARNING: 'xcrun devicectl list devices' failed. " + "No physical device detection available.", tee=True) + return None + + # Log the full output for debugging + log("devicectl output:") + for line in (result.stdout or "").splitlines(): + log_raw(f" {line}") + + # Try JSON output for structured parsing + json_result = run_cmd( + ["xcrun", "devicectl", "list", "devices", "--json-output", "/dev/stdout"], + check=False, + ) + if json_result.returncode == 0 and json_result.stdout: + try: + import json + data = json.loads(json_result.stdout) + devices = data.get("result", {}).get("devices", []) + for device in devices: + conn = device.get("connectionProperties", {}) + transport = conn.get("transportType", "") + name = device.get("deviceProperties", {}).get("name", "unknown") + device_udid = device.get("identifier", "") + if transport in ("wired", "localNetwork", "wifi") and device_udid: + log(f"Found connected device: {name} (UDID: {device_udid}, " + f"transport: {transport})", tee=True) + return device_udid + except Exception as e: + log(f"JSON parsing failed: {e}", tee=True) + + log("No connected physical devices found.", tee=True) + return None + + # --- Main --- def main(): @@ -317,6 +371,11 @@ def main(): workitem_root = os.environ.get("HELIX_WORKITEM_ROOT", ".") correlation_payload = os.environ.get("HELIX_CORRELATION_PAYLOAD", ".") + # Determine target device type from iOSRid env var (set by .proj). + # ios-arm64 → physical device, iossimulator-* → simulator + ios_rid = os.environ.get("IOS_RID", "iossimulator-arm64") + is_physical_device = (ios_rid == "ios-arm64") + # The simulator device name can be overridden via env var; default to # "iPhone 16" which is available on current macOS Helix images. device_name = os.environ.get("IOS_SIMULATOR_DEVICE", "iPhone 16") @@ -336,6 +395,8 @@ def main(): } log_raw("=== iOS HELIX SETUP START ===", tee=True) + log(f"Target device type: {'physical device' if is_physical_device else 'simulator'} " + f"(IOS_RID={ios_rid})", tee=True) # Step 1: Configure the .NET SDK from the correlation payload ctx["dotnet_exe"] = setup_dotnet(correlation_payload) @@ -344,11 +405,24 @@ def main(): # Step 2: Select the correct Xcode version select_xcode() - # Step 3: Validate simulator runtimes are available - validate_simulator_runtimes() - - # Step 4: Boot the target simulator device - boot_simulator(device_name) + # Step 3 & 4: Device-type-specific setup + if is_physical_device: + # Detect and validate the connected physical device + device_udid = detect_physical_device() + if not device_udid: + log("WARNING: No physical device found. Build may still succeed " + "but deploy will fail.", tee=True) + else: + # Log the detected UDID for diagnostics. Note: os.environ changes + # in this Python process do NOT persist to subsequent Helix commands + # (test.py, post.py). runner.py re-detects the device independently + # via iOSHelper.detect_connected_device(). + os.environ["IOS_DEVICE_UDID"] = device_udid + log(f"IOS_DEVICE_UDID detected: {device_udid}", tee=True) + else: + # Simulator: validate runtimes and boot the device + validate_simulator_runtimes() + boot_simulator(device_name) # Step 5: Install the maui-ios workload # Must happen BEFORE restore because restore needs workload packs diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py index 4b7e520262a..8caac58c67d 100644 --- a/src/scenarios/shared/ioshelper.py +++ b/src/scenarios/shared/ioshelper.py @@ -1,4 +1,6 @@ +import json import os +import re import time import glob import subprocess @@ -11,6 +13,92 @@ def __init__(self): self.bundle_id = None self.device_id = None self.app_bundle_path = None + self.is_physical_device = False + + @staticmethod + def detect_connected_device(): + """Detect a connected physical iOS device and return its UDID. + + Checks IOS_DEVICE_UDID environment variable first. If not set, + auto-detects using 'xcrun devicectl list devices' and returns the + UDID of the first connected device. + + Returns the UDID string, or None if no device is found. + """ + # Prefer explicit env var + udid = os.environ.get('IOS_DEVICE_UDID', '').strip() + if udid: + getLogger().info("Using IOS_DEVICE_UDID from environment: %s", udid) + return udid + + # Auto-detect via devicectl (Xcode 15+) + getLogger().info("Auto-detecting connected iOS device via 'xcrun devicectl list devices'...") + try: + result = subprocess.run( + ['xcrun', 'devicectl', 'list', 'devices', '--json-output', '/dev/stdout'], + capture_output=True, text=True, timeout=30 + ) + if result.returncode != 0: + getLogger().warning("devicectl list devices failed (exit %d): %s", + result.returncode, result.stderr) + return None + + data = json.loads(result.stdout) + # devicectl JSON output has "result.devices" array with "identifier" (UDID) + # and "connectionProperties.transportType" to filter for USB-connected devices + devices = data.get('result', {}).get('devices', []) + for device in devices: + conn = device.get('connectionProperties', {}) + transport = conn.get('transportType', '') + state = device.get('deviceProperties', {}).get('developerModeStatus', '') + name = device.get('deviceProperties', {}).get('name', 'unknown') + device_udid = device.get('identifier', '') + + # Only consider locally-connected (wired/WiFi) devices, not + # paired watches or other peripherals + if transport in ('wired', 'localNetwork', 'wifi') and device_udid: + getLogger().info("Found connected device: %s (UDID: %s, transport: %s, devMode: %s)", + name, device_udid, transport, state) + return device_udid + + getLogger().warning("No connected iOS devices found in devicectl output") + return None + + except subprocess.TimeoutExpired: + getLogger().warning("devicectl list devices timed out") + return None + except (json.JSONDecodeError, KeyError) as e: + getLogger().warning("Failed to parse devicectl JSON output: %s", e) + # Fall back to text parsing of non-JSON output + return iOSHelper._detect_device_fallback() + + @staticmethod + def _detect_device_fallback(): + """Fallback device detection using text output from devicectl. + + Used when JSON parsing fails (e.g., older Xcode versions that don't + support --json-output). + """ + try: + result = subprocess.run( + ['xcrun', 'devicectl', 'list', 'devices'], + capture_output=True, text=True, timeout=30 + ) + if result.returncode != 0: + return None + + # Look for lines with a UUID pattern (device UDID) + # Example line: " PERFIOS-01 00008101-001A09223E08001E ..." + for line in (result.stdout or '').splitlines(): + match = re.search(r'([0-9A-Fa-f]{8}-[0-9A-Fa-f]{16}|[0-9A-Fa-f]{25,40})', line) + if match: + udid = match.group(1) + getLogger().info("Fallback detection found device UDID: %s (from: %s)", + udid, line.strip()) + return udid + return None + except Exception: + return None def setup_simulator(self, bundle_id, app_bundle_path, device_id='booted'): """Boot the iOS simulator and install the app bundle.""" @@ -41,10 +129,12 @@ def setup_physical_device(self, bundle_id, app_bundle_path, device_id): """Set up a physical iOS device for testing. Installs the app bundle on the connected physical device using devicectl. + Requires Xcode 15+ for the 'xcrun devicectl' toolchain. """ self.bundle_id = bundle_id self.device_id = device_id self.app_bundle_path = app_bundle_path + self.is_physical_device = True getLogger().info("Installing app bundle on physical device %s: %s", device_id, app_bundle_path) RunCommand(['xcrun', 'devicectl', 'device', 'install', 'app', @@ -121,6 +211,15 @@ def uninstall_app(self, bundle_id): getLogger().info("Uninstalling app: %s", bundle_id) RunCommand(['xcrun', 'simctl', 'uninstall', self.device_id, bundle_id], verbose=True).run() + def uninstall_app_physical(self, bundle_id): + """Uninstall the app from a physical device.""" + getLogger().info("Uninstalling app from physical device: %s", bundle_id) + try: + RunCommand(['xcrun', 'devicectl', 'device', 'uninstall', 'app', + '--device', self.device_id, bundle_id], verbose=True).run() + except subprocess.CalledProcessError: + getLogger().debug("Uninstall returned error (app may not be installed), ignoring.") + def terminate_app(self, bundle_id): """Terminate the app on the simulator (ignore errors).""" getLogger().info("Terminating app: %s", bundle_id) @@ -129,6 +228,15 @@ def terminate_app(self, bundle_id): except subprocess.CalledProcessError: getLogger().debug("Terminate returned error (app may not be running), ignoring.") + def terminate_app_physical(self, bundle_id): + """Terminate the app on a physical device (ignore errors).""" + getLogger().info("Terminating app on physical device: %s", bundle_id) + try: + RunCommand(['xcrun', 'devicectl', 'device', 'process', 'terminate', + '--device', self.device_id, '--bundle-id', bundle_id], verbose=True).run() + except subprocess.CalledProcessError: + getLogger().debug("Terminate returned error (app may not be running), ignoring.") + def close_simulator(self, skip_uninstall=False): """Clean up the simulator session. @@ -140,6 +248,26 @@ def close_simulator(self, skip_uninstall=False): self.terminate_app(self.bundle_id) self.uninstall_app(self.bundle_id) + def close_physical_device(self, skip_uninstall=False): + """Clean up the physical device session. + + Terminates and uninstalls the app unless skip_uninstall is True. + """ + if not skip_uninstall: + getLogger().info("Stopping app for uninstall on physical device") + self.terminate_app_physical(self.bundle_id) + self.uninstall_app_physical(self.bundle_id) + + def cleanup(self, skip_uninstall=False): + """Clean up the device session (simulator or physical). + + Dispatches to the appropriate cleanup method based on device type. + """ + if self.is_physical_device: + self.close_physical_device(skip_uninstall=skip_uninstall) + else: + self.close_simulator(skip_uninstall=skip_uninstall) + def find_app_bundle(self, build_output_dir, app_name, configuration='Debug'): """Find the .app bundle in the build output directory. diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index a0a96a5a938..2742fc91b50 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -184,7 +184,8 @@ def parseargs(self): iosinnerloopparser.add_argument('--configuration', '-c', help='Build configuration', dest='configuration', default='Debug') iosinnerloopparser.add_argument('--msbuild-args', help='Additional MSBuild arguments', dest='msbuildargs', default='') iosinnerloopparser.add_argument('--bundle-id', help='iOS bundle identifier', dest='bundleid') - iosinnerloopparser.add_argument('--device-id', help='iOS Simulator device ID', dest='deviceid', default='booted') + iosinnerloopparser.add_argument('--device-id', help='iOS device ID (UDID for physical device, simulator ID or "booted" for simulator)', dest='deviceid', default='booted') + iosinnerloopparser.add_argument('--device-type', choices=['simulator', 'device'], help='Target device type: simulator (default) or physical device. Auto-detected from RuntimeIdentifier if not set.', dest='devicetype', default=None) iosinnerloopparser.add_argument('--inner-loop-iterations', help='Number of incremental build+deploy+startup iterations (1+)', type=int, default=10, dest='innerloopiterations') self.add_common_arguments(iosinnerloopparser) @@ -221,6 +222,14 @@ def parseargs(self): self.bundleid = args.bundleid self.deviceid = args.deviceid self.innerloopiterations = args.innerloopiterations + # Determine device type: explicit arg, or infer from RuntimeIdentifier + # in msbuildargs (ios-arm64 → device, iossimulator-* → simulator) + if args.devicetype: + self.devicetype = args.devicetype + elif 'RuntimeIdentifier=ios-arm64' in self.msbuildargs: + self.devicetype = 'device' + else: + self.devicetype = 'simulator' if self.testtype == const.DEVICESTARTUP: self.packagepath = args.packagepath @@ -1030,7 +1039,8 @@ def merge_build_and_startup(build_report_path, startup_results, final_report_pat def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, bundleid, app_bundle, - scenarioprefix, startup, traits, iosHelper): + scenarioprefix, startup, traits, iosHelper, + is_physical_device=False): """Run one incremental build+deploy+startup iteration. edit_pairs is a list of (dest_path, original_content, modified_content) tuples. @@ -1060,11 +1070,13 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, getLogger().info("Incremental build: %s" % ' '.join(incremental_cmd)) subprocess.run(incremental_cmd, check=True) - # Install app on simulator - iosHelper.install_app(app_bundle) - - # Measure startup - ms = iosHelper.measure_cold_startup(bundleid) + # Install and measure startup — dispatch based on device type + if is_physical_device: + iosHelper.install_app_physical(app_bundle) + ms = iosHelper.measure_cold_startup_physical(bundleid) + else: + iosHelper.install_app(app_bundle) + ms = iosHelper.measure_cold_startup(bundleid) getLogger().info("Incremental iteration %d/%d: build+deploy done, startup: %d ms" % (iteration, num_iterations, ms)) # Parse this iteration's binlog → temp build report @@ -1098,6 +1110,16 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, raise Exception("For iOS inner loop measurements, --csproj-path must be provided.") if not self.bundleid: raise Exception("For iOS inner loop measurements, --bundle-id must be provided.") + is_physical = (self.devicetype == 'device') + if is_physical and self.deviceid == 'booted': + # Physical device requires a real UDID — auto-detect if not provided + detected = iOSHelper.detect_connected_device() + if not detected: + raise Exception("Physical device mode requires a device UDID. " + "Set --device-id or IOS_DEVICE_UDID, or connect a device.") + self.deviceid = detected + getLogger().info("Auto-detected physical device UDID: %s" % self.deviceid) + getLogger().info("iOS inner loop device type: %s (device-id: %s)" % (self.devicetype, self.deviceid)) scenarioprefix = self.scenarioname or "MAUI iOS Build and Deploy" os.makedirs(const.TRACEDIR, exist_ok=True) @@ -1123,14 +1145,19 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, getLogger().info("First build: %s" % ' '.join(first_cmd)) subprocess.run(first_cmd, check=True) - # --- Simulator setup and first deploy --- + # --- Device/simulator setup and first deploy --- iosHelper = iOSHelper() try: app_bundle = iosHelper.find_app_bundle(project_dir, exename, self.configuration) - iosHelper.setup_simulator(self.bundleid, app_bundle, self.deviceid) + + if is_physical: + iosHelper.setup_physical_device(self.bundleid, app_bundle, self.deviceid) + first_startup_ms = iosHelper.measure_cold_startup_physical(self.bundleid) + else: + iosHelper.setup_simulator(self.bundleid, app_bundle, self.deviceid) + first_startup_ms = iosHelper.measure_cold_startup(self.bundleid) # --- First startup measurement --- - first_startup_ms = iosHelper.measure_cold_startup(self.bundleid) getLogger().info("First deploy startup: %d ms" % first_startup_ms) # --- Parse first build report --- @@ -1179,7 +1206,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, iteration, num_iterations, base_cmd, edit_pairs, self.bundleid, app_bundle, scenarioprefix, startup, self.traits, - iosHelper) + iosHelper, is_physical_device=is_physical) incremental_startup_results.append(ms) intermediate_files.append(iter_binlog) @@ -1266,4 +1293,4 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, sys.exit(upload_code) finally: - iosHelper.close_simulator(skip_uninstall=True) \ No newline at end of file + iosHelper.cleanup(skip_uninstall=True) \ No newline at end of file From edab33fea86b8661d506e70a70300610cde9df72 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 16:54:43 +0200 Subject: [PATCH 08/95] Fix review findings: devicectl commands, UDID regex, edit paths, Xcode sort Fix 1 (Major): Replace non-existent 'devicectl terminate --bundle-id' with '--terminate-existing' flag on launch command. Make terminate_app_physical() a no-op with documentation explaining why. Fix 2 (Medium): Write devicectl JSON output to temp file instead of /dev/stdout, which mixes human-readable table and JSON. Applied in both ioshelper.py and setup_helix.py with proper temp file cleanup. Fix 3 (Medium): Add standard UUID pattern (8-4-4-4-12) to UDID regex in _detect_device_fallback() for CoreDevice UUID format compatibility. Fix 4 (Medium): Normalize MAUI template to always use Pages/ subdirectory in pre.py. If template puts MainPage files at root, move them to Pages/. Add explanatory comment in .proj documenting the coupling. Fix 5 (Minor): Use tuple-of-ints version sort for Xcode selection instead of string comparison (fixes 16.10 < 16.2 ordering bug). Fix 6 (Minor): Make simulator boot failure fatal with sys.exit(1). Add dynamic fallback to latest available iPhone simulator before failing. Fix 7 (Nit): Add missing trailing newline to runner.py. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../maui_scenarios_ios_innerloop.proj | 4 + src/scenarios/mauiiosinnerloop/pre.py | 14 +++ src/scenarios/mauiiosinnerloop/setup_helix.py | 118 ++++++++++++++---- src/scenarios/shared/ioshelper.py | 46 +++---- src/scenarios/shared/runner.py | 2 +- 5 files changed, 136 insertions(+), 48 deletions(-) diff --git a/eng/performance/maui_scenarios_ios_innerloop.proj b/eng/performance/maui_scenarios_ios_innerloop.proj index fee7cbf197c..b8c6d6a32c2 100644 --- a/eng/performance/maui_scenarios_ios_innerloop.proj +++ b/eng/performance/maui_scenarios_ios_innerloop.proj @@ -74,6 +74,10 @@ $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" + $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id com.companyname.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --device-type simulator --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) export IOS_RID=$(iOSRid);$(Python) post.py output.log diff --git a/src/scenarios/mauiiosinnerloop/pre.py b/src/scenarios/mauiiosinnerloop/pre.py index 1614988de14..585eec9c2a9 100644 --- a/src/scenarios/mauiiosinnerloop/pre.py +++ b/src/scenarios/mauiiosinnerloop/pre.py @@ -299,6 +299,9 @@ def strip_non_ios_tfms(csproj_path: str, framework: str): # --- Modified MainPage.xaml.cs: add a debug line in the constructor --- # The template may place MainPage in either the root or Pages/ subdirectory. + # Normalize to ALWAYS use Pages/ so that the hardcoded --edit-dest paths in + # maui_scenarios_ios_innerloop.proj ("app/Pages/MainPage.xaml.cs") are valid. + pages_dir = os.path.join(const.APPDIR, 'Pages') cs_candidates = [ os.path.join(const.APPDIR, 'Pages', 'MainPage.xaml.cs'), os.path.join(const.APPDIR, 'MainPage.xaml.cs'), @@ -314,6 +317,17 @@ def strip_non_ios_tfms(csproj_path: str, framework: str): f"searched: {cs_candidates}" ) + # If MainPage files are at the root, move them into Pages/ so that the + # .proj's hardcoded edit-dest paths are always correct. + if os.path.dirname(os.path.abspath(cs_original)) != os.path.abspath(pages_dir): + os.makedirs(pages_dir, exist_ok=True) + for fname in ['MainPage.xaml.cs', 'MainPage.xaml']: + src_file = os.path.join(const.APPDIR, fname) + if os.path.exists(src_file): + shutil.move(src_file, os.path.join(pages_dir, fname)) + logger.info(f"Moved {src_file} → {os.path.join(pages_dir, fname)}") + cs_original = os.path.join(pages_dir, 'MainPage.xaml.cs') + cs_modified = os.path.join(src_dir, 'MainPage.xaml.cs') with open(cs_original, 'r') as f: cs_content = f.read() diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py index 2c391c64c6f..935250ffdf8 100644 --- a/src/scenarios/mauiiosinnerloop/setup_helix.py +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -126,8 +126,15 @@ def select_xcode(): run_cmd(["xcodebuild", "-version"], check=False) return - # Sort by version number (Xcode_16.2.app → key on "16.2") - candidates.sort(key=lambda p: p.rsplit("_", 1)[-1].replace(".app", "")) + # Sort by version number (Xcode_16.2.app → key on "16.2"). + # Use tuple-of-ints to get correct version ordering (e.g., 16.10 > 16.2). + def _xcode_version_key(path): + ver = path.rsplit("_", 1)[-1].replace(".app", "") + try: + return tuple(int(x) for x in ver.split('.')) + except ValueError: + return (0,) + candidates.sort(key=_xcode_version_key) xcode_path = candidates[-1] log(f"Selected Xcode: {xcode_path}", tee=True) @@ -174,11 +181,40 @@ def validate_simulator_runtimes(): "Simulator-based testing will fail.", tee=True) +def _find_latest_iphone_simulator(): + """Find the latest available iPhone simulator device name. + + Parses 'xcrun simctl list devices available' output to find iPhone devices + and returns the last one (typically the latest model). + Returns the device name string, or None if none found. + """ + import re + result = run_cmd(["xcrun", "simctl", "list", "devices", "available"], check=False) + if result.returncode != 0 or not result.stdout: + return None + + # Match lines like " iPhone 16 Pro Max (UUID) (Shutdown)" + iphone_names = [] + for line in result.stdout.splitlines(): + m = re.match(r'\s+(iPhone\s+\d+[^(]*?)\s+\(', line) + if m: + iphone_names.append(m.group(1).strip()) + + if not iphone_names: + return None + + # Return the last match — simctl lists devices in order, so the last + # iPhone entry is typically the latest model. + return iphone_names[-1] + + def boot_simulator(device_name): """Boot the target iOS simulator device. Handles the case where the device is already booted (exit code 149 from simctl boot = "Unable to boot device in current state: Booted"). + If the requested device fails to boot, tries the latest available iPhone + simulator as a fallback. Exits with code 1 if no simulator can be booted. """ log_raw("=== SIMULATOR BOOT ===", tee=True) log(f"Booting simulator device: '{device_name}'", tee=True) @@ -194,10 +230,33 @@ def boot_simulator(device_name): # Already booted — not an error log(f"Simulator '{device_name}' is already booted (OK).") else: - log(f"WARNING: Failed to boot simulator '{device_name}' " + log(f"Failed to boot simulator '{device_name}' " f"(exit code {result.returncode}). " - "Available devices:", tee=True) - run_cmd(["xcrun", "simctl", "list", "devices", "available"], check=False) + "Trying dynamic fallback...", tee=True) + + # Try to find and boot the latest available iPhone simulator + fallback = _find_latest_iphone_simulator() + if fallback and fallback != device_name: + log(f"Attempting fallback device: '{fallback}'", tee=True) + fb_result = run_cmd( + ["xcrun", "simctl", "boot", fallback], + check=False, + ) + if fb_result.returncode == 0: + log(f"Fallback simulator '{fallback}' booted successfully.", tee=True) + elif "Booted" in (fb_result.stdout or "") or fb_result.returncode == 149: + log(f"Fallback simulator '{fallback}' is already booted (OK).", tee=True) + else: + log(f"ERROR: Fallback simulator '{fallback}' also failed to boot " + f"(exit code {fb_result.returncode}).", tee=True) + run_cmd(["xcrun", "simctl", "list", "devices", "available"], check=False) + _dump_log() + sys.exit(1) + else: + log("ERROR: No fallback iPhone simulator found. Available devices:", tee=True) + run_cmd(["xcrun", "simctl", "list", "devices", "available"], check=False) + _dump_log() + sys.exit(1) # Log booted devices for confirmation log("Currently booted devices:") @@ -332,26 +391,35 @@ def detect_physical_device(): log_raw(f" {line}") # Try JSON output for structured parsing - json_result = run_cmd( - ["xcrun", "devicectl", "list", "devices", "--json-output", "/dev/stdout"], - check=False, - ) - if json_result.returncode == 0 and json_result.stdout: - try: - import json - data = json.loads(json_result.stdout) - devices = data.get("result", {}).get("devices", []) - for device in devices: - conn = device.get("connectionProperties", {}) - transport = conn.get("transportType", "") - name = device.get("deviceProperties", {}).get("name", "unknown") - device_udid = device.get("identifier", "") - if transport in ("wired", "localNetwork", "wifi") and device_udid: - log(f"Found connected device: {name} (UDID: {device_udid}, " - f"transport: {transport})", tee=True) - return device_udid - except Exception as e: - log(f"JSON parsing failed: {e}", tee=True) + # Write to temp file instead of /dev/stdout because devicectl mixes + # human-readable table text and JSON when writing to stdout. + import tempfile + json_tmp = tempfile.mktemp(suffix='.json', prefix='devicectl_') + try: + json_result = run_cmd( + ["xcrun", "devicectl", "list", "devices", "--json-output", json_tmp], + check=False, + ) + if json_result.returncode == 0 and os.path.exists(json_tmp): + try: + import json + with open(json_tmp, 'r') as f: + data = json.load(f) + devices = data.get("result", {}).get("devices", []) + for device in devices: + conn = device.get("connectionProperties", {}) + transport = conn.get("transportType", "") + name = device.get("deviceProperties", {}).get("name", "unknown") + device_udid = device.get("identifier", "") + if transport in ("wired", "localNetwork", "wifi") and device_udid: + log(f"Found connected device: {name} (UDID: {device_udid}, " + f"transport: {transport})", tee=True) + return device_udid + except Exception as e: + log(f"JSON parsing failed: {e}", tee=True) + finally: + if os.path.exists(json_tmp): + os.remove(json_tmp) log("No connected physical devices found.", tee=True) return None diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py index 8caac58c67d..6c9e6e9da5b 100644 --- a/src/scenarios/shared/ioshelper.py +++ b/src/scenarios/shared/ioshelper.py @@ -33,9 +33,12 @@ def detect_connected_device(): # Auto-detect via devicectl (Xcode 15+) getLogger().info("Auto-detecting connected iOS device via 'xcrun devicectl list devices'...") + json_tmp = None try: + import tempfile + json_tmp = tempfile.mktemp(suffix='.json', prefix='devicectl_') result = subprocess.run( - ['xcrun', 'devicectl', 'list', 'devices', '--json-output', '/dev/stdout'], + ['xcrun', 'devicectl', 'list', 'devices', '--json-output', json_tmp], capture_output=True, text=True, timeout=30 ) if result.returncode != 0: @@ -43,7 +46,8 @@ def detect_connected_device(): result.returncode, result.stderr) return None - data = json.loads(result.stdout) + with open(json_tmp, 'r') as f: + data = json.load(f) # devicectl JSON output has "result.devices" array with "identifier" (UDID) # and "connectionProperties.transportType" to filter for USB-connected devices devices = data.get('result', {}).get('devices', []) @@ -71,6 +75,9 @@ def detect_connected_device(): getLogger().warning("Failed to parse devicectl JSON output: %s", e) # Fall back to text parsing of non-JSON output return iOSHelper._detect_device_fallback() + finally: + if json_tmp and os.path.exists(json_tmp): + os.remove(json_tmp) @staticmethod def _detect_device_fallback(): @@ -88,9 +95,10 @@ def _detect_device_fallback(): return None # Look for lines with a UUID pattern (device UDID) - # Example line: " PERFIOS-01 00008101-001A09223E08001E ..." + # CoreDevice UUIDs use standard 8-4-4-4-12 format (e.g., 5AE7F3E5-C6A0-5FBE-BF3F-29CD735AAA0B). + # Old-style Apple UDIDs use 8-16 hex or 25-40 hex chars. for line in (result.stdout or '').splitlines(): - match = re.search(r'([0-9A-Fa-f]{8}-[0-9A-Fa-f]{16}|[0-9A-Fa-f]{25,40})', line) + match = re.search(r'([0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}|[0-9A-Fa-f]{8}-[0-9A-Fa-f]{16}|[0-9A-Fa-f]{25,40})', line) if match: udid = match.group(1) getLogger().info("Fallback detection found device UDID: %s (from: %s)", @@ -186,21 +194,15 @@ def measure_cold_startup(self, bundle_id): def measure_cold_startup_physical(self, bundle_id): """Measure app cold startup time on a physical device in milliseconds. - Terminates any running instance, waits briefly, then launches the app via devicectl. + Uses --terminate-existing to kill any running instance before launching. + devicectl's 'process terminate' only accepts --pid (not --bundle-id), + so we rely on launch's --terminate-existing flag instead. Returns wall-clock time for the launch command in milliseconds as int. """ - getLogger().info("Terminating app for cold startup on physical device: %s", bundle_id) - try: - RunCommand(['xcrun', 'devicectl', 'device', 'process', 'terminate', - '--device', self.device_id, '--bundle-id', bundle_id], verbose=True).run() - except subprocess.CalledProcessError: - getLogger().debug("Terminate returned error (app may not be running), ignoring.") - - time.sleep(0.5) - - getLogger().info("Launching app on physical device: %s", bundle_id) + getLogger().info("Launching app on physical device (with --terminate-existing): %s", bundle_id) start = time.time() RunCommand(['xcrun', 'devicectl', 'device', 'process', 'launch', + '--terminate-existing', '--device', self.device_id, bundle_id], verbose=True).run() elapsed_ms = (time.time() - start) * 1000 getLogger().info("Cold startup time: %d ms", int(elapsed_ms)) @@ -229,13 +231,13 @@ def terminate_app(self, bundle_id): getLogger().debug("Terminate returned error (app may not be running), ignoring.") def terminate_app_physical(self, bundle_id): - """Terminate the app on a physical device (ignore errors).""" - getLogger().info("Terminating app on physical device: %s", bundle_id) - try: - RunCommand(['xcrun', 'devicectl', 'device', 'process', 'terminate', - '--device', self.device_id, '--bundle-id', bundle_id], verbose=True).run() - except subprocess.CalledProcessError: - getLogger().debug("Terminate returned error (app may not be running), ignoring.") + """No-op: devicectl 'process terminate' requires --pid (not --bundle-id). + + Termination of running instances is handled by the --terminate-existing + flag on 'devicectl device process launch' in measure_cold_startup_physical(). + This method is kept for API symmetry with terminate_app() but does nothing. + """ + getLogger().debug("terminate_app_physical is a no-op — launch uses --terminate-existing") def close_simulator(self, skip_uninstall=False): """Clean up the simulator session. diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 2742fc91b50..d2de7530a35 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -1293,4 +1293,4 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, sys.exit(upload_code) finally: - iosHelper.cleanup(skip_uninstall=True) \ No newline at end of file + iosHelper.cleanup(skip_uninstall=True) From 99421705cc7cf4cf95e03cfbf77d4c7b06c74c6c Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 17:13:16 +0200 Subject: [PATCH 09/95] Quality fixes: tempfile.mkstemp, devicectl fallback, report guard - Replace deprecated tempfile.mktemp() with tempfile.mkstemp() in both ioshelper.py and setup_helix.py to avoid TOCTOU race condition. - Fix unreachable fallback in detect_connected_device(): when devicectl exits non-zero (e.g., older Xcode without --json-output), call _detect_device_fallback() instead of returning None immediately. - Guard against missing JSON report in runner.py IOSINNERLOOP branch: Startup.cs only writes reports when PERFLAB_INLAB=1, so local runs would crash with FileNotFoundError. Now degrades gracefully with empty counters and a warning. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiiosinnerloop/setup_helix.py | 3 +- src/scenarios/shared/ioshelper.py | 7 +++- src/scenarios/shared/runner.py | 40 ++++++++++++++----- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py index 935250ffdf8..353f05da39d 100644 --- a/src/scenarios/mauiiosinnerloop/setup_helix.py +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -394,7 +394,8 @@ def detect_physical_device(): # Write to temp file instead of /dev/stdout because devicectl mixes # human-readable table text and JSON when writing to stdout. import tempfile - json_tmp = tempfile.mktemp(suffix='.json', prefix='devicectl_') + fd, json_tmp = tempfile.mkstemp(suffix='.json', prefix='devicectl_') + os.close(fd) try: json_result = run_cmd( ["xcrun", "devicectl", "list", "devices", "--json-output", json_tmp], diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py index 6c9e6e9da5b..240fafe29e6 100644 --- a/src/scenarios/shared/ioshelper.py +++ b/src/scenarios/shared/ioshelper.py @@ -36,7 +36,8 @@ def detect_connected_device(): json_tmp = None try: import tempfile - json_tmp = tempfile.mktemp(suffix='.json', prefix='devicectl_') + fd, json_tmp = tempfile.mkstemp(suffix='.json', prefix='devicectl_') + os.close(fd) result = subprocess.run( ['xcrun', 'devicectl', 'list', 'devices', '--json-output', json_tmp], capture_output=True, text=True, timeout=30 @@ -44,7 +45,9 @@ def detect_connected_device(): if result.returncode != 0: getLogger().warning("devicectl list devices failed (exit %d): %s", result.returncode, result.stderr) - return None + # Non-zero exit likely means --json-output is unsupported (older Xcode). + # Fall back to text-based parsing. + return iOSHelper._detect_device_fallback() with open(json_tmp, 'r') as f: data = json.load(f) diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index d2de7530a35..f6edda722ee 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -1020,9 +1020,18 @@ def run(self): import upload def merge_build_and_startup(build_report_path, startup_results, final_report_path): - """Load the build metrics report, append a startup time counter, write to final path.""" - with open(build_report_path, 'r') as f: - report = json.load(f) + """Load the build metrics report, append a startup time counter, write to final path. + + If the build report doesn't exist (e.g., local runs without PERFLAB_INLAB=1), + creates a minimal report containing only the startup counter. + """ + if not os.path.exists(build_report_path): + getLogger().warning("Build report not found at %s. " + "Creating minimal report with startup data only." % build_report_path) + report = {"tests": [{"counters": []}]} + else: + with open(build_report_path, 'r') as f: + report = json.load(f) startup_counter = { "name": "Time to Main", "topCounter": True, @@ -1089,14 +1098,23 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, upload_to_perflab_container=False) startup.parsetraces(traits) - # Extract build counters and test metadata from temp report - with open(iter_report, 'r') as f: - iter_data = json.load(f) - test_obj = iter_data["tests"][0] - counters = test_obj["counters"] - # Return test metadata (without counters) for building the final report - test_metadata = test_obj.copy() - test_metadata["counters"] = [] + # Extract build counters and test metadata from temp report. + # The report is only written when PERFLAB_INLAB=1 (by Startup.cs). + # For local runs without that env var, degrade gracefully with empty counters. + if not os.path.exists(iter_report): + getLogger().warning("Build report not found (expected at %s). " + "This is normal for local runs without PERFLAB_INLAB=1. " + "Returning empty counters." % iter_report) + counters = [] + test_metadata = {"counters": []} + else: + with open(iter_report, 'r') as f: + iter_data = json.load(f) + test_obj = iter_data["tests"][0] + counters = test_obj["counters"] + # Return test metadata (without counters) for building the final report + test_metadata = test_obj.copy() + test_metadata["counters"] = [] # Clean up temp report (leave binlog for later cleanup) if os.path.exists(iter_report): From d21e9b719bf255d5cf2ce8bf6c93597f1c2d5b25 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 17:36:18 +0200 Subject: [PATCH 10/95] [TEMP] Disable all pipeline jobs except iOS Inner Loop Temporarily disable all other scenario jobs to speed up CI iteration while validating the new MAUI iOS Inner Loop scenario. This change should be reverted before merging. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-perf-jobs.yml | 494 ++++++++++++++++---------------- 1 file changed, 248 insertions(+), 246 deletions(-) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 834892cc975..209fa3400e1 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -17,7 +17,7 @@ jobs: # Public correctness jobs ###################################################### -- ${{ if parameters.runPublicJobs }}: +- ${{ if false }}: # [TEMP] was: parameters.runPublicJobs # Scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml @@ -326,244 +326,245 @@ jobs: - ${{ if parameters.runPrivateJobs }}: - # Scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: scenarios - projectFileName: scenarios.proj - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] disabled - all entries before iOS Inner Loop + # Scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Affinitized Scenario benchmarks (Initially just PDN) - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - win-arm64-ampere - isPublic: false - jobParameters: - runKind: scenarios - projectFileName: scenarios_affinitized.proj - channels: - - main - - 9.0 - - 8.0 - additionalJobIdentifier: 'Affinity_85' - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Affinitized Scenario benchmarks (Initially just PDN) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - win-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios_affinitized.proj + channels: + - main + - 9.0 + - 8.0 + additionalJobIdentifier: 'Affinity_85' + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (Mono Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: Mono - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (Mono Default) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: Mono + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (CoreCLR Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (CoreCLR Default) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (CoreCLR NativeAOT) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: NativeAOT - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (CoreCLR NativeAOT) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: NativeAOT + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (Mono Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: Mono - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (Mono Default) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: Mono + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (CoreCLR Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (CoreCLR Default) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (CoreCLR NativeAOT) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: NativeAOT - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (CoreCLR NativeAOT) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: NativeAOT + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (Mono - Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: Mono_Debug - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (Mono - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (CoreCLR - Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (CoreCLR - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (Mono - Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: Mono_Debug - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (Mono - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (CoreCLR - Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (CoreCLR - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui iOS Inner Loop (Mono - Default) - Debug - template: /eng/pipelines/templates/build-machine-matrix.yml @@ -606,30 +607,31 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # NativeAOT scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - isPublic: false - jobParameters: - runKind: nativeaot_scenarios - projectFileName: nativeaot_scenarios.proj - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] disabled + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + isPublic: false + jobParameters: + runKind: nativeaot_scenarios + projectFileName: nativeaot_scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} ################################################ # Scheduled Private jobs ################################################ # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if parameters.runScheduledPrivateJobs }}: +- ${{ if false }}: # [TEMP] was: parameters.runScheduledPrivateJobs # SDK scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml From 25a2846ddefd24c845a292dac71133af2f42fc70 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 18:12:24 +0200 Subject: [PATCH 11/95] Fix iOS Inner Loop build output capture and logging - Capture dotnet build output instead of crashing on CalledProcessError - Create traces/ directory before first build - Fix setup_helix.py to write output.log (matches .proj expectation) - Improve error handling for build failures Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiiosinnerloop/setup_helix.py | 2 +- src/scenarios/shared/runner.py | 26 ++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py index 353f05da39d..28bf1a44c31 100644 --- a/src/scenarios/mauiiosinnerloop/setup_helix.py +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -435,7 +435,7 @@ def main(): upload_root = os.environ.get("HELIX_WORKITEM_UPLOAD_ROOT") if upload_root: os.makedirs(upload_root, exist_ok=True) - _logfile = open(os.path.join(upload_root, "setup_helix.log"), "a") + _logfile = open(os.path.join(upload_root, "output.log"), "a") workitem_root = os.environ.get("HELIX_WORKITEM_ROOT", ".") correlation_payload = os.environ.get("HELIX_CORRELATION_PAYLOAD", ".") diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index f6edda722ee..7490e3850a5 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -1077,7 +1077,16 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, iter_binlog = os.path.join(const.TRACEDIR, iter_binlog_name) incremental_cmd = base_cmd + [f'-bl:{iter_binlog}'] getLogger().info("Incremental build: %s" % ' '.join(incremental_cmd)) - subprocess.run(incremental_cmd, check=True) + try: + result = subprocess.run(incremental_cmd) + getLogger().info("Incremental build exit code: %d" % result.returncode) + if result.returncode != 0: + getLogger().error("Incremental build FAILED (iteration %d, exit code %d). Command: %s" % (iteration, result.returncode, ' '.join(incremental_cmd))) + raise subprocess.CalledProcessError(result.returncode, incremental_cmd) + except subprocess.CalledProcessError: + getLogger().error("dotnet build failed during incremental iteration %d. " + "Check the build output above and the binlog at: %s" % (iteration, iter_binlog)) + raise # Install and measure startup — dispatch based on device type if is_physical_device: @@ -1159,9 +1168,18 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, exename = self.traits.exename # --- First build + deploy --- - first_cmd = base_cmd + [f'-bl:{first_binlog}'] - getLogger().info("First build: %s" % ' '.join(first_cmd)) - subprocess.run(first_cmd, check=True) + try: + first_cmd = base_cmd + [f'-bl:{first_binlog}'] + getLogger().info("First build: %s" % ' '.join(first_cmd)) + result = subprocess.run(first_cmd) + getLogger().info("First build exit code: %d" % result.returncode) + if result.returncode != 0: + getLogger().error("First build FAILED (exit code %d). Command: %s" % (result.returncode, ' '.join(first_cmd))) + raise subprocess.CalledProcessError(result.returncode, first_cmd) + except subprocess.CalledProcessError: + getLogger().error("dotnet build failed for iOS inner loop. " + "Check the build output above and the binlog at: %s" % first_binlog) + raise # --- Device/simulator setup and first deploy --- iosHelper = iOSHelper() From 53e647e0814592404d75e0ea08f1c11229b3eb59 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 18:30:02 +0200 Subject: [PATCH 12/95] Capture and print dotnet build output for iOS Inner Loop The dotnet build stdout/stderr wasn't appearing in Helix console logs, making it impossible to diagnose build failures. Explicitly capture and print build output through Python logging. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/runner.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 7490e3850a5..5b1fccf7a61 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -1012,7 +1012,6 @@ def run(self): elif self.testtype == const.IOSINNERLOOP: import hashlib - import subprocess from shutil import copytree from performance.common import runninginlab from performance.constants import UPLOAD_CONTAINER, UPLOAD_STORAGE_URI, UPLOAD_QUEUE @@ -1055,7 +1054,6 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, edit_pairs is a list of (dest_path, original_content, modified_content) tuples. Returns (startup_ms, counters_list, binlog_path, test_metadata). """ - import subprocess getLogger().info("=== Incremental iteration %d/%d ===" % (iteration, num_iterations)) @@ -1072,18 +1070,14 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, content_hash = hashlib.md5(original.encode()).hexdigest()[:8] getLogger().info("Restored original source: %s (hash=%s, len=%d)" % (dest, content_hash, len(original))) - # Incremental build with per-iteration binlog + # Incremental build with per-iteration binlog. + # Use RunCommand to capture and log build output (see first build comment). iter_binlog_name = 'incremental-build-and-deploy-%d.binlog' % iteration iter_binlog = os.path.join(const.TRACEDIR, iter_binlog_name) incremental_cmd = base_cmd + [f'-bl:{iter_binlog}'] - getLogger().info("Incremental build: %s" % ' '.join(incremental_cmd)) try: - result = subprocess.run(incremental_cmd) - getLogger().info("Incremental build exit code: %d" % result.returncode) - if result.returncode != 0: - getLogger().error("Incremental build FAILED (iteration %d, exit code %d). Command: %s" % (iteration, result.returncode, ' '.join(incremental_cmd))) - raise subprocess.CalledProcessError(result.returncode, incremental_cmd) - except subprocess.CalledProcessError: + RunCommand(incremental_cmd, verbose=True).run() + except CalledProcessError: getLogger().error("dotnet build failed during incremental iteration %d. " "Check the build output above and the binlog at: %s" % (iteration, iter_binlog)) raise @@ -1153,7 +1147,8 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, first_binlog = os.path.join(const.TRACEDIR, 'first-build-and-deploy.binlog') # Build the base MSBuild command (no -t:Install for iOS — plain dotnet build) - base_cmd = ['dotnet', 'build', self.csprojpath] + # -v:n (normal verbosity) ensures MSBuild errors/warnings appear in the log + base_cmd = ['dotnet', 'build', self.csprojpath, '-v:n'] if self.configuration: base_cmd.extend(['-c', self.configuration]) if self.framework: @@ -1168,15 +1163,13 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, exename = self.traits.exename # --- First build + deploy --- + # Use RunCommand (verbose=True) to capture and log build output line-by-line + # through Python's logging, which Helix captures. Plain subprocess.run() + # inherits stdout/stderr but they don't appear in Helix console logs. try: first_cmd = base_cmd + [f'-bl:{first_binlog}'] - getLogger().info("First build: %s" % ' '.join(first_cmd)) - result = subprocess.run(first_cmd) - getLogger().info("First build exit code: %d" % result.returncode) - if result.returncode != 0: - getLogger().error("First build FAILED (exit code %d). Command: %s" % (result.returncode, ' '.join(first_cmd))) - raise subprocess.CalledProcessError(result.returncode, first_cmd) - except subprocess.CalledProcessError: + RunCommand(first_cmd, verbose=True).run() + except CalledProcessError: getLogger().error("dotnet build failed for iOS inner loop. " "Check the build output above and the binlog at: %s" % first_binlog) raise From bb1e6d26cba92d4963d4454b4677e54f006a2d09 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 20:18:11 +0200 Subject: [PATCH 13/95] Increase iOS Inner Loop Helix timeout to 2.5 hours Build 2943141 hit the 90-minute timeout. iOS first build with AOT compilation can take 30+ minutes, plus 3 incremental iterations. Increasing to 2.5 hours to allow full completion. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_scenarios_ios_innerloop.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/maui_scenarios_ios_innerloop.proj b/eng/performance/maui_scenarios_ios_innerloop.proj index b8c6d6a32c2..f405720bb41 100644 --- a/eng/performance/maui_scenarios_ios_innerloop.proj +++ b/eng/performance/maui_scenarios_ios_innerloop.proj @@ -34,7 +34,7 @@ - 01:30 + 02:30 From 565f8eb7f13d4c93529510dd82402eb2b4a81480 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 20:44:06 +0200 Subject: [PATCH 14/95] Bypass Xcode version validation for iOS Inner Loop Helix machines have Xcode 26.2 but the iOS SDK requires 26.3. The minor version difference shouldn't affect build correctness, so bypass the check with ValidateXcodeVersion=false. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_scenarios_ios_innerloop.proj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/performance/maui_scenarios_ios_innerloop.proj b/eng/performance/maui_scenarios_ios_innerloop.proj index f405720bb41..86ef31bbb49 100644 --- a/eng/performance/maui_scenarios_ios_innerloop.proj +++ b/eng/performance/maui_scenarios_ios_innerloop.proj @@ -15,6 +15,10 @@ can be set via pipeline parameter when device support is enabled. --> iossimulator-arm64 <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(iOSRid) + + <_MSBuildArgs>$(_MSBuildArgs) /p:ValidateXcodeVersion=false $(RuntimeFlavor)_Default 3 From 60d89540cd311a0ecc5f7eb2e7ac7931ec0694f1 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 21:07:09 +0200 Subject: [PATCH 15/95] Fix iOS simulator RID for Intel x64 Helix machines Mac.iPhone.17.Perf queue uses Intel x64 machines which need iossimulator-x64, not iossimulator-arm64. Add architecture detection in setup_helix.py and update default RID in .proj. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../maui_scenarios_ios_innerloop.proj | 6 ++++-- src/scenarios/mauiiosinnerloop/setup_helix.py | 20 ++++++++++++++++++- src/scenarios/shared/runner.py | 8 ++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/eng/performance/maui_scenarios_ios_innerloop.proj b/eng/performance/maui_scenarios_ios_innerloop.proj index 86ef31bbb49..98ad3ab33f2 100644 --- a/eng/performance/maui_scenarios_ios_innerloop.proj +++ b/eng/performance/maui_scenarios_ios_innerloop.proj @@ -12,8 +12,10 @@ <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false - iossimulator-arm64 + can be set via pipeline parameter when device support is enabled. + Default is iossimulator-x64 because the Mac.iPhone.17.Perf Helix + queue runs on Intel x64 machines (e.g. DNCENGMAC045). --> + iossimulator-x64 <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(iOSRid) <_MSBuildArgs>$(_MSBuildArgs) /p:ValidateXcodeVersion=false + + <_MSBuildArgs>$(_MSBuildArgs) /p:MtouchLink=None $(RuntimeFlavor)_Default 3 diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py index 4ab6fb4e81c..9e3c58359ce 100644 --- a/src/scenarios/mauiiosinnerloop/setup_helix.py +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -161,6 +161,37 @@ def _xcode_version_key(path): run_cmd(["xcodebuild", "-version"], check=False) +def _validate_xcode_version(min_major=26): + """Fail fast if the active Xcode is too old for iOS inner loop builds. + + Parses the version from 'xcodebuild -version' (e.g. "Xcode 15.0" → 15) + and exits with a clear diagnostic if the major version is below *min_major*. + """ + result = run_cmd(["xcodebuild", "-version"], check=False) + if result.returncode != 0 or not result.stdout: + log("WARNING: Could not determine Xcode version — skipping check.", tee=True) + return + + import re + m = re.search(r"Xcode\s+(\d+)\.(\d+)", result.stdout) + if not m: + log("WARNING: Could not parse Xcode version from output — skipping check.", + tee=True) + return + + major, minor = int(m.group(1)), int(m.group(2)) + if major < min_major: + log(f"ERROR: Xcode version {major}.{minor} is too old. " + f"iOS inner loop requires Xcode {min_major}.0 or later. " + "This Helix machine cannot run iOS inner loop measurements.", + tee=True) + _dump_log() + sys.exit(1) + + log(f"Xcode version {major}.{minor} meets minimum requirement " + f"(>= {min_major}.0)", tee=True) + + def validate_simulator_runtimes(): """Check that iOS simulator runtimes are available on this machine.""" log_raw("=== SIMULATOR RUNTIME VALIDATION ===", tee=True) @@ -492,6 +523,12 @@ def main(): # Step 2: Select the correct Xcode version select_xcode() + # Step 2b: Validate minimum Xcode version for iOS inner loop builds. + # Machines with Xcode < 26 cannot build .NET MAUI iOS apps targeting the + # iOS 26+ SDK. Fail fast with a clear message instead of waiting 10+ min + # for ILLink to crash with MT0180. + _validate_xcode_version() + # Step 3 & 4: Device-type-specific setup if is_physical_device: # Detect and validate the connected physical device From 68d6b4c46b0bdd971a551af080670a9bdcb9bf4e Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 22:37:40 +0200 Subject: [PATCH 19/95] Re-enable all pipeline jobs after iOS inner loop CI validation Remove temporary ${{ if false }}: wrappers that disabled all jobs except iOS inner loop during iterative CI debugging. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-perf-jobs.yml | 494 ++++++++++++++++---------------- 1 file changed, 246 insertions(+), 248 deletions(-) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 209fa3400e1..834892cc975 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -17,7 +17,7 @@ jobs: # Public correctness jobs ###################################################### -- ${{ if false }}: # [TEMP] was: parameters.runPublicJobs +- ${{ if parameters.runPublicJobs }}: # Scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml @@ -326,245 +326,244 @@ jobs: - ${{ if parameters.runPrivateJobs }}: - - ${{ if false }}: # [TEMP] disabled - all entries before iOS Inner Loop - # Scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: scenarios - projectFileName: scenarios.proj - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Affinitized Scenario benchmarks (Initially just PDN) - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - win-arm64-ampere - isPublic: false - jobParameters: - runKind: scenarios - projectFileName: scenarios_affinitized.proj - channels: - - main - - 9.0 - - 8.0 - additionalJobIdentifier: 'Affinity_85' - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Affinitized Scenario benchmarks (Initially just PDN) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - win-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios_affinitized.proj + channels: + - main + - 9.0 + - 8.0 + additionalJobIdentifier: 'Affinity_85' + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (Mono Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: Mono - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (Mono Default) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: Mono + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (CoreCLR Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (CoreCLR Default) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (CoreCLR NativeAOT) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: NativeAOT - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (CoreCLR NativeAOT) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: NativeAOT + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (Mono Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: Mono - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (Mono Default) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: Mono + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (CoreCLR Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (CoreCLR Default) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (CoreCLR NativeAOT) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: NativeAOT - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (CoreCLR NativeAOT) - Release + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: NativeAOT + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (Mono - Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: Mono_Debug - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (Mono - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (CoreCLR - Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui Android scenario benchmarks (CoreCLR - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (Mono - Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: Mono_Debug - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (Mono - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (CoreCLR - Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + # Maui iOS scenario benchmarks (CoreCLR - Default) - Debug + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui iOS Inner Loop (Mono - Default) - Debug - template: /eng/pipelines/templates/build-machine-matrix.yml @@ -607,31 +606,30 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # NativeAOT scenario benchmarks - - ${{ if false }}: # [TEMP] disabled - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - isPublic: false - jobParameters: - runKind: nativeaot_scenarios - projectFileName: nativeaot_scenarios.proj - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + isPublic: false + jobParameters: + runKind: nativeaot_scenarios + projectFileName: nativeaot_scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} ################################################ # Scheduled Private jobs ################################################ # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if false }}: # [TEMP] was: parameters.runScheduledPrivateJobs +- ${{ if parameters.runScheduledPrivateJobs }}: # SDK scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml From ba4e0d15592d98f80eedbf2b1240189bc89799d7 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Fri, 3 Apr 2026 23:38:38 +0200 Subject: [PATCH 20/95] Add install timing, CoreCLR config, and device support for iOS inner loop - Separate install from simulator/device setup in ioshelper.py - Capture install time for first and incremental deploys in runner.py - Add "Install Time" counter to both perf reports - Add CoreCLR Debug job entry in pipeline YAML - Add device (ios-arm64) job entries for both Mono and CoreCLR - Wire iOSRid env var through to MSBuild for device builds - [TEMP] Disable non-iOS-inner-loop jobs for CI validation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-perf-jobs.yml | 833 ++++++++++++++++-------------- scripts/run_performance_job.py | 6 + src/scenarios/shared/ioshelper.py | 27 +- src/scenarios/shared/runner.py | 45 +- 4 files changed, 513 insertions(+), 398 deletions(-) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 834892cc975..ea82215fd02 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -327,207 +327,295 @@ jobs: - ${{ if parameters.runPrivateJobs }}: # Scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: scenarios - projectFileName: scenarios.proj - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Affinitized Scenario benchmarks (Initially just PDN) - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - win-arm64-ampere - isPublic: false - jobParameters: - runKind: scenarios - projectFileName: scenarios_affinitized.proj - channels: - - main - - 9.0 - - 8.0 - additionalJobIdentifier: 'Affinity_85' - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - win-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios_affinitized.proj + channels: + - main + - 9.0 + - 8.0 + additionalJobIdentifier: 'Affinity_85' + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui Android scenario benchmarks (Mono Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: Mono - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: Mono + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui Android scenario benchmarks (CoreCLR Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui Android scenario benchmarks (CoreCLR NativeAOT) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: NativeAOT - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: NativeAOT + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui iOS scenario benchmarks (Mono Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: Mono - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: Mono + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui iOS scenario benchmarks (CoreCLR Default) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: Default - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui iOS scenario benchmarks (CoreCLR NativeAOT) - Release - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - channels: - - main - runtimeFlavor: coreclr - codeGenType: NativeAOT - buildConfig: Release - additionalJobIdentifier: CoreCLR - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: NativeAOT + buildConfig: Release + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Maui Android scenario benchmarks (Mono - Default) - Debug + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android scenario benchmarks (CoreCLR - Default) - Debug + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS scenario benchmarks (Mono - Default) - Debug + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: mono + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: Mono_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS scenario benchmarks (CoreCLR - Default) - Debug + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + channels: + - main + runtimeFlavor: coreclr + codeGenType: Default + buildConfig: Debug + additionalJobIdentifier: CoreCLR_Debug + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Inner Loop (Mono - Default) - Debug - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy + - osx-x64-ios-arm64 isPublic: false jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios_innerloop.proj channels: - main runtimeFlavor: mono codeGenType: Default buildConfig: Debug - additionalJobIdentifier: Mono_Debug + additionalJobIdentifier: Mono_InnerLoop ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # Maui Android scenario benchmarks (CoreCLR - Default) - Debug + # Maui iOS Inner Loop (CoreCLR - Default) - Debug - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy + - osx-x64-ios-arm64 isPublic: false jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios_innerloop.proj channels: - main runtimeFlavor: coreclr codeGenType: Default buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug + additionalJobIdentifier: CoreCLR_InnerLoop ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS scenario benchmarks (Mono - Default) - Debug + # Maui iOS Inner Loop (Mono - Default) - Debug - Device - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml @@ -535,18 +623,20 @@ jobs: - osx-x64-ios-arm64 isPublic: false jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios_innerloop.proj channels: - main runtimeFlavor: mono codeGenType: Default buildConfig: Debug - additionalJobIdentifier: Mono_Debug + additionalJobIdentifier: Mono_InnerLoop_Device + runEnvVars: + - iOSRid=ios-arm64 ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - - # Maui iOS scenario benchmarks (CoreCLR - Default) - Debug + + # Maui iOS Inner Loop (CoreCLR - Default) - Debug - Device - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml @@ -554,33 +644,16 @@ jobs: - osx-x64-ios-arm64 isPublic: false jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj + runKind: maui_scenarios_ios_innerloop + projectFileName: maui_scenarios_ios_innerloop.proj channels: - main runtimeFlavor: coreclr codeGenType: Default buildConfig: Debug - additionalJobIdentifier: CoreCLR_Debug - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # Maui iOS Inner Loop (Mono - Default) - Debug - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios_innerloop - projectFileName: maui_scenarios_ios_innerloop.proj - channels: - - main - runtimeFlavor: mono - codeGenType: Default - buildConfig: Debug - additionalJobIdentifier: Mono_InnerLoop + additionalJobIdentifier: CoreCLR_InnerLoop_Device + runEnvVars: + - iOSRid=ios-arm64 ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} @@ -606,23 +679,24 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # NativeAOT scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - isPublic: false - jobParameters: - runKind: nativeaot_scenarios - projectFileName: nativeaot_scenarios.proj - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + isPublic: false + jobParameters: + runKind: nativeaot_scenarios + projectFileName: nativeaot_scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} ################################################ # Scheduled Private jobs @@ -632,40 +706,42 @@ jobs: - ${{ if parameters.runScheduledPrivateJobs }}: # SDK scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - - win-x86-viper - #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available - isPublic: false - jobParameters: - runKind: sdk_scenarios - projectFileName: sdk_scenarios.proj - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + - win-x86-viper + #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available + isPublic: false + jobParameters: + runKind: sdk_scenarios + projectFileName: sdk_scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Blazor 3.2 scenario benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64-viper - isPublic: false - jobParameters: - runKind: blazor_scenarios - projectFileName: blazor_scenarios.proj - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-viper + isPublic: false + jobParameters: + runKind: blazor_scenarios + projectFileName: blazor_scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # F# benchmarks - ${{ if false }}: # skipping, no useful benchmarks there currently @@ -689,139 +765,145 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: fsharpmicro - targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj - runCategories: 'FSharpMicro' - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: fsharpmicro + targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj + runCategories: 'FSharpMicro' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # bepuphysics benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: bepuphysics - targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: bepuphysics + targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runCategories: 'BepuPhysics' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # ImageSharp benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: imagesharp - targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: imagesharp + targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runCategories: 'ImageSharp' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Akade.IndexedSet benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: akadeindexedset - targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj - runCategories: 'AkadeIndexedSet' - channels: - - main - - 9.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: akadeindexedset + targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj + runCategories: 'AkadeIndexedSet' + channels: + - main + - 9.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # ML.NET benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: mlnet - targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: - - main - - 9.0 - - 8.0 - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: mlnet + targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runCategories: 'mldotnet' + channels: + - main + - 9.0 + - 8.0 + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # Roslyn benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: roslyn - targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - 9.0 - - 8.0 - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: roslyn + targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + - 9.0 + - 8.0 + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # ILLink benchmarks # disabled because of: https://github.com/dotnet/performance/issues/3569 @@ -844,22 +926,23 @@ jobs: - 8.0 # Powershell benchmarks - - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-performance-job.yml - buildMachines: - - win-x64-viper - - ubuntu-x64-viper - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: powershell - targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj - runCategories: 'Public Internal' - channels: - - main - - 9.0 - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: # [TEMP] Disabled for iOS inner loop CI validation + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64-viper + - ubuntu-x64-viper + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: powershell + targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj + runCategories: 'Public Internal' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index e531d4158e7..38c2db61a7f 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -1170,6 +1170,12 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_ os.environ["CodegenType"] = args.codegen_type or '' os.environ["BuildConfig"] = args.build_config or DEFAULT_BUILD_CONFIG + # Propagate run_env_vars to os.environ so they reach MSBuild + # evaluation as properties (e.g., iOSRid for device builds). + if args.run_env_vars: + for key, value in args.run_env_vars.items(): + os.environ[key] = value + # TODO: See if these commands are needed for linux as they were being called before but were failing. if args.os_group == "windows" or args.os_group == "osx": break_system_packages = ["--break-system-packages"] if args.os_group == "osx" else [] diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py index 240fafe29e6..2cd997162cb 100644 --- a/src/scenarios/shared/ioshelper.py +++ b/src/scenarios/shared/ioshelper.py @@ -112,7 +112,12 @@ def _detect_device_fallback(): return None def setup_simulator(self, bundle_id, app_bundle_path, device_id='booted'): - """Boot the iOS simulator and install the app bundle.""" + """Prepare the iOS simulator for testing. + + Verifies the simulator is booted and uninstalls any existing app with + the bundle ID. Does NOT install the app — call install_app() separately + so install timing can be captured independently. + """ self.bundle_id = bundle_id self.device_id = device_id self.app_bundle_path = app_bundle_path @@ -131,26 +136,26 @@ def setup_simulator(self, bundle_id, app_bundle_path, device_id='booted'): else: getLogger().info("Using already-booted simulator (device_id='booted')") - # Install app - getLogger().info("Installing app bundle: %s", app_bundle_path) - RunCommand(['xcrun', 'simctl', 'install', device_id, app_bundle_path], verbose=True).run() - getLogger().info("Completed install.") + # Uninstall any existing app to ensure a clean install + getLogger().info("Uninstalling any existing app: %s", bundle_id) + try: + RunCommand(['xcrun', 'simctl', 'uninstall', device_id, bundle_id], verbose=True).run() + except subprocess.CalledProcessError: + getLogger().debug("Uninstall returned error (app may not be installed), ignoring.") def setup_physical_device(self, bundle_id, app_bundle_path, device_id): """Set up a physical iOS device for testing. - Installs the app bundle on the connected physical device using devicectl. - Requires Xcode 15+ for the 'xcrun devicectl' toolchain. + Configures the device for deployment. Does NOT install the app — + call install_app_physical() separately so install timing can be + captured independently. Requires Xcode 15+ for 'xcrun devicectl'. """ self.bundle_id = bundle_id self.device_id = device_id self.app_bundle_path = app_bundle_path self.is_physical_device = True - getLogger().info("Installing app bundle on physical device %s: %s", device_id, app_bundle_path) - RunCommand(['xcrun', 'devicectl', 'device', 'install', 'app', - '--device', device_id, app_bundle_path], verbose=True).run() - getLogger().info("Completed install on physical device.") + getLogger().info("Physical device setup complete for device %s", device_id) def install_app(self, app_bundle_path): """Install the app bundle and return install time in milliseconds.""" diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index d3663cc7c7c..10c3dd45203 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -1026,19 +1026,27 @@ def run(self): from shared.util import helixuploaddir import upload - def merge_build_and_startup(build_report_path, startup_results, final_report_path): - """Load the build metrics report, append a startup time counter, write to final path. + def merge_build_deploy_and_startup(build_report_path, install_results, startup_results, final_report_path): + """Load the build metrics report, append install time and startup time counters, write to final path. If the build report doesn't exist (e.g., local runs without PERFLAB_INLAB=1), - creates a minimal report containing only the startup counter. + creates a minimal report containing only the install and startup counters. """ if not os.path.exists(build_report_path): getLogger().warning("Build report not found at %s. " - "Creating minimal report with startup data only." % build_report_path) + "Creating minimal report with install and startup data only." % build_report_path) report = {"tests": [{"counters": []}]} else: with open(build_report_path, 'r') as f: report = json.load(f) + install_counter = { + "name": "Install Time", + "topCounter": False, + "defaultCounter": False, + "higherIsBetter": False, + "metricName": "ms", + "results": install_results + } startup_counter = { "name": "Time to Main", "topCounter": True, @@ -1048,6 +1056,7 @@ def merge_build_and_startup(build_report_path, startup_results, final_report_pat "results": startup_results } # Report structure: { "tests": [ { "counters": [...] } ] } + report["tests"][0]["counters"].append(install_counter) report["tests"][0]["counters"].append(startup_counter) with open(final_report_path, 'w') as f: json.dump(report, f, indent=2) @@ -1060,7 +1069,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, """Run one incremental build+deploy+startup iteration. edit_pairs is a list of (dest_path, original_content, modified_content) tuples. - Returns (startup_ms, counters_list, binlog_path, test_metadata). + Returns (startup_ms, install_ms, counters_list, binlog_path, test_metadata). """ getLogger().info("=== Incremental iteration %d/%d ===" % (iteration, num_iterations)) @@ -1092,12 +1101,12 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, # Install and measure startup — dispatch based on device type if is_physical_device: - iosHelper.install_app_physical(app_bundle) + install_ms = iosHelper.install_app_physical(app_bundle) ms = iosHelper.measure_cold_startup_physical(bundleid) else: - iosHelper.install_app(app_bundle) + install_ms = iosHelper.install_app(app_bundle) ms = iosHelper.measure_cold_startup(bundleid) - getLogger().info("Incremental iteration %d/%d: build+deploy done, startup: %d ms" % (iteration, num_iterations, ms)) + getLogger().info("Incremental iteration %d/%d: build+deploy done, install: %.1f ms, startup: %d ms" % (iteration, num_iterations, install_ms, ms)) # Parse this iteration's binlog → temp build report iter_report_name = 'incremental-build-report-%d.json' % iteration @@ -1141,7 +1150,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, os.remove(iter_report) getLogger().info("Removed temp report: %s" % iter_report) - return ms, counters, iter_binlog, test_metadata + return ms, install_ms, counters, iter_binlog, test_metadata # --- Validate inputs --- if not self.csprojpath: @@ -1197,13 +1206,15 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, if is_physical: iosHelper.setup_physical_device(self.bundleid, app_bundle, self.deviceid) + first_install_ms = iosHelper.install_app_physical(app_bundle) first_startup_ms = iosHelper.measure_cold_startup_physical(self.bundleid) else: iosHelper.setup_simulator(self.bundleid, app_bundle, self.deviceid) + first_install_ms = iosHelper.install_app(app_bundle) first_startup_ms = iosHelper.measure_cold_startup(self.bundleid) # --- First startup measurement --- - getLogger().info("First deploy startup: %d ms" % first_startup_ms) + getLogger().info("First deploy install: %.1f ms, startup: %d ms" % (first_install_ms, first_startup_ms)) # --- Parse first build report --- startup = StartupWrapper() @@ -1218,7 +1229,7 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, # Merge first build metrics + startup → first e2e report first_e2e_report = os.path.join(const.TRACEDIR, 'first-debug-e2e-perf-lab-report.json') - merge_build_and_startup(first_build_report, [first_startup_ms], first_e2e_report) + merge_build_deploy_and_startup(first_build_report, [first_install_ms], [first_startup_ms], first_e2e_report) # --- Incremental loop --- num_iterations = self.innerloopiterations @@ -1242,18 +1253,20 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, raise Exception("No edit-src/edit-dest specified; incremental builds require file pairs to toggle") incremental_startup_results = [] + incremental_install_results = [] aggregated_counters = {} # counter_name -> aggregated counter dict report_template = None # test metadata from first parsed report intermediate_files = [] # files to clean up for iteration in range(1, num_iterations + 1): - ms, counters, iter_binlog, test_metadata = run_incremental_iteration( + ms, install_ms, counters, iter_binlog, test_metadata = run_incremental_iteration( iteration, num_iterations, base_cmd, edit_pairs, self.bundleid, app_bundle, scenarioprefix, startup, self.traits, iosHelper, is_physical_device=is_physical) incremental_startup_results.append(ms) + incremental_install_results.append(install_ms) intermediate_files.append(iter_binlog) # Save test metadata from the first iteration @@ -1276,6 +1289,14 @@ def run_incremental_iteration(iteration, num_iterations, base_cmd, edit_pairs, # --- Aggregate incremental results --- incremental_e2e_report = os.path.join(const.TRACEDIR, 'incremental-debug-e2e-perf-lab-report.json') final_counters = list(aggregated_counters.values()) + final_counters.append({ + "name": "Install Time", + "topCounter": False, + "defaultCounter": False, + "higherIsBetter": False, + "metricName": "ms", + "results": incremental_install_results + }) final_counters.append({ "name": "Time to Main", "topCounter": True, From 60f11067b395de8dc9f7aa13193ae40e744f483f Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Sat, 4 Apr 2026 00:11:19 +0200 Subject: [PATCH 21/95] Add fallback for workload install version skew in pre.py When the dynamically-resolved manifest references SDK packs not yet propagated to NuGet feeds, fall back to installing without the rollback file. This avoids CI being blocked by transient feed propagation delays. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiiosinnerloop/pre.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/scenarios/mauiiosinnerloop/pre.py b/src/scenarios/mauiiosinnerloop/pre.py index 585eec9c2a9..396e2a70be6 100644 --- a/src/scenarios/mauiiosinnerloop/pre.py +++ b/src/scenarios/mauiiosinnerloop/pre.py @@ -94,8 +94,17 @@ def install_maui_ios_workload(precommands: PreCommands): f.write(json.dumps(rollback_dict, indent=4)) logger.info("Created rollback_maui.json file") - # Install maui-ios (not 'maui') — only installs iOS components - precommands.install_workload('maui-ios', ['--from-rollback-file', 'rollback_maui.json']) + # Install maui-ios (not 'maui') — only installs iOS components. + # When a new manifest is published to the feed, referenced SDK packs may + # not have propagated to all NuGet feeds yet, causing "package NOT FOUND". + # Fall back to installing without the rollback file, which lets the SDK + # resolve a recent stable version that is already fully available. + try: + precommands.install_workload('maui-ios', ['--from-rollback-file', 'rollback_maui.json']) + except Exception as e: + logger.warning(f"Workload install with rollback file failed (possible NuGet version skew): {e}") + logger.info("Retrying without rollback file (will use SDK default version)...") + precommands.install_workload('maui-ios') logger.info("########## Finished installing maui-ios workload ##########") def check_xcode_compatibility(framework: str): From 995fe9456f72eb9b17f39e7d736a70f0e1364701 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Sat, 4 Apr 2026 00:41:41 +0200 Subject: [PATCH 22/95] Add workload install fallback to setup_helix.py When the rollback file references SDK packs not yet propagated to NuGet feeds, retry without the rollback file. Matches the fallback pattern already added to pre.py. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiiosinnerloop/setup_helix.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py index 9e3c58359ce..cc9e0f9952b 100644 --- a/src/scenarios/mauiiosinnerloop/setup_helix.py +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -325,6 +325,24 @@ def install_workload(ctx): install_args.append("--ignore-failed-sources") result = run_cmd(install_args, check=False) + if result.returncode != 0 and os.path.isfile(rollback_file): + # When a new manifest is published to the feed, referenced SDK packs + # may not have propagated to all NuGet feeds yet, causing + # "package NOT FOUND". Retry without the rollback file so the SDK + # resolves a recent stable version that is already fully available. + log(f"WARNING: Workload install with rollback file failed " + f"(exit code {result.returncode}, possible NuGet version skew)", tee=True) + log("Retrying without rollback file (will use SDK default version)...", tee=True) + + retry_args = [ + ctx["dotnet_exe"], "workload", "install", "maui-ios", + ] + if os.path.isfile(nuget_config): + retry_args.extend(["--configfile", nuget_config]) + retry_args.append("--ignore-failed-sources") + + result = run_cmd(retry_args, check=False) + if result.returncode != 0: log(f"WORKLOAD INSTALL FAILED (exit code {result.returncode})", tee=True) _dump_log() From ae4c34cfe1ecfc1b6b64f4a8aa71d65fcd47238f Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Sat, 4 Apr 2026 01:40:09 +0200 Subject: [PATCH 23/95] Exclude simulator work item from device jobs The simulator HelixWorkItem was unconditionally included, even when iOSRid=ios-arm64. This caused the simulator to receive device RID in _MSBuildArgs, producing ARM64 binaries that can't install on a simulator. Add Condition to exclude it from device jobs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/performance/maui_scenarios_ios_innerloop.proj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/performance/maui_scenarios_ios_innerloop.proj b/eng/performance/maui_scenarios_ios_innerloop.proj index d13434e2959..1443f87ddfd 100644 --- a/eng/performance/maui_scenarios_ios_innerloop.proj +++ b/eng/performance/maui_scenarios_ios_innerloop.proj @@ -78,10 +78,10 @@ <_MacEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$PATH - - + + $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" - $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" - + $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" || exit $? $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id com.companyname.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --device-type simulator --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) export IOS_RID=$(iOSRid);$(Python) post.py output.log @@ -104,7 +100,7 @@ EnableCodeSigning=false). --> - $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" + $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" || exit $? $(Python) test.py iosinnerloop --csproj-path app/MauiiOSInnerLoop.csproj --edit-src "src/MainPage.xaml.cs;src/MainPage.xaml" --edit-dest "app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml" --bundle-id com.companyname.mauiiosinnerloop -f $(PERFLAB_Framework)-ios -c Debug --msbuild-args "$(_MSBuildArgs)" --device-type device --inner-loop-iterations $(InnerLoopIterations) --scenario-name "%(Identity)" $(ScenarioArgs) export IOS_RID=$(iOSRid);$(Python) post.py output.log diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 38c2db61a7f..63b6dba60ee 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -1340,6 +1340,15 @@ def get_work_item_command_for_artifact_dir(artifact_dir: str): scenario_arguments=scenario_arguments or None) if args.send_to_helix: + # Re-apply run_env_vars so they reach SendToHelix MSBuild evaluation. + # The env was snapshot/restored earlier (environ_copy), and while + # os.environ.update() preserves new keys, some shell wrappers may not + # inherit them reliably. Explicitly re-setting ensures properties like + # iOSRid (used in .proj ItemGroup conditions) are available. + if args.run_env_vars: + for key, value in args.run_env_vars.items(): + os.environ[key] = value + perf_send_to_helix(perf_send_to_helix_args) results_glob = os.path.join(helix_results_destination_dir, '**', '*perf-lab-report.json') From 5c092e4587a5f6b0f52cbd5bd834e94f8f9579eb Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Sat, 4 Apr 2026 18:54:52 +0200 Subject: [PATCH 26/95] Pass iOSRid as explicit MSBuild property to fix device job dispatch Env var inheritance through msbuild.sh/tools.sh is unreliable for iOSRid. Add ios_rid field to PerfSendToHelixArgs and pass it as /p:iOSRid= on the MSBuild command line so it reaches .proj evaluation deterministically. Also set it via set_environment_variables as belt-and-suspenders. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/run_performance_job.py | 1 + scripts/send_to_helix.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 63b6dba60ee..d12fe07fff6 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -1336,6 +1336,7 @@ def get_work_item_command_for_artifact_dir(artifact_dir: str): only_sanity_check=args.only_sanity_check, ios_strip_symbols=args.ios_strip_symbols, ios_llvm_build=args.ios_llvm_build, + ios_rid=args.run_env_vars.get("iOSRid"), fail_on_test_failure=fail_on_test_failure, scenario_arguments=scenario_arguments or None) diff --git a/scripts/send_to_helix.py b/scripts/send_to_helix.py index f3266623e3a..d88ff0b0986 100644 --- a/scripts/send_to_helix.py +++ b/scripts/send_to_helix.py @@ -71,6 +71,7 @@ class PerfSendToHelixArgs: linking_type: Optional[str] = None python: Optional[str] = None affinity: Optional[str] = None + ios_rid: Optional[str] = None ios_strip_symbols: Optional[bool] = None ios_llvm_build: Optional[bool] = None scenario_arguments: Optional[list[str]] = None @@ -111,6 +112,7 @@ def set_env_var(name: str, value: Union[str, bool, list[str], timedelta, int, No set_env_var("RuntimeFlavor", self.runtime_flavor) set_env_var("CodegenType", self.codegen_type) set_env_var("LinkingType", self.linking_type) + set_env_var("iOSRid", self.ios_rid) set_env_var("iOSStripSymbols", self.ios_strip_symbols) set_env_var("iOSLlvmBuild", self.ios_llvm_build) set_env_var("TargetCsproj", self.target_csproj) @@ -142,5 +144,11 @@ def perf_send_to_helix(args: PerfSendToHelixArgs): binlog_dest = os.path.join(args.performance_repo_dir, "artifacts", "log", args.build_config, "SendToHelix.binlog") send_params = [args.project_file, "/restore", "/t:Test", f"/bl:{binlog_dest}"] + # Pass iOSRid explicitly as an MSBuild property so it reaches .proj + # evaluation reliably. Env var inheritance through msbuild.sh/tools.sh + # is unreliable for this property. + if args.ios_rid: + send_params.append(f"/p:iOSRid={args.ios_rid}") + run_msbuild_command(send_params, warn_as_error=False) From edc2ccb86e049c59d64ca2260a4f5b16512fe898 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Sat, 4 Apr 2026 19:33:43 +0200 Subject: [PATCH 27/95] Rename simulator job identifiers for clarity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mono_InnerLoop → Mono_InnerLoop_Simulator CoreCLR_InnerLoop → CoreCLR_InnerLoop_Simulator Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/sdk-perf-jobs.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index ea82215fd02..e6b24018e19 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -577,7 +577,7 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS Inner Loop (Mono - Default) - Debug + # Maui iOS Inner Loop (Mono - Default) - Debug - Simulator - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml @@ -592,11 +592,11 @@ jobs: runtimeFlavor: mono codeGenType: Default buildConfig: Debug - additionalJobIdentifier: Mono_InnerLoop + additionalJobIdentifier: Mono_InnerLoop_Simulator ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # Maui iOS Inner Loop (CoreCLR - Default) - Debug + # Maui iOS Inner Loop (CoreCLR - Default) - Debug - Simulator - template: /eng/pipelines/templates/build-machine-matrix.yml parameters: jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml @@ -611,7 +611,7 @@ jobs: runtimeFlavor: coreclr codeGenType: Default buildConfig: Debug - additionalJobIdentifier: CoreCLR_InnerLoop + additionalJobIdentifier: CoreCLR_InnerLoop_Simulator ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} From d8b4ff17667a1c301489402ae597909a405780f8 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Sat, 4 Apr 2026 23:44:33 +0200 Subject: [PATCH 28/95] Switch ioshelper.py from simctl/devicectl to mlaunch for install/launch Replace simctl (simulator) and devicectl (device) install/launch commands with mlaunch to match the real Visual Studio F5 developer experience: - Simulator: --launchsim combines install + launch (install_app returns 0) - Device: --installdev for install, --launchdev for launch - Device cleanup: --uninstalldevbundleid replaces devicectl uninstall - Simulator cleanup: unchanged (simctl terminate + uninstall) - Added _resolve_mlaunch() to find mlaunch from iOS SDK packs Device detection (devicectl) and simulator management (simctl boot/ terminate/uninstall) remain unchanged. The install_app/measure_cold_startup API is preserved so runner.py requires no changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/ioshelper.py | 85 ++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 17 deletions(-) diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py index 858ff2861e6..8fa734976b3 100644 --- a/src/scenarios/shared/ioshelper.py +++ b/src/scenarios/shared/ioshelper.py @@ -14,16 +14,49 @@ class iOSHelper: """Unified helper for iOS simulator and physical device operations. Callers use the same API (setup_device, install_app, measure_cold_startup, - cleanup) regardless of device type. The helper dispatches to simctl - (simulator) or devicectl (physical) commands internally. + cleanup) regardless of device type. + + Install and launch use mlaunch (the same tool Visual Studio uses for F5) + to match the real developer inner-loop experience: + - Simulator: mlaunch --launchsim (combines install + launch) + - Device: mlaunch --installdev / --launchdev (separate steps) + Device detection still uses devicectl; simulator management uses simctl. """ + _mlaunch_path = None # resolved once, cached for the process + def __init__(self): self.bundle_id = None self.device_id = None self.app_bundle_path = None self.is_physical_device = False + # ── mlaunch Resolution ──────────────────────────────────────────── + + @staticmethod + def _resolve_mlaunch(): + """Resolve the mlaunch binary from the iOS SDK pack. + + Searches $DOTNET_ROOT/packs/Microsoft.iOS.Sdk.*/tools/bin/mlaunch, + falling back to ~/.dotnet if DOTNET_ROOT is unset. Caches the result. + """ + if iOSHelper._mlaunch_path is not None: + return iOSHelper._mlaunch_path + + dotnet_root = os.environ.get('DOTNET_ROOT', os.path.expanduser('~/.dotnet')) + pattern = os.path.join(dotnet_root, 'packs', 'Microsoft.iOS.Sdk.*', '*', 'tools', 'bin', 'mlaunch') + matches = sorted(glob.glob(pattern)) + if not matches: + raise FileNotFoundError( + f"mlaunch not found. Searched: {pattern}\n" + f"Ensure the iOS SDK workload is installed (dotnet workload install ios)." + ) + # Use the last match (highest version when sorted lexicographically) + mlaunch = matches[-1] + getLogger().info("Resolved mlaunch: %s", mlaunch) + iOSHelper._mlaunch_path = mlaunch + return mlaunch + # ── Device Detection ───────────────────────────────────────────── @staticmethod @@ -132,12 +165,19 @@ def setup_device(self, bundle_id, app_bundle_path, device_id='booted', is_physic # ── Unified Operations ─────────────────────────────────────────── def install_app(self, app_bundle_path): - """Install the app bundle and return wall-clock install time in ms.""" - if self.is_physical_device: - cmd = ['xcrun', 'devicectl', 'device', 'install', 'app', - '--device', self.device_id, app_bundle_path] - else: - cmd = ['xcrun', 'simctl', 'install', self.device_id, app_bundle_path] + """Install the app bundle and return wall-clock install time in ms. + + Device: mlaunch --installdev (separate from launch, matching F5). + Simulator: no-op — mlaunch --launchsim combines install + launch, + so the install cost is captured in measure_cold_startup() instead. + """ + if not self.is_physical_device: + getLogger().info("Simulator: skipping install (--launchsim handles it)") + return 0 + + mlaunch = self._resolve_mlaunch() + cmd = [mlaunch, '--installdev', app_bundle_path, + '--devname', self.device_id] start = time.time() RunCommand(cmd, verbose=True).run() @@ -148,17 +188,23 @@ def install_app(self, app_bundle_path): def measure_cold_startup(self, bundle_id): """Measure app cold startup time in ms (int). - Terminates any running instance, waits briefly, then launches. - For physical devices, uses --terminate-existing on the launch command - since devicectl's 'process terminate' requires --pid not --bundle-id. + Uses mlaunch to match the real F5 developer experience: + - Simulator: mlaunch --launchsim (installs + launches in one step) + - Device: mlaunch --launchdev (install was done separately) + + Terminates any running instance first. For simulator this uses + simctl terminate (mlaunch has no simulator terminate command). """ + mlaunch = self._resolve_mlaunch() + if self.is_physical_device: - cmd = ['xcrun', 'devicectl', 'device', 'process', 'launch', - '--terminate-existing', '--device', self.device_id, bundle_id] + cmd = [mlaunch, '--launchdev', self.app_bundle_path, + '--devname', self.device_id] else: self._run_quiet(['xcrun', 'simctl', 'terminate', self.device_id, bundle_id]) time.sleep(0.5) - cmd = ['xcrun', 'simctl', 'launch', self.device_id, bundle_id] + cmd = [mlaunch, '--launchsim', self.app_bundle_path, + '--device', f':v2:udid={self.device_id}'] start = time.time() RunCommand(cmd, verbose=True).run() @@ -167,12 +213,17 @@ def measure_cold_startup(self, bundle_id): return elapsed_ms def cleanup(self, skip_uninstall=False): - """Clean up the device session (simulator or physical).""" + """Clean up the device session (simulator or physical). + + Device uses mlaunch --uninstalldevbundleid. Simulator keeps simctl + (mlaunch has no simulator terminate/uninstall commands). + """ if skip_uninstall: return if self.is_physical_device: - self._run_quiet(['xcrun', 'devicectl', 'device', 'uninstall', 'app', - '--device', self.device_id, self.bundle_id]) + mlaunch = self._resolve_mlaunch() + self._run_quiet([mlaunch, '--uninstalldevbundleid', self.bundle_id, + '--devname', self.device_id]) else: self._run_quiet(['xcrun', 'simctl', 'terminate', self.device_id, self.bundle_id]) self._run_quiet(['xcrun', 'simctl', 'uninstall', self.device_id, self.bundle_id]) From c398150819ace620e7783d3865ab7fb1d5d92a61 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Sat, 4 Apr 2026 23:53:03 +0200 Subject: [PATCH 29/95] Use --installsim for granular simulator install timing Instead of making install_app() a no-op for simulator, use mlaunch --installsim to get a separate install measurement. measure_cold_startup() still uses --launchsim for launch timing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/ioshelper.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py index 8fa734976b3..7b5b2bb32cf 100644 --- a/src/scenarios/shared/ioshelper.py +++ b/src/scenarios/shared/ioshelper.py @@ -18,8 +18,8 @@ class iOSHelper: Install and launch use mlaunch (the same tool Visual Studio uses for F5) to match the real developer inner-loop experience: - - Simulator: mlaunch --launchsim (combines install + launch) - - Device: mlaunch --installdev / --launchdev (separate steps) + - Simulator: mlaunch --installsim / --launchsim + - Device: mlaunch --installdev / --launchdev Device detection still uses devicectl; simulator management uses simctl. """ @@ -167,17 +167,17 @@ def setup_device(self, bundle_id, app_bundle_path, device_id='booted', is_physic def install_app(self, app_bundle_path): """Install the app bundle and return wall-clock install time in ms. - Device: mlaunch --installdev (separate from launch, matching F5). - Simulator: no-op — mlaunch --launchsim combines install + launch, - so the install cost is captured in measure_cold_startup() instead. + Device: mlaunch --installdev + Simulator: mlaunch --installsim """ - if not self.is_physical_device: - getLogger().info("Simulator: skipping install (--launchsim handles it)") - return 0 - mlaunch = self._resolve_mlaunch() - cmd = [mlaunch, '--installdev', app_bundle_path, - '--devname', self.device_id] + + if self.is_physical_device: + cmd = [mlaunch, '--installdev', app_bundle_path, + '--devname', self.device_id] + else: + cmd = [mlaunch, '--installsim', app_bundle_path, + '--device', f':v2:udid={self.device_id}'] start = time.time() RunCommand(cmd, verbose=True).run() @@ -189,8 +189,8 @@ def measure_cold_startup(self, bundle_id): """Measure app cold startup time in ms (int). Uses mlaunch to match the real F5 developer experience: - - Simulator: mlaunch --launchsim (installs + launches in one step) - - Device: mlaunch --launchdev (install was done separately) + - Simulator: mlaunch --launchsim + - Device: mlaunch --launchdev Terminates any running instance first. For simulator this uses simctl terminate (mlaunch has no simulator terminate command). From 8e89a1f28285bea9ffd4c91e371bf753a3558221 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Sun, 5 Apr 2026 11:55:07 +0200 Subject: [PATCH 30/95] Re-select Xcode to match iOS SDK's required version after workload install After installing the maui-ios workload, read _RecommendedXcodeVersion from the SDK's Versions.props and switch to the matching Xcode_*.app if the currently active Xcode doesn't match. This handles the case where Helix agents have a newer Xcode than the SDK requires. Falls back gracefully to the already-selected Xcode if no matching installation is found. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiiosinnerloop/setup_helix.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py index cc9e0f9952b..69fe8104e60 100644 --- a/src/scenarios/mauiiosinnerloop/setup_helix.py +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -351,6 +351,91 @@ def install_workload(ctx): log("maui-ios workload installed successfully") +def _reselect_xcode_for_sdk(framework): + """Re-select Xcode to match the installed iOS SDK's required version. + + After workload install, the iOS SDK pack's Versions.props declares + _RecommendedXcodeVersion. If the currently active Xcode doesn't match, + look for a matching Xcode_*.app and switch to it. + """ + import glob + import re + + log_raw("=== XCODE SDK COMPATIBILITY CHECK ===", tee=True) + + # 1. Get the active Xcode version + result = run_cmd(["xcodebuild", "-version"], check=False) + if result.returncode != 0 or not result.stdout: + log("WARNING: Could not determine active Xcode version — skipping", tee=True) + return + m = re.search(r'Xcode\s+(\d+\.\d+)', result.stdout) + if not m: + log("WARNING: Could not parse Xcode version — skipping", tee=True) + return + active_xcode = m.group(1) + + # 2. Find the SDK's _RecommendedXcodeVersion + dotnet_root = os.environ.get("DOTNET_ROOT", "") + if not dotnet_root: + log("WARNING: DOTNET_ROOT not set — skipping Xcode SDK check", tee=True) + return + tfm_prefix = framework.split('-')[0] if '-' in framework else framework + search = os.path.join( + dotnet_root, 'packs', f'Microsoft.iOS.Sdk.{tfm_prefix}_*', + '*', 'targets', 'Microsoft.iOS.Sdk.Versions.props' + ) + props_files = sorted(glob.glob(search)) + if not props_files: + log(f"WARNING: No iOS SDK Versions.props found ({search}) — skipping", tee=True) + return + with open(props_files[-1], 'r') as f: + content = f.read() + rec = re.search(r'<_RecommendedXcodeVersion>([^<]+)', content) + if not rec: + log("WARNING: _RecommendedXcodeVersion not found in Versions.props", tee=True) + return + required_xcode = rec.group(1) + + # 3. Compare major.minor + active_mm = '.'.join(active_xcode.split('.')[:2]) + required_mm = '.'.join(required_xcode.split('.')[:2]) + if active_mm == required_mm: + log(f"Xcode {active_xcode} matches SDK requirement ({required_xcode})", tee=True) + return + + log(f"Xcode mismatch: active={active_xcode}, SDK requires={required_xcode}. " + f"Looking for Xcode_{required_mm}*.app ...", tee=True) + + # 4. Find a matching Xcode installation + find_result = run_cmd( + ["find", "/Applications", "-maxdepth", "1", "-type", "d", + "-name", f"Xcode_{required_mm}*.app"], + check=False, + ) + candidates = [l.strip() for l in (find_result.stdout or "").splitlines() if l.strip()] + if not candidates: + log(f"WARNING: No Xcode_{required_mm}*.app found in /Applications. " + f"Continuing with Xcode {active_xcode} — build may fail.", tee=True) + return + + # Pick the best match (sort by version, take highest patch) + def _ver_key(path): + ver = path.rsplit("_", 1)[-1].replace(".app", "") + try: + return tuple(int(x) for x in ver.split('.')) + except ValueError: + return (0,) + candidates.sort(key=_ver_key) + target = candidates[-1] + + log(f"Switching Xcode to: {target}", tee=True) + result = run_cmd(["sudo", "xcode-select", "-s", target], check=False) + if result.returncode != 0: + log(f"WARNING: xcode-select -s failed — setting DEVELOPER_DIR instead", tee=True) + os.environ["DEVELOPER_DIR"] = os.path.join(target, "Contents", "Developer") + run_cmd(["xcodebuild", "-version"], check=False) + + def restore_packages(ctx): """Restore NuGet packages for the app project. @@ -570,6 +655,10 @@ def main(): # Must happen BEFORE restore because restore needs workload packs install_workload(ctx) + # Step 5b: Re-select Xcode to match the installed iOS SDK's requirement. + # Step 2 picks the highest Xcode, but the SDK may need an older one. + _reselect_xcode_for_sdk(framework) + # Step 6: Restore NuGet packages restore_packages(ctx) From 5d80e1f41bda9bfc580aef466ae706ff89bb22f6 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Sun, 5 Apr 2026 17:48:22 +0200 Subject: [PATCH 31/95] Resolve booted simulator UDID for mlaunch and remove ValidateXcodeVersion bypass mlaunch requires a real simulator UDID, not the simctl shortcut 'booted'. Add _resolve_booted_simulator_udid() that queries simctl to find the actual UDID of the booted simulator before passing it to mlaunch. Also remove ValidateXcodeVersion=false from the .proj file since setup_helix.py now re-selects the correct Xcode version after workload install. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../maui_scenarios_ios_innerloop.proj | 5 +-- src/scenarios/shared/ioshelper.py | 38 ++++++++++++++++++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/eng/performance/maui_scenarios_ios_innerloop.proj b/eng/performance/maui_scenarios_ios_innerloop.proj index 44f9003ce4d..fa76b20d2b0 100644 --- a/eng/performance/maui_scenarios_ios_innerloop.proj +++ b/eng/performance/maui_scenarios_ios_innerloop.proj @@ -17,10 +17,7 @@ queue runs on Intel x64 machines (e.g. DNCENGMAC045). --> iossimulator-x64 <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(iOSRid) - - <_MSBuildArgs>$(_MSBuildArgs) /p:ValidateXcodeVersion=false + diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py index 7b5b2bb32cf..af77d987b5e 100644 --- a/src/scenarios/shared/ioshelper.py +++ b/src/scenarios/shared/ioshelper.py @@ -131,6 +131,32 @@ def _detect_via_devicectl_text(): except Exception: return None + # ── Simulator UDID Resolution ──────────────────────────────────── + + @staticmethod + def _resolve_booted_simulator_udid(): + """Return the UDID of the first booted simulator. + + mlaunch requires a real UDID (--device :v2:udid=); it does + not understand simctl's "booted" shortcut. This method queries + ``simctl list devices booted`` to find the actual UDID. + """ + try: + result = subprocess.run( + ['xcrun', 'simctl', 'list', 'devices', 'booted', '-j'], + capture_output=True, text=True, timeout=15 + ) + if result.returncode != 0: + return None + data = json.loads(result.stdout) + for runtime_devices in data.get('devices', {}).values(): + for dev in runtime_devices: + if dev.get('state', '').lower() == 'booted': + return dev['udid'] + return None + except Exception: + return None + # ── Unified Device Setup ───────────────────────────────────────── def setup_device(self, bundle_id, app_bundle_path, device_id='booted', is_physical=False): @@ -160,7 +186,17 @@ def setup_device(self, bundle_id, app_bundle_path, device_id='booted', is_physic if result.returncode != 0 and 'already booted' not in (result.stderr or '').lower(): raise subprocess.CalledProcessError(result.returncode, result.args, result.stdout, result.stderr) - self._run_quiet(['xcrun', 'simctl', 'uninstall', device_id, bundle_id]) + # Resolve the actual UDID — mlaunch needs a real UDID, not "booted" + if self.device_id == 'booted': + resolved = self._resolve_booted_simulator_udid() + if not resolved: + raise RuntimeError( + "Could not resolve booted simulator UDID. " + "Ensure a simulator is booted (setup_helix.py should have done this).") + getLogger().info("Resolved booted simulator UDID: %s", resolved) + self.device_id = resolved + + self._run_quiet(['xcrun', 'simctl', 'uninstall', self.device_id, bundle_id]) # ── Unified Operations ─────────────────────────────────────────── From 6756d0393c779dbfbf2a11a9b2e84e330cd0e239 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Sun, 5 Apr 2026 20:27:55 +0200 Subject: [PATCH 32/95] Run --launchsim as background process to prevent timeout mlaunch --launchsim blocks until the app exits, but MAUI GUI apps never exit on their own. Run it via subprocess.Popen and poll for the app to appear in the simulator's process list via simctl spawn launchctl list. Terminate the mlaunch process once startup is detected. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/shared/ioshelper.py | 63 +++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py index af77d987b5e..586651a3f68 100644 --- a/src/scenarios/shared/ioshelper.py +++ b/src/scenarios/shared/ioshelper.py @@ -225,8 +225,10 @@ def measure_cold_startup(self, bundle_id): """Measure app cold startup time in ms (int). Uses mlaunch to match the real F5 developer experience: - - Simulator: mlaunch --launchsim - - Device: mlaunch --launchdev + - Simulator: mlaunch --launchsim (run as background process since + it blocks until the app exits; we detect launch via simctl and + then terminate the process) + - Device: mlaunch --launchdev (returns immediately with PID) Terminates any running instance first. For simulator this uses simctl terminate (mlaunch has no simulator terminate command). @@ -236,17 +238,56 @@ def measure_cold_startup(self, bundle_id): if self.is_physical_device: cmd = [mlaunch, '--launchdev', self.app_bundle_path, '--devname', self.device_id] - else: - self._run_quiet(['xcrun', 'simctl', 'terminate', self.device_id, bundle_id]) - time.sleep(0.5) - cmd = [mlaunch, '--launchsim', self.app_bundle_path, - '--device', f':v2:udid={self.device_id}'] + start = time.time() + RunCommand(cmd, verbose=True).run() + elapsed_ms = int((time.time() - start) * 1000) + getLogger().info("Cold startup: %d ms", elapsed_ms) + return elapsed_ms + + # Simulator: --launchsim blocks until the app exits, so run it + # in a subprocess and poll for the app to appear in the process list. + self._run_quiet(['xcrun', 'simctl', 'terminate', self.device_id, bundle_id]) + time.sleep(0.5) + cmd = [mlaunch, '--launchsim', self.app_bundle_path, + '--device', f':v2:udid={self.device_id}'] + getLogger().info("$ %s", ' '.join(cmd)) start = time.time() - RunCommand(cmd, verbose=True).run() - elapsed_ms = int((time.time() - start) * 1000) - getLogger().info("Cold startup: %d ms", elapsed_ms) - return elapsed_ms + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + # Poll until the app is running in the simulator (max 120s) + app_name = os.path.splitext(os.path.basename(self.app_bundle_path))[0] + timeout = 120 + poll_interval = 0.5 + elapsed = 0.0 + while elapsed < timeout: + # Check if mlaunch exited with an error + ret = proc.poll() + if ret is not None and ret != 0: + stdout = proc.stdout.read().decode() if proc.stdout else '' + stderr = proc.stderr.read().decode() if proc.stderr else '' + raise subprocess.CalledProcessError(ret, cmd, stdout, stderr) + + # Check if the app process is running via simctl + check = subprocess.run( + ['xcrun', 'simctl', 'spawn', self.device_id, 'launchctl', 'list'], + capture_output=True, text=True, timeout=10 + ) + if bundle_id in (check.stdout or ''): + break + time.sleep(poll_interval) + elapsed = time.time() - start + + elapsed_ms = int((time.time() - start) * 1000) + getLogger().info("Cold startup: %d ms", elapsed_ms) + return elapsed_ms + finally: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() def cleanup(self, skip_uninstall=False): """Clean up the device session (simulator or physical). From 803ec217ebac3874ec52a216f21d3ac7da3b22fb Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Mon, 6 Apr 2026 17:23:59 +0200 Subject: [PATCH 33/95] Add post-build code signing for physical device deployment Mirror the signing flow from maui_scenarios_ios.proj: disable automatic code signing during dotnet build (EnableCodeSigning=false), then copy embedded.mobileprovision into the .app bundle and run the Helix-provided 'sign' tool before mlaunch --installdev. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../maui_scenarios_ios_innerloop.proj | 18 +++++----- src/scenarios/shared/ioshelper.py | 35 +++++++++++++++++++ src/scenarios/shared/runner.py | 4 +++ 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/eng/performance/maui_scenarios_ios_innerloop.proj b/eng/performance/maui_scenarios_ios_innerloop.proj index fa76b20d2b0..b18a7135759 100644 --- a/eng/performance/maui_scenarios_ios_innerloop.proj +++ b/eng/performance/maui_scenarios_ios_innerloop.proj @@ -17,7 +17,11 @@ queue runs on Intel x64 machines (e.g. DNCENGMAC045). --> iossimulator-x64 <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(iOSRid) - + + <_MSBuildArgs Condition="'$(iOSRid)' == 'ios-arm64'">$(_MSBuildArgs) /p:EnableCodeSigning=false @@ -88,13 +92,11 @@ + iPhones (Mac.iPhone.17.Perf queue). + Code signing is disabled during 'dotnet build' (EnableCodeSigning=false) + and handled post-build: ioshelper copies embedded.mobileprovision into + the .app and runs the Helix-provided 'sign' tool — same flow as + maui_scenarios_ios.proj device startup scenarios. --> $(_MacEnvVars);export IOS_RID=$(iOSRid);$(Python) setup_helix.py $(PERFLAB_Framework)-ios "$(_MSBuildArgs)" || exit $? diff --git a/src/scenarios/shared/ioshelper.py b/src/scenarios/shared/ioshelper.py index 586651a3f68..3b242f8dd39 100644 --- a/src/scenarios/shared/ioshelper.py +++ b/src/scenarios/shared/ioshelper.py @@ -198,6 +198,41 @@ def setup_device(self, bundle_id, app_bundle_path, device_id='booted', is_physic self._run_quiet(['xcrun', 'simctl', 'uninstall', self.device_id, bundle_id]) + # ── Device Code Signing ────────────────────────────────────────── + + def sign_app_for_device(self, app_bundle_path): + """Sign the .app bundle for physical device deployment. + + Mirrors the signing flow from maui_scenarios_ios.proj device startup: + 1. Copy embedded.mobileprovision into the .app bundle + 2. Run the Helix-provided 'sign' tool + + Both 'embedded.mobileprovision' and 'sign' are pre-installed on the + Mac.iPhone.17.Perf Helix machines. The build must use + EnableCodeSigning=false so MSBuild skips automatic signing. + + No-op for simulator builds. + """ + if not self.is_physical_device: + return + + import shutil + provision_src = 'embedded.mobileprovision' + provision_dst = os.path.join(app_bundle_path, 'embedded.mobileprovision') + + if not os.path.exists(provision_src): + getLogger().warning( + "embedded.mobileprovision not found in working directory. " + "Device signing may fail if the Helix machine doesn't have it.") + else: + shutil.copy2(provision_src, provision_dst) + getLogger().info("Copied provisioning profile into %s", app_bundle_path) + + app_name = os.path.basename(app_bundle_path) + app_dir = os.path.dirname(os.path.abspath(app_bundle_path)) + getLogger().info("Signing %s for device deployment", app_name) + RunCommand(['sign', app_name], verbose=True).run(working_directory=app_dir) + # ── Unified Operations ─────────────────────────────────────────── def install_app(self, app_bundle_path): diff --git a/src/scenarios/shared/runner.py b/src/scenarios/shared/runner.py index 5057f9fb04f..7fe97fe6984 100644 --- a/src/scenarios/shared/runner.py +++ b/src/scenarios/shared/runner.py @@ -1101,6 +1101,7 @@ def run(self): try: app_bundle = iosHelper.find_app_bundle(project_dir, exename, self.configuration) iosHelper.setup_device(self.bundleid, app_bundle, self.deviceid, is_physical=is_physical) + iosHelper.sign_app_for_device(app_bundle) first_install_ms = iosHelper.install_app(app_bundle) first_startup_ms = iosHelper.measure_cold_startup(self.bundleid) getLogger().info("First deploy: install=%.1f ms, startup=%d ms", first_install_ms, first_startup_ms) @@ -1161,6 +1162,9 @@ def run(self): getLogger().error("Incremental build %d failed. Binlog: %s", iteration, iter_binlog) raise + # Sign (device only — no-op for simulator) + iosHelper.sign_app_for_device(app_bundle) + # Install + startup install_ms = iosHelper.install_app(app_bundle) startup_ms = iosHelper.measure_cold_startup(self.bundleid) From 2bcbb64baf10cfaba69b726997ae15c206bebdb1 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 7 Apr 2026 13:04:35 +0200 Subject: [PATCH 34/95] =?UTF-8?q?Make=20Xcode=20handling=20diagnostic-only?= =?UTF-8?q?=20on=20Helix=20=E2=80=94=20match=20Device=20Startup=20scenario?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The working 'Device Startup - iOS' scenario (maui_scenarios_ios.proj) does NO Xcode manipulation on the Helix machine — it trusts the system default that's already configured. Our setup_helix.py was aggressively manipulating Xcode (selecting highest version, validating min_major=26, re-selecting after workload install), which could switch AWAY from a working Xcode. Changes: - Replace select_xcode() body with diagnostic-only logging (xcode-select -p and xcodebuild -version) — no more sudo xcode-select -s on Helix - Remove _validate_xcode_version() — the hard-coded min_major=26 check rejects machines that the working scenario accepts just fine - Remove _reselect_xcode_for_sdk() — post-workload Xcode switching based on _RecommendedXcodeVersion may switch away from a working config - Remove corresponding calls in main() (Steps 2b and 5b) - Update module docstring to document the Xcode strategy and explain it matches the Device Startup scenario approach The build-agent-side Xcode selection in PreparePayloadWorkItem (.proj file) is unchanged — that runs on the build agent, not Helix, and matches the same pattern used by the working scenario. --- src/scenarios/mauiiosinnerloop/setup_helix.py | 206 ++---------------- 1 file changed, 23 insertions(+), 183 deletions(-) diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py index 69fe8104e60..a2cf26d0ab0 100644 --- a/src/scenarios/mauiiosinnerloop/setup_helix.py +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -4,12 +4,19 @@ Runs on the Helix machine BEFORE test.py. Bootstraps the macOS environment for iOS builds: 1. Configure DOTNET_ROOT and PATH from the correlation payload SDK. - 2. Select the correct Xcode version (highest versioned Xcode_*.app). + 2. Log the system Xcode version (diagnostic only — no switching). 3. Validate iOS simulator runtime availability. 4. Boot the target iOS simulator device. 5. Install the maui-ios workload. 6. Restore NuGet packages for the app project. 7. Disable Spotlight indexing on the workitem directory. + +Xcode selection strategy: this script trusts the system-default Xcode that +is already configured on the Helix machine. The build agent's +PreparePayloadWorkItem (in maui_scenarios_ios_innerloop.proj) selects the +highest versioned Xcode_*.app — that is the only place Xcode switching +happens. This matches the working "Device Startup - iOS" scenario +(maui_scenarios_ios.proj), which does no Xcode manipulation on Helix. """ import os @@ -100,98 +107,23 @@ def setup_dotnet(correlation_payload): def select_xcode(): - """Select the highest versioned Xcode_*.app installation. + """Log the current system Xcode for diagnostics (no switching). - Follows the same pattern as maui_scenarios_ios.proj PreparePayloadWorkItem: - find /Applications -maxdepth 1 -type d -name 'Xcode_*.app' | sort ... | tail -1 + The working "Device Startup - iOS" scenario (maui_scenarios_ios.proj) + does NO Xcode manipulation on the Helix machine — it trusts the system + default. We follow the same approach: Xcode selection only happens on + the build agent via PreparePayloadWorkItem in the .proj file. - This avoids runner-image symlink aliases that don't work with the iOS SDK. - If XCODE_PATH env var is already set, uses that instead of auto-detecting. + This function only logs what's already configured so we have diagnostic + info if builds fail. """ - log_raw("=== XCODE SELECTION ===", tee=True) - - xcode_path = os.environ.get("XCODE_PATH", "") - if not xcode_path: - # Auto-detect: find highest versioned Xcode_*.app - result = run_cmd( - ["find", "/Applications", "-maxdepth", "1", "-type", "d", - "-name", "Xcode_*.app"], - check=False, - ) - candidates = [line.strip() for line in (result.stdout or "").splitlines() - if line.strip()] - if not candidates: - log("WARNING: No Xcode_*.app found in /Applications. " - "Falling back to system default Xcode.", tee=True) - run_cmd(["xcode-select", "-p"], check=False) - run_cmd(["xcodebuild", "-version"], check=False) - return - - # Sort by version number (Xcode_16.2.app → key on "16.2"). - # Use tuple-of-ints to get correct version ordering (e.g., 16.10 > 16.2). - def _xcode_version_key(path): - ver = path.rsplit("_", 1)[-1].replace(".app", "") - try: - return tuple(int(x) for x in ver.split('.')) - except ValueError: - return (0,) - candidates.sort(key=_xcode_version_key) - xcode_path = candidates[-1] - - log(f"Selected Xcode: {xcode_path}", tee=True) - - if not os.path.isdir(os.path.join(xcode_path, "Contents", "Developer")): - log(f"WARNING: {xcode_path} does not look like a valid Xcode installation " - "(missing Contents/Developer)", tee=True) - return - - # Use sudo xcode-select -s to switch the system Xcode (same as .proj pattern) - result = run_cmd( - ["sudo", "xcode-select", "-s", xcode_path], - check=False, - ) - if result.returncode != 0: - log(f"WARNING: xcode-select -s failed (exit {result.returncode}). " - "Falling back to DEVELOPER_DIR.", tee=True) - os.environ["DEVELOPER_DIR"] = os.path.join(xcode_path, "Contents", "Developer") - else: - log(f"Xcode switched to: {xcode_path}") - - # Log the active Xcode version for diagnostics + log_raw("=== XCODE DIAGNOSTICS ===", tee=True) + log("Trusting system-default Xcode (matches Device Startup scenario approach).", + tee=True) + run_cmd(["xcode-select", "-p"], check=False) run_cmd(["xcodebuild", "-version"], check=False) -def _validate_xcode_version(min_major=26): - """Fail fast if the active Xcode is too old for iOS inner loop builds. - - Parses the version from 'xcodebuild -version' (e.g. "Xcode 15.0" → 15) - and exits with a clear diagnostic if the major version is below *min_major*. - """ - result = run_cmd(["xcodebuild", "-version"], check=False) - if result.returncode != 0 or not result.stdout: - log("WARNING: Could not determine Xcode version — skipping check.", tee=True) - return - - import re - m = re.search(r"Xcode\s+(\d+)\.(\d+)", result.stdout) - if not m: - log("WARNING: Could not parse Xcode version from output — skipping check.", - tee=True) - return - - major, minor = int(m.group(1)), int(m.group(2)) - if major < min_major: - log(f"ERROR: Xcode version {major}.{minor} is too old. " - f"iOS inner loop requires Xcode {min_major}.0 or later. " - "This Helix machine cannot run iOS inner loop measurements.", - tee=True) - _dump_log() - sys.exit(1) - - log(f"Xcode version {major}.{minor} meets minimum requirement " - f"(>= {min_major}.0)", tee=True) - - def validate_simulator_runtimes(): """Check that iOS simulator runtimes are available on this machine.""" log_raw("=== SIMULATOR RUNTIME VALIDATION ===", tee=True) @@ -351,91 +283,6 @@ def install_workload(ctx): log("maui-ios workload installed successfully") -def _reselect_xcode_for_sdk(framework): - """Re-select Xcode to match the installed iOS SDK's required version. - - After workload install, the iOS SDK pack's Versions.props declares - _RecommendedXcodeVersion. If the currently active Xcode doesn't match, - look for a matching Xcode_*.app and switch to it. - """ - import glob - import re - - log_raw("=== XCODE SDK COMPATIBILITY CHECK ===", tee=True) - - # 1. Get the active Xcode version - result = run_cmd(["xcodebuild", "-version"], check=False) - if result.returncode != 0 or not result.stdout: - log("WARNING: Could not determine active Xcode version — skipping", tee=True) - return - m = re.search(r'Xcode\s+(\d+\.\d+)', result.stdout) - if not m: - log("WARNING: Could not parse Xcode version — skipping", tee=True) - return - active_xcode = m.group(1) - - # 2. Find the SDK's _RecommendedXcodeVersion - dotnet_root = os.environ.get("DOTNET_ROOT", "") - if not dotnet_root: - log("WARNING: DOTNET_ROOT not set — skipping Xcode SDK check", tee=True) - return - tfm_prefix = framework.split('-')[0] if '-' in framework else framework - search = os.path.join( - dotnet_root, 'packs', f'Microsoft.iOS.Sdk.{tfm_prefix}_*', - '*', 'targets', 'Microsoft.iOS.Sdk.Versions.props' - ) - props_files = sorted(glob.glob(search)) - if not props_files: - log(f"WARNING: No iOS SDK Versions.props found ({search}) — skipping", tee=True) - return - with open(props_files[-1], 'r') as f: - content = f.read() - rec = re.search(r'<_RecommendedXcodeVersion>([^<]+)', content) - if not rec: - log("WARNING: _RecommendedXcodeVersion not found in Versions.props", tee=True) - return - required_xcode = rec.group(1) - - # 3. Compare major.minor - active_mm = '.'.join(active_xcode.split('.')[:2]) - required_mm = '.'.join(required_xcode.split('.')[:2]) - if active_mm == required_mm: - log(f"Xcode {active_xcode} matches SDK requirement ({required_xcode})", tee=True) - return - - log(f"Xcode mismatch: active={active_xcode}, SDK requires={required_xcode}. " - f"Looking for Xcode_{required_mm}*.app ...", tee=True) - - # 4. Find a matching Xcode installation - find_result = run_cmd( - ["find", "/Applications", "-maxdepth", "1", "-type", "d", - "-name", f"Xcode_{required_mm}*.app"], - check=False, - ) - candidates = [l.strip() for l in (find_result.stdout or "").splitlines() if l.strip()] - if not candidates: - log(f"WARNING: No Xcode_{required_mm}*.app found in /Applications. " - f"Continuing with Xcode {active_xcode} — build may fail.", tee=True) - return - - # Pick the best match (sort by version, take highest patch) - def _ver_key(path): - ver = path.rsplit("_", 1)[-1].replace(".app", "") - try: - return tuple(int(x) for x in ver.split('.')) - except ValueError: - return (0,) - candidates.sort(key=_ver_key) - target = candidates[-1] - - log(f"Switching Xcode to: {target}", tee=True) - result = run_cmd(["sudo", "xcode-select", "-s", target], check=False) - if result.returncode != 0: - log(f"WARNING: xcode-select -s failed — setting DEVELOPER_DIR instead", tee=True) - os.environ["DEVELOPER_DIR"] = os.path.join(target, "Contents", "Developer") - run_cmd(["xcodebuild", "-version"], check=False) - - def restore_packages(ctx): """Restore NuGet packages for the app project. @@ -623,15 +470,12 @@ def main(): ctx["dotnet_exe"] = setup_dotnet(correlation_payload) print_diagnostics() - # Step 2: Select the correct Xcode version + # Step 2: Log the system Xcode for diagnostics (no switching). + # Xcode selection happens on the build agent (PreparePayloadWorkItem in + # the .proj file). On Helix we trust the system default — same approach + # as the working "Device Startup - iOS" scenario. select_xcode() - # Step 2b: Validate minimum Xcode version for iOS inner loop builds. - # Machines with Xcode < 26 cannot build .NET MAUI iOS apps targeting the - # iOS 26+ SDK. Fail fast with a clear message instead of waiting 10+ min - # for ILLink to crash with MT0180. - _validate_xcode_version() - # Step 3 & 4: Device-type-specific setup if is_physical_device: # Detect and validate the connected physical device @@ -655,10 +499,6 @@ def main(): # Must happen BEFORE restore because restore needs workload packs install_workload(ctx) - # Step 5b: Re-select Xcode to match the installed iOS SDK's requirement. - # Step 2 picks the highest Xcode, but the SDK may need an older one. - _reselect_xcode_for_sdk(framework) - # Step 6: Restore NuGet packages restore_packages(ctx) From c2a65e3d6e78601b80613bf98acee24a5dd1c442 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 7 Apr 2026 14:26:58 +0200 Subject: [PATCH 35/95] Add --skip-manifest-update to workload install retry on Helix The retry path in setup_helix.py was missing --skip-manifest-update, causing the SDK to pull a newer manifest that references packs not yet published to all NuGet feeds (microsoft.ios.sdk.net10.0_26.4 v26.4.10245). This matches the build agent's PreCommands.install_workload() default. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/scenarios/mauiiosinnerloop/setup_helix.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/scenarios/mauiiosinnerloop/setup_helix.py b/src/scenarios/mauiiosinnerloop/setup_helix.py index a2cf26d0ab0..cb5aef15996 100644 --- a/src/scenarios/mauiiosinnerloop/setup_helix.py +++ b/src/scenarios/mauiiosinnerloop/setup_helix.py @@ -268,6 +268,11 @@ def install_workload(ctx): retry_args = [ ctx["dotnet_exe"], "workload", "install", "maui-ios", + # --skip-manifest-update prevents the SDK from pulling a newer + # manifest that may reference packs not yet published to all feeds. + # This matches PreCommands.install_workload() default behavior on + # the build agent (precommands.py). + "--skip-manifest-update", ] if os.path.isfile(nuget_config): retry_args.extend(["--configfile", nuget_config]) From 4253fcdb6cef38c95cda3e0cdae64327f695b395 Mon Sep 17 00:00:00 2001 From: David Nguyen <87228593+davidnguyen-tech@users.noreply.github.com> Date: Tue, 7 Apr 2026 17:28:35 +0200 Subject: [PATCH 36/95] Resolve sign tool path for Helix device deployment The HelixWorkItem runner sets a minimal PATH that excludes /usr/local/bin, where the 'sign' tool lives on Mac.iPhone.17.Perf machines. The XHarness runner (used by Device Startup scenarios) has a broader PATH, which is why sign works there but fails in our HelixWorkItem with FileNotFoundError. Two-layer fix: 1. Add /usr/local/bin to the PATH export in _MacEnvVars (proj file) so the tool is found by default in the shell environment. 2. Make sign_app_for_device() resilient: resolve the sign binary via shutil.which, then fall back to known locations and login shell discovery, with a clear error message if all lookups fail. --- .../maui_scenarios_ios_innerloop.proj | 5 +++- src/scenarios/shared/ioshelper.py | 29 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/eng/performance/maui_scenarios_ios_innerloop.proj b/eng/performance/maui_scenarios_ios_innerloop.proj index b18a7135759..3c9aef6f499 100644 --- a/eng/performance/maui_scenarios_ios_innerloop.proj +++ b/eng/performance/maui_scenarios_ios_innerloop.proj @@ -76,7 +76,10 @@ os.environ in a Python script (setup_helix.py) do not persist to subsequent commands (test.py, post.py) in the Helix shell session. --> - <_MacEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$PATH + + <_MacEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:/usr/local/bin:$PATH - - <_MacEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:/usr/local/bin:$PATH + <_MacEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$PATH