From da0ee89387a8c9b96a242029e3e72e9e55fe864f Mon Sep 17 00:00:00 2001 From: Matthew Leibowitz Date: Tue, 27 Feb 2024 22:21:02 +0200 Subject: [PATCH] Migrate to 1ES PT --- scripts/azure-pipelines.yml | 69 ++- scripts/azure-templates-bootstrapper.yml | 193 ++++---- scripts/azure-templates-linux-matrix.yml | 8 +- scripts/azure-templates-merger.yml | 44 ++ scripts/azure-templates-stages.yml | 577 +++++++++++------------ scripts/azure-templates-wasm-matrix.yml | 15 +- 6 files changed, 478 insertions(+), 428 deletions(-) create mode 100644 scripts/azure-templates-merger.yml diff --git a/scripts/azure-pipelines.yml b/scripts/azure-pipelines.yml index 0575203ec3..8cafe9f22e 100644 --- a/scripts/azure-pipelines.yml +++ b/scripts/azure-pipelines.yml @@ -10,39 +10,48 @@ pr: parameters: - name: buildExternals - displayName: 'The specific native artifacts to use for this build.' + displayName: 'The Build ID containing the specific native artifacts to use:' type: string default: 'latest' - - name: VM_IMAGE_HOST + - name: buildAgentHost + displayName: 'The generic host build agent configuration:' type: object default: pool: - name: Azure Pipelines - vmImage: ubuntu-20.04 - - name: VM_IMAGE_WINDOWS + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + os: windows + - name: buildAgentWindows + displayName: 'The Windows build agent configuration:' type: object default: pool: - name: Azure Pipelines - vmImage: windows-2022 - - name: VM_IMAGE_MAC + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + os: windows + - name: buildAgentMac + displayName: 'The macOS build agent configuration:' type: object default: pool: name: Azure Pipelines vmImage: macos-13 - - name: VM_IMAGE_LINUX + os: macos + - name: buildAgentLinux + displayName: 'The Linus build agent configuration:' type: object default: pool: - name: Azure Pipelines - vmImage: ubuntu-20.04 + name: AzurePipelines-EO + image: 1ESPT-Ubuntu20.04 + os: linux - name: runCompliance + displayName: 'Run post-build compliance tasks (such as API Scan)' type: boolean default: false variables: - - template: azure-pipelines-variables.yml + - template: /scripts/azure-pipelines-variables.yml@self resources: repositories: @@ -51,17 +60,27 @@ resources: name: xamarin/yaml-templates endpoint: xamarin ref: refs/heads/main + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release -stages: - - template: azure-templates-stages.yml - parameters: - buildPipelineType: 'build' - buildExternals: ${{ parameters.buildExternals }} - runCompliance: ${{ parameters.runCompliance }} - VM_IMAGE_HOST: ${{ parameters.VM_IMAGE_HOST }} - VM_IMAGE_WINDOWS: ${{ parameters.VM_IMAGE_WINDOWS }} - VM_IMAGE_WINDOWS_NATIVE: ${{ parameters.VM_IMAGE_WINDOWS }} - VM_IMAGE_MAC: ${{ parameters.VM_IMAGE_MAC }} - VM_IMAGE_MAC_NATIVE: ${{ parameters.VM_IMAGE_MAC }} - VM_IMAGE_LINUX: ${{ parameters.VM_IMAGE_LINUX }} - VM_IMAGE_LINUX_NATIVE: ${{ parameters.VM_IMAGE_LINUX }} +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: ${{ parameters.buildAgentHost.pool }} + customBuildTags: + - ES365AIMigrationTooling + stages: + - template: /scripts/azure-templates-stages.yml@self + parameters: + buildPipelineType: 'build' + buildExternals: ${{ parameters.buildExternals }} + runCompliance: ${{ parameters.runCompliance }} + buildAgentHost: ${{ parameters.buildAgentHost }} + buildAgentWindows: ${{ parameters.buildAgentWindows }} + buildAgentWindowsNative: ${{ parameters.buildAgentWindows }} + buildAgentMac: ${{ parameters.buildAgentMac }} + buildAgentMacNative: ${{ parameters.buildAgentMac }} + buildAgentLinux: ${{ parameters.buildAgentLinux }} + buildAgentLinuxNative: ${{ parameters.buildAgentLinux }} \ No newline at end of file diff --git a/scripts/azure-templates-bootstrapper.yml b/scripts/azure-templates-bootstrapper.yml index 8e43dce46a..45659e0ab7 100644 --- a/scripts/azure-templates-bootstrapper.yml +++ b/scripts/azure-templates-bootstrapper.yml @@ -1,7 +1,7 @@ parameters: name: '' # in the form type_platform_host displayName: '' # the human name - vmImage: '' # the VM image + buildAgent: '' # the configuration for the build agent packages: '' # any additional packages target: '' # the bootstrapper target dependsOn: [] # the dependiencies @@ -31,25 +31,54 @@ parameters: installEmsdk: false # whether or not to install the Emscripten SDK installNinja: false # whether or not to install the ninja build system artifactName: '' # the name of the artifact to merge this run into + additionalArtifacts: [] # the additional artifacts to publish tools: [] # any additional .net global tools + skipInstall: false # whether or not to install any tools + skipSteps: false # whether or not to run any steps jobs: - job: ${{ parameters.name }} displayName: ${{ parameters.displayName }} timeoutInMinutes: 180 - pool: ${{ parameters.vmImage.pool }} + pool: ${{ parameters.buildAgent.pool }} dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - ${{ if ne(parameters.vmImage.variables, '') }}: - ${{ parameters.vmImage.variables }} + ${{ if ne(parameters.buildAgent.variables, '') }}: + ${{ parameters.buildAgent.variables }} ${{ if ne(length(parameters.variables), 0) }}: ${{ parameters.variables }} + templateContext: + sdl: + spotBugs: + enabled: false + binskim: + break: false + outputParentDirectory: 'output' + outputs: + - output: pipelineArtifact + displayName: 'Publish the ${{ parameters.name }} artifacts' + condition: or(${{ parameters.shouldPublish }}, failed()) + artifactName: ${{ parameters.name }} + targetPath: 'output' + - ${{ if ne(parameters.artifactName, '') }}: + - output: pipelineArtifact + displayName: 'Publish the combined ${{ parameters.artifactName }} artifacts' + artifactName: ${{ parameters.artifactName }} + targetPath: 'output' + - ${{ each additionalArtifact in parameters.additionalArtifacts }}: + - output: pipelineArtifact + displayName: 'Publish the ${{ additionalArtifact.name }} artifacts' + ${{ if eq(additionalArtifact.always, 'true') }}: + condition: always() + artifactName: ${{ additionalArtifact.name }} + targetPath: ${{ additionalArtifact.path }} + steps: # prepare - checkout: self submodules: recursive - - template: azure-templates-variables.yml + - template: /scripts/azure-templates-variables.yml@self # checkout required skia PR - pwsh: .\scripts\checkout-skia.ps1 -GitHubToken $(GitHub.Token.PublicAccess) @@ -57,7 +86,7 @@ jobs: condition: eq(variables['Build.Reason'], 'PullRequest') - ${{ if eq(parameters.buildPipelineType, 'tests') }}: - - template: azure-templates-github-status.yml + - template: /scripts/azure-templates-github-status.yml@self parameters: state: 'pending' @@ -66,13 +95,13 @@ jobs: displayName: Determine build type # provisioning steps - - ${{ if ne(parameters.vmImage.provisioningSteps, '') }}: - - ${{ parameters.vmImage.provisioningSteps }} + - ${{ if ne(parameters.buildAgent.provisioningSteps, '') }}: + - ${{ parameters.buildAgent.provisioningSteps }} - ${{ if ne(length(parameters.provisioningSteps), 0) }}: - ${{ parameters.provisioningSteps }} # install any packages on linux - - ${{ if and(eq(parameters.docker, ''), endsWith(parameters.name, '_linux')) }}: + - ${{ if and(eq(parameters.docker, ''), endsWith(parameters.name, '_linux'), ne(parameters.skipInstall, 'true')) }}: - bash: | sudo apt update sudo apt install -y ${{ parameters.packages }} @@ -81,7 +110,7 @@ jobs: condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], ''), ne('${{ parameters.packages }}', '')) # install extra bits for the native builds - - ${{ if startsWith(parameters.name, 'native_') }}: + - ${{ if and(startsWith(parameters.name, 'native_'), ne(parameters.skipInstall, 'true')) }}: # switch to the correct Python version - task: UsePythonVersion@0 displayName: Switch to the correct Python version @@ -120,7 +149,7 @@ jobs: condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install extra bits for the managed builds - - ${{ if not(startsWith(parameters.name, 'native_')) }}: + - ${{ if and(not(startsWith(parameters.name, 'native_')), ne(parameters.skipInstall, 'true')) }}: # install ninja - ${{ if eq(parameters.installNinja, 'true') }}: - pwsh: .\scripts\install-ninja.ps1 @@ -201,7 +230,7 @@ jobs: displayName: Install the .NET workloads # install the mac tools - - ${{ if endsWith(parameters.name, '_macos') }}: + - ${{ if and(endsWith(parameters.name, '_macos'), ne(parameters.skipInstall, 'true')) }}: - ${{ if not(startsWith(parameters.name, 'native_')) }}: - bash: sudo ./scripts/select-xcode.sh $(XCODE_VERSION) displayName: Switch to the latest Xcode @@ -212,14 +241,14 @@ jobs: condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install the linux tools - - ${{ if and(eq(parameters.installEmsdk, 'true'), endsWith(parameters.name, '_linux')) }}: + - ${{ if and(eq(parameters.installEmsdk, 'true'), endsWith(parameters.name, '_linux'), ne(parameters.skipInstall, 'true')) }}: - bash: ./scripts/install-emsdk.sh $(EMSCRIPTEN_VERSION) displayName: Install the Emscripten SDK retryCountOnTaskFailure: 3 condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install the Windows tools - - ${{ if endsWith(parameters.name, '_windows') }}: + - ${{ if and(endsWith(parameters.name, '_windows'), ne(parameters.skipInstall, 'true')) }}: # select the correct/latest version of Visual Studio - pwsh: .\scripts\select-vs.ps1 displayName: Select Visual Studio @@ -245,18 +274,19 @@ jobs: condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # install any .NET global tools - - ${{ each tool in parameters.tools }}: - - pwsh: dotnet tool install -g ${{ tool }} - displayName: Install ${{ tool }} - retryCountOnTaskFailure: 3 - condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) + - ${{ if ne(parameters.skipInstall, 'true') }}: + - ${{ each tool in parameters.tools }}: + - pwsh: dotnet tool install -g ${{ tool }} + displayName: Install ${{ tool }} + retryCountOnTaskFailure: 3 + condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # download artifacts - - template: azure-templates-download-artifacts.yml + - template: /scripts/azure-templates-download-artifacts.yml@self parameters: condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) artifacts: ${{ parameters.requiredArtifacts }} - - template: azure-templates-download-artifacts.yml + - template: /scripts/azure-templates-download-artifacts.yml@self parameters: condition: and(succeeded(), ne(variables['DOWNLOAD_EXTERNALS'], '')) sourceBuildId: $(DOWNLOAD_EXTERNALS) @@ -264,88 +294,65 @@ jobs: - name: ${{ parameters.name }} # pre-build steps - - ${{ if ne(parameters.vmImage.preBuildSteps, '') }}: - - ${{ parameters.vmImage.preBuildSteps }} + - ${{ if ne(parameters.buildAgent.preBuildSteps, '') }}: + - ${{ parameters.buildAgent.preBuildSteps }} - ${{ if ne(length(parameters.preBuildSteps), 0) }}: - ${{ parameters.preBuildSteps }} - # build - - ${{ if eq(parameters.docker, '') }}: - - ${{ if endsWith(parameters.name, '_windows') }}: - - pwsh: | - Get-Content $PSCommandPath - dotnet tool restore - ${{ parameters.initScript }} - dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} + # actual build + - ${{ if ne(parameters.skipSteps, 'true') }}: + - ${{ if eq(parameters.docker, '') }}: + - ${{ if endsWith(parameters.name, '_windows') }}: + - pwsh: | + Get-Content $PSCommandPath + dotnet tool restore + ${{ parameters.initScript }} + dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} - env: - JavaSdkDirectory: $(JAVA_HOME) - LLVM_HOME: $(LLVM_HOME) - # There seems to be a bug in some verions of mspdbcmf.exe. This looks to be fixed in a VS preview. - AppxSymbolPackageEnabled: false - displayName: Run the bootstrapper for ${{ parameters.target }} - retryCountOnTaskFailure: ${{ parameters.retryCount }} + env: + JavaSdkDirectory: $(JAVA_HOME) + LLVM_HOME: $(LLVM_HOME) + # There seems to be a bug in some verions of mspdbcmf.exe. This looks to be fixed in a VS preview. + AppxSymbolPackageEnabled: false + displayName: Run the bootstrapper for ${{ parameters.target }} + retryCountOnTaskFailure: ${{ parameters.retryCount }} + condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) + - ${{ if not(endsWith(parameters.name, '_windows')) }}: + - bash: | + cat ${BASH_SOURCE[0]} + dotnet tool restore + ${{ parameters.initScript }} + dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} + + env: + JavaSdkDirectory: $(JAVA_HOME) + displayName: Run the bootstrapper for ${{ parameters.target }} + retryCountOnTaskFailure: ${{ parameters.retryCount }} + condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) + - ${{ if ne(parameters.docker, '') }}: + - task: 1ES.BuildContainerImage@1 + displayName: Build the Docker image for ${{ parameters.docker }} condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) - - ${{ if not(endsWith(parameters.name, '_windows')) }}: + inputs: + dockerfile: ${{ parameters.docker }}/Dockerfile + context: ${{ parameters.docker }} + image: skiasharp:skiasharp + buildArguments: --tag skiasharp ${{ parameters.dockerArgs }} - bash: | - cat ${BASH_SOURCE[0]} - dotnet tool restore - ${{ parameters.initScript }} - dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} - - env: - JavaSdkDirectory: $(JAVA_HOME) - displayName: Run the bootstrapper for ${{ parameters.target }} + echo dotnet tool restore > cmd.sh + echo dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} >> cmd.sh + sed -i 's/--gnArgs=\" \"//' cmd.sh + cat cmd.sh + displayName: Generate the script for the Docker image + condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) + - bash: | + docker run --rm --name skiasharp --volume $(pwd):/work skiasharp /bin/bash /work/cmd.sh + displayName: Run the bootstrapper for ${{ parameters.target }} using the Docker image retryCountOnTaskFailure: ${{ parameters.retryCount }} condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) - - ${{ if ne(parameters.docker, '') }}: - - task: Docker@2 - displayName: Build the Docker image for ${{ parameters.docker }} - condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) - inputs: - command: build - buildContext: ${{ parameters.docker }} - dockerfile: ${{ parameters.docker }}/Dockerfile - arguments: --tag skiasharp ${{ parameters.dockerArgs }} - - bash: | - echo dotnet tool restore > cmd.sh - echo dotnet cake --target=${{ parameters.target }} --verbosity=${{ parameters.verbosity }} --configuration=${{ coalesce(parameters.configuration, 'Release') }} ${{ parameters.additionalArgs }} >> cmd.sh - sed -i 's/--gnArgs=\" \"//' cmd.sh - cat cmd.sh - displayName: Generate the script for the Docker image - condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) - - bash: | - docker run --rm --name skiasharp --volume $(pwd):/work skiasharp /bin/bash /work/cmd.sh - displayName: Run the bootstrapper for ${{ parameters.target }} using the Docker image - retryCountOnTaskFailure: ${{ parameters.retryCount }} - condition: and(succeeded(), eq(variables['DOWNLOAD_EXTERNALS'], '')) # post-build steps - ${{ parameters.postBuildSteps }} - # publish artifacts - - task: PublishBuildArtifacts@1 - displayName: Publish the ${{ parameters.name }} artifacts - condition: or(${{ parameters.shouldPublish }}, failed()) - retryCountOnTaskFailure: 3 - inputs: - artifactName: ${{ parameters.name }} - pathToPublish: 'output' - - ${{ if ne(parameters.artifactName, '') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish the combined ${{ parameters.artifactName }} artifacts - retryCountOnTaskFailure: 3 - inputs: - artifactName: ${{ parameters.artifactName }} - pathToPublish: 'output' - - ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(parameters.buildPipelineType, 'tests'), ne(variables['System.PullRequest.IsFork'], 'true'), or(and(eq(variables['Build.Reason'], 'Schedule'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'))), parameters.runCompliance)) }}: - - task: ComponentGovernanceComponentDetection@0 - displayName: Run component detection - condition: always() - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - ${{ if eq(parameters.buildPipelineType, 'tests') }}: - - template: azure-templates-github-status.yml + - template: /scripts/azure-templates-github-status.yml@self diff --git a/scripts/azure-templates-linux-matrix.yml b/scripts/azure-templates-linux-matrix.yml index 6bad3fbbd1..47a5e6ddd0 100644 --- a/scripts/azure-templates-linux-matrix.yml +++ b/scripts/azure-templates-linux-matrix.yml @@ -2,7 +2,7 @@ parameters: artifactName: '' # the name of the artifact to merge this run into buildExternals: '' # the build number to download externals from buildPipelineType: 'both' # the type of build pipeline setup - vmImage: '' # the VM image + buildAgent: '' # the configuration for the build agent builds: - name: '' desc: '' @@ -20,15 +20,15 @@ parameters: jobs: - ${{ each build in parameters.builds }}: - ${{ each item in parameters.matrix }}: - - template: azure-templates-bootstrapper.yml + - template: /scripts/azure-templates-bootstrapper.yml@self parameters: name: ${{ replace(replace(format('native_linux_{0}_{1}_{2}_{3}_linux', item.arch, item.variant, build.name, item.alt), '__', '_'), '__', '_') }} displayName: Linux ${{ replace(replace(replace(replace(replace(format('({0}|{1}|{2}|{3})', item.arch, item.variant, build.name, item.alt), '||', '|'), '||', '|'), '(|', '('), '|)', ')'), '|', ', ') }} buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.vmImage }} + buildAgent: ${{ parameters.buildAgent }} docker: ${{ item.docker }} dockerArgs: ${{ item.dockerArgs }} target: ${{ coalesce(item.target, 'externals-linux') }} additionalArgs: --buildarch=${{ item.arch }} --variant=${{ coalesce(item.variant, 'linux') }}${{ build.name }} --gnArgs="\"${{ build.gnArgs }} ${{ item.gnArgs }}\"" ${{ build.additionalArgs }} ${{ item.additionalArgs }} - artifactName: ${{ parameters.artifactName }} + artifactName: ${{ parameters.artifactName }} \ No newline at end of file diff --git a/scripts/azure-templates-merger.yml b/scripts/azure-templates-merger.yml new file mode 100644 index 0000000000..615fd8903c --- /dev/null +++ b/scripts/azure-templates-merger.yml @@ -0,0 +1,44 @@ +parameters: + name: '' # in the form type_platform_host + displayName: '' # the human name + buildAgent: '' # the configuration for the build agent + buildPipelineType: 'both' # the type of build pipeline setup + requiredArtifacts: [] # the artifacts that this build needs to download + matrixArtifacts: [] # the artifacts that this build needs to download + +jobs: + - template: /scripts/azure-templates-bootstrapper.yml@self + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + buildPipelineType: ${{ parameters.buildPipelineType }} + buildAgent: ${{ parameters.buildAgent }} + skipInstall: true + skipSteps: true + requiredArtifacts: ${{ parameters.requiredArtifacts }} + preBuildSteps: + - pwsh: az devops configure --defaults organization=$(System.TeamFoundationCollectionUri) project=$(System.TeamProject) --use-git-aliases true + - ${{ each artifactJson in parameters.matrixArtifacts }}: + - pwsh: | + $artifactJson=@' + ${{ artifactJson }} + '@ + + echo $artifactJson + + $json = ConvertFrom-Json $artifactJson + $objects = $json | Get-Member -MemberType NoteProperty + $names = $objects | ForEach-Object { $json."$($_.Name)".name } + + Write-Host "Found $($names.Length) items:" + $names | ForEach-Object { Write-Host " - $_" } + + $dir = "$(Build.ArtifactStagingDirectory)" + $id = "$(Build.BuildId)" + foreach ($name in $names) { + Write-Host "Downloading '$name'..." + az pipelines runs artifact download --artifact-name "$name" --path "$dir" --run-id "$id" + } + Write-Host "Downloads complete." + env: + AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) diff --git a/scripts/azure-templates-stages.yml b/scripts/azure-templates-stages.yml index ff53526a17..1d270b05c2 100644 --- a/scripts/azure-templates-stages.yml +++ b/scripts/azure-templates-stages.yml @@ -5,19 +5,19 @@ parameters: - name: buildExternals type: string default: 'latest' - - name: VM_IMAGE_HOST + - name: buildAgentHost type: object - - name: VM_IMAGE_WINDOWS + - name: buildAgentWindows type: object - - name: VM_IMAGE_WINDOWS_NATIVE + - name: buildAgentWindowsNative type: object - - name: VM_IMAGE_MAC + - name: buildAgentMac type: object - - name: VM_IMAGE_MAC_NATIVE + - name: buildAgentMacNative type: object - - name: VM_IMAGE_LINUX + - name: buildAgentLinux type: object - - name: VM_IMAGE_LINUX_NATIVE + - name: buildAgentLinuxNative type: object - name: runCompliance type: boolean @@ -29,263 +29,244 @@ stages: jobs: - job: prepare # Prepare Build displayName: Prepare Build - pool: ${{ parameters.VM_IMAGE_HOST.pool }} + pool: ${{ parameters.buildAgentHost.pool }} steps: - checkout: none - - template: azure-templates-variables.yml + - template: /scripts/azure-templates-variables.yml@self parameters: updateBuild: true - ${{ if eq(parameters.buildPipelineType, 'build') }}: - - template: azure-templates-github-status.yml + - template: /scripts/azure-templates-github-status.yml@self parameters: context: 'SkiaSharp-Tests' state: 'pending' displayName: Queue up the status for the tests pipeline - ${{ if eq(parameters.buildPipelineType, 'tests') }}: - - template: azure-templates-github-status.yml + - template: /scripts/azure-templates-github-status.yml@self - ${{ if ne(parameters.buildPipelineType, 'tests') }}: - stage: native_windows displayName: Native Windows dependsOn: prepare jobs: - - template: azure-templates-bootstrapper.yml # Build Native Android|x86 (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|x86 (Win) parameters: name: native_android_x86_windows displayName: Android x86 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-android additionalArgs: --buildarch=x86 - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native Android|x64 (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|x64 (Win) parameters: name: native_android_x64_windows displayName: Android x64 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-android additionalArgs: --buildarch=x64 - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native Android|arm (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|arm (Win) parameters: name: native_android_arm_windows displayName: Android arm buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-android additionalArgs: --buildarch=arm - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native Android|arm64 (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|arm64 (Win) parameters: name: native_android_arm64_windows displayName: Android arm64 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-android additionalArgs: --buildarch=arm64 - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native Tizen (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Tizen (Win) parameters: name: native_tizen_windows displayName: Tizen buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-tizen - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native Win32|x86 (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|x86 (Win) parameters: name: native_win32_x86_windows displayName: Win32 x86 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-windows additionalArgs: --buildarch=x86 - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native Win32|x64 (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|x64 (Win) parameters: name: native_win32_x64_windows displayName: Win32 x64 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-windows additionalArgs: --buildarch=x64 - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native Win32|arm64 (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|arm64 (Win) parameters: name: native_win32_arm64_windows displayName: Win32 arm64 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-windows additionalArgs: --buildarch=arm64 - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native Win32|x86 (Win + MSVC) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|x86 (Win + MSVC) parameters: name: native_win32_x86_msvc_windows displayName: Win32 x86 [MSVC] buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + vmImage: ${{ parameters.buildAgentWindowsNative }} target: externals-windows additionalArgs: --buildarch=x86 --llvm="msvc" - artifactName: native_msvc installLlvm: false - - template: azure-templates-bootstrapper.yml # Build Native Win32|x64 (Win + MSVC) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|x64 (Win + MSVC) parameters: name: native_win32_x64_msvc_windows displayName: Win32 x64 [MSVC] buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + vmImage: ${{ parameters.buildAgentWindowsNative }} target: externals-windows additionalArgs: --buildarch=x64 --llvm="msvc" - artifactName: native_msvc installLlvm: false - - template: azure-templates-bootstrapper.yml # Build Native Win32|arm64 (Win + MSVC) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Win32|arm64 (Win + MSVC) parameters: name: native_win32_arm64_msvc_windows displayName: Win32 arm64 [MSVC] buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + vmImage: ${{ parameters.buildAgentWindowsNative }} target: externals-windows additionalArgs: --buildarch=arm64 --llvm="msvc" - artifactName: native_msvc installLlvm: false - - template: azure-templates-bootstrapper.yml # Build Native WinUI|x86 (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native WinUI|x86 (Win) parameters: name: native_winui_x86_windows displayName: WinUI x86 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-winui additionalArgs: --buildarch=x86 - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native WinUI|x64 (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native WinUI|x64 (Win) parameters: name: native_winui_x64_windows displayName: WinUI x64 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-winui additionalArgs: --buildarch=x64 - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native WinUI|arm64 (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native WinUI|arm64 (Win) parameters: name: native_winui_arm64_windows displayName: WinUI arm64 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-winui additionalArgs: --buildarch=arm64 - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native NanoServer|x64 (Win) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native NanoServer|x64 (Win) parameters: name: native_win32_x64_nanoserver_windows displayName: Nano Server x64 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS_NATIVE }} + buildAgent: ${{ parameters.buildAgentWindowsNative }} target: externals-nanoserver additionalArgs: --buildarch=x64 - artifactName: native - ${{ if ne(parameters.buildPipelineType, 'tests') }}: - stage: native_macos displayName: Native macOS dependsOn: prepare jobs: - - template: azure-templates-bootstrapper.yml # Build Native Android|x86 (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|x86 (macOS) parameters: name: native_android_x86_macos displayName: Android x86 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} + buildAgent: ${{ parameters.buildAgentMacNative }} target: externals-android additionalArgs: --buildarch=x86 - - template: azure-templates-bootstrapper.yml # Build Native Android|x64 (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|x64 (macOS) parameters: name: native_android_x64_macos displayName: Android x64 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} + buildAgent: ${{ parameters.buildAgentMacNative }} target: externals-android additionalArgs: --buildarch=x64 - - template: azure-templates-bootstrapper.yml # Build Native Android|arm (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|arm (macOS) parameters: name: native_android_arm_macos displayName: Android arm buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} + buildAgent: ${{ parameters.buildAgentMacNative }} target: externals-android additionalArgs: --buildarch=arm - - template: azure-templates-bootstrapper.yml # Build Native Android|arm64 (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Android|arm64 (macOS) parameters: name: native_android_arm64_macos displayName: Android arm64 buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} + buildAgent: ${{ parameters.buildAgentMacNative }} target: externals-android additionalArgs: --buildarch=arm64 - - template: azure-templates-bootstrapper.yml # Build Native iOS (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native iOS (macOS) parameters: name: native_ios_macos displayName: iOS buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} + buildAgent: ${{ parameters.buildAgentMacNative }} target: externals-ios - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native Mac Catalyst (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Mac Catalyst (macOS) parameters: name: native_maccatalyst_macos displayName: Mac Catalyst buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} + buildAgent: ${{ parameters.buildAgentMacNative }} target: externals-maccatalyst - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native macOS (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native macOS (macOS) parameters: name: native_macos_macos displayName: macOS buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} + buildAgent: ${{ parameters.buildAgentMacNative }} target: externals-macos - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native tvOS (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native tvOS (macOS) parameters: name: native_tvos_macos displayName: tvOS buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} + buildAgent: ${{ parameters.buildAgentMacNative }} target: externals-tvos - artifactName: native - - template: azure-templates-bootstrapper.yml # Build Native Tizen (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Tizen (macOS) parameters: name: native_tizen_macos displayName: Tizen buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC_NATIVE }} + buildAgent: ${{ parameters.buildAgentMacNative }} target: externals-tizen - ${{ if ne(parameters.buildPipelineType, 'tests') }}: @@ -293,12 +274,11 @@ stages: displayName: Native Linux dependsOn: prepare jobs: - - template: azure-templates-linux-matrix.yml # Build Native Linux (Linux) + - template: /scripts/azure-templates-linux-matrix.yml@self # Build Native Linux (Linux) parameters: - artifactName: native buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_LINUX_NATIVE }} + buildAgent: ${{ parameters.buildAgentLinuxNative }} builds: - name: '' - name: nodeps @@ -318,13 +298,13 @@ stages: docker: scripts/Docker/debian9/clang-cross dockerArgs: --build-arg TOOLCHAIN_ARCH=aarch64-linux-gnu --build-arg TOOLCHAIN_ARCH_SHORT=arm64 target: externals-linux-clang-cross - - template: azure-templates-bootstrapper.yml # Build Native Tizen (Linux) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Native Tizen (Linux) parameters: name: native_tizen_linux displayName: Tizen buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_LINUX_NATIVE }} + buildAgent: ${{ parameters.buildAgentLinuxNative }} packages: $(TIZEN_LINUX_PACKAGES) target: externals-tizen @@ -333,12 +313,11 @@ stages: displayName: Native WASM dependsOn: prepare jobs: - - template: azure-templates-wasm-matrix.yml # Build Native WASM (Linux) + - template: /scripts/azure-templates-wasm-matrix.yml@self # Build Native WASM (Linux) parameters: buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_LINUX_NATIVE }} - artifactName: native + buildAgent: ${{ parameters.buildAgentLinuxNative }} emscripten: - 2.0.6: displayName: 2.0.6 @@ -392,20 +371,65 @@ stages: features: _wasmeh,simd,mt - ${{ if ne(parameters.buildPipelineType, 'tests') }}: - - stage: managed - displayName: Build Managed + - stage: native + displayName: Native + variables: + nativeLinuxJobs: $[ convertToJson(stageDependencies.native_linux) ] + nativeWasmJobs: $[ convertToJson(stageDependencies.native_wasm) ] dependsOn: - native_windows - native_macos - native_linux - native_wasm jobs: - - template: azure-templates-bootstrapper.yml # Build Managed (Windows) + - template: /scripts/azure-templates-merger.yml@self # Merge Native Artifacts + parameters: + name: native + displayName: Merge Native Artifacts + buildPipelineType: ${{ parameters.buildPipelineType }} + buildAgent: ${{ parameters.buildAgentHost }} + requiredArtifacts: + # Android + - name: native_android_x86_windows + - name: native_android_x64_windows + - name: native_android_arm_windows + - name: native_android_arm64_windows + # Tizen + - name: native_tizen_windows + # Win32 + - name: native_win32_x86_windows + - name: native_win32_x64_windows + - name: native_win32_arm64_windows + # WinUI + - name: native_winui_x86_windows + - name: native_winui_x64_windows + - name: native_winui_arm64_windows + # Nano Server + - name: native_win32_x64_nanoserver_windows + # iOS + - name: native_ios_macos + # Mac Catalyst + - name: native_maccatalyst_macos + # macOS + - name: native_macos_macos + # tvOS + - name: native_tvos_macos + matrixArtifacts: + - $(nativeLinuxJobs) + - $(nativeWasmJobs) + + - ${{ if ne(parameters.buildPipelineType, 'tests') }}: + - stage: managed + displayName: Build Managed + dependsOn: + - native + jobs: + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Managed (Windows) parameters: name: managed_windows displayName: Managed (Windows) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS}} + buildAgent: ${{ parameters.buildAgentWindows}} target: libs additionalArgs: --skipExternals="all" requiredArtifacts: @@ -414,12 +438,12 @@ stages: postBuildSteps: - pwsh: Remove-Item ./output/native/ -Recurse -Force -ErrorAction Continue displayName: Delete the native folder - - template: azure-templates-bootstrapper.yml # Build Managed (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Managed (macOS) parameters: name: managed_macos displayName: Managed (macOS) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC }} + buildAgent: ${{ parameters.buildAgentMac }} target: libs additionalArgs: --skipExternals="all" requiredArtifacts: @@ -428,12 +452,12 @@ stages: postBuildSteps: - pwsh: Remove-Item ./output/native/ -Recurse -Force -ErrorAction Continue displayName: Delete the native folder - - template: azure-templates-bootstrapper.yml # Build Managed (Linux) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Managed (Linux) parameters: name: managed_linux displayName: Managed (Linux) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_LINUX }} + buildAgent: ${{ parameters.buildAgentLinux }} packages: $(MANAGED_LINUX_PACKAGES) target: libs additionalArgs: --skipExternals="all" @@ -453,43 +477,35 @@ stages: - native_linux - native_wasm jobs: - - template: azure-templates-bootstrapper.yml # Package NuGets + - template: /scripts/azure-templates-bootstrapper.yml@self # Package NuGets parameters: name: package_normal_windows displayName: Package NuGets buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS}} + buildAgent: ${{ parameters.buildAgentWindows}} target: nuget-normal additionalArgs: --skipExternals="all" requiredArtifacts: - name: native postBuildSteps: - - task: PublishBuildArtifacts@1 - displayName: Publish the nuget artifacts - inputs: - artifactName: nuget - pathToPublish: 'output/nugets' - - task: PublishBuildArtifacts@1 - displayName: Publish the SignList.xml into nuget artifacts - inputs: - artifactName: nuget - pathToPublish: 'scripts\SignList.xml' - - task: PublishBuildArtifacts@1 - displayName: Publish the symbols nuget artifacts - inputs: - artifactName: nuget_symbols - pathToPublish: 'output/nugets-symbols' - pwsh: | Remove-Item ./output/native/ -Recurse -Force Remove-Item ./output/nugets/ -Recurse -Force Remove-Item ./output/nugets-symbols/ -Recurse -Force displayName: Delete the pre-published folders - - template: azure-templates-bootstrapper.yml # Package Special NuGets + additionalArtifacts: + - name: nuget + path: 'output/nugets' + - name: nuget + path: 'scripts/SignList.xml' + - name: nuget_symbols + path: 'output/nugets-symbols' + - template: /scripts/azure-templates-bootstrapper.yml@self # Package Special NuGets parameters: name: package_special_windows displayName: Package Special NuGets buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS}} + buildAgent: ${{ parameters.buildAgentWindows}} dependsOn: - package_normal_windows target: nuget-special @@ -500,16 +516,14 @@ stages: - name: nuget_symbols dir: nugets-symbols postBuildSteps: - - task: PublishBuildArtifacts@1 - displayName: Publish the special nuget artifacts - inputs: - artifactName: nuget_special - pathToPublish: 'output/nugets-special' - pwsh: | Remove-Item ./output/nugets/ -Recurse -Force Remove-Item ./output/nugets-special/ -Recurse -Force Remove-Item ./output/nugets-symbols/ -Recurse -Force displayName: Delete the pre-published folders + additionalArtifacts: + - name: nuget_special + path: 'output/nugets-special' - ${{ if ne(parameters.buildPipelineType, 'build') }}: - stage: api_diff @@ -519,12 +533,12 @@ stages: ${{ if eq(parameters.buildPipelineType, 'both') }}: dependsOn: package jobs: - - template: azure-templates-bootstrapper.yml # API Diff + - template: /scripts/azure-templates-bootstrapper.yml@self # API Diff parameters: name: api_diff_windows displayName: API Diff buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS}} + buildAgent: ${{ parameters.buildAgentWindows}} target: docs-api-diff additionalArgs: --nugetDiffPrerelease=$(NUGET_DIFF_PRERELEASE) shouldPublish: false @@ -534,42 +548,36 @@ stages: preBuildSteps: - pwsh: .\scripts\install-gtk.ps1 displayName: Install GTK# 2.12 - postBuildSteps: - - task: PublishBuildArtifacts@1 - displayName: Publish the API diffs - condition: always() - inputs: - artifactName: api-diff - pathToPublish: '$(Build.SourcesDirectory)\output\api-diff' - - task: PublishBuildArtifacts@1 - displayName: Publish the changelogs - condition: always() - inputs: - artifactName: changelogs - pathToPublish: '$(Build.SourcesDirectory)\changelogs' + additionalArtifacts: + - name: api-diff + always: true + path: '$(Build.SourcesDirectory)\output\api-diff' + - name: changelogs + always: true + path: '$(Build.SourcesDirectory)\changelogs' - - ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(parameters.buildPipelineType, 'tests'), ne(variables['System.PullRequest.IsFork'], 'true')) }}: - - stage: signing - displayName: Sign NuGets - dependsOn: package - jobs: - - template: sign-artifacts/jobs/v2.yml@xamarin-templates - parameters: - ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: - signType: 'Real' - ${{ if not(or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'))) }}: - signType: 'Test' + # - ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(parameters.buildPipelineType, 'tests'), ne(variables['System.PullRequest.IsFork'], 'true')) }}: + # - stage: signing + # displayName: Sign NuGets + # dependsOn: package + # jobs: + # - template: sign-artifacts/jobs/v2.yml@xamarin-templates + # parameters: + # ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: + # signType: 'Real' + # ${{ if not(or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'))) }}: + # signType: 'Test' - - ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(parameters.buildPipelineType, 'tests'), ne(variables['System.PullRequest.IsFork'], 'true')) }}: - - stage: sbom - displayName: 'Software Bill of Materials' - dependsOn: signing - jobs: - - template: compliance/sbom/job.v1.yml@xamarin-templates # Software Bill of Materials (SBOM): https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/ado-sbom-generator - parameters: - artifactNames: ['nuget'] - packageName: 'SkiaSharp' - packageFilter: '*.nupkg' + # - ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(parameters.buildPipelineType, 'tests'), ne(variables['System.PullRequest.IsFork'], 'true')) }}: + # - stage: sbom + # displayName: 'Software Bill of Materials' + # dependsOn: signing + # jobs: + # - template: compliance/sbom/job.v1.yml@xamarin-templates # Software Bill of Materials (SBOM): https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/ado-sbom-generator + # parameters: + # artifactNames: ['nuget'] + # packageName: 'SkiaSharp' + # packageFilter: '*.nupkg' - ${{ if ne(parameters.buildPipelineType, 'build') }}: - stage: tests @@ -583,12 +591,12 @@ stages: - native_linux - native_wasm jobs: - - template: azure-templates-bootstrapper.yml # Tests|netfx (Windows) + - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|netfx (Windows) parameters: name: tests_netfx_windows displayName: Windows (.NET Framework) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS}} + buildAgent: ${{ parameters.buildAgentWindows}} target: tests-netfx additionalArgs: --skipExternals="all" --coverage=$(ENABLE_CODE_COVERAGE) shouldPublish: false @@ -604,18 +612,16 @@ stages: testResultsFormat: xUnit testResultsFiles: 'output/logs/testlogs/**/TestResults.xml' testRunTitle: 'Windows .NET Framework Tests' - - task: PublishBuildArtifacts@1 - displayName: Publish the test logs - condition: always() - inputs: - artifactName: testlogs_netfx_windows - pathToPublish: 'output/logs/testlogs' - - template: azure-templates-bootstrapper.yml # Tests|netcore (Windows) + additionalArtifacts: + - name: testlogs_netfx_windows + always: true + path: 'output/logs/testlogs' + - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|netcore (Windows) parameters: name: tests_netcore_windows displayName: Windows (.NET Core) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS}} + buildAgent: ${{ parameters.buildAgentWindows}} target: tests-netcore additionalArgs: --skipExternals="all" --coverage=$(ENABLE_CODE_COVERAGE) shouldPublish: false @@ -631,23 +637,18 @@ stages: testResultsFormat: xUnit testResultsFiles: 'output/logs/testlogs/**/TestResults.xml' testRunTitle: 'Windows .NET Core Tests' - - task: PublishBuildArtifacts@1 - displayName: Publish the test logs - condition: always() - inputs: - artifactName: testlogs_netcore_windows - pathToPublish: 'output/logs/testlogs' - - task: PublishBuildArtifacts@1 - displayName: 'Publish the code coverage results' - inputs: - artifactName: coverage_netcore_windows - pathToPublish: 'output/coverage' - - template: azure-templates-bootstrapper.yml # Tests|netcore (macOS) + additionalArtifacts: + - name: testlogs_netcore_windows + always: true + path: 'output/logs/testlogs' + - name: coverage_netcore_windows + path: 'output/coverage' + - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|netcore (macOS) parameters: name: tests_netcore_macos displayName: macOS (.NET Core) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC }} + buildAgent: ${{ parameters.buildAgentMac }} target: tests-netcore additionalArgs: --skipExternals="all" --coverage=$(ENABLE_CODE_COVERAGE) shouldPublish: false @@ -661,23 +662,18 @@ stages: testResultsFormat: xUnit testResultsFiles: 'output/logs/testlogs/**/TestResults.xml' testRunTitle: 'macOS .NET Core Tests' - - task: PublishBuildArtifacts@1 - displayName: Publish the test logs - condition: always() - inputs: - artifactName: testlogs_netcore_macos - pathToPublish: 'output/logs/testlogs' - - task: PublishBuildArtifacts@1 - displayName: 'Publish the code coverage results' - inputs: - artifactName: coverage_netcore_macos - pathToPublish: 'output/coverage' - - template: azure-templates-bootstrapper.yml # Tests|netcore (Linux) + additionalArtifacts: + - name: testlogs_netcore_macos + always: true + path: 'output/logs/testlogs' + - name: coverage_netcore_macos + path: 'output/coverage' + - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|netcore (Linux) parameters: name: tests_netcore_linux displayName: Linux (.NET Core) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_LINUX }} + buildAgent: ${{ parameters.buildAgentLinux }} packages: $(MANAGED_LINUX_PACKAGES) target: tests-netcore additionalArgs: --skipExternals="all" --coverage=$(ENABLE_CODE_COVERAGE) @@ -693,23 +689,18 @@ stages: testResultsFormat: xUnit testResultsFiles: 'output/logs/testlogs/**/TestResults.xml' testRunTitle: 'Linux .NET Core Tests' - - task: PublishBuildArtifacts@1 - displayName: Publish the test logs - condition: always() - inputs: - artifactName: testlogs_netcore_linux - pathToPublish: 'output/logs/testlogs' - - task: PublishBuildArtifacts@1 - displayName: 'Publish the code coverage results' - inputs: - artifactName: coverage_netcore_linux - pathToPublish: 'output/coverage' - - template: azure-templates-bootstrapper.yml # Tests|android (macOS) + additionalArtifacts: + - name: testlogs_netcore_linux + always: true + path: 'output/logs/testlogs' + - name: coverage_netcore_linux + path: 'output/coverage' + - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|android (macOS) parameters: name: tests_android_macos displayName: Android (macOS) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC }} + buildAgent: ${{ parameters.buildAgentMac }} target: tests-android additionalArgs: --device=android-emulator-64 --deviceVersion=$(ANDROID_TEST_DEVICE_VERSION) --skipExternals="all" --coverage=$(ENABLE_CODE_COVERAGE) shouldPublish: false @@ -731,25 +722,23 @@ stages: testResultsFormat: xUnit testResultsFiles: 'output/logs/testlogs/**/TestResults.xml' testRunTitle: 'Android Tests (API $(ANDROID_TEST_DEVICE_VERSION))' - - task: PublishBuildArtifacts@1 - displayName: Publish the test logs - condition: always() - inputs: - artifactName: testlogs_android_$(ANDROID_TEST_DEVICE_VERSION) - pathToPublish: 'output/logs/testlogs' - - template: azure-templates-bootstrapper.yml # Tests|ios (macOS) + additionalArtifacts: + - name: testlogs_android_$(ANDROID_TEST_DEVICE_VERSION) + always: true + path: 'output/logs/testlogs' + - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|ios (macOS) parameters: name: tests_ios_macos displayName: iOS (macOS) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC }} + buildAgent: ${{ parameters.buildAgentMac }} target: tests-ios additionalArgs: --device=ios-simulator-64 --deviceVersion=$(IOS_TEST_DEVICE_VERSION) --skipExternals="all" --coverage=$(ENABLE_CODE_COVERAGE) shouldPublish: false requiredArtifacts: - name: native_ios_macos preBuildSteps: - - template: azure-templates-provisioning-profiles.yml + - template: /scripts/azure-templates-provisioning-profiles.yml@self postBuildSteps: - task: PublishTestResults@2 displayName: Publish the iOS test results @@ -758,45 +747,41 @@ stages: testResultsFormat: xUnit testResultsFiles: 'output/logs/testlogs/**/TestResults.xml' testRunTitle: 'iOS Tests (v$(IOS_TEST_DEVICE_VERSION))' - - task: PublishBuildArtifacts@1 - displayName: Publish the test logs - condition: always() - inputs: - artifactName: testlogs_ios_$(IOS_TEST_DEVICE_VERSION) - pathToPublish: 'output/logs/testlogs' - - template: azure-templates-bootstrapper.yml # Tests|maccatalyst (macOS) + additionalArtifacts: + - name: testlogs_ios_$(IOS_TEST_DEVICE_VERSION) + always: true + path: 'output/logs/testlogs' + - template: /scripts/azure-templates-bootstrapper.yml@self # Tests|maccatalyst (macOS) parameters: name: tests_maccatalyst_macos displayName: Mac Catalyst (macOS) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC }} + buildAgent: ${{ parameters.buildAgentMac }} target: tests-maccatalyst additionalArgs: --device=maccatalyst --skipExternals="all" --coverage=$(ENABLE_CODE_COVERAGE) shouldPublish: false requiredArtifacts: - name: native_maccatalyst_macos preBuildSteps: - - template: azure-templates-provisioning-profiles.yml + - template: /scripts/azure-templates-provisioning-profiles.yml@self postBuildSteps: - task: PublishTestResults@2 - displayName: Publish the iOMac CatalystS test results + displayName: Publish the Mac Catalyst test results condition: always() inputs: testResultsFormat: xUnit testResultsFiles: 'output/logs/testlogs/**/TestResults.xml' testRunTitle: 'Mac Catalyst Tests' - - task: PublishBuildArtifacts@1 - displayName: Publish the test logs - condition: always() - inputs: - artifactName: testlogs_maccatalyst - pathToPublish: 'output/logs/testlogs' - - template: azure-templates-bootstrapper.yml # Tests [WASM] (Linux) + additionalArtifacts: + - name: testlogs_maccatalyst + always: true + path: 'output/logs/testlogs' + - template: /scripts/azure-templates-bootstrapper.yml@self # Tests [WASM] (Linux) parameters: name: tests_wasm_linux displayName: WASM (Linux) buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_LINUX }} + buildAgent: ${{ parameters.buildAgentLinux }} packages: $(MANAGED_LINUX_PACKAGES) ninja-build target: tests-wasm additionalArgs: --skipExternals="all" --coverage=false --chromedriver=$(CHROMEWEBDRIVER) @@ -813,27 +798,25 @@ stages: testResultsFormat: xUnit testResultsFiles: 'output/logs/testlogs/**/*.xml' testRunTitle: 'Linux WASM Tests' - - task: PublishBuildArtifacts@1 - displayName: Publish the test logs - condition: always() - inputs: - artifactName: testlogs_wasm - pathToPublish: 'output/logs/testlogs' + additionalArtifacts: + - name: testlogs_wasm + always: true + path: 'output/logs/testlogs' # TODO: add tests for linux alpine # TODO: add tests for linux no dependencies # TODO: add tests for windows nano server - job: coverage_reports # Coverage Reports displayName: Coverage Reports - pool: ${{ parameters.VM_IMAGE_HOST.pool }} + pool: ${{ parameters.buildAgentHost.pool }} dependsOn: - tests_netcore_windows - tests_netcore_macos - tests_netcore_linux steps: - checkout: self - - template: azure-templates-variables.yml + - template: /scripts/azure-templates-variables.yml@self - ${{ if ne(parameters.buildPipelineType, 'both') }}: - - template: azure-templates-github-status.yml + - template: /scripts/azure-templates-github-status.yml@self parameters: state: 'pending' - task: DownloadBuildArtifacts@1 @@ -857,7 +840,7 @@ stages: codeCoverageTool: Cobertura summaryFileLocation: 'output/**/Cobertura.xml' - ${{ if ne(parameters.buildPipelineType, 'both') }}: - - template: azure-templates-github-status.yml + - template: /scripts/azure-templates-github-status.yml@self - ${{ if ne(parameters.buildPipelineType, 'build') }}: - stage: samples @@ -867,12 +850,12 @@ stages: ${{ if eq(parameters.buildPipelineType, 'both') }}: dependsOn: package jobs: - - template: azure-templates-bootstrapper.yml # Build Samples (Windows) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Samples (Windows) parameters: name: samples_windows displayName: Windows buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_WINDOWS}} + buildAgent: ${{ parameters.buildAgentWindows}} target: samples requiredArtifacts: - name: nuget @@ -880,12 +863,12 @@ stages: postBuildSteps: - pwsh: Remove-Item ./output/nugets/ -Recurse -Force -ErrorAction Continue displayName: Delete the nugets folder - - template: azure-templates-bootstrapper.yml # Build Samples (macOS) + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Samples (macOS) parameters: name: samples_macos displayName: macOS buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_MAC }} + buildAgent: ${{ parameters.buildAgentMac }} target: samples installNinja: true shouldPublish: false @@ -893,13 +876,13 @@ stages: - name: nuget dir: nugets preBuildSteps: - - template: azure-templates-provisioning-profiles.yml - - template: azure-templates-bootstrapper.yml # Build Samples (Linux) + - template: /scripts/azure-templates-provisioning-profiles.yml@self + - template: /scripts/azure-templates-bootstrapper.yml@self # Build Samples (Linux) parameters: name: samples_linux displayName: Linux buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.VM_IMAGE_LINUX }} + buildAgent: ${{ parameters.buildAgentLinux }} packages: $(MANAGED_LINUX_PACKAGES) target: samples shouldPublish: false @@ -909,53 +892,53 @@ stages: installEmsdk: true initScript: source ~/emsdk/emsdk_env.sh - - ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(parameters.buildPipelineType, 'tests'), ne(variables['System.PullRequest.IsFork'], 'true'), or(and(eq(variables['Build.Reason'], 'Schedule'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'))), parameters.runCompliance)) }}: - - template: security/full/v1.yml@xamarin-templates - parameters: - stageDependsOn: - - package - complianceEnabled: true - complianceTimeoutInMinutes: 480 - scanArtifacts: - - nuget - - nuget_symbols - - native_msvc - antiMalwareEnabled: true - binSkimEnabled: false - policheckExclusionFile: $(Build.SourcesDirectory)\scripts\guardian\PoliCheckExclusions.xml - policheckGdnSuppressionFilesFolder: $(Build.SourcesDirectory)\scripts\guardian - credScanEnabled: true - credScanSuppressionFile: $(Build.SourcesDirectory)\scripts\guardian\CredScanSuppressions.json - sourceGdnSuppressionFile: $(Build.SourcesDirectory)\scripts\guardian\source.gdnsuppress - tsaConfigFile: $(Build.SourcesDirectory)\scripts\guardian\tsaoptions-v2.json - tsaReportBranch: $(Build.SourceBranch) - enableCodeInspector: true - apiScanEnabled: true - apiScanSoftwareName: 'SkiaSharp' - apiScanSoftwareVersionNum: $(SKIASHARP_MAJOR_VERSION) - apiScanPreserveLogsFolder: true - apiScanSurrogateConfigurationFolder: $(Build.ArtifactStagingDirectory)\APIScanSurrogates - preScanSteps: - - pwsh: | - $softwareFolder = "$(Build.ArtifactStagingDirectory)\binaries-to-scan" - $surrogateFile = "$(Build.SourcesDirectory)\scripts\guardian\APIScanSurrogates.in.xml" - $destFolder = "$(Build.ArtifactStagingDirectory)\APIScanSurrogates" - $destFile = "$destFolder\APIScanSurrogates.xml" - New-Item -ItemType Directory -Force -Path $destFolder | Out-Null - $surrogateContents = (Get-Content $surrogateFile) - $surrogateContents = $surrogateContents.Replace("{SOFTWARE_FOLDER}", $softwareFolder) - $surrogateContents | Set-Content $destFile - displayName: Generate the surrogate files - - pwsh: | - $nupkgs = (Get-ChildItem "$(Build.ArtifactStagingDirectory)\binaries-to-scan\*\*.*nupkg") - foreach ($nupkg in $nupkgs) { - $filename = $nupkg.Name.TrimEnd('.nupkg') - $dest = "$(Build.ArtifactStagingDirectory)\binaries-to-scan\nuget_symbols-extracted\$filename" - Write-Host "Extracting '$nupkg' to '$dest'..." - Expand-Archive $nupkg $dest - Remove-Item $nupkg - } - displayName: Extract all the .nupkg files + # - ${{ if and(eq(variables['System.TeamProject'], 'devdiv'), ne(parameters.buildPipelineType, 'tests'), ne(variables['System.PullRequest.IsFork'], 'true'), or(and(eq(variables['Build.Reason'], 'Schedule'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'))), parameters.runCompliance)) }}: + # - template: security/full/v1.yml@xamarin-templates + # parameters: + # stageDependsOn: + # - package + # complianceEnabled: true + # complianceTimeoutInMinutes: 480 + # scanArtifacts: + # - nuget + # - nuget_symbols + # - native_msvc + # antiMalwareEnabled: true + # binSkimEnabled: false + # policheckExclusionFile: $(Build.SourcesDirectory)\scripts\guardian\PoliCheckExclusions.xml + # policheckGdnSuppressionFilesFolder: $(Build.SourcesDirectory)\scripts\guardian + # credScanEnabled: true + # credScanSuppressionFile: $(Build.SourcesDirectory)\scripts\guardian\CredScanSuppressions.json + # sourceGdnSuppressionFile: $(Build.SourcesDirectory)\scripts\guardian\source.gdnsuppress + # tsaConfigFile: $(Build.SourcesDirectory)\scripts\guardian\tsaoptions-v2.json + # tsaReportBranch: $(Build.SourceBranch) + # enableCodeInspector: true + # apiScanEnabled: true + # apiScanSoftwareName: 'SkiaSharp' + # apiScanSoftwareVersionNum: $(SKIASHARP_MAJOR_VERSION) + # apiScanPreserveLogsFolder: true + # apiScanSurrogateConfigurationFolder: $(Build.ArtifactStagingDirectory)\APIScanSurrogates + # preScanSteps: + # - pwsh: | + # $softwareFolder = "$(Build.ArtifactStagingDirectory)\binaries-to-scan" + # $surrogateFile = "$(Build.SourcesDirectory)\scripts\guardian\APIScanSurrogates.in.xml" + # $destFolder = "$(Build.ArtifactStagingDirectory)\APIScanSurrogates" + # $destFile = "$destFolder\APIScanSurrogates.xml" + # New-Item -ItemType Directory -Force -Path $destFolder | Out-Null + # $surrogateContents = (Get-Content $surrogateFile) + # $surrogateContents = $surrogateContents.Replace("{SOFTWARE_FOLDER}", $softwareFolder) + # $surrogateContents | Set-Content $destFile + # displayName: Generate the surrogate files + # - pwsh: | + # $nupkgs = (Get-ChildItem "$(Build.ArtifactStagingDirectory)\binaries-to-scan\*\*.*nupkg") + # foreach ($nupkg in $nupkgs) { + # $filename = $nupkg.Name.TrimEnd('.nupkg') + # $dest = "$(Build.ArtifactStagingDirectory)\binaries-to-scan\nuget_symbols-extracted\$filename" + # Write-Host "Extracting '$nupkg' to '$dest'..." + # Expand-Archive $nupkg $dest + # Remove-Item $nupkg + # } + # displayName: Extract all the .nupkg files - ${{ if eq(parameters.buildPipelineType, 'tests') }}: - stage: finalize @@ -967,11 +950,11 @@ stages: jobs: - job: finalize # Finalize Build displayName: Finalize Build - pool: ${{ parameters.VM_IMAGE_HOST.pool }} + pool: ${{ parameters.buildAgentHost.pool }} steps: - checkout: none - - template: azure-templates-variables.yml - - template: azure-templates-github-status.yml + - template: /scripts/azure-templates-variables.yml@self + - template: /scripts/azure-templates-github-status.yml@self parameters: context: 'SkiaSharp-Tests' displayName: Update the final status for the tests pipeline diff --git a/scripts/azure-templates-wasm-matrix.yml b/scripts/azure-templates-wasm-matrix.yml index 52730c71e4..b4cb6a8817 100644 --- a/scripts/azure-templates-wasm-matrix.yml +++ b/scripts/azure-templates-wasm-matrix.yml @@ -2,26 +2,23 @@ parameters: artifactName: '' # the name of the artifact to merge this run into buildExternals: '' # the build number to download externals from buildPipelineType: false - vmImage: '' # the VM image + buildAgent: '' # the configuration for the build agent emscripten: [ ] jobs: - ${{ each version in parameters.emscripten }}: - - template: azure-templates-bootstrapper.yml + - template: /scripts/azure-templates-bootstrapper.yml@self parameters: name: native_wasm_${{ replace(version.displayName, '.', '_') }}_linux displayName: WASM (${{ version.displayName }}) buildExternals: ${{ parameters.buildExternals }} buildPipelineType: ${{ parameters.buildPipelineType }} - vmImage: ${{ parameters.vmImage }} + buildAgent: ${{ parameters.buildAgent }} docker: scripts/Docker/wasm target: externals-wasm dockerArgs: --build-arg EMSCRIPTEN_VERSION=${{ version.version }} additionalArgs: --emscriptenVersion=${{ version.version }} --emscriptenFeatures="${{ version.features }}" artifactName: ${{ parameters.artifactName }} - postBuildSteps: - - task: PublishBuildArtifacts@1 - displayName: Publish the native_wasm_linux artifacts - inputs: - artifactName: native_wasm_linux - pathToPublish: 'output' \ No newline at end of file + additionalArtifacts: + - name: native_wasm_linux + path: 'output' \ No newline at end of file