diff --git a/build.ps1 b/build.ps1 index b98e2a5fd..7f44828e3 100755 --- a/build.ps1 +++ b/build.ps1 @@ -34,6 +34,10 @@ param( [switch] $GenerateMatrixJson, + [Parameter(Mandatory, ParameterSetName="UpdateBuildYaml")] + [switch] + $UpdateBuildYaml, + [Parameter(ParameterSetName="GenerateMatrixJson")] [switch] $FullJson, @@ -108,6 +112,7 @@ param( $Version, [Parameter(ParameterSetName="GenerateMatrixJson")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateManifestLists")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -115,6 +120,7 @@ param( $StableVersion, [Parameter(ParameterSetName="GenerateMatrixJson")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateManifestLists")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -122,6 +128,7 @@ param( $LtsVersion, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -129,6 +136,7 @@ param( $PreviewVersion, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [Parameter(ParameterSetName="GenerateTagsYaml")] [ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')] @@ -147,12 +155,14 @@ param( $ForcePesterInstall, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [string] [ValidateSet('All','OnlyAcr','NoAcr')] $Acr, [Parameter(ParameterSetName="GenerateManifestLists")] + [Parameter(ParameterSetName="UpdateBuildYaml")] [Parameter(ParameterSetName="GenerateMatrixJson")] [string] [ValidateSet('All','Linux','Windows')] @@ -203,6 +213,7 @@ DynamicParam { Add-ParameterAttribute -ParameterSetName 'GetTagsByChannel' -Attributes $channelAttributes Add-ParameterAttribute -ParameterSetName 'DupeCheck' -Attributes $channelAttributes Add-ParameterAttribute -ParameterSetName 'GenerateMatrixJson' -Attributes $channelAttributes -Mandatory $false + Add-ParameterAttribute -ParameterSetName "UpdateBuildYaml" -Attributes $channelAttributes -Mandatory $false Add-ParameterAttribute -ParameterSetName 'GenerateTagsYaml' -Attributes $channelAttributes Add-ParameterAttribute -ParameterSetName 'GenerateManifestLists' -Attributes $channelAttributes @@ -227,7 +238,7 @@ Begin { $ENV:DOCKER_BUILDKIT = 1 - if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1) + if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'UpdateBuildYaml', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1) { throw "Multiple Channels are not supported in this parameter set" } @@ -404,7 +415,7 @@ End { } } } - elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent) { + elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent) { if($Acr -eq 'OnlyAcr' -and !$useAcr) { continue @@ -437,7 +448,7 @@ End { $osVersion = $allMeta.meta.osVersion $manifestLists = $allMeta.meta.ManifestLists - if($osVersion -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent) + if($osVersion -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent) { if ($osVersion) { $osVersion = $osVersion.replace('${fromTag}', $actualTagData.fromTag) @@ -525,8 +536,8 @@ End { Invoke-PesterWrapper -Script $testsPath -OutputFile $logPath -ExtraParams $extraParams } - Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose - if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') { + Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose + if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') { $dockerImagesToScan = '' # print local image names # used only with the -Build @@ -665,6 +676,47 @@ End { } } + $channelsUsed = @{} + if ($UpdateBuildYaml.IsPresent) { + foreach ($repo in $tagGroups.Keys | Sort-Object) { + $channelGroups = $tagGroups.$repo | Group-Object -Property Channel + foreach($channelGroup in $channelGroups) + { + $channelName = $channelGroup.Name + $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' + $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" + + if (!$channelsUsed.Contains($channelName)) + { + # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. + # But, we only want 1 ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) so only populate start of yaml file once per channel. + $channelReleaseStageFileExists = Test-Path $channelReleaseStagePath + if ($channelReleaseStageFileExists) + { + Remove-Item -Path $channelReleaseStagePath + } + + New-Item -Type File -Path $channelReleaseStagePath + + Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath + $channelsUsed.Add($channelName, $channelGroup.Values) + } + + $osGroups = $channelGroup.Group | Group-Object -Property os + foreach ($osGroup in $osGroups) { + $architectureGroups = $osGroup.Group | Group-Object -Property Architecture + foreach ($architectureGroup in $architectureGroups) { + # Note: For each image to be built (including test-deps images) the ReleaseStage.yml file needs to have a template call for the image. + foreach ($tag in $architectureGroup.Group | Sort-Object -Property dockerfile) { + Write-Verbose -Verbose "calling method to populate template call in yaml file for channel: $channelName for image: $($tag.Name)" + Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag + } + } + } + } + } + } + if ($GenerateManifestLists.IsPresent) { $manifestLists = @() $tags = @() diff --git a/tools/buildHelper/buildHelper.psm1 b/tools/buildHelper/buildHelper.psm1 index 1d63eb98c..85bc26d1b 100644 --- a/tools/buildHelper/buildHelper.psm1 +++ b/tools/buildHelper/buildHelper.psm1 @@ -1030,3 +1030,89 @@ Function ConvertTo-SortedDictionary { } return $sortedDictionary } + +function Get-StartOfYamlPopulated { + param( + [string] + $Channel, + + [string] + $YamlFilePath + ) + + if (!$YamlFilePath) + { + throw "Yaml file $YamlFilePath provided as parameter cannot be found." + } + + $doubleSpace = " "*2 + + Add-Content -Path $YamlFilePath -Value "parameters:" + Add-Content -Path $YamlFilePath -Value "- name: channel" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: 'preview'" + Add-Content -Path $YamlFilePath -Value "- name: channelPath" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: ''" + Add-Content -Path $YamlFilePath -Value "- name: vmImage" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: PSMMSUbuntu20.04-Secure" + Add-Content -Path $YamlFilePath -Value "stages:" + Add-Content -Path $YamlFilePath -Value "- stage: StageGenerateBuild_$Channel" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)dependsOn: ['StageResolveVersionandYaml']" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)displayName: Build $Channel" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)jobs:" +} + +function Get-TemplatePopulatedYaml { + param( + [string] + $YamlFilePath, + + [psobject] + $ImageInfo + ) + + if (!$YamlFilePath) + { + throw "Yaml file $YamlFilePath provided as parameter cannot be found." + } + + $doubleSpace = " "*2 + $fourSpace = " "*4 + $sixSpace = " "*6 + + $imageName = $ImageInfo.Name + $artifactSuffix = $ImageInfo.Name.ToString().Replace("\", "_").Replace("-","_") + $architecture = $ImageInfo.Architecture + $poolOS = $ImageInfo.IsLinux ? "linux" : "windows" + $archBasedJobName = "Build_$($poolOS)_$($architecture)" + + if ($architecture -eq "arm32") + { + $architecture = "arm64" # we need to use hostArchicture arm64 for pool for arm32 + } + + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)- template: /.vsts-ci/releasePhase.yml@self" + Add-Content -Path $YamlFilePath -Value "$($fourSpace)parameters:" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)archName: '$archBasedJobName'" # ie: Build_Linux_arm32 + Add-Content -Path $YamlFilePath -Value "$($sixSpace)imageName: $imageName" # ie. imageName: alpine317\test-deps (since this differs from artifactSuffix for test-deps images only, we have a separate entry as the yaml param) + Add-Content -Path $YamlFilePath -Value "$($sixSpace)artifactSuffix: $artifactSuffix" # i.e artifactSuffix: alpine317_test_deps + Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolOS: '$poolOS'" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostArchitecture: '$architecture'" + if ($poolOS -eq "linux") + { + # only need to specify buildKitValue=1 for linux + Add-Content -Path $YamlFilePath -Value "$($sixSpace)buildKitValue: 1" + } + else + { + if ($imageName -contains "2022") + { + Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostVersion: '1ESWindows2022'" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)windowsContainerImageValue: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'" + } + + Add-Content -Path $YamlFilePath -Value "$($sixSpace)maxParallel: 3" + } + + Add-Content -Path $YamlFilePath -Value "$($sixSpace)channel: `${{ parameters.channel }}" + Add-Content -Path $YamlFilePath -Value "$($sixSpace)channelPath: `${{ parameters.channelPath }}" +} diff --git a/tools/updateYamls/updateBuildYamls.ps1 b/tools/updateYamls/updateBuildYamls.ps1 new file mode 100644 index 000000000..ebbb51853 --- /dev/null +++ b/tools/updateYamls/updateBuildYamls.ps1 @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +param( + [string] + $StableVersion, + + [string] + $PreviewVersion, + + [string] + $LtsVersion +) + +if (!($stableVersion -eq "")) +{ + if ($stableVersion -notmatch '\d+\.\d+\.\d+$') { + throw "stable release tag is not for a stable build: '$stableVersion'" + } +} + +if (!($previewVersion -eq "")) +{ + if ($previewVersion -notmatch '\d+\.\d+\.\d+-(preview|rc)\.\d+$') { + throw "preview release tag is not for a preview build: '$previewVersion'" + } +} + +if (!($ltsVersion -eq "")) +{ + if ($ltsVersion -notmatch '\d+\.\d+\.\d+$') { + throw "lts release tag is not for a lts build: '$ltsVersion'" + } +} + +function Update-ChannelReleaseStageYaml { + param( + [Parameter(Mandatory)] + [ValidateSet('stable', 'preview', 'lts')] + [string] + $Channel, + + [string] + $Version + ) + + $toolsFolderPath = Split-Path -Parent $PSScriptRoot + $repoRoot = Split-Path -Parent $toolsFolderPath + $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' + $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' + Import-Module $buildHelperModulePath + $buildScriptPath = Join-Path -Path $repoRoot -ChildPath "build.ps1" + if (!($Version -eq "")) + { + Write-Verbose -Verbose "using $Channel version: $Version" + if ($Channel -eq "stable") + { + & $buildScriptPath -UpdateBuildYaml -Channel $Channel -StableVersion $Version -Verbose -Acr All -OsFilter All + } + elseif ($Channel -eq "preview") + { + & $buildScriptPath -UpdateBuildYaml -Channel $Channel -PreviewVersion $Version -Verbose -Acr All -OsFilter All + } + else #Channel lts + { + & $buildScriptPath -UpdateBuildYaml -Channel $Channel -LtsVersion $Version -Verbose -Acr All -OsFilter All + } + } + else + { + Write-Verbose -Verbose "using $Channel only" + & $buildScriptPath -UpdateBuildYaml -Channel $Channel -Verbose -Acr All -OsFilter All + } +} + + +# Update stableReleaseStage.yml +Update-ChannelReleaseStageYaml -Channel stable -Version $stableVersion + +# Update previewReleaseStage.yml +Update-ChannelReleaseStageYaml -Channel preview -Version $previewVersion + +# Update ltsReleaseStage.yml +Update-ChannelReleaseStageYaml -Channel lts -Version $ltsVersion + diff --git a/tools/updateYamls/updateYamls.md b/tools/updateYamls/updateYamls.md new file mode 100644 index 000000000..83cb3409e --- /dev/null +++ b/tools/updateYamls/updateYamls.md @@ -0,0 +1,15 @@ +# Update Build Yamls + +## Context + +This `updateBuildYamls.ps1` script needs to be run before the build pipeline for PowerShell-Docker is kicked off. Running the script will produce a channel based yaml file, like <channel>ReleaseStage.yml for each channel. Then a PR must be created with these newly added/updated yaml files as the build will rely on them. + +## Running the Script + +To update the releaseStage.yml file for all the channels, run `./updateBuildYamls.ps1 -StableVersion -PreviewVersion -LtsVersion `. + +If you want the channel versions from channels.json to be used, simply omit the -*Version parameters and run, `./updateBuildYamls.ps1`. + +## Notes + +The versions provided must match versioning syntax rules for stable and preview versions. Valid examples include 7.4.2 (stable) and v7.4.0-preview.5 (preview)