diff --git a/.vsts-ci/templatesReleasePipeline/ltsReleaseGetBuiltImages.yml b/.vsts-ci/templatesReleasePipeline/ltsReleaseGetBuiltImages.yml new file mode 100644 index 000000000..b50ceec46 --- /dev/null +++ b/.vsts-ci/templatesReleasePipeline/ltsReleaseGetBuiltImages.yml @@ -0,0 +1,11 @@ +parameters: +- name: channel + default: 'lts' +- name: artifactNames + type: object + default: ['drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_alpine316','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_alpine317','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_debian11','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_mariner2','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_ubi8','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_ubuntu2004','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_ubuntu2204','drop_StageGenerateBuild_lts_Job_Build_Build_linux_arm32_ubuntu2004_arm32v7','drop_StageGenerateBuild_lts_Job_Build_Build_linux_arm32_ubuntu2204_arm32v7','drop_StageGenerateBuild_lts_Job_Build_Build_windows_amd64_nanoserver1809','drop_StageGenerateBuild_lts_Job_Build_Build_windows_amd64_nanoserver2022','drop_StageGenerateBuild_lts_Job_Build_Build_windows_amd64_windowsserver2022','drop_StageGenerateBuild_lts_Job_Build_Build_windows_amd64_windowsservercore2022','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_alpine316_test_deps','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_alpine317_test_deps','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_debian11_test_deps','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_ubi8_test_deps','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_ubuntu2004_test_deps','drop_StageGenerateBuild_lts_Job_Build_Build_linux_amd64_ubuntu2204_test_deps'] +stages: +- template: /.vsts-ci/templatesReleasePipeline/releaseTestPrepStage.yml@self + parameters: + channel: ${{ parameters.channel }} + artifactNames: ${{ parameters.artifactNames }} diff --git a/.vsts-ci/templatesReleasePipeline/previewReleaseGetBuiltImages.yml b/.vsts-ci/templatesReleasePipeline/previewReleaseGetBuiltImages.yml new file mode 100644 index 000000000..74c01639a --- /dev/null +++ b/.vsts-ci/templatesReleasePipeline/previewReleaseGetBuiltImages.yml @@ -0,0 +1,11 @@ +parameters: +- name: channel + default: 'preview' +- name: artifactNames + type: object + default: ['drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_alpine316','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_alpine317','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_debian11','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_debian12','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_mariner2','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_ubi8','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_ubi9','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_ubuntu2004','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_ubuntu2204','drop_StageGenerateBuild_preview_Job_Build_Build_linux_arm32_ubuntu2004_arm32v7','drop_StageGenerateBuild_preview_Job_Build_Build_linux_arm32_ubuntu2204_arm32v7','drop_StageGenerateBuild_preview_Job_Build_Build_linux_arm64_mariner2_arm64','drop_StageGenerateBuild_preview_Job_Build_Build_windows_amd64_windowsserver2022','drop_StageGenerateBuild_preview_Job_Build_Build_windows_amd64_windowsservercore2022','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_alpine316_test_deps','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_alpine317_test_deps','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_debian11_test_deps','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_debian12_test_deps','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_ubi8_test_deps','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_ubi9_test_deps','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_ubuntu2004_test_deps','drop_StageGenerateBuild_preview_Job_Build_Build_linux_amd64_ubuntu2204_test_deps'] +stages: +- template: /.vsts-ci/templatesReleasePipeline/releaseTestPrepStage.yml@self + parameters: + channel: ${{ parameters.channel }} + artifactNames: ${{ parameters.artifactNames }} diff --git a/.vsts-ci/templatesReleasePipeline/stableReleaseGetBuiltImages.yml b/.vsts-ci/templatesReleasePipeline/stableReleaseGetBuiltImages.yml new file mode 100644 index 000000000..4d6de49f7 --- /dev/null +++ b/.vsts-ci/templatesReleasePipeline/stableReleaseGetBuiltImages.yml @@ -0,0 +1,11 @@ +parameters: +- name: channel + default: 'stable' +- name: artifactNames + type: object + default: ['drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_alpine316','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_alpine317','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_debian11','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_debian12','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_mariner2','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_ubi8','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_ubi9','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_ubuntu2004','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_ubuntu2204','drop_StageGenerateBuild_stable_Job_Build_Build_linux_arm32_ubuntu2004_arm32v7','drop_StageGenerateBuild_stable_Job_Build_Build_linux_arm32_ubuntu2204_arm32v7','drop_StageGenerateBuild_stable_Job_Build_Build_linux_arm64_mariner2_arm64','drop_StageGenerateBuild_stable_Job_Build_Build_windows_amd64_windowsserver2022','drop_StageGenerateBuild_stable_Job_Build_Build_windows_amd64_windowsservercore2022','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_alpine316_test_deps','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_alpine317_test_deps','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_debian11_test_deps','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_debian12_test_deps','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_ubi8_test_deps','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_ubi9_test_deps','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_ubuntu2004_test_deps','drop_StageGenerateBuild_stable_Job_Build_Build_linux_amd64_ubuntu2204_test_deps'] +stages: +- template: /.vsts-ci/templatesReleasePipeline/releaseTestPrepStage.yml@self + parameters: + channel: ${{ parameters.channel }} + artifactNames: ${{ parameters.artifactNames }} diff --git a/build.ps1 b/build.ps1 index 514d8d65c..73f2dc3fd 100755 --- a/build.ps1 +++ b/build.ps1 @@ -708,6 +708,8 @@ End { } $channelsUsed = @{} + $releaseChannelsUsed = @{} + $emptyArr = @() if ($UpdateBuildYaml.IsPresent) { foreach ($repo in $tagGroups.Keys | Sort-Object) { $channelGroups = $tagGroups.$repo | Group-Object -Property Channel @@ -715,8 +717,13 @@ End { { $channelName = $channelGroup.Name $ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci' + $releaseTemplatesFolder = Join-Path -Path $ciFolder -ChildPath "templatesReleasePipeline" + # Build yaml file path $channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml" + # Release yaml file path + $channelReleaseOperationYamlPath = Join-Path -Path $releaseTemplatesFolder -ChildPath "$($channelName)ReleaseGetBuiltImages.yml" + if (!$channelsUsed.Contains($channelName)) { # Note: channelGroup contains entry for a channels' regular and channel's test-deps images. @@ -733,6 +740,19 @@ End { $channelsUsed.Add($channelName, $channelGroup.Values) } + if (!$releaseChannelsUsed.Contains($channelName)) + { + # For release operation yaml: + $channelReleaseOperationYamlFileExists = Test-Path $channelReleaseOperationYamlPath + if ($channelReleaseOperationYamlFileExists) + { + Remove-Item -Path $channelReleaseOperationYamlPath + } + + New-Item -Type File -Path $channelReleaseOperationYamlPath + $releaseChannelsUsed.Add($channelName, $emptyArr) + } + $osGroups = $channelGroup.Group | Group-Object -Property os foreach ($osGroup in $osGroups) { $architectureGroups = $osGroup.Group | Group-Object -Property Architecture @@ -741,11 +761,21 @@ End { foreach ($tag in $architectureGroup.Group | Sort-Object -Property dockerfile) { Write-Verbose -Verbose "calling method to populate template call in yaml file for channel: $channelName for image: $($tag.Name)" Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag + + # for release yaml, need to collect tag object (containing metadata for each image) + $releaseChannelsUsed[$channelName] += $tag } } } } } + + $channelsFound = $releaseChannelsUsed.Keys + foreach ($channelKey in $channelsFound) + { + $imgInfoObjArr = $releaseChannelsUsed[$channelKey] + Get-ReleaseYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseOperationYamlPath -ImageInfoObjects $imgInfoObjArr + } } if ($GenerateManifestLists.IsPresent) { diff --git a/tools/buildHelper/buildHelper.psm1 b/tools/buildHelper/buildHelper.psm1 index 6776dd308..7bcd78be8 100644 --- a/tools/buildHelper/buildHelper.psm1 +++ b/tools/buildHelper/buildHelper.psm1 @@ -1244,3 +1244,68 @@ function Get-TemplatePopulatedYaml { Add-Content -Path $YamlFilePath -Value "$($sixSpace)channel: `${{ parameters.channel }}" Add-Content -Path $YamlFilePath -Value "$($sixSpace)channelPath: `${{ parameters.channelPath }}" } + +function Get-ReleaseYamlPopulated { + param( + [string] + $Channel, + + [string] + $YamlFilePath, + + [object[]] + $ImageInfoObjects + ) + + if (!$YamlFilePath) + { + throw "Yaml file $YamlFilePath provided as parameter cannot be found." + } + + $defaultArtifactsValue = Get-DefaultArtifactNamesString -Channel $Channel -ImageInfoObjects $ImageInfoObjects + if ($defaultArtifactsValue -eq "[]") + { + throw "Default artifact names string $defaultArtifactsValue was empty." + } + + $doubleSpace = " "*2 + $fourSpace = " "*4 + + Add-Content -Path $YamlFilePath -Value "parameters:" + Add-Content -Path $YamlFilePath -Value "- name: channel" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: '$Channel'" + Add-Content -Path $YamlFilePath -Value "- name: artifactNames" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)type: object" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: $defaultArtifactsValue" + Add-Content -Path $YamlFilePath -Value "stages:" + Add-Content -Path $YamlFilePath -Value "- template: /.vsts-ci/templatesReleasePipeline/releaseTestPrepStage.yml@self" + Add-Content -Path $YamlFilePath -Value "$($doubleSpace)parameters:" + Add-Content -Path $YamlFilePath -Value "$($fourSpace)channel: `${{ parameters.channel }}" + Add-Content -Path $YamlFilePath -Value "$($fourSpace)artifactNames: `${{ parameters.artifactNames }}" +} + +function Get-DefaultArtifactNamesString { + param( + [string] + $Channel, + + [object[]] + $ImageInfoObjects + ) + + $defaultArtifactString = "[" + foreach ($img in $ImageInfoObjects) + { + $architecture = $img.Architecture + $poolOS = $img.IsLinux ? "linux" : "windows" + $archBasedJobName = "Build_$($poolOS)_$($architecture)" + $artifactSuffix = $img.Name.ToString().Replace("\", "_").Replace("-","_").Replace(".","") + + $joiningDelimeter = $defaultArtifactString.Length -gt 1 ? "," : "" + $artifactName = "$joiningDelimeter" + "'" + "drop_StageGenerateBuild_$($Channel)_Job_Build_" + $archBasedJobName + "_$artifactSuffix" + "'" + $defaultArtifactString += $artifactName + } + + $defaultArtifactString += "]" + return $defaultArtifactString +} diff --git a/tools/updateYamls/updateYamls.md b/tools/updateYamls/updateYamls.md index 83cb3409e..f5e4e421d 100644 --- a/tools/updateYamls/updateYamls.md +++ b/tools/updateYamls/updateYamls.md @@ -2,13 +2,14 @@ ## Context -This `updateBuildYamls.ps1` script needs to be run before the build pipeline for PowerShell-Docker is kicked off. Running the script will produce a channel based yaml file, like <channel>ReleaseStage.yml for each channel. Then a PR must be created with these newly added/updated yaml files as the build will rely on them. +This `updateBuildYamls.ps1` script needs to be run before the build and release pipelines for PowerShell-Docker are kicked off. Running the script will produce channel-based yaml files, like <channel>ReleaseStage.yml for build and templatesReleasePipeline/<channel>ReleaseGetBuiltImages.yml for release, for each channel. Then a PR must be created with these newly added/updated yaml files as the build and release will rely on them. ## Running the Script -To update the releaseStage.yml file for all the channels, run `./updateBuildYamls.ps1 -StableVersion -PreviewVersion -LtsVersion `. +To update the <channel>releaseStage.yml and templatesReleasePipeline/<channel>ReleaseGetBuiltImages.yml files for all the channels, run: + `./updateBuildYamls.ps1 -StableVersion -PreviewVersion -LtsVersion `. -If you want the channel versions from channels.json to be used, simply omit the -*Version parameters and run, `./updateBuildYamls.ps1`. +If you want the channel versions from channels.json to be used, simply omit the -*Version parameters and run: `./updateBuildYamls.ps1`. ## Notes