Skip to content

Script populate yaml #803

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 24, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 57 additions & 5 deletions build.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ param(
[switch]
$GenerateMatrixJson,

[Parameter(Mandatory, ParameterSetName="UpdateBuildYaml")]
[switch]
$UpdateBuildYaml,

[Parameter(ParameterSetName="GenerateMatrixJson")]
[switch]
$FullJson,
Expand Down Expand Up @@ -108,27 +112,31 @@ param(
$Version,

[Parameter(ParameterSetName="GenerateMatrixJson")]
[Parameter(ParameterSetName="UpdateBuildYaml")]
[Parameter(ParameterSetName="GenerateManifestLists")]
[Parameter(ParameterSetName="GenerateTagsYaml")]
[ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
[string]
$StableVersion,

[Parameter(ParameterSetName="GenerateMatrixJson")]
[Parameter(ParameterSetName="UpdateBuildYaml")]
[Parameter(ParameterSetName="GenerateManifestLists")]
[Parameter(ParameterSetName="GenerateTagsYaml")]
[ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
[string]
$LtsVersion,

[Parameter(ParameterSetName="GenerateManifestLists")]
[Parameter(ParameterSetName="UpdateBuildYaml")]
[Parameter(ParameterSetName="GenerateMatrixJson")]
[Parameter(ParameterSetName="GenerateTagsYaml")]
[ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
[string]
$PreviewVersion,

[Parameter(ParameterSetName="GenerateManifestLists")]
[Parameter(ParameterSetName="UpdateBuildYaml")]
[Parameter(ParameterSetName="GenerateMatrixJson")]
[Parameter(ParameterSetName="GenerateTagsYaml")]
[ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
Expand All @@ -147,12 +155,14 @@ param(
$ForcePesterInstall,

[Parameter(ParameterSetName="GenerateManifestLists")]
[Parameter(ParameterSetName="UpdateBuildYaml")]
[Parameter(ParameterSetName="GenerateMatrixJson")]
[string]
[ValidateSet('All','OnlyAcr','NoAcr')]
$Acr,

[Parameter(ParameterSetName="GenerateManifestLists")]
[Parameter(ParameterSetName="UpdateBuildYaml")]
[Parameter(ParameterSetName="GenerateMatrixJson")]
[string]
[ValidateSet('All','Linux','Windows')]
Expand Down Expand Up @@ -203,6 +213,7 @@ DynamicParam {
Add-ParameterAttribute -ParameterSetName 'GetTagsByChannel' -Attributes $channelAttributes
Add-ParameterAttribute -ParameterSetName 'DupeCheck' -Attributes $channelAttributes
Add-ParameterAttribute -ParameterSetName 'GenerateMatrixJson' -Attributes $channelAttributes -Mandatory $false
Add-ParameterAttribute -ParameterSetName "UpdateBuildYaml" -Attributes $channelAttributes -Mandatory $false
Add-ParameterAttribute -ParameterSetName 'GenerateTagsYaml' -Attributes $channelAttributes
Add-ParameterAttribute -ParameterSetName 'GenerateManifestLists' -Attributes $channelAttributes

Expand All @@ -227,7 +238,7 @@ Begin {

$ENV:DOCKER_BUILDKIT = 1

if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1)
if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'UpdateBuildYaml', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1)
{
throw "Multiple Channels are not supported in this parameter set"
}
Expand Down Expand Up @@ -404,7 +415,7 @@ End {
}
}
}
elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent) {
elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent) {
if($Acr -eq 'OnlyAcr' -and !$useAcr)
{
continue
Expand Down Expand Up @@ -437,7 +448,7 @@ End {

$osVersion = $allMeta.meta.osVersion
$manifestLists = $allMeta.meta.ManifestLists
if($osVersion -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent)
if($osVersion -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent)
{
if ($osVersion) {
$osVersion = $osVersion.replace('${fromTag}', $actualTagData.fromTag)
Expand Down Expand Up @@ -525,8 +536,8 @@ End {
Invoke-PesterWrapper -Script $testsPath -OutputFile $logPath -ExtraParams $extraParams
}

Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose
if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') {
Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose
if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') {
$dockerImagesToScan = ''
# print local image names
# used only with the -Build
Expand Down Expand Up @@ -665,6 +676,47 @@ End {
}
}

$channelsUsed = @{}
if ($UpdateBuildYaml.IsPresent) {
foreach ($repo in $tagGroups.Keys | Sort-Object) {
$channelGroups = $tagGroups.$repo | Group-Object -Property Channel
foreach($channelGroup in $channelGroups)
{
$channelName = $channelGroup.Name
$ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci'
$channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml"

if (!$channelsUsed.Contains($channelName))
{
# Note: channelGroup contains entry for a channels' regular and channel's test-deps images.
# But, we only want 1 <channel>ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) so only populate start of yaml file once per channel.
$channelReleaseStageFileExists = Test-Path $channelReleaseStagePath
if ($channelReleaseStageFileExists)
{
Remove-Item -Path $channelReleaseStagePath
}

New-Item -Type File -Path $channelReleaseStagePath

Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath
$channelsUsed.Add($channelName, $channelGroup.Values)
}

$osGroups = $channelGroup.Group | Group-Object -Property os
foreach ($osGroup in $osGroups) {
$architectureGroups = $osGroup.Group | Group-Object -Property Architecture
foreach ($architectureGroup in $architectureGroups) {
# Note: For each image to be built (including test-deps images) the <channel>ReleaseStage.yml file needs to have a template call for the image.
foreach ($tag in $architectureGroup.Group | Sort-Object -Property dockerfile) {
Write-Verbose -Verbose "calling method to populate template call in yaml file for channel: $channelName for image: $($tag.Name)"
Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag
}
}
}
}
}
}

if ($GenerateManifestLists.IsPresent) {
$manifestLists = @()
$tags = @()
Expand Down
86 changes: 86 additions & 0 deletions tools/buildHelper/buildHelper.psm1
Original file line number Diff line number Diff line change
Expand Up @@ -1030,3 +1030,89 @@ Function ConvertTo-SortedDictionary {
}
return $sortedDictionary
}

function Get-StartOfYamlPopulated {
param(
[string]
$Channel,

[string]
$YamlFilePath
)

if (!$YamlFilePath)
{
throw "Yaml file $YamlFilePath provided as parameter cannot be found."
}

$doubleSpace = " "*2

Add-Content -Path $YamlFilePath -Value "parameters:"
Add-Content -Path $YamlFilePath -Value "- name: channel"
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: 'preview'"
Add-Content -Path $YamlFilePath -Value "- name: channelPath"
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: ''"
Add-Content -Path $YamlFilePath -Value "- name: vmImage"
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: PSMMSUbuntu20.04-Secure"
Add-Content -Path $YamlFilePath -Value "stages:"
Add-Content -Path $YamlFilePath -Value "- stage: StageGenerateBuild_$Channel"
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)dependsOn: ['StageResolveVersionandYaml']"
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)displayName: Build $Channel"
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)jobs:"
}

function Get-TemplatePopulatedYaml {
param(
[string]
$YamlFilePath,

[psobject]
$ImageInfo
)

if (!$YamlFilePath)
{
throw "Yaml file $YamlFilePath provided as parameter cannot be found."
}

$doubleSpace = " "*2
$fourSpace = " "*4
$sixSpace = " "*6

$imageName = $ImageInfo.Name
$artifactSuffix = $ImageInfo.Name.ToString().Replace("\", "_").Replace("-","_")
$architecture = $ImageInfo.Architecture
$poolOS = $ImageInfo.IsLinux ? "linux" : "windows"
$archBasedJobName = "Build_$($poolOS)_$($architecture)"

if ($architecture -eq "arm32")
{
$architecture = "arm64" # we need to use hostArchicture arm64 for pool for arm32
}

Add-Content -Path $YamlFilePath -Value "$($doubleSpace)- template: /.vsts-ci/releasePhase.yml@self"
Add-Content -Path $YamlFilePath -Value "$($fourSpace)parameters:"
Add-Content -Path $YamlFilePath -Value "$($sixSpace)archName: '$archBasedJobName'" # ie: Build_Linux_arm32
Add-Content -Path $YamlFilePath -Value "$($sixSpace)imageName: $imageName" # ie. imageName: alpine317\test-deps (since this differs from artifactSuffix for test-deps images only, we have a separate entry as the yaml param)
Add-Content -Path $YamlFilePath -Value "$($sixSpace)artifactSuffix: $artifactSuffix" # i.e artifactSuffix: alpine317_test_deps
Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolOS: '$poolOS'"
Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostArchitecture: '$architecture'"
if ($poolOS -eq "linux")
{
# only need to specify buildKitValue=1 for linux
Add-Content -Path $YamlFilePath -Value "$($sixSpace)buildKitValue: 1"
}
else
{
if ($imageName -contains "2022")
{
Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostVersion: '1ESWindows2022'"
Add-Content -Path $YamlFilePath -Value "$($sixSpace)windowsContainerImageValue: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'"
}

Add-Content -Path $YamlFilePath -Value "$($sixSpace)maxParallel: 3"
}

Add-Content -Path $YamlFilePath -Value "$($sixSpace)channel: `${{ parameters.channel }}"
Add-Content -Path $YamlFilePath -Value "$($sixSpace)channelPath: `${{ parameters.channelPath }}"
}
85 changes: 85 additions & 0 deletions tools/updateYamls/updateBuildYamls.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

param(
[string]
$StableVersion,

[string]
$PreviewVersion,

[string]
$LtsVersion
)

if (!($stableVersion -eq ""))
{
if ($stableVersion -notmatch '\d+\.\d+\.\d+$') {
throw "stable release tag is not for a stable build: '$stableVersion'"
}
}

if (!($previewVersion -eq ""))
{
if ($previewVersion -notmatch '\d+\.\d+\.\d+-(preview|rc)\.\d+$') {
throw "preview release tag is not for a preview build: '$previewVersion'"
}
}

if (!($ltsVersion -eq ""))
{
if ($ltsVersion -notmatch '\d+\.\d+\.\d+$') {
throw "lts release tag is not for a lts build: '$ltsVersion'"
}
}

function Update-ChannelReleaseStageYaml {
param(
[Parameter(Mandatory)]
[ValidateSet('stable', 'preview', 'lts')]
[string]
$Channel,

[string]
$Version
)

$toolsFolderPath = Split-Path -Parent $PSScriptRoot
$repoRoot = Split-Path -Parent $toolsFolderPath
$buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper'
$buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1'
Import-Module $buildHelperModulePath
$buildScriptPath = Join-Path -Path $repoRoot -ChildPath "build.ps1"
if (!($Version -eq ""))
{
Write-Verbose -Verbose "using $Channel version: $Version"
if ($Channel -eq "stable")
{
& $buildScriptPath -UpdateBuildYaml -Channel $Channel -StableVersion $Version -Verbose -Acr All -OsFilter All
}
elseif ($Channel -eq "preview")
{
& $buildScriptPath -UpdateBuildYaml -Channel $Channel -PreviewVersion $Version -Verbose -Acr All -OsFilter All
}
else #Channel lts
{
& $buildScriptPath -UpdateBuildYaml -Channel $Channel -LtsVersion $Version -Verbose -Acr All -OsFilter All
}
}
else
{
Write-Verbose -Verbose "using $Channel only"
& $buildScriptPath -UpdateBuildYaml -Channel $Channel -Verbose -Acr All -OsFilter All
}
}


# Update stableReleaseStage.yml
Update-ChannelReleaseStageYaml -Channel stable -Version $stableVersion

# Update previewReleaseStage.yml
Update-ChannelReleaseStageYaml -Channel preview -Version $previewVersion

# Update ltsReleaseStage.yml
Update-ChannelReleaseStageYaml -Channel lts -Version $ltsVersion

15 changes: 15 additions & 0 deletions tools/updateYamls/updateYamls.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Update Build Yamls

## Context

This `updateBuildYamls.ps1` script needs to be run before the build pipeline for PowerShell-Docker is kicked off. Running the script will produce a channel based yaml file, like &lt;channel&gt;ReleaseStage.yml for each channel. Then a PR must be created with these newly added/updated yaml files as the build will rely on them.

## Running the Script

To update the releaseStage.yml file for all the channels, run `./updateBuildYamls.ps1 -StableVersion <stableVersion> -PreviewVersion <previewVersion> -LtsVersion <ltsVersion>`.

If you want the channel versions from channels.json to be used, simply omit the -*Version parameters and run, `./updateBuildYamls.ps1`.

## Notes

The versions provided must match versioning syntax rules for stable and preview versions. Valid examples include 7.4.2 (stable) and v7.4.0-preview.5 (preview)