Skip to content

Commit 1fce34a

Browse files
anamnavianamnavi
authored
Script populate yaml (#803)
* add script and markdown file * fix typo in md file * add copyright header
1 parent 744090c commit 1fce34a

File tree

4 files changed

+243
-5
lines changed

4 files changed

+243
-5
lines changed

‎build.ps1

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ param(
3434
[switch]
3535
$GenerateMatrixJson,
3636

37+
[Parameter(Mandatory, ParameterSetName="UpdateBuildYaml")]
38+
[switch]
39+
$UpdateBuildYaml,
40+
3741
[Parameter(ParameterSetName="GenerateMatrixJson")]
3842
[switch]
3943
$FullJson,
@@ -108,27 +112,31 @@ param(
108112
$Version,
109113

110114
[Parameter(ParameterSetName="GenerateMatrixJson")]
115+
[Parameter(ParameterSetName="UpdateBuildYaml")]
111116
[Parameter(ParameterSetName="GenerateManifestLists")]
112117
[Parameter(ParameterSetName="GenerateTagsYaml")]
113118
[ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
114119
[string]
115120
$StableVersion,
116121

117122
[Parameter(ParameterSetName="GenerateMatrixJson")]
123+
[Parameter(ParameterSetName="UpdateBuildYaml")]
118124
[Parameter(ParameterSetName="GenerateManifestLists")]
119125
[Parameter(ParameterSetName="GenerateTagsYaml")]
120126
[ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
121127
[string]
122128
$LtsVersion,
123129

124130
[Parameter(ParameterSetName="GenerateManifestLists")]
131+
[Parameter(ParameterSetName="UpdateBuildYaml")]
125132
[Parameter(ParameterSetName="GenerateMatrixJson")]
126133
[Parameter(ParameterSetName="GenerateTagsYaml")]
127134
[ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
128135
[string]
129136
$PreviewVersion,
130137

131138
[Parameter(ParameterSetName="GenerateManifestLists")]
139+
[Parameter(ParameterSetName="UpdateBuildYaml")]
132140
[Parameter(ParameterSetName="GenerateMatrixJson")]
133141
[Parameter(ParameterSetName="GenerateTagsYaml")]
134142
[ValidatePattern('(\d+\.){2}\d(-\w+(\.\d+)?)?')]
@@ -147,12 +155,14 @@ param(
147155
$ForcePesterInstall,
148156

149157
[Parameter(ParameterSetName="GenerateManifestLists")]
158+
[Parameter(ParameterSetName="UpdateBuildYaml")]
150159
[Parameter(ParameterSetName="GenerateMatrixJson")]
151160
[string]
152161
[ValidateSet('All','OnlyAcr','NoAcr')]
153162
$Acr,
154163

155164
[Parameter(ParameterSetName="GenerateManifestLists")]
165+
[Parameter(ParameterSetName="UpdateBuildYaml")]
156166
[Parameter(ParameterSetName="GenerateMatrixJson")]
157167
[string]
158168
[ValidateSet('All','Linux','Windows')]
@@ -203,6 +213,7 @@ DynamicParam {
203213
Add-ParameterAttribute -ParameterSetName 'GetTagsByChannel' -Attributes $channelAttributes
204214
Add-ParameterAttribute -ParameterSetName 'DupeCheck' -Attributes $channelAttributes
205215
Add-ParameterAttribute -ParameterSetName 'GenerateMatrixJson' -Attributes $channelAttributes -Mandatory $false
216+
Add-ParameterAttribute -ParameterSetName "UpdateBuildYaml" -Attributes $channelAttributes -Mandatory $false
206217
Add-ParameterAttribute -ParameterSetName 'GenerateTagsYaml' -Attributes $channelAttributes
207218
Add-ParameterAttribute -ParameterSetName 'GenerateManifestLists' -Attributes $channelAttributes
208219

@@ -227,7 +238,7 @@ Begin {
227238

228239
$ENV:DOCKER_BUILDKIT = 1
229240

230-
if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1)
241+
if ($PSCmdlet.ParameterSetName -notin 'GenerateMatrixJson', 'UpdateBuildYaml', 'GenerateTagsYaml', 'DupeCheck', 'GenerateManifestLists' -and $Channel.Count -gt 1)
231242
{
232243
throw "Multiple Channels are not supported in this parameter set"
233244
}
@@ -404,7 +415,7 @@ End {
404415
}
405416
}
406417
}
407-
elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent) {
418+
elseif ($GenerateTagsYaml.IsPresent -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent) {
408419
if($Acr -eq 'OnlyAcr' -and !$useAcr)
409420
{
410421
continue
@@ -437,7 +448,7 @@ End {
437448

438449
$osVersion = $allMeta.meta.osVersion
439450
$manifestLists = $allMeta.meta.ManifestLists
440-
if($osVersion -or $GenerateMatrixJson.IsPresent -or $GenerateManifestLists.IsPresent)
451+
if($osVersion -or $GenerateMatrixJson.IsPresent -or $UpdateBuildYaml.IsPresent -or $GenerateManifestLists.IsPresent)
441452
{
442453
if ($osVersion) {
443454
$osVersion = $osVersion.replace('${fromTag}', $actualTagData.fromTag)
@@ -525,8 +536,8 @@ End {
525536
Invoke-PesterWrapper -Script $testsPath -OutputFile $logPath -ExtraParams $extraParams
526537
}
527538

528-
Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose
529-
if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') {
539+
Write-Verbose "!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and '$($PSCmdlet.ParameterSetName)' -notlike '*All'" -Verbose
540+
if (!$GenerateTagsYaml -and !$GenerateMatrixJson -and !$UpdateBuildYaml -and !$GenerateManifestLists -and $PSCmdlet.ParameterSetName -notlike '*All') {
530541
$dockerImagesToScan = ''
531542
# print local image names
532543
# used only with the -Build
@@ -665,6 +676,47 @@ End {
665676
}
666677
}
667678

679+
$channelsUsed = @{}
680+
if ($UpdateBuildYaml.IsPresent) {
681+
foreach ($repo in $tagGroups.Keys | Sort-Object) {
682+
$channelGroups = $tagGroups.$repo | Group-Object -Property Channel
683+
foreach($channelGroup in $channelGroups)
684+
{
685+
$channelName = $channelGroup.Name
686+
$ciFolder = Join-Path -Path $PSScriptRoot -ChildPath '.vsts-ci'
687+
$channelReleaseStagePath = Join-Path -Path $ciFolder -ChildPath "$($channelName)ReleaseStage.yml"
688+
689+
if (!$channelsUsed.Contains($channelName))
690+
{
691+
# Note: channelGroup contains entry for a channels' regular and channel's test-deps images.
692+
# But, we only want 1 <channel>ReleaseStage.yml file to be created per channel (ie 1 stableReleaseStage.yml) so only populate start of yaml file once per channel.
693+
$channelReleaseStageFileExists = Test-Path $channelReleaseStagePath
694+
if ($channelReleaseStageFileExists)
695+
{
696+
Remove-Item -Path $channelReleaseStagePath
697+
}
698+
699+
New-Item -Type File -Path $channelReleaseStagePath
700+
701+
Get-StartOfYamlPopulated -Channel $channelName -YamlFilePath $channelReleaseStagePath
702+
$channelsUsed.Add($channelName, $channelGroup.Values)
703+
}
704+
705+
$osGroups = $channelGroup.Group | Group-Object -Property os
706+
foreach ($osGroup in $osGroups) {
707+
$architectureGroups = $osGroup.Group | Group-Object -Property Architecture
708+
foreach ($architectureGroup in $architectureGroups) {
709+
# Note: For each image to be built (including test-deps images) the <channel>ReleaseStage.yml file needs to have a template call for the image.
710+
foreach ($tag in $architectureGroup.Group | Sort-Object -Property dockerfile) {
711+
Write-Verbose -Verbose "calling method to populate template call in yaml file for channel: $channelName for image: $($tag.Name)"
712+
Get-TemplatePopulatedYaml -YamlFilePath $channelReleaseStagePath -ImageInfo $tag
713+
}
714+
}
715+
}
716+
}
717+
}
718+
}
719+
668720
if ($GenerateManifestLists.IsPresent) {
669721
$manifestLists = @()
670722
$tags = @()

‎tools/buildHelper/buildHelper.psm1

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1030,3 +1030,89 @@ Function ConvertTo-SortedDictionary {
10301030
}
10311031
return $sortedDictionary
10321032
}
1033+
1034+
function Get-StartOfYamlPopulated {
1035+
param(
1036+
[string]
1037+
$Channel,
1038+
1039+
[string]
1040+
$YamlFilePath
1041+
)
1042+
1043+
if (!$YamlFilePath)
1044+
{
1045+
throw "Yaml file $YamlFilePath provided as parameter cannot be found."
1046+
}
1047+
1048+
$doubleSpace = " "*2
1049+
1050+
Add-Content -Path $YamlFilePath -Value "parameters:"
1051+
Add-Content -Path $YamlFilePath -Value "- name: channel"
1052+
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: 'preview'"
1053+
Add-Content -Path $YamlFilePath -Value "- name: channelPath"
1054+
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: ''"
1055+
Add-Content -Path $YamlFilePath -Value "- name: vmImage"
1056+
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)default: PSMMSUbuntu20.04-Secure"
1057+
Add-Content -Path $YamlFilePath -Value "stages:"
1058+
Add-Content -Path $YamlFilePath -Value "- stage: StageGenerateBuild_$Channel"
1059+
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)dependsOn: ['StageResolveVersionandYaml']"
1060+
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)displayName: Build $Channel"
1061+
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)jobs:"
1062+
}
1063+
1064+
function Get-TemplatePopulatedYaml {
1065+
param(
1066+
[string]
1067+
$YamlFilePath,
1068+
1069+
[psobject]
1070+
$ImageInfo
1071+
)
1072+
1073+
if (!$YamlFilePath)
1074+
{
1075+
throw "Yaml file $YamlFilePath provided as parameter cannot be found."
1076+
}
1077+
1078+
$doubleSpace = " "*2
1079+
$fourSpace = " "*4
1080+
$sixSpace = " "*6
1081+
1082+
$imageName = $ImageInfo.Name
1083+
$artifactSuffix = $ImageInfo.Name.ToString().Replace("\", "_").Replace("-","_")
1084+
$architecture = $ImageInfo.Architecture
1085+
$poolOS = $ImageInfo.IsLinux ? "linux" : "windows"
1086+
$archBasedJobName = "Build_$($poolOS)_$($architecture)"
1087+
1088+
if ($architecture -eq "arm32")
1089+
{
1090+
$architecture = "arm64" # we need to use hostArchicture arm64 for pool for arm32
1091+
}
1092+
1093+
Add-Content -Path $YamlFilePath -Value "$($doubleSpace)- template: /.vsts-ci/releasePhase.yml@self"
1094+
Add-Content -Path $YamlFilePath -Value "$($fourSpace)parameters:"
1095+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)archName: '$archBasedJobName'" # ie: Build_Linux_arm32
1096+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)imageName: $imageName" # ie. imageName: alpine317\test-deps (since this differs from artifactSuffix for test-deps images only, we have a separate entry as the yaml param)
1097+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)artifactSuffix: $artifactSuffix" # i.e artifactSuffix: alpine317_test_deps
1098+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolOS: '$poolOS'"
1099+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostArchitecture: '$architecture'"
1100+
if ($poolOS -eq "linux")
1101+
{
1102+
# only need to specify buildKitValue=1 for linux
1103+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)buildKitValue: 1"
1104+
}
1105+
else
1106+
{
1107+
if ($imageName -contains "2022")
1108+
{
1109+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)poolHostVersion: '1ESWindows2022'"
1110+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)windowsContainerImageValue: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'"
1111+
}
1112+
1113+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)maxParallel: 3"
1114+
}
1115+
1116+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)channel: `${{ parameters.channel }}"
1117+
Add-Content -Path $YamlFilePath -Value "$($sixSpace)channelPath: `${{ parameters.channelPath }}"
1118+
}

‎tools/updateYamls/updateBuildYamls.ps1

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# Licensed under the MIT License.
3+
4+
param(
5+
[string]
6+
$StableVersion,
7+
8+
[string]
9+
$PreviewVersion,
10+
11+
[string]
12+
$LtsVersion
13+
)
14+
15+
if (!($stableVersion -eq ""))
16+
{
17+
if ($stableVersion -notmatch '\d+\.\d+\.\d+$') {
18+
throw "stable release tag is not for a stable build: '$stableVersion'"
19+
}
20+
}
21+
22+
if (!($previewVersion -eq ""))
23+
{
24+
if ($previewVersion -notmatch '\d+\.\d+\.\d+-(preview|rc)\.\d+$') {
25+
throw "preview release tag is not for a preview build: '$previewVersion'"
26+
}
27+
}
28+
29+
if (!($ltsVersion -eq ""))
30+
{
31+
if ($ltsVersion -notmatch '\d+\.\d+\.\d+$') {
32+
throw "lts release tag is not for a lts build: '$ltsVersion'"
33+
}
34+
}
35+
36+
function Update-ChannelReleaseStageYaml {
37+
param(
38+
[Parameter(Mandatory)]
39+
[ValidateSet('stable', 'preview', 'lts')]
40+
[string]
41+
$Channel,
42+
43+
[string]
44+
$Version
45+
)
46+
47+
$toolsFolderPath = Split-Path -Parent $PSScriptRoot
48+
$repoRoot = Split-Path -Parent $toolsFolderPath
49+
$buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper'
50+
$buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1'
51+
Import-Module $buildHelperModulePath
52+
$buildScriptPath = Join-Path -Path $repoRoot -ChildPath "build.ps1"
53+
if (!($Version -eq ""))
54+
{
55+
Write-Verbose -Verbose "using $Channel version: $Version"
56+
if ($Channel -eq "stable")
57+
{
58+
& $buildScriptPath -UpdateBuildYaml -Channel $Channel -StableVersion $Version -Verbose -Acr All -OsFilter All
59+
}
60+
elseif ($Channel -eq "preview")
61+
{
62+
& $buildScriptPath -UpdateBuildYaml -Channel $Channel -PreviewVersion $Version -Verbose -Acr All -OsFilter All
63+
}
64+
else #Channel lts
65+
{
66+
& $buildScriptPath -UpdateBuildYaml -Channel $Channel -LtsVersion $Version -Verbose -Acr All -OsFilter All
67+
}
68+
}
69+
else
70+
{
71+
Write-Verbose -Verbose "using $Channel only"
72+
& $buildScriptPath -UpdateBuildYaml -Channel $Channel -Verbose -Acr All -OsFilter All
73+
}
74+
}
75+
76+
77+
# Update stableReleaseStage.yml
78+
Update-ChannelReleaseStageYaml -Channel stable -Version $stableVersion
79+
80+
# Update previewReleaseStage.yml
81+
Update-ChannelReleaseStageYaml -Channel preview -Version $previewVersion
82+
83+
# Update ltsReleaseStage.yml
84+
Update-ChannelReleaseStageYaml -Channel lts -Version $ltsVersion
85+

‎tools/updateYamls/updateYamls.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Update Build Yamls
2+
3+
## Context
4+
5+
This `updateBuildYamls.ps1` script needs to be run before the build pipeline for PowerShell-Docker is kicked off. Running the script will produce a channel based yaml file, like &lt;channel&gt;ReleaseStage.yml for each channel. Then a PR must be created with these newly added/updated yaml files as the build will rely on them.
6+
7+
## Running the Script
8+
9+
To update the releaseStage.yml file for all the channels, run `./updateBuildYamls.ps1 -StableVersion <stableVersion> -PreviewVersion <previewVersion> -LtsVersion <ltsVersion>`.
10+
11+
If you want the channel versions from channels.json to be used, simply omit the -*Version parameters and run, `./updateBuildYamls.ps1`.
12+
13+
## Notes
14+
15+
The versions provided must match versioning syntax rules for stable and preview versions. Valid examples include 7.4.2 (stable) and v7.4.0-preview.5 (preview)

0 commit comments

Comments
 (0)