Use channel tags to make manifest list more automatic (#743)

* Use channel tags to make manifest list more automatic

* Add unit test to verify we mix manifest list channels

* Update unittests/generateManifestLists.tests.ps1

* Update unittests/generateManifestLists.tests.ps1

Co-authored-by: Aditya Patwardhan <adityap@microsoft.com>

* add new format of manifest list

* update how we create the manifest create cmdline

* fix release tests

* mark amazon linux as unstable

* mark mariner as arm for test purposes

* remove community channel

* mark the right image

* remove community CI

* skip tasks when matrix is empty

---------

Co-authored-by: Aditya Patwardhan <adityap@microsoft.com>

Author: TravisEz13

.vsts-ci/manifestSteps.yml

@@ -7,6 +7,10 @@ steps:
 - powershell: 'Get-ChildItem env:'
   displayName: 'Capture Environment'
 
+- pwsh: |
+    $PSVersionTable
+  displayName: Capture pwsh Version
+
 - powershell: |
       $dockerConfigFolder = "$env:userprofile/.docker"
       if(!(Test-Path $dockerConfigFolder)){ $null = new-item -Type Directory -Path $dockerConfigFolder}

.vsts-ci/templatesReleasePipeline/releaseJob.yml

@@ -102,6 +102,7 @@ jobs:
               }
             }
       displayName: 'release - $(channel) - $(imagename)'
+      condition: and( succeeded(), ne(variables['Channel'],''))
 
     - pwsh: 'az logout'
       displayName: 'az logout'

.vsts-ci/templatesReleasePipeline/testJob.yml

@@ -84,6 +84,8 @@ jobs:
             }
             .\build.ps1 -test -Channel $(Channel) -Pull -ImageName $(dockerHost) -name '$(ImageName)' -Version $(Version) -TestLogPostfix windows-$(Channel) -Repository $(dockerNamespace)/powershell @extraParams
       displayName: 'Run Tests for $(Channel) - $(ImageName)'
+      condition: and( succeeded(), ne(variables['Channel'],''))
+
 
     - task: PublishTestResults@2
       displayName: 'Publish Test Results **/test*.xml'

.vsts-ci/templatesReleasePipeline/testStage.yml

@@ -53,17 +53,17 @@ stages:
 
       - template: ./testJob.yml
         parameters:
-          jobName: ${{ parameters.channel }}_linux
+          jobName: ${{ parameters.channel }}_linux_arm64
           matrix: dependencies.GenerateYaml_${{ parameters.channel }}.outputs['matrix.matrix_${{ parameters.channel }}_linux_arm64']
           vmImage: ubuntu-latest
           dependsOn:
             - GenerateYaml_${{ parameters.channel }}
 
       - template: ./testJob.yml
         parameters:
-          jobName: ${{ parameters.channel }}_linux
+          jobName: ${{ parameters.channel }}_linux_arm32
           matrix: dependencies.GenerateYaml_${{ parameters.channel }}.outputs['matrix.matrix_${{ parameters.channel }}_linux_arm32']
           vmImage: ubuntu-latest
           dependsOn:
             - GenerateYaml_${{ parameters.channel }}
-            
+

build.ps1

@@ -668,10 +668,25 @@ End {
     if ($GenerateManifestLists.IsPresent) {
         $manifestLists = @()
         $tags = @()
+
         foreach ($repo in $tagGroups.Keys | Sort-Object) {
             $channelGroups = $tagGroups.$repo | Group-Object -Property Channel
             foreach ($channelGroup in $channelGroups) {
                 $channelName = $channelGroup.Name
+                switch($channelName) {
+                    'stable' {
+                        $channelTag = 'latest'
+                        $channelTagPrefix = ''
+                        $channelTagPostfix = ''
+                    }
+
+                    default {
+                        $channelTag = $channelName.ToLower()
+                        $channelTagPrefix = $channelTag + '-'
+                        $channelTagPostfix = '-' + $channelTag
+                    }
+                }
+
                 Write-Verbose "generating $channelName json"
                 $osGroups = $channelGroup.Group | Group-Object -Property os
                 foreach ($osGroup in $osGroups) {
@@ -681,14 +696,31 @@ End {
                     foreach ($tag in $osGroup.Group | Where-Object { $_.Name -notlike '*/*' } | Sort-Object -Property ManifestLists) {
                         if ($tag.ManifestLists) {
                             foreach ($manifestList in $tag.ManifestLists) {
+                                $originalManifestList = $manifestList
+                                if ($manifestList -notlike '*${channel*') {
+                                    Write-Warning "Issue found with $osName $manifestList $($tag.Tags)"
+                                    throw 'ManifestLists entries must contain on of: ${channelTag} ${channelTagPrefix} ${channelTagPostfix}'
+                                }
+
+                                $manifestList = $manifestList -replace '\${channelTag}', $channelTag
+                                $manifestList = $manifestList -replace '\${channelTagPrefix}', $channelTagPrefix
+                                $manifestList = $manifestList -replace '\${channelTagPostfix}', $channelTagPostfix
+
+                                if (!$manifestList) {
+                                    throw "error formatting $originalManifestList"
+                                }
 
                                 if ($manifestLists -notcontains $manifestList) {
                                     $manifestLists += $manifestList
                                 }
 
-                                $tag | Add-Member -MemberType NoteProperty -Value $repo -Name 'Repo'
-
-                                $tags += $tag
+                                $tags += [PSCustomObject]@{
+                                    Repo = $repo
+                                    FormattedManifestList = $manifestList
+                                    Tags = $tag.Tags
+                                    Channel = $tag.Channel
+                                    ContinueOnError = $tag.ContinueOnError
+                                }
                             }
                         }
                     }
@@ -707,7 +739,7 @@ End {
                 Repo         = ""
             }
 
-            foreach ($tag in $tags | Where-Object { $_.ManifestLists -contains $manifestList }) {
+            foreach ($tag in $tags | Where-Object { $_.FormattedManifestList -eq $manifestList }) {
                 if (-not $matrix.ContainsKey($manifestList)) {
                     $matrix.Add($manifestList, @{ })
                 }

createManifest.ps1

@@ -27,26 +27,31 @@ param(
 
     [switch]
     $SkipPush
-
 )
 
 $first = $true
 $manifestList = @()
 foreach($tag in $TagList)
 {
+    $arguments = @()
     $amend = ""
     if (!$first) {
-        $amend = '--amend'
+        $arguments += '--amend'
     }
+    $arguments += @(
+        "$ContainerRegistry/${Image}:$ManifestTag"
+        "$ContainerRegistry/${Image}:$tag"
+    )
 
-    Write-Verbose -Message "running: docker manifest create $ammend $ContainerRegistry/${Image}:$ManifestTag $ContainerRegistry/${Image}:$tag" -Verbose
-    docker manifest create $amend $ContainerRegistry/${Image}:$ManifestTag "$ContainerRegistry/${Image}:$tag"
+    Write-Verbose -Message "running: docker manifest create $arguments" -Verbose
+    docker manifest create $arguments
     $first = $false
 }
 
 # Create the manifest
 
 # Inspect (print) the manifest
+Write-Verbose -Verbose "capturing the manifest..."
 docker manifest inspect $ContainerRegistry/${Image}:$ManifestTag
 
 # push the manifest

release/7-2/nanoserver1809/meta.json

@@ -12,7 +12,7 @@
         "nanoserver-#shorttag#"
     ],
     "manifestLists": [
-        "nanoserver"
+        "${channelTagPrefix}nanoserver"
     ],
     "Base64EncodePackageUrl": true,
     "TestProperties": {

release/7-2/nanoserver2022/meta.json

@@ -12,7 +12,7 @@
         "nanoserver-#shorttag#"
     ],
     "manifestLists": [
-        "nanoserver"
+        "${channelTagPrefix}nanoserver"
     ],
     "Base64EncodePackageUrl": true,
     "TestProperties": {

release/7-2/ubi8/meta.json

@@ -12,7 +12,8 @@
         {"Tag": "8"}
     ],
     "manifestLists": [
-        "ubi"
+        "ubi${channelTagPostfix}",
+        "${channelTagPrefix}ubi"
     ],
     "SubImage": "test-deps",
     "TestProperties": {

release/7-2/ubuntu22.04-arm32v7/meta.json

@@ -20,7 +20,7 @@
         "Arm32": true
     },
     "manifestLists": [
-        "latest"
+        "${channelTag}"
     ],
     "EndOfLife": "2023-04-02",
     "DistributionState": "Validated",

release/7-2/ubuntu22.04/meta.json

@@ -13,7 +13,7 @@
         {"Tag": "22.04"}
     ],
     "manifestLists": [
-        "latest"
+        "${channelTag}"
     ],
     "TestProperties": {
         "size": 343

release/7-2/windowsserver2022/meta.json

@@ -13,7 +13,7 @@
         "windowsserver-#shorttag#"
     ],
     "manifestLists": [
-        "windowsserver"
+        "${channelTagPrefix}windowsserver"
     ],
     "TestProperties": {
         "size": 1

release/7-2/windowsservercore2022/meta.json

@@ -13,7 +13,7 @@
         "windowsservercore-#shorttag#"
     ],
     "manifestLists": [
-        "latest"
+        "${channelTag}"
     ],
     "TestProperties": {
         "size": 1

release/7-3/mariner2-arm64/meta.json

@@ -12,7 +12,7 @@
     ],
     "TestProperties": {
         "size": 404,
-        "arm64": true
+        "Arm32": true
     },
     "EndOfLife": "2023-12-14",
     "DistributionState": "Validating",

release/7-3/nanoserver2022/meta.json

@@ -9,7 +9,7 @@
     ],
     "ShortDistroName": "nanoserver",
     "manifestLists": [
-        "nanoserver"
+        "${channelTagPrefix}nanoserver"
     ],
     "Base64EncodePackageUrl": true,
     "TestProperties": {

release/7-3/ubi9/meta.json

@@ -9,7 +9,8 @@
         {"Tag": "9"}
     ],
     "manifestLists": [
-        "ubi-preview"
+        "ubi${channelTagPostfix}",
+        "${channelTagPrefix}ubi"
     ],
     "SubImage": "test-deps",
     "TestProperties": {

release/7-3/ubuntu22.04-arm32v7/meta.json

@@ -16,7 +16,7 @@
         "Arm32": true
     },
     "manifestLists": [
-        "preview"
+        "${channelTag}"
     ],
     "EndOfLife": "2023-04-02",
     "DistributionState": "Validated",

release/7-3/ubuntu22.04/meta.json

@@ -10,7 +10,7 @@
         {"Tag": "22.04"}
     ],
     "manifestLists": [
-        "preview"
+        "${channelTag}"
     ],
     "TestProperties": {
         "size": 338

release/7-3/windowsserver2022/meta.json

@@ -10,7 +10,7 @@
     "Base64EncodePackageUrl": true,
     "ShortDistroName": "windowsserver",
     "manifestLists": [
-        "windowsserver"
+        "${channelTagPrefix}windowsserver"
     ],
     "TestProperties": {
         "size": 1

release/7-3/windowsservercore2022/meta.json

@@ -8,7 +8,7 @@
         }
     ],
     "manifestLists": [
-        "preview"
+        "${channelTag}"
     ],
     "Base64EncodePackageUrl": true,
     "ShortDistroName": "windowsservercore",

release/7-4/mariner2-arm64/meta.json

@@ -12,7 +12,7 @@
     ],
     "TestProperties": {
         "size": 404,
-        "arm64": true
+        "Arm32": true
     },
     "EndOfLife": "2023-12-14",
     "DistributionState": "Validating",

release/7-4/ubi9/meta.json

@@ -9,7 +9,8 @@
         {"Tag": "9"}
     ],
     "manifestLists": [
-        "ubi-preview"
+        "ubi${channelTagPostfix}",
+        "${channelTagPrefix}ubi"
     ],
     "SubImage": "test-deps",
     "TestProperties": {

release/7-4/ubuntu22.04-arm32v7/meta.json

@@ -16,7 +16,7 @@
         "Arm32": true
     },
     "manifestLists": [
-        "preview"
+        "${channelTag}"
     ],
     "EndOfLife": "2023-04-02",
     "DistributionState": "Validated",

release/7-4/ubuntu22.04/meta.json

@@ -10,7 +10,7 @@
         {"Tag": "22.04"}
     ],
     "manifestLists": [
-        "preview"
+        "${channelTag}"
     ],
     "TestProperties": {
         "size": 338

release/7-4/windowsserver2022/meta.json

@@ -10,7 +10,7 @@
     "Base64EncodePackageUrl": true,
     "ShortDistroName": "windowsserver",
     "manifestLists": [
-        "windowsserver"
+        "${channelTagPrefix}windowsserver"
     ],
     "TestProperties": {
         "size": 1

release/7-4/windowsservercore2022/meta.json

@@ -8,7 +8,7 @@
         }
     ],
     "manifestLists": [
-        "preview"
+        "${channelTag}"
     ],
     "Base64EncodePackageUrl": true,
     "ShortDistroName": "windowsservercore",

release/unstable/7-4/nanoserver2022/meta.json

@@ -9,7 +9,7 @@
     ],
     "ShortDistroName": "nanoserver",
     "manifestLists": [
-        "nanoserver"
+        "${channelTagPrefix}nanoserver"
     ],
     "Base64EncodePackageUrl": true,
     "TestProperties": {

release/community-stable/amazonlinux/docker/Dockerfile renamed to release/unstable/amazonlinux/docker/Dockerfile

@@ -266,7 +266,7 @@ class DockerImageMetaData {
     [bool]
     $ContinueOnError = $false
 
-    [string]
+    [string[]]
     $ManifestLists = @()
 
     [bool]

release/community-stable/amazonlinux/meta.json renamed to release/unstable/amazonlinux/meta.json

@@ -18,11 +18,5 @@
         "tagPrefix": "lts",
         "pwshInstallVersion": "7-lts",
         "packageTag": "-lts"
-    },
-    {
-        "name": "community-stable",
-        "path": "release/community-stable",
-        "pwshInstallVersion": "7",
-        "packageTag": ""
     }
 ]

release/community-stable/amazonlinux/test-deps/docker/Dockerfile renamed to release/unstable/amazonlinux/test-deps/docker/Dockerfile

@@ -13,4 +13,24 @@ Describe "build.ps1 -GenerateManifestLists" {
         $json | Should -Not -BeNullOrEmpty
         $json | ConvertFrom-Json -AsHashtable | Should -Not -BeNullOrEmpty
     }
+
+    It "Each tag should only be in one channel" {
+        $json = & $buildScript -GenerateManifestLists -Channel stable, preview, lts -OsFilter All | ConvertFrom-Json
+        $tags = @{}
+        $json | ForEach-Object {
+            $channel = $_.channel
+            foreach ($tag in $_.tags) {
+                if (!$tags.ContainsKey($tag)) {
+                    $tags[$tag] = @()
+                }
+                if ($tags[$tag] -notcontains $channel) {
+                    $tags[$tag] += $channel
+                }
+            }
+        }
+
+        foreach($tag in $tags.Keys) {
+            $tags.$tag.count | should -Be 1 -Because "$Tag should not be in multiple channels ($($tags.$tag))"
+        }
+    }
 }