Generate conformance report and upload to release (#1017)

Problem: We want to be able to upload our conformance report for every release, so we can keep the upstream Gateway API tracker for our project up to date.

Solution: Generate the conformance report whenever the tests are run. When a release occurs, we'll upload to the release artifacts.

Author: sjberman

.github/workflows/conformance.yml

@@ -5,6 +5,8 @@ on:
     branches:
       - main
       - release-*
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+*"
   pull_request:
     branches:
       - main
@@ -27,6 +29,8 @@ jobs:
   conformance-tests:
     name: Gateway Conformance Tests
     runs-on: ubuntu-22.04
+    permissions:
+      contents: write # needed for uploading release artifacts
     steps:
       - name: Checkout Repository
         uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
@@ -139,5 +143,20 @@ jobs:
         working-directory: ./conformance
 
       - name: Run conformance tests
-        run: make run-conformance-tests TAG=${{ github.sha }}
+        run: make run-conformance-tests TAG=${{ github.sha }} VERSION=${{ github.ref_name }}
+        working-directory: ./conformance
+
+      - name: Wait for release to exist
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: lewagon/wait-on-check-action@e106e5c43e8ca1edea6383a39a01c5ca495fd812 # v1.3.1
+        with:
+          ref: ${{ github.ref }}
+          check-name: 'Build Binary'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload profile to release
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh release upload ${{ github.ref_name }} conformance-profile.yaml
         working-directory: ./conformance

.gitignore

@@ -11,6 +11,7 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 cover.html
+conformance/conformance-profile.yaml
 
 # Dependency directories (remove the comment below to include it)
 # vendor/

conformance/Makefile

@@ -76,8 +76,11 @@ run-conformance-tests: ## Run conformance tests
 	kubectl run -i conformance \
 		--image=$(PREFIX):$(TAG) --image-pull-policy=Never \
 		--overrides='{ "spec": { "serviceAccountName": "conformance" }  }' \
-		--restart=Never -- go test -v . -tags conformance -args --gateway-class=$(GATEWAY_CLASS) --debug \
-						        --supported-features=$(SUPPORTED_FEATURES)
+		--restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \
+						        --supported-features=$(SUPPORTED_FEATURES) --version=$(VERSION) \
+								--report-output=output.txt; cat output.txt" | tee output.txt
+	sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml
+	rm output.txt
 
 .PHONY: cleanup-conformance-tests
 cleanup-conformance-tests: ## Clean up conformance tests fixtures

conformance/tests/conformance_test.go

@@ -18,7 +18,7 @@ limitations under the License.
 package tests
 
 import (
-	"strings"
+	"os"
 	"testing"
 
 	. "github.com/onsi/gomega"
@@ -27,9 +27,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/gateway-api/apis/v1alpha2"
 	"sigs.k8s.io/gateway-api/apis/v1beta1"
+	"sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
 	"sigs.k8s.io/gateway-api/conformance/tests"
 	"sigs.k8s.io/gateway-api/conformance/utils/flags"
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
+	"sigs.k8s.io/yaml"
 )
 
 func TestConformance(t *testing.T) {
@@ -43,39 +45,54 @@ func TestConformance(t *testing.T) {
 	g.Expect(v1alpha2.AddToScheme(client.Scheme())).To(Succeed())
 	g.Expect(v1beta1.AddToScheme(client.Scheme())).To(Succeed())
 
-	supportedFeatures := parseSupportedFeatures(*flags.SupportedFeatures)
-	exemptFeatures := parseSupportedFeatures(*flags.ExemptFeatures)
+	supportedFeatures := suite.ParseSupportedFeatures(*flags.SupportedFeatures)
+	exemptFeatures := suite.ParseSupportedFeatures(*flags.ExemptFeatures)
 
 	t.Logf(`Running conformance tests with %s GatewayClass\n cleanup: %t\n`+
 		`debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]`,
 		*flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug,
 		*flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures)
 
-	cSuite := suite.New(suite.Options{
-		Client:                     client,
-		GatewayClassName:           *flags.GatewayClassName,
-		Debug:                      *flags.ShowDebug,
-		CleanupBaseResources:       *flags.CleanupBaseResources,
-		SupportedFeatures:          supportedFeatures,
-		ExemptFeatures:             exemptFeatures,
-		EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
+	expSuite, err := suite.NewExperimentalConformanceTestSuite(suite.ExperimentalConformanceOptions{
+		Options: suite.Options{
+			Client:                     client,
+			GatewayClassName:           *flags.GatewayClassName,
+			Debug:                      *flags.ShowDebug,
+			CleanupBaseResources:       *flags.CleanupBaseResources,
+			SupportedFeatures:          supportedFeatures,
+			ExemptFeatures:             exemptFeatures,
+			EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures,
+		},
+		Implementation: v1alpha1.Implementation{
+			Organization: "nginxinc",
+			Project:      "nginx-kubernetes-gateway",
+			URL:          "https://github.com/nginxinc/nginx-kubernetes-gateway",
+			Version:      *flags.ImplementationVersion,
+			Contact: []string{
+				"https://github.com/nginxinc/nginx-kubernetes-gateway/discussions/new/choose",
+			},
+		},
+		ConformanceProfiles: sets.New(suite.HTTPConformanceProfileName),
 	})
-	cSuite.Setup(t)
-	cSuite.Run(t, tests.ConformanceTests)
-}
+	g.Expect(err).To(Not(HaveOccurred()))
+
+	expSuite.Setup(t)
+	err = expSuite.Run(t, tests.ConformanceTests)
+	g.Expect(err).To(Not(HaveOccurred()))
+
+	report, err := expSuite.Report()
+	g.Expect(err).To(Not(HaveOccurred()))
+
+	yamlReport, err := yaml.Marshal(report)
+	g.Expect(err).ToNot(HaveOccurred())
+
+	f, err := os.Create(*flags.ReportOutput)
+	g.Expect(err).ToNot(HaveOccurred())
+	defer f.Close()
+
+	_, err = f.WriteString("CONFORMANCE PROFILE\n")
+	g.Expect(err).ToNot(HaveOccurred())
 
-// parseSupportedFeatures parses flag arguments and converts the string to
-// sets.Set[suite.SupportedFeature]
-// FIXME(kate-osborn): Use exported ParseSupportedFeatures function
-// https://github.com/kubernetes-sigs/gateway-api/blob/63e423cf1b837991d2747742199d90863a98b0c3/conformance/utils/suite/suite.go#L235
-// once it's released. https://github.com/nginxinc/nginx-kubernetes-gateway/issues/779
-func parseSupportedFeatures(f string) sets.Set[suite.SupportedFeature] {
-	if f == "" {
-		return nil
-	}
-	res := sets.Set[suite.SupportedFeature]{}
-	for _, value := range strings.Split(f, ",") {
-		res.Insert(suite.SupportedFeature(value))
-	}
-	return res
+	_, err = f.Write(yamlReport)
+	g.Expect(err).ToNot(HaveOccurred())
 }

go.mod

@@ -21,6 +21,7 @@ require (
 	sigs.k8s.io/controller-runtime v0.16.0
 	sigs.k8s.io/controller-tools v0.13.0
 	sigs.k8s.io/gateway-api v0.8.0
+	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
@@ -88,5 +89,4 @@ require (
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
 )