diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index cc07ea23ab..e792d1311c 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -5,6 +5,8 @@ on: branches: - main - release-* + tags: + - "v[0-9]+.[0-9]+.[0-9]+*" pull_request: branches: - main @@ -27,6 +29,8 @@ jobs: conformance-tests: name: Gateway Conformance Tests runs-on: ubuntu-22.04 + permissions: + contents: write # needed for uploading release artifacts steps: - name: Checkout Repository uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 @@ -139,5 +143,20 @@ jobs: working-directory: ./conformance - name: Run conformance tests - run: make run-conformance-tests TAG=${{ github.sha }} + run: make run-conformance-tests TAG=${{ github.sha }} VERSION=${{ github.ref_name }} + working-directory: ./conformance + + - name: Wait for release to exist + if: startsWith(github.ref, 'refs/tags/') + uses: lewagon/wait-on-check-action@e106e5c43e8ca1edea6383a39a01c5ca495fd812 # v1.3.1 + with: + ref: ${{ github.ref }} + check-name: 'Build Binary' + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload profile to release + if: startsWith(github.ref, 'refs/tags/') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release upload ${{ github.ref_name }} conformance-profile.yaml working-directory: ./conformance diff --git a/.gitignore b/.gitignore index 7b75ba804a..7252302fda 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out cover.html +conformance/conformance-profile.yaml # Dependency directories (remove the comment below to include it) # vendor/ diff --git a/conformance/Makefile b/conformance/Makefile index 3de7883ca1..72e11145df 100644 --- a/conformance/Makefile +++ b/conformance/Makefile @@ -76,8 +76,11 @@ run-conformance-tests: ## Run conformance tests kubectl run -i conformance \ --image=$(PREFIX):$(TAG) --image-pull-policy=Never \ --overrides='{ "spec": { "serviceAccountName": "conformance" } }' \ - --restart=Never -- go test -v . -tags conformance -args --gateway-class=$(GATEWAY_CLASS) --debug \ - --supported-features=$(SUPPORTED_FEATURES) + --restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \ + --supported-features=$(SUPPORTED_FEATURES) --version=$(VERSION) \ + --report-output=output.txt; cat output.txt" | tee output.txt + sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml + rm output.txt .PHONY: cleanup-conformance-tests cleanup-conformance-tests: ## Clean up conformance tests fixtures diff --git a/conformance/tests/conformance_test.go b/conformance/tests/conformance_test.go index f51c9de80a..dc3aae7baa 100644 --- a/conformance/tests/conformance_test.go +++ b/conformance/tests/conformance_test.go @@ -18,7 +18,7 @@ limitations under the License. package tests import ( - "strings" + "os" "testing" . "github.com/onsi/gomega" @@ -27,9 +27,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/apis/v1beta1" + "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1" "sigs.k8s.io/gateway-api/conformance/tests" "sigs.k8s.io/gateway-api/conformance/utils/flags" "sigs.k8s.io/gateway-api/conformance/utils/suite" + "sigs.k8s.io/yaml" ) func TestConformance(t *testing.T) { @@ -43,39 +45,54 @@ func TestConformance(t *testing.T) { g.Expect(v1alpha2.AddToScheme(client.Scheme())).To(Succeed()) g.Expect(v1beta1.AddToScheme(client.Scheme())).To(Succeed()) - supportedFeatures := parseSupportedFeatures(*flags.SupportedFeatures) - exemptFeatures := parseSupportedFeatures(*flags.ExemptFeatures) + supportedFeatures := suite.ParseSupportedFeatures(*flags.SupportedFeatures) + exemptFeatures := suite.ParseSupportedFeatures(*flags.ExemptFeatures) t.Logf(`Running conformance tests with %s GatewayClass\n cleanup: %t\n`+ `debug: %t\n enable all features: %t \n supported features: [%v]\n exempt features: [%v]`, *flags.GatewayClassName, *flags.CleanupBaseResources, *flags.ShowDebug, *flags.EnableAllSupportedFeatures, *flags.SupportedFeatures, *flags.ExemptFeatures) - cSuite := suite.New(suite.Options{ - Client: client, - GatewayClassName: *flags.GatewayClassName, - Debug: *flags.ShowDebug, - CleanupBaseResources: *flags.CleanupBaseResources, - SupportedFeatures: supportedFeatures, - ExemptFeatures: exemptFeatures, - EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures, + expSuite, err := suite.NewExperimentalConformanceTestSuite(suite.ExperimentalConformanceOptions{ + Options: suite.Options{ + Client: client, + GatewayClassName: *flags.GatewayClassName, + Debug: *flags.ShowDebug, + CleanupBaseResources: *flags.CleanupBaseResources, + SupportedFeatures: supportedFeatures, + ExemptFeatures: exemptFeatures, + EnableAllSupportedFeatures: *flags.EnableAllSupportedFeatures, + }, + Implementation: v1alpha1.Implementation{ + Organization: "nginxinc", + Project: "nginx-kubernetes-gateway", + URL: "https://github.com/nginxinc/nginx-kubernetes-gateway", + Version: *flags.ImplementationVersion, + Contact: []string{ + "https://github.com/nginxinc/nginx-kubernetes-gateway/discussions/new/choose", + }, + }, + ConformanceProfiles: sets.New(suite.HTTPConformanceProfileName), }) - cSuite.Setup(t) - cSuite.Run(t, tests.ConformanceTests) -} + g.Expect(err).To(Not(HaveOccurred())) + + expSuite.Setup(t) + err = expSuite.Run(t, tests.ConformanceTests) + g.Expect(err).To(Not(HaveOccurred())) + + report, err := expSuite.Report() + g.Expect(err).To(Not(HaveOccurred())) + + yamlReport, err := yaml.Marshal(report) + g.Expect(err).ToNot(HaveOccurred()) + + f, err := os.Create(*flags.ReportOutput) + g.Expect(err).ToNot(HaveOccurred()) + defer f.Close() + + _, err = f.WriteString("CONFORMANCE PROFILE\n") + g.Expect(err).ToNot(HaveOccurred()) -// parseSupportedFeatures parses flag arguments and converts the string to -// sets.Set[suite.SupportedFeature] -// FIXME(kate-osborn): Use exported ParseSupportedFeatures function -// https://github.com/kubernetes-sigs/gateway-api/blob/63e423cf1b837991d2747742199d90863a98b0c3/conformance/utils/suite/suite.go#L235 -// once it's released. https://github.com/nginxinc/nginx-kubernetes-gateway/issues/779 -func parseSupportedFeatures(f string) sets.Set[suite.SupportedFeature] { - if f == "" { - return nil - } - res := sets.Set[suite.SupportedFeature]{} - for _, value := range strings.Split(f, ",") { - res.Insert(suite.SupportedFeature(value)) - } - return res + _, err = f.Write(yamlReport) + g.Expect(err).ToNot(HaveOccurred()) } diff --git a/go.mod b/go.mod index 42510c6e6f..3b8a90220b 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( sigs.k8s.io/controller-runtime v0.16.0 sigs.k8s.io/controller-tools v0.13.0 sigs.k8s.io/gateway-api v0.8.0 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -88,5 +89,4 @@ require ( k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect )