Add conformance tests to the CI pipeline

Author: ciarams87

.github/workflows/ci.yml

@@ -144,10 +144,88 @@ jobs:
           path: ${{ github.workspace }}/dist
           key: nginx-kubernetes-gateway-${{ github.run_id }}-${{ github.run_number }}
 
+  conformance-tests:
+    name: Gateway Conformance Tests
+    runs-on: ubuntu-22.04
+    needs: [vars, binary]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+
+      - name: Fetch Cached Artifacts
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          path: ${{ github.workspace }}/dist
+          key: nginx-kubernetes-gateway-${{ github.run_id }}-${{ github.run_number }}
+
+      - name: Docker Buildx
+        uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v2.7.0
+      
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-kubernetes-gateway
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: Prepare NKG files
+        run: |
+          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
+          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          yq -i "with(.spec.template.spec.containers[0]; .image = \"${nkg_prefix}:${nkg_tag}\" | .imagePullPolicy = \"Never\")" deploy/manifests/deployment.yaml
+
+      - name: Build Docker Image
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
+        with:
+          file: build/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          context: "."
+          target: goreleaser
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          pull: true
+      
+      - name: Build Test Docker Image
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
+        with:
+          file: conformance/tests/Dockerfile
+          tags: conformance-test-runner:${{ github.sha }}
+          context: "."
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          pull: true
+
+      - name: Deploy Kubernetes
+        id: k8s
+        run: |
+          kind create cluster --image kindest/node:v1.27.1 --kubeconfig kube-${{ github.run_id }}
+          echo "KUBECONFIG=kube-${{ github.run_id }}" >> "$GITHUB_ENV"
+        working-directory: ./conformance
+
+      - name: Setup conformance tests
+        run: |
+          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
+          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          make install-nkg-local-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} BUILD_NKG=false
+        working-directory: ./conformance
+
+      - name: Run conformance tests
+        run: |
+          make run-conformance-tests TAG=${{ github.sha }}
+        working-directory: ./conformance
+        continue-on-error: true
+
   build:
     name: Build Image
     runs-on: ubuntu-22.04
-    needs: [vars, binary]
+    needs: [vars, binary, conformance-tests]
     steps:
       - name: Checkout Repository
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3

conformance/Makefile

@@ -4,6 +4,7 @@ GATEWAY_CLASS = nginx
 SUPPORTED_FEATURES = HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect
 KIND_KUBE_CONFIG_FOLDER = $${HOME}/.kube/kind
 TAG = latest
+BUILD_NKG = true
 PREFIX = conformance-test-runner
 NKG_DEPLOYMENT_MANIFEST=../deploy/manifests/deployment.yaml
 NGINX_IMAGE=$(shell yq '.spec.template.spec.containers[1].image as $$nginx_ver | $$nginx_ver' $(NKG_DEPLOYMENT_MANIFEST))
@@ -29,8 +30,10 @@ preload-nginx-container: ## Preload NGINX container on configured kind cluster
 
 .PHONY: build-and-load-images
 build-and-load-images: preload-nginx-container ## Build NKG container and load it and NGINX container on configured kind cluster
+ifeq ($(BUILD_NKG),true)
 	yq -i 'with(.spec.template.spec.containers[0]; .image = "$(NKG_PREFIX):$(NKG_TAG)" | .imagePullPolicy = "Never")' $(NKG_DEPLOYMENT_MANIFEST)
 	cd .. && make PREFIX=$(NKG_PREFIX) TAG=$(NKG_TAG) container
+endif
 	kind load docker-image $(NKG_PREFIX):$(NKG_TAG)
 
 .PHONY: prepare-nkg-dependencies

conformance/README.md

@@ -35,6 +35,7 @@ uninstall-nkg                  Uninstall NKG on configured kind cluster
 | ------------- | ------------- | ------------- |
 | TAG | latest  | The tag for the conformance test image |
 | PREFIX | conformance-test-runner | The prefix for the conformance test image |
+| BUILD_NKG | true | Flag to indicate if the local NKG image needs to be built |
 | NKG_TAG  | edge  | The tag for the locally built NKG image |
 | NKG_PREFIX | nginx-kubernetes-gateway  | The prefix for the locally built NKG image |
 | KIND_KUBE_CONFIG_FOLDER | ~/.kube/kind  | The location of the kubeconfig folder |
@@ -56,6 +57,21 @@ $ make create-kind-cluster
 $ make install-nkg-local-build
 ```
 
+**Note:** You can optionally skip the actual *build* step by setting the BUILD_NKG flag to "false". However, if choosing 
+this option, the following step *must* be completed manually *before* the build step:
+ * Set NKG_PREFIX=<nkg_repo_name> NKG_TAG=<nkg_image_tag> to preferred values.
+ * Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block.
+ * Save the changes.
+ ```
+ .
+ ..
+ containers:
+ - image: <nkg_repo_name>:<nkg_image_tag>
+   imagePullPolicy: Never
+ ..
+ .
+ ```
+
 #### *Option 2* Install Nginx Kubernetes Gateway from edge to configured kind cluster
 Instead of the above command, you can skip the build NKG image step and prepare the environment to instead
 use the `edge` image