From c72687620cdc11df4682f25a8f9c1c726836cb1a Mon Sep 17 00:00:00 2001 From: Ciara Stacke Date: Wed, 28 Jun 2023 09:54:13 +0100 Subject: [PATCH 1/3] Add conformance tests to the CI pipeline --- .github/workflows/ci.yml | 80 +++++++++++++++++++++++++++++++++++++++- conformance/Makefile | 3 ++ conformance/README.md | 16 ++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dba4ad7fd2..0a4859d078 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,10 +144,88 @@ jobs: path: ${{ github.workspace }}/dist key: nginx-kubernetes-gateway-${{ github.run_id }}-${{ github.run_number }} + conformance-tests: + name: Gateway Conformance Tests + runs-on: ubuntu-22.04 + needs: [vars, binary] + steps: + - name: Checkout Repository + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + + - name: Fetch Cached Artifacts + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ${{ github.workspace }}/dist + key: nginx-kubernetes-gateway-${{ github.run_id }}-${{ github.run_number }} + + - name: Docker Buildx + uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v2.7.0 + + - name: Docker meta + id: meta + uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0 + with: + images: | + name=ghcr.io/nginxinc/nginx-kubernetes-gateway + tags: | + type=semver,pattern={{version}} + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: Prepare NKG files + run: | + nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) + nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) + yq -i "with(.spec.template.spec.containers[0]; .image = \"${nkg_prefix}:${nkg_tag}\" | .imagePullPolicy = \"Never\")" deploy/manifests/deployment.yaml + + - name: Build Docker Image + uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 + with: + file: build/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + context: "." + target: goreleaser + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + pull: true + + - name: Build Test Docker Image + uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 + with: + file: conformance/tests/Dockerfile + tags: conformance-test-runner:${{ github.sha }} + context: "." + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + pull: true + + - name: Deploy Kubernetes + id: k8s + run: | + kind create cluster --image kindest/node:v1.27.1 --kubeconfig kube-${{ github.run_id }} + echo "KUBECONFIG=kube-${{ github.run_id }}" >> "$GITHUB_ENV" + working-directory: ./conformance + + - name: Setup conformance tests + run: | + nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) + nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) + make install-nkg-local-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} BUILD_NKG=false + working-directory: ./conformance + + - name: Run conformance tests + run: | + make run-conformance-tests TAG=${{ github.sha }} + working-directory: ./conformance + continue-on-error: true + build: name: Build Image runs-on: ubuntu-22.04 - needs: [vars, binary] + needs: [vars, binary, conformance-tests] steps: - name: Checkout Repository uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 diff --git a/conformance/Makefile b/conformance/Makefile index ea3a0e98a4..73e43cbb37 100644 --- a/conformance/Makefile +++ b/conformance/Makefile @@ -4,6 +4,7 @@ GATEWAY_CLASS = nginx SUPPORTED_FEATURES = HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect KIND_KUBE_CONFIG_FOLDER = $${HOME}/.kube/kind TAG = latest +BUILD_NKG = true PREFIX = conformance-test-runner NKG_DEPLOYMENT_MANIFEST=../deploy/manifests/deployment.yaml NGINX_IMAGE=$(shell yq '.spec.template.spec.containers[1].image as $$nginx_ver | $$nginx_ver' $(NKG_DEPLOYMENT_MANIFEST)) @@ -29,8 +30,10 @@ preload-nginx-container: ## Preload NGINX container on configured kind cluster .PHONY: build-and-load-images build-and-load-images: preload-nginx-container ## Build NKG container and load it and NGINX container on configured kind cluster +ifeq ($(BUILD_NKG),true) yq -i 'with(.spec.template.spec.containers[0]; .image = "$(NKG_PREFIX):$(NKG_TAG)" | .imagePullPolicy = "Never")' $(NKG_DEPLOYMENT_MANIFEST) cd .. && make PREFIX=$(NKG_PREFIX) TAG=$(NKG_TAG) container +endif kind load docker-image $(NKG_PREFIX):$(NKG_TAG) .PHONY: prepare-nkg-dependencies diff --git a/conformance/README.md b/conformance/README.md index eecc911dc6..fb22293f3e 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -35,6 +35,7 @@ uninstall-nkg Uninstall NKG on configured kind cluster | ------------- | ------------- | ------------- | | TAG | latest | The tag for the conformance test image | | PREFIX | conformance-test-runner | The prefix for the conformance test image | +| BUILD_NKG | true | Flag to indicate if the local NKG image needs to be built | | NKG_TAG | edge | The tag for the locally built NKG image | | NKG_PREFIX | nginx-kubernetes-gateway | The prefix for the locally built NKG image | | KIND_KUBE_CONFIG_FOLDER | ~/.kube/kind | The location of the kubeconfig folder | @@ -56,6 +57,21 @@ $ make create-kind-cluster $ make install-nkg-local-build ``` +**Note:** You can optionally skip the actual *build* step by setting the BUILD_NKG flag to "false". However, if choosing +this option, the following step *must* be completed manually *before* the build step: + * Set NKG_PREFIX= NKG_TAG= to preferred values. + * Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block. + * Save the changes. + ``` + . + .. + containers: + - image: : + imagePullPolicy: Never + .. + . + ``` + #### *Option 2* Install Nginx Kubernetes Gateway from edge to configured kind cluster Instead of the above command, you can skip the build NKG image step and prepare the environment to instead use the `edge` image From 5156aba2511fa7348ea77f33f83ae395b8c09001 Mon Sep 17 00:00:00 2001 From: Ciara Stacke Date: Thu, 29 Jun 2023 14:40:36 +0100 Subject: [PATCH 2/3] Reconfigure make targets and define kubernetes version only once --- .github/workflows/ci.yml | 7 ++++--- Makefile | 3 ++- conformance/Makefile | 30 ++++++++++++++++++++---------- conformance/README.md | 22 +++++++++++++--------- conformance/tests/Dockerfile | 2 ++ 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a4859d078..532672bdb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,7 +177,8 @@ jobs: run: | nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) - yq -i "with(.spec.template.spec.containers[0]; .image = \"${nkg_prefix}:${nkg_tag}\" | .imagePullPolicy = \"Never\")" deploy/manifests/deployment.yaml + make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} + working-directory: ./conformance - name: Build Docker Image uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1 @@ -205,7 +206,7 @@ jobs: - name: Deploy Kubernetes id: k8s run: | - kind create cluster --image kindest/node:v1.27.1 --kubeconfig kube-${{ github.run_id }} + make create-kind-cluster KIND_KUBE_CONFIG=kube-${{ github.run_id }} echo "KUBECONFIG=kube-${{ github.run_id }}" >> "$GITHUB_ENV" working-directory: ./conformance @@ -213,7 +214,7 @@ jobs: run: | nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1) nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) - make install-nkg-local-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} BUILD_NKG=false + make install-nkg-local-no-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} working-directory: ./conformance - name: Run conformance tests diff --git a/Makefile b/Makefile index 9b893be60e..0571eb4711 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,8 @@ deps: ## Add missing and remove unused modules, verify deps and download them to .PHONY: create-kind-cluster create-kind-cluster: ## Create a kind cluster - kind create cluster --image kindest/node:v1.27.1 + $(eval KIND_IMAGE=$(shell grep -m1 'FROM kindest/node' NKG_TAG= to preferred values. * Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block. @@ -71,10 +77,8 @@ this option, the following step *must* be completed manually *before* the build .. . ``` - -#### *Option 2* Install Nginx Kubernetes Gateway from edge to configured kind cluster -Instead of the above command, you can skip the build NKG image step and prepare the environment to instead -use the `edge` image +#### *Option 3* Install Nginx Kubernetes Gateway from edge to configured kind cluster +You can also skip the build NKG image step and prepare the environment to instead use the `edge` image ```bash $ make install-nkg-edge diff --git a/conformance/tests/Dockerfile b/conformance/tests/Dockerfile index 672872bbdc..023907417d 100644 --- a/conformance/tests/Dockerfile +++ b/conformance/tests/Dockerfile @@ -1,4 +1,6 @@ # syntax=docker/dockerfile:1.5 +# this is here so we can grab the latest version of kind and have dependabot keep it up to date +FROM kindest/node:v1.27.3 FROM golang:1.20 From 7ed62c98a8b8509d01491ffda23ecb11bf205c66 Mon Sep 17 00:00:00 2001 From: Ciara Stacke Date: Thu, 29 Jun 2023 18:42:00 +0100 Subject: [PATCH 3/3] Add build binary to conformance tests --- .github/workflows/ci.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 532672bdb9..09dd6e7491 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,16 +147,15 @@ jobs: conformance-tests: name: Gateway Conformance Tests runs-on: ubuntu-22.04 - needs: [vars, binary] + needs: vars steps: - name: Checkout Repository uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - name: Fetch Cached Artifacts - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + - name: Setup Golang Environment + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: - path: ${{ github.workspace }}/dist - key: nginx-kubernetes-gateway-${{ github.run_id }}-${{ github.run_number }} + go-version-file: go.mod - name: Docker Buildx uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v2.7.0 @@ -179,6 +178,14 @@ jobs: nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2) make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag} working-directory: ./conformance + + - name: Build binary + uses: goreleaser/goreleaser-action@336e29918d653399e599bfca99fadc1d7ffbc9f7 # v4.3.0 + with: + version: latest + args: ${{ startsWith(github.ref, 'refs/tags/') && 'release' || 'build --snapshot' }} --clean + env: + GOPATH: ${{ needs.vars.outputs.go_path }} - name: Build Docker Image uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1