Run conformance tests in pod on cluster

Author: ciarams87

conformance/Makefile

@@ -1,11 +1,9 @@
 NKG_TAG = edge
 NKG_PREFIX = nginx-kubernetes-gateway
-GATEWAY_CLASS = nginx
-SUPPORTED_FEATURES = HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect
-EXEMPT_FEATURES = ReferenceGrant
 KIND_KUBE_CONFIG_FOLDER = $${HOME}/.kube/kind
 TAG = latest
 PREFIX = conformance-test-runner
+NGINX_IMAGE=$(shell yq '.spec.template.spec.containers[1].image as $$nginx_ver | $$nginx_ver' ../deploy/manifests/deployment.yaml)
 .DEFAULT_GOAL := help
 
 .PHONY: help
@@ -14,17 +12,20 @@ help: Makefile ## Display this help
 
 .PHONY: build-test-runner-image
 build-test-runner-image: ## Build conformance test runner image
-	docker build -t $(PREFIX):$(TAG) -f tests/Dockerfile ..
+	docker build -t $(PREFIX):$(TAG) -f tests/Dockerfile .. 
 
 .PHONY: create-kind-cluster
 create-kind-cluster: ## Create a kind cluster
-	kind create cluster --image kindest/node:v1.27.1
+	kind create cluster --image kindest/node:v1.27.1 
 	kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG_FOLDER)/config
 
 .PHONY: prepare-nkg
-prepare-nkg: ## Build and load NKG container on configured kind cluster
+prepare-nkg: ## Build and load NKG and NGINX containers on configured kind cluster
 	cd .. && make PREFIX=$(NKG_PREFIX) TAG=$(NKG_TAG) container
 	kind load docker-image $(NKG_PREFIX):$(NKG_TAG)
+	docker pull $(NGINX_IMAGE)
+	kind load docker-image $(NGINX_IMAGE)
+	yq -i 'with(.spec.template.spec.containers[0]; .image = "$(NKG_PREFIX):$(NKG_TAG)" | .imagePullPolicy = "Never")' ../deploy/manifests/deployment.yaml
 
 .PHONY: install-nkg
 install-nkg: ## Install NKG with provisioner on configured kind cluster
@@ -36,16 +37,17 @@ install-nkg: ## Install NKG with provisioner on configured kind cluster
 	kubectl apply -f ../deploy/manifests/rbac.yaml
 	kubectl apply -f ../deploy/manifests/gatewayclass.yaml
 	kubectl apply -f ../deploy/manifests/service/nodeport.yaml
-	kubectl apply -f provisioner/provisioner.yaml
-
-.PHONY: update-test-kind-config 
-update-test-kind-config: ## Update kind config
-	sed -ir "s|server:.*|server: https://kind-control-plane:6443|" $(KIND_KUBE_CONFIG_FOLDER)/config
+	yq '(select(di != 3))' provisioner/provisioner.yaml | kubectl apply -f -
+	yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(NKG_PREFIX):$(NKG_TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' provisioner/provisioner.yaml | kubectl apply -f -
 
 .PHONY: run-conformance-tests
-run-conformance-tests: update-test-kind-config ## Run conformance tests
-	docker run --network=kind --rm -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) \
-		go test -timeout 25m -v . -tags conformance -args --gateway-class=$(GATEWAY_CLASS) --debug --supported-features=$(SUPPORTED_FEATURES) --exempt-features=$(EXEMPT_FEATURES)
+run-conformance-tests: ## Run conformance tests
+	kind load docker-image $(PREFIX):$(TAG)
+	yq '(select(di != 3))' tests/conformance_spec.yaml | kubectl apply -f -
+	yq '(select(.spec.containers[].image) | .spec.containers[].image="$(PREFIX):$(TAG)")' tests/conformance_spec.yaml | kubectl apply -f -
+	sleep 5
+	kubectl logs conformance --follow
+	kubectl delete -f tests/conformance_spec.yaml
 
 .PHONY: uninstall-nkg
 uninstall-nkg: ## Uninstall NKG on configured kind cluster
@@ -54,7 +56,11 @@ uninstall-nkg: ## Uninstall NKG on configured kind cluster
 	kubectl delete -f ../deploy/manifests/namespace.yaml
 	kubectl delete clusterrole nginx-gateway-provisioner
 	kubectl delete clusterrolebinding nginx-gateway-provisioner
-	
+
+.PHONY: undo-image-update
+undo-image-update: ## Undo the NKG image name and tag in deployment manifest
+	git checkout -- ../deploy/manifests/deployment.yaml
+
 .PHONY: delete-kind-cluster
 delete-kind-cluster: ## Delete kind cluster
 	kind delete cluster

conformance/README.md

@@ -5,6 +5,7 @@
 * [kind](https://kind.sigs.k8s.io/).
 * Docker.
 * Golang.
+* [yq](https://github.com/mikefarah/yq/#macos--linux-via-homebrew)
 
 **Note**: all commands in steps below are executed from the ```conformance``` directory
 
@@ -29,49 +30,38 @@ update-test-kind-config        Update kind config
 $ make create-kind-cluster
 ```
 
-### Step 2 - Update NKG deployment and provisioner manifests
-**Note**: this step is only required when user wants to run conformance tests using locally built image of Nginx Kubernetes Gateway
-* Set NKG_PREFIX=<repo_name> NKG_TAG=<image_tag> to preferred values.
-* Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block.
-* Navigate to `conformance/provisioner` and update values in `provisioner.yaml` as specified in below code-block.
-* Save the changes.
-```
-.
-..
-containers:
-- image: <repo_name>:<image_tag>
-  imagePullPolicy: Never
-..
-.
-```
-
-### Step 3 - Build and load Nginx Kubernetes Gateway container to configured kind cluster
-**Note**: this step is only required when user wants to run conformance tests using locally built image of Nginx Kubernetes Gateway
+### Step 2 - Build Nginx Kubernetes Gateway container and load it and the NGINX container to configured kind cluster
 
 ```bash
 $ make NKG_PREFIX=<repo_name> NKG_TAG=<image_tag> prepare-nkg
 
 ```
-### Step 4 - Build conformance test runner image
+### Step 3 - Build conformance test runner image
 ```bash
 $ make build-test-runner-image
 ```
 
-### Step 5 - Install Nginx Kubernetes Gateway
+### Step 4 - Install Nginx Kubernetes Gateway
 ```bash
-$ make install-nkg
+$ make NKG_PREFIX=<repo_name> NKG_TAG=<image_tag> install-nkg
 ```
 
-### Step 6 - Run Gateway conformance tests
+### Step 5 - Run Gateway conformance tests
 ```bash
-$ make run-conformance-tests
+$ make NKG_PREFIX=<repo_name> NKG_TAG=<image_tag> run-conformance-tests
 ```
 
-### Step 7 - Uninstall Nginx Kubernetes Gateway
+### Step 6 - Uninstall Nginx Kubernetes Gateway
 ```bash
 $ make uninstall-nkg
 ```
 
+### Step 7 - Revert changes to the NKG deployment manifest
+**Warning**: `make undo-image-update` will hard reset changes to the deploy/manifests/deployment.yaml file!
+```bash
+$ make undo-image-update
+```
+
 ### Step 8 - Delete kind cluster
 ```bash
 $ make delete-kind-cluster

conformance/tests/Dockerfile

@@ -9,3 +9,5 @@ COPY --link go.sum /go/src/github.com/nginxinc/nginx-kubernetes-gateway/
 RUN go mod download
 
 COPY --link conformance/tests /go/src/github.com/nginxinc/nginx-kubernetes-gateway/conformance/tests/
+
+CMD go test -timeout 25m -v . -tags conformance -args --gateway-class=${GATEWAY_CLASS} --debug --supported-features=${SUPPORTED_FEATURES} --exempt-features=${EXEMPT_FEATURES}

conformance/tests/conformance_spec.yaml

@@ -0,0 +1,82 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: conformance
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: conformance
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - pods
+  - secrets
+  - services
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+- apiGroups:
+  - gateway.networking.k8s.io
+  resources:
+  - gatewayclasses
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - gateway.networking.k8s.io
+  resources:
+  - gateways
+  - httproutes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nginx-conformance
+subjects:
+- kind: ServiceAccount
+  name: conformance
+  namespace: default
+roleRef:
+  kind: ClusterRole
+  name: conformance
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: conformance
+spec:
+  containers:
+  - name: conformance
+    image: conformance-test-runner:latest
+    imagePullPolicy: Never
+    env:
+      - name: GATEWAY_CLASS
+        value: nginx
+      - name: SUPPORTED_FEATURES
+        value: HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect
+      - name: EXEMPT_FEATURES
+        value: ReferenceGrant
+  serviceAccountName: conformance
+  restartPolicy: Never