Automate Zero Downtime Scale tests (#2259)

Problem: We want our NFR tests to be fully automated to save developer time for each release cycle, and have a repetitive way of running the tests.

Solution: Automate the zero downtime scaling test. No longer collecting logs as done previously, because error logs would be unreliable to collect for pods that are scaling down (we don't have persistence that we can easily use in the automation to gather historic logs). Ultimately we are still ensuring that traffic is flowing and status updates occur, which are the important pieces here.

Author: sjberman

tests/Makefile

@@ -97,7 +97,7 @@ run-tests-on-vm: ## Run the functional tests on a GCP VM
 
 .PHONY: nfr-test
 nfr-test: ## Run the NFR tests on a GCP VM
-	NFR=true bash scripts/run-tests-gcp-vm.sh
+	NFR=true CI=$(CI) bash scripts/run-tests-gcp-vm.sh
 
 .PHONY: start-longevity-test
 start-longevity-test: export START_LONGEVITY=true
@@ -110,7 +110,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 .PHONY: .vm-nfr-test
 .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`)
 	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v \
-	    --label-filter "nfr" $(GINKGO_FLAGS) ./suite --  --gateway-api-version=$(GW_API_VERSION) \
+	    --label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite --  --gateway-api-version=$(GW_API_VERSION) \
 		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
 		--plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
 		--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \

tests/README.md

@@ -744,3 +744,32 @@ func (rm *ResourceManager) WaitForPodsToBeReadyWithCount(ctx context.Context, na
 		},
 	)
 }
+
+// WaitForGatewayObservedGeneration waits for the provided Gateway's ObservedGeneration to equal the expected value.
+func (rm *ResourceManager) WaitForGatewayObservedGeneration(
+	ctx context.Context,
+	namespace,
+	name string,
+	generation int,
+) error {
+	return wait.PollUntilContextCancel(
+		ctx,
+		500*time.Millisecond,
+		true, /* poll immediately */
+		func(ctx context.Context) (bool, error) {
+			var gw v1.Gateway
+			key := types.NamespacedName{Namespace: namespace, Name: name}
+			if err := rm.K8sClient.Get(ctx, key, &gw); err != nil {
+				return false, err
+			}
+
+			for _, cond := range gw.Status.Conditions {
+				if cond.ObservedGeneration == int64(generation) {
+					return true, nil
+				}
+			}
+
+			return false, nil
+		},
+	)
+}

tests/framework/resourcemanager.go

@@ -23,7 +23,13 @@ if [ $retcode -ne 0 ]; then
 fi
 
 if [ "${NFR}" = "true" ]; then
-    gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --recurse username@${RESOURCE_NAME}:~/nginx-gateway-fabric/tests/results .
+    ## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config
+    if [ "${CI}" = "false" ]; then
+      gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh > /dev/null
+      rsync -ave 'ssh -F ngf-gcp.ssh' username@${RESOURCE_NAME}.${GKE_CLUSTER_ZONE}.${GKE_PROJECT}:~/nginx-gateway-fabric/tests/results .
+    else
+      gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --recurse username@${RESOURCE_NAME}:~/nginx-gateway-fabric/tests/results .
+    fi
 fi
 
 ## If tearing down the longevity test, we need to collect logs from gcloud and add to the results

tests/zero-downtime-scaling/results/1.0.0/1.0.0.md renamed to tests/results/zero-downtime-scale/1.0.0/1.0.0.md

@@ -25,7 +25,3 @@ affinity:
       labelSelector:
         matchLabels:
           app.kubernetes.io/name: nginx-gateway
-
-service:
-  annotations:
-    networking.gke.io/load-balancer-type: "Internal"

tests/zero-downtime-scaling/results/1.0.0/10-node/abrupt-down-http.png renamed to tests/results/zero-downtime-scale/1.0.0/10-node/abrupt-down-http.png

@@ -1,18 +1,11 @@
 nginxGateway:
-  image:
-    repository: ghcr.io/nginxinc/nginx-gateway-fabric
-    tag: edge # change this tag if you are testing a different version
-    pullPolicy: IfNotPresent
   lifecycle:
     preStop:
       exec:
         command:
         - /usr/bin/gateway
         - sleep
         - --duration=40s
-  config:
-    logging:
-      level: debug
 
 nginx:
   lifecycle:
@@ -31,7 +24,3 @@ affinity:
       labelSelector:
         matchLabels:
           app.kubernetes.io/name: nginx-gateway
-
-service:
-  annotations:
-    networking.gke.io/load-balancer-type: "Internal"

tests/zero-downtime-scaling/results/1.0.0/10-node/abrupt-down-https.png renamed to tests/results/zero-downtime-scale/1.0.0/10-node/abrupt-down-https.png

@@ -0,0 +1,18 @@
+nginxGateway:
+  lifecycle:
+    preStop:
+      exec:
+        command:
+        - /usr/bin/gateway
+        - sleep
+        - --duration=40s
+
+nginx:
+  lifecycle:
+    preStop:
+      exec:
+        command:
+        - /bin/sleep
+        - "40"
+
+terminationGracePeriodSeconds: 50