Fix NFR upgrade tests with NGINX Plus (#2967)

Problem: When first deploying NGF edge version in the upgrade tests, NGINX Plus was not properly enabled. Also, once deployed, the upgraded version did not use the proper usage endpoint.

Solution: Ensure that the upgrade tests deploy the edge version of the NGINX Plus image, and set the proper usage endpoint once upgraded.

Author: sjberman

tests/Makefile

@@ -130,7 +130,8 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
 		--pull-policy=$(PULL_POLICY) --service-type=$(GW_SERVICE_TYPE) \
 		--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) --plus-enabled=$(PLUS_ENABLED) \
-		--plus-license-file-name=$(PLUS_LICENSE_FILE) --plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT)
+		--plus-license-file-name=$(PLUS_LICENSE_FILE) --plus-usage-endpoint=$(PLUS_USAGE_ENDPOINT) \
+		--gke-project=$(GKE_PROJECT)
 
 .PHONY: test
 test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster

tests/framework/ngf.go

@@ -148,6 +148,7 @@ func UpgradeNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) {
 
 	args = append(args, setImageArgs(cfg)...)
 	args = append(args, setTelemetryArgs(cfg)...)
+	args = append(args, setPlusUsageEndpointArg(cfg)...)
 	fullArgs := append(args, extraArgs...) //nolint:gocritic
 
 	GinkgoWriter.Printf("Upgrading NGF with command: helm %v\n", strings.Join(fullArgs, " "))

tests/scripts/remote-scripts/run-nfr-tests.sh

@@ -10,7 +10,7 @@ elif [ "${STOP_LONGEVITY}" == "true" ]; then
     GINKGO_LABEL="longevity-teardown"
 fi
 
-cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" PLUS_USAGE_ENDPOINT="${PLUS_USAGE_ENDPOINT}"
+cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" PLUS_USAGE_ENDPOINT="${PLUS_USAGE_ENDPOINT}" GKE_PROJECT="${GKE_PROJECT}"
 
 if [ "${START_LONGEVITY}" == "true" ]; then
     suite/scripts/longevity-wrk.sh

tests/scripts/vars.env-example

@@ -18,6 +18,7 @@ GINKGO_FLAGS=<optional flags to pass to the go test command>
 SOURCE_IP_RANGE=<IPs that should be allowed SSH to the VM, e.g. 1.2.3.4/32 or $(curl -sS -4 icanhazip.com)/32>
 ADD_VM_IP_AUTH_NETWORKS=<if set to true, the script will add the VM IP to the list of the GKE cluster master authorized networks>
 PLUS_ENABLED=<enable nginx plus>
+PLUS_USAGE_ENDPOINT=<endpoint for reporting NGINX Plus usage>
 NGF_VERSION=<version of NGF being tested. Defaults to value of TAG if not set>
 GKE_MACHINE_TYPE=<the node type for the gke cluster, defaults to e2-medium>
 GKE_NUM_NODES=<the number of nodes in the gke cluster, defaults to 3>

tests/suite/system_suite_test.go

@@ -64,6 +64,7 @@ var (
 	plusLicenseFileName      = flag.String("plus-license-file-name", "", "File name containing the NGINX Plus JWT")
 	plusUsageEndpoint        = flag.String("plus-usage-endpoint", "", "Endpoint for reporting NGINX Plus usage")
 	clusterName              = flag.String("cluster-name", "kind", "Cluster name")
+	gkeProject               = flag.String("gke-project", "", "GKE Project name")
 )
 
 var (
@@ -84,6 +85,8 @@ var (
 	logs              string
 )
 
+var formatNginxPlusEdgeImagePath = "us-docker.pkg.dev/%s/nginx-gateway-fabric/nginx-plus"
+
 const (
 	releaseName           = "ngf-test"
 	ngfNamespace          = "nginx-gateway"
@@ -206,18 +209,24 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo
 		Telemetry:         cfg.telemetry,
 	}
 
+	switch {
 	// if we aren't installing from the public charts, then set the custom images
-	if !strings.HasPrefix(cfg.chartPath, "oci://") {
+	case !strings.HasPrefix(cfg.chartPath, "oci://"):
 		installCfg.NgfImageRepository = *ngfImageRepository
 		installCfg.NginxImageRepository = *nginxImageRepository
 		if *plusEnabled && cfg.nfr {
 			installCfg.NginxImageRepository = *nginxPlusImageRepository
 		}
 		installCfg.ImageTag = *imageTag
 		installCfg.ImagePullPolicy = *imagePullPolicy
-	} else if version == "edge" {
+	case version == "edge":
 		chartVersion = "0.0.0-edge"
 		installCfg.ChartVersion = chartVersion
+		if *plusEnabled && cfg.nfr {
+			installCfg.NginxImageRepository = fmt.Sprintf(formatNginxPlusEdgeImagePath, *gkeProject)
+		}
+	case *plusEnabled && cfg.nfr:
+		installCfg.NginxImageRepository = fmt.Sprintf(formatNginxPlusEdgeImagePath, *gkeProject)
 	}
 
 	output, err := framework.InstallGatewayAPI(cfg.gwAPIVersion)

tests/suite/upgrade_test.go

@@ -101,6 +101,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() {
 			ServiceType:          *serviceType,
 			IsGKEInternalLB:      *isGKEInternalLB,
 			Plus:                 *plusEnabled,
+			PlusUsageEndpoint:    *plusUsageEndpoint,
 		}
 
 		type metricsResults struct {