Add dataplane performance automated test

Author: ciarams87

tests/Makefile

@@ -4,6 +4,8 @@ NGINX_PREFIX = $(PREFIX)/nginx
 PULL_POLICY=Never
 GW_API_VERSION ?= 1.0.0
 K8S_VERSION ?= latest ## Expected format: 1.24 (major.minor) or latest
+GW_SERVICE_TYPE=NodePort
+GW_SVC_GKE_INTERNAL=false
 
 .PHONY: help
 help: Makefile ## Display this help
@@ -24,8 +26,20 @@ load-images: ## Load NGF and NGINX images on configured kind cluster
 test: ## Run the system tests against your default k8s cluster
 	go test -v ./suite -args --gateway-api-version=$(GW_API_VERSION) --image-tag=$(TAG) \
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --pull-policy=$(PULL_POLICY) \
-		--k8s-version=$(K8S_VERSION)
+		--k8s-version=$(K8S_VERSION) --service-type=$(GW_SERVICE_TYPE) --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
 
 .PHONY: delete-kind-cluster
 delete-kind-cluster: ## Delete kind cluster
 	kind delete cluster
+
+.PHONY: reset-etc-hosts
+reset-etc-hosts: ## Reset the /etc/hosts file to delete the entry for cafe.example.com
+	sudo sed -i -e '/cafe.example.com/d' /etc/hosts
+
+.PHONY: run-tests-on-vm
+run-tests-on-vm: ## Run the tests on a GCP VM
+	bash utils/run-tests-gcp-vm.sh
+
+.PHONY: cleanup-vm
+cleanup-vm: ## Delete the test GCP VM
+	bash utils/cleanup-vm.sh

tests/README.md

@@ -33,20 +33,24 @@ build-images                   Build NGF and NGINX images
 create-kind-cluster            Create a kind cluster
 delete-kind-cluster            Delete kind cluster
 help                           Display this help
+install-gcp-deps               Install dependencies on a GCP VM. To be ran only from a VM.
 load-images                    Load NGF and NGINX images on configured kind cluster
+reset-etc-hosts                Reset the /etc/hosts file to delete the entry for cafe.example.com
 test                           Run the system tests against your default k8s cluster
 ```
 
 **Note:** The following variables are configurable when running the below `make` commands:
 
-| Variable | Default | Description |
-|----------|---------|-------------|
-| TAG      | edge    | tag for the locally built NGF images |
-| PREFIX   | nginx-gateway-fabric | prefix for the locally built NGF image |
-| NGINX_PREFIX | nginx-gateway-fabric/nginx | prefix for the locally built NGINX image |
-| PULL_POLICY | Never | NGF image pull policy |
-| GW_API_VERSION | 1.0.0 | version of Gateway API resources to install |
-| K8S_VERSION | latest | version of k8s that the tests are run on |
+| Variable            | Default                    | Description                                                    |
+| ------------------- | -------------------------- | -------------------------------------------------------------- |
+| TAG                 | edge                       | tag for the locally built NGF images                           |
+| PREFIX              | nginx-gateway-fabric       | prefix for the locally built NGF image                         |
+| NGINX_PREFIX        | nginx-gateway-fabric/nginx | prefix for the locally built NGINX image                       |
+| PULL_POLICY         | Never                      | NGF image pull policy                                          |
+| GW_API_VERSION      | 1.0.0                      | version of Gateway API resources to install                    |
+| K8S_VERSION         | latest                     | version of k8s that the tests are run on                       |
+| GW_SERVICE_TYPE     | NodePort                   | Type of Service that should be created                         |
+| GW_SVC_GKE_INTERNAL | false                      | Specifies if the LoadBalancer should be a GKE internal service |
 
 ## Step 1 - Create a Kubernetes cluster
 
@@ -74,10 +78,28 @@ make build-images load-images TAG=$(whoami)
 
 ## Step 3 - Run the tests
 
+### 3a - Run the tests locally
+
+The tests require `sudo` access locally to create an entry in the `/etc/hosts` file.
+
+```makefile
+sudo make test TAG=$(whoami)
+```
+
+### 3b - Run the tests on a GKE cluster from a GCP VM
+
+This step only applies if you would like to run the tests from a GCP based VM. The VM should be created in the same
+zone as your GKE cluster, and requires a service account that has Kubernetes admin permissions. Additionally, you need
+ssh access to the VM and the VM needs to have network access to the Kubernetes control node.
+
+Before running the below `make` command, populate the required env vars in `utils/vars.env`.
+
 ```makefile
-make test TAG=$(whoami)
+make run-tests-on-vm
 ```
 
+### Common test amendments
+
 To run a specific test, you can "focus" it by adding the `F` prefix to the name. For example:
 
 ```go
@@ -112,8 +134,22 @@ XIt("runs some test", func(){
 })
 ```
 
-## Step 4 - Delete kind cluster
+## Step 4 - Cleanup
 
-```makefile
-make delete-kind-cluster
-```
+1. Delete kind cluster, if required
+
+    ```makefile
+    make delete-kind-cluster
+    ```
+
+2. Remove entries from `/etc/hosts`, if required
+
+    ```makefile
+    make reset-etc-hosts
+    ```
+
+3. Delete the cloud VM, if required
+
+    ```makefile
+    make cleanup-vm
+    ```

tests/dataplane-performance/setup.md

@@ -29,6 +29,7 @@ type InstallationConfig struct {
 	ImageTag             string
 	ImagePullPolicy      string
 	ServiceType          string
+	IsGKEInternalLB      bool
 }
 
 // InstallGatewayAPI installs the specified version of the Gateway API resources.
@@ -110,6 +111,9 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) {
 
 	if cfg.ServiceType != "" {
 		args = append(args, formatValueSet("service.type", cfg.ServiceType)...)
+		if cfg.ServiceType == "LoadBalancer" && cfg.IsGKEInternalLB {
+			args = append(args, formatValueSet(`service.annotations.networking\.gke\.io\/load-balancer-type`, "Internal")...)
+		}
 	}
 
 	fullArgs := append(args, extraArgs...)

tests/framework/ngf.go

@@ -11,9 +11,9 @@ import (
 )
 
 // Get sends a GET request to the specified url.
-// It resolves to localhost (where the NGF port-forward is running) instead of using DNS.
+// It resolves to the specified address instead of using DNS.
 // The status and body of the response is returned, or an error.
-func Get(url string, timeout time.Duration) (int, string, error) {
+func Get(url string, address string, timeout time.Duration) (int, string, error) {
 	dialer := &net.Dialer{}
 
 	http.DefaultTransport.(*http.Transport).DialContext = func(
@@ -23,7 +23,7 @@ func Get(url string, timeout time.Duration) (int, string, error) {
 	) (net.Conn, error) {
 		split := strings.Split(addr, ":")
 		port := split[len(split)-1]
-		return dialer.DialContext(ctx, network, fmt.Sprintf("127.0.0.1:%s", port))
+		return dialer.DialContext(ctx, network, fmt.Sprintf("%s:%s", address, port))
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)

tests/framework/request.go

@@ -48,6 +48,18 @@ type ResourceManager struct {
 	TimeoutConfig TimeoutConfig
 }
 
+// ClusterInfo holds the cluster metadata
+type ClusterInfo struct {
+	K8sVersion      string
+	MemoryPerNode   string
+	GkeInstanceType string
+	GkeZone         string
+	NodeCount       int
+	CPUCountPerNode int64
+	MaxPodsPerNode  int64
+	IsGKE           bool
+}
+
 // Apply creates or updates Kubernetes resources defined as Go objects.
 func (rm *ResourceManager) Apply(resources []client.Object) error {
 	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout)
@@ -311,3 +323,86 @@ func (rm *ResourceManager) waitForRoutesToBeReady(ctx context.Context, namespace
 		},
 	)
 }
+
+// GetLBIPAddress gets the IP or Hostname from the Loadbalancer service.
+func (rm *ResourceManager) GetLBIPAddress(namespace string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout)
+	defer cancel()
+	var serviceList core.ServiceList
+	var address string
+	if err := rm.K8sClient.List(ctx, &serviceList, client.InNamespace(namespace)); err != nil {
+		return "", err
+	}
+	var nsName types.NamespacedName
+
+	for _, svc := range serviceList.Items {
+		if svc.Spec.Type == core.ServiceTypeLoadBalancer {
+			nsName = types.NamespacedName{Namespace: svc.GetNamespace(), Name: svc.GetName()}
+			if err := rm.waitForLBStatusToBeReady(ctx, nsName); err != nil {
+				return "", fmt.Errorf("Error getting status from LoadBalancer service: %w", err)
+			}
+		}
+	}
+
+	if nsName.Name != "" {
+		var lbService core.Service
+
+		if err := rm.K8sClient.Get(ctx, nsName, &lbService); err != nil {
+			return "", fmt.Errorf("Error getting LoadBalancer service: %w", err)
+		}
+		if lbService.Status.LoadBalancer.Ingress[0].IP != "" {
+			address = lbService.Status.LoadBalancer.Ingress[0].IP
+		} else if lbService.Status.LoadBalancer.Ingress[0].Hostname != "" {
+			address = lbService.Status.LoadBalancer.Ingress[0].Hostname
+		}
+		return address, nil
+	}
+	return "", nil
+}
+
+func (rm *ResourceManager) waitForLBStatusToBeReady(ctx context.Context, svcNsName types.NamespacedName) error {
+	return wait.PollUntilContextCancel(
+		ctx,
+		500*time.Millisecond,
+		true, /* poll immediately */
+		func(ctx context.Context) (bool, error) {
+			var svc core.Service
+			if err := rm.K8sClient.Get(ctx, svcNsName, &svc); err != nil {
+				return false, err
+			}
+			if len(svc.Status.LoadBalancer.Ingress) > 0 {
+				return true, nil
+			}
+
+			return false, nil
+		},
+	)
+}
+
+// GetClusterInfo retrieves node info and Kubernetes version from the cluster
+func (rm *ResourceManager) GetClusterInfo() (ClusterInfo, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout)
+	defer cancel()
+	var nodes core.NodeList
+	ci := &ClusterInfo{}
+	if err := rm.K8sClient.List(ctx, &nodes); err != nil {
+		return *ci, fmt.Errorf("error getting nodes: %w", err)
+	}
+
+	ci.NodeCount = len(nodes.Items)
+
+	node := nodes.Items[0]
+	ci.K8sVersion = node.Status.NodeInfo.KubeletVersion
+	ci.CPUCountPerNode, _ = node.Status.Capacity.Cpu().AsInt64()
+	ci.MemoryPerNode = node.Status.Capacity.Memory().String()
+	ci.MaxPodsPerNode, _ = node.Status.Capacity.Pods().AsInt64()
+	providerID := node.Spec.ProviderID
+
+	if strings.Split(providerID, "://")[0] == "gce" {
+		ci.IsGKE = true
+		ci.GkeInstanceType = node.Labels["beta.kubernetes.io/instance-type"]
+		ci.GkeZone = node.Labels["topology.kubernetes.io/zone"]
+	}
+
+	return *ci, nil
+}