Automate Graceful Recovery NFR (#1832)

Problem: We want to automate the Graceful Recovery NFR.

Solution: Extend existing automation to cover the following test cases in the Graceful Recovery NFR: "restart nginx-gateway container" and "restart NGINX container".

Testing: Test correctly matches results of manual run of graceful-recovery NFR test.

Author: bjee19

tests/Makefile

@@ -14,7 +14,7 @@ GINKGO_FLAGS=
 NGF_VERSION=
 CI=false
 TELEMETRY_ENDPOINT=
-TELEMETRY_ENDPOINT_INSECURE=
+TELEMETRY_ENDPOINT_INSECURE=false
 
 ifneq ($(GINKGO_LABEL),)
     override GINKGO_FLAGS += --label-filter "$(GINKGO_LABEL)"

tests/framework/request.go

@@ -3,6 +3,7 @@ package framework
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net"
 	"net/http"
@@ -34,7 +35,18 @@ func Get(url, address string, timeout time.Duration) (int, string, error) {
 		return 0, "", err
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	var resp *http.Response
+	if strings.HasPrefix(url, "https") {
+		customTransport := http.DefaultTransport.(*http.Transport).Clone()
+		// similar to how in our examples with https requests we run our curl command
+		// we turn off verification of the certificate, we do the same here
+		customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // for https test traffic
+		client := &http.Client{Transport: customTransport}
+		resp, err = client.Do(req)
+	} else {
+		resp, err = http.DefaultClient.Do(req)
+	}
+
 	if err != nil {
 		return 0, "", err
 	}

tests/framework/resourcemanager.go

@@ -124,12 +124,12 @@ func (rm *ResourceManager) ApplyFromFiles(files []string, namespace string) erro
 }
 
 // Delete deletes Kubernetes resources defined as Go objects.
-func (rm *ResourceManager) Delete(resources []client.Object) error {
+func (rm *ResourceManager) Delete(resources []client.Object, opts ...client.DeleteOption) error {
 	for _, resource := range resources {
 		ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteTimeout)
 		defer cancel()
 
-		if err := rm.K8sClient.Delete(ctx, resource); err != nil && !apierrors.IsNotFound(err) {
+		if err := rm.K8sClient.Delete(ctx, resource, opts...); err != nil && !apierrors.IsNotFound(err) {
 			return fmt.Errorf("error deleting resource: %w", err)
 		}
 	}
@@ -159,7 +159,7 @@ func (rm *ResourceManager) readAndHandleObjects(
 	files []string,
 ) error {
 	for _, file := range files {
-		data, err := rm.getFileContents(file)
+		data, err := rm.GetFileContents(file)
 		if err != nil {
 			return err
 		}
@@ -187,9 +187,9 @@ func (rm *ResourceManager) readAndHandleObjects(
 	return nil
 }
 
-// getFileContents takes a string that can either be a local file
+// GetFileContents takes a string that can either be a local file
 // path or an https:// URL to YAML manifests and provides the contents.
-func (rm *ResourceManager) getFileContents(file string) (*bytes.Buffer, error) {
+func (rm *ResourceManager) GetFileContents(file string) (*bytes.Buffer, error) {
 	if strings.HasPrefix(file, "http://") {
 		return nil, fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", file)
 	} else if strings.HasPrefix(file, "https://") {
@@ -314,7 +314,7 @@ func (rm *ResourceManager) waitForRoutesToBeReady(ctx context.Context, namespace
 
 			var numParents, readyCount int
 			for _, route := range routeList.Items {
-				numParents += len(route.Status.Parents)
+				numParents += len(route.Spec.ParentRefs)
 				for _, parent := range route.Status.Parents {
 					for _, cond := range parent.Conditions {
 						if cond.Type == string(v1.RouteConditionAccepted) && cond.Status == metav1.ConditionTrue {

tests/framework/timeout.go

@@ -17,15 +17,23 @@ type TimeoutConfig struct {
 
 	// RequestTimeout represents the maximum time for making an HTTP Request with the roundtripper.
 	RequestTimeout time.Duration
+
+	// ContainerRestartTimeout represents the maximum time for a Kubernetes Container to restart.
+	ContainerRestartTimeout time.Duration
+
+	// GetLeaderLeaseTimeout represents the maximum time for NGF to retrieve the leader lease.
+	GetLeaderLeaseTimeout time.Duration
 }
 
 // DefaultTimeoutConfig populates a TimeoutConfig with the default values.
 func DefaultTimeoutConfig() TimeoutConfig {
 	return TimeoutConfig{
-		CreateTimeout:        60 * time.Second,
-		DeleteTimeout:        10 * time.Second,
-		GetTimeout:           10 * time.Second,
-		ManifestFetchTimeout: 10 * time.Second,
-		RequestTimeout:       10 * time.Second,
+		CreateTimeout:           60 * time.Second,
+		DeleteTimeout:           10 * time.Second,
+		GetTimeout:              10 * time.Second,
+		ManifestFetchTimeout:    10 * time.Second,
+		RequestTimeout:          10 * time.Second,
+		ContainerRestartTimeout: 10 * time.Second,
+		GetLeaderLeaseTimeout:   60 * time.Second,
 	}
 }