Fix graceful recovery tests (#2440)

Problem: Graceful recovery tests were failing in the pipeline

Solutions:
- Increase timeout when testing for passing/failing traffic after a container or pod restart. 
The timeout was set to 20 seconds, with a polling interval of 500ms, but the per-request 
timeout was 10 seconds. This means we were only retrying the request twice when a request 
timed out because it couldn't connect. Increasing the overall timeout to 60 seconds increased 
the number of retries and eliminated the errors.
- Remove the kubectl drain flag --delete-local-data since it no longer exists and
 replace it with --delete-emptydir-data.

Author: kate-osborn

tests/framework/timeout.go

@@ -32,6 +32,9 @@ type TimeoutConfig struct {
 
 	// GetStatusTimeout represents the maximum time for NGF to update the status of a resource.
 	GetStatusTimeout time.Duration
+
+	// TestForTrafficTimeout represents the maximum time for NGF to test for passing or failing traffic.
+	TestForTrafficTimeout time.Duration
 }
 
 // DefaultTimeoutConfig populates a TimeoutConfig with the default values.
@@ -47,5 +50,6 @@ func DefaultTimeoutConfig() TimeoutConfig {
 		ContainerRestartTimeout: 10 * time.Second,
 		GetLeaderLeaseTimeout:   60 * time.Second,
 		GetStatusTimeout:        60 * time.Second,
+		TestForTrafficTimeout:   60 * time.Second,
 	}
 }

tests/suite/graceful_recovery_test.go

@@ -82,7 +82,7 @@ var _ = Describe("Graceful Recovery test", Ordered, Label("graceful-recovery"),
 			func() error {
 				return checkForWorkingTraffic(teaURL, coffeeURL)
 			}).
-			WithTimeout(timeoutConfig.RequestTimeout * 2).
+			WithTimeout(timeoutConfig.TestForTrafficTimeout).
 			WithPolling(500 * time.Millisecond).
 			Should(Succeed())
 	})
@@ -133,17 +133,18 @@ func runRestartNodeTest(teaURL, coffeeURL string, files []string, ns *core.Names
 	}
 
 	if drain {
-		_, err := exec.Command(
+		output, err := exec.Command(
 			"kubectl",
 			"drain",
 			kindNodeName,
 			"--ignore-daemonsets",
-			"--delete-local-data",
+			"--delete-emptydir-data",
 		).CombinedOutput()
-		Expect(err).ToNot(HaveOccurred())
 
-		_, err = exec.Command("kubectl", "delete", "node", kindNodeName).CombinedOutput()
-		Expect(err).ToNot(HaveOccurred())
+		Expect(err).ToNot(HaveOccurred(), string(output))
+
+		output, err = exec.Command("kubectl", "delete", "node", kindNodeName).CombinedOutput()
+		Expect(err).ToNot(HaveOccurred(), string(output))
 	}
 
 	_, err = exec.Command("docker", "restart", containerName).CombinedOutput()
@@ -287,6 +288,7 @@ func checkForFailingTraffic(teaURL, coffeeURL string) error {
 
 func expectRequestToSucceed(appURL, address string, responseBodyMessage string) error {
 	status, body, err := framework.Get(appURL, address, timeoutConfig.RequestTimeout)
+
 	if status != http.StatusOK {
 		return errors.New("http status was not 200")
 	}
@@ -320,7 +322,7 @@ func checkNGFFunctionality(teaURL, coffeeURL, ngfPodName, containerName string,
 		func() error {
 			return checkForWorkingTraffic(teaURL, coffeeURL)
 		}).
-		WithTimeout(timeoutConfig.RequestTimeout * 2).
+		WithTimeout(timeoutConfig.TestForTrafficTimeout).
 		WithPolling(500 * time.Millisecond).
 		Should(Succeed())
 
@@ -330,7 +332,7 @@ func checkNGFFunctionality(teaURL, coffeeURL, ngfPodName, containerName string,
 		func() error {
 			return checkForFailingTraffic(teaURL, coffeeURL)
 		}).
-		WithTimeout(timeoutConfig.RequestTimeout).
+		WithTimeout(timeoutConfig.TestForTrafficTimeout).
 		WithPolling(500 * time.Millisecond).
 		Should(Succeed())
 
@@ -341,7 +343,7 @@ func checkNGFFunctionality(teaURL, coffeeURL, ngfPodName, containerName string,
 		func() error {
 			return checkForWorkingTraffic(teaURL, coffeeURL)
 		}).
-		WithTimeout(timeoutConfig.RequestTimeout * 2).
+		WithTimeout(timeoutConfig.TestForTrafficTimeout).
 		WithPolling(500 * time.Millisecond).
 		Should(Succeed())