OTel kuttl test: move check for 5 minute metric to last to avoid failures.

Author: dsessler7

testing/kuttl/e2e/otel-logging-and-metrics/04-assert-repo-host-does-not-logs.yaml renamed to testing/kuttl/e2e/otel-logging-and-metrics/02-assert-repo-host-does-not-logs.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- files/03--annotate-cluster.yaml
+assert:
+- files/03-backup-completed.yaml

testing/kuttl/e2e/otel-logging-and-metrics/03--backup.yaml

@@ -6,7 +6,8 @@ commands:
 # and 5s queries are present, as well as patroni metrics.
 # Then, check the collector logs for patroni, pgbackrest, and postgres logs.
 # Finally, ensure the monitoring user exists and is configured.
-- script: |
+- timeout: 400
+  script: |
     retry() { bash -ceu 'printf "$1\nSleeping...\n" && sleep 5' - "$@"; }
     check_containers_ready() { bash -ceu 'echo "$1" | jq -e ".[] | select(.type==\"ContainersReady\") | .status==\"True\""' - "$@"; }
     contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; }
@@ -22,21 +23,6 @@ commands:
       exit 1
     }
 
-    scrape_metrics=$(kubectl exec "${pod}" -c collector -n "${NAMESPACE}" -- \
-      curl --insecure --silent http://localhost:9187/metrics)
-    { contains "${scrape_metrics}" 'ccp_connection_stats_active'; } || {
-      retry "5 second metric not found"
-      exit 1
-    }
-    { contains "${scrape_metrics}" 'ccp_database_size_bytes'; } || {
-      retry "5 minute metric not found"
-      exit 1
-    }
-    { contains "${scrape_metrics}" 'patroni_postgres_running'; } || {
-      retry "patroni metric not found"
-      exit 1
-    }
-
     logs=$(kubectl logs "${pod}" --namespace "${NAMESPACE}" -c collector | grep InstrumentationScope)
     { contains "${logs}" 'InstrumentationScope patroni'; } || {
       retry "patroni logs not found"
@@ -51,6 +37,21 @@ commands:
       exit 1
     }
 
+    scrape_metrics=$(kubectl exec "${pod}" -c collector -n "${NAMESPACE}" -- \
+      curl --insecure --silent http://localhost:9187/metrics)
+    { contains "${scrape_metrics}" 'ccp_connection_stats_active'; } || {
+      retry "5 second metric not found"
+      exit 1
+    }
+    { contains "${scrape_metrics}" 'patroni_postgres_running'; } || {
+      retry "patroni metric not found"
+      exit 1
+    }
+    { contains "${scrape_metrics}" 'ccp_database_size_bytes'; } || {
+      retry "5 minute metric not found"
+      exit 1
+    }
+
     kubectl exec --stdin "${pod}" --namespace "${NAMESPACE}" -c database \
       -- psql -qb --set ON_ERROR_STOP=1 --file=- <<'SQL'
       DO $$