From f7f99879c1ad9333d9f7aa9699de3c5086720d6b Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Thu, 3 Aug 2023 10:42:09 -0700 Subject: [PATCH 1/2] Add log to gsutil fetch. --- scripts/gha/integration_testing/ftl_gha_validator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/gha/integration_testing/ftl_gha_validator.py b/scripts/gha/integration_testing/ftl_gha_validator.py index 58119d2315..5216e01a18 100644 --- a/scripts/gha/integration_testing/ftl_gha_validator.py +++ b/scripts/gha/integration_testing/ftl_gha_validator.py @@ -79,7 +79,11 @@ def _gcs_list_dir(gcs_path): """Recursively returns a list of contents for a directory on GCS.""" args = [GSUTIL, "ls", "-r", gcs_path] logging.info("Listing GCS contents: %s", " ".join(args)) - result = subprocess.run(args=args, capture_output=True, text=True, check=True) + try: + result = subprocess.run(args=args, capture_output=True, text=True, check=True) + except subprocess.CalledProcessError as e: + print("Error: %s" % result.stderr) + raise e return result.stdout.splitlines() @@ -88,7 +92,11 @@ def _gcs_read_file(gcs_path): """Extracts the contents of a file on GCS.""" args = [GSUTIL, "cat", gcs_path] logging.info("Reading GCS file: %s", " ".join(args)) - result = subprocess.run(args=args, capture_output=True, text=True, check=True) + try: + result = subprocess.run(args=args, capture_output=True, text=True, check=True) + except subprocess.CalledProcessError as e: + print("Error: %s" % result.stderr) + raise e return result.stdout From 2b89f688764c49670374febbcef359e89c84e667 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Thu, 3 Aug 2023 12:59:36 -0700 Subject: [PATCH 2/2] Fix for an error listing log files. On newer iOS devices, gsutil would return an error like this: CommandException: Cloud folder gs://test-lab-ipzct6ijdt6r6-isxnu6bmc400i/2023-08-03_19:36:33.075378_nNGB/iphone11pro-16.5-en-portrait/CrashLogs/CoreCapture/WiFi/[2023-08-03_14,19,44.533766]=BCMWLAN Firmware Rx DMA Stall~reason=(0x1)/ contains a wildcard; gsutil does not currently support objects with wildcards in their name. It interprets the brackets as a wildcard. However, even though it returns a failure, it's still returning a valid file list. So, catch the error and return the list anyway if it looks like a file list. --- .../gha/integration_testing/ftl_gha_validator.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/gha/integration_testing/ftl_gha_validator.py b/scripts/gha/integration_testing/ftl_gha_validator.py index 5216e01a18..dfa6118ebb 100644 --- a/scripts/gha/integration_testing/ftl_gha_validator.py +++ b/scripts/gha/integration_testing/ftl_gha_validator.py @@ -82,8 +82,15 @@ def _gcs_list_dir(gcs_path): try: result = subprocess.run(args=args, capture_output=True, text=True, check=True) except subprocess.CalledProcessError as e: - print("Error: %s" % result.stderr) - raise e + # It's possible to have a CalledProcessError but still have gotten a file list. + # Check the stdout to see if we got lines that look GCS file paths, and ignore + # the error if so. + output = e.output.splitlines() + if len(output) > 1 and gcs_path in output[0]: + return output + else: + print("Error: %s" % e.stderr) + raise e return result.stdout.splitlines() @@ -95,7 +102,7 @@ def _gcs_read_file(gcs_path): try: result = subprocess.run(args=args, capture_output=True, text=True, check=True) except subprocess.CalledProcessError as e: - print("Error: %s" % result.stderr) + print("Error: %s" % e.stderr) raise e return result.stdout