From 8e6c0ff45817470b26f610a252a70bc8470b19e1 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 24 Jul 2023 15:11:02 -0700 Subject: [PATCH 1/6] Add --firestore flag to report Firestore history. --- scripts/gha/report_build_status.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/gha/report_build_status.py b/scripts/gha/report_build_status.py index 1bc1b5cab4..bce7286872 100644 --- a/scripts/gha/report_build_status.py +++ b/scripts/gha/report_build_status.py @@ -112,6 +112,10 @@ "summary_include_crashes", True, "Whether to include CRASH/TIMEOUT in the test summary.") +flags.DEFINE_bool( + "firestore", False, + "Report on Firestore tests rather than on general tests.") + _WORKFLOW_TESTS = 'integration_tests.yml' _WORKFLOW_PACKAGING = 'cpp-packaging.yml' _TRIGGER_USER = 'firebase-workflow-trigger[bot]' @@ -380,11 +384,10 @@ def main(argv): if run['status'] != 'completed': continue if run['day'] < start_date or run['day'] > end_date: continue run['duration'] = dateutil.parser.parse(run['updated_at'], ignoretz=True) - run['date'] - if general_test_time in str(run['date']): + compare_test_time = firestore_test_time if FLAGS.firestore else general_test_time + if compare_test_time in str(run['date']): source_tests[day] = run all_days.add(day) - # elif firestore_test_time in str(run['date']): - # firestore_tests[day] = run workflow_id = _WORKFLOW_PACKAGING all_runs = firebase_github.list_workflow_runs(FLAGS.token, workflow_id, _BRANCH, 'schedule', _LIMIT) From 2cf3ead79062914e07d4890db5569452f35648bf Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 24 Jul 2023 15:19:30 -0700 Subject: [PATCH 2/6] Add Firestore report to nightly cron job. --- .github/workflows/build-report.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml index 9776e62f65..9966a6a439 100644 --- a/.github/workflows/build-report.yml +++ b/.github/workflows/build-report.yml @@ -13,7 +13,12 @@ env: jobs: generate-report: + strategy: + fail-fast: false + matrix: + test_set: [ general, firestore ] runs-on: ubuntu-20.04 + name: generate-report-${{ matrix.test_set }} steps: - name: Setup python uses: actions/setup-python@v4 @@ -36,14 +41,24 @@ jobs: timeout_minutes: 20 max_attempts: 3 shell: bash - command: python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache + command: | + extra_flags= + if [[ "${{ matrix.test_set }}" == "firestore" ]]; then + extra_flags=--firestore + fi + python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache ${extra_flags} - name: Fetch extended GitHub jobs (with retry) uses: nick-invision/retry@v2 with: timeout_minutes: 80 max_attempts: 3 shell: bash - command: python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache + command: | + extra_flags= + if [[ "${{ matrix.test_set }}" == "firestore" ]]; then + extra_flags=--firestore + fi + python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache ${extra_flags} - name: Generate report files run: | python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --output_markdown --read_cache build_status_short.cache > report_short.md @@ -88,4 +103,9 @@ jobs: timeout_minutes: 5 max_attempts: 3 shell: bash - command: python3 scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title '[C++] Nightly Integration Testing Report' --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md + command: | + issue_title='[C++] Nightly Integration Testing Report' + if [[ "${{ matrix.test_set }}" == "firestore" ]]; then + issue_title='[C++] Nightly Integration Testing Report for Firestore' + fi + python3 scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title "${issue_title}" --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md From 6393ae98f767c45f0a402fc9492bf6cc2ce55885 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 24 Jul 2023 15:22:30 -0700 Subject: [PATCH 3/6] Remove multiline commands. --- .github/workflows/build-report.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml index 9966a6a439..2b652f2ac7 100644 --- a/.github/workflows/build-report.yml +++ b/.github/workflows/build-report.yml @@ -16,9 +16,9 @@ jobs: strategy: fail-fast: false matrix: - test_set: [ general, firestore ] + extra_flag: [ nofirestore, firestore ] runs-on: ubuntu-20.04 - name: generate-report-${{ matrix.test_set }} + name: generate-report-${{ matrix.extra_flag }} steps: - name: Setup python uses: actions/setup-python@v4 @@ -41,24 +41,14 @@ jobs: timeout_minutes: 20 max_attempts: 3 shell: bash - command: | - extra_flags= - if [[ "${{ matrix.test_set }}" == "firestore" ]]; then - extra_flags=--firestore - fi - python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache ${extra_flags} + command: python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache --${{ matrix.extra_flag }} - name: Fetch extended GitHub jobs (with retry) uses: nick-invision/retry@v2 with: timeout_minutes: 80 max_attempts: 3 shell: bash - command: | - extra_flags= - if [[ "${{ matrix.test_set }}" == "firestore" ]]; then - extra_flags=--firestore - fi - python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache ${extra_flags} + command: python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache --${{ matrix.extra_flag }} - name: Generate report files run: | python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --output_markdown --read_cache build_status_short.cache > report_short.md @@ -105,7 +95,7 @@ jobs: shell: bash command: | issue_title='[C++] Nightly Integration Testing Report' - if [[ "${{ matrix.test_set }}" == "firestore" ]]; then + if [[ "${{ matrix.extra_flag }}" == "firestore" ]]; then issue_title='[C++] Nightly Integration Testing Report for Firestore' fi python3 scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title "${issue_title}" --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md From 7821f6329fde79f1464625283a06f2de2056dd4d Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 24 Jul 2023 15:23:50 -0700 Subject: [PATCH 4/6] Revert "Remove multiline commands." This reverts commit 6393ae98f767c45f0a402fc9492bf6cc2ce55885. --- .github/workflows/build-report.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml index 2b652f2ac7..9966a6a439 100644 --- a/.github/workflows/build-report.yml +++ b/.github/workflows/build-report.yml @@ -16,9 +16,9 @@ jobs: strategy: fail-fast: false matrix: - extra_flag: [ nofirestore, firestore ] + test_set: [ general, firestore ] runs-on: ubuntu-20.04 - name: generate-report-${{ matrix.extra_flag }} + name: generate-report-${{ matrix.test_set }} steps: - name: Setup python uses: actions/setup-python@v4 @@ -41,14 +41,24 @@ jobs: timeout_minutes: 20 max_attempts: 3 shell: bash - command: python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache --${{ matrix.extra_flag }} + command: | + extra_flags= + if [[ "${{ matrix.test_set }}" == "firestore" ]]; then + extra_flags=--firestore + fi + python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache ${extra_flags} - name: Fetch extended GitHub jobs (with retry) uses: nick-invision/retry@v2 with: timeout_minutes: 80 max_attempts: 3 shell: bash - command: python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache --${{ matrix.extra_flag }} + command: | + extra_flags= + if [[ "${{ matrix.test_set }}" == "firestore" ]]; then + extra_flags=--firestore + fi + python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache ${extra_flags} - name: Generate report files run: | python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --output_markdown --read_cache build_status_short.cache > report_short.md @@ -95,7 +105,7 @@ jobs: shell: bash command: | issue_title='[C++] Nightly Integration Testing Report' - if [[ "${{ matrix.extra_flag }}" == "firestore" ]]; then + if [[ "${{ matrix.test_set }}" == "firestore" ]]; then issue_title='[C++] Nightly Integration Testing Report for Firestore' fi python3 scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title "${issue_title}" --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md From 9c5990da056f60538a5fc1920cb6eee2c8115fe7 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 24 Jul 2023 15:23:59 -0700 Subject: [PATCH 5/6] Fix tabbing. --- .github/workflows/build-report.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml index 9966a6a439..6148047220 100644 --- a/.github/workflows/build-report.yml +++ b/.github/workflows/build-report.yml @@ -42,11 +42,11 @@ jobs: max_attempts: 3 shell: bash command: | - extra_flags= - if [[ "${{ matrix.test_set }}" == "firestore" ]]; then - extra_flags=--firestore - fi - python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache ${extra_flags} + extra_flags= + if [[ "${{ matrix.test_set }}" == "firestore" ]]; then + extra_flags=--firestore + fi + python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache ${extra_flags} - name: Fetch extended GitHub jobs (with retry) uses: nick-invision/retry@v2 with: @@ -54,11 +54,11 @@ jobs: max_attempts: 3 shell: bash command: | - extra_flags= - if [[ "${{ matrix.test_set }}" == "firestore" ]]; then - extra_flags=--firestore - fi - python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache ${extra_flags} + extra_flags= + if [[ "${{ matrix.test_set }}" == "firestore" ]]; then + extra_flags=--firestore + fi + python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache ${extra_flags} - name: Generate report files run: | python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --output_markdown --read_cache build_status_short.cache > report_short.md @@ -104,8 +104,8 @@ jobs: max_attempts: 3 shell: bash command: | - issue_title='[C++] Nightly Integration Testing Report' - if [[ "${{ matrix.test_set }}" == "firestore" ]]; then - issue_title='[C++] Nightly Integration Testing Report for Firestore' - fi - python3 scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title "${issue_title}" --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md + issue_title='[C++] Nightly Integration Testing Report' + if [[ "${{ matrix.test_set }}" == "firestore" ]]; then + issue_title='[C++] Nightly Integration Testing Report for Firestore' + fi + python3 scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title "${issue_title}" --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md From 2376f7e0e6b8fec0fac6efe205193c4051fd5ccb Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 24 Jul 2023 15:55:14 -0700 Subject: [PATCH 6/6] Fix packaging test detection for Firestore. --- .github/workflows/build-report.yml | 3 +++ scripts/gha/report_build_status.py | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml index 6148047220..31ed97d141 100644 --- a/.github/workflows/build-report.yml +++ b/.github/workflows/build-report.yml @@ -20,6 +20,9 @@ jobs: runs-on: ubuntu-20.04 name: generate-report-${{ matrix.test_set }} steps: + - name: Pause 5 minutes to offset Firestore fetches. + if: ${{ matrix.test_set == 'firestore' }} + run: sleep 300 - name: Setup python uses: actions/setup-python@v4 with: diff --git a/scripts/gha/report_build_status.py b/scripts/gha/report_build_status.py index bce7286872..76b6b974df 100644 --- a/scripts/gha/report_build_status.py +++ b/scripts/gha/report_build_status.py @@ -429,9 +429,15 @@ def main(argv): with progress.bar.Bar('Downloading triggered workflow logs...', max=len(package_tests_all)) as bar: for run in package_tests_all: day = str(run['date'].date()) - if day in package_tests and int(package_tests[day]['id']) < int(run['id']): - bar.next() - continue + if day in package_tests: + # Packaging triggers two tests. For Firestore, we want the larger run ID (the second run triggered). + if FLAGS.firestore and int(package_tests[day]['id']) > int(run['id']): + bar.next() + continue + # For general tests we want the smaller run ID (the first run triggered). + if not FLAGS.firestore and int(package_tests[day]['id']) < int(run['id']): + bar.next() + continue packaging_run = 0