From 7803a78e120b7e966cb8e4774ec6d70e03d73747 Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Wed, 19 Mar 2025 01:40:03 +0000 Subject: [PATCH] Cache fuchsia checkout in a docker layer This allows the Fuchsia checkout to be cached and shared across builds, which avoids any rate limits with checking out Fuchsia. Note though that the Fuchsia checkout is about 27GiB as of checkout, which might cause separate problems. try-job: x86_64-fuchsia --- .../host-x86_64/dist-various-2/Dockerfile | 2 + .../host-x86_64/x86_64-fuchsia/Dockerfile | 2 + .../x86_64-fuchsia/build-fuchsia.sh | 63 +------------- .../x86_64-fuchsia/checkout-fuchsia.sh | 83 +++++++++++++++++++ src/ci/docker/scripts/checkout-fuchsia.sh | 83 +++++++++++++++++++ 5 files changed, 171 insertions(+), 62 deletions(-) create mode 100755 src/ci/docker/host-x86_64/x86_64-fuchsia/checkout-fuchsia.sh create mode 100755 src/ci/docker/scripts/checkout-fuchsia.sh diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile index 03ec77f507e75..461108b24bc6a 100644 --- a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile @@ -84,6 +84,8 @@ WORKDIR /tmp COPY scripts/shared.sh /tmp/ COPY scripts/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh +COPY scripts/checkout-fuchsia.sh /tmp/ +RUN /tmp/checkout-fuchsia.sh COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/ RUN /tmp/build-solaris-toolchain.sh x86_64 amd64 solaris-i386 pc RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc sun diff --git a/src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile b/src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile index 0cae83a85b3a4..dbdfdb2619c5d 100644 --- a/src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile @@ -37,6 +37,8 @@ WORKDIR /tmp COPY scripts/shared.sh /tmp/ COPY scripts/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh +COPY scripts/checkout-fuchsia.sh /tmp/ +RUN /tmp/checkout-fuchsia.sh ENV CARGO_TARGET_X86_64_UNKNOWN_FUCHSIA_AR /usr/local/bin/llvm-ar ENV CARGO_TARGET_X86_64_UNKNOWN_FUCHSIA_RUSTFLAGS \ diff --git a/src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh b/src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh index d0a138b79033b..5ccd38d16111e 100755 --- a/src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh +++ b/src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh @@ -19,68 +19,7 @@ set -euf -o pipefail -# Set this variable to 1 to disable updating the Fuchsia checkout. This is -# useful for making local changes. You can find the Fuchsia checkout in -# `obj/x86_64-fuchsia/fuchsia` in your local checkout after running this -# job for the first time. -KEEP_CHECKOUT= - -# Any upstream refs that should be cherry-picked. This can be used to include -# Gerrit changes from https://fxrev.dev during development (click the "Download" -# button on a changelist to see the cherry pick ref). Example: -# PICK_REFS=(refs/changes/71/1054071/2 refs/changes/74/1054574/2) -PICK_REFS=() - -# The commit hash of Fuchsia's integration.git to check out. This controls the -# commit hash of fuchsia.git and some other repos in the "monorepo" checkout, in -# addition to versions of prebuilts. It should be bumped regularly by the -# Fuchsia team – we aim for every 1-2 months. -INTEGRATION_SHA=f6f83d3e3852209f7752be55694006afbe979e50 - -checkout=fuchsia -jiri=.jiri_root/bin/jiri - -set -x - -if [ -z "$KEEP_CHECKOUT" ]; then - # This script will: - # - create a directory named "fuchsia" if it does not exist - # - download "jiri" to "fuchsia/.jiri_root/bin" - curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" \ - | base64 --decode \ - | bash -s $checkout - - cd $checkout - - $jiri init \ - -partial=true \ - -analytics-opt=false \ - . - - $jiri import \ - -name=integration \ - -revision=$INTEGRATION_SHA \ - -overwrite=true \ - flower \ - "https://fuchsia.googlesource.com/integration" - - if [ -d ".git" ]; then - # Wipe out any local changes if we're reusing a checkout. - git checkout --force JIRI_HEAD - fi - - $jiri update -autoupdate=false - - echo integration commit = $(git -C integration rev-parse HEAD) - - for git_ref in "${PICK_REFS[@]}"; do - git fetch https://fuchsia.googlesource.com/fuchsia $git_ref - git cherry-pick --no-commit FETCH_HEAD - done -else - echo Reusing existing Fuchsia checkout - cd $checkout -fi +cd fuchsia # Run the script inside the Fuchsia checkout responsible for building Fuchsia. # You can change arguments to the build by setting KEEP_CHECKOUT=1 above and diff --git a/src/ci/docker/host-x86_64/x86_64-fuchsia/checkout-fuchsia.sh b/src/ci/docker/host-x86_64/x86_64-fuchsia/checkout-fuchsia.sh new file mode 100755 index 0000000000000..423783326eddd --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-fuchsia/checkout-fuchsia.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# Downloads and builds the Fuchsia operating system using a toolchain installed +# in $RUST_INSTALL_DIR. +# +# You may run this script locally using Docker with the following command: +# +# $ src/ci/docker/run.sh x86_64-fuchsia +# +# Alternatively, from within the container with --dev, assuming you have made it +# as far as building the toolchain with the above command: +# +# $ src/ci/docker/run.sh --dev x86_64-fuchsia +# docker# git config --global --add safe.directory /checkout/obj/fuchsia +# docker# ../src/ci/docker/host-x86_64/x86_64-fuchsia/checkout-fuchsia.sh +# +# Also see the docs in the rustc-dev-guide for more info: +# https://github.com/rust-lang/rustc-dev-guide/pull/1989 + +set -euf -o pipefail + +# Set this variable to 1 to disable updating the Fuchsia checkout. This is +# useful for making local changes. You can find the Fuchsia checkout in +# `obj/x86_64-fuchsia/fuchsia` in your local checkout after running this +# job for the first time. +KEEP_CHECKOUT= + +# Any upstream refs that should be cherry-picked. This can be used to include +# Gerrit changes from https://fxrev.dev during development (click the "Download" +# button on a changelist to see the cherry pick ref). Example: +# PICK_REFS=(refs/changes/71/1054071/2 refs/changes/74/1054574/2) +PICK_REFS=() + +# The commit hash of Fuchsia's integration.git to check out. This controls the +# commit hash of fuchsia.git and some other repos in the "monorepo" checkout, in +# addition to versions of prebuilts. It should be bumped regularly by the +# Fuchsia team – we aim for every 1-2 months. +INTEGRATION_SHA=f6f83d3e3852209f7752be55694006afbe979e50 + +checkout=fuchsia +jiri=.jiri_root/bin/jiri + +set -x + +if [ -z "$KEEP_CHECKOUT" ]; then + # This script will: + # - create a directory named "fuchsia" if it does not exist + # - download "jiri" to "fuchsia/.jiri_root/bin" + curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" \ + | base64 --decode \ + | bash -s $checkout + + cd $checkout + + $jiri init \ + -partial=true \ + -analytics-opt=false \ + . + + $jiri import \ + -name=integration \ + -revision=$INTEGRATION_SHA \ + -overwrite=true \ + flower \ + "https://fuchsia.googlesource.com/integration" + + if [ -d ".git" ]; then + # Wipe out any local changes if we're reusing a checkout. + git checkout --force JIRI_HEAD + fi + + $jiri update -autoupdate=false + + echo integration commit = $(git -C integration rev-parse HEAD) + + for git_ref in "${PICK_REFS[@]}"; do + git fetch https://fuchsia.googlesource.com/fuchsia $git_ref + git cherry-pick --no-commit FETCH_HEAD + done +else + echo Reusing existing Fuchsia checkout + cd $checkout +fi diff --git a/src/ci/docker/scripts/checkout-fuchsia.sh b/src/ci/docker/scripts/checkout-fuchsia.sh new file mode 100755 index 0000000000000..9eb98952f5ccb --- /dev/null +++ b/src/ci/docker/scripts/checkout-fuchsia.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# Downloads and builds the Fuchsia operating system using a toolchain installed +# in $RUST_INSTALL_DIR. +# +# You may run this script locally using Docker with the following command: +# +# $ src/ci/docker/run.sh x86_64-fuchsia +# +# Alternatively, from within the container with --dev, assuming you have made it +# as far as building the toolchain with the above command: +# +# $ src/ci/docker/run.sh --dev x86_64-fuchsia +# docker# git config --global --add safe.directory /checkout/obj/fuchsia +# docker# ../src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh +# +# Also see the docs in the rustc-dev-guide for more info: +# https://github.com/rust-lang/rustc-dev-guide/pull/1989 + +set -euf -o pipefail + +# Set this variable to 1 to disable updating the Fuchsia checkout. This is +# useful for making local changes. You can find the Fuchsia checkout in +# `obj/x86_64-fuchsia/fuchsia` in your local checkout after running this +# job for the first time. +KEEP_CHECKOUT= + +# Any upstream refs that should be cherry-picked. This can be used to include +# Gerrit changes from https://fxrev.dev during development (click the "Download" +# button on a changelist to see the cherry pick ref). Example: +# PICK_REFS=(refs/changes/71/1054071/2 refs/changes/74/1054574/2) +PICK_REFS=() + +# The commit hash of Fuchsia's integration.git to check out. This controls the +# commit hash of fuchsia.git and some other repos in the "monorepo" checkout, in +# addition to versions of prebuilts. It should be bumped regularly by the +# Fuchsia team – we aim for every 1-2 months. +INTEGRATION_SHA=f6f83d3e3852209f7752be55694006afbe979e50 + +checkout=fuchsia +jiri=.jiri_root/bin/jiri + +set -x + +if [ -z "$KEEP_CHECKOUT" ]; then + # This script will: + # - create a directory named "fuchsia" if it does not exist + # - download "jiri" to "fuchsia/.jiri_root/bin" + curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" \ + | base64 --decode \ + | bash -s $checkout + + cd $checkout + + $jiri init \ + -partial=true \ + -analytics-opt=false \ + . + + $jiri import \ + -name=integration \ + -revision=$INTEGRATION_SHA \ + -overwrite=true \ + flower \ + "https://fuchsia.googlesource.com/integration" + + if [ -d ".git" ]; then + # Wipe out any local changes if we're reusing a checkout. + git checkout --force JIRI_HEAD + fi + + $jiri update -autoupdate=false + + echo integration commit = $(git -C integration rev-parse HEAD) + + for git_ref in "${PICK_REFS[@]}"; do + git fetch https://fuchsia.googlesource.com/fuchsia $git_ref + git cherry-pick --no-commit FETCH_HEAD + done +else + echo Reusing existing Fuchsia checkout + cd $checkout +fi