From fa6c880238a24302081fb9a3142347b829f4377e Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 8 May 2025 19:28:51 -0400 Subject: [PATCH 01/56] Add static libraries to post-install script --- swift-ci/sdks/android/scripts/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 879aa1e4..dafdaf2c 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -525,6 +525,11 @@ for swiftrt in ${DESTINATION}/../swift-resources/usr/lib/swift-*/android/*/swift mkdir -p ${DESTINATION}/usr/lib/swift/android/${arch} cp -a ${swiftrt} ${DESTINATION}/usr/lib/swift/android/${arch} done +for swiftrt in ${DESTINATION}/../swift-resources/usr/lib/swift_static-*/android/*/swiftrt.o; do + arch=$(basename $(dirname ${swiftrt})) + mkdir -p ${DESTINATION}/usr/lib/swift_static/android/${arch} + cp -a ${swiftrt} ${DESTINATION}/usr/lib/swift_static/android/${arch} +done echo "$(basename $0): success: ndk-sysroot ${ANDROID_NDK_DESC} to Android SDK" EOF From 3999f1cbaa6775fc585cac95dbfcf9b5c3c0a857 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 10 May 2025 15:49:16 -0400 Subject: [PATCH 02/56] Add post-install script to SDK bundle --- swift-ci/sdks/android/scripts/build.sh | 74 ++++++++++++++++---------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index dafdaf2c..b92130ea 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -326,8 +326,9 @@ for arch in $archs; do -DOPENSSL_INCLUDE_DIR=$sdk_root/usr/include \ -DOPENSSL_SSL_LIBRARY=$sdk_root/usr/lib/libssl.a \ -DOPENSSL_CRYPTO_LIBRARY=$sdk_root/usr/lib/libcrypto.a \ - -DCURL_USE_OPENSSL=ON \ -DCURLSSLOPT_NATIVE_CA=ON \ + -DCURL_USE_OPENSSL=ON \ + -DCURL_USE_LIBSSH2=OFF \ -DTHREADS_PREFER_PTHREAD_FLAG=OFF \ -DCMAKE_THREAD_PREFER_PTHREAD=OFF \ -DCMAKE_THREADS_PREFER_PTHREAD_FLAG=OFF \ @@ -486,7 +487,7 @@ if [[ ${INCLUDE_NDK_SYSROOT} != 1 ]]; then #ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" mkdir scripts/ cat > scripts/setup-android-sdk.sh <<'EOF' -#/bin/sh +#/bin/bash # this script will setup the ndk-sysroot with links to the # local installation indicated by ANDROID_NDK_HOME set -e @@ -494,44 +495,61 @@ if [ -z "${ANDROID_NDK_HOME}" ]; then echo "$(basename $0): error: missing environment variable ANDROID_NDK_HOME" exit 1 fi -PREBUILT="${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt" -if [ ! -d "${PREBUILT}" ]; then - echo "$(basename $0): error: ANDROID_NDK_HOME not found: ${PREBUILT}" +ndk_prebuilt="${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt" +if [ ! -d "${ndk_prebuilt}" ]; then + echo "$(basename $0): error: ANDROID_NDK_HOME not found: ${ndk_prebuilt}" + exit 1 +fi + +#Pkg.Revision = 27.0.12077973 +#Pkg.Revision = 28.1.13356709 +ndk_version=$(grep '^Pkg.Revision = ' "${ANDROID_NDK_HOME}/source.properties" | cut -f3- -d' ' | cut -f 1 -d '.') +if [[ "${ndk_version}" -lt 27 ]]; then + echo "$(basename $0): error: minimum NDK version 27 required; found ${ndk_version} in ${ANDROID_NDK_HOME}/source.properties" exit 1 fi -DESTINATION=$(dirname $(dirname $(realpath $0)))/ndk-sysroot -# clear out any previous NDK setup -rm -rf ${DESTINATION} -# copy vs. link the NDK files +cd $(dirname $(dirname $(realpath $0))) +swift_resources=swift-resources +ndk_sysroot=ndk-sysroot + +if [[ -d "${ndk_sysroot}" ]]; then + # clear out any previous NDK setup + rm -rf ${ndk_sysroot} + ndk_re="re-" +fi + + +# link vs. copy the NDK files SWIFT_ANDROID_NDK_LINK=${SWIFT_ANDROID_NDK_LINK:-1} -if [[ "${SWIFT_ANDROID_NDK_LINK}" != 1 ]]; then - ANDROID_NDK_DESC="copied" - cp -a ${PREBUILT}/*/sysroot ${DESTINATION} -else - ANDROID_NDK_DESC="linked" - mkdir -p ${DESTINATION}/usr/lib - ln -s $(realpath ${PREBUILT}/*/sysroot/usr/include) ${DESTINATION}/usr/include - for triplePath in ${PREBUILT}/*/sysroot/usr/lib/*; do +if [[ "${SWIFT_ANDROID_NDK_LINK}" == 1 ]]; then + ndk_action="${ndk_re}linked" + mkdir -p ${ndk_sysroot}/usr/lib + ln -s ${ndk_prebuilt}/*/sysroot/usr/include ${ndk_sysroot}/usr/include + for triplePath in ${ndk_prebuilt}/*/sysroot/usr/lib/*; do triple=$(basename ${triplePath}) - ln -s $(realpath ${triplePath}) ${DESTINATION}/usr/lib/${triple} + ln -s ${triplePath} ${ndk_sysroot}/usr/lib/${triple} done +else + ndk_action="${ndk_re}copied" + cp -a ${ndk_prebuilt}/*/sysroot ${ndk_sysroot} fi # copy each architecture's swiftrt.o into the sysroot, # working around https://github.com/swiftlang/swift/pull/79621 -for swiftrt in ${DESTINATION}/../swift-resources/usr/lib/swift-*/android/*/swiftrt.o; do - arch=$(basename $(dirname ${swiftrt})) - mkdir -p ${DESTINATION}/usr/lib/swift/android/${arch} - cp -a ${swiftrt} ${DESTINATION}/usr/lib/swift/android/${arch} -done -for swiftrt in ${DESTINATION}/../swift-resources/usr/lib/swift_static-*/android/*/swiftrt.o; do - arch=$(basename $(dirname ${swiftrt})) - mkdir -p ${DESTINATION}/usr/lib/swift_static/android/${arch} - cp -a ${swiftrt} ${DESTINATION}/usr/lib/swift_static/android/${arch} +for folder in swift swift_static; do + for swiftrt in ${swift_resources}/usr/lib/${folder}-*/android/*/swiftrt.o; do + arch=$(basename $(dirname ${swiftrt})) + mkdir -p ${ndk_sysroot}/usr/lib/${folder}/android/${arch} + if [[ "${SWIFT_ANDROID_NDK_LINK}" == 1 ]]; then + ln -s ../../../../../../${swiftrt} ${ndk_sysroot}/usr/lib/${folder}/android/${arch}/ + else + cp -a ${swiftrt} ${ndk_sysroot}/usr/lib/${folder}/android/${arch}/ + fi + done done -echo "$(basename $0): success: ndk-sysroot ${ANDROID_NDK_DESC} to Android SDK" +echo "$(basename $0): success: ndk-sysroot ${ndk_action} to Android NDK at ${ndk_prebuilt}" EOF chmod +x scripts/setup-android-sdk.sh else From 9427b7504bdb74e51f16c6fc4753a280c5975fcc Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Mon, 12 May 2025 13:50:39 -0400 Subject: [PATCH 03/56] Add post-install script to SDK bundle --- swift-ci/sdks/android/build | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build index a2834f71..c2810508 100755 --- a/swift-ci/sdks/android/build +++ b/swift-ci/sdks/android/build @@ -26,10 +26,13 @@ elif [ "${BUILD_VERSION}" = 'devel' ]; then LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" -else +elif [ "${BUILD_VERSION}" = 'trunk' ]; then LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" SWIFT_BRANCH="development" +else + echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + exit 1 fi SWIFT_BASE=$SWIFT_TAG-$HOST_OS From 63e404cdc22e6fec93bd918b87dbd766c62f132f Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 14 May 2025 23:08:18 -0400 Subject: [PATCH 04/56] --- .github/workflows/pull_request.yml | 6 +- swift-ci/sdks/android/build | 141 +++++++---------------------- swift-ci/sdks/android/build-local | 114 +++++++++++++++++++++++ 3 files changed, 152 insertions(+), 109 deletions(-) create mode 100755 swift-ci/sdks/android/build-local diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 582b2b9b..4ecf2d81 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -64,10 +64,14 @@ jobs: - name: Install Dependencies run: | sudo apt install -q ninja-build patchelf - - name: Build Android SDK + - name: Build Android SDK in Docker working-directory: swift-ci/sdks/android run: | BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build + - name: Build Android SDK + working-directory: swift-ci/sdks/android + run: | + BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build-local - name: Get artifact info id: info shell: bash diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build index c2810508..f3f3ff8c 100755 --- a/swift-ci/sdks/android/build +++ b/swift-ci/sdks/android/build @@ -1,114 +1,39 @@ -#!/bin/bash -e -# Swift Android SDK: Top-level Build Script - -# default architectures to build for -TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} - -ANDROID_NDK_VERSION=android-ndk-r27c -ANDROID_API=28 - -SDKROOT=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk -mkdir -p ${SDKROOT} - -# Install a Swift host toolchain if it isn't already present -SWIFT_ROOT=${SDKROOT}/host-toolchain -HOST_OS=ubuntu$(lsb_release -sr) - -#BUILD_VERSION='release' -#BUILD_VERSION='trunk' -BUILD_VERSION=${BUILD_VERSION:-'devel'} - -if [ "${BUILD_VERSION}" = 'release' ]; then - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) - SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" -elif [ "${BUILD_VERSION}" = 'devel' ]; then - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) - SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" -elif [ "${BUILD_VERSION}" = 'trunk' ]; then - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) - SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="development" -else - echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" - exit 1 +#!/bin/bash +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Top-level Build Script +# +# ===----------------------------------------------------------------------=== + +if [[ "$DOCKER" == "" ]]; then + DOCKER=docker fi -SWIFT_BASE=$SWIFT_TAG-$HOST_OS - -if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then - mkdir -p $SWIFT_ROOT - pushd $SWIFT_ROOT - - SWIFT_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" - wget -q $SWIFT_URL - tar xzf $SWIFT_BASE.tar.gz - - popd -fi - -#HOST_TOOLCHAIN=${HOST_TOOLCHAIN:-$(dirname $(dirname $(which swiftc)))} -HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr -$HOST_TOOLCHAIN/bin/swift --version - -# ensure the correct Swift is first in the PATH -export PATH=$HOST_TOOLCHAIN/bin:$PATH - -export ANDROID_NDK_HOME=${SDKROOT}/ndk/${ANDROID_NDK_VERSION} -export ANDROID_NDK=${ANDROID_NDK_HOME} - -if [[ ! -d ${ANDROID_NDK_HOME} ]]; then - mkdir -p $(dirname ${ANDROID_NDK_HOME}) - pushd $(dirname ${ANDROID_NDK_HOME}) - NDKFILE=$(basename $ANDROID_NDK_HOME)-linux.zip - wget -q https://dl.google.com/android/repository/${NDKFILE} - unzip -q ${NDKFILE} - popd -fi - -mkdir -p ${SDKROOT}/products +case $(arch) in + arm64|aarch64) + OS_ARCH_SUFFIX=-aarch64 + ;; + amd64|x86_64) + OS_ARCH_SUFFIX= + ;; + *) + echo "Unknown architecture $(arch)" + exit 1 + ;; +esac + +# Build the Docker image +$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX -t swift-android . # Check-out the sources -if [[ ! -d ${SDKROOT}/src ]]; then - scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} -fi - -# fetch and apply the patches -PATCHDIR=${SDKROOT}/patches -if [[ ! -d ${PATCHDIR} ]]; then - git clone https://github.com/finagolfin/swift-android-sdk.git ${PATCHDIR} - - # TODO: need to selectively apply patches based on release or not release - pushd ${SDKROOT}/src/swift-project - echo "Applying patches" - - # patch the patch, which seems to only be needed for an API less than 28 - # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 - perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $PATCHDIR/swift-android.patch - - # remove the need to link in android-execinfo - perl -pi -e 's/dispatch android-execinfo/dispatch/g' $PATCHDIR/swift-android.patch - - git apply -v $PATCHDIR/swift-android.patch - # swift-android-ci.patch is not needed, since it is only used for llbuild, etc. - #git apply -C1 $PATCHDIR/swift-android-ci.patch - #git apply -v $PATCHDIR/swift-android-ci-release.patch - if [ "${BUILD_VERSION}" = 'release' ]; then - git apply -v $PATCHDIR/swift-android-testing-release.patch - else - git apply -v $PATCHDIR/swift-android-testing-except-release.patch - fi - - perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift - # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) - perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift - - # need to un-apply libandroid-spawn since we don't need it for API28+ - perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt - perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt - popd -fi +scripts/fetch-source.sh --source-dir source -./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} +mkdir -p products +# Run the build +$DOCKER run -it --rm \ + -v ./source:/source \ + -v ./products:/products \ + swift-android \ + /scripts/build.sh --source-dir /source --products-dir /products diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local new file mode 100755 index 00000000..c2810508 --- /dev/null +++ b/swift-ci/sdks/android/build-local @@ -0,0 +1,114 @@ +#!/bin/bash -e +# Swift Android SDK: Top-level Build Script + +# default architectures to build for +TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} + +ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_API=28 + +SDKROOT=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk +mkdir -p ${SDKROOT} + +# Install a Swift host toolchain if it isn't already present +SWIFT_ROOT=${SDKROOT}/host-toolchain +HOST_OS=ubuntu$(lsb_release -sr) + +#BUILD_VERSION='release' +#BUILD_VERSION='trunk' +BUILD_VERSION=${BUILD_VERSION:-'devel'} + +if [ "${BUILD_VERSION}" = 'release' ]; then + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) + SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" +elif [ "${BUILD_VERSION}" = 'devel' ]; then + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) + SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" +elif [ "${BUILD_VERSION}" = 'trunk' ]; then + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) + SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="development" +else + echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + exit 1 +fi + +SWIFT_BASE=$SWIFT_TAG-$HOST_OS + +if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then + mkdir -p $SWIFT_ROOT + pushd $SWIFT_ROOT + + SWIFT_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" + wget -q $SWIFT_URL + tar xzf $SWIFT_BASE.tar.gz + + popd +fi + +#HOST_TOOLCHAIN=${HOST_TOOLCHAIN:-$(dirname $(dirname $(which swiftc)))} +HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr +$HOST_TOOLCHAIN/bin/swift --version + +# ensure the correct Swift is first in the PATH +export PATH=$HOST_TOOLCHAIN/bin:$PATH + +export ANDROID_NDK_HOME=${SDKROOT}/ndk/${ANDROID_NDK_VERSION} +export ANDROID_NDK=${ANDROID_NDK_HOME} + +if [[ ! -d ${ANDROID_NDK_HOME} ]]; then + mkdir -p $(dirname ${ANDROID_NDK_HOME}) + pushd $(dirname ${ANDROID_NDK_HOME}) + NDKFILE=$(basename $ANDROID_NDK_HOME)-linux.zip + wget -q https://dl.google.com/android/repository/${NDKFILE} + unzip -q ${NDKFILE} + popd +fi + +mkdir -p ${SDKROOT}/products + +# Check-out the sources +if [[ ! -d ${SDKROOT}/src ]]; then + scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} +fi + +# fetch and apply the patches +PATCHDIR=${SDKROOT}/patches +if [[ ! -d ${PATCHDIR} ]]; then + git clone https://github.com/finagolfin/swift-android-sdk.git ${PATCHDIR} + + # TODO: need to selectively apply patches based on release or not release + pushd ${SDKROOT}/src/swift-project + echo "Applying patches" + + # patch the patch, which seems to only be needed for an API less than 28 + # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 + perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $PATCHDIR/swift-android.patch + + # remove the need to link in android-execinfo + perl -pi -e 's/dispatch android-execinfo/dispatch/g' $PATCHDIR/swift-android.patch + + git apply -v $PATCHDIR/swift-android.patch + # swift-android-ci.patch is not needed, since it is only used for llbuild, etc. + #git apply -C1 $PATCHDIR/swift-android-ci.patch + #git apply -v $PATCHDIR/swift-android-ci-release.patch + if [ "${BUILD_VERSION}" = 'release' ]; then + git apply -v $PATCHDIR/swift-android-testing-release.patch + else + git apply -v $PATCHDIR/swift-android-testing-except-release.patch + fi + + perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift + # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) + perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift + + # need to un-apply libandroid-spawn since we don't need it for API28+ + perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt + perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt + popd +fi + +./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} + From b583ff8bef847e61069b58d65a9df0ca124a8b38 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Wed, 14 May 2025 23:42:43 -0400 Subject: [PATCH 05/56] --- .gitmodules | 3 + swift-ci/sdks/android/Dockerfile | 88 +++++++++++++++++++ swift-ci/sdks/android/resources/patches | 1 + swift-ci/sdks/android/scripts/install-ndk.sh | 13 +++ .../sdks/android/scripts/install-swift.sh | 56 ++++++++++++ 5 files changed, 161 insertions(+) create mode 100644 .gitmodules create mode 100644 swift-ci/sdks/android/Dockerfile create mode 160000 swift-ci/sdks/android/resources/patches create mode 100755 swift-ci/sdks/android/scripts/install-ndk.sh create mode 100755 swift-ci/sdks/android/scripts/install-swift.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..f3724176 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "swift-ci/sdks/android/resources/patches"] + path = swift-ci/sdks/android/resources/patches + url = https://github.com/swift-android-sdk/swift-android-sdk diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile new file mode 100644 index 00000000..c62b179b --- /dev/null +++ b/swift-ci/sdks/android/Dockerfile @@ -0,0 +1,88 @@ +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Docker-based build +# +# ===----------------------------------------------------------------------=== + +FROM ubuntu:24.04 + +# Versions to fetch + +ARG SWIFT_VERSION=scheme:release/6.2 +ARG LIBXML2_VERSION=2.12.7 +ARG CURL_VERSION=8.7.1 +ARG BORINGSSL_VERSION=fips-20220613 +ARG ICU_VERSION=maint/maint-69 +ARG ZLIB_VERSION=1.3.1 + +# Architecture to build on (empty means x86-64) +ARG OS_ARCH_SUFFIX= + +# ............................................................................ + +# Install development tools +RUN apt-get -q update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q install -y \ + build-essential \ + cmake \ + ninja-build \ + python3 \ + golang \ + git \ + gnupg2 \ + libsqlite3-dev \ + libcurl4-openssl-dev \ + libedit-dev \ + libicu-dev \ + libncurses5-dev \ + libpython3-dev \ + libsqlite3-dev \ + libxml2-dev \ + uuid-dev \ + uuid-runtime \ + tzdata \ + curl \ + && rm -rf /var/lib/apt-lists/* + +# Install Swift +ARG SWIFT_SIGNING_KEY=E813C892820A6FA13755B268F167DF1ACF9CE069 +ARG SWIFT_PLATFORM=ubuntu +ARG OS_MAJOR_VER=24 +ARG OS_MINOR_VER=04 +ARG SWIFT_WEBROOT=https://download.swift.org/swift-6.2-branch + +ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \ + SWIFT_PLATFORM=$SWIFT_PLATFORM \ + OS_MAJOR_VER=$OS_MAJOR_VER \ + OS_MINOR_VER=$OS_MINOR_VER \ + OS_VER=$SWIFT_PLATFORM$OS_MAJOR_VER.$OS_MINOR_VER \ + SWIFT_WEBROOT="$SWIFT_WEBROOT/$SWIFT_PLATFORM$OS_MAJOR_VER$OS_MINOR_VER$OS_ARCH_SUFFIX" + +COPY scripts/install-swift.sh /scripts/install-swift.sh +RUN chmod ugo+x /scripts/install-swift.sh + +RUN /scripts/install-swift.sh + +ENV PATH="/usr/local/swift/bin:${PATH}" + +RUN /scripts/install-ndk.sh + +ENV SWIFT_VERSION=$SWIFT_VERSION \ + LIBXML2_VERSION=$LIBXML2_VERSION \ + CURL_VERSION=$CURL_VERSION \ + BORINGSSL_VERSION=$BORINGSSL_VERSION \ + ICU_VERSION=$ICU_VERSION \ + ZLIB_VERSION=$ZLIB_VERSION + +COPY scripts /scripts +RUN chmod ugo+x /scripts/* + +COPY resources /resources + +# Create a user +RUN groupadd -g 998 build-user && \ + useradd -m -r -u 998 -g build-user build-user + +USER build-user + +WORKDIR /home/build-user diff --git a/swift-ci/sdks/android/resources/patches b/swift-ci/sdks/android/resources/patches new file mode 160000 index 00000000..8ceadf9f --- /dev/null +++ b/swift-ci/sdks/android/resources/patches @@ -0,0 +1 @@ +Subproject commit 8ceadf9f64bfc6149e66f2712f90551f4cc045cd diff --git a/swift-ci/sdks/android/scripts/install-ndk.sh b/swift-ci/sdks/android/scripts/install-ndk.sh new file mode 100755 index 00000000..92a44208 --- /dev/null +++ b/swift-ci/sdks/android/scripts/install-ndk.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Install NDK +# +# ===----------------------------------------------------------------------=== + +set -e + +echo "Installing Android NDK" + + diff --git a/swift-ci/sdks/android/scripts/install-swift.sh b/swift-ci/sdks/android/scripts/install-swift.sh new file mode 100755 index 00000000..c1175d54 --- /dev/null +++ b/swift-ci/sdks/android/scripts/install-swift.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Install Swift +# +# ===----------------------------------------------------------------------=== + +set -e + +echo "Installing Swift" + +# Get latest toolchain info +latest_build=$(curl -s ${SWIFT_WEBROOT}/latest-build.yml) +download=$(echo "$latest_build" | grep '^download: ' | sed 's/^download: //g') +download_signature=$(echo "$latest_build " | grep '^download_signature: ' | sed 's/^download_signature: //g') +download_dir=$(echo "$latest_build" | grep '^dir: ' | sed 's/^dir: //g') + +echo "Latest build is ${download_dir}" + +# Make a temporary directory +tmpdir=$(mktemp -d) +function cleanup { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +pushd "$tmpdir" >/dev/null +export GNUPGHOME="$tmpdir" + +# Fetch the toolchain and signature +echo "Going to fetch ${SWIFT_WEBROOT}/${download_dir}/${download}" + +curl -fsSL "${SWIFT_WEBROOT}/${download_dir}/${download}" -o toolchain.tar.gz + +echo "Going to fetch ${SWIFT_WEBROOT}/${download_dir}/${download_signature}" + +curl -fsSL "${SWIFT_WEBROOT}/${download_dir}/${download_signature}" -o toolchain.sig + +echo "Fetching keys" + +curl -fsSL --compressed https://swift.org/keys/all-keys.asc | gpg --import - + +echo "Verifying signature" + +gpg --batch --verify toolchain.sig toolchain.tar.gz + +# Extract and install the toolchain +echo "Extracting Swift" + +mkdir -p /usr/local/swift +tar -xzf toolchain.tar.gz --directory /usr/local/swift --strip-components=2 +chmod -R o+r /usr/local/swift/lib/swift + +popd >/dev/null + From 57e06fe426969ffd23ae4d79926c31e07f7134ee Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 00:21:30 -0400 Subject: [PATCH 06/56] --- .github/workflows/pull_request.yml | 7 +-- swift-ci/sdks/android/Dockerfile | 11 ++++- swift-ci/sdks/android/build-local | 36 --------------- swift-ci/sdks/android/scripts/build.sh | 47 ++++++++++++++++++++ swift-ci/sdks/android/scripts/install-ndk.sh | 7 +++ 5 files changed, 67 insertions(+), 41 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 4ecf2d81..15962aaf 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -61,13 +61,14 @@ jobs: df -h - name: Checkout repository uses: actions/checkout@v4 - - name: Install Dependencies - run: | - sudo apt install -q ninja-build patchelf - name: Build Android SDK in Docker + continue-on-error: true working-directory: swift-ci/sdks/android run: | BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build + - name: Install Dependencies + run: | + sudo apt install -q ninja-build patchelf - name: Build Android SDK working-directory: swift-ci/sdks/android run: | diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index c62b179b..1bf26d67 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -42,6 +42,7 @@ RUN apt-get -q update \ uuid-runtime \ tzdata \ curl \ + unzip \ && rm -rf /var/lib/apt-lists/* # Install Swift @@ -60,12 +61,18 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \ COPY scripts/install-swift.sh /scripts/install-swift.sh RUN chmod ugo+x /scripts/install-swift.sh - RUN /scripts/install-swift.sh - ENV PATH="/usr/local/swift/bin:${PATH}" + +ARG NDK_VERSION=android-ndk-r27c + +ENV NDK_VERSION=$NDK_VERSION + +COPY scripts/install-ndk.sh /scripts/install-ndk.sh +RUN chmod ugo+x /scripts/install-ndk.sh RUN /scripts/install-ndk.sh +ENV ANDROID_NDK_HOME="/usr/local/ndk/${NDK_VERSION}" ENV SWIFT_VERSION=$SWIFT_VERSION \ LIBXML2_VERSION=$LIBXML2_VERSION \ diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index c2810508..8dfe5028 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -74,41 +74,5 @@ if [[ ! -d ${SDKROOT}/src ]]; then scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} fi -# fetch and apply the patches -PATCHDIR=${SDKROOT}/patches -if [[ ! -d ${PATCHDIR} ]]; then - git clone https://github.com/finagolfin/swift-android-sdk.git ${PATCHDIR} - - # TODO: need to selectively apply patches based on release or not release - pushd ${SDKROOT}/src/swift-project - echo "Applying patches" - - # patch the patch, which seems to only be needed for an API less than 28 - # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 - perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $PATCHDIR/swift-android.patch - - # remove the need to link in android-execinfo - perl -pi -e 's/dispatch android-execinfo/dispatch/g' $PATCHDIR/swift-android.patch - - git apply -v $PATCHDIR/swift-android.patch - # swift-android-ci.patch is not needed, since it is only used for llbuild, etc. - #git apply -C1 $PATCHDIR/swift-android-ci.patch - #git apply -v $PATCHDIR/swift-android-ci-release.patch - if [ "${BUILD_VERSION}" = 'release' ]; then - git apply -v $PATCHDIR/swift-android-testing-release.patch - else - git apply -v $PATCHDIR/swift-android-testing-except-release.patch - fi - - perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift - # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) - perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift - - # need to un-apply libandroid-spawn since we don't need it for API28+ - perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt - perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt - popd -fi - ./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index b92130ea..4d371669 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -238,6 +238,53 @@ function run() { "$@" } +header "Patching Sources" + +quiet_pushd ${source_dir} + PATCHDIR="${resource_dir}/patches" + patch=$(realpath "${PATCHDIR}/swift-android.patch") + + # patch the patch, which seems to only be needed for an API less than 28 + # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 + perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $patch + + # remove the need to link in android-execinfo + perl -pi -e 's/dispatch android-execinfo/dispatch/g' $patch + + if git apply --reverse --check "$patch" >/dev/null 2>&1; then + echo "already patched" + elif git apply "$patch" >/dev/null 2>&1; then + echo "done" + else + echo "failed" + exit 1 + fi + + if [ "${BUILD_VERSION}" = 'release' ]; then + testing_patch=$(realpath "${PATCHDIR}/swift-android-testing-release.patch") + else + testing_patch=$(realpath "${PATCHDIR}/swift-android-testing-except-release.patch") + fi + + if git apply --reverse --check "$testing_patch" >/dev/null 2>&1; then + echo "already patched" + elif git apply "$testing_patch" >/dev/null 2>&1; then + echo "done" + else + echo "failed" + exit 1 + fi + + perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift + # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) + perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift + + # need to un-apply libandroid-spawn since we don't need it for API28+ + perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt + perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt + +quiet_popd + for arch in $archs; do # enable short-circuiting the individual builds if [[ ! -z "$SWIFT_ANDROID_ARCHIVEONLY" ]]; then diff --git a/swift-ci/sdks/android/scripts/install-ndk.sh b/swift-ci/sdks/android/scripts/install-ndk.sh index 92a44208..0476cb97 100755 --- a/swift-ci/sdks/android/scripts/install-ndk.sh +++ b/swift-ci/sdks/android/scripts/install-ndk.sh @@ -10,4 +10,11 @@ set -e echo "Installing Android NDK" +mkdir -p /usr/local/ndk +pushd /usr/local/ndk >/dev/null +NDKFILE=${NDK_VERSION}-linux.zip +curl -fsSL "https://dl.google.com/android/repository/${NDKFILE}" -o ${NDKFILE} +unzip -q ${NDKFILE} +rm ${NDKFILE} +popd >/dev/null From a7bcbd41ca953fc887f63984d2aac4bd8466e3d7 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 00:52:01 -0400 Subject: [PATCH 07/56] --- .github/workflows/pull_request.yml | 3 +++ swift-ci/sdks/android/Dockerfile | 6 ++++-- swift-ci/sdks/android/build | 18 +++++++++++++++++- swift-ci/sdks/android/scripts/install-ndk.sh | 11 ++++++++++- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 15962aaf..29517010 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -61,6 +61,9 @@ jobs: df -h - name: Checkout repository uses: actions/checkout@v4 + with: + submodules: 'true' + - run: ls resources/patches/swift-android.patch - name: Build Android SDK in Docker continue-on-error: true working-directory: swift-ci/sdks/android diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 1bf26d67..24f0ff7e 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -18,6 +18,9 @@ ARG ZLIB_VERSION=1.3.1 # Architecture to build on (empty means x86-64) ARG OS_ARCH_SUFFIX= +# the branch name to use for the Swift build +ARG SWIFT_BRANCH= + # ............................................................................ # Install development tools @@ -50,7 +53,7 @@ ARG SWIFT_SIGNING_KEY=E813C892820A6FA13755B268F167DF1ACF9CE069 ARG SWIFT_PLATFORM=ubuntu ARG OS_MAJOR_VER=24 ARG OS_MINOR_VER=04 -ARG SWIFT_WEBROOT=https://download.swift.org/swift-6.2-branch +ARG SWIFT_WEBROOT=https://download.swift.org/$SWIFT_BRANCH ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \ SWIFT_PLATFORM=$SWIFT_PLATFORM \ @@ -64,7 +67,6 @@ RUN chmod ugo+x /scripts/install-swift.sh RUN /scripts/install-swift.sh ENV PATH="/usr/local/swift/bin:${PATH}" - ARG NDK_VERSION=android-ndk-r27c ENV NDK_VERSION=$NDK_VERSION diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build index f3f3ff8c..968e40ef 100755 --- a/swift-ci/sdks/android/build +++ b/swift-ci/sdks/android/build @@ -23,8 +23,24 @@ case $(arch) in ;; esac +case "${BUILD_VERSION}" in + release) + SWIFT_BRANCH="swift-6.1-branch" + ;; + devel) + SWIFT_BRANCH="swift-6.2-branch" + ;; + trunk) + SWIFT_BRANCH="development" + ;; + *) + echo "Unknown build version: ${BUILD_VERSION}" + exit 1 + ;; +esac + # Build the Docker image -$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX -t swift-android . +$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_BRANCH=$SWIFT_BRANCH -t swift-android . # Check-out the sources scripts/fetch-source.sh --source-dir source diff --git a/swift-ci/sdks/android/scripts/install-ndk.sh b/swift-ci/sdks/android/scripts/install-ndk.sh index 0476cb97..960b9ea2 100755 --- a/swift-ci/sdks/android/scripts/install-ndk.sh +++ b/swift-ci/sdks/android/scripts/install-ndk.sh @@ -12,9 +12,18 @@ echo "Installing Android NDK" mkdir -p /usr/local/ndk pushd /usr/local/ndk >/dev/null + NDKFILE=${NDK_VERSION}-linux.zip -curl -fsSL "https://dl.google.com/android/repository/${NDKFILE}" -o ${NDKFILE} + +NDKURL="https://dl.google.com/android/repository/${NDKFILE}" +echo "Going to fetch ${NDKURL}" + +curl -fsSL "${NDKURL}" -o ${NDKFILE} + +echo "Extracting NDK" unzip -q ${NDKFILE} + rm ${NDKFILE} + popd >/dev/null From d103d49c5f791ef9189e3229b81aa87c0ba5790c Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 00:53:32 -0400 Subject: [PATCH 08/56] --- .github/workflows/pull_request.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 29517010..dc4e2838 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -63,7 +63,9 @@ jobs: uses: actions/checkout@v4 with: submodules: 'true' - - run: ls resources/patches/swift-android.patch + - name: "Check Patches" + working-directory: swift-ci/sdks/android + run: ls resources/patches/ - name: Build Android SDK in Docker continue-on-error: true working-directory: swift-ci/sdks/android From 01d008ba28593b4935e4f370914c000320a3a5c5 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 07:15:02 -0400 Subject: [PATCH 09/56] --- .github/workflows/pull_request.yml | 1 + swift-ci/sdks/android/scripts/build.sh | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index dc4e2838..b8f004cb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -67,6 +67,7 @@ jobs: working-directory: swift-ci/sdks/android run: ls resources/patches/ - name: Build Android SDK in Docker + if: false continue-on-error: true working-directory: swift-ci/sdks/android run: | diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 4d371669..d3ed84ec 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -168,6 +168,8 @@ fi script_dir=$(dirname -- "${BASH_SOURCE[0]}") resource_dir="${script_dir}/../resources" +patches_dir=$(realpath "${script_dir}/../patches") +ls -la ${patches_dir} # Find the version numbers of the various dependencies function describe { @@ -241,8 +243,7 @@ function run() { header "Patching Sources" quiet_pushd ${source_dir} - PATCHDIR="${resource_dir}/patches" - patch=$(realpath "${PATCHDIR}/swift-android.patch") + patch="${patches_dir}/swift-android.patch" # patch the patch, which seems to only be needed for an API less than 28 # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 @@ -261,9 +262,9 @@ quiet_pushd ${source_dir} fi if [ "${BUILD_VERSION}" = 'release' ]; then - testing_patch=$(realpath "${PATCHDIR}/swift-android-testing-release.patch") + testing_patch="${patches_dir}/swift-android-testing-release.patch" else - testing_patch=$(realpath "${PATCHDIR}/swift-android-testing-except-release.patch") + testing_patch="${patches_dir}/swift-android-testing-except-release.patch" fi if git apply --reverse --check "$testing_patch" >/dev/null 2>&1; then From 7d4999814931c06f644dda2d247b15f355b0d867 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 08:09:52 -0400 Subject: [PATCH 10/56] --- .github/workflows/pull_request.yml | 10 ++++------ swift-ci/sdks/android/scripts/build.sh | 5 +++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index b8f004cb..66fc9f82 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -66,12 +66,6 @@ jobs: - name: "Check Patches" working-directory: swift-ci/sdks/android run: ls resources/patches/ - - name: Build Android SDK in Docker - if: false - continue-on-error: true - working-directory: swift-ci/sdks/android - run: | - BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build - name: Install Dependencies run: | sudo apt install -q ninja-build patchelf @@ -79,6 +73,10 @@ jobs: working-directory: swift-ci/sdks/android run: | BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build-local + - name: Build Android SDK in Docker + working-directory: swift-ci/sdks/android + run: | + BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build - name: Get artifact info id: info shell: bash diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index d3ed84ec..88b8b952 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -167,8 +167,9 @@ if ! swiftc=$(which swiftc); then fi script_dir=$(dirname -- "${BASH_SOURCE[0]}") -resource_dir="${script_dir}/../resources" -patches_dir=$(realpath "${script_dir}/../patches") +resource_dir=$(realpath "${script_dir}/../resources") +patches_dir="${resource_dir}/patches" + ls -la ${patches_dir} # Find the version numbers of the various dependencies From 797c594630eaf90ba3c0e84127f94b92f96eac31 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 08:31:06 -0400 Subject: [PATCH 11/56] --- .github/workflows/pull_request.yml | 10 +++++++--- swift-ci/sdks/android/build-local | 8 ++++---- swift-ci/sdks/android/scripts/build.sh | 2 -- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 66fc9f82..05c977da 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -37,7 +37,7 @@ jobs: run: ./build android-build: - name: Build Android ${{ matrix.swift-version }} ${{ matrix.arch }} SDK + name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK strategy: fail-fast: false matrix: @@ -48,6 +48,7 @@ jobs: # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs arch: ['x86_64', ''] swift-version: ['release', 'devel', 'trunk'] + build-type: ['docker', 'local'] runs-on: ubuntu-24.04 steps: - name: Free Disk Space @@ -67,13 +68,16 @@ jobs: working-directory: swift-ci/sdks/android run: ls resources/patches/ - name: Install Dependencies + if: ${{ matrix.build-type == 'local' }} run: | sudo apt install -q ninja-build patchelf - - name: Build Android SDK + - name: Build Android SDK (Local) + if: ${{ matrix.build-type == 'local' }} working-directory: swift-ci/sdks/android run: | BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build-local - - name: Build Android SDK in Docker + - name: Build Android SDK (Docker) + if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android run: | BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 8dfe5028..15039f36 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -69,10 +69,10 @@ fi mkdir -p ${SDKROOT}/products -# Check-out the sources -if [[ ! -d ${SDKROOT}/src ]]; then - scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} -fi +./scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} + +echo "CHECKING SOURCES: ${SDKROOT}/src" +ls -la ${SDKROOT}/src ./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 88b8b952..091c176f 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -170,8 +170,6 @@ script_dir=$(dirname -- "${BASH_SOURCE[0]}") resource_dir=$(realpath "${script_dir}/../resources") patches_dir="${resource_dir}/patches" -ls -la ${patches_dir} - # Find the version numbers of the various dependencies function describe { pushd $1 >/dev/null 2>&1 From ca2b1a14f07cdab9a87d97664628652dcb15c1cc Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 09:42:36 -0400 Subject: [PATCH 12/56] --- swift-ci/sdks/android/build-local | 3 --- swift-ci/sdks/android/scripts/build.sh | 14 +++++--------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 15039f36..fca59a5a 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -71,8 +71,5 @@ mkdir -p ${SDKROOT}/products ./scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} -echo "CHECKING SOURCES: ${SDKROOT}/src" -ls -la ${SDKROOT}/src - ./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 091c176f..ea6a28bf 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -109,7 +109,6 @@ declare_package libxml2 "libxml2" "MIT" \ declare_package curl "curl" "MIT" "https://curl.se" declare_package boringssl "boringssl" "OpenSSL AND ISC AND MIT" \ "https://boringssl.googlesource.com/boringssl/" -declare_package zlib "zlib" "Zlib" "https://zlib.net" # Parse command line arguments android_sdk_version=0.1 @@ -201,8 +200,6 @@ curl_version=${curl_desc#curl-} boringssl_version=$(describe ${source_dir}/boringssl) -zlib_version=$(versionFromTag ${source_dir}/zlib) - function quiet_pushd { pushd "$1" >/dev/null 2>&1 } @@ -232,7 +229,6 @@ echo " - Swift ${swift_version}" echo " - libxml2 ${libxml2_version}" echo " - curl ${curl_version}" echo " - BoringSSL ${boringssl_version}" -echo " - zlib ${zlib_version}" function run() { echo "$@" @@ -241,7 +237,7 @@ function run() { header "Patching Sources" -quiet_pushd ${source_dir} +quiet_pushd ${source_dir}/swift-project patch="${patches_dir}/swift-android.patch" # patch the patch, which seems to only be needed for an API less than 28 @@ -251,9 +247,9 @@ quiet_pushd ${source_dir} # remove the need to link in android-execinfo perl -pi -e 's/dispatch android-execinfo/dispatch/g' $patch - if git apply --reverse --check "$patch" >/dev/null 2>&1; then + if git apply --reverse --check "$patch" ; then echo "already patched" - elif git apply "$patch" >/dev/null 2>&1; then + elif git apply "$patch" ; then echo "done" else echo "failed" @@ -266,9 +262,9 @@ quiet_pushd ${source_dir} testing_patch="${patches_dir}/swift-android-testing-except-release.patch" fi - if git apply --reverse --check "$testing_patch" >/dev/null 2>&1; then + if git apply --reverse --check "$testing_patch" ; then echo "already patched" - elif git apply "$testing_patch" >/dev/null 2>&1; then + elif git apply "$testing_patch" ; then echo "done" else echo "failed" From 78c5f7a9a15836d594641c3cebcf1b6dd1aa9696 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 10:32:29 -0400 Subject: [PATCH 13/56] --- swift-ci/sdks/android/Dockerfile | 6 ++-- swift-ci/sdks/android/build | 21 +++++++++---- swift-ci/sdks/android/build-local | 41 +++++++++++++++----------- swift-ci/sdks/android/scripts/build.sh | 39 +++++++++++------------- 4 files changed, 59 insertions(+), 48 deletions(-) diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 24f0ff7e..2c1d4dbd 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -67,14 +67,14 @@ RUN chmod ugo+x /scripts/install-swift.sh RUN /scripts/install-swift.sh ENV PATH="/usr/local/swift/bin:${PATH}" -ARG NDK_VERSION=android-ndk-r27c +ARG ANDROID_NDK_VERSION=android-ndk-r27c -ENV NDK_VERSION=$NDK_VERSION +ENV ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION COPY scripts/install-ndk.sh /scripts/install-ndk.sh RUN chmod ugo+x /scripts/install-ndk.sh RUN /scripts/install-ndk.sh -ENV ANDROID_NDK_HOME="/usr/local/ndk/${NDK_VERSION}" +ENV ANDROID_NDK_HOME="/usr/local/ndk/${ANDROID_NDK_VERSION}" ENV SWIFT_VERSION=$SWIFT_VERSION \ LIBXML2_VERSION=$LIBXML2_VERSION \ diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build index 968e40ef..76e9c1c2 100755 --- a/swift-ci/sdks/android/build +++ b/swift-ci/sdks/android/build @@ -25,25 +25,34 @@ esac case "${BUILD_VERSION}" in release) - SWIFT_BRANCH="swift-6.1-branch" + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) + SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" ;; devel) - SWIFT_BRANCH="swift-6.2-branch" + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) + SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" ;; trunk) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) + SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" SWIFT_BRANCH="development" ;; *) - echo "Unknown build version: ${BUILD_VERSION}" + echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" exit 1 ;; esac +ANDROID_API=28 +ANDROID_NDK_VERSION=android-ndk-r27c + # Build the Docker image -$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_BRANCH=$SWIFT_BRANCH -t swift-android . +$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_BRANCH=$SWIFT_BRANCH --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . # Check-out the sources -scripts/fetch-source.sh --source-dir source +scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} mkdir -p products @@ -52,4 +61,4 @@ $DOCKER run -it --rm \ -v ./source:/source \ -v ./products:/products \ swift-android \ - /scripts/build.sh --source-dir /source --products-dir /products + /scripts/build.sh --source-dir /source --products-dir /products --android-api ${ANDROID_API} --host-toolchain /usr/local/swift --archs ${TARGET_ARCHS} diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index fca59a5a..7a333707 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -16,24 +16,29 @@ HOST_OS=ubuntu$(lsb_release -sr) #BUILD_VERSION='release' #BUILD_VERSION='trunk' -BUILD_VERSION=${BUILD_VERSION:-'devel'} - -if [ "${BUILD_VERSION}" = 'release' ]; then - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) - SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" -elif [ "${BUILD_VERSION}" = 'devel' ]; then - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) - SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" -elif [ "${BUILD_VERSION}" = 'trunk' ]; then - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) - SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="development" -else - echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" - exit 1 -fi +#BUILD_VERSION=${BUILD_VERSION:-'devel'} + +case "${BUILD_VERSION}" in + release) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) + SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" + ;; + devel) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) + SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" + ;; + trunk) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) + SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="development" + ;; + *) + echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + exit 1 + ;; +esac SWIFT_BASE=$SWIFT_TAG-$HOST_OS diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index ea6a28bf..b2d567bc 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -238,23 +238,14 @@ function run() { header "Patching Sources" quiet_pushd ${source_dir}/swift-project - patch="${patches_dir}/swift-android.patch" + swift_android_patch="${patches_dir}/swift-android.patch" # patch the patch, which seems to only be needed for an API less than 28 # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 - perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $patch + perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch # remove the need to link in android-execinfo - perl -pi -e 's/dispatch android-execinfo/dispatch/g' $patch - - if git apply --reverse --check "$patch" ; then - echo "already patched" - elif git apply "$patch" ; then - echo "done" - else - echo "failed" - exit 1 - fi + perl -pi -e 's/dispatch android-execinfo/dispatch/g' $swift_android_patch if [ "${BUILD_VERSION}" = 'release' ]; then testing_patch="${patches_dir}/swift-android-testing-release.patch" @@ -262,14 +253,21 @@ quiet_pushd ${source_dir}/swift-project testing_patch="${patches_dir}/swift-android-testing-except-release.patch" fi - if git apply --reverse --check "$testing_patch" ; then - echo "already patched" - elif git apply "$testing_patch" ; then - echo "done" - else - echo "failed" - exit 1 - fi + for patch in "$swift_android_patch" "$testing_patch"; do + echo "applying patch $patch in $PWD…" + + if git apply --reverse --check "$patch" >/dev/null 2>&1 ; then + echo "already patched" + elif git apply "$patch" ; then + echo "done" + else + echo "failed to apply patch $patch in $PWD" + # tolerate failed patches in trunk for now + if [[ "${BUILD_VERSION}" != 'trunk' ]]; then + exit 1 + fi + fi + done perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) @@ -689,4 +687,3 @@ if [[ -z "$SWIFT_ANDROID_ARCHIVEONLY" ]]; then fi groupend - From 4927569981e9df81368eb3b1ef0f47e3af2466fa Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 11:22:30 -0400 Subject: [PATCH 14/56] --- swift-ci/sdks/android/Dockerfile | 10 ++++------ swift-ci/sdks/android/build | 9 +++++++-- swift-ci/sdks/android/build-local | 4 ++-- swift-ci/sdks/android/scripts/build.sh | 9 +++------ .../sdks/android/scripts/install-swift.sh | 19 ++++++------------- 5 files changed, 22 insertions(+), 29 deletions(-) diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 2c1d4dbd..9d8de687 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -18,8 +18,8 @@ ARG ZLIB_VERSION=1.3.1 # Architecture to build on (empty means x86-64) ARG OS_ARCH_SUFFIX= -# the branch name to use for the Swift build -ARG SWIFT_BRANCH= +# the Swift toolchain URL to download +ARG SWIFT_TOOLCHAIN_URL= # ............................................................................ @@ -53,18 +53,16 @@ ARG SWIFT_SIGNING_KEY=E813C892820A6FA13755B268F167DF1ACF9CE069 ARG SWIFT_PLATFORM=ubuntu ARG OS_MAJOR_VER=24 ARG OS_MINOR_VER=04 -ARG SWIFT_WEBROOT=https://download.swift.org/$SWIFT_BRANCH ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \ SWIFT_PLATFORM=$SWIFT_PLATFORM \ OS_MAJOR_VER=$OS_MAJOR_VER \ OS_MINOR_VER=$OS_MINOR_VER \ - OS_VER=$SWIFT_PLATFORM$OS_MAJOR_VER.$OS_MINOR_VER \ - SWIFT_WEBROOT="$SWIFT_WEBROOT/$SWIFT_PLATFORM$OS_MAJOR_VER$OS_MINOR_VER$OS_ARCH_SUFFIX" + OS_VER=$SWIFT_PLATFORM$OS_MAJOR_VER.$OS_MINOR_VER COPY scripts/install-swift.sh /scripts/install-swift.sh RUN chmod ugo+x /scripts/install-swift.sh -RUN /scripts/install-swift.sh +RUN /scripts/install-swift.sh "${SWIFT_TOOLCHAIN_URL}" ENV PATH="/usr/local/swift/bin:${PATH}" ARG ANDROID_NDK_VERSION=android-ndk-r27c diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build index 76e9c1c2..c8b18dd1 100755 --- a/swift-ci/sdks/android/build +++ b/swift-ci/sdks/android/build @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -ex # # ===----------------------------------------------------------------------=== # @@ -45,11 +45,16 @@ case "${BUILD_VERSION}" in ;; esac +# FIXME: this is the host operating system, but we need the Docker OS +HOST_OS=ubuntu24.04 +SWIFT_BASE=$SWIFT_TAG-$HOST_OS +SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" + ANDROID_API=28 ANDROID_NDK_VERSION=android-ndk-r27c # Build the Docker image -$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_BRANCH=$SWIFT_BRANCH --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . +$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . # Check-out the sources scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 7a333707..fc9e0cff 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -41,13 +41,13 @@ case "${BUILD_VERSION}" in esac SWIFT_BASE=$SWIFT_TAG-$HOST_OS +SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then mkdir -p $SWIFT_ROOT pushd $SWIFT_ROOT - SWIFT_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" - wget -q $SWIFT_URL + wget -q $SWIFT_TOOLCHAIN_URL tar xzf $SWIFT_BASE.tar.gz popd diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index b2d567bc..6782969d 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -256,16 +256,13 @@ quiet_pushd ${source_dir}/swift-project for patch in "$swift_android_patch" "$testing_patch"; do echo "applying patch $patch in $PWD…" - if git apply --reverse --check "$patch" >/dev/null 2>&1 ; then + if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then echo "already patched" - elif git apply "$patch" ; then + elif git apply -C1 "$patch" ; then echo "done" else echo "failed to apply patch $patch in $PWD" - # tolerate failed patches in trunk for now - if [[ "${BUILD_VERSION}" != 'trunk' ]]; then - exit 1 - fi + exit 1 fi done diff --git a/swift-ci/sdks/android/scripts/install-swift.sh b/swift-ci/sdks/android/scripts/install-swift.sh index c1175d54..03832000 100755 --- a/swift-ci/sdks/android/scripts/install-swift.sh +++ b/swift-ci/sdks/android/scripts/install-swift.sh @@ -8,15 +8,8 @@ set -e -echo "Installing Swift" - -# Get latest toolchain info -latest_build=$(curl -s ${SWIFT_WEBROOT}/latest-build.yml) -download=$(echo "$latest_build" | grep '^download: ' | sed 's/^download: //g') -download_signature=$(echo "$latest_build " | grep '^download_signature: ' | sed 's/^download_signature: //g') -download_dir=$(echo "$latest_build" | grep '^dir: ' | sed 's/^dir: //g') - -echo "Latest build is ${download_dir}" +SWIFT_TOOLCHAIN_URL=$1 +echo "Installing Swift from: ${SWIFT_TOOLCHAIN_URL}" # Make a temporary directory tmpdir=$(mktemp -d) @@ -29,13 +22,13 @@ pushd "$tmpdir" >/dev/null export GNUPGHOME="$tmpdir" # Fetch the toolchain and signature -echo "Going to fetch ${SWIFT_WEBROOT}/${download_dir}/${download}" +echo "Going to fetch ${SWIFT_TOOLCHAIN_URL}" -curl -fsSL "${SWIFT_WEBROOT}/${download_dir}/${download}" -o toolchain.tar.gz +curl -fsSL "${SWIFT_TOOLCHAIN_URL}" -o toolchain.tar.gz -echo "Going to fetch ${SWIFT_WEBROOT}/${download_dir}/${download_signature}" +echo "Going to fetch ${SWIFT_TOOLCHAIN_URL}.sig" -curl -fsSL "${SWIFT_WEBROOT}/${download_dir}/${download_signature}" -o toolchain.sig +curl -fsSL "${SWIFT_TOOLCHAIN_URL}.sig" -o toolchain.sig echo "Fetching keys" From 23db56912bf7ac5e62b8ba65a126da2458afd514 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 11:37:14 -0400 Subject: [PATCH 15/56] --- swift-ci/sdks/android/Dockerfile | 4 ++-- swift-ci/sdks/android/scripts/install-ndk.sh | 8 +++++++- swift-ci/sdks/android/scripts/install-swift.sh | 6 +++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 9d8de687..98718e3a 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -62,10 +62,10 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \ COPY scripts/install-swift.sh /scripts/install-swift.sh RUN chmod ugo+x /scripts/install-swift.sh -RUN /scripts/install-swift.sh "${SWIFT_TOOLCHAIN_URL}" +RUN /scripts/install-swift.sh ENV PATH="/usr/local/swift/bin:${PATH}" -ARG ANDROID_NDK_VERSION=android-ndk-r27c +ARG ANDROID_NDK_VERSION= ENV ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION diff --git a/swift-ci/sdks/android/scripts/install-ndk.sh b/swift-ci/sdks/android/scripts/install-ndk.sh index 960b9ea2..876d730a 100755 --- a/swift-ci/sdks/android/scripts/install-ndk.sh +++ b/swift-ci/sdks/android/scripts/install-ndk.sh @@ -13,7 +13,13 @@ echo "Installing Android NDK" mkdir -p /usr/local/ndk pushd /usr/local/ndk >/dev/null -NDKFILE=${NDK_VERSION}-linux.zip +if [[ "${ANDROID_NDK_VERSION}" == "" ]]; then + echo "$0: Missing ANDROID_NDK_VERSION environment" + exit 1 +fi + + +NDKFILE=${ANDROID_NDK_VERSION}-linux.zip NDKURL="https://dl.google.com/android/repository/${NDKFILE}" echo "Going to fetch ${NDKURL}" diff --git a/swift-ci/sdks/android/scripts/install-swift.sh b/swift-ci/sdks/android/scripts/install-swift.sh index 03832000..eb47f41a 100755 --- a/swift-ci/sdks/android/scripts/install-swift.sh +++ b/swift-ci/sdks/android/scripts/install-swift.sh @@ -8,7 +8,11 @@ set -e -SWIFT_TOOLCHAIN_URL=$1 +if [[ "${SWIFT_TOOLCHAIN_URL}" == "" ]]; then + echo "$0: Missing SWIFT_TOOLCHAIN_URL environment" + exit 1 +fi + echo "Installing Swift from: ${SWIFT_TOOLCHAIN_URL}" # Make a temporary directory From b51597b4c487158213a6c425a94c3149f54e709c Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 11:50:45 -0400 Subject: [PATCH 16/56] --- .github/workflows/pull_request.yml | 10 +-- swift-ci/sdks/android/build | 69 ------------------- swift-ci/sdks/android/build-docker | 45 ++++++++++++ swift-ci/sdks/android/build-local | 41 +++-------- .../sdks/android/scripts/toolchain-vars.sh | 27 ++++++++ 5 files changed, 83 insertions(+), 109 deletions(-) delete mode 100755 swift-ci/sdks/android/build create mode 100755 swift-ci/sdks/android/build-docker create mode 100644 swift-ci/sdks/android/scripts/toolchain-vars.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 05c977da..c5ec1841 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -64,23 +64,17 @@ jobs: uses: actions/checkout@v4 with: submodules: 'true' - - name: "Check Patches" - working-directory: swift-ci/sdks/android - run: ls resources/patches/ - - name: Install Dependencies - if: ${{ matrix.build-type == 'local' }} - run: | - sudo apt install -q ninja-build patchelf - name: Build Android SDK (Local) if: ${{ matrix.build-type == 'local' }} working-directory: swift-ci/sdks/android run: | + sudo apt install -q ninja-build patchelf BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build-local - name: Build Android SDK (Docker) if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android run: | - BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build + BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build-docker - name: Get artifact info id: info shell: bash diff --git a/swift-ci/sdks/android/build b/swift-ci/sdks/android/build deleted file mode 100755 index c8b18dd1..00000000 --- a/swift-ci/sdks/android/build +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -ex -# -# ===----------------------------------------------------------------------=== -# -# Swift Android SDK: Top-level Build Script -# -# ===----------------------------------------------------------------------=== - -if [[ "$DOCKER" == "" ]]; then - DOCKER=docker -fi - -case $(arch) in - arm64|aarch64) - OS_ARCH_SUFFIX=-aarch64 - ;; - amd64|x86_64) - OS_ARCH_SUFFIX= - ;; - *) - echo "Unknown architecture $(arch)" - exit 1 - ;; -esac - -case "${BUILD_VERSION}" in - release) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) - SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" - ;; - devel) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) - SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" - ;; - trunk) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) - SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="development" - ;; - *) - echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" - exit 1 - ;; -esac - -# FIXME: this is the host operating system, but we need the Docker OS -HOST_OS=ubuntu24.04 -SWIFT_BASE=$SWIFT_TAG-$HOST_OS -SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" - -ANDROID_API=28 -ANDROID_NDK_VERSION=android-ndk-r27c - -# Build the Docker image -$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . - -# Check-out the sources -scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} - -mkdir -p products - -# Run the build -$DOCKER run -it --rm \ - -v ./source:/source \ - -v ./products:/products \ - swift-android \ - /scripts/build.sh --source-dir /source --products-dir /products --android-api ${ANDROID_API} --host-toolchain /usr/local/swift --archs ${TARGET_ARCHS} diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker new file mode 100755 index 00000000..296c9388 --- /dev/null +++ b/swift-ci/sdks/android/build-docker @@ -0,0 +1,45 @@ +#!/bin/bash -ex +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Docker Container Build Script +# +# ===----------------------------------------------------------------------=== + +if [[ "$DOCKER" == "" ]]; then + DOCKER=docker +fi + +case $(arch) in + arm64|aarch64) + OS_ARCH_SUFFIX=-aarch64 + ;; + amd64|x86_64) + OS_ARCH_SUFFIX= + ;; + *) + echo "Unknown architecture $(arch)" + exit 1 + ;; +esac + +HOST_OS=ubuntu24.04 +source ./scripts/toolchain-vars.sh + +ANDROID_API=28 +ANDROID_NDK_VERSION=android-ndk-r27c + +# Build the Docker image +$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . + +# Check-out the sources +scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} + +mkdir -p products + +# Run the build +$DOCKER run -it --rm \ + -v ./source:/source \ + -v ./products:/products \ + swift-android \ + /scripts/build.sh --source-dir /source --products-dir /products --android-api ${ANDROID_API} --host-toolchain /usr/local/swift --archs ${TARGET_ARCHS} diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index fc9e0cff..a8dedc3e 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -1,5 +1,10 @@ -#!/bin/bash -e -# Swift Android SDK: Top-level Build Script +#!/bin/bash -ex +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Local (non-Docker-containerized) Build Script +# +# ===----------------------------------------------------------------------=== # default architectures to build for TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} @@ -11,38 +16,10 @@ SDKROOT=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk mkdir -p ${SDKROOT} # Install a Swift host toolchain if it isn't already present -SWIFT_ROOT=${SDKROOT}/host-toolchain HOST_OS=ubuntu$(lsb_release -sr) +source ./scripts/toolchain-vars.sh -#BUILD_VERSION='release' -#BUILD_VERSION='trunk' -#BUILD_VERSION=${BUILD_VERSION:-'devel'} - -case "${BUILD_VERSION}" in - release) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) - SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" - ;; - devel) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) - SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" - ;; - trunk) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) - SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" - SWIFT_BRANCH="development" - ;; - *) - echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" - exit 1 - ;; -esac - -SWIFT_BASE=$SWIFT_TAG-$HOST_OS -SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" - +SWIFT_ROOT=${SDKROOT}/host-toolchain if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then mkdir -p $SWIFT_ROOT pushd $SWIFT_ROOT diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh new file mode 100644 index 00000000..276c56cb --- /dev/null +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e + +case "${BUILD_VERSION}" in + release) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) + SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" + ;; + devel) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) + SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" + ;; + trunk) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) + SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" + SWIFT_BRANCH="development" + ;; + *) + echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + exit 1 + ;; +esac + +SWIFT_BASE=$SWIFT_TAG-$HOST_OS +SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" + From baa2b09ad4adab97ecc3fbbd816e185c2837dc9f Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 11:57:17 -0400 Subject: [PATCH 17/56] --- swift-ci/sdks/android/build-docker | 8 ++++++++ swift-ci/sdks/android/scripts/build.sh | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 296c9388..75487d98 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -32,6 +32,14 @@ ANDROID_NDK_VERSION=android-ndk-r27c # Build the Docker image $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . +echo "Testing docker container" +$DOCKER run -it --rm \ + -v ./source:/source \ + -v ./products:/products \ + swift-android \ + echo "Docker Test" +echo "Done testing docker container" + # Check-out the sources scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 6782969d..23564b7c 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -1,6 +1,6 @@ #!/bin/bash # Swift SDK for Android: Build Script -set -e +set -ex # Docker sets TERM to xterm if using a pty; we probably want # xterm-256color, otherwise we only get eight colors From 3a2dd25eb7cb167a6835a4dd2edcbca3ec4e6422 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 13:20:46 -0400 Subject: [PATCH 18/56] --- swift-ci/sdks/android/build-docker | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 75487d98..62644604 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -33,7 +33,7 @@ ANDROID_NDK_VERSION=android-ndk-r27c $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . echo "Testing docker container" -$DOCKER run -it --rm \ +$DOCKER run --rm \ -v ./source:/source \ -v ./products:/products \ swift-android \ @@ -46,7 +46,7 @@ scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} mkdir -p products # Run the build -$DOCKER run -it --rm \ +$DOCKER run --rm \ -v ./source:/source \ -v ./products:/products \ swift-android \ From c72982507ab2301120cd334d324d626f31da6d56 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 13:38:47 -0400 Subject: [PATCH 19/56] --- swift-ci/sdks/android/build-docker | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 62644604..67e2c79e 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -32,21 +32,13 @@ ANDROID_NDK_VERSION=android-ndk-r27c # Build the Docker image $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . -echo "Testing docker container" -$DOCKER run --rm \ - -v ./source:/source \ - -v ./products:/products \ - swift-android \ - echo "Docker Test" -echo "Done testing docker container" - # Check-out the sources scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} mkdir -p products # Run the build -$DOCKER run --rm \ +$DOCKER run -i --rm \ -v ./source:/source \ -v ./products:/products \ swift-android \ From db2c0dbc51434b8b9ee4483b9a60d084b4724139 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 14:28:07 -0400 Subject: [PATCH 20/56] --- swift-ci/sdks/android/build-docker | 10 ++++++++-- swift-ci/sdks/android/build-local | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 67e2c79e..2f205193 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # # ===----------------------------------------------------------------------=== # @@ -42,4 +42,10 @@ $DOCKER run -i --rm \ -v ./source:/source \ -v ./products:/products \ swift-android \ - /scripts/build.sh --source-dir /source --products-dir /products --android-api ${ANDROID_API} --host-toolchain /usr/local/swift --archs ${TARGET_ARCHS} + /scripts/build.sh \ + --source-dir /source \ + --products-dir /products \ + --host-toolchain /usr/local/swift \ + --android-api ${ANDROID_API} \ + --ndk-home /usr/local/ndk/${ANDROID_NDK_VERSION} \ + --archs ${TARGET_ARCHS} diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index a8dedc3e..20d345d6 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # # ===----------------------------------------------------------------------=== # From ae612f274c072e1cf375183ce0c2270b7865c54b Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 15:04:23 -0400 Subject: [PATCH 21/56] --- swift-ci/sdks/android/build-docker | 4 +- swift-ci/sdks/android/build-local | 2 + swift-ci/sdks/android/scripts/build.sh | 45 -------------- .../sdks/android/scripts/patch-sources.sh | 62 +++++++++++++++++++ 4 files changed, 67 insertions(+), 46 deletions(-) create mode 100755 swift-ci/sdks/android/scripts/patch-sources.sh diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 2f205193..5291ef35 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -33,7 +33,9 @@ ANDROID_NDK_VERSION=android-ndk-r27c $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . # Check-out the sources -scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} +./scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} + +./scripts/patch-sources.sh source mkdir -p products diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 20d345d6..eed84a25 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -53,5 +53,7 @@ mkdir -p ${SDKROOT}/products ./scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} +./scripts/patch-sources.sh ${SDKROOT}/src + ./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 23564b7c..0046044a 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -165,10 +165,6 @@ if ! swiftc=$(which swiftc); then exit 1 fi -script_dir=$(dirname -- "${BASH_SOURCE[0]}") -resource_dir=$(realpath "${script_dir}/../resources") -patches_dir="${resource_dir}/patches" - # Find the version numbers of the various dependencies function describe { pushd $1 >/dev/null 2>&1 @@ -235,47 +231,6 @@ function run() { "$@" } -header "Patching Sources" - -quiet_pushd ${source_dir}/swift-project - swift_android_patch="${patches_dir}/swift-android.patch" - - # patch the patch, which seems to only be needed for an API less than 28 - # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 - perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch - - # remove the need to link in android-execinfo - perl -pi -e 's/dispatch android-execinfo/dispatch/g' $swift_android_patch - - if [ "${BUILD_VERSION}" = 'release' ]; then - testing_patch="${patches_dir}/swift-android-testing-release.patch" - else - testing_patch="${patches_dir}/swift-android-testing-except-release.patch" - fi - - for patch in "$swift_android_patch" "$testing_patch"; do - echo "applying patch $patch in $PWD…" - - if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then - echo "already patched" - elif git apply -C1 "$patch" ; then - echo "done" - else - echo "failed to apply patch $patch in $PWD" - exit 1 - fi - done - - perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift - # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) - perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift - - # need to un-apply libandroid-spawn since we don't need it for API28+ - perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt - perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt - -quiet_popd - for arch in $archs; do # enable short-circuiting the individual builds if [[ ! -z "$SWIFT_ANDROID_ARCHIVEONLY" ]]; then diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh new file mode 100755 index 00000000..936a6a8b --- /dev/null +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Swift Android SDK: Patch Sources +set -e + +source_dir=$1 + +if [[ ! -d "${source_dir}" ]]; then + echo "$0: source_dir ${source_dir} does not exist" + exit 1 +fi + +script_dir=$(dirname -- "${BASH_SOURCE[0]}") +resource_dir=$(realpath "${script_dir}/../resources") +patches_dir="${resource_dir}/patches" + +if [[ ! -d "${patches_dir}" ]]; then + echo "$0: patches_dir ${patches_dir} does not exist" + exit 1 +fi + +if [[ "${BUILD_VERSION}" == "" ]]; then + echo "$0: BUILD_VERSION environment is not set" + exit 1 +fi + +cd ${source_dir}/swift-project +swift_android_patch="${patches_dir}/swift-android.patch" + +# patch the patch, which seems to only be needed for an API less than 28 +# https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 +perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch + +# remove the need to link in android-execinfo +perl -pi -e 's/dispatch android-execinfo/dispatch/g' $swift_android_patch + +if [ "${BUILD_VERSION}" = 'release' ]; then + testing_patch="${patches_dir}/swift-android-testing-release.patch" +else + testing_patch="${patches_dir}/swift-android-testing-except-release.patch" +fi + +for patch in "$swift_android_patch" "$testing_patch"; do + echo "applying patch $patch in $PWD…" + + if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then + echo "already patched" + elif git apply -C1 "$patch" ; then + echo "done" + else + echo "failed to apply patch $patch in $PWD" + exit 1 + fi +done + +perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift +# disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) +perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift + +# need to un-apply libandroid-spawn since we don't need it for API28+ +perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt +perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt + From 7fee5ff35f93c00c266019392b6e0557e6ac6ca4 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 15:36:23 -0400 Subject: [PATCH 22/56] --- swift-ci/sdks/android/build-docker | 12 +++++++----- swift-ci/sdks/android/scripts/build.sh | 17 +++++------------ 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 5291ef35..19139af6 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -6,6 +6,12 @@ # # ===----------------------------------------------------------------------=== +# default architectures to build for +TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} + +ANDROID_NDK_VERSION=android-ndk-r27c +ANDROID_API=28 + if [[ "$DOCKER" == "" ]]; then DOCKER=docker fi @@ -26,15 +32,11 @@ esac HOST_OS=ubuntu24.04 source ./scripts/toolchain-vars.sh -ANDROID_API=28 -ANDROID_NDK_VERSION=android-ndk-r27c - # Build the Docker image $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . -# Check-out the sources +# Check-out and patch the sources ./scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} - ./scripts/patch-sources.sh source mkdir -p products diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 0046044a..549c4033 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -232,11 +232,6 @@ function run() { } for arch in $archs; do - # enable short-circuiting the individual builds - if [[ ! -z "$SWIFT_ANDROID_ARCHIVEONLY" ]]; then - continue - fi - case $arch in armv7) target_host="arm-linux-androideabi"; compiler_target_host="armv7a-linux-androideabi$android_api"; android_abi="armeabi-v7a" ;; aarch64) target_host="aarch64-linux-android"; compiler_target_host="$target_host$android_api"; android_abi="arm64-v8a" ;; @@ -629,13 +624,11 @@ EOF quiet_popd -if [[ -z "$SWIFT_ANDROID_ARCHIVEONLY" ]]; then - header "Outputting compressed bundle" +header "Outputting compressed bundle" - quiet_pushd "${build_dir}" - mkdir -p "${products_dir}" - tar czf "${products_dir}/${bundle}.tar.gz" "${bundle}" - quiet_popd -fi +quiet_pushd "${build_dir}" + mkdir -p "${products_dir}" + tar czf "${products_dir}/${bundle}.tar.gz" "${bundle}" +quiet_popd groupend From c9655b1035916aa84ae5a61468422e451d63d2bd Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 16:19:36 -0400 Subject: [PATCH 23/56] --- swift-ci/sdks/android/build-local | 2 -- .../sdks/android/scripts/patch-sources.sh | 33 ++++++++++++------- .../sdks/android/scripts/toolchain-vars.sh | 23 +++++++++++-- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index eed84a25..dcd5cb75 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -52,8 +52,6 @@ fi mkdir -p ${SDKROOT}/products ./scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} - ./scripts/patch-sources.sh ${SDKROOT}/src ./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} - diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index 936a6a8b..72e6cdc9 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -18,11 +18,6 @@ if [[ ! -d "${patches_dir}" ]]; then exit 1 fi -if [[ "${BUILD_VERSION}" == "" ]]; then - echo "$0: BUILD_VERSION environment is not set" - exit 1 -fi - cd ${source_dir}/swift-project swift_android_patch="${patches_dir}/swift-android.patch" @@ -33,13 +28,28 @@ perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch # remove the need to link in android-execinfo perl -pi -e 's/dispatch android-execinfo/dispatch/g' $swift_android_patch -if [ "${BUILD_VERSION}" = 'release' ]; then - testing_patch="${patches_dir}/swift-android-testing-release.patch" -else - testing_patch="${patches_dir}/swift-android-testing-except-release.patch" -fi +case "${BUILD_VERSION}" in + release) + testing_patch="${patches_dir}/swift-android-testing-release.patch" + ;; + devel) + testing_patch="${patches_dir}/swift-android-testing-except-release.patch" + ;; + trunk) + testing_patch="${patches_dir}/swift-android-testing-except-release.patch" + dispatch_patch="${patches_dir}/swift-android-trunk-libdispatch.patch" + ;; + *) + echo "$0: invalid BUILD_VERSION=${BUILD_VERSION}" + exit 1 + ;; +esac + +for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do + if [[ "${patch}" == "" ]]; then + continue + fi -for patch in "$swift_android_patch" "$testing_patch"; do echo "applying patch $patch in $PWD…" if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then @@ -59,4 +69,3 @@ perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Test # need to un-apply libandroid-spawn since we don't need it for API28+ perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt - diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index 276c56cb..856c6b70 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -1,18 +1,35 @@ #!/bin/bash -e +# +# ===----------------------------------------------------------------------=== +# +# Swift Android SDK: Toolchain source variables +# +# ===----------------------------------------------------------------------=== + +# This script is meant to be sourced from another script that sets the +# BUILD_VERSION environment variable to one of "release", "devel", or "trunk" +# and will set check the latest builds for each build type in order +# to provide information about the Swift tag name in use and where to +# obtain the latest toolchain for building. + +# TODO: we could instead use the latest-build.yml files for this, like: +# https://download.swift.org/swift-6.2-branch/ubuntu2404/latest-build.yml +# https://download.swift.org/development/ubuntu2404/latest-build.yml +# but there doesn't seem to be one for the current release build. case "${BUILD_VERSION}" in release) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/releases | grep -m1 swift-6.1 | cut -d- -f2) SWIFT_TAG="swift-${LATEST_TOOLCHAIN_VERSION}-RELEASE" SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-release" ;; devel) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/tags | grep -m1 swift-6.2-DEV | cut -d- -f8-10) SWIFT_TAG="swift-6.2-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" SWIFT_BRANCH="swift-$(echo $SWIFT_TAG | cut -d- -f2)-branch" ;; trunk) - LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/apple/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) + LATEST_TOOLCHAIN_VERSION=$(curl -sL https://github.com/swiftlang/swift/tags | grep -m1 swift-DEV | cut -d- -f7-9) SWIFT_TAG="swift-DEVELOPMENT-SNAPSHOT-${LATEST_TOOLCHAIN_VERSION}-a" SWIFT_BRANCH="development" ;; From ec2cd6d274f85fcf79df2da8e986441306c67ee8 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 18:54:46 -0400 Subject: [PATCH 24/56] --- swift-ci/sdks/android/scripts/build.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 549c4033..8972d838 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -483,12 +483,30 @@ if [ -z "${ANDROID_NDK_HOME}" ]; then echo "$(basename $0): error: missing environment variable ANDROID_NDK_HOME" exit 1 fi + ndk_prebuilt="${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt" if [ ! -d "${ndk_prebuilt}" ]; then echo "$(basename $0): error: ANDROID_NDK_HOME not found: ${ndk_prebuilt}" exit 1 fi +ndk_sysroot=${ndk_prebuilt}/linux-x86_64/sysroot + +ln -sf "${host_toolchain}/lib/swift" "${ndk_sysroot}/usr/lib" + +# ensure that the expected compiler resource is found +ls -la s/usr/local/ndk/android-ndk-r27c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift/android/x86_64/swiftrt.o + +## need to fixup swiftrt.o in ANDROID_NDK_HOME +#host_swift_dir=$(dirname $(dirname $(which swift))) +#for folder in swift swift_static; do +# for swiftrt in ${swift_resources}/usr/lib/${folder}-*/android/*/swiftrt.o; do +# arch=$(basename $(dirname ${swiftrt})) +# mkdir -p ${ndk_sysroot}/usr/lib/${folder}/android/${arch} +# cp -av ${swiftrt} ${ndk_sysroot}/usr/lib/${folder}/android/${arch}/ +# done +#done + #Pkg.Revision = 27.0.12077973 #Pkg.Revision = 28.1.13356709 ndk_version=$(grep '^Pkg.Revision = ' "${ANDROID_NDK_HOME}/source.properties" | cut -f3- -d' ' | cut -f 1 -d '.') From 5cc85658ce2cdf5bffcaef76e73d2519d03abeea Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 19:17:21 -0400 Subject: [PATCH 25/56] --- swift-ci/sdks/android/scripts/build.sh | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 8972d838..2da146a8 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -210,6 +210,10 @@ HOST=linux-x86_64 #HOST=$(uname -s -m | tr '[:upper:]' '[:lower:]' | tr ' ' '-') ndk_installation=$ndk_home/toolchains/llvm/prebuilt/$HOST +# ANDROID_NDK env needed by the swift-android.patch for: +# call ln -sf "${SWIFT_BUILD_PATH}/lib/swift" "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib" +export ANDROID_NDK=$ndk_home + echo "Swift found at ${swift_dir}" echo "Host toolchain found at ${host_toolchain}" ${host_toolchain}/bin/swift --version @@ -492,21 +496,6 @@ fi ndk_sysroot=${ndk_prebuilt}/linux-x86_64/sysroot -ln -sf "${host_toolchain}/lib/swift" "${ndk_sysroot}/usr/lib" - -# ensure that the expected compiler resource is found -ls -la s/usr/local/ndk/android-ndk-r27c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift/android/x86_64/swiftrt.o - -## need to fixup swiftrt.o in ANDROID_NDK_HOME -#host_swift_dir=$(dirname $(dirname $(which swift))) -#for folder in swift swift_static; do -# for swiftrt in ${swift_resources}/usr/lib/${folder}-*/android/*/swiftrt.o; do -# arch=$(basename $(dirname ${swiftrt})) -# mkdir -p ${ndk_sysroot}/usr/lib/${folder}/android/${arch} -# cp -av ${swiftrt} ${ndk_sysroot}/usr/lib/${folder}/android/${arch}/ -# done -#done - #Pkg.Revision = 27.0.12077973 #Pkg.Revision = 28.1.13356709 ndk_version=$(grep '^Pkg.Revision = ' "${ANDROID_NDK_HOME}/source.properties" | cut -f3- -d' ' | cut -f 1 -d '.') From 3893262b857e6bb79dc17b834894bd1dd825ed8a Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 23:14:03 -0400 Subject: [PATCH 26/56] --- swift-ci/sdks/android/build-local | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index dcd5cb75..888d45cb 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -38,7 +38,6 @@ $HOST_TOOLCHAIN/bin/swift --version export PATH=$HOST_TOOLCHAIN/bin:$PATH export ANDROID_NDK_HOME=${SDKROOT}/ndk/${ANDROID_NDK_VERSION} -export ANDROID_NDK=${ANDROID_NDK_HOME} if [[ ! -d ${ANDROID_NDK_HOME} ]]; then mkdir -p $(dirname ${ANDROID_NDK_HOME}) @@ -51,7 +50,14 @@ fi mkdir -p ${SDKROOT}/products -./scripts/fetch-source.sh --source-dir ${SDKROOT}/src --swift-tag ${SWIFT_TAG} -./scripts/patch-sources.sh ${SDKROOT}/src - -./scripts/build.sh --products-dir ${SDKROOT}/products --source-dir ${SDKROOT}/src --build-dir ${SDKROOT}/build --ndk-home ${ANDROID_NDK_HOME} --android-api ${ANDROID_API} --host-toolchain ${HOST_TOOLCHAIN} --archs ${TARGET_ARCHS} ${@} +./scripts/fetch-source.sh --source-dir ${SDKROOT}/source --swift-tag ${SWIFT_TAG} +./scripts/patch-sources.sh ${SDKROOT}/source + +./scripts/build.sh \ + --source-dir ${SDKROOT}/source \ + --products-dir ${SDKROOT}/products + --build-dir ${SDKROOT}/build \ + --host-toolchain ${HOST_TOOLCHAIN} \ + --android-api ${ANDROID_API} \ + --ndk-home ${ANDROID_NDK_HOME} \ + --archs ${TARGET_ARCHS} ${@} From b20c5463cad418f47b2ce0a14a057df5ba07e633 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Thu, 15 May 2025 23:40:38 -0400 Subject: [PATCH 27/56] --- swift-ci/sdks/android/build-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 888d45cb..4052910d 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -55,7 +55,7 @@ mkdir -p ${SDKROOT}/products ./scripts/build.sh \ --source-dir ${SDKROOT}/source \ - --products-dir ${SDKROOT}/products + --products-dir ${SDKROOT}/products \ --build-dir ${SDKROOT}/build \ --host-toolchain ${HOST_TOOLCHAIN} \ --android-api ${ANDROID_API} \ From e9f41fdcd308db7f82c6ceb24b14de79aefd0948 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 00:21:37 -0400 Subject: [PATCH 28/56] --- swift-ci/sdks/android/build-docker | 1 - swift-ci/sdks/android/build-local | 4 +++- swift-ci/sdks/android/scripts/patch-sources.sh | 12 ++++++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 19139af6..d5a1666f 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -41,7 +41,6 @@ $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLC mkdir -p products -# Run the build $DOCKER run -i --rm \ -v ./source:/source \ -v ./products:/products \ diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 4052910d..25589134 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -48,11 +48,13 @@ if [[ ! -d ${ANDROID_NDK_HOME} ]]; then popd fi -mkdir -p ${SDKROOT}/products +# Check-out and patch the sources ./scripts/fetch-source.sh --source-dir ${SDKROOT}/source --swift-tag ${SWIFT_TAG} ./scripts/patch-sources.sh ${SDKROOT}/source +mkdir -p ${SDKROOT}/products + ./scripts/build.sh \ --source-dir ${SDKROOT}/source \ --products-dir ${SDKROOT}/products \ diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index 72e6cdc9..cdbb686b 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -26,7 +26,11 @@ swift_android_patch="${patches_dir}/swift-android.patch" perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch # remove the need to link in android-execinfo -perl -pi -e 's/dispatch android-execinfo/dispatch/g' $swift_android_patch +perl -pi -e 's;dispatch android-execinfo;dispatch;g' $swift_android_patch + +# debug symbolic link setup +perl -pi -e 's;call ln -sf;call ln -svf;g' $swift_android_patch +perl -pi -e 's%linux-x86_64/sysroot/usr/lib"%linux-x86_64/sysroot/usr/lib"; echo "VALIDATING SYMBOLIC LINK"; ls -la "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib"; ls -la "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift"; %g' $swift_android_patch case "${BUILD_VERSION}" in release) @@ -64,8 +68,8 @@ done perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) -perl -pi -e 's/os\(Android\)/os\(AndroidDISABLED\)/g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift +perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift # need to un-apply libandroid-spawn since we don't need it for API28+ -perl -pi -e 's/MATCHES "Android"/MATCHES "AndroidDISABLED"/g' llbuild/lib/llvm/Support/CMakeLists.txt -perl -pi -e 's/ STREQUAL Android\)/ STREQUAL AndroidDISABLED\)/g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt +perl -pi -e 's;MATCHES "Android";MATCHES "AndroidDISABLED";g' llbuild/lib/llvm/Support/CMakeLists.txt +perl -pi -e 's; STREQUAL Android\); STREQUAL AndroidDISABLED\);g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt From d17d5f22e8183472788749d16887c4da85a1fedb Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 01:03:55 -0400 Subject: [PATCH 29/56] --- swift-ci/sdks/android/build-docker | 4 ++++ swift-ci/sdks/android/build-local | 4 ++++ swift-ci/sdks/android/scripts/patch-sources.sh | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index d5a1666f..4e5b4e71 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -41,6 +41,10 @@ $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLC mkdir -p products +echo "build-script-impl start =========" +cat $(find source -name build-script-impl) +echo "build-script-impl end =========" + $DOCKER run -i --rm \ -v ./source:/source \ -v ./products:/products \ diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 25589134..90b9b4db 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -55,6 +55,10 @@ fi mkdir -p ${SDKROOT}/products +echo "build-script-impl start =========" +cat $(find ${SDKROOT}/source -name build-script-impl) +echo "build-script-impl end =========" + ./scripts/build.sh \ --source-dir ${SDKROOT}/source \ --products-dir ${SDKROOT}/products \ diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index cdbb686b..560c00f8 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -30,7 +30,7 @@ perl -pi -e 's;dispatch android-execinfo;dispatch;g' $swift_android_patch # debug symbolic link setup perl -pi -e 's;call ln -sf;call ln -svf;g' $swift_android_patch -perl -pi -e 's%linux-x86_64/sysroot/usr/lib"%linux-x86_64/sysroot/usr/lib"; echo "VALIDATING SYMBOLIC LINK"; ls -la "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib"; ls -la "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift"; %g' $swift_android_patch +perl -pi -e 's%linux-x86_64/sysroot/usr/lib"%linux-x86_64/sysroot/usr/lib"; echo "VALIDATING SYMBOLIC LINK"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib"; ls -la "\${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/swift"; %g' $swift_android_patch case "${BUILD_VERSION}" in release) From 7bfc73462d83cacf3cc7044b9202b8e2bd313eba Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 08:39:03 -0400 Subject: [PATCH 30/56] --- swift-ci/sdks/android/build-docker | 4 ++-- swift-ci/sdks/android/scripts/patch-sources.sh | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 4e5b4e71..41391799 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -36,8 +36,8 @@ source ./scripts/toolchain-vars.sh $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . # Check-out and patch the sources -./scripts/fetch-source.sh --source-dir source --swift-tag ${SWIFT_TAG} -./scripts/patch-sources.sh source +./scripts/fetch-source.sh --source-dir ${PWD}/source --swift-tag ${SWIFT_TAG} +./scripts/patch-sources.sh ${PWD}/source mkdir -p products diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index 560c00f8..f9b0dd02 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -55,15 +55,16 @@ for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do fi echo "applying patch $patch in $PWD…" + git apply -C1 "$patch" - if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then - echo "already patched" - elif git apply -C1 "$patch" ; then - echo "done" - else - echo "failed to apply patch $patch in $PWD" - exit 1 - fi + #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then + # echo "already patched" + #elif git apply -C1 "$patch" ; then + # echo "done" + #else + # echo "failed to apply patch $patch in $PWD" + # exit 1 + #fi done perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift From bbce180658586bef4148492838ec2ee0884025ce Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 09:45:06 -0400 Subject: [PATCH 31/56] --- swift-ci/sdks/android/build-docker | 5 ++++- swift-ci/sdks/android/build-local | 4 +++- swift-ci/sdks/android/scripts/patch-sources.sh | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 41391799..cca6c78d 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash -ex # # ===----------------------------------------------------------------------=== # @@ -42,8 +42,11 @@ $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLC mkdir -p products echo "build-script-impl start =========" +ls -la $(find ${SDKROOT}/source -name build-script-impl) cat $(find source -name build-script-impl) echo "build-script-impl end =========" +grep 'VALIDATING SYMBOLIC LINK' $(find ${SDKROOT}/source -name build-script-impl) + $DOCKER run -i --rm \ -v ./source:/source \ diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 90b9b4db..4d0da118 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash -ex # # ===----------------------------------------------------------------------=== # @@ -56,8 +56,10 @@ fi mkdir -p ${SDKROOT}/products echo "build-script-impl start =========" +ls -la $(find ${SDKROOT}/source -name build-script-impl) cat $(find ${SDKROOT}/source -name build-script-impl) echo "build-script-impl end =========" +grep 'VALIDATING SYMBOLIC LINK' $(find ${SDKROOT}/source -name build-script-impl) ./scripts/build.sh \ --source-dir ${SDKROOT}/source \ diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index f9b0dd02..4fabb9ad 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -55,7 +55,7 @@ for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do fi echo "applying patch $patch in $PWD…" - git apply -C1 "$patch" + git apply -v -C1 "$patch" #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then # echo "already patched" From 713ac904428fb219e08051ffcf32b36197883a95 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 09:49:43 -0400 Subject: [PATCH 32/56] Update submodules --- swift-ci/sdks/android/resources/patches | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/resources/patches b/swift-ci/sdks/android/resources/patches index 8ceadf9f..d0828093 160000 --- a/swift-ci/sdks/android/resources/patches +++ b/swift-ci/sdks/android/resources/patches @@ -1 +1 @@ -Subproject commit 8ceadf9f64bfc6149e66f2712f90551f4cc045cd +Subproject commit d08280938ff70b790bbd0f559c0b420b518877ca From f73aff4604b0d4c691a5999d0228c2cbff65ccdf Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 10:14:03 -0400 Subject: [PATCH 33/56] --- swift-ci/sdks/android/build-docker | 4 ++-- swift-ci/sdks/android/scripts/patch-sources.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index cca6c78d..97b4ce09 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -42,10 +42,10 @@ $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLC mkdir -p products echo "build-script-impl start =========" -ls -la $(find ${SDKROOT}/source -name build-script-impl) +ls -la $(find source -name build-script-impl) cat $(find source -name build-script-impl) echo "build-script-impl end =========" -grep 'VALIDATING SYMBOLIC LINK' $(find ${SDKROOT}/source -name build-script-impl) +grep 'VALIDATING SYMBOLIC LINK' $(find source -name build-script-impl) $DOCKER run -i --rm \ diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index 4fabb9ad..889c0bbe 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -55,7 +55,7 @@ for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do fi echo "applying patch $patch in $PWD…" - git apply -v -C1 "$patch" + git apply -v --unsafe-paths -C1 "$patch" #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then # echo "already patched" From 5347bfd7f2045976cc7eb35fe266b0d0323335e1 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 10:53:20 -0400 Subject: [PATCH 34/56] --- swift-ci/sdks/android/scripts/patch-sources.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index 889c0bbe..2629009d 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -21,6 +21,10 @@ fi cd ${source_dir}/swift-project swift_android_patch="${patches_dir}/swift-android.patch" +ls -la swift/utils/build-script-impl +ls -la ${swift_android_patch} +ls -la swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift + # patch the patch, which seems to only be needed for an API less than 28 # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch @@ -55,7 +59,9 @@ for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do fi echo "applying patch $patch in $PWD…" - git apply -v --unsafe-paths -C1 "$patch" + # first check to make sure the patches can apply and fail if not + git apply -v --check -C1 "$patch" + git apply -v -C1 "$patch" #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then # echo "already patched" From f5ad6af43043ba41afe83b4db50707517cfc6599 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 11:19:41 -0400 Subject: [PATCH 35/56] --- swift-ci/sdks/android/build-docker | 28 ++++++++----------- swift-ci/sdks/android/build-local | 6 ---- .../sdks/android/scripts/patch-sources.sh | 11 +++++--- 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 97b4ce09..7df7ad47 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -11,6 +11,14 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 +HOST_OS=ubuntu24.04 +source ./scripts/toolchain-vars.sh + +# Check-out and patch the sources +./scripts/fetch-source.sh --source-dir ${PWD}/source --swift-tag ${SWIFT_TAG} +./scripts/patch-sources.sh ${PWD}/source + +mkdir -p products if [[ "$DOCKER" == "" ]]; then DOCKER=docker @@ -29,29 +37,15 @@ case $(arch) in ;; esac -HOST_OS=ubuntu24.04 -source ./scripts/toolchain-vars.sh +CONTAINER_NAME="swift-android" # Build the Docker image -$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t swift-android . - -# Check-out and patch the sources -./scripts/fetch-source.sh --source-dir ${PWD}/source --swift-tag ${SWIFT_TAG} -./scripts/patch-sources.sh ${PWD}/source - -mkdir -p products - -echo "build-script-impl start =========" -ls -la $(find source -name build-script-impl) -cat $(find source -name build-script-impl) -echo "build-script-impl end =========" -grep 'VALIDATING SYMBOLIC LINK' $(find source -name build-script-impl) - +$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t ${CONTAINER_NAME} . $DOCKER run -i --rm \ -v ./source:/source \ -v ./products:/products \ - swift-android \ + ${CONTAINER_NAME} \ /scripts/build.sh \ --source-dir /source \ --products-dir /products \ diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 4d0da118..53563fcf 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -55,12 +55,6 @@ fi mkdir -p ${SDKROOT}/products -echo "build-script-impl start =========" -ls -la $(find ${SDKROOT}/source -name build-script-impl) -cat $(find ${SDKROOT}/source -name build-script-impl) -echo "build-script-impl end =========" -grep 'VALIDATING SYMBOLIC LINK' $(find ${SDKROOT}/source -name build-script-impl) - ./scripts/build.sh \ --source-dir ${SDKROOT}/source \ --products-dir ${SDKROOT}/products \ diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index 2629009d..3cccfb6d 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -21,10 +21,6 @@ fi cd ${source_dir}/swift-project swift_android_patch="${patches_dir}/swift-android.patch" -ls -la swift/utils/build-script-impl -ls -la ${swift_android_patch} -ls -la swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift - # patch the patch, which seems to only be needed for an API less than 28 # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch @@ -61,6 +57,7 @@ for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do echo "applying patch $patch in $PWD…" # first check to make sure the patches can apply and fail if not git apply -v --check -C1 "$patch" + echo "git apply --check result: $0" git apply -v -C1 "$patch" #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then @@ -73,6 +70,11 @@ for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do #fi done +# validate the patches +ls -la swift/utils/build-script-impl +grep 'VALIDATING SYMBOLIC LINK' swift/utils/build-script-impl + + perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift @@ -80,3 +82,4 @@ perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Test # need to un-apply libandroid-spawn since we don't need it for API28+ perl -pi -e 's;MATCHES "Android";MATCHES "AndroidDISABLED";g' llbuild/lib/llvm/Support/CMakeLists.txt perl -pi -e 's; STREQUAL Android\); STREQUAL AndroidDISABLED\);g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt + From 6f79e4c3a60dca045f41003ecd50c7d7a212352f Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 11:28:20 -0400 Subject: [PATCH 36/56] --- swift-ci/sdks/android/build-docker | 15 ++++++++++----- swift-ci/sdks/android/build-local | 22 +++++++++++----------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 7df7ad47..ea60bd2b 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -11,14 +11,19 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 + +# note that WORKDIR must not be under the current checkout or the patches will fail to apply +WORKDIR=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk +mkdir -p ${WORKDIR} + HOST_OS=ubuntu24.04 source ./scripts/toolchain-vars.sh # Check-out and patch the sources -./scripts/fetch-source.sh --source-dir ${PWD}/source --swift-tag ${SWIFT_TAG} -./scripts/patch-sources.sh ${PWD}/source +./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} +./scripts/patch-sources.sh ${WORKDIR}/source -mkdir -p products +mkdir -p ${WORKDIR}/products if [[ "$DOCKER" == "" ]]; then DOCKER=docker @@ -43,8 +48,8 @@ CONTAINER_NAME="swift-android" $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t ${CONTAINER_NAME} . $DOCKER run -i --rm \ - -v ./source:/source \ - -v ./products:/products \ + -v ${WORKDIR}/source:/source \ + -v ${WORKDIR}/products:/products \ ${CONTAINER_NAME} \ /scripts/build.sh \ --source-dir /source \ diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 53563fcf..119c5f13 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -12,14 +12,14 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 -SDKROOT=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk -mkdir -p ${SDKROOT} +# note that WORKDIR must not be under the current checkout or the patches will fail to apply +WORKDIR=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk +mkdir -p ${WORKDIR} -# Install a Swift host toolchain if it isn't already present HOST_OS=ubuntu$(lsb_release -sr) source ./scripts/toolchain-vars.sh -SWIFT_ROOT=${SDKROOT}/host-toolchain +SWIFT_ROOT=${WORKDIR}/host-toolchain if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then mkdir -p $SWIFT_ROOT pushd $SWIFT_ROOT @@ -37,7 +37,7 @@ $HOST_TOOLCHAIN/bin/swift --version # ensure the correct Swift is first in the PATH export PATH=$HOST_TOOLCHAIN/bin:$PATH -export ANDROID_NDK_HOME=${SDKROOT}/ndk/${ANDROID_NDK_VERSION} +export ANDROID_NDK_HOME=${WORKDIR}/ndk/${ANDROID_NDK_VERSION} if [[ ! -d ${ANDROID_NDK_HOME} ]]; then mkdir -p $(dirname ${ANDROID_NDK_HOME}) @@ -50,15 +50,15 @@ fi # Check-out and patch the sources -./scripts/fetch-source.sh --source-dir ${SDKROOT}/source --swift-tag ${SWIFT_TAG} -./scripts/patch-sources.sh ${SDKROOT}/source +./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG} +./scripts/patch-sources.sh ${WORKDIR}/source -mkdir -p ${SDKROOT}/products +mkdir -p ${WORKDIR}/products ./scripts/build.sh \ - --source-dir ${SDKROOT}/source \ - --products-dir ${SDKROOT}/products \ - --build-dir ${SDKROOT}/build \ + --source-dir ${WORKDIR}/source \ + --products-dir ${WORKDIR}/products \ + --build-dir ${WORKDIR}/build \ --host-toolchain ${HOST_TOOLCHAIN} \ --android-api ${ANDROID_API} \ --ndk-home ${ANDROID_NDK_HOME} \ From 054ca78caf21f1ff984a513030aea3f8e0e77b09 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 12:16:57 -0400 Subject: [PATCH 37/56] --- swift-ci/sdks/android/scripts/build.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 2da146a8..3d2df479 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -208,10 +208,22 @@ header "Swift Android SDK build script" swift_dir=$(realpath $(dirname "$swiftc")/..) HOST=linux-x86_64 #HOST=$(uname -s -m | tr '[:upper:]' '[:lower:]' | tr ' ' '-') + +# in a Docker container, the pre-installed NDK is read-only, +# but the build script needs to write to it to work around +# https://github.com/swiftlang/swift-driver/pull/1822 +# so we copy it to a read-write location for the purposes of the build +# this can all be removed once that PR lands +mkdir -p ${build_dir}/ndk/ +ndk_home_tmp=${build_dir}/ndk/$(basename $ndk_home) +cp -a $ndk_home $ndk_home_tmp +ndk_home=$ndk_home_tmp + ndk_installation=$ndk_home/toolchains/llvm/prebuilt/$HOST # ANDROID_NDK env needed by the swift-android.patch for: # call ln -sf "${SWIFT_BUILD_PATH}/lib/swift" "${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib" +export ANDROID_NDK_HOME=$ndk_home export ANDROID_NDK=$ndk_home echo "Swift found at ${swift_dir}" From 247f6955113f1c4454aac5de4468d40a398c0541 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 13:02:35 -0400 Subject: [PATCH 38/56] --- .github/workflows/pull_request.yml | 3 +++ swift-ci/sdks/android/Dockerfile | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c5ec1841..7fca437d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -95,6 +95,9 @@ jobs: if [[ ! -z "${{ matrix.arch }}" ]]; then ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')" fi + if [[ "${{ matrix.build-type }}" == 'local' ]]; then + ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')-local" + fi ARTIFACT_NAME="${ARTIFACT_NAME}${ARTIFACT_EXT}" # There is no way to prevent even a single-file artifact from being zipped: diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 98718e3a..1f74c6ab 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -41,6 +41,7 @@ RUN apt-get -q update \ libpython3-dev \ libsqlite3-dev \ libxml2-dev \ + rsync \ uuid-dev \ uuid-runtime \ tzdata \ From e11112e6e030dd74e3e76591c7fb5fddda21905d Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 16:58:34 -0400 Subject: [PATCH 39/56] --- .github/workflows/pull_request.yml | 7 ++++--- swift-ci/sdks/android/Dockerfile | 11 ++--------- swift-ci/sdks/android/build-docker | 2 +- swift-ci/sdks/android/scripts/build.sh | 5 +++++ 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7fca437d..6369cec0 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -91,13 +91,14 @@ jobs: ARTIFACT_EXT=".artifactbundle.tar.gz" ARTIFACT_NAME="$(basename ${ARTIFACT_PATH} ${ARTIFACT_EXT})" + # depending on whether we are building locally or in a container, add a maker to the name + if [[ "${{ matrix.build-type }}" == 'local' ]]; then + ARTIFACT_NAME="${ARTIFACT_NAME}-local" + fi # artifacts need a unique name so we suffix with the matrix arch(s) if [[ ! -z "${{ matrix.arch }}" ]]; then ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')" fi - if [[ "${{ matrix.build-type }}" == 'local' ]]; then - ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')-local" - fi ARTIFACT_NAME="${ARTIFACT_NAME}${ARTIFACT_EXT}" # There is no way to prevent even a single-file artifact from being zipped: diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 1f74c6ab..5f59f812 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -6,15 +6,6 @@ FROM ubuntu:24.04 -# Versions to fetch - -ARG SWIFT_VERSION=scheme:release/6.2 -ARG LIBXML2_VERSION=2.12.7 -ARG CURL_VERSION=8.7.1 -ARG BORINGSSL_VERSION=fips-20220613 -ARG ICU_VERSION=maint/maint-69 -ARG ZLIB_VERSION=1.3.1 - # Architecture to build on (empty means x86-64) ARG OS_ARCH_SUFFIX= @@ -82,6 +73,8 @@ ENV SWIFT_VERSION=$SWIFT_VERSION \ ICU_VERSION=$ICU_VERSION \ ZLIB_VERSION=$ZLIB_VERSION +ENV SWIFT_BUILD_DOCKER="1" + COPY scripts /scripts RUN chmod ugo+x /scripts/* diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index ea60bd2b..6f4b3577 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -49,7 +49,7 @@ $DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLC $DOCKER run -i --rm \ -v ${WORKDIR}/source:/source \ - -v ${WORKDIR}/products:/products \ + -v ${WORKDIR}/products:/products:rw \ ${CONTAINER_NAME} \ /scripts/build.sh \ --source-dir /source \ diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 3d2df479..077aa017 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -168,6 +168,11 @@ fi # Find the version numbers of the various dependencies function describe { pushd $1 >/dev/null 2>&1 + # this is needed for docker containers or else we get the error: + # fatal: detected dubious ownership in repository at '/source/curl' + if [[ "${SWIFT_BUILD_DOCKER}" == "1" ]]; then + git config --global --add safe.directory $(pwd) + fi git describe --tags popd >/dev/null 2>&1 } From 634929f930f44aea7e748c7d7b1670816e676e26 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 18:11:44 -0400 Subject: [PATCH 40/56] --- swift-ci/sdks/android/scripts/build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 077aa017..8f155e26 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -247,6 +247,13 @@ echo " - libxml2 ${libxml2_version}" echo " - curl ${curl_version}" echo " - BoringSSL ${boringssl_version}" +# make sure the products_dir is writeable +mount +ls -lad $products_dir +touch $products_dir/products_dir_write_test.tmp +rm $products_dir/products_dir_write_test.tmp +#chown -R $(id -u):$(id -g) $products_dir + function run() { echo "$@" "$@" From 2972a4e67f904188b5457df46845a8b0723793da Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 18:34:04 -0400 Subject: [PATCH 41/56] --- swift-ci/sdks/android/build-docker | 1 + swift-ci/sdks/android/scripts/build.sh | 56 ++++++-------------------- 2 files changed, 13 insertions(+), 44 deletions(-) diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 6f4b3577..68b7b168 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -24,6 +24,7 @@ source ./scripts/toolchain-vars.sh ./scripts/patch-sources.sh ${WORKDIR}/source mkdir -p ${WORKDIR}/products +chmod ugo+rwx ${WORKDIR}/products if [[ "$DOCKER" == "" ]]; then DOCKER=docker diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 8f155e26..71800f13 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -248,7 +248,6 @@ echo " - curl ${curl_version}" echo " - BoringSSL ${boringssl_version}" # make sure the products_dir is writeable -mount ls -lad $products_dir touch $products_dir/products_dir_write_test.tmp rm $products_dir/products_dir_write_test.tmp @@ -494,15 +493,16 @@ for arch in $archs; do rsync -a ${sdk_staging}/${arch}/usr ${swift_res_root} done +rm -r ${swift_res_root}/usr/share/{doc,man} +rm -r ${sdk_staging} + +# create an install script to set up the NDK links +#ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" +mkdir scripts/ + ndk_sysroot="ndk-sysroot" -# whether to include the ndk-sysroot in the SDK bundle -INCLUDE_NDK_SYSROOT=${INCLUDE_NDK_SYSROOT:-0} -if [[ ${INCLUDE_NDK_SYSROOT} != 1 ]]; then - # if we do not include the NDK, then create an install script - #ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" - mkdir scripts/ - cat > scripts/setup-android-sdk.sh <<'EOF' +cat > scripts/setup-android-sdk.sh <<'EOF' #/bin/bash # this script will setup the ndk-sysroot with links to the # local installation indicated by ANDROID_NDK_HOME @@ -518,8 +518,6 @@ if [ ! -d "${ndk_prebuilt}" ]; then exit 1 fi -ndk_sysroot=${ndk_prebuilt}/linux-x86_64/sysroot - #Pkg.Revision = 27.0.12077973 #Pkg.Revision = 28.1.13356709 ndk_version=$(grep '^Pkg.Revision = ' "${ANDROID_NDK_HOME}/source.properties" | cut -f3- -d' ' | cut -f 1 -d '.') @@ -528,7 +526,7 @@ if [[ "${ndk_version}" -lt 27 ]]; then exit 1 fi -cd $(dirname $(dirname $(realpath $0))) +cd $(dirname $(dirname $(realpath -- "${BASH_SOURCE[0]}"))) swift_resources=swift-resources ndk_sysroot=ndk-sysroot @@ -538,7 +536,6 @@ if [[ -d "${ndk_sysroot}" ]]; then ndk_re="re-" fi - # link vs. copy the NDK files SWIFT_ANDROID_NDK_LINK=${SWIFT_ANDROID_NDK_LINK:-1} if [[ "${SWIFT_ANDROID_NDK_LINK}" == 1 ]]; then @@ -570,38 +567,8 @@ done echo "$(basename $0): success: ndk-sysroot ${ndk_action} to Android NDK at ${ndk_prebuilt}" EOF - chmod +x scripts/setup-android-sdk.sh -else - COPY_NDK_SYSROOT=${COPY_NDK_SYSROOT:-1} - if [[ ${COPY_NDK_SYSROOT} == 1 ]]; then - cp -a ${ndk_installation}/sysroot ${ndk_sysroot} - else - # rather than copying the sysroot, we can instead make links to - # the various sub-folders this won't work for the distribution, - # since the NDK is going to be located in different places - # for different machines - mkdir -p ${ndk_sysroot}/usr/lib - ln -sv $(realpath ${ndk_installation}/sysroot/usr/include) ${ndk_sysroot}/usr/include - for triplePath in ${ndk_installation}/sysroot/usr/lib/*; do - triple=$(basename ${triplePath}) - ln -sv $(realpath ${triplePath}) ${ndk_sysroot}/usr/lib/${triple} - ls ${ndk_sysroot}/usr/lib/${triple}/ - done - fi - # need to manually copy over swiftrt.o or else: - # error: link command failed with exit code 1 (use -v to see invocation) - # clang: error: no such file or directory: '${HOME}/.swiftpm/swift-sdks/swift-6.2-DEVELOPMENT-SNAPSHOT-2025-04-24-a-android-0.1.artifactbundle/swift-android/ndk-sysroot/usr/lib/swift/android/x86_64/swiftrt.o' - # see: https://github.com/swiftlang/swift-driver/pull/1822#issuecomment-2762811807 - # should be fixed by: https://github.com/swiftlang/swift/pull/79621 - for arch in $archs; do - mkdir -p ${ndk_sysroot}/usr/lib/swift/android/${arch} - ln -srv ${swift_res_root}/usr/lib/swift-${arch}/android/${arch}/swiftrt.o ${ndk_sysroot}/usr/lib/swift/android/${arch}/swiftrt.o - done -fi - -rm -r ${swift_res_root}/usr/share/{doc,man} -rm -r ${sdk_staging} +chmod +x scripts/setup-android-sdk.sh cat > swift-sdk.json < Date: Fri, 16 May 2025 19:36:47 -0400 Subject: [PATCH 42/56] --- .github/workflows/pull_request.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6369cec0..943b4e29 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -80,11 +80,14 @@ jobs: shell: bash run: | set -ex + ls -la ${{ runner.temp }} + ls -la ${{ runner.temp }}/swift-android-sdk + ls -la ${{ runner.temp }}/swift-android-sdk/products SWIFT_ROOT=$(dirname ${{ runner.temp }}/swift-android-sdk/host-toolchain/*/usr) echo "swift-root=${SWIFT_ROOT}" >> $GITHUB_OUTPUT echo "swift-path=${SWIFT_ROOT}/usr/bin/swift" >> $GITHUB_OUTPUT - ARTIFACT_BUILD=$(realpath ${{ runner.temp }}/swift-android-sdk/build/*.artifactbundle) + #ARTIFACT_BUILD=$(realpath ${{ runner.temp }}/swift-android-sdk/build/*.artifactbundle) ARTIFACT_PATH=$(realpath ${{ runner.temp }}/swift-android-sdk/products/*.artifactbundle.tar.gz) echo "artifact-path=${ARTIFACT_PATH}" >> $GITHUB_OUTPUT echo "sdk-id=x86_64-unknown-linux-android28" >> $GITHUB_OUTPUT @@ -108,7 +111,7 @@ jobs: echo "artifact-name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT # show an abridged tree - tree ${ARTIFACT_BUILD} --filesfirst --prune -P 'Android.swiftmodule' -P 'libswiftAndroid.*' -P 'libFoundation.*' -P 'swiftrt.o' -P 'swift*.json' -P 'info.json' -P 'api-level.h' -P 'android.modulemap' -P 'SwiftAndroidNDK.h' -P 'bridging.modulemap' -P 'linux' -P 'libclang*.a' -P 'libunwind.a' -P 'libclang_rt.builtins-*-android.a' + #tree ${ARTIFACT_BUILD} --filesfirst --prune -P 'Android.swiftmodule' -P 'libswiftAndroid.*' -P 'libFoundation.*' -P 'swiftrt.o' -P 'swift*.json' -P 'info.json' -P 'api-level.h' -P 'android.modulemap' -P 'SwiftAndroidNDK.h' -P 'bridging.modulemap' -P 'linux' -P 'libclang*.a' -P 'libunwind.a' -P 'libclang_rt.builtins-*-android.a' - name: Upload SDK artifactbundle uses: actions/upload-artifact@v4 with: From 889fbc3314a16f643c043caf85ed462ed4c65c04 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 21:15:54 -0400 Subject: [PATCH 43/56] --- .github/workflows/pull_request.yml | 24 ++++++++++++++----- swift-ci/sdks/android/Dockerfile | 2 +- .../sdks/android/scripts/install-swift.sh | 14 +++++++---- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 943b4e29..df622b48 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -60,6 +60,12 @@ jobs: sudo docker image prune --all --force sudo docker builder prune -a df -h + - name: Setup + run: | + # these variabes are used by build-docker and build-local + # to determine which Swift version to build for + echo "BUILD_VERSION=${{ matrix.swift-version }}" >> $GITHUB_ENV + echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV - name: Checkout repository uses: actions/checkout@v4 with: @@ -69,12 +75,22 @@ jobs: working-directory: swift-ci/sdks/android run: | sudo apt install -q ninja-build patchelf - BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build-local + ./build-local - name: Build Android SDK (Docker) if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android run: | - BUILD_VERSION=${{ matrix.swift-version }} TARGET_ARCHS=${{ matrix.arch }} ./build-docker + ./build-docker + - name: Install Host Toolchain + if: ${{ matrix.build-type == 'docker' }} + working-directory: swift-ci/sdks/android + run: | + # when building in a Docker container, we don't have a host toolchain + # locally, but we need one in order to run the SDK validation tests + source ./scripts/toolchain-vars.sh + mkdir -p ${{ runner.temp }}/swift-android-sdk/host-toolchain + ./scripts/install-swift.sh ${{ runner.temp }}/swift-android-sdk/host-toolchain/swift + ${{ runner.temp }}/swift-android-sdk/host-toolchain/swift/bin/swift --version - name: Get artifact info id: info shell: bash @@ -87,7 +103,6 @@ jobs: echo "swift-root=${SWIFT_ROOT}" >> $GITHUB_OUTPUT echo "swift-path=${SWIFT_ROOT}/usr/bin/swift" >> $GITHUB_OUTPUT - #ARTIFACT_BUILD=$(realpath ${{ runner.temp }}/swift-android-sdk/build/*.artifactbundle) ARTIFACT_PATH=$(realpath ${{ runner.temp }}/swift-android-sdk/products/*.artifactbundle.tar.gz) echo "artifact-path=${ARTIFACT_PATH}" >> $GITHUB_OUTPUT echo "sdk-id=x86_64-unknown-linux-android28" >> $GITHUB_OUTPUT @@ -109,9 +124,6 @@ jobs: # so the actual artifact download will look like: # swift-6.1-RELEASE_android-0.1-x86_64.artifactbundle.tar.gz.zip echo "artifact-name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT - - # show an abridged tree - #tree ${ARTIFACT_BUILD} --filesfirst --prune -P 'Android.swiftmodule' -P 'libswiftAndroid.*' -P 'libFoundation.*' -P 'swiftrt.o' -P 'swift*.json' -P 'info.json' -P 'api-level.h' -P 'android.modulemap' -P 'SwiftAndroidNDK.h' -P 'bridging.modulemap' -P 'linux' -P 'libclang*.a' -P 'libunwind.a' -P 'libclang_rt.builtins-*-android.a' - name: Upload SDK artifactbundle uses: actions/upload-artifact@v4 with: diff --git a/swift-ci/sdks/android/Dockerfile b/swift-ci/sdks/android/Dockerfile index 5f59f812..2be3fc37 100644 --- a/swift-ci/sdks/android/Dockerfile +++ b/swift-ci/sdks/android/Dockerfile @@ -54,7 +54,7 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \ COPY scripts/install-swift.sh /scripts/install-swift.sh RUN chmod ugo+x /scripts/install-swift.sh -RUN /scripts/install-swift.sh +RUN /scripts/install-swift.sh /usr/local/swift ENV PATH="/usr/local/swift/bin:${PATH}" ARG ANDROID_NDK_VERSION= diff --git a/swift-ci/sdks/android/scripts/install-swift.sh b/swift-ci/sdks/android/scripts/install-swift.sh index eb47f41a..bcd83b4f 100755 --- a/swift-ci/sdks/android/scripts/install-swift.sh +++ b/swift-ci/sdks/android/scripts/install-swift.sh @@ -13,7 +13,13 @@ if [[ "${SWIFT_TOOLCHAIN_URL}" == "" ]]; then exit 1 fi -echo "Installing Swift from: ${SWIFT_TOOLCHAIN_URL}" +destination=$1 +if [[ "${destination}" == "" ]]; then + echo "$0: Usage: $(basename $0) " + exit 1 +fi + +echo "Installing Swift from: ${SWIFT_TOOLCHAIN_URL} into: ${destination}" # Make a temporary directory tmpdir=$(mktemp -d) @@ -45,9 +51,9 @@ gpg --batch --verify toolchain.sig toolchain.tar.gz # Extract and install the toolchain echo "Extracting Swift" -mkdir -p /usr/local/swift -tar -xzf toolchain.tar.gz --directory /usr/local/swift --strip-components=2 -chmod -R o+r /usr/local/swift/lib/swift +mkdir -p ${destination} +tar -xzf toolchain.tar.gz --directory ${destination} --strip-components=2 +chmod -R o+r ${destination}/lib/swift popd >/dev/null From 10db0efe927bb866bfa9b8c28079769ffad409d5 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 22:02:31 -0400 Subject: [PATCH 44/56] --- .github/workflows/pull_request.yml | 6 ++---- swift-ci/sdks/android/README.md | 14 +++++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index df622b48..12efbdf4 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -88,6 +88,7 @@ jobs: # when building in a Docker container, we don't have a host toolchain # locally, but we need one in order to run the SDK validation tests source ./scripts/toolchain-vars.sh + export SWIFT_TOOLCHAIN_URL=${SWIFT_TOOLCHAIN_URL} mkdir -p ${{ runner.temp }}/swift-android-sdk/host-toolchain ./scripts/install-swift.sh ${{ runner.temp }}/swift-android-sdk/host-toolchain/swift ${{ runner.temp }}/swift-android-sdk/host-toolchain/swift/bin/swift --version @@ -96,9 +97,6 @@ jobs: shell: bash run: | set -ex - ls -la ${{ runner.temp }} - ls -la ${{ runner.temp }}/swift-android-sdk - ls -la ${{ runner.temp }}/swift-android-sdk/products SWIFT_ROOT=$(dirname ${{ runner.temp }}/swift-android-sdk/host-toolchain/*/usr) echo "swift-root=${SWIFT_ROOT}" >> $GITHUB_OUTPUT echo "swift-path=${SWIFT_ROOT}/usr/bin/swift" >> $GITHUB_OUTPUT @@ -143,7 +141,7 @@ jobs: ${{ steps.info.outputs.swift-path }} sdk install ${{ steps.info.outputs.artifact-path }} ${{ steps.info.outputs.swift-path }} sdk configure --show-configuration $(${{ steps.info.outputs.swift-path }} sdk list | head -n 1) ${{ steps.info.outputs.sdk-id }} # recent releases require that ANDROID_NDK_ROOT *not* be set - # see https://github.com/finagolfin/swift-android-sdk/issues/207 + # see https://github.com/swiftlang/swift-driver/pull/1879 echo "ANDROID_NDK_ROOT=" >> $GITHUB_ENV - name: Create Demo Project diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md index 531aa88a..cd658cfd 100644 --- a/swift-ci/sdks/android/README.md +++ b/swift-ci/sdks/android/README.md @@ -1,11 +1,19 @@ -# Build scripts for Swift Android SDK +# Dockerfile-based build for Swift Android SDK + +This is a Dockerfile-based build set-up for the Swift Android SDK. + +To build the Docker container and run the + +``` +$ docker build -t swift-android . +``` This folder contains scripts to build a Swift Android SDK in the form of an artifactbundle. ## Running -The top-level `./build` script installs a host toolchain and the +The top-level `./build-docker` script installs a host toolchain and the Android NDK, and then invokes `scripts/fetch-source.sh` which will fetch tagged sources for libxml2, curl, boringssl, and swift. @@ -27,7 +35,7 @@ whereas building for all the architectures takes over an hour. To build an artifactbundle for just the `x86_64` architecture, run: ``` -TARGET_ARCHS=x86_64 ./build +TARGET_ARCHS=aarch64 ./build-docker ``` ## Installing and validating the SDK From 32c442b071531115f33b8ee2e36ccc16ac56c745 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Fri, 16 May 2025 23:01:20 -0400 Subject: [PATCH 45/56] --- .github/workflows/pull_request.yml | 2 +- swift-ci/sdks/android/scripts/toolchain-vars.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 12efbdf4..4dd94b2e 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -87,8 +87,8 @@ jobs: run: | # when building in a Docker container, we don't have a host toolchain # locally, but we need one in order to run the SDK validation tests + HOST_OS=ubuntu$(lsb_release -sr) source ./scripts/toolchain-vars.sh - export SWIFT_TOOLCHAIN_URL=${SWIFT_TOOLCHAIN_URL} mkdir -p ${{ runner.temp }}/swift-android-sdk/host-toolchain ./scripts/install-swift.sh ${{ runner.temp }}/swift-android-sdk/host-toolchain/swift ${{ runner.temp }}/swift-android-sdk/host-toolchain/swift/bin/swift --version diff --git a/swift-ci/sdks/android/scripts/toolchain-vars.sh b/swift-ci/sdks/android/scripts/toolchain-vars.sh index 856c6b70..c1001931 100644 --- a/swift-ci/sdks/android/scripts/toolchain-vars.sh +++ b/swift-ci/sdks/android/scripts/toolchain-vars.sh @@ -40,5 +40,5 @@ case "${BUILD_VERSION}" in esac SWIFT_BASE=$SWIFT_TAG-$HOST_OS -SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" +export SWIFT_TOOLCHAIN_URL="https://download.swift.org/$SWIFT_BRANCH/$(echo $HOST_OS | tr -d '.')/$SWIFT_TAG/$SWIFT_BASE.tar.gz" From d12b81dd71a5588579c425de18e13a8afa3177e1 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 09:49:18 -0400 Subject: [PATCH 46/56] --- .github/workflows/pull_request.yml | 9 +++++---- swift-ci/sdks/android/build-local | 8 +------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 4dd94b2e..2d945a9d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -85,13 +85,14 @@ jobs: if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android run: | - # when building in a Docker container, we don't have a host toolchain - # locally, but we need one in order to run the SDK validation tests + # when building in a Docker container, we don't have a local host toolchain, + # but we need one in order to run the SDK validation tests, so we install it now HOST_OS=ubuntu$(lsb_release -sr) source ./scripts/toolchain-vars.sh mkdir -p ${{ runner.temp }}/swift-android-sdk/host-toolchain - ./scripts/install-swift.sh ${{ runner.temp }}/swift-android-sdk/host-toolchain/swift - ${{ runner.temp }}/swift-android-sdk/host-toolchain/swift/bin/swift --version + ./scripts/install-swift.sh ${{ runner.temp }}/swift-android-sdk/host-toolchain + ls ${{ runner.temp }}/swift-android-sdk/host-toolchain + ${{ runner.temp }}/swift-android-sdk/host-toolchain/*/usr/bin/swift --version - name: Get artifact info id: info shell: bash diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 119c5f13..10336062 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -21,13 +21,7 @@ source ./scripts/toolchain-vars.sh SWIFT_ROOT=${WORKDIR}/host-toolchain if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then - mkdir -p $SWIFT_ROOT - pushd $SWIFT_ROOT - - wget -q $SWIFT_TOOLCHAIN_URL - tar xzf $SWIFT_BASE.tar.gz - - popd + ./scripts/install-swift.sh ${SWIFT_ROOT} fi #HOST_TOOLCHAIN=${HOST_TOOLCHAIN:-$(dirname $(dirname $(which swiftc)))} From 736570bad210d9a19581829f5f51b5e97daecc82 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 10:02:52 -0400 Subject: [PATCH 47/56] --- .github/workflows/pull_request.yml | 2 +- swift-ci/sdks/android/build-local | 5 ++--- swift-ci/sdks/android/scripts/install-swift.sh | 5 ----- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2d945a9d..8f9b2a47 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -90,7 +90,7 @@ jobs: HOST_OS=ubuntu$(lsb_release -sr) source ./scripts/toolchain-vars.sh mkdir -p ${{ runner.temp }}/swift-android-sdk/host-toolchain - ./scripts/install-swift.sh ${{ runner.temp }}/swift-android-sdk/host-toolchain + ./scripts/install-swift.sh ${{ runner.temp }}/swift-android-sdk/host-toolchain/$SWIFT_BASE/usr ls ${{ runner.temp }}/swift-android-sdk/host-toolchain ${{ runner.temp }}/swift-android-sdk/host-toolchain/*/usr/bin/swift --version - name: Get artifact info diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 10336062..1effb3f7 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -20,12 +20,11 @@ HOST_OS=ubuntu$(lsb_release -sr) source ./scripts/toolchain-vars.sh SWIFT_ROOT=${WORKDIR}/host-toolchain +HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then - ./scripts/install-swift.sh ${SWIFT_ROOT} + ./scripts/install-swift.sh ${HOST_TOOLCHAIN} fi -#HOST_TOOLCHAIN=${HOST_TOOLCHAIN:-$(dirname $(dirname $(which swiftc)))} -HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr $HOST_TOOLCHAIN/bin/swift --version # ensure the correct Swift is first in the PATH diff --git a/swift-ci/sdks/android/scripts/install-swift.sh b/swift-ci/sdks/android/scripts/install-swift.sh index bcd83b4f..0acd9e02 100755 --- a/swift-ci/sdks/android/scripts/install-swift.sh +++ b/swift-ci/sdks/android/scripts/install-swift.sh @@ -33,24 +33,19 @@ export GNUPGHOME="$tmpdir" # Fetch the toolchain and signature echo "Going to fetch ${SWIFT_TOOLCHAIN_URL}" - curl -fsSL "${SWIFT_TOOLCHAIN_URL}" -o toolchain.tar.gz echo "Going to fetch ${SWIFT_TOOLCHAIN_URL}.sig" - curl -fsSL "${SWIFT_TOOLCHAIN_URL}.sig" -o toolchain.sig echo "Fetching keys" - curl -fsSL --compressed https://swift.org/keys/all-keys.asc | gpg --import - echo "Verifying signature" - gpg --batch --verify toolchain.sig toolchain.tar.gz # Extract and install the toolchain echo "Extracting Swift" - mkdir -p ${destination} tar -xzf toolchain.tar.gz --directory ${destination} --strip-components=2 chmod -R o+r ${destination}/lib/swift From 68ecf818c904abfa7e31e60ab740436ee26ab14a Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 11:05:37 -0400 Subject: [PATCH 48/56] --- .github/workflows/pull_request.yml | 20 ++++++++------- swift-ci/sdks/android/README.md | 35 ++++++++++++++++++++------ swift-ci/sdks/android/build-docker | 10 ++++++-- swift-ci/sdks/android/build-local | 12 ++++++--- swift-ci/sdks/android/scripts/build.sh | 2 +- 5 files changed, 56 insertions(+), 23 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 8f9b2a47..8ca2a68a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -61,11 +61,13 @@ jobs: sudo docker builder prune -a df -h - name: Setup + id: config run: | # these variabes are used by build-docker and build-local # to determine which Swift version to build for echo "BUILD_VERSION=${{ matrix.swift-version }}" >> $GITHUB_ENV echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV + echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV - name: Checkout repository uses: actions/checkout@v4 with: @@ -75,12 +77,12 @@ jobs: working-directory: swift-ci/sdks/android run: | sudo apt install -q ninja-build patchelf - ./build-local + ./build-local ${BUILD_VERSION} ${WORKDIR} - name: Build Android SDK (Docker) if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android run: | - ./build-docker + ./build-docker ${BUILD_VERSION} ${WORKDIR} - name: Install Host Toolchain if: ${{ matrix.build-type == 'docker' }} working-directory: swift-ci/sdks/android @@ -89,20 +91,20 @@ jobs: # but we need one in order to run the SDK validation tests, so we install it now HOST_OS=ubuntu$(lsb_release -sr) source ./scripts/toolchain-vars.sh - mkdir -p ${{ runner.temp }}/swift-android-sdk/host-toolchain - ./scripts/install-swift.sh ${{ runner.temp }}/swift-android-sdk/host-toolchain/$SWIFT_BASE/usr - ls ${{ runner.temp }}/swift-android-sdk/host-toolchain - ${{ runner.temp }}/swift-android-sdk/host-toolchain/*/usr/bin/swift --version + mkdir -p ${WORKDIR}/host-toolchain + ./scripts/install-swift.sh ${WORKDIR}/host-toolchain/$SWIFT_BASE/usr + ls ${WORKDIR}/host-toolchain + ${WORKDIR}/host-toolchain/*/usr/bin/swift --version - name: Get artifact info id: info shell: bash run: | set -ex - SWIFT_ROOT=$(dirname ${{ runner.temp }}/swift-android-sdk/host-toolchain/*/usr) + SWIFT_ROOT=$(dirname ${WORKDIR}/host-toolchain/*/usr) echo "swift-root=${SWIFT_ROOT}" >> $GITHUB_OUTPUT echo "swift-path=${SWIFT_ROOT}/usr/bin/swift" >> $GITHUB_OUTPUT - ARTIFACT_PATH=$(realpath ${{ runner.temp }}/swift-android-sdk/products/*.artifactbundle.tar.gz) + ARTIFACT_PATH=$(realpath ${WORKDIR}/products/*.artifactbundle.tar.gz) echo "artifact-path=${ARTIFACT_PATH}" >> $GITHUB_OUTPUT echo "sdk-id=x86_64-unknown-linux-android28" >> $GITHUB_OUTPUT @@ -133,7 +135,7 @@ jobs: run: | # need to free up some space or else when installing we get: No space left on device df -h - rm -rf ${{ runner.temp }}/swift-android-sdk/{build,src} + rm -rf ${WORKDIR}/{build,src} df -h - name: Install artifactbundle shell: bash diff --git a/swift-ci/sdks/android/README.md b/swift-ci/sdks/android/README.md index cd658cfd..10d4db3e 100644 --- a/swift-ci/sdks/android/README.md +++ b/swift-ci/sdks/android/README.md @@ -2,14 +2,34 @@ This is a Dockerfile-based build set-up for the Swift Android SDK. -To build the Docker container and run the +The top-level `./build-docker` script will create a +Docker container and install a host toolchain and the +Android NDK, and then invoke `scripts/fetch-source.sh` which will +fetch tagged sources for libxml2, curl, boringssl, and swift. + +It can be run with: + +``` +$ ./build-docker +``` + +for example: ``` -$ docker build -t swift-android . +$ ./build-docker release /tmp/android-sdk ``` -This folder contains scripts to build a Swift Android SDK -in the form of an artifactbundle. +This will create an Ubuntu 24.04 container with the necessary dependencies +to build the Android SDK, including a Swift host toolchain and the +Android NDK that will be used for cross-compilation. + +The `version` argument can be one of the following values: + +| version | Swift version | +| --- | --- | +| `release` | swift-6.1-RELEASE | +| `devel` | swift-6.2-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | +| `trunk` | swift-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd | ## Running @@ -35,7 +55,7 @@ whereas building for all the architectures takes over an hour. To build an artifactbundle for just the `x86_64` architecture, run: ``` -TARGET_ARCHS=aarch64 ./build-docker +TARGET_ARCHS=aarch64 ./build-docker release /tmp/android-sdk ``` ## Installing and validating the SDK @@ -46,6 +66,5 @@ will create and upload an installable SDK named something like: The workflow will also install the SDK locally and use [swift-android-action](https://github.com/marketplace/actions/swift-android-action) -to build and test various Swift packages in an Android emulator. - - +to build and test various Swift packages in an Android emulator using the +freshly-created SDK bundle. diff --git a/swift-ci/sdks/android/build-docker b/swift-ci/sdks/android/build-docker index 68b7b168..fb3b377a 100755 --- a/swift-ci/sdks/android/build-docker +++ b/swift-ci/sdks/android/build-docker @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # # ===----------------------------------------------------------------------=== # @@ -12,9 +12,15 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 +export BUILD_VERSION=${1} # note that WORKDIR must not be under the current checkout or the patches will fail to apply -WORKDIR=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk +WORKDIR=${2} +if [[ "${WORKDIR}" == '' ]]; then + echo "Usage: $(basename $0) " + exit 1 +fi mkdir -p ${WORKDIR} +WORKDIR=$(realpath ${WORKDIR}) HOST_OS=ubuntu24.04 source ./scripts/toolchain-vars.sh diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 1effb3f7..0be600ce 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # # ===----------------------------------------------------------------------=== # @@ -12,16 +12,22 @@ TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7} ANDROID_NDK_VERSION=android-ndk-r27c ANDROID_API=28 +export BUILD_VERSION=${1} # note that WORKDIR must not be under the current checkout or the patches will fail to apply -WORKDIR=${RUNNER_TEMP:-${TMPDIR:-'/tmp'}}/swift-android-sdk +WORKDIR=${1} +if [[ "${WORKDIR}" == '' ]]; then + echo "Usage: $(basename $0) " + exit 1 +fi mkdir -p ${WORKDIR} +WORKDIR=$(realpath ${WORKDIR}) HOST_OS=ubuntu$(lsb_release -sr) source ./scripts/toolchain-vars.sh SWIFT_ROOT=${WORKDIR}/host-toolchain HOST_TOOLCHAIN=$SWIFT_ROOT/$SWIFT_BASE/usr -if [[ ! -d "$SWIFT_ROOT/$SWIFT_BASE" ]]; then +if [[ ! -d "$HOST_TOOLCHAIN" ]]; then ./scripts/install-swift.sh ${HOST_TOOLCHAIN} fi diff --git a/swift-ci/sdks/android/scripts/build.sh b/swift-ci/sdks/android/scripts/build.sh index 71800f13..20877272 100755 --- a/swift-ci/sdks/android/scripts/build.sh +++ b/swift-ci/sdks/android/scripts/build.sh @@ -1,6 +1,6 @@ #!/bin/bash # Swift SDK for Android: Build Script -set -ex +set -e # Docker sets TERM to xterm if using a pty; we probably want # xterm-256color, otherwise we only get eight colors From f18311f3126f1f2f24a6355fa6594ca0e5bfb3e1 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 11:38:29 -0400 Subject: [PATCH 49/56] --- swift-ci/sdks/android/scripts/patch-sources.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index 3cccfb6d..f4b57d93 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -56,9 +56,9 @@ for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do echo "applying patch $patch in $PWD…" # first check to make sure the patches can apply and fail if not - git apply -v --check -C1 "$patch" - echo "git apply --check result: $0" - git apply -v -C1 "$patch" + patch -v --check -C1 < "$patch" + echo "patch --check result: $0" + patch -v -C1 < "$patch" #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then # echo "already patched" From ad934ff2e4d40f42eaf256f4ce39d8d9f76a6dbb Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 14:07:55 -0400 Subject: [PATCH 50/56] --- .github/workflows/pull_request.yml | 5 ++++- .../sdks/android/scripts/patch-sources.sh | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 8ca2a68a..72a708cc 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -67,7 +67,8 @@ jobs: # to determine which Swift version to build for echo "BUILD_VERSION=${{ matrix.swift-version }}" >> $GITHUB_ENV echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV - echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV + #echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV + echo "WORKDIR=/tmp/swift-android-sdk" >> $GITHUB_ENV - name: Checkout repository uses: actions/checkout@v4 with: @@ -136,6 +137,8 @@ jobs: # need to free up some space or else when installing we get: No space left on device df -h rm -rf ${WORKDIR}/{build,src} + sudo docker image prune --all --force + sudo docker builder prune -a df -h - name: Install artifactbundle shell: bash diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index f4b57d93..d646d92e 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -21,6 +21,10 @@ fi cd ${source_dir}/swift-project swift_android_patch="${patches_dir}/swift-android.patch" +ls -la swift/utils/build-script-impl +ls -la ${swift_android_patch} +ls -la swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift + # patch the patch, which seems to only be needed for an API less than 28 # https://github.com/finagolfin/swift-android-sdk/blob/main/swift-android.patch#L110 perl -pi -e 's/#if os\(Windows\)/#if os\(Android\)/g' $swift_android_patch @@ -56,9 +60,9 @@ for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do echo "applying patch $patch in $PWD…" # first check to make sure the patches can apply and fail if not - patch -v --check -C1 < "$patch" - echo "patch --check result: $0" - patch -v -C1 < "$patch" + git apply -v --check -C1 "$patch" + echo "git apply --check result: $0" + git apply -v -C1 "$patch" #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then # echo "already patched" @@ -70,11 +74,6 @@ for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do #fi done -# validate the patches -ls -la swift/utils/build-script-impl -grep 'VALIDATING SYMBOLIC LINK' swift/utils/build-script-impl - - perl -pi -e 's%String\(cString: getpass%\"fake\" //%' swiftpm/Sources/PackageRegistryCommand/PackageRegistryCommand+Auth.swift # disable backtrace() for Android (needs either API33+ or libandroid-execinfo, or to manually add in backtrace backport) perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Testing/SourceAttribution/Backtrace.swift @@ -83,3 +82,7 @@ perl -pi -e 's;os\(Android\);os\(AndroidDISABLED\);g' swift-testing/Sources/Test perl -pi -e 's;MATCHES "Android";MATCHES "AndroidDISABLED";g' llbuild/lib/llvm/Support/CMakeLists.txt perl -pi -e 's; STREQUAL Android\); STREQUAL AndroidDISABLED\);g' swift-corelibs-foundation/Sources/Foundation/CMakeLists.txt +# validate the patches +ls -la swift/utils/build-script-impl +grep 'VALIDATING SYMBOLIC LINK' swift/utils/build-script-impl + From 249e9c6d7b759c7e518e4c990cbb96148e043eda Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 14:12:17 -0400 Subject: [PATCH 51/56] --- swift-ci/sdks/android/scripts/patch-sources.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/swift-ci/sdks/android/scripts/patch-sources.sh b/swift-ci/sdks/android/scripts/patch-sources.sh index d646d92e..ee425c6a 100755 --- a/swift-ci/sdks/android/scripts/patch-sources.sh +++ b/swift-ci/sdks/android/scripts/patch-sources.sh @@ -61,8 +61,7 @@ for patch in "$swift_android_patch" "$testing_patch" "$dispatch_patch"; do echo "applying patch $patch in $PWD…" # first check to make sure the patches can apply and fail if not git apply -v --check -C1 "$patch" - echo "git apply --check result: $0" - git apply -v -C1 "$patch" + git apply --no-index -v -C1 "$patch" #if git apply -C1 --reverse --check "$patch" >/dev/null 2>&1 ; then # echo "already patched" From de874cb56899bfe57811387ed7b5e8b996cd2292 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 14:12:26 -0400 Subject: [PATCH 52/56] --- .github/workflows/pull_request.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 72a708cc..376e48e8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -67,8 +67,7 @@ jobs: # to determine which Swift version to build for echo "BUILD_VERSION=${{ matrix.swift-version }}" >> $GITHUB_ENV echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV - #echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV - echo "WORKDIR=/tmp/swift-android-sdk" >> $GITHUB_ENV + echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV - name: Checkout repository uses: actions/checkout@v4 with: From a18ff070ed3cf2b44112db161a4d51ceafd425b6 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 15:26:12 -0400 Subject: [PATCH 53/56] --- swift-ci/sdks/android/build-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index 0be600ce..f89874e8 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -14,7 +14,7 @@ ANDROID_API=28 export BUILD_VERSION=${1} # note that WORKDIR must not be under the current checkout or the patches will fail to apply -WORKDIR=${1} +WORKDIR=${2} if [[ "${WORKDIR}" == '' ]]; then echo "Usage: $(basename $0) " exit 1 From 6683bbf94f837d69337750079967eb2eb33a8186 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 15:52:57 -0400 Subject: [PATCH 54/56] --- swift-ci/sdks/android/build-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift-ci/sdks/android/build-local b/swift-ci/sdks/android/build-local index f89874e8..30bd4f92 100755 --- a/swift-ci/sdks/android/build-local +++ b/swift-ci/sdks/android/build-local @@ -61,4 +61,4 @@ mkdir -p ${WORKDIR}/products --host-toolchain ${HOST_TOOLCHAIN} \ --android-api ${ANDROID_API} \ --ndk-home ${ANDROID_NDK_HOME} \ - --archs ${TARGET_ARCHS} ${@} + --archs ${TARGET_ARCHS} From ea800d20497b5bc4f957077df6ab1c0aad85ebe0 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 17:18:37 -0400 Subject: [PATCH 55/56] --- .github/workflows/pull_request.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 376e48e8..6524d13a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -8,8 +8,6 @@ env: jobs: build: name: Build Docker images - # disabled for CI testing - if: false runs-on: ubuntu-latest steps: - name: Checkout repository @@ -26,8 +24,6 @@ jobs: static-linux-build: name: Build Static Linux image - # disabled for CI testing - if: false runs-on: ubuntu-latest steps: - name: Checkout repository @@ -41,14 +37,15 @@ jobs: strategy: fail-fast: false matrix: + build-type: ['docker'] + #build-type: ['docker', 'local'] # blank arch builds all (aarch64,x86_64,armv7) - #arch: [''] + arch: [''] # builds only x86_64 to speed up the validation #arch: ['x86_64'] # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs - arch: ['x86_64', ''] + #arch: ['x86_64', ''] swift-version: ['release', 'devel', 'trunk'] - build-type: ['docker', 'local'] runs-on: ubuntu-24.04 steps: - name: Free Disk Space From f3c8ad18bea4726a098d53c302b1fa5a9841dfdd Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Sat, 17 May 2025 17:37:35 -0400 Subject: [PATCH 56/56] --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6524d13a..13c57b39 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -132,7 +132,7 @@ jobs: run: | # need to free up some space or else when installing we get: No space left on device df -h - rm -rf ${WORKDIR}/{build,src} + rm -rf ${WORKDIR}/{build,source} sudo docker image prune --all --force sudo docker builder prune -a df -h