take 3

Author: clee2000

.circleci/config.yml

@@ -1 +1,317 @@
+# run python regenerate.py to generate config.yml from config.yml.in
+
 version: 2.1
+
+executors:
+  windows-with-nvidia-gpu:
+    machine:
+      resource_class: windows.gpu.nvidia.medium
+      image: windows-server-2019-nvidia:stable
+      shell: bash.exe
+
+install_official_git_client: &install_official_git_client
+  name: Install Official Git Client
+  no_output_timeout: "1h"
+  command: |
+    set -e
+    sudo apt-get -qq update
+    sudo apt-get -qq install openssh-client git
+
+# This system setup script is meant to run before the CI-related scripts, e.g.,
+# installing Git client, checking out code, setting up CI env, and
+# building/testing.
+setup_linux_system_environment: &setup_linux_system_environment
+  name: Set Up System Environment
+  no_output_timeout: "1h"
+  command: |
+    set -ex
+
+    # Stop background apt updates.  Hypothetically, the kill should not
+    # be necessary, because stop is supposed to send a kill signal to
+    # the process, but we've added it for good luck.  Also
+    # hypothetically, it's supposed to be unnecessary to wait for
+    # the process to block.  We also have that line for good luck.
+    # If you like, try deleting them and seeing if it works.
+    sudo systemctl stop apt-daily.service || true
+    sudo systemctl kill --kill-who=all apt-daily.service || true
+
+    sudo systemctl stop unattended-upgrades.service || true
+    sudo systemctl kill --kill-who=all unattended-upgrades.service || true
+
+    # wait until `apt-get update` has been killed
+    while systemctl is-active --quiet apt-daily.service
+    do
+      sleep 1;
+    done
+    while systemctl is-active --quiet unattended-upgrades.service
+    do
+      sleep 1;
+    done
+
+    # See if we actually were successful
+    systemctl list-units --all | cat
+
+    sudo apt-get purge -y unattended-upgrades
+
+    cat /etc/apt/sources.list
+
+    ps auxfww | grep [a]pt
+    ps auxfww | grep dpkg
+
+pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
+  machine:
+    image: ubuntu-2004-cuda-11.4:202110-01
+  steps:
+  - checkout
+  - run:
+      <<: *setup_linux_system_environment
+  - run:
+      name: Set Up CI Environment
+      no_output_timeout: "1h"
+      command: |
+        set -e
+
+        sudo apt-get -y update
+        sudo apt-get -y install expect-dev moreutils
+
+        sudo pip3 -q install awscli==1.16.35
+
+         if [ -n "${CUDA_VERSION}" ]; then
+           nvidia-smi
+         fi
+
+        # This IAM user only allows read-write access to ECR
+        export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_ONLY}
+        export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_ONLY}
+        eval $(aws ecr get-login --region us-east-1 --no-include-email)
+  - run:
+      name: Build
+      no_output_timeout: "20h"
+      command: |
+        set -e
+
+        # for some reason, pip installs it in a different place than what is looked at in the py file
+        sudo pip3 install requests --target=/opt/circleci/.pyenv/versions/3.9.4/lib/python3.9/site-packages
+        export pyTorchDockerImageTag=$(python3 .jenkins/get_docker_tag.py)
+        echo "PyTorchDockerImageTag: "${pyTorchDockerImageTag}
+
+        cat >/home/circleci/project/ci_build_script.sh \<<EOL
+        # =================== The following code will be executed inside Docker container ===================
+        set -ex
+
+        .jenkins/build.sh
+        # =================== The above code will be executed inside Docker container ===================
+        EOL
+        chmod +x /home/circleci/project/ci_build_script.sh
+
+        export DOCKER_IMAGE=${DOCKER_IMAGE}:${pyTorchDockerImageTag}
+        echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
+        docker pull ${DOCKER_IMAGE} >/dev/null
+        if [ -n "${CUDA_VERSION}" ]; then
+          export id=$(docker run --gpus all -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+        else
+          export id=$(docker run -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
+        fi
+
+        echo "declare -x JOB_BASE_NAME=${CIRCLE_JOB}" > /home/circleci/project/env
+        echo "declare -x COMMIT_ID=${CIRCLE_SHA1}" >> /home/circleci/project/env
+        echo "declare -x COMMIT_SOURCE=${CIRCLE_BRANCH}" >> /home/circleci/project/env
+        # DANGER! DO NOT REMOVE THE `set +x` SETTING HERE!
+        set +x
+        if [[ "$CIRCLE_BRANCH" == master || "$CIRCLE_BRANCH" == main ]]; then
+          if [ -z "${CIRCLECI_AWS_ACCESS_KEY_FOR_PYTORCH_TUTORIAL_BUILD_MASTER_S3_BUCKET}" ]; then exit 1; fi
+          if [ -z "${CIRCLECI_AWS_SECRET_KEY_FOR_PYTORCH_TUTORIAL_BUILD_MASTER_S3_BUCKET}" ]; then exit 1; fi
+          if [ -z "${GITHUB_PYTORCHBOT_USERNAME}" ]; then exit 1; fi
+          if [ -z "${GITHUB_PYTORCHBOT_TOKEN}" ]; then exit 1; fi
+
+          echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_PYTORCH_TUTORIAL_BUILD_MASTER_S3_BUCKET}" >> /home/circleci/project/env
+          echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_PYTORCH_TUTORIAL_BUILD_MASTER_S3_BUCKET}" >> /home/circleci/project/env
+          echo "declare -x GITHUB_PYTORCHBOT_USERNAME=${GITHUB_PYTORCHBOT_USERNAME}" >> /home/circleci/project/env
+          echo "declare -x GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" >> /home/circleci/project/env
+        else
+          echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_PYTORCH_TUTORIAL_BUILD_PR_S3_BUCKET}" >> /home/circleci/project/env
+          echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_PYTORCH_TUTORIAL_BUILD_PR_S3_BUCKET}" >> /home/circleci/project/env
+        fi
+        set -x
+
+        echo 'rm /opt/cache/bin/*' | docker exec -u root -i "$id" bash
+        docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
+
+        export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
+        echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
+        # Copy docs with plot to a docs dir
+        if docker exec -it "$id" sh -c "test -d ./workspace/docs_with_plot/docs/"; then
+            mkdir /home/circleci/project/docs
+            docker cp "$id:/var/lib/jenkins/workspace/docs_with_plot/docs/." /home/circleci/project/docs
+            echo "Directory copied successfully"
+        else
+            echo "No docs_with_plot directory. Skipping..."
+        fi
+
+  - store_artifacts:
+      path: ./docs
+      destination: tutorials
+
+pytorch_tutorial_build_worker_defaults: &pytorch_tutorial_build_worker_defaults
+  environment:
+    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda12.1-cudnn8-py3-gcc9"
+    CUDA_VERSION: "9"
+  resource_class: gpu.nvidia.small
+  <<: *pytorch_tutorial_build_defaults
+
+pytorch_tutorial_build_manager_defaults: &pytorch_tutorial_build_manager_defaults
+  environment:
+    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda12.1-cudnn8-py3-gcc9"
+  resource_class: medium
+
+
+  <<: *pytorch_tutorial_build_defaults
+
+pytorch_windows_build_worker: &pytorch_windows_build_worker
+  executor: windows-with-nvidia-gpu
+  steps:
+    - checkout
+    - run:
+        name: Install Cuda
+        no_output_timeout: 30m
+        command: |
+          .circleci/scripts/windows_cuda_install.sh
+    - run:
+        name: Generate cache key
+        # This will refresh cache on Sundays, build should generate new cache.
+        command: echo "$(date +"%Y-%U")" > .circleci-weekly
+    - restore_cache:
+        keys:
+          - data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
+    - run:
+        name: test
+        no_output_timeout: "1h"
+        command: |
+          .circleci/scripts/build_for_windows.sh
+    - save_cache:
+        key: data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
+        paths:
+          - advanced_source/data
+          - beginner_source/data
+          - intermediate_source/data
+          - prototype_source/data
+
+jobs:
+  pytorch_tutorial_pr_build_manager:
+    <<: *pytorch_tutorial_build_manager_defaults
+  pytorch_tutorial_pr_build_worker_0:
+    <<: *pytorch_tutorial_build_worker_defaults
+    resource_class: gpu.nvidia.small.multi
+  pytorch_tutorial_pr_build_worker_1:
+    <<: *pytorch_tutorial_build_worker_defaults
+    resource_class: gpu.nvidia.large
+  pytorch_tutorial_pr_build_worker_10:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_11:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_12:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_13:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_14:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_15:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_16:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_17:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_18:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_19:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_2:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_3:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_4:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_5:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_6:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_7:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_8:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_pr_build_worker_9:
+    <<: *pytorch_tutorial_build_worker_defaults
+
+  pytorch_tutorial_trunk_build_manager:
+    <<: *pytorch_tutorial_build_manager_defaults
+  pytorch_tutorial_trunk_build_worker_0:
+    <<: *pytorch_tutorial_build_worker_defaults
+    resource_class: gpu.nvidia.small.multi
+  pytorch_tutorial_trunk_build_worker_1:
+    <<: *pytorch_tutorial_build_worker_defaults
+    resource_class: gpu.nvidia.large
+  pytorch_tutorial_trunk_build_worker_10:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_11:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_12:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_13:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_14:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_15:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_16:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_17:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_18:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_19:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_2:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_3:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_4:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_5:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_6:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_7:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_8:
+    <<: *pytorch_tutorial_build_worker_defaults
+  pytorch_tutorial_trunk_build_worker_9:
+    <<: *pytorch_tutorial_build_worker_defaults
+
+  pytorch_tutorial_windows_pr_build_worker_0:
+    <<: *pytorch_windows_build_worker
+  pytorch_tutorial_windows_pr_build_worker_1:
+    <<: *pytorch_windows_build_worker
+  pytorch_tutorial_windows_pr_build_worker_2:
+    <<: *pytorch_windows_build_worker
+  pytorch_tutorial_windows_pr_build_worker_3:
+    <<: *pytorch_windows_build_worker
+  pytorch_tutorial_windows_trunk_build_worker_0:
+    <<: *pytorch_windows_build_worker
+  pytorch_tutorial_windows_trunk_build_worker_1:
+    <<: *pytorch_windows_build_worker
+  pytorch_tutorial_windows_trunk_build_worker_2:
+    <<: *pytorch_windows_build_worker
+  pytorch_tutorial_windows_trunk_build_worker_3:
+    <<: *pytorch_windows_build_worker
+
+workflows:
+  build:
+    when: << pipeline.parameters.this_should_never_be_true >>
+    jobs:
+      # Build jobs that only run on PR
+      - pytorch_tutorial_pr_build_worker_0:
+          filters:
+            branches:
+              ignore:
+              - master
+              - main