Skip to content

Commit a5ed0b0

Browse files
zou3519zou3519
committed
Set up CI
Copied from torchvision. ghstack-source-id: 2158902 Pull Request resolved: #87
1 parent 015c15d commit a5ed0b0

File tree

4 files changed

+163
-0
lines changed

4 files changed

+163
-0
lines changed

‎.github/scripts/setup-env.sh

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#!/usr/bin/env bash
2+
3+
set -euxo pipefail
4+
5+
# Prepare conda
6+
set +x && eval "$($(which conda) shell.bash hook)" && set -x
7+
8+
# Setup the OS_TYPE environment variable that should be used for conditions involving the OS below.
9+
case $(uname) in
10+
Linux)
11+
OS_TYPE=linux
12+
;;
13+
Darwin)
14+
OS_TYPE=macos
15+
;;
16+
MSYS*)
17+
OS_TYPE=windows
18+
;;
19+
*)
20+
echo "Unknown OS type:" $(uname)
21+
exit 1
22+
;;
23+
esac
24+
25+
if [[ "${OS_TYPE}" == "macos" && $(uname -m) == x86_64 ]]; then
26+
echo '::group::Uninstall system JPEG libraries on macOS'
27+
# The x86 macOS runners, e.g. the GitHub Actions native "macos-12" runner, has some JPEG and PNG libraries
28+
# installed by default that interfere with our build. We uninstall them here and use the one from conda below.
29+
IMAGE_LIBS=$(brew list | grep -E "jpeg|png")
30+
for lib in $IMAGE_LIBS; do
31+
brew uninstall --ignore-dependencies --force "${lib}"
32+
done
33+
echo '::endgroup::'
34+
fi
35+
36+
echo '::group::Create build environment'
37+
# See https://github.com/pytorch/vision/issues/7296 for ffmpeg
38+
conda create \
39+
--name ci \
40+
--quiet --yes \
41+
python="${PYTHON_VERSION}" pip \
42+
ninja cmake
43+
conda activate ci
44+
pip install --progress-bar=off --upgrade setuptools
45+
46+
echo '::endgroup::'
47+
48+
if [[ "${OS_TYPE}" == windows && "${GPU_ARCH_TYPE}" == cuda ]]; then
49+
echo '::group::Install VisualStudio CUDA extensions on Windows'
50+
if [[ "${VC_YEAR:-}" == "2022" ]]; then
51+
TARGET_DIR="/c/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/MSBuild/Microsoft/VC/v170/BuildCustomizations"
52+
else
53+
TARGET_DIR="/c/Program Files (x86)/Microsoft Visual Studio/2019/BuildTools/MSBuild/Microsoft/VC/v160/BuildCustomizations"
54+
fi
55+
mkdir -p "${TARGET_DIR}"
56+
cp -r "${CUDA_HOME}/MSBuildExtensions/"* "${TARGET_DIR}"
57+
echo '::endgroup::'
58+
fi
59+
60+
echo '::group::Install PyTorch'
61+
# TODO: Can we maybe have this as environment variable in the job template? For example, `IS_RELEASE`.
62+
if [[ (${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
63+
CHANNEL=test
64+
else
65+
CHANNEL=nightly
66+
fi
67+
68+
case $GPU_ARCH_TYPE in
69+
cpu)
70+
GPU_ARCH_ID="cpu"
71+
;;
72+
cuda)
73+
VERSION_WITHOUT_DOT=$(echo "${GPU_ARCH_VERSION}" | sed 's/\.//')
74+
GPU_ARCH_ID="cu${VERSION_WITHOUT_DOT}"
75+
;;
76+
*)
77+
echo "Unknown GPU_ARCH_TYPE=${GPU_ARCH_TYPE}"
78+
exit 1
79+
;;
80+
esac
81+
PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${CHANNEL}/${GPU_ARCH_ID}"
82+
pip install --progress-bar=off --pre torch --index-url="${PYTORCH_WHEEL_INDEX}"
83+
84+
if [[ $GPU_ARCH_TYPE == 'cuda' ]]; then
85+
python -c "import torch; exit(not torch.cuda.is_available())"
86+
fi
87+
echo '::endgroup::'
88+
89+
echo '::group::Install third party dependencies prior to extension-cpp install'
90+
# Installing with `easy_install`, e.g. `python setup.py install` or `python setup.py develop`, has some quirks when
91+
# when pulling in third-party dependencies. For example:
92+
# - On Windows, we often hit an SSL error although `pip` can install just fine.
93+
# - It happily pulls in pre-releases, which can lead to more problems down the line.
94+
# `pip` does not unless explicitly told to do so.
95+
# Thus, we use `easy_install` to extract the third-party dependencies here and install them upfront with `pip`.
96+
python setup.py egg_info
97+
# The requires.txt cannot be used with `pip install -r` directly. The requirements are listed at the top and the
98+
# optional dependencies come in non-standard syntax after a blank line. Thus, we just extract the header.
99+
sed -e '/^$/,$d' *.egg-info/requires.txt | tee requirements.txt
100+
pip install --progress-bar=off -r requirements.txt
101+
echo '::endgroup::'
102+
103+
echo '::group::Install extension-cpp'
104+
python setup.py develop
105+
echo '::endgroup::'
106+
107+
echo '::group::Collect environment information'
108+
conda list
109+
python -m torch.utils.collect_env
110+
echo '::endgroup::'

‎.github/scripts/unittest.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
./.github/scripts/setup-env.sh
6+
7+
# Activate conda environment
8+
eval "$($(which conda) shell.bash hook)" && conda deactivate && conda activate ci
9+
10+
echo '::group::Install testing utilities'
11+
pip install --progress-bar=off pytest pytest-mock pytest-cov expecttest numpy
12+
echo '::endgroup::'
13+
14+
pytest test/ --junit-xml="${RUNNER_TEST_RESULTS_DIR}/test-results.xml" -v

‎.github/workflows/tests.yml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Tests
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- master
8+
workflow_dispatch:
9+
10+
jobs:
11+
unittests-linux:
12+
strategy:
13+
matrix:
14+
python-version:
15+
- "3.11"
16+
runner: ["linux.12xlarge"]
17+
gpu-arch-type: ["cpu"]
18+
include:
19+
- python-version: 3.8
20+
runner: linux.g5.4xlarge.nvidia.gpu
21+
gpu-arch-type: cuda
22+
gpu-arch-version: "11.8"
23+
fail-fast: false
24+
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
25+
with:
26+
repository: pytorch/extension-cpp
27+
runner: ${{ matrix.runner }}
28+
gpu-arch-type: ${{ matrix.gpu-arch-type }}
29+
gpu-arch-version: ${{ matrix.gpu-arch-version }}
30+
timeout: 120
31+
script: |
32+
set -euo pipefail
33+
34+
export PYTHON_VERSION=${{ matrix.python-version }}
35+
export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }}
36+
export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }}
37+
38+
./.github/scripts/unittest.sh

‎test/test_extension.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ def _opcheck(self, device):
6262
def test_opcheck_cpu(self):
6363
self._opcheck("cpu")
6464

65+
@unittest.skipIf(not torch.cuda.is_available(), "requires cuda")
6566
def test_opcheck_cuda(self):
6667
self._opcheck("cuda")
6768

0 commit comments

Comments
 (0)