From 9bdfa01f001dea9693b3eba93bd562185f4cf609 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Wed, 8 Sep 2021 23:18:06 -0400 Subject: [PATCH 1/6] try windows with auto workers --- ci/azure/windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 05ba7c57ad6c4..e6fe76f224908 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -12,13 +12,13 @@ jobs: ENV_FILE: ci/deps/azure-windows-38.yaml CONDA_PY: "38" PATTERN: "not slow and not network" - PYTEST_WORKERS: 2 # GH-42236 + PYTEST_WORKERS: "auto" py39: ENV_FILE: ci/deps/azure-windows-39.yaml CONDA_PY: "39" PATTERN: "not slow and not network and not high_memory" - PYTEST_WORKERS: 2 # GH-42236 + PYTEST_WORKERS: "auto" steps: - powershell: | From cf7550449c8c5623ebab14cb96ee7c2639bb1caf Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Thu, 9 Sep 2021 07:48:58 -0400 Subject: [PATCH 2/6] change pattern for skipping multiple tests --- ci/azure/windows.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index e6fe76f224908..b186a0f82fa31 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -11,14 +11,12 @@ jobs: py38_np18: ENV_FILE: ci/deps/azure-windows-38.yaml CONDA_PY: "38" - PATTERN: "not slow and not network" - PYTEST_WORKERS: "auto" + PATTERN: "not (slow or network)" py39: ENV_FILE: ci/deps/azure-windows-39.yaml CONDA_PY: "39" - PATTERN: "not slow and not network and not high_memory" - PYTEST_WORKERS: "auto" + PATTERN: "not (slow or network or high_memory)" steps: - powershell: | From a7d65b6020fee1acaa24d446d4abedcff1cff148 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Thu, 9 Sep 2021 09:10:36 -0400 Subject: [PATCH 3/6] put back 2 workers --- ci/azure/windows.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index b186a0f82fa31..8480607fa1878 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -12,11 +12,13 @@ jobs: ENV_FILE: ci/deps/azure-windows-38.yaml CONDA_PY: "38" PATTERN: "not (slow or network)" + PYTEST_WORKERS: 2 # GH-42236 py39: ENV_FILE: ci/deps/azure-windows-39.yaml CONDA_PY: "39" PATTERN: "not (slow or network or high_memory)" + PYTEST_WORKERS: 2 # GH-42236 steps: - powershell: | From 70bdf08d97ad8e3139fee533b08cf1d1cf0b121f Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Thu, 9 Sep 2021 10:43:46 -0400 Subject: [PATCH 4/6] use one worker. Use wmic to get CPU info --- ci/azure/windows.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 8480607fa1878..a53ef75f208a3 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -12,13 +12,13 @@ jobs: ENV_FILE: ci/deps/azure-windows-38.yaml CONDA_PY: "38" PATTERN: "not (slow or network)" - PYTEST_WORKERS: 2 # GH-42236 + PYTEST_WORKERS: 1 # GH-43468 py39: ENV_FILE: ci/deps/azure-windows-39.yaml CONDA_PY: "39" PATTERN: "not (slow or network or high_memory)" - PYTEST_WORKERS: 2 # GH-42236 + PYTEST_WORKERS: 1 # GH-43468 steps: - powershell: | @@ -39,6 +39,7 @@ jobs: displayName: 'Build' - bash: | source activate pandas-dev + wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed ci/run_tests.sh displayName: 'Test' - task: PublishTestResults@2 From 6c3075ed2d230a6211a126d41468f5a6203147ca Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Thu, 9 Sep 2021 22:20:58 -0400 Subject: [PATCH 5/6] try splitting test targets via REs --- azure-pipelines.yml | 1 + ci/azure/windows.yml | 28 ++++++++++++++++++++++------ ci/run_tests.sh | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e16fd36ac8f98..6c685d09ab55a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,6 +17,7 @@ pr: variables: PYTEST_WORKERS: auto + PYTEST_TARGET: pandas jobs: # Mac and Linux use the same template diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index a53ef75f208a3..f7a6a76dd0b6f 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -8,17 +8,33 @@ jobs: vmImage: ${{ parameters.vmImage }} strategy: matrix: - py38_np18: + py38_np18_1: ENV_FILE: ci/deps/azure-windows-38.yaml CONDA_PY: "38" - PATTERN: "not (slow or network)" - PYTEST_WORKERS: 1 # GH-43468 + PATTERN: "not slow and not network" + PYTEST_WORKERS: 2 # GH-42236 + PYTEST_TARGET: "pandas/tests/[a-i]*" - py39: + py38_np18_2: + ENV_FILE: ci/deps/azure-windows-38.yaml + CONDA_PY: "38" + PATTERN: "not slow and not network" + PYTEST_WORKERS: 2 # GH-42236 + PYTEST_TARGET: "pandas/tests/[j-z]*" + + py39_1: + ENV_FILE: ci/deps/azure-windows-39.yaml + CONDA_PY: "39" + PATTERN: "not slow and not network and not high_memory" + PYTEST_WORKERS: 2 # GH-42236 + PYTEST_TARGET: "pandas/tests/[a-i]*" + + py39_2: ENV_FILE: ci/deps/azure-windows-39.yaml CONDA_PY: "39" - PATTERN: "not (slow or network or high_memory)" - PYTEST_WORKERS: 1 # GH-43468 + PATTERN: "not slow and not network and not high_memory" + PYTEST_WORKERS: 2 # GH-42236 + PYTEST_TARGET: "pandas/tests/[j-z]*" steps: - powershell: | diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 8304b89499465..53a2bf151d3bf 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -19,7 +19,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas" +PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ $(uname) != "Linux" && $(uname) != "Darwin" ]]; then # GH#37455 windows py38 build appears to be running out of memory From a5d56b91c349658fe648ae18f5c5eff27197291d Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Thu, 9 Sep 2021 23:09:41 -0400 Subject: [PATCH 6/6] add target for github workflows --- .github/workflows/ci.yml | 1 + .github/workflows/posix.yml | 1 + .github/workflows/python-dev.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7a26a7905799..93c17997a95f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,6 +168,7 @@ jobs: PANDAS_DATA_MANAGER: array PATTERN: ${{ matrix.pattern }} PYTEST_WORKERS: "auto" + PYTEST_TARGET: pandas run: | source activate pandas-dev ci/run_tests.sh diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index b707d583ab96a..f37f31686ef69 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -44,6 +44,7 @@ jobs: LC_ALL: ${{ matrix.settings[4] }} PANDAS_TESTING_MODE: ${{ matrix.settings[5] }} TEST_ARGS: ${{ matrix.settings[6] }} + PYTEST_TARGET: pandas concurrency: group: ${{ github.ref }}-${{ matrix.settings[0] }} cancel-in-progress: ${{github.event_name == 'pull_request'}} diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 7429155fe5023..596c3b6df9d49 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -17,6 +17,7 @@ env: PANDAS_CI: 1 PATTERN: "not slow and not network and not clipboard" COVERAGE: true + PYTEST_TARGET: pandas jobs: build: