diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
index 583619826265c..925284341a7f4 100644
--- a/.github/workflows/code-checks.yml
+++ b/.github/workflows/code-checks.yml
@@ -116,16 +116,23 @@ jobs:
 
     - name: Set up Conda
       uses: ./.github/actions/setup-conda
+      with:
+        environment-file: ci/deps/asv.yaml
 
-    - name: Build Pandas
-      id: build
-      uses: ./.github/actions/build_pandas
+    - name: Run ASV benchmarks
+      run: |
+        git remote -v
+
+    - name: Run ASV benchmarks
+      run: |
+        git branch --show-current
 
     - name: Run ASV benchmarks
       run: |
         cd asv_bench
+        git fetch origin main:main
         asv machine --yes
-        asv run --quick --dry-run --strict --durations=30 --python=same
+        asv run --quick --dry-run --strict --durations=30
 
   build_docker_dev_environment:
     name: Build Docker Dev Environment
diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json
index 4a0c882640eb6..74328d706893e 100644
--- a/asv_bench/asv.conf.json
+++ b/asv_bench/asv.conf.json
@@ -50,6 +50,7 @@
         "numexpr": [],
         "pytables": [null, ""],  // platform dependent, see excludes below
         "pyarrow": [],
+        "pytorch": [],
         "tables": [null, ""],
         "openpyxl": [],
         "xlsxwriter": [],
diff --git a/ci/deps/asv.yaml b/ci/deps/asv.yaml
new file mode 100644
index 0000000000000..59ef551100507
--- /dev/null
+++ b/ci/deps/asv.yaml
@@ -0,0 +1,35 @@
+# Dependencies used in ASV performance benchmarks
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  - python=3.8
+
+  # test dependencies
+  - cython>=0.29.32
+
+  # required dependencies
+  - python-dateutil
+  - numpy
+  - pytz
+
+  # optional dependencies
+  - jinja2
+  - matplotlib
+  - numba
+  - numexpr
+  - openpyxl
+  - odfpy
+  - pyarrow
+  - pytables
+  - scipy
+  - sqlalchemy
+  - xlrd
+  - xlsxwriter
+
+  # downstream packages
+  - py
+  - pytorch
+
+  # benchmarks
+  - asv
diff --git a/environment.yml b/environment.yml
index 3c1df67782132..4524961a634f6 100644
--- a/environment.yml
+++ b/environment.yml
@@ -69,7 +69,6 @@ dependencies:
   - pandas-datareader
   - pyyaml
   - py
-  - pytorch
 
   # local testing dependencies
   - moto
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 30ee35db05c50..a5d531f616061 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -58,7 +58,6 @@ coverage
 pandas-datareader
 pyyaml
 py
-torch
 moto
 flask
 asv