Merge branch 'main' into bug-agg-nonunique-col

Author: luke396

.github/actions/setup-conda/action.yml

@@ -9,20 +9,9 @@ inputs:
   extra-specs:
     description: Extra packages to install
     required: false
-  pyarrow-version:
-    description: If set, overrides the PyArrow version in the Conda environment to the given string.
-    required: false
 runs:
   using: composite
   steps:
-    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
-      run: |
-        grep -q '  - pyarrow' ${{ inputs.environment-file }}
-        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
-        cat ${{ inputs.environment-file }}
-      shell: bash
-      if: ${{ inputs.pyarrow-version }}
-
     - name: Install ${{ inputs.environment-file }}
       uses: mamba-org/provision-with-micromamba@v12
       with:

.github/workflows/dependabot.yml renamed to .github/dependabot.yml

@@ -35,7 +35,7 @@ jobs:
         python-version: '3.9'
 
     - name: Run pre-commit
-      uses: pre-commit/action@v2.0.3
+      uses: pre-commit/action@v3.0.0
       with:
         extra_args: --verbose --all-files
 
@@ -93,7 +93,7 @@ jobs:
       if: ${{ steps.build.outcome == 'success' && always() }}
 
     - name: Typing + pylint
-      uses: pre-commit/action@v2.0.3
+      uses: pre-commit/action@v3.0.0
       with:
         extra_args: --verbose --hook-stage manual --all-files
       if: ${{ steps.build.outcome == 'success' && always() }}

.github/workflows/code-checks.yml

@@ -52,7 +52,6 @@ jobs:
       uses: ./.github/actions/setup-conda
       with:
         environment-file: ci/deps/${{ matrix.env_file }}
-        pyarrow-version: ${{ matrix.os == 'macos-latest' && '9' || '' }}
 
     - name: Build Pandas
       uses: ./.github/actions/build_pandas

.github/workflows/macos-windows.yml

@@ -13,7 +13,7 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/stale@v4
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity. Please [update](https://pandas.pydata.org/pandas-docs/stable/development/contributing.html#updating-your-pull-request) and respond to this comment if you're still interested in working on this."

.github/workflows/stale-pr.yml

@@ -26,8 +26,8 @@ jobs:
     strategy:
       matrix:
         env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml, actions-311.yaml]
-        pattern: ["not single_cpu", "single_cpu"]
-        pyarrow_version: ["8", "9", "10"]
+        # Prevent the include jobs from overriding other jobs
+        pattern: [""]
         include:
           - name: "Downstream Compat"
             env_file: actions-38-downstream_compat.yaml
@@ -75,21 +75,11 @@ jobs:
             # TODO(cython3): Re-enable once next-beta(after beta 1) comes out
             # There are some warnings failing the build with -werror
             pandas_ci: "0"
-        exclude:
-          - env_file: actions-38.yaml
-            pyarrow_version: "8"
-          - env_file: actions-38.yaml
-            pyarrow_version: "9"
-          - env_file: actions-39.yaml
-            pyarrow_version: "8"
-          - env_file: actions-39.yaml
-            pyarrow_version: "9"
-          - env_file: actions-310.yaml
-            pyarrow_version: "8"
-          - env_file: actions-310.yaml
-            pyarrow_version: "9"
+          - name: "Pyarrow Nightly"
+            env_file: actions-311-pyarrownightly.yaml
+            pattern: "not slow and not network and not single_cpu"
       fail-fast: false
-    name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }}
+    name: ${{ matrix.name || matrix.env_file }}
     env:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
@@ -100,14 +90,14 @@ jobs:
       PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
       PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
-      PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }}
+      PYTEST_WORKERS: 'auto'
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
       IS_PYPY: ${{ contains(matrix.env_file, 'pypy') }}
       # TODO: re-enable coverage on pypy, its slow
       COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }}
       cancel-in-progress: true
 
     services:
@@ -166,12 +156,24 @@ jobs:
       uses: ./.github/actions/setup-conda
       with:
         environment-file: ${{ env.ENV_FILE }}
-        pyarrow-version: ${{ matrix.pyarrow_version }}
 
     - name: Build Pandas
+      id: build
       uses: ./.github/actions/build_pandas
 
-    - name: Test
+    - name: Test (not single_cpu)
+      uses: ./.github/actions/run-tests
+      # TODO: Don't continue on error for PyPy
+      continue-on-error: ${{ env.IS_PYPY == 'true' }}
+      env:
+        # Set pattern to not single_cpu if not already set
+        PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}
+
+    - name: Test (single_cpu)
       uses: ./.github/actions/run-tests
       # TODO: Don't continue on error for PyPy
       continue-on-error: ${{ env.IS_PYPY == 'true' }}
+      env:
+        PATTERN: 'single_cpu'
+        PYTEST_WORKERS: 1
+      if: ${{ matrix.pattern == '' && (always() && steps.build.outcome == 'success')}}

.github/workflows/ubuntu.yml

@@ -71,7 +71,7 @@ jobs:
           fetch-depth: 0
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.9.0
+        uses: pypa/cibuildwheel@v2.12.1
         env:
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 

.github/workflows/wheels.yml

@@ -41,7 +41,7 @@ dependencies:
   - psycopg2>=2.8.6
   - pymysql>=1.0.2
   - pytables>=3.6.1
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat>=1.1.2
   - python-snappy>=0.6.0
   - pyxlsb>=1.0.8

ci/deps/actions-310.yaml

@@ -0,0 +1,29 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11
+
+  # build dependencies
+  - versioneer[toml]
+  - cython>=0.29.33
+
+  # test dependencies
+  - pytest>=7.0.0
+  - pytest-cov
+  - pytest-xdist>=2.2.0
+  - hypothesis>=6.34.2
+  - pytest-asyncio>=0.17.0
+
+  # required dependencies
+  - python-dateutil
+  - numpy
+  - pytz
+  - pip
+
+  - pip:
+    - "tzdata>=2022.1"
+    - "--extra-index-url https://pypi.fury.io/arrow-nightlies/"
+    - "--prefer-binary"
+    - "--pre"
+    - "pyarrow"

ci/deps/actions-311-pyarrownightly.yaml

@@ -41,7 +41,7 @@ dependencies:
   - psycopg2>=2.8.6
   - pymysql>=1.0.2
   # - pytables>=3.8.0  # first version that supports 3.11
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat>=1.1.2
   - python-snappy>=0.6.0
   - pyxlsb>=1.0.8

ci/deps/actions-311.yaml

@@ -39,7 +39,7 @@ dependencies:
   - openpyxl<3.1.1, >=3.0.7
   - odfpy>=1.4.1
   - psycopg2>=2.8.6
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql>=1.0.2
   - pyreadstat>=1.1.2
   - pytables>=3.6.1

ci/deps/actions-38-downstream_compat.yaml

@@ -39,7 +39,7 @@ dependencies:
   - odfpy>=1.4.1
   - pandas-gbq>=0.15.0
   - psycopg2>=2.8.6
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql>=1.0.2
   - pyreadstat>=1.1.2
   - pytables>=3.6.1

ci/deps/actions-38.yaml

@@ -40,7 +40,7 @@ dependencies:
   - pandas-gbq>=0.15.0
   - psycopg2>=2.8.6
   - pymysql>=1.0.2
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat>=1.1.2
   - pytables>=3.6.1
   - python-snappy>=0.6.0

ci/deps/actions-39.yaml

@@ -39,7 +39,7 @@ dependencies:
   - odfpy>=1.4.1
   - pandas-gbq>=0.15.0
   - psycopg2>=2.8.6
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql>=1.0.2
   # Not provided on ARM
   #- pyreadstat

ci/deps/circle-38-arm64.yaml

@@ -10,6 +10,14 @@
   font-size: 0.9rem;
 }
 
+.gs-data-header {
+  background-color: var(--pst-color-on-surface);
+}
+
+.gs-data-list {
+  background-color: var(--pst-color-on-background);
+}
+
 .gs-data-title .badge {
   margin: 10px;
   padding: 5px;
@@ -57,45 +65,33 @@
   margin-top: -5px;
 }
 .gs-callout-remember {
-  border-left-color: #f0ad4e;
+  border-left-color: var(--pst-color-secondary);
   align-items: center;
   font-size: 1.2rem;
 }
 .gs-callout-remember h4 {
-  color: #f0ad4e;
+  color:  var(--pst-color-secondary);
 }
 
 /* reference to user guide */
 .gs-torefguide {
   align-items: center;
   font-size: 0.9rem;
+  background-color: var(--pst-color-on-background);
+  border-radius: .25rem;
+  padding: 2px;
 }
 
 .gs-torefguide .badge {
-  background-color: #130654;
-  margin: 10px 10px 10px 0px;
+  background-color: var(--pst-color-primary);
+  margin: 10px 10px 10px 10px;
   padding: 5px;
 }
 
-.gs-torefguide a {
-  margin-left: 5px;
-  color: #130654;
-  border-bottom: 1px solid #FFCA00f3;
-  box-shadow: 0px -10px 0px #FFCA00f3 inset;
-}
-
 .gs-torefguide p {
   margin-top: 1rem;
 }
 
-.gs-torefguide a:hover {
-  margin-left: 5px;
-  color: grey;
-  text-decoration: none;
-  border-bottom: 1px solid #b2ff80f3;
-  box-shadow: 0px -10px 0px #b2ff80f3 inset;
-}
-
 /* question-task environment */
 
 ul.task-bullet, ol.custom-bullet{
@@ -113,14 +109,14 @@ ul.task-bullet > li:before {
     margin-left:-2em;
     background-position:center;
     background-repeat:no-repeat;
-    background-color: #130654;
+    background-color: var(--pst-color-primary);
     border-radius: 50%;
     background-size:100%;
     background-image:url('../question_mark_noback.svg');
   }
 
 ul.task-bullet > li {
-  border-left: 1px solid #130654;
+  border-left: 1px solid var(--pst-color-primary);
   padding-left:1em;
 }
 
@@ -132,7 +128,7 @@ ul.task-bullet > li > p:first-child {
 /* Getting started index page */
 
 .comparison-card {
-  background:#FFF;
+  background-color: var(--pst-color-background);
   border-radius:0;
   padding: 30px 10px 10px 10px;
   margin: 10px 0px;
@@ -146,6 +142,7 @@ ul.task-bullet > li > p:first-child {
   margin: 10px;
   margin-bottom: 20px;
   height: 72px;
+  background: none !important;
 }
 
 .comparison-card-excel .card-img-top, .comparison-card-stata .card-img-top, .comparison-card-sas .card-img-top {
@@ -154,7 +151,7 @@ ul.task-bullet > li > p:first-child {
 
 .comparison-card .card-footer {
   border: none;
-  background-color: transparent;
+  background-color: var(--pst-color-background);
 }
 
 .install-block {
@@ -236,15 +233,16 @@ ul.task-bullet > li > p:first-child {
 
 .tutorial-card .card-header {
   cursor: pointer;
-  background-color: transparent;
+  background-color: var(--pst-color-surface);
+  border: 1px solid var(--pst-color-border)
 }
 
 .tutorial-card .card-body {
-  background-color: transparent;
+  background-color: var(--pst-color-on-background);
 }
 
 .tutorial-card .badge {
-  background-color: #130654;
+  background-color: var(--pst-color-primary);
   margin: 10px 10px 10px 10px;
   padding: 5px;
 }
@@ -253,8 +251,9 @@ ul.task-bullet > li > p:first-child {
   margin: 0px;
 }
 
+
 .tutorial-card .gs-badge-link a {
-  color: white;
+  color: var(--pst-color-text-base);
   text-decoration: none;
 }
 

doc/source/_static/css/getting_started.css

@@ -25,6 +25,7 @@ table {
 .intro-card .card-img-top {
   margin: 10px;
   height: 52px;
+  background: none !important;
 }
 
 .intro-card .card-header {
@@ -48,5 +49,5 @@ table {
 }
 
 .card, .card img {
-  background-color: transparent !important;
+  background-color: var(--pst-color-background);
 }