From 21eb96a1ffccb29628ad8a57f802e24839777353 Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 19 Dec 2021 01:43:28 +0100 Subject: [PATCH 01/10] Add mambaforge to github actions ci --- .github/workflows/database.yml | 2 ++ .github/workflows/posix.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index 3fdca951ae058..f937d7b289351 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -88,6 +88,8 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: + mamba-version: "*" + channels: conda-forge,defaults activate-environment: pandas-dev channel-priority: strict environment-file: ${{ matrix.ENV_FILE }} diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 9839eacc046f9..251514d39424f 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -81,6 +81,8 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: + mamba-version: "*" + channels: conda-forge,defaults activate-environment: pandas-dev channel-priority: flexible environment-file: ${{ env.ENV_FILE }} From 780e121dfc3daea61b19dc25473b7bc176be314f Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 19 Dec 2021 01:56:27 +0100 Subject: [PATCH 02/10] Add to checks ci --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bb82ed5d6816..b97682ac38b2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,8 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: + mamba-version: "*" + channels: conda-forge,defaults activate-environment: pandas-dev channel-priority: strict environment-file: ${{ env.ENV_FILE }} @@ -104,6 +106,8 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: + mamba-version: "*" + channels: conda-forge,defaults activate-environment: pandas-dev channel-priority: strict environment-file: ${{ env.ENV_FILE }} From 2227ee620711362b0f3668dea3905032fe6b6681 Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 19 Dec 2021 01:59:12 +0100 Subject: [PATCH 03/10] Also use for windows builds --- ci/setup_env.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 2e16bc6545161..fe21ab501266b 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -48,6 +48,7 @@ conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false conda install pip conda # create conda to create a historical artifact for pip & setuptools conda update -n base conda +conda install -y -c conda-forge mamba echo "conda info -a" conda info -a @@ -62,8 +63,8 @@ conda list conda remove --all -q -y -n pandas-dev echo -echo "conda env create -q --file=${ENV_FILE}" -time conda env create -q --file="${ENV_FILE}" +echo "mamba env create -q --file=${ENV_FILE}" +time mamba env create -q --file="${ENV_FILE}" if [[ "$BITS32" == "yes" ]]; then From 3b2ec07bc65b33189ca2e46ceb96f15ca7fa850d Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 19 Dec 2021 11:19:09 +0100 Subject: [PATCH 04/10] Restrict pyarrow to greater than pulled version --- ci/deps/actions-39.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index dc897453114f1..8616e134bdac3 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -25,7 +25,7 @@ dependencies: - numexpr - numpy - openpyxl - - pyarrow + - pyarrow>2.0.0 - pytables - python-dateutil - pytz From 241826f2d1a93d90c79a15fa2ab8cab4a59f3103 Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 19 Dec 2021 13:34:28 +0100 Subject: [PATCH 05/10] Try changing order --- ci/deps/actions-39.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 8616e134bdac3..4b00ee71e4643 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -21,9 +21,9 @@ dependencies: - jinja2 - lxml - matplotlib + - numpy - numba - numexpr - - numpy - openpyxl - pyarrow>2.0.0 - pytables From 223cb6879a2f32cb9083ca420ab7d99b76336b38 Mon Sep 17 00:00:00 2001 From: phofl Date: Sun, 19 Dec 2021 14:18:38 +0100 Subject: [PATCH 06/10] Remove arrow pin --- ci/deps/actions-39.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 4b00ee71e4643..e586f48750870 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -25,7 +25,7 @@ dependencies: - numba - numexpr - openpyxl - - pyarrow>2.0.0 + - pyarrow - pytables - python-dateutil - pytz From 66516f7999413cc7421f8dac10b76a6d6163d499 Mon Sep 17 00:00:00 2001 From: phofl Date: Mon, 20 Dec 2021 21:47:10 +0100 Subject: [PATCH 07/10] Change order --- ci/deps/actions-39.yaml | 5 +++-- environment.yml | 8 ++++---- requirements-dev.txt | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index e586f48750870..5b2baa8fb393a 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -11,6 +11,9 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 + - numpy + - pyarrow + # pandas dependencies - beautifulsoup4 - bottleneck @@ -21,11 +24,9 @@ dependencies: - jinja2 - lxml - matplotlib - - numpy - numba - numexpr - openpyxl - - pyarrow - pytables - python-dateutil - pytz diff --git a/environment.yml b/environment.yml index 30d05ab7700ff..0735ce8d7dd4e 100644 --- a/environment.yml +++ b/environment.yml @@ -74,6 +74,10 @@ dependencies: - notebook>=6.0.3 - pip + - fastparquet>=0.4.0 # pandas.read_parquet, DataFrame.to_parquet + - pyarrow>=1.0.1 # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather + - python-snappy # required by pyarrow + # optional - blosc - bottleneck>=1.3.1 @@ -99,10 +103,6 @@ dependencies: - xlwt - odfpy - - fastparquet>=0.4.0 # pandas.read_parquet, DataFrame.to_parquet - - pyarrow>=1.0.1 # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather - - python-snappy # required by pyarrow - - pytables>=3.6.1 # pandas.read_hdf, DataFrame.to_hdf - s3fs>=0.4.0 # file IO when using 's3://...' path - aiobotocore<2.0.0 # GH#44311 pinned to fix docbuild diff --git a/requirements-dev.txt b/requirements-dev.txt index 5673becbbe1cb..229224b84e3ba 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -47,6 +47,9 @@ ipywidgets nbformat notebook>=6.0.3 pip +fastparquet>=0.4.0 +pyarrow>=1.0.1 +python-snappy blosc bottleneck>=1.3.1 ipykernel @@ -64,9 +67,6 @@ xlrd xlsxwriter xlwt odfpy -fastparquet>=0.4.0 -pyarrow>=1.0.1 -python-snappy tables>=3.6.1 s3fs>=0.4.0 aiobotocore<2.0.0 From 177f1e4180e17fee970126d9c8c95ef478cef00c Mon Sep 17 00:00:00 2001 From: phofl Date: Mon, 20 Dec 2021 22:00:49 +0100 Subject: [PATCH 08/10] Pin pyarrow --- ci/deps/actions-39-slow.yaml | 2 +- ci/deps/actions-39.yaml | 5 +++-- environment.yml | 8 ++++---- requirements-dev.txt | 6 +++--- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ci/deps/actions-39-slow.yaml b/ci/deps/actions-39-slow.yaml index cb54210e81f23..23cc9de14f007 100644 --- a/ci/deps/actions-39-slow.yaml +++ b/ci/deps/actions-39-slow.yaml @@ -26,7 +26,7 @@ dependencies: - numexpr - numpy - openpyxl - - pyarrow + - pyarrow>2.0.1 - pytables - python-dateutil - pytz diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 5b2baa8fb393a..bcfabbe000e02 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -11,8 +11,7 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - - numpy - - pyarrow + # pandas dependencies - beautifulsoup4 @@ -24,9 +23,11 @@ dependencies: - jinja2 - lxml - matplotlib + - numpy - numba - numexpr - openpyxl + - pyarrow>2.0.1 - pytables - python-dateutil - pytz diff --git a/environment.yml b/environment.yml index 0735ce8d7dd4e..b8ba4338e9f36 100644 --- a/environment.yml +++ b/environment.yml @@ -74,10 +74,6 @@ dependencies: - notebook>=6.0.3 - pip - - fastparquet>=0.4.0 # pandas.read_parquet, DataFrame.to_parquet - - pyarrow>=1.0.1 # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather - - python-snappy # required by pyarrow - # optional - blosc - bottleneck>=1.3.1 @@ -103,6 +99,10 @@ dependencies: - xlwt - odfpy + - fastparquet>=0.4.0 # pandas.read_parquet, DataFrame.to_parquet + - pyarrow>2.0.1 # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather + - python-snappy # required by pyarrow + - pytables>=3.6.1 # pandas.read_hdf, DataFrame.to_hdf - s3fs>=0.4.0 # file IO when using 's3://...' path - aiobotocore<2.0.0 # GH#44311 pinned to fix docbuild diff --git a/requirements-dev.txt b/requirements-dev.txt index 229224b84e3ba..2cae0732f7939 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -47,9 +47,6 @@ ipywidgets nbformat notebook>=6.0.3 pip -fastparquet>=0.4.0 -pyarrow>=1.0.1 -python-snappy blosc bottleneck>=1.3.1 ipykernel @@ -67,6 +64,9 @@ xlrd xlsxwriter xlwt odfpy +fastparquet>=0.4.0 +pyarrow>2.0.1 +python-snappy tables>=3.6.1 s3fs>=0.4.0 aiobotocore<2.0.0 From 40a3ca3bc035d3da150f4ce0ebcaf3a72c0baf9d Mon Sep 17 00:00:00 2001 From: phofl Date: Wed, 22 Dec 2021 00:18:29 +0100 Subject: [PATCH 09/10] Add comment --- ci/deps/actions-39.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index bcfabbe000e02..68e729f726beb 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -11,8 +11,6 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - - # pandas dependencies - beautifulsoup4 - bottleneck @@ -23,7 +21,7 @@ dependencies: - jinja2 - lxml - matplotlib - - numpy + - numpy # move up to solve before numba - numba - numexpr - openpyxl From e3d3fa5b610e8b078d2d80269cccb15d3e170adb Mon Sep 17 00:00:00 2001 From: phofl Date: Wed, 22 Dec 2021 10:30:00 +0100 Subject: [PATCH 10/10] Remove defaults channel --- .github/workflows/ci.yml | 4 ++-- .github/workflows/database.yml | 2 +- .github/workflows/posix.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b97682ac38b2d..b9a85e25e60ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: mamba-version: "*" - channels: conda-forge,defaults + channels: conda-forge activate-environment: pandas-dev channel-priority: strict environment-file: ${{ env.ENV_FILE }} @@ -107,7 +107,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: mamba-version: "*" - channels: conda-forge,defaults + channels: conda-forge activate-environment: pandas-dev channel-priority: strict environment-file: ${{ env.ENV_FILE }} diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index f937d7b289351..294091ec9852e 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -89,7 +89,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: mamba-version: "*" - channels: conda-forge,defaults + channels: conda-forge activate-environment: pandas-dev channel-priority: strict environment-file: ${{ matrix.ENV_FILE }} diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 251514d39424f..3fa9341bd0fef 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -82,7 +82,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: mamba-version: "*" - channels: conda-forge,defaults + channels: conda-forge activate-environment: pandas-dev channel-priority: flexible environment-file: ${{ env.ENV_FILE }}