Merge branch 'main' into sas/decompress3

Author: jonashaag

.github/actions/build_pandas/action.yml

@@ -6,8 +6,8 @@ runs:
 
     - name: Environment Detail
       run: |
-        conda info
-        conda list
+        micromamba info
+        micromamba list
       shell: bash -el {0}
 
     - name: Build Pandas

.github/actions/setup-conda/action.yml

@@ -6,8 +6,8 @@ inputs:
   environment-name:
     description: Name to use for the Conda environment
     default: test
-  python-version:
-    description: Python version to install
+  extra-specs:
+    description: Extra packages to install
     required: false
   pyarrow-version:
     description: If set, overrides the PyArrow version in the Conda environment to the given string.
@@ -24,14 +24,13 @@ runs:
       if: ${{ inputs.pyarrow-version }}
 
     - name: Install ${{ inputs.environment-file }}
-      uses: conda-incubator/setup-miniconda@v2.1.1
+      uses: mamba-org/provision-with-micromamba@v12
       with:
         environment-file: ${{ inputs.environment-file }}
-        activate-environment: ${{ inputs.environment-name }}
-        python-version: ${{ inputs.python-version }}
-        channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
+        environment-name: ${{ inputs.environment-name }}
+        extra-specs: ${{ inputs.extra-specs }}
         channels: conda-forge
-        mamba-version: "0.24"
-        use-mamba: true
-        use-only-tar-bz2: true
+        channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
         condarc-file: ci/condarc.yml
+        cache-env: true
+        cache-downloads: true

.github/workflows/asv-bot.yml

@@ -33,12 +33,6 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Cache conda
-        uses: actions/cache@v3
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
-
         # Although asv sets up its own env, deps are still needed
         # during discovery process
       - name: Set up Conda

.github/workflows/code-checks.yml

@@ -52,12 +52,6 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Cache conda
-      uses: actions/cache@v3
-      with:
-        path: ~/conda_pkgs_dir
-        key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
-
     - name: Set up Conda
       uses: ./.github/actions/setup-conda
 
@@ -117,12 +111,6 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Cache conda
-      uses: actions/cache@v3
-      with:
-        path: ~/conda_pkgs_dir
-        key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
-
     - name: Set up Conda
       uses: ./.github/actions/setup-conda
 

.github/workflows/sdist.yml

@@ -62,9 +62,10 @@ jobs:
     - name: Set up Conda
       uses: ./.github/actions/setup-conda
       with:
-        environment-file: ""
+        environment-file: false
         environment-name: pandas-sdist
-        python-version: ${{ matrix.python-version }}
+        extra-specs: |
+          python =${{ matrix.python-version }}
 
     - name: Install pandas from sdist
       run: |

.github/workflows/ubuntu.yml

@@ -134,15 +134,6 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Cache conda
-      uses: actions/cache@v3
-      env:
-        CACHE_NUMBER: 0
-      with:
-        path: ~/conda_pkgs_dir
-        key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
-          hashFiles('${{ env.ENV_FILE }}') }}
-
     - name: Extra installs
       # xsel for clipboard tests
       run: sudo apt-get update && sudo apt-get install -y xsel ${{ env.EXTRA_APT }}

asv_bench/benchmarks/indexing.py

@@ -157,25 +157,39 @@ def time_boolean_rows_boolean(self):
 
 
 class DataFrameNumericIndexing:
-    def setup(self):
+
+    params = [
+        (Int64Index, UInt64Index, Float64Index),
+        ("unique_monotonic_inc", "nonunique_monotonic_inc"),
+    ]
+    param_names = ["index_dtype", "index_structure"]
+
+    def setup(self, index, index_structure):
+        N = 10**5
+        indices = {
+            "unique_monotonic_inc": index(range(N)),
+            "nonunique_monotonic_inc": index(
+                list(range(55)) + [54] + list(range(55, N - 1))
+            ),
+        }
         self.idx_dupe = np.array(range(30)) * 99
-        self.df = DataFrame(np.random.randn(100000, 5))
+        self.df = DataFrame(np.random.randn(N, 5), index=indices[index_structure])
         self.df_dup = concat([self.df, 2 * self.df, 3 * self.df])
-        self.bool_indexer = [True] * 50000 + [False] * 50000
+        self.bool_indexer = [True] * (N // 2) + [False] * (N - N // 2)
 
-    def time_iloc_dups(self):
+    def time_iloc_dups(self, index, index_structure):
         self.df_dup.iloc[self.idx_dupe]
 
-    def time_loc_dups(self):
+    def time_loc_dups(self, index, index_structure):
         self.df_dup.loc[self.idx_dupe]
 
-    def time_iloc(self):
+    def time_iloc(self, index, index_structure):
         self.df.iloc[:100, 0]
 
-    def time_loc(self):
+    def time_loc(self, index, index_structure):
         self.df.loc[:100, 0]
 
-    def time_bool_indexer(self):
+    def time_bool_indexer(self, index, index_structure):
         self.df[self.bool_indexer]
 
 

asv_bench/benchmarks/io/excel.py

@@ -47,6 +47,25 @@ def time_write_excel(self, engine):
         writer.save()
 
 
+class WriteExcelStyled:
+    params = ["openpyxl", "xlsxwriter"]
+    param_names = ["engine"]
+
+    def setup(self, engine):
+        self.df = _generate_dataframe()
+
+    def time_write_excel_style(self, engine):
+        bio = BytesIO()
+        bio.seek(0)
+        writer = ExcelWriter(bio, engine=engine)
+        df_style = self.df.style
+        df_style.applymap(lambda x: "border: red 1px solid;")
+        df_style.applymap(lambda x: "color: blue")
+        df_style.applymap(lambda x: "border-color: green black", subset=["float1"])
+        df_style.to_excel(writer, sheet_name="Sheet1")
+        writer.save()
+
+
 class ReadExcel:
 
     params = ["xlrd", "openpyxl", "odf"]

asv_bench/benchmarks/io/sql.py

@@ -39,6 +39,8 @@ def setup(self, connection):
             index=tm.makeStringIndex(N),
         )
         self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df["date"] = self.df["datetime"].dt.date
+        self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
         self.df.to_sql(self.table_name, self.con, if_exists="replace")
 
@@ -53,7 +55,16 @@ class WriteSQLDtypes:
 
     params = (
         ["sqlalchemy", "sqlite"],
-        ["float", "float_with_nan", "string", "bool", "int", "datetime"],
+        [
+            "float",
+            "float_with_nan",
+            "string",
+            "bool",
+            "int",
+            "date",
+            "time",
+            "datetime",
+        ],
     )
     param_names = ["connection", "dtype"]
 
@@ -78,6 +89,8 @@ def setup(self, connection, dtype):
             index=tm.makeStringIndex(N),
         )
         self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df["date"] = self.df["datetime"].dt.date
+        self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
         self.df.to_sql(self.table_name, self.con, if_exists="replace")
 
@@ -105,6 +118,8 @@ def setup(self):
             index=tm.makeStringIndex(N),
         )
         self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df["date"] = self.df["datetime"].dt.date
+        self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
         self.df.to_sql(self.table_name, self.con, if_exists="replace")
 
@@ -122,7 +137,16 @@ def time_read_sql_table_parse_dates(self):
 
 class ReadSQLTableDtypes:
 
-    params = ["float", "float_with_nan", "string", "bool", "int", "datetime"]
+    params = [
+        "float",
+        "float_with_nan",
+        "string",
+        "bool",
+        "int",
+        "date",
+        "time",
+        "datetime",
+    ]
     param_names = ["dtype"]
 
     def setup(self, dtype):
@@ -141,6 +165,8 @@ def setup(self, dtype):
             index=tm.makeStringIndex(N),
         )
         self.df.loc[1000:3000, "float_with_nan"] = np.nan
+        self.df["date"] = self.df["datetime"].dt.date
+        self.df["time"] = self.df["datetime"].dt.time
         self.df["datetime_string"] = self.df["datetime"].astype(str)
         self.df.to_sql(self.table_name, self.con, if_exists="replace")
 

asv_bench/benchmarks/tslibs/strftime.py

@@ -0,0 +1,64 @@
+import numpy as np
+
+import pandas as pd
+from pandas import offsets
+
+
+class DatetimeStrftime:
+    timeout = 1500
+    params = [1000, 10000]
+    param_names = ["obs"]
+
+    def setup(self, obs):
+        d = "2018-11-29"
+        dt = "2018-11-26 11:18:27.0"
+        self.data = pd.DataFrame(
+            {
+                "dt": [np.datetime64(dt)] * obs,
+                "d": [np.datetime64(d)] * obs,
+                "r": [np.random.uniform()] * obs,
+            }
+        )
+
+    def time_frame_date_to_str(self, obs):
+        self.data["d"].astype(str)
+
+    def time_frame_date_formatting_default(self, obs):
+        self.data["d"].dt.strftime(date_format="%Y-%m-%d")
+
+    def time_frame_date_formatting_custom(self, obs):
+        self.data["d"].dt.strftime(date_format="%Y---%m---%d")
+
+    def time_frame_datetime_to_str(self, obs):
+        self.data["dt"].astype(str)
+
+    def time_frame_datetime_formatting_default_date_only(self, obs):
+        self.data["dt"].dt.strftime(date_format="%Y-%m-%d")
+
+    def time_frame_datetime_formatting_default(self, obs):
+        self.data["dt"].dt.strftime(date_format="%Y-%m-%d %H:%M:%S")
+
+    def time_frame_datetime_formatting_default_with_float(self, obs):
+        self.data["dt"].dt.strftime(date_format="%Y-%m-%d %H:%M:%S.%f")
+
+    def time_frame_datetime_formatting_custom(self, obs):
+        self.data["dt"].dt.strftime(date_format="%Y-%m-%d --- %H:%M:%S")
+
+
+class BusinessHourStrftime:
+    timeout = 1500
+    params = [1000, 10000]
+    param_names = ["obs"]
+
+    def setup(self, obs):
+        self.data = pd.DataFrame(
+            {
+                "off": [offsets.BusinessHour()] * obs,
+            }
+        )
+
+    def time_frame_offset_str(self, obs):
+        self.data["off"].apply(str)
+
+    def time_frame_offset_repr(self, obs):
+        self.data["off"].apply(repr)

doc/source/getting_started/tutorials.rst

@@ -118,3 +118,4 @@ Various tutorials
 * `Pandas and Python: Top 10, by Manish Amde <https://manishamde.github.io/blog/2013/03/07/pandas-and-python-top-10/>`_
 * `Pandas DataFrames Tutorial, by Karlijn Willems <https://www.datacamp.com/community/tutorials/pandas-tutorial-dataframe-python>`_
 * `A concise tutorial with real life examples <https://tutswiki.com/pandas-cookbook/chapter1/>`_
+* `430+ Searchable Pandas recipes by Isshin Inada <https://skytowner.com/explore/pandas_recipes_reference>`_

doc/source/reference/general_functions.rst

@@ -23,6 +23,7 @@ Data manipulations
    merge_asof
    concat
    get_dummies
+   from_dummies
    factorize
    unique
    wide_to_long

doc/source/user_guide/io.rst

@@ -2559,16 +2559,29 @@ Let's look at a few examples.
 
 Read a URL with no options:
 
-.. ipython:: python
+.. code-block:: ipython
 
-   url = "https://www.fdic.gov/resources/resolutions/bank-failures/failed-bank-list"
-   dfs = pd.read_html(url)
-   dfs
+   In [320]: "https://www.fdic.gov/resources/resolutions/bank-failures/failed-bank-list"
+   In [321]: pd.read_html(url)
+   Out[321]:
+   [                         Bank NameBank           CityCity StateSt  ...              Acquiring InstitutionAI Closing DateClosing FundFund
+    0                    Almena State Bank             Almena      KS  ...                          Equity Bank    October 23, 2020    10538
+    1           First City Bank of Florida  Fort Walton Beach      FL  ...            United Fidelity Bank, fsb    October 16, 2020    10537
+    2                 The First State Bank      Barboursville      WV  ...                       MVB Bank, Inc.       April 3, 2020    10536
+    3                   Ericson State Bank            Ericson      NE  ...           Farmers and Merchants Bank   February 14, 2020    10535
+    4     City National Bank of New Jersey             Newark      NJ  ...                      Industrial Bank    November 1, 2019    10534
+    ..                                 ...                ...     ...  ...                                  ...                 ...      ...
+    558                 Superior Bank, FSB           Hinsdale      IL  ...                Superior Federal, FSB       July 27, 2001     6004
+    559                Malta National Bank              Malta      OH  ...                    North Valley Bank         May 3, 2001     4648
+    560    First Alliance Bank & Trust Co.         Manchester      NH  ...  Southern New Hampshire Bank & Trust    February 2, 2001     4647
+    561  National State Bank of Metropolis         Metropolis      IL  ...              Banterra Bank of Marion   December 14, 2000     4646
+    562                   Bank of Honolulu           Honolulu      HI  ...                   Bank of the Orient    October 13, 2000     4645
+
+    [563 rows x 7 columns]]
 
 .. note::
 
-   The data from the above URL changes every Monday so the resulting data above
-   and the data below may be slightly different.
+   The data from the above URL changes every Monday so the resulting data above may be slightly different.
 
 Read in the content of the file from the above URL and pass it to ``read_html``
 as a string: