Merge remote-tracking branch 'upstream/master' into CI-check-for-no-internal-functions-across-modules

Author: MomIsBestFriend

asv_bench/benchmarks/frame_methods.py

@@ -619,4 +619,17 @@ def time_select_dtypes(self, n):
         self.df.select_dtypes(include="int")
 
 
+class MemoryUsage:
+    def setup(self):
+        self.df = DataFrame(np.random.randn(100000, 2), columns=list("AB"))
+        self.df2 = self.df.copy()
+        self.df2["A"] = self.df2["A"].astype("object")
+
+    def time_memory_usage(self):
+        self.df.memory_usage(deep=True)
+
+    def time_memory_usage_object_dtype(self):
+        self.df2.memory_usage(deep=True)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/indexing.py

@@ -308,6 +308,31 @@ def time_frame_getitem_single_column_int(self):
         self.df_int_col[0]
 
 
+class IndexSingleRow:
+    params = [True, False]
+    param_names = ["unique_cols"]
+
+    def setup(self, unique_cols):
+        arr = np.arange(10 ** 7).reshape(-1, 10)
+        df = DataFrame(arr)
+        dtypes = ["u1", "u2", "u4", "u8", "i1", "i2", "i4", "i8", "f8", "f4"]
+        for i, d in enumerate(dtypes):
+            df[i] = df[i].astype(d)
+
+        if not unique_cols:
+            # GH#33032 single-row lookups with non-unique columns were
+            #  15x slower than with unique columns
+            df.columns = ["A", "A"] + list(df.columns[2:])
+
+        self.df = df
+
+    def time_iloc_row(self, unique_cols):
+        self.df.iloc[10000]
+
+    def time_loc_row(self, unique_cols):
+        self.df.loc[10000]
+
+
 class AssignTimeseriesIndex:
     def setup(self):
         N = 100000

asv_bench/benchmarks/series_methods.py

@@ -223,27 +223,27 @@ def time_series_datetimeindex_repr(self):
 
 class All:
 
-    params = [[10 ** 3, 10 ** 6], ["fast", "slow"]]
-    param_names = ["N", "case"]
+    params = [[10 ** 3, 10 ** 6], ["fast", "slow"], ["bool", "boolean"]]
+    param_names = ["N", "case", "dtype"]
 
-    def setup(self, N, case):
+    def setup(self, N, case, dtype):
         val = case != "fast"
-        self.s = Series([val] * N)
+        self.s = Series([val] * N, dtype=dtype)
 
-    def time_all(self, N, case):
+    def time_all(self, N, case, dtype):
         self.s.all()
 
 
 class Any:
 
-    params = [[10 ** 3, 10 ** 6], ["fast", "slow"]]
-    param_names = ["N", "case"]
+    params = [[10 ** 3, 10 ** 6], ["fast", "slow"], ["bool", "boolean"]]
+    param_names = ["N", "case", "dtype"]
 
-    def setup(self, N, case):
+    def setup(self, N, case, dtype):
         val = case == "fast"
-        self.s = Series([val] * N)
+        self.s = Series([val] * N, dtype=dtype)
 
-    def time_any(self, N, case):
+    def time_any(self, N, case, dtype):
         self.s.any()
 
 
@@ -265,11 +265,14 @@ class NanOps:
             "prod",
         ],
         [10 ** 3, 10 ** 6],
-        ["int8", "int32", "int64", "float64"],
+        ["int8", "int32", "int64", "float64", "Int64", "boolean"],
     ]
     param_names = ["func", "N", "dtype"]
 
     def setup(self, func, N, dtype):
+        if func == "argmax" and dtype in {"Int64", "boolean"}:
+            # Skip argmax for nullable int since this doesn't work yet (GH-24382)
+            raise NotImplementedError
         self.s = Series([1] * N, dtype=dtype)
         self.func = getattr(self.s, func)
 

asv_bench/benchmarks/sparse.py

@@ -2,7 +2,8 @@
 import scipy.sparse
 
 import pandas as pd
-from pandas import MultiIndex, Series, SparseArray, date_range
+from pandas import MultiIndex, Series, date_range
+from pandas.arrays import SparseArray
 
 
 def make_array(size, dense_proportion, fill_value, dtype):

asv_bench/benchmarks/stat_ops.py

@@ -7,11 +7,17 @@
 
 class FrameOps:
 
-    params = [ops, ["float", "int"], [0, 1]]
+    params = [ops, ["float", "int", "Int64"], [0, 1]]
     param_names = ["op", "dtype", "axis"]
 
     def setup(self, op, dtype, axis):
-        df = pd.DataFrame(np.random.randn(100000, 4)).astype(dtype)
+        if op == "mad" and dtype == "Int64" and axis == 1:
+            # GH-33036
+            raise NotImplementedError
+        values = np.random.randn(100000, 4)
+        if dtype == "Int64":
+            values = values.astype(int)
+        df = pd.DataFrame(values).astype(dtype)
         self.df_func = getattr(df, op)
 
     def time_op(self, op, dtype, axis):

asv_bench/benchmarks/timeseries.py

@@ -336,15 +336,33 @@ def time_infer_quarter(self):
 
 class ToDatetimeFormat:
     def setup(self):
-        self.s = Series(["19MAY11", "19MAY11:00:00:00"] * 100000)
+        N = 100000
+        self.s = Series(["19MAY11", "19MAY11:00:00:00"] * N)
         self.s2 = self.s.str.replace(":\\S+$", "")
 
+        self.same_offset = ["10/11/2018 00:00:00.045-07:00"] * N
+        self.diff_offset = [
+            f"10/11/2018 00:00:00.045-0{offset}:00" for offset in range(10)
+        ] * int(N / 10)
+
     def time_exact(self):
         to_datetime(self.s2, format="%d%b%y")
 
     def time_no_exact(self):
         to_datetime(self.s, format="%d%b%y", exact=False)
 
+    def time_same_offset(self):
+        to_datetime(self.same_offset, format="%m/%d/%Y %H:%M:%S.%f%z")
+
+    def time_different_offset(self):
+        to_datetime(self.diff_offset, format="%m/%d/%Y %H:%M:%S.%f%z")
+
+    def time_same_offset_to_utc(self):
+        to_datetime(self.same_offset, format="%m/%d/%Y %H:%M:%S.%f%z", utc=True)
+
+    def time_different_offset_to_utc(self):
+        to_datetime(self.diff_offset, format="%m/%d/%Y %H:%M:%S.%f%z", utc=True)
+
 
 class ToDatetimeCache:
 

ci/code_checks.sh

@@ -287,18 +287,12 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
     pytest -q --doctest-modules pandas/core/groupby/groupby.py -k"-cumcount -describe -pipe"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    MSG='Doctests datetimes.py' ; echo $MSG
-    pytest -q --doctest-modules pandas/core/tools/datetimes.py
+    MSG='Doctests tools' ; echo $MSG
+    pytest -q --doctest-modules pandas/core/tools/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    MSG='Doctests top-level reshaping functions' ; echo $MSG
-    pytest -q --doctest-modules \
-        pandas/core/reshape/concat.py \
-        pandas/core/reshape/pivot.py \
-        pandas/core/reshape/reshape.py \
-        pandas/core/reshape/tile.py \
-        pandas/core/reshape/melt.py \
-        -k"-crosstab -pivot_table -cut"
+    MSG='Doctests reshaping functions' ; echo $MSG
+    pytest -q --doctest-modules pandas/core/reshape/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Doctests interval classes' ; echo $MSG
@@ -333,6 +327,14 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
     MSG='Doctests generic.py' ; echo $MSG
     pytest -q --doctest-modules pandas/core/generic.py
     RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Doctests tseries' ; echo $MSG
+    pytest -q --doctest-modules pandas/tseries/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Doctests computation' ; echo $MSG
+    pytest -q --doctest-modules pandas/core/computation/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi
 
 ### DOCSTRINGS ###

doc/source/development/contributing_docstring.rst

@@ -160,7 +160,7 @@ backticks. The following are considered inline code:
 
 .. _docstring.short_summary:
 
-Section 1: Short summary
+Section 1: short summary
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 The short summary is a single sentence that expresses what the function does in
@@ -228,7 +228,7 @@ infinitive verb.
 
 .. _docstring.extended_summary:
 
-Section 2: Extended summary
+Section 2: extended summary
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The extended summary provides details on what the function does. It should not
@@ -259,7 +259,7 @@ their use cases, if it is not too generic.
 
 .. _docstring.parameters:
 
-Section 3: Parameters
+Section 3: parameters
 ~~~~~~~~~~~~~~~~~~~~~
 
 The details of the parameters will be added in this section. This section has
@@ -424,7 +424,7 @@ For axis, the convention is to use something like:
 
 .. _docstring.returns:
 
-Section 4: Returns or Yields
+Section 4: returns or yields
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If the method returns a value, it will be documented in this section. Also
@@ -505,7 +505,7 @@ If the method yields its value:
 
 .. _docstring.see_also:
 
-Section 5: See Also
+Section 5: see also
 ~~~~~~~~~~~~~~~~~~~
 
 This section is used to let users know about pandas functionality
@@ -583,7 +583,7 @@ For example:
 
 .. _docstring.notes:
 
-Section 6: Notes
+Section 6: notes
 ~~~~~~~~~~~~~~~~
 
 This is an optional section used for notes about the implementation of the
@@ -597,7 +597,7 @@ This section follows the same format as the extended summary section.
 
 .. _docstring.examples:
 
-Section 7: Examples
+Section 7: examples
 ~~~~~~~~~~~~~~~~~~~
 
 This is one of the most important sections of a docstring, despite being
@@ -998,4 +998,4 @@ mapping function names to docstrings. Wherever possible, we prefer using
 
 See ``pandas.core.generic.NDFrame.fillna`` for an example template, and
 ``pandas.core.series.Series.fillna`` and ``pandas.core.generic.frame.fillna``
-for the filled versions.
+for the filled versions.

doc/source/development/developer.rst

@@ -62,7 +62,7 @@ for each column, *including the index columns*. This has JSON form:
 
 See below for the detailed specification for these.
 
-Index Metadata Descriptors
+Index metadata descriptors
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``RangeIndex`` can be stored as metadata only, not requiring serialization. The
@@ -89,7 +89,7 @@ with other column names) a disambiguating name with pattern matching
 columns, ``name`` attribute is always stored in the column descriptors as
 above.
 
-Column Metadata
+Column metadata
 ~~~~~~~~~~~~~~~
 
 ``pandas_type`` is the logical type of the column, and is one of:
@@ -182,4 +182,4 @@ As an example of fully-formed metadata:
     'creator': {
       'library': 'pyarrow',
       'version': '0.13.0'
-    }}
+    }}

doc/source/development/extending.rst

@@ -210,7 +210,7 @@ will
 
 .. _extending.extension.ufunc:
 
-NumPy Universal Functions
+NumPy universal functions
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 :class:`Series` implements ``__array_ufunc__``. As part of the implementation,
@@ -501,4 +501,4 @@ registers the default "matplotlib" backend as follows.
 
 
 More information on how to implement a third-party plotting backend can be found at
-https://github.com/pandas-dev/pandas/blob/master/pandas/plotting/__init__.py#L1.
+https://github.com/pandas-dev/pandas/blob/master/pandas/plotting/__init__.py#L1.

doc/source/development/maintaining.rst

@@ -1,7 +1,7 @@
 .. _maintaining:
 
 ******************
-Pandas Maintenance
+pandas maintenance
 ******************
 
 This guide is for pandas' maintainers. It may also be interesting to contributors
@@ -41,7 +41,7 @@ reading.
 
 .. _maintaining.triage:
 
-Issue Triage
+Issue triage
 ------------
 
 
@@ -123,7 +123,7 @@ Here's a typical workflow for triaging a newly opened issue.
 
 .. _maintaining.closing:
 
-Closing Issues
+Closing issues
 --------------
 
 Be delicate here: many people interpret closing an issue as us saying that the
@@ -132,7 +132,7 @@ respond or self-close their issue if it's determined that the behavior is not a
 or the feature is out of scope. Sometimes reporters just go away though, and
 we'll close the issue after the conversation has died.
 
-Reviewing Pull Requests
+Reviewing pull requests
 -----------------------
 
 Anybody can review a pull request: regular contributors, triagers, or core-team
@@ -144,7 +144,7 @@ members. Here are some guidelines to check.
 * User-facing changes should have a whatsnew in the appropriate file.
 * Regression tests should reference the original GitHub issue number like ``# GH-1234``.
 
-Cleaning up old Issues
+Cleaning up old issues
 ----------------------
 
 Every open issue in pandas has a cost. Open issues make finding duplicates harder,
@@ -164,7 +164,7 @@ If an older issue lacks a reproducible example, label it as "Needs Info" and
 ask them to provide one (or write one yourself if possible). If one isn't
 provide reasonably soon, close it according to the policies in :ref:`maintaining.closing`.
 
-Cleaning up old Pull Requests
+Cleaning up old pull requests
 -----------------------------
 
 Occasionally, contributors are unable to finish off a pull request.

doc/source/development/meeting.rst

@@ -1,7 +1,7 @@
 .. _meeting:
 
 ==================
-Developer Meetings
+Developer meetings
 ==================
 
 We hold regular developer meetings on the second Wednesday
@@ -29,4 +29,3 @@ You can subscribe to this calendar with the following links:
 
 Additionally, we'll sometimes have one-off meetings on specific topics.
 These will be published on the same calendar.
-

doc/source/getting_started/intro_tutorials/01_table_oriented.rst

@@ -26,7 +26,7 @@ documentation.
         </li>
     </ul>
 
-Pandas data table representation
+pandas data table representation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. image:: ../../_static/schemas/01_table_dataframe.svg