Merge remote-tracking branch 'upstream/master' into sa-errors

Author: dilex42

asv_bench/benchmarks/frame_ctor.py

@@ -1,5 +1,6 @@
 import numpy as np
 
+import pandas as pd
 from pandas import DataFrame, MultiIndex, Series, Timestamp, date_range
 
 from .pandas_vb_common import tm
@@ -118,4 +119,48 @@ def time_frame_from_range(self):
         self.df = DataFrame(self.data)
 
 
+class FromArrays:
+
+    goal_time = 0.2
+
+    def setup(self):
+        N_rows = 1000
+        N_cols = 1000
+        self.float_arrays = [np.random.randn(N_rows) for _ in range(N_cols)]
+        self.sparse_arrays = [
+            pd.arrays.SparseArray(np.random.randint(0, 2, N_rows), dtype="float64")
+            for _ in range(N_cols)
+        ]
+        self.int_arrays = [
+            pd.array(np.random.randint(1000, size=N_rows), dtype="Int64")
+            for _ in range(N_cols)
+        ]
+        self.index = pd.Index(range(N_rows))
+        self.columns = pd.Index(range(N_cols))
+
+    def time_frame_from_arrays_float(self):
+        self.df = DataFrame._from_arrays(
+            self.float_arrays,
+            index=self.index,
+            columns=self.columns,
+            verify_integrity=False,
+        )
+
+    def time_frame_from_arrays_int(self):
+        self.df = DataFrame._from_arrays(
+            self.int_arrays,
+            index=self.index,
+            columns=self.columns,
+            verify_integrity=False,
+        )
+
+    def time_frame_from_arrays_sparse(self):
+        self.df = DataFrame._from_arrays(
+            self.sparse_arrays,
+            index=self.index,
+            columns=self.columns,
+            verify_integrity=False,
+        )
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/rolling.py

@@ -11,7 +11,7 @@ class Methods:
         ["int", "float"],
         ["median", "mean", "max", "min", "std", "count", "skew", "kurt", "sum"],
     )
-    param_names = ["contructor", "window", "dtype", "method"]
+    param_names = ["constructor", "window", "dtype", "method"]
 
     def setup(self, constructor, window, dtype, method):
         N = 10 ** 5
@@ -72,7 +72,7 @@ class ExpandingMethods:
         ["int", "float"],
         ["median", "mean", "max", "min", "std", "count", "skew", "kurt", "sum"],
     )
-    param_names = ["contructor", "window", "dtype", "method"]
+    param_names = ["constructor", "window", "dtype", "method"]
 
     def setup(self, constructor, dtype, method):
         N = 10 ** 5
@@ -86,7 +86,7 @@ def time_expanding(self, constructor, dtype, method):
 class EWMMethods:
 
     params = (["DataFrame", "Series"], [10, 1000], ["int", "float"], ["mean", "std"])
-    param_names = ["contructor", "window", "dtype", "method"]
+    param_names = ["constructor", "window", "dtype", "method"]
 
     def setup(self, constructor, window, dtype, method):
         N = 10 ** 5
@@ -104,7 +104,7 @@ class VariableWindowMethods(Methods):
         ["int", "float"],
         ["median", "mean", "max", "min", "std", "count", "skew", "kurt", "sum"],
     )
-    param_names = ["contructor", "window", "dtype", "method"]
+    param_names = ["constructor", "window", "dtype", "method"]
 
     def setup(self, constructor, window, dtype, method):
         N = 10 ** 5

asv_bench/benchmarks/sparse.py

@@ -45,7 +45,6 @@ def time_sparse_array(self, dense_proportion, fill_value, dtype):
 class SparseDataFrameConstructor:
     def setup(self):
         N = 1000
-        self.arr = np.arange(N)
         self.sparse = scipy.sparse.rand(N, N, 0.005)
 
     def time_from_scipy(self):

ci/azure/posix.yml

@@ -24,7 +24,7 @@ jobs:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
           CONDA_PY: "36"
           PATTERN: "slow"
-          # pandas does not use the language (zh_CN), but should support diferent encodings (utf8)
+          # pandas does not use the language (zh_CN), but should support different encodings (utf8)
           # we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any
           LANG: "zh_CN.utf8"
           LC_ALL: "zh_CN.utf8"

ci/code_checks.sh

@@ -102,9 +102,17 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
 
     MSG='Check for use of not concatenated strings' ; echo $MSG
     if [[ "$GITHUB_ACTIONS" == "true" ]]; then
-        $BASE_DIR/scripts/validate_string_concatenation.py --format="[error]{source_path}:{line_number}:{msg}" .
+        $BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate" --format="##[error]{source_path}:{line_number}:{msg}" .
     else
-        $BASE_DIR/scripts/validate_string_concatenation.py .
+        $BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate" .
+    fi
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for strings with wrong placed spaces' ; echo $MSG
+    if [[ "$GITHUB_ACTIONS" == "true" ]]; then
+        $BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace" --format="##[error]{source_path}:{line_number}:{msg}" .
+    else
+        $BASE_DIR/scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace" .
     fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 

doc/source/conf.py

@@ -195,7 +195,7 @@
 
 # The theme to use for HTML and HTML Help pages.  Major themes that come with
 # Sphinx are currently 'default' and 'sphinxdoc'.
-html_theme = "pandas_sphinx_theme"
+html_theme = "pydata_sphinx_theme"
 
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths

doc/source/development/internals.rst

@@ -89,16 +89,10 @@ pandas extends NumPy's type system with custom types, like ``Categorical`` or
 datetimes with a timezone, so we have multiple notions of "values". For 1-D
 containers (``Index`` classes and ``Series``) we have the following convention:
 
-* ``cls._ndarray_values`` is *always* a NumPy ``ndarray``. Ideally,
-  ``_ndarray_values`` is cheap to compute. For example, for a ``Categorical``,
-  this returns the codes, not the array of objects.
 * ``cls._values`` refers is the "best possible" array. This could be an
-  ``ndarray``, ``ExtensionArray``, or in ``Index`` subclass (note: we're in the
-  process of removing the index subclasses here so that it's always an
-  ``ndarray`` or ``ExtensionArray``).
+  ``ndarray`` or ``ExtensionArray``.
 
-So, for example, ``Series[category]._values`` is a ``Categorical``, while
-``Series[category]._ndarray_values`` is the underlying codes.
+So, for example, ``Series[category]._values`` is a ``Categorical``.
 
 .. _ref-subclassing-pandas:
 

doc/source/getting_started/intro_tutorials/02_read_write.rst

@@ -225,7 +225,7 @@ The method :meth:`~DataFrame.info` provides technical information about a
     <div class="d-flex flex-row bg-light gs-torefguide">
         <span class="badge badge-info">To user guide</span>
 
-For a complete overview of the input and output possibilites from and to pandas, see the user guide section about :ref:`reader and writer functions <io>`.
+For a complete overview of the input and output possibilities from and to pandas, see the user guide section about :ref:`reader and writer functions <io>`.
 
 .. raw:: html
 

doc/source/getting_started/intro_tutorials/03_subset_data.rst

@@ -88,7 +88,7 @@ name of the column of interest.
     </ul>
 
 Each column in a :class:`DataFrame` is a :class:`Series`. As a single column is
-selected, the returned object is a pandas :class:`DataFrame`. We can verify this
+selected, the returned object is a pandas :class:`Series`. We can verify this
 by checking the type of the output:
 
 .. ipython:: python
@@ -101,7 +101,7 @@ And have a look at the ``shape`` of the output:
 
     titanic["Age"].shape
 
-:attr:`DataFrame.shape` is an attribute (remember :ref:`tutorial on reading and writing <10min_tut_02_read_write>`, do not use parantheses for attributes) of a
+:attr:`DataFrame.shape` is an attribute (remember :ref:`tutorial on reading and writing <10min_tut_02_read_write>`, do not use parentheses for attributes) of a
 pandas ``Series`` and ``DataFrame`` containing the number of rows and
 columns: *(nrows, ncolumns)*. A pandas Series is 1-dimensional and only
 the number of rows is returned.

doc/source/getting_started/intro_tutorials/08_combine_dataframes.rst

@@ -165,7 +165,7 @@ index. For example:
 .. note::
     The existence of multiple row/column indices at the same time
     has not been mentioned within these tutorials. *Hierarchical indexing*
-    or *MultiIndex* is an advanced and powerfull pandas feature to analyze
+    or *MultiIndex* is an advanced and powerful pandas feature to analyze
     higher dimensional data.
 
     Multi-indexing is out of scope for this pandas introduction. For the

doc/source/getting_started/intro_tutorials/10_text_data.rst

@@ -188,7 +188,7 @@ Which passenger of the titanic has the longest name?
 
     titanic["Name"].str.len()
 
-To get the longest name we first have to get the lenghts of each of the
+To get the longest name we first have to get the lengths of each of the
 names in the ``Name`` column. By using pandas string methods, the
 :meth:`Series.str.len` function is applied to each of the names individually
 (element-wise).

doc/source/index.rst.template

@@ -119,7 +119,6 @@ programming language.
     :titlesonly:
 {% endif %}
 {% if not single_doc %}
-    What's New in 1.1.0 <whatsnew/v1.1.0>
     getting_started/index
     user_guide/index
     {% endif -%}

doc/source/reference/extensions.rst

@@ -37,7 +37,6 @@ objects.
       api.extensions.ExtensionArray._from_factorized
       api.extensions.ExtensionArray._from_sequence
       api.extensions.ExtensionArray._from_sequence_of_strings
-      api.extensions.ExtensionArray._ndarray_values
       api.extensions.ExtensionArray._reduce
       api.extensions.ExtensionArray._values_for_argsort
       api.extensions.ExtensionArray._values_for_factorize

doc/source/user_guide/dsintro.rst

@@ -406,7 +406,7 @@ From a list of dataclasses
 
 Data Classes as introduced in `PEP557 <https://www.python.org/dev/peps/pep-0557>`__,
 can be passed into the DataFrame constructor.
-Passing a list of dataclasses is equivilent to passing a list of dictionaries.
+Passing a list of dataclasses is equivalent to passing a list of dictionaries.
 
 Please be aware, that that all values in the list should be dataclasses, mixing
 types in the list would result in a TypeError.

doc/source/user_guide/indexing.rst

@@ -374,7 +374,7 @@ For getting values with a boolean array:
    df1.loc['a'] > 0
    df1.loc[:, df1.loc['a'] > 0]
 
-NA values in a boolean array propogate as ``False``:
+NA values in a boolean array propagate as ``False``:
 
 .. versionchanged:: 1.0.2
 

doc/source/user_guide/io.rst

@@ -5005,7 +5005,7 @@ Possible values are:
   This usually provides better performance for analytic databases
   like *Presto* and *Redshift*, but has worse performance for
   traditional SQL backend if the table contains many columns.
-  For more information check the SQLAlchemy `documention
+  For more information check the SQLAlchemy `documentation
   <https://docs.sqlalchemy.org/en/latest/core/dml.html#sqlalchemy.sql.expression.Insert.values.params.*args>`__.
 - callable with signature ``(pd_table, conn, keys, data_iter)``:
   This can be used to implement a more performant insertion method based on

doc/source/user_guide/scale.rst

@@ -246,6 +246,7 @@ We'll import ``dask.dataframe`` and notice that the API feels similar to pandas.
 We can use Dask's ``read_parquet`` function, but provide a globstring of files to read in.
 
 .. ipython:: python
+   :okwarning:
 
    import dask.dataframe as dd
 
@@ -258,7 +259,7 @@ Inspecting the ``ddf`` object, we see a few things
 * There are familiar methods like ``.groupby``, ``.sum``, etc.
 * There are new attributes like ``.npartitions`` and ``.divisions``
 
-The partitions and divisions are how Dask parallizes computation. A **Dask**
+The partitions and divisions are how Dask parallelizes computation. A **Dask**
 DataFrame is made up of many **Pandas** DataFrames. A single method call on a
 Dask DataFrame ends up making many pandas method calls, and Dask knows how to
 coordinate everything to get the result.

doc/source/user_guide/style.ipynb

@@ -620,8 +620,8 @@
     "aligns = ['left','zero','mid']\n",
     "for align in aligns:\n",
     "    row = \"<tr><th>{}</th>\".format(align)\n",
-    "    for serie in [test1,test2,test3]:\n",
-    "        s = serie.copy()\n",
+    "    for series in [test1,test2,test3]:\n",
+    "        s = series.copy()\n",
     "        s.name=''\n",
     "        row += \"<td>{}</td>\".format(s.to_frame().style.bar(align=align, \n",
     "                                                           color=['#d65f5f', '#5fba7d'], \n",

doc/source/whatsnew/v0.24.0.rst

@@ -397,7 +397,7 @@ Other enhancements
 - :func:`~DataFrame.to_csv`, :func:`~Series.to_csv`, :func:`~DataFrame.to_json`, and :func:`~Series.to_json` now support ``compression='infer'`` to infer compression based on filename extension (:issue:`15008`).
   The default compression for ``to_csv``, ``to_json``, and ``to_pickle`` methods has been updated to ``'infer'`` (:issue:`22004`).
 - :meth:`DataFrame.to_sql` now supports writing ``TIMESTAMP WITH TIME ZONE`` types for supported databases. For databases that don't support timezones, datetime data will be stored as timezone unaware local timestamps. See the :ref:`io.sql_datetime_data` for implications (:issue:`9086`).
-- :func:`to_timedelta` now supports iso-formated timedelta strings (:issue:`21877`)
+- :func:`to_timedelta` now supports iso-formatted timedelta strings (:issue:`21877`)
 - :class:`Series` and :class:`DataFrame` now support :class:`Iterable` objects in the constructor (:issue:`2193`)
 - :class:`DatetimeIndex` has gained the :attr:`DatetimeIndex.timetz` attribute. This returns the local time with timezone information. (:issue:`21358`)
 - :meth:`~Timestamp.round`, :meth:`~Timestamp.ceil`, and :meth:`~Timestamp.floor` for :class:`DatetimeIndex` and :class:`Timestamp`

doc/source/whatsnew/v1.0.1.rst

@@ -16,7 +16,7 @@ Fixed regressions
 ~~~~~~~~~~~~~~~~~
 
 - Fixed regression in :class:`DataFrame` setting values with a slice (e.g. ``df[-4:] = 1``) indexing by label instead of position (:issue:`31469`)
-- Fixed regression when indexing a ``Series`` or ``DataFrame`` indexed by ``DatetimeIndex`` with a slice containg a :class:`datetime.date` (:issue:`31501`)
+- Fixed regression when indexing a ``Series`` or ``DataFrame`` indexed by ``DatetimeIndex`` with a slice containing a :class:`datetime.date` (:issue:`31501`)
 - Fixed regression in ``DataFrame.__setitem__`` raising an ``AttributeError`` with a :class:`MultiIndex` and a non-monotonic indexer (:issue:`31449`)
 - Fixed regression in :class:`Series` multiplication when multiplying a numeric :class:`Series` with >10000 elements with a timedelta-like scalar (:issue:`31457`)
 - Fixed regression in ``.groupby().agg()`` raising an ``AssertionError`` for some reductions like ``min`` on object-dtype columns (:issue:`31522`)

doc/source/whatsnew/v1.1.0.rst

@@ -224,6 +224,10 @@ Performance improvements
 - The internal index method :meth:`~Index._shallow_copy` now copies cached attributes over to the new index,
   avoiding creating these again on the new index. This can speed up many operations that depend on creating copies of
   existing indexes (:issue:`28584`, :issue:`32640`, :issue:`32669`)
+- Significant performance improvement when creating a :class:`DataFrame` with
+  sparse values from ``scipy.sparse`` matrices using the
+  :meth:`DataFrame.sparse.from_spmatrix` constructor (:issue:`32821`,
+  :issue:`32825`,  :issue:`32826`, :issue:`32856`, :issue:`32858`).
 
 .. ---------------------------------------------------------------------------
 
@@ -303,6 +307,7 @@ Indexing
 - Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` when indexing with an integer key on a object-dtype :class:`Index` that is not all-integers (:issue:`31905`)
 - Bug in :meth:`DataFrame.iloc.__setitem__` on a :class:`DataFrame` with duplicate columns incorrectly setting values for all matching columns (:issue:`15686`, :issue:`22036`)
 - Bug in :meth:`DataFrame.loc:` and :meth:`Series.loc` with a :class:`DatetimeIndex`, :class:`TimedeltaIndex`, or :class:`PeriodIndex` incorrectly allowing lookups of non-matching datetime-like dtypes (:issue:`32650`)
+- Bug in :meth:`Series.__getitem__` indexing with non-standard scalars, e.g. ``np.dtype`` (:issue:`32684`)
 
 Missing
 ^^^^^^^
@@ -334,18 +339,21 @@ MultiIndex
 
 I/O
 ^^^
-- Bug in :meth:`read_json` where integer overflow was occuring when json contains big number strings. (:issue:`30320`)
+- Bug in :meth:`read_json` where integer overflow was occurring when json contains big number strings. (:issue:`30320`)
 - `read_csv` will now raise a ``ValueError`` when the arguments `header` and `prefix` both are not `None`. (:issue:`27394`)
 - Bug in :meth:`DataFrame.to_json` was raising ``NotFoundError`` when ``path_or_buf`` was an S3 URI (:issue:`28375`)
 - Bug in :meth:`DataFrame.to_parquet` overwriting pyarrow's default for
   ``coerce_timestamps``; following pyarrow's default allows writing nanosecond
   timestamps with ``version="2.0"`` (:issue:`31652`).
 - Bug in :meth:`read_csv` was raising `TypeError` when `sep=None` was used in combination with `comment` keyword (:issue:`31396`)
 - Bug in :class:`HDFStore` that caused it to set to ``int64`` the dtype of a ``datetime64`` column when reading a DataFrame in Python 3 from fixed format written in Python 2 (:issue:`31750`)
+- Bug in :meth:`DataFrame.to_json` where ``Timedelta`` objects would not be serialized correctly with ``date_format="iso"`` (:issue:`28256`)
 - :func:`read_csv` will raise a ``ValueError`` when the column names passed in `parse_dates` are missing in the Dataframe (:issue:`31251`)
 - Bug in :meth:`read_excel` where a UTF-8 string with a high surrogate would cause a segmentation violation (:issue:`23809`)
 - Bug in :meth:`read_csv` was causing a file descriptor leak on an empty file (:issue:`31488`)
 - Bug in :meth:`read_csv` was causing a segfault when there were blank lines between the header and data rows (:issue:`28071`)
+- Bug in :meth:`read_csv` was raising a misleading exception on a permissions issue (:issue:`23784`)
+- Bug in :meth:`read_csv` was raising an ``IndexError`` when header=None and 2 extra data columns
 
 
 Plotting
@@ -377,7 +385,7 @@ Reshaping
 - :meth:`DataFrame.replace` and :meth:`Series.replace` will raise a ``TypeError`` if ``to_replace`` is not an expected type. Previously the ``replace`` would fail silently (:issue:`18634`)
 - Bug in :meth:`DataFrame.apply` where callback was called with :class:`Series` parameter even though ``raw=True`` requested. (:issue:`32423`)
 - Bug in :meth:`DataFrame.pivot_table` losing timezone information when creating a :class:`MultiIndex` level from a column with timezone-aware dtype (:issue:`32558`)
-
+- :meth:`DataFrame.agg` now provides more descriptive ``SpecificationError`` message when attempting to aggregating non-existant column (:issue:`32755`)
 
 Sparse
 ^^^^^^
@@ -402,6 +410,8 @@ Other
 - Fixed :func:`pandas.testing.assert_series_equal` to correctly raise if left object is a different subclass with ``check_series_type=True`` (:issue:`32670`).
 - :meth:`IntegerArray.astype` now supports ``datetime64`` dtype (:issue:32538`)
 - Fixed bug in :func:`pandas.testing.assert_series_equal` where dtypes were checked for ``Interval`` and ``ExtensionArray`` operands when ``check_dtype`` was ``False`` (:issue:`32747`)
+- Bug in :meth:`Series.map` not raising on invalid ``na_action`` (:issue:`32815`)
+- Bug in :meth:`DataFrame.__dir__` caused a segfault when using unicode surrogates in a column name (:issue:`25509`)
 
 .. ---------------------------------------------------------------------------
 

doc/sphinxext/announce.py

@@ -68,8 +68,21 @@ def get_authors(revision_range):
         revision_range = f"{lst_release}..{cur_release}"
 
     # authors, in current release and previous to current release.
-    cur = set(re.findall(pat, this_repo.git.shortlog("-s", revision_range), re.M))
-    pre = set(re.findall(pat, this_repo.git.shortlog("-s", lst_release), re.M))
+    # We need two passes over the log for cur and prev, one to get the
+    # "Co-authored by" commits, which come from backports by the bot,
+    # and one for regular commits.
+    xpr = re.compile(r"Co-authored-by: (?P<name>[^<]+) ")
+    cur = set(
+        xpr.findall(
+            this_repo.git.log("--grep=Co-authored", "--pretty=%b", revision_range)
+        )
+    )
+    cur |= set(re.findall(pat, this_repo.git.shortlog("-s", revision_range), re.M))
+
+    pre = set(
+        xpr.findall(this_repo.git.log("--grep=Co-authored", "--pretty=%b", lst_release))
+    )
+    pre |= set(re.findall(pat, this_repo.git.shortlog("-s", lst_release), re.M))
 
     # Homu is the author of auto merges, clean him out.
     cur.discard("Homu")