Merge remote-tracking branch 'upstream/master' into GH36666

Author: avinashpancham

.github/workflows/ci.yml

@@ -37,12 +37,6 @@ jobs:
         ci/code_checks.sh lint
       if: always()
 
-    - name: Dependencies consistency
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh dependencies
-      if: always()
-
     - name: Checks on imported code
       run: |
         source activate pandas-dev

ci/code_checks.sh

@@ -15,11 +15,10 @@
 #   $ ./ci/code_checks.sh code          # checks on imported code
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
-#   $ ./ci/code_checks.sh dependencies  # check that dependencies are consistent
 #   $ ./ci/code_checks.sh typing	# run static type analysis
 
-[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependencies" || "$1" == "typing" ]] || \
-    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|dependencies|typing]"; exit 9999; }
+[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "typing" ]] || \
+    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|typing]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
 RET=0
@@ -38,6 +37,12 @@ function invgrep {
     return $((! $EXIT_STATUS))
 }
 
+function check_namespace {
+    local -r CLASS="${1}"
+    grep -R -l --include "*.py" " ${CLASS}(" pandas/tests | xargs grep -n "pd\.${CLASS}("
+    test $? -gt 0
+}
+
 if [[ "$GITHUB_ACTIONS" == "true" ]]; then
     FLAKE8_FORMAT="##[error]%(path)s:%(row)s:%(col)s:%(code)s:%(text)s"
     INVGREP_PREPEND="##[error]"
@@ -48,31 +53,6 @@ fi
 ### LINTING ###
 if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
 
-    echo "black --version"
-    black --version
-
-    MSG='Checking black formatting' ; echo $MSG
-    black . --check
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    # `setup.cfg` contains the list of error codes that are being ignored in flake8
-
-    echo "flake8 --version"
-    flake8 --version
-
-    # pandas/_libs/src is C code, so no need to search there.
-    MSG='Linting .py code' ; echo $MSG
-    flake8 --format="$FLAKE8_FORMAT" .
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Linting .pyx and .pxd code' ; echo $MSG
-    flake8 --format="$FLAKE8_FORMAT" pandas --append-config=flake8/cython.cfg
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Linting .pxi.in' ; echo $MSG
-    flake8 --format="$FLAKE8_FORMAT" pandas/_libs --append-config=flake8/cython-template.cfg
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
     # Check that cython casting is of the form `<type>obj` as opposed to `<type> obj`;
     # it doesn't make a difference, but we want to be internally consistent.
     # Note: this grep pattern is (intended to be) equivalent to the python
@@ -125,19 +105,6 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    echo "isort --version-number"
-    isort --version-number
-
-    # Imports - Check formatting using isort see setup.cfg for settings
-    MSG='Check import format using isort' ; echo $MSG
-    ISORT_CMD="isort --quiet --check-only pandas asv_bench scripts web"
-    if [[ "$GITHUB_ACTIONS" == "true" ]]; then
-        eval $ISORT_CMD | awk '{print "##[error]" $0}'; RET=$(($RET + ${PIPESTATUS[0]}))
-    else
-        eval $ISORT_CMD
-    fi
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
 fi
 
 ### PATTERNS ###
@@ -234,6 +201,13 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     MSG='Check code for instances of os.remove' ; echo $MSG
     invgrep -R --include="*.py*" --exclude "common.py" --exclude "test_writers.py" --exclude "test_store.py" -E "os\.remove" pandas/tests/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for inconsistent use of pandas namespace in tests' ; echo $MSG
+    check_namespace "Series"
+    RET=$(($RET + $?))
+    check_namespace "DataFrame"
+    RET=$(($RET + $?))
+    echo $MSG "DONE"
 fi
 
 ### CODE ###
@@ -354,15 +328,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
 fi
 
-### DEPENDENCIES ###
-if [[ -z "$CHECK" || "$CHECK" == "dependencies" ]]; then
-
-    MSG='Check that requirements-dev.txt has been generated from environment.yml' ; echo $MSG
-    $BASE_DIR/scripts/generate_pip_deps_from_conda.py --compare --azure
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-fi
-
 ### TYPING ###
 if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then
 
@@ -374,5 +339,4 @@ if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi
 
-
 exit $RET

doc/source/getting_started/index.rst

@@ -533,7 +533,7 @@ pandas has great support for time series and has an extensive set of tools for w
             <div id="collapseTen" class="collapse" data-parent="#accordion">
                 <div class="card-body">
 
-Data sets do not only contain numerical data. pandas provides a wide range of functions to cleaning textual data and extract useful information from it.
+Data sets do not only contain numerical data. pandas provides a wide range of functions to clean textual data and extract useful information from it.
 
 .. raw:: html
 

doc/source/getting_started/intro_tutorials/10_text_data.rst

@@ -66,15 +66,15 @@ How to manipulate textual data?
     <ul class="task-bullet">
         <li>
 
-Make all name characters lowercase
+Make all name characters lowercase.
 
 .. ipython:: python
 
     titanic["Name"].str.lower()
 
 To make each of the strings in the ``Name`` column lowercase, select the ``Name`` column
-(see :ref:`tutorial on selection of data <10min_tut_03_subset>`), add the ``str`` accessor and
-apply the ``lower`` method. As such, each of the strings is converted element wise.
+(see the :ref:`tutorial on selection of data <10min_tut_03_subset>`), add the ``str`` accessor and
+apply the ``lower`` method. As such, each of the strings is converted element-wise.
 
 .. raw:: html
 
@@ -86,15 +86,15 @@ having a ``dt`` accessor, a number of
 specialized string methods are available when using the ``str``
 accessor. These methods have in general matching names with the
 equivalent built-in string methods for single elements, but are applied
-element-wise (remember :ref:`element wise calculations <10min_tut_05_columns>`?)
+element-wise (remember :ref:`element-wise calculations <10min_tut_05_columns>`?)
 on each of the values of the columns.
 
 .. raw:: html
 
     <ul class="task-bullet">
         <li>
 
-Create a new column ``Surname`` that contains the surname of the Passengers by extracting the part before the comma.
+Create a new column ``Surname`` that contains the surname of the passengers by extracting the part before the comma.
 
 .. ipython:: python
 
@@ -135,7 +135,7 @@ More information on extracting parts of strings is available in the user guide s
     <ul class="task-bullet">
         <li>
 
-Extract the passenger data about the Countesses on board of the Titanic.
+Extract the passenger data about the countesses on board of the Titanic.
 
 .. ipython:: python
 
@@ -145,15 +145,15 @@ Extract the passenger data about the Countesses on board of the Titanic.
 
     titanic[titanic["Name"].str.contains("Countess")]
 
-(*Interested in her story? See *\ `Wikipedia <https://en.wikipedia.org/wiki/No%C3%ABl_Leslie,_Countess_of_Rothes>`__\ *!*)
+(*Interested in her story? See* `Wikipedia <https://en.wikipedia.org/wiki/No%C3%ABl_Leslie,_Countess_of_Rothes>`__\ *!*)
 
 The string method :meth:`Series.str.contains` checks for each of the values in the
 column ``Name`` if the string contains the word ``Countess`` and returns
-for each of the values ``True`` (``Countess`` is part of the name) of
+for each of the values ``True`` (``Countess`` is part of the name) or
 ``False`` (``Countess`` is not part of the name). This output can be used
 to subselect the data using conditional (boolean) indexing introduced in
 the :ref:`subsetting of data tutorial <10min_tut_03_subset>`. As there was
-only one Countess on the Titanic, we get one row as a result.
+only one countess on the Titanic, we get one row as a result.
 
 .. raw:: html
 
@@ -220,7 +220,7 @@ we can do a selection using the ``loc`` operator, introduced in the
     <ul class="task-bullet">
         <li>
 
-In the "Sex" column, replace values of "male" by "M" and values of "female" by "F"
+In the "Sex" column, replace values of "male" by "M" and values of "female" by "F".
 
 .. ipython:: python
 
@@ -256,7 +256,7 @@ a ``dictionary`` to define the mapping ``{from : to}``.
         <h4>REMEMBER</h4>
 
 -  String methods are available using the ``str`` accessor.
--  String methods work element wise and can be used for conditional
+-  String methods work element-wise and can be used for conditional
    indexing.
 -  The ``replace`` method is a convenient method to convert values
    according to a given dictionary.

doc/source/index.rst.template

@@ -17,7 +17,7 @@ pandas documentation
 `Source Repository <https://github.com/pandas-dev/pandas>`__ |
 `Issues & Ideas <https://github.com/pandas-dev/pandas/issues>`__ |
 `Q&A Support <https://stackoverflow.com/questions/tagged/pandas>`__ |
-`Mailing List <https://groups.google.com/forum/#!forum/pydata>`__
+`Mailing List <https://groups.google.com/g/pydata>`__
 
 :mod:`pandas` is an open source, BSD-licensed library providing high-performance,
 easy-to-use data structures and data analysis tools for the `Python <https://www.python.org/>`__

doc/source/user_guide/computation.rst

@@ -652,9 +652,9 @@ parameter:
     :header: "``closed``", "Description", "Default for"
     :widths: 20, 30, 30
 
-    ``right``, close right endpoint, time-based windows
+    ``right``, close right endpoint,
     ``left``, close left endpoint,
-    ``both``, close both endpoints, fixed windows
+    ``both``, close both endpoints,
     ``neither``, open endpoints,
 
 For example, having the right endpoint open is useful in many problems that require that there is no contamination
@@ -681,9 +681,6 @@ from present information back to past information. This allows the rolling windo
 
    df
 
-Currently, this feature is only implemented for time-based windows.
-For fixed windows, the closed parameter cannot be set and the rolling window will always have both endpoints closed.
-
 .. _stats.iter_rolling_window:
 
 Iteration over window:

doc/source/whatsnew/v1.1.4.rst

@@ -22,6 +22,7 @@ Fixed regressions
 - Fixed regression in :class:`RollingGroupby` causing a segmentation fault with Index of dtype object (:issue:`36727`)
 - Fixed regression in :meth:`DataFrame.resample(...).apply(...)` raised ``AttributeError`` when input was a :class:`DataFrame` and only a :class:`Series` was evaluated (:issue:`36951`)
 - Fixed regression in :class:`PeriodDtype` comparing both equal and unequal to its string representation (:issue:`37265`)
+- Fixed regression in certain offsets (:meth:`pd.offsets.Day() <pandas.tseries.offsets.Day>` and below) no longer being hashable (:issue:`37267`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.2.0.rst

@@ -221,6 +221,8 @@ Other enhancements
 - :meth:`Rolling.var()` and :meth:`Rolling.std()` use Kahan summation and Welfords Method to avoid numerical issues (:issue:`37051`)
 - :meth:`DataFrame.plot` now recognizes ``xlabel`` and ``ylabel`` arguments for plots of type ``scatter`` and ``hexbin`` (:issue:`37001`)
 - :class:`DataFrame` now supports ``divmod`` operation (:issue:`37165`)
+- :meth:`DataFrame.to_parquet` now returns a ``bytes`` object when no ``path`` argument is passed (:issue:`37105`)
+- :class:`Rolling` now supports the ``closed`` argument for fixed windows (:issue:`34315`)
 
 .. _whatsnew_120.api_breaking.python:
 
@@ -410,7 +412,7 @@ Conversion
 Strings
 ^^^^^^^
 - Bug in :meth:`Series.to_string`, :meth:`DataFrame.to_string`, and :meth:`DataFrame.to_latex` adding a leading space when ``index=False`` (:issue:`24980`)
--
+- Bug in :func:`to_numeric` raising a ``TypeError`` when attempting to convert a string dtype :class:`Series` containing only numeric strings and ``NA`` (:issue:`37262`)
 -
 
 
@@ -517,7 +519,7 @@ ExtensionArray
 - Fixed Bug where :class:`DataFrame` column set to scalar extension type via a dict instantion was considered an object type rather than the extension type (:issue:`35965`)
 - Fixed bug where ``astype()`` with equal dtype and ``copy=False`` would return a new object (:issue:`284881`)
 - Fixed bug when applying a NumPy ufunc with multiple outputs to a :class:`pandas.arrays.IntegerArray` returning None (:issue:`36913`)
-
+- Fixed an inconsistency in :class:`PeriodArray`'s ``__init__`` signature to those of :class:`DatetimeArray` and :class:`TimedeltaArray` (:issue:`37289`)
 
 Other
 ^^^^^

pandas/_libs/lib.pyx

@@ -2019,7 +2019,7 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
         elif util.is_bool_object(val):
             floats[i] = uints[i] = ints[i] = bools[i] = val
             seen.bool_ = True
-        elif val is None:
+        elif val is None or val is C_NA:
             seen.saw_null()
             floats[i] = complexes[i] = NaN
         elif hasattr(val, '__len__') and len(val) == 0:

pandas/_libs/tslibs/offsets.pyx

@@ -791,6 +791,11 @@ cdef class Tick(SingleConstructorOffset):
     def is_anchored(self) -> bool:
         return False
 
+    # This is identical to BaseOffset.__hash__, but has to be redefined here
+    # for Python 3, because we've redefined __eq__.
+    def __hash__(self) -> int:
+        return hash(self._params)
+
     # --------------------------------------------------------------------
     # Comparison and Arithmetic Methods
 

pandas/_libs/window/indexers.pyx

@@ -43,16 +43,14 @@ def calculate_variable_window_bounds(
     (ndarray[int64], ndarray[int64])
     """
     cdef:
-        bint left_closed = False
-        bint right_closed = False
-        int index_growth_sign = 1
+        bint left_closed = False, right_closed = False
         ndarray[int64_t, ndim=1] start, end
-        int64_t start_bound, end_bound
+        int64_t start_bound, end_bound, index_growth_sign = 1
         Py_ssize_t i, j
 
-    # if windows is variable, default is 'right', otherwise default is 'both'
+    # default is 'right'
     if closed is None:
-        closed = 'right' if index is not None else 'both'
+        closed = 'right'
 
     if closed in ['right', 'both']:
         right_closed = True

pandas/_typing.py

@@ -38,6 +38,8 @@
     from pandas.core.indexes.base import Index
     from pandas.core.series import Series
 
+    from pandas.io.formats.format import EngFormatter
+
 # array-like
 
 AnyArrayLike = TypeVar("AnyArrayLike", "ExtensionArray", "Index", "Series", np.ndarray)
@@ -127,6 +129,10 @@
 EncodingVar = TypeVar("EncodingVar", str, None, Optional[str])
 
 
+# type of float formatter in DataFrameFormatter
+FloatFormatType = Union[str, Callable, "EngFormatter"]
+
+
 @dataclass
 class IOargs(Generic[ModeVar, EncodingVar]):
     """

pandas/conftest.py

@@ -34,7 +34,7 @@
 import pandas.util._test_decorators as td
 
 import pandas as pd
-from pandas import DataFrame
+from pandas import DataFrame, Series
 import pandas._testing as tm
 from pandas.core import ops
 from pandas.core.indexes.api import Index, MultiIndex
@@ -529,6 +529,23 @@ def series_with_simple_index(index):
     return _create_series(index)
 
 
+@pytest.fixture
+def series_with_multilevel_index():
+    """
+    Fixture with a Series with a 2-level MultiIndex.
+    """
+    arrays = [
+        ["bar", "bar", "baz", "baz", "qux", "qux", "foo", "foo"],
+        ["one", "two", "one", "two", "one", "two", "one", "two"],
+    ]
+    tuples = zip(*arrays)
+    index = MultiIndex.from_tuples(tuples)
+    data = np.random.randn(8)
+    ser = Series(data, index=index)
+    ser[3] = np.NaN
+    return ser
+
+
 _narrow_dtypes = [
     np.float16,
     np.float32,