Merge branch 'master' into value_counts_part1

Author: DataInformer

asv_bench/benchmarks/tslibs/resolution.py

@@ -0,0 +1,50 @@
+"""
+ipython analogue:
+
+tr = TimeResolution()
+mi = pd.MultiIndex.from_product(tr.params[:-1] + ([str(x) for x in tr.params[-1]],))
+df = pd.DataFrame(np.nan, index=mi, columns=["mean", "stdev"])
+
+for unit in tr.params[0]:
+    for size in tr.params[1]:
+        for tz in tr.params[2]:
+            tr.setup(unit, size, tz)
+            key = (unit, size, str(tz))
+            print(key)
+
+            val = %timeit -o tr.time_get_resolution(unit, size, tz)
+
+            df.loc[key] = (val.average, val.stdev)
+
+"""
+from datetime import timedelta, timezone
+
+from dateutil.tz import gettz, tzlocal
+import numpy as np
+import pytz
+
+from pandas._libs.tslibs.resolution import get_resolution
+
+
+class TimeResolution:
+    params = (
+        ["D", "h", "m", "s", "us", "ns"],
+        [1, 100, 10 ** 4, 10 ** 6],
+        [
+            None,
+            timezone.utc,
+            timezone(timedelta(minutes=60)),
+            pytz.timezone("US/Pacific"),
+            gettz("Asia/Tokyo"),
+            tzlocal(),
+        ],
+    )
+    param_names = ["unit", "size", "tz"]
+
+    def setup(self, unit, size, tz):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        arr = arr.view(f"M8[{unit}]").astype("M8[ns]").view("i8")
+        self.i8data = arr
+
+    def time_get_resolution(self, unit, size, tz):
+        get_resolution(self.i8data, tz)

asv_bench/benchmarks/tslibs/timestamp.py

@@ -1,17 +1,29 @@
-import datetime
+from datetime import datetime, timedelta, timezone
 
-import dateutil
+from dateutil.tz import gettz, tzlocal, tzutc
 import numpy as np
 import pytz
 
 from pandas import Timestamp
 
+# One case for each type of tzinfo object that has its own code path
+#  in tzconversion code.
+_tzs = [
+    None,
+    pytz.timezone("Europe/Amsterdam"),
+    gettz("US/Central"),
+    pytz.UTC,
+    tzutc(),
+    timezone(timedelta(minutes=60)),
+    tzlocal(),
+]
+
 
 class TimestampConstruction:
     def setup(self):
         self.npdatetime64 = np.datetime64("2020-01-01 00:00:00")
-        self.dttime_unaware = datetime.datetime(2020, 1, 1, 0, 0, 0)
-        self.dttime_aware = datetime.datetime(2020, 1, 1, 0, 0, 0, 0, pytz.UTC)
+        self.dttime_unaware = datetime(2020, 1, 1, 0, 0, 0)
+        self.dttime_aware = datetime(2020, 1, 1, 0, 0, 0, 0, pytz.UTC)
         self.ts = Timestamp("2020-01-01 00:00:00")
 
     def time_parse_iso8601_no_tz(self):
@@ -49,7 +61,6 @@ def time_from_pd_timestamp(self):
 
 
 class TimestampProperties:
-    _tzs = [None, pytz.timezone("Europe/Amsterdam"), pytz.UTC, dateutil.tz.tzutc()]
     _freqs = [None, "B"]
     params = [_tzs, _freqs]
     param_names = ["tz", "freq"]
@@ -63,9 +74,6 @@ def time_tz(self, tz, freq):
     def time_dayofweek(self, tz, freq):
         self.ts.dayofweek
 
-    def time_weekday_name(self, tz, freq):
-        self.ts.day_name
-
     def time_dayofyear(self, tz, freq):
         self.ts.dayofyear
 
@@ -108,9 +116,12 @@ def time_microsecond(self, tz, freq):
     def time_month_name(self, tz, freq):
         self.ts.month_name()
 
+    def time_weekday_name(self, tz, freq):
+        self.ts.day_name()
+
 
 class TimestampOps:
-    params = [None, "US/Eastern", pytz.UTC, dateutil.tz.tzutc()]
+    params = _tzs
     param_names = ["tz"]
 
     def setup(self, tz):
@@ -148,7 +159,7 @@ def time_ceil(self, tz):
 
 class TimestampAcrossDst:
     def setup(self):
-        dt = datetime.datetime(2016, 3, 27, 1)
+        dt = datetime(2016, 3, 27, 1)
         self.tzinfo = pytz.timezone("CET").localize(dt, is_dst=False).tzinfo
         self.ts2 = Timestamp(dt)
 

asv_bench/benchmarks/tslibs/tslib.py

@@ -0,0 +1,55 @@
+"""
+ipython analogue:
+
+tr = TimeIntsToPydatetime()
+mi = pd.MultiIndex.from_product(
+    tr.params[:-1] + ([str(x) for x in tr.params[-1]],)
+)
+df = pd.DataFrame(np.nan, index=mi, columns=["mean", "stdev"])
+for box in tr.params[0]:
+    for size in tr.params[1]:
+        for tz in tr.params[2]:
+            tr.setup(box, size, tz)
+            key = (box, size, str(tz))
+            print(key)
+            val = %timeit -o tr.time_ints_to_pydatetime(box, size, tz)
+            df.loc[key] = (val.average, val.stdev)
+"""
+from datetime import timedelta, timezone
+
+from dateutil.tz import gettz, tzlocal
+import numpy as np
+import pytz
+
+from pandas._libs.tslib import ints_to_pydatetime
+
+_tzs = [
+    None,
+    timezone.utc,
+    timezone(timedelta(minutes=60)),
+    pytz.timezone("US/Pacific"),
+    gettz("Asia/Tokyo"),
+    tzlocal(),
+]
+_sizes = [0, 1, 100, 10 ** 4, 10 ** 6]
+
+
+class TimeIntsToPydatetime:
+    params = (
+        ["time", "date", "datetime", "timestamp"],
+        _sizes,
+        _tzs,
+    )
+    param_names = ["box", "size", "tz"]
+    # TODO: fold? freq?
+
+    def setup(self, box, size, tz):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+    def time_ints_to_pydatetime(self, box, size, tz):
+        if box == "date":
+            # ints_to_pydatetime does not allow non-None tz with date;
+            #  this will mean doing some duplicate benchmarks
+            tz = None
+        ints_to_pydatetime(self.i8data, tz, box=box)

doc/source/user_guide/computation.rst

@@ -597,6 +597,18 @@ You can view other examples of ``BaseIndexer`` subclasses `here <https://github.
 
 .. versionadded:: 1.1
 
+One subclass of note within those examples is the ``VariableOffsetWindowIndexer`` that allows
+rolling operations over a non-fixed offset like a ``BusinessDay``.
+
+.. ipython:: python
+
+   from pandas.api.indexers import VariableOffsetWindowIndexer
+   df = pd.DataFrame(range(10), index=pd.date_range('2020', periods=10))
+   offset = pd.offsets.BDay(1)
+   indexer = VariableOffsetWindowIndexer(index=df.index, offset=offset)
+   df
+   df.rolling(indexer).sum()
+
 For some problems knowledge of the future is available for analysis. For example, this occurs when
 each data point is a full time series read from an experiment, and the task is to extract underlying
 conditions. In these cases it can be useful to perform forward-looking rolling window computations.

doc/source/whatsnew/v1.1.0.rst

@@ -328,6 +328,7 @@ Other enhancements
 - :meth:`DataFrame.to_html` and :meth:`DataFrame.to_string`'s ``col_space`` parameter now accepts a list or dict to change only some specific columns' width (:issue:`28917`).
 - :meth:`DataFrame.to_excel` can now also write OpenOffice spreadsheet (.ods) files (:issue:`27222`)
 - :meth:`~Series.explode` now accepts ``ignore_index`` to reset the index, similarly to :meth:`pd.concat` or :meth:`DataFrame.sort_values` (:issue:`34932`).
+- :meth:`read_csv` now accepts string values like "0", "0.0", "1", "1.0" as convertible to the nullable boolean dtype (:issue:`34859`)
 
 .. ---------------------------------------------------------------------------
 
@@ -673,6 +674,7 @@ Other API changes
 - ``loc`` lookups with an object-dtype :class:`Index` and an integer key will now raise ``KeyError`` instead of ``TypeError`` when key is missing (:issue:`31905`)
 - Using a :func:`pandas.api.indexers.BaseIndexer` with ``count``, ``min``, ``max``, ``median``, ``skew``,  ``cov``, ``corr`` will now return correct results for any monotonic :func:`pandas.api.indexers.BaseIndexer` descendant (:issue:`32865`)
 - Added a :func:`pandas.api.indexers.FixedForwardWindowIndexer` class to support forward-looking windows during ``rolling`` operations.
+- Added a :func:`pandas.api.indexers.VariableOffsetWindowIndexer` class to support ``rolling`` operations with non-fixed offsets (:issue:`34994`)
 - Added :class:`pandas.errors.InvalidIndexError` (:issue:`34570`).
 - :meth:`DataFrame.swaplevels` now raises a  ``TypeError`` if the axis is not a :class:`MultiIndex`.
   Previously an ``AttributeError`` was raised (:issue:`31126`)
@@ -692,9 +694,9 @@ Other API changes
 - :func: `pandas.api.dtypes.is_string_dtype` no longer incorrectly identifies categorical series as string.
 - :func:`read_excel` no longer takes ``**kwds`` arguments. This means that passing in keyword ``chunksize`` now raises a ``TypeError``
   (previously raised a ``NotImplementedError``), while passing in keyword ``encoding`` now raises a ``TypeError`` (:issue:`34464`)
-- :func: `merge` now checks ``suffixes`` parameter type to be ``tuple`` and raises ``TypeError``, whereas before a ``list`` or ``set`` were accepted and that the ``set`` could produce unexpected results (:issue:`33740`)
 - :class:`Period` no longer accepts tuples for the ``freq`` argument (:issue:`34658`)
 - :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` now raises ValueError if ``limit_direction`` is 'forward' or 'both' and ``method`` is 'backfill' or 'bfill' or ``limit_direction`` is 'backward' or 'both' and ``method`` is 'pad' or 'ffill' (:issue:`34746`)
+- The :class:`DataFrame` constructor no longer accepts a list of ``DataFrame`` objects. Because of changes to NumPy, ``DataFrame`` objects are now consistently treated as 2D objects, so a list of ``DataFrames`` is considered 3D, and no longer acceptible for the ``DataFrame`` constructor (:issue:`32289`).
 
 
 Increased minimum versions for dependencies
@@ -785,6 +787,7 @@ Deprecations
 - :meth:`DataFrame.to_dict` has deprecated accepting short names for ``orient`` in future versions (:issue:`32515`)
 - :meth:`Categorical.to_dense` is deprecated and will be removed in a future version, use ``np.asarray(cat)`` instead (:issue:`32639`)
 - The ``fastpath`` keyword in the ``SingleBlockManager`` constructor is deprecated and will be removed in a future version (:issue:`33092`)
+- Providing ``suffixes`` as a ``set`` in :func:`pandas.merge` is deprecated. Provide a tuple instead (:issue:`33740`, :issue:`34741`).
 - :meth:`Index.is_mixed` is deprecated and will be removed in a future version, check ``index.inferred_type`` directly instead (:issue:`32922`)
 
 - Passing any arguments but the first one to  :func:`read_html` as
@@ -816,6 +819,7 @@ Deprecations
 - :meth:`util.testing.assert_almost_equal` now accepts both relative and absolute
   precision through the ``rtol``, and ``atol`` parameters, thus deprecating the
   ``check_less_precise`` parameter. (:issue:`13357`).
+- :func:`DataFrame.melt` accepting a value_name that already exists is deprecated, and will be removed in a future version (:issue:`34731`)
 
 .. ---------------------------------------------------------------------------
 
@@ -1041,6 +1045,9 @@ I/O
 - Bug in :meth:`read_excel` for ODS files removes 0.0 values (:issue:`27222`)
 - Bug in :meth:`ujson.encode` was raising an `OverflowError` with numbers larger than sys.maxsize (:issue: `34395`)
 - Bug in :meth:`HDFStore.append_to_multiple` was raising a ``ValueError`` when the min_itemsize parameter is set (:issue:`11238`)
+- Bug in :meth:`~HDFStore.create_table` now raises an error when `column` argument was not specified in `data_columns` on input (:issue:`28156`)
+- :meth:`read_json` now could read line-delimited json file from a file url while `lines` and `chunksize` are set.
+- Bug in :meth:`DataFrame.to_sql` when reading DataFrames with ``-np.inf`` entries with MySQL now has a more explicit ``ValueError`` (:issue:`34431`)
 
 Plotting
 ^^^^^^^^
@@ -1120,6 +1127,7 @@ ExtensionArray
 ^^^^^^^^^^^^^^
 
 - Fixed bug where :meth:`Series.value_counts` would raise on empty input of ``Int64`` dtype (:issue:`33317`)
+- Fixed bug in :func:`concat` when concatenating DataFrames with non-overlaping columns resulting in object-dtype columns rather than preserving the extension dtype (:issue:`27692`, :issue:`33027`)
 - Fixed bug where :meth:`StringArray.isna` would return ``False`` for NA values when ``pandas.options.mode.use_inf_as_na`` was set to ``True`` (:issue:`33655`)
 - Fixed bug in :class:`Series` construction with EA dtype and index but no data or scalar data fails (:issue:`26469`)
 - Fixed bug that caused :meth:`Series.__repr__()` to crash for extension types whose elements are multidimensional arrays (:issue:`33770`).

pandas/_libs/lib.pyx

@@ -73,7 +73,7 @@ from pandas._libs.tslibs.nattype cimport (
 )
 from pandas._libs.tslibs.conversion cimport convert_to_tsobject
 from pandas._libs.tslibs.timedeltas cimport convert_to_timedelta64
-from pandas._libs.tslibs.timezones cimport get_timezone, tz_compare
+from pandas._libs.tslibs.timezones cimport tz_compare
 from pandas._libs.tslibs.period cimport is_period_object
 from pandas._libs.tslibs.offsets cimport is_offset_object
 
@@ -1789,7 +1789,7 @@ def is_datetime_with_singletz_array(values: ndarray) -> bool:
     for i in range(n):
         base_val = values[i]
         if base_val is not NaT:
-            base_tz = get_timezone(getattr(base_val, 'tzinfo', None))
+            base_tz = getattr(base_val, 'tzinfo', None)
             break
 
     for j in range(i, n):