Merge branch 'master' of https://github.com/pandas-dev/pandas

Author: jacobaustin123

ci/code_checks.sh

@@ -292,10 +292,6 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
     pytest -q --doctest-modules pandas/core/generic.py
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    MSG='Doctests groupby.py' ; echo $MSG
-    pytest -q --doctest-modules pandas/core/groupby/groupby.py -k"-cumcount -describe -pipe"
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
     MSG='Doctests series.py' ; echo $MSG
     pytest -q --doctest-modules pandas/core/series.py
     RET=$(($RET + $?)) ; echo $MSG "DONE"
@@ -318,6 +314,10 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
     pytest -q --doctest-modules pandas/core/dtypes/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Doctests groupby' ; echo $MSG
+    pytest -q --doctest-modules pandas/core/groupby/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Doctests indexes' ; echo $MSG
     pytest -q --doctest-modules pandas/core/indexes/
     RET=$(($RET + $?)) ; echo $MSG "DONE"

doc/source/user_guide/io.rst

@@ -285,14 +285,18 @@ chunksize : int, default ``None``
 Quoting, compression, and file format
 +++++++++++++++++++++++++++++++++++++
 
-compression : {``'infer'``, ``'gzip'``, ``'bz2'``, ``'zip'``, ``'xz'``, ``None``}, default ``'infer'``
+compression : {``'infer'``, ``'gzip'``, ``'bz2'``, ``'zip'``, ``'xz'``, ``None``, ``dict``}, default ``'infer'``
   For on-the-fly decompression of on-disk data. If 'infer', then use gzip,
   bz2, zip, or xz if filepath_or_buffer is a string ending in '.gz', '.bz2',
   '.zip', or '.xz', respectively, and no decompression otherwise. If using 'zip',
   the ZIP file must contain only one data file to be read in.
-  Set to ``None`` for no decompression.
+  Set to ``None`` for no decompression. Can also be a dict with key ``'method'``
+  set to one of {``'zip'``, ``'gzip'``, ``'bz2'``}, and other keys set to
+  compression settings. As an example, the following could be passed for
+  faster compression: ``compression={'method': 'gzip', 'compresslevel': 1}``.
 
   .. versionchanged:: 0.24.0 'infer' option added and set to default.
+  .. versionchanged:: 1.1.0 dict option extended to support ``gzip`` and ``bz2``.
 thousands : str, default ``None``
   Thousands separator.
 decimal : str, default ``'.'``
@@ -3347,6 +3351,12 @@ The compression type can be an explicit parameter or be inferred from the file e
 If 'infer', then use ``gzip``, ``bz2``, ``zip``, or ``xz`` if filename ends in ``'.gz'``, ``'.bz2'``, ``'.zip'``, or
 ``'.xz'``, respectively.
 
+The compression parameter can also be a ``dict`` in order to pass options to the
+compression protocol. It must have a ``'method'`` key set to the name
+of the compression protocol, which must be one of
+{``'zip'``, ``'gzip'``, ``'bz2'``}. All other key-value pairs are passed to
+the underlying compression library.
+
 .. ipython:: python
 
    df = pd.DataFrame({
@@ -3383,6 +3393,15 @@ The default is to 'infer':
    rt = pd.read_pickle("s1.pkl.bz2")
    rt
 
+Passing options to the compression protocol in order to speed up compression:
+
+.. ipython:: python
+
+   df.to_pickle(
+       "data.pkl.gz",
+       compression={"method": "gzip", 'compresslevel': 1}
+   )
+
 .. ipython:: python
    :suppress:
 

doc/source/user_guide/timeseries.rst

@@ -772,6 +772,7 @@ There are several time/date properties that one can access from ``Timestamp`` or
     week,"The week ordinal of the year"
     dayofweek,"The number of the day of the week with Monday=0, Sunday=6"
     weekday,"The number of the day of the week with Monday=0, Sunday=6"
+    isocalendar,"The ISO 8601 year, week and day of the date"
     quarter,"Quarter of the date: Jan-Mar = 1, Apr-Jun = 2, etc."
     days_in_month,"The number of days in the month of the datetime"
     is_month_start,"Logical indicating if first day of month (defined by frequency)"
@@ -786,6 +787,15 @@ Furthermore, if you have a ``Series`` with datetimelike values, then you can
 access these properties via the ``.dt`` accessor, as detailed in the section
 on :ref:`.dt accessors<basics.dt_accessors>`.
 
+.. versionadded:: 1.1.0
+
+You may obtain the year, week and day components of the ISO year from the ISO 8601 standard:
+
+.. ipython:: python
+
+   idx = pd.date_range(start='2019-12-29', freq='D', periods=4)
+   idx.to_series().dt.isocalendar
+
 .. _timeseries.offsets:
 
 DateOffset objects

doc/source/whatsnew/v0.4.x.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_04x:
 
-v.0.4.1 through v0.4.3 (September 25 - October 9, 2011)
--------------------------------------------------------
+Versions 0.4.1 through 0.4.3 (September 25 - October 9, 2011)
+-------------------------------------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.5.0.rst

@@ -1,8 +1,8 @@
 
 .. _whatsnew_050:
 
-v.0.5.0 (October 24, 2011)
---------------------------
+Version 0.5.0 (October 24, 2011)
+--------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.6.0.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_060:
 
-v.0.6.0 (November 25, 2011)
----------------------------
+Version 0.6.0 (November 25, 2011)
+---------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v1.1.0.rst

@@ -126,9 +126,16 @@ Other enhancements
 - :class:`Series.str` now has a `fullmatch` method that matches a regular expression against the entire string in each row of the series, similar to `re.fullmatch` (:issue:`32806`).
 - :meth:`DataFrame.sample` will now also allow array-like and BitGenerator objects to be passed to ``random_state`` as seeds (:issue:`32503`)
 - :meth:`MultiIndex.union` will now raise `RuntimeWarning` if the object inside are unsortable, pass `sort=False` to suppress this warning (:issue:`33015`)
+- :class:`Series.dt` and :class:`DatatimeIndex` now have an `isocalendar` accessor that returns a :class:`DataFrame` with year, week, and day calculated according to the ISO 8601 calendar (:issue:`33206`).
 - The :meth:`DataFrame.to_feather` method now supports additional keyword
   arguments (e.g. to set the compression) that are added in pyarrow 0.17
   (:issue:`33422`).
+- :meth:`DataFrame.to_csv`, :meth:`DataFrame.to_pickle`,
+  and :meth:`DataFrame.to_json` now support passing a dict of
+  compression arguments when using the ``gzip`` and ``bz2`` protocols.
+  This can be used to set a custom compression level, e.g.,
+  ``df.to_csv(path, compression={'method': 'gzip', 'compresslevel': 1}``
+  (:issue:`33196`)
 
 .. ---------------------------------------------------------------------------
 
@@ -409,7 +416,7 @@ Performance improvements
   sparse values from ``scipy.sparse`` matrices using the
   :meth:`DataFrame.sparse.from_spmatrix` constructor (:issue:`32821`,
   :issue:`32825`,  :issue:`32826`, :issue:`32856`, :issue:`32858`).
-- Performance improvement in reductions (sum, min, max) for nullable (integer and boolean) dtypes (:issue:`30982`, :issue:`33261`).
+- Performance improvement in reductions (sum, prod, min, max) for nullable (integer and boolean) dtypes (:issue:`30982`, :issue:`33261`, :issue:`33442`).
 
 
 .. ---------------------------------------------------------------------------
@@ -441,6 +448,8 @@ Datetimelike
 - Bug in :meth:`DatetimeIndex.searchsorted` not accepting a ``list`` or :class:`Series` as its argument (:issue:`32762`)
 - Bug where :meth:`PeriodIndex` raised when passed a :class:`Series` of strings (:issue:`26109`)
 - Bug in :class:`Timestamp` arithmetic when adding or subtracting a ``np.ndarray`` with ``timedelta64`` dtype (:issue:`33296`)
+- Bug in :meth:`DatetimeIndex.to_period` not infering the frequency when called with no arguments (:issue:`33358`)
+
 
 Timedelta
 ^^^^^^^^^
@@ -505,6 +514,7 @@ Indexing
 - Bug in :meth:`DataFrame.iloc.__setitem__` creating a new array instead of overwriting ``Categorical`` values in-place (:issue:`32831`)
 - Bug in :meth:`DataFrame.copy` _item_cache not invalidated after copy causes post-copy value updates to not be reflected (:issue:`31784`)
 - Bug in `Series.__getitem__` with an integer key and a :class:`MultiIndex` with leading integer level failing to raise ``KeyError`` if the key is not present in the first level (:issue:`33355`)
+- Bug in :meth:`DataFrame.iloc` when slicing a single column-:class:`DataFrame`` with ``ExtensionDtype`` (e.g. ``df.iloc[:, :1]``) returning an invalid result (:issue:`32957`)
 
 Missing
 ^^^^^^^
@@ -623,6 +633,7 @@ Other
 - Bug in :meth:`DataFrame.to_records` incorrectly losing timezone information in timezone-aware ``datetime64`` columns (:issue:`32535`)
 - Fixed :func:`pandas.testing.assert_series_equal` to correctly raise if left object is a different subclass with ``check_series_type=True`` (:issue:`32670`).
 - :meth:`IntegerArray.astype` now supports ``datetime64`` dtype (:issue:32538`)
+- Getting a missing attribute in a query/eval string raises the correct ``AttributeError`` (:issue:`32408`)
 - Fixed bug in :func:`pandas.testing.assert_series_equal` where dtypes were checked for ``Interval`` and ``ExtensionArray`` operands when ``check_dtype`` was ``False`` (:issue:`32747`)
 - Bug in :meth:`Series.map` not raising on invalid ``na_action`` (:issue:`32815`)
 - Bug in :meth:`DataFrame.__dir__` caused a segfault when using unicode surrogates in a column name (:issue:`25509`)

pandas/_libs/reshape.pyx

@@ -36,7 +36,7 @@ ctypedef fused reshape_t:
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def unstack(reshape_t[:, :] values, uint8_t[:] mask,
+def unstack(reshape_t[:, :] values, const uint8_t[:] mask,
             Py_ssize_t stride, Py_ssize_t length, Py_ssize_t width,
             reshape_t[:, :] new_values, uint8_t[:, :] new_mask):
     """

pandas/_libs/tslibs/ccalendar.pxd

@@ -2,9 +2,11 @@ from cython cimport Py_ssize_t
 
 from numpy cimport int64_t, int32_t
 
+ctypedef (int32_t, int32_t, int32_t) iso_calendar_t
 
 cdef int dayofweek(int y, int m, int d) nogil
 cdef bint is_leapyear(int64_t year) nogil
 cpdef int32_t get_days_in_month(int year, Py_ssize_t month) nogil
 cpdef int32_t get_week_of_year(int year, int month, int day) nogil
+cpdef iso_calendar_t get_iso_calendar(int year, int month, int day) nogil
 cpdef int32_t get_day_of_year(int year, int month, int day) nogil

pandas/_libs/tslibs/ccalendar.pyx

@@ -150,33 +150,65 @@ cpdef int32_t get_week_of_year(int year, int month, int day) nogil:
     -------
     week_of_year : int32_t
 
+    Notes
+    -----
+    Assumes the inputs describe a valid date.
+    """
+    return get_iso_calendar(year, month, day)[1]
+
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+cpdef iso_calendar_t get_iso_calendar(int year, int month, int day) nogil:
+    """
+    Return the year, week, and day of year corresponding to ISO 8601
+
+    Parameters
+    ----------
+    year : int
+    month : int
+    day : int
+
+    Returns
+    -------
+    year : int32_t
+    week : int32_t
+    day : int32_t
+
     Notes
     -----
     Assumes the inputs describe a valid date.
     """
     cdef:
         int32_t doy, dow
-        int woy
+        int32_t iso_year, iso_week
 
     doy = get_day_of_year(year, month, day)
     dow = dayofweek(year, month, day)
 
     # estimate
-    woy = (doy - 1) - dow + 3
-    if woy >= 0:
-        woy = woy // 7 + 1
+    iso_week = (doy - 1) - dow + 3
+    if iso_week >= 0:
+        iso_week = iso_week // 7 + 1
 
     # verify
-    if woy < 0:
-        if (woy > -2) or (woy == -2 and is_leapyear(year - 1)):
-            woy = 53
+    if iso_week < 0:
+        if (iso_week > -2) or (iso_week == -2 and is_leapyear(year - 1)):
+            iso_week = 53
         else:
-            woy = 52
-    elif woy == 53:
+            iso_week = 52
+    elif iso_week == 53:
         if 31 - day + dow < 3:
-            woy = 1
+            iso_week = 1
+
+    iso_year = year
+    if iso_week == 1 and doy > 7:
+        iso_year += 1
+
+    elif iso_week >= 52 and doy < 7:
+        iso_year -= 1
 
-    return woy
+    return iso_year, iso_week, dow + 1
 
 
 @cython.wraparound(False)

pandas/_libs/tslibs/fields.pyx

@@ -8,14 +8,14 @@ from cython import Py_ssize_t
 
 import numpy as np
 cimport numpy as cnp
-from numpy cimport ndarray, int64_t, int32_t, int8_t
+from numpy cimport ndarray, int64_t, int32_t, int8_t, uint32_t
 cnp.import_array()
 
 from pandas._libs.tslibs.ccalendar import (
     get_locale_names, MONTHS_FULL, DAYS_FULL, DAY_SECONDS)
 from pandas._libs.tslibs.ccalendar cimport (
     get_days_in_month, is_leapyear, dayofweek, get_week_of_year,
-    get_day_of_year)
+    get_day_of_year, get_iso_calendar, iso_calendar_t)
 from pandas._libs.tslibs.np_datetime cimport (
     npy_datetimestruct, pandas_timedeltastruct, dt64_to_dtstruct,
     td64_to_tdstruct)
@@ -670,3 +670,42 @@ cpdef isleapyear_arr(ndarray years):
                       np.logical_and(years % 4 == 0,
                                      years % 100 > 0))] = 1
     return out.view(bool)
+
+
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def build_isocalendar_sarray(const int64_t[:] dtindex):
+    """
+    Given a int64-based datetime array, return the ISO 8601 year, week, and day
+    as a structured array.
+    """
+    cdef:
+        Py_ssize_t i, count = len(dtindex)
+        npy_datetimestruct dts
+        ndarray[uint32_t] iso_years, iso_weeks, days
+        iso_calendar_t ret_val
+
+    sa_dtype = [
+        ("year", "u4"),
+        ("week", "u4"),
+        ("day", "u4"),
+    ]
+
+    out = np.empty(count, dtype=sa_dtype)
+
+    iso_years = out["year"]
+    iso_weeks = out["week"]
+    days = out["day"]
+
+    with nogil:
+        for i in range(count):
+            if dtindex[i] == NPY_NAT:
+                ret_val = 0, 0, 0
+            else:
+                dt64_to_dtstruct(dtindex[i], &dts)
+                ret_val = get_iso_calendar(dts.year, dts.month, dts.day)
+
+            iso_years[i] = ret_val[0]
+            iso_weeks[i] = ret_val[1]
+            days[i] = ret_val[2]
+    return out