Merge remote-tracking branch 'upstream/master' into numpy-ea

Author: TomAugspurger

doc/source/api.rst

@@ -371,13 +371,14 @@
 
 
 intersphinx_mapping = {
-    'statsmodels': ('http://www.statsmodels.org/devel/', None),
+    'dateutil': ("https://dateutil.readthedocs.io/en/latest/", None),
     'matplotlib': ('https://matplotlib.org/', None),
+    'numpy': ('https://docs.scipy.org/doc/numpy/', None),
     'pandas-gbq': ('https://pandas-gbq.readthedocs.io/en/latest/', None),
+    'py': ('https://pylib.readthedocs.io/en/latest/', None),
     'python': ('https://docs.python.org/3/', None),
-    'numpy': ('https://docs.scipy.org/doc/numpy/', None),
     'scipy': ('https://docs.scipy.org/doc/scipy/reference/', None),
-    'py': ('https://pylib.readthedocs.io/en/latest/', None)
+    'statsmodels': ('http://www.statsmodels.org/devel/', None),
 }
 import glob
 autosummary_generate = glob.glob("*.rst")

doc/source/conf.py

@@ -66,12 +66,21 @@ pandas objects can be split on any of their axes. The abstract definition of
 grouping is to provide a mapping of labels to group names. To create a GroupBy
 object (more on what the GroupBy object is later), you may do the following:
 
-.. code-block:: python
+.. ipython:: python
+
+    df = pd.DataFrame([('bird', 'Falconiformes', 389.0),
+                       ('bird', 'Psittaciformes', 24.0),
+                       ('mammal', 'Carnivora', 80.2),
+                       ('mammal', 'Primates', np.nan),
+                       ('mammal', 'Carnivora', 58)],
+                      index=['falcon', 'parrot', 'lion', 'monkey', 'leopard'],
+                      columns=('class', 'order', 'max_speed'))
+    df
 
-   # default is axis=0
-   >>> grouped = obj.groupby(key)
-   >>> grouped = obj.groupby(key, axis=1)
-   >>> grouped = obj.groupby([key1, key2])
+    # default is axis=0
+    grouped = df.groupby('class')
+    grouped = df.groupby('order', axis='columns')
+    grouped = df.groupby(['class', 'order'])
 
 The mapping can be specified many different ways:
 
@@ -239,7 +248,7 @@ the length of the ``groups`` dict, so it is largely just a convenience:
 .. ipython::
 
    @verbatim
-   In [1]: gb.<TAB>
+   In [1]: gb.<TAB>  # noqa: E225, E999
    gb.agg        gb.boxplot    gb.cummin     gb.describe   gb.filter     gb.get_group  gb.height     gb.last       gb.median     gb.ngroups    gb.plot       gb.rank       gb.std        gb.transform
    gb.aggregate  gb.count      gb.cumprod    gb.dtype      gb.first      gb.groups     gb.hist       gb.max        gb.min        gb.nth        gb.prod       gb.resample   gb.sum        gb.var
    gb.apply      gb.cummax     gb.cumsum     gb.fillna     gb.gender     gb.head       gb.indices    gb.mean       gb.name       gb.ohlc       gb.quantile   gb.size       gb.tail       gb.weight
@@ -1300,12 +1309,17 @@ Now, to find prices per store/product, we can simply do:
 Piping can also be expressive when you want to deliver a grouped object to some
 arbitrary function, for example:
 
-.. code-block:: python
+.. ipython:: python
+
+   def mean(groupby):
+       return groupby.mean()
 
-   df.groupby(['Store', 'Product']).pipe(report_func)
+   df.groupby(['Store', 'Product']).pipe(mean)
 
-where ``report_func`` takes a GroupBy object and creates a report
-from that.
+where ``mean`` takes a GroupBy object and finds the mean of the Revenue and Quantity
+columns repectively for each Store-Product combination. The ``mean`` function can
+be any function that takes in a GroupBy object; the ``.pipe`` will pass the GroupBy
+object as a parameter into the function you specify.
 
 Examples
 --------

doc/source/groupby.rst

@@ -855,40 +855,40 @@ into ``freq`` keyword arguments. The available date offsets and associated frequ
     :header: "Date Offset", "Frequency String", "Description"
     :widths: 15, 15, 65
 
-    ``DateOffset``, None, "Generic offset class, defaults to 1 calendar day"
-    ``BDay`` or ``BusinessDay``, ``'B'``,"business day (weekday)"
-    ``CDay`` or ``CustomBusinessDay``, ``'C'``, "custom business day"
-    ``Week``, ``'W'``, "one week, optionally anchored on a day of the week"
-    ``WeekOfMonth``, ``'WOM'``, "the x-th day of the y-th week of each month"
-    ``LastWeekOfMonth``, ``'LWOM'``, "the x-th day of the last week of each month"
-    ``MonthEnd``, ``'M'``, "calendar month end"
-    ``MonthBegin``, ``'MS'``, "calendar month begin"
-    ``BMonthEnd`` or ``BusinessMonthEnd``, ``'BM'``, "business month end"
-    ``BMonthBegin`` or ``BusinessMonthBegin``, ``'BMS'``, "business month begin"
-    ``CBMonthEnd`` or ``CustomBusinessMonthEnd``, ``'CBM'``, "custom business month end"
-    ``CBMonthBegin`` or ``CustomBusinessMonthBegin``, ``'CBMS'``, "custom business month begin"
-    ``SemiMonthEnd``, ``'SM'``, "15th (or other day_of_month) and calendar month end"
-    ``SemiMonthBegin``, ``'SMS'``, "15th (or other day_of_month) and calendar month begin"
-    ``QuarterEnd``, ``'Q'``, "calendar quarter end"
-    ``QuarterBegin``, ``'QS'``, "calendar quarter begin"
-    ``BQuarterEnd``, ``'BQ``, "business quarter end"
-    ``BQuarterBegin``, ``'BQS'``, "business quarter begin"
-    ``FY5253Quarter``, ``'REQ'``, "retail (aka 52-53 week) quarter"
-    ``YearEnd``, ``'A'``, "calendar year end"
-    ``YearBegin``, ``'AS'`` or ``'BYS'``,"calendar year begin"
-    ``BYearEnd``, ``'BA'``, "business year end"
-    ``BYearBegin``, ``'BAS'``, "business year begin"
-    ``FY5253``, ``'RE'``, "retail (aka 52-53 week) year"
-    ``Easter``, None, "Easter holiday"
-    ``BusinessHour``, ``'BH'``, "business hour"
-    ``CustomBusinessHour``, ``'CBH'``, "custom business hour"
-    ``Day``, ``'D'``, "one absolute day"
-    ``Hour``, ``'H'``, "one hour"
-    ``Minute``, ``'T'`` or ``'min'``,"one minute"
-    ``Second``, ``'S'``, "one second"
-    ``Milli``, ``'L'`` or ``'ms'``, "one millisecond"
-    ``Micro``, ``'U'`` or ``'us'``, "one microsecond"
-    ``Nano``, ``'N'``, "one nanosecond"
+    :class:`~pandas.tseries.offsets.DateOffset`, None, "Generic offset class, defaults to 1 calendar day"
+    :class:`~pandas.tseries.offsets.BDay` or :class:`~pandas.tseries.offsets.BusinessDay`, ``'B'``,"business day (weekday)"
+    :class:`~pandas.tseries.offsets.CDay` or :class:`~pandas.tseries.offsets.CustomBusinessDay`, ``'C'``, "custom business day"
+    :class:`~pandas.tseries.offsets.Week`, ``'W'``, "one week, optionally anchored on a day of the week"
+    :class:`~pandas.tseries.offsets.WeekOfMonth`, ``'WOM'``, "the x-th day of the y-th week of each month"
+    :class:`~pandas.tseries.offsets.LastWeekOfMonth`, ``'LWOM'``, "the x-th day of the last week of each month"
+    :class:`~pandas.tseries.offsets.MonthEnd`, ``'M'``, "calendar month end"
+    :class:`~pandas.tseries.offsets.MonthBegin`, ``'MS'``, "calendar month begin"
+    :class:`~pandas.tseries.offsets.BMonthEnd` or :class:`~pandas.tseries.offsets.BusinessMonthEnd`, ``'BM'``, "business month end"
+    :class:`~pandas.tseries.offsets.BMonthBegin` or :class:`~pandas.tseries.offsets.BusinessMonthBegin`, ``'BMS'``, "business month begin"
+    :class:`~pandas.tseries.offsets.CBMonthEnd` or :class:`~pandas.tseries.offsets.CustomBusinessMonthEnd`, ``'CBM'``, "custom business month end"
+    :class:`~pandas.tseries.offsets.CBMonthBegin` or :class:`~pandas.tseries.offsets.CustomBusinessMonthBegin`, ``'CBMS'``, "custom business month begin"
+    :class:`~pandas.tseries.offsets.SemiMonthEnd`, ``'SM'``, "15th (or other day_of_month) and calendar month end"
+    :class:`~pandas.tseries.offsets.SemiMonthBegin`, ``'SMS'``, "15th (or other day_of_month) and calendar month begin"
+    :class:`~pandas.tseries.offsets.QuarterEnd`, ``'Q'``, "calendar quarter end"
+    :class:`~pandas.tseries.offsets.QuarterBegin`, ``'QS'``, "calendar quarter begin"
+    :class:`~pandas.tseries.offsets.BQuarterEnd`, ``'BQ``, "business quarter end"
+    :class:`~pandas.tseries.offsets.BQuarterBegin`, ``'BQS'``, "business quarter begin"
+    :class:`~pandas.tseries.offsets.FY5253Quarter`, ``'REQ'``, "retail (aka 52-53 week) quarter"
+    :class:`~pandas.tseries.offsets.YearEnd`, ``'A'``, "calendar year end"
+    :class:`~pandas.tseries.offsets.YearBegin`, ``'AS'`` or ``'BYS'``,"calendar year begin"
+    :class:`~pandas.tseries.offsets.BYearEnd`, ``'BA'``, "business year end"
+    :class:`~pandas.tseries.offsets.BYearBegin`, ``'BAS'``, "business year begin"
+    :class:`~pandas.tseries.offsets.FY5253`, ``'RE'``, "retail (aka 52-53 week) year"
+    :class:`~pandas.tseries.offsets.Easter`, None, "Easter holiday"
+    :class:`~pandas.tseries.offsets.BusinessHour`, ``'BH'``, "business hour"
+    :class:`~pandas.tseries.offsets.CustomBusinessHour`, ``'CBH'``, "custom business hour"
+    :class:`~pandas.tseries.offsets.Day`, ``'D'``, "one absolute day"
+    :class:`~pandas.tseries.offsets.Hour`, ``'H'``, "one hour"
+    :class:`~pandas.tseries.offsets.Minute`, ``'T'`` or ``'min'``,"one minute"
+    :class:`~pandas.tseries.offsets.Second`, ``'S'``, "one second"
+    :class:`~pandas.tseries.offsets.Milli`, ``'L'`` or ``'ms'``, "one millisecond"
+    :class:`~pandas.tseries.offsets.Micro`, ``'U'`` or ``'us'``, "one microsecond"
+    :class:`~pandas.tseries.offsets.Nano`, ``'N'``, "one nanosecond"
 
 ``DateOffsets`` additionally have :meth:`rollforward` and :meth:`rollback`
 methods for moving a date forward or backward respectively to a valid offset

doc/source/timeseries.rst

@@ -374,7 +374,7 @@ Other Enhancements
 - :meth:`read_excel()` now accepts ``usecols`` as a list of column names or callable (:issue:`18273`)
 - :meth:`MultiIndex.to_flat_index` has been added to flatten multiple levels into a single-level :class:`Index` object.
 - :meth:`DataFrame.to_stata` and :class:`pandas.io.stata.StataWriter117` can write mixed sting columns to Stata strl format (:issue:`23633`)
-- :meth:`DataFrame.between_time` and :meth:`DataFrame.at_time` have gained the an ``axis`` parameter (:issue:`8839`)
+- :meth:`DataFrame.between_time` and :meth:`DataFrame.at_time` have gained the ``axis`` parameter (:issue:`8839`)
 - The ``scatter_matrix``, ``andrews_curves``, ``parallel_coordinates``, ``lag_plot``, ``autocorrelation_plot``, ``bootstrap_plot``, and ``radviz`` plots from the ``pandas.plotting`` module are now accessible from calling :meth:`DataFrame.plot` (:issue:`11978`)
 - :class:`IntervalIndex` has gained the :attr:`~IntervalIndex.is_overlapping` attribute to indicate if the ``IntervalIndex`` contains any overlapping intervals (:issue:`23309`)
 
@@ -1539,6 +1539,7 @@ Notice how we now instead output ``np.nan`` itself instead of a stringified form
 - Bug in :meth:`DataFrame.to_dict` when the resulting dict contains non-Python scalars in the case of numeric data (:issue:`23753`)
 - :func:`DataFrame.to_string()`, :func:`DataFrame.to_html()`, :func:`DataFrame.to_latex()` will correctly format output when a string is passed as the ``float_format`` argument (:issue:`21625`, :issue:`22270`)
 - Bug in :func:`read_csv` that caused it to raise ``OverflowError`` when trying to use 'inf' as ``na_value`` with integer index column (:issue:`17128`)
+- Bug in :func:`read_fwf` in which the compression type of a file was not being properly inferred (:issue:`22199`)
 - Bug in :func:`pandas.io.json.json_normalize` that caused it to raise ``TypeError`` when two consecutive elements of ``record_path`` are dicts (:issue:`22706`)
 - Bug in :meth:`DataFrame.to_stata`, :class:`pandas.io.stata.StataWriter` and :class:`pandas.io.stata.StataWriter117` where a exception would leave a partially written and invalid dta file (:issue:`23573`)
 - Bug in :meth:`DataFrame.to_stata` and :class:`pandas.io.stata.StataWriter117` that produced invalid files when using strLs with non-ASCII characters (:issue:`23573`)

doc/source/whatsnew/v0.24.0.rst

@@ -401,7 +401,7 @@ def _read(filepath_or_buffer, kwds):
         encoding = re.sub('_', '-', encoding).lower()
         kwds['encoding'] = encoding
 
-    compression = kwds.get('compression')
+    compression = kwds.get('compression', 'infer')
     compression = _infer_compression(filepath_or_buffer, compression)
     filepath_or_buffer, _, compression, should_close = get_filepath_or_buffer(
         filepath_or_buffer, encoding, compression)

pandas/io/parsers.py

@@ -555,20 +555,26 @@ def test_default_delimiter():
     tm.assert_frame_equal(result, expected)
 
 
-@pytest.mark.parametrize("compression", ["gzip", "bz2"])
-def test_fwf_compression(compression):
+@pytest.mark.parametrize("infer", [True, False, None])
+def test_fwf_compression(compression_only, infer):
     data = """1111111111
     2222222222
     3333333333""".strip()
 
+    compression = compression_only
+    extension = "gz" if compression == "gzip" else compression
+
     kwargs = dict(widths=[5, 5], names=["one", "two"])
     expected = read_fwf(StringIO(data), **kwargs)
 
     if compat.PY3:
         data = bytes(data, encoding="utf-8")
 
-    with tm.ensure_clean() as path:
+    with tm.ensure_clean(filename="tmp." + extension) as path:
         tm.write_to_compressed(compression, path, data)
 
-        result = read_fwf(path, compression=compression, **kwargs)
+        if infer is not None:
+            kwargs["compression"] = "infer" if infer else compression
+
+        result = read_fwf(path, **kwargs)
         tm.assert_frame_equal(result, expected)

pandas/tests/io/parser/test_read_fwf.py

@@ -53,7 +53,6 @@ exclude =
     doc/source/basics.rst
     doc/source/contributing_docstring.rst
     doc/source/enhancingperf.rst
-    doc/source/groupby.rst
 
 
 [yapf]