Merge remote-tracking branch 'upstream/master' into mcmali-s3-pub-test

Author: alimcmaster1

.devcontainer.json

@@ -17,7 +17,9 @@
 		"python.linting.pylintEnabled": false,
 		"python.linting.mypyEnabled": true,
 		"python.testing.pytestEnabled": true,
-		"python.testing.cwd": "pandas/tests"
+		"python.testing.pytestArgs": [
+			"pandas"
+		]
 	},
 
 	// Add the IDs of extensions you want installed when the container is created in the array below.

ci/deps/azure-36-locale.yaml

@@ -15,7 +15,6 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - gcsfs
   - html5lib
   - ipython
   - jinja2
@@ -31,7 +30,6 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
   - scipy
   - xarray
   - xlrd

ci/deps/azure-37-locale.yaml

@@ -27,7 +27,6 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
   - scipy
   - xarray
   - xlrd

ci/deps/azure-windows-37.yaml

@@ -15,7 +15,8 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
-  - gcsfs
+  - fsspec>=0.7.4
+  - gcsfs>=0.6.0
   - html5lib
   - jinja2
   - lxml
@@ -28,7 +29,7 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - scipy
   - sqlalchemy
   - xlrd

ci/deps/travis-36-cov.yaml

@@ -18,7 +18,8 @@ dependencies:
   - cython>=0.29.16
   - dask
   - fastparquet>=0.3.2
-  - gcsfs
+  - fsspec>=0.7.4
+  - gcsfs>=0.6.0
   - geopandas
   - html5lib
   - matplotlib
@@ -35,7 +36,7 @@ dependencies:
   - pytables
   - python-snappy
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - scikit-learn
   - scipy
   - sqlalchemy

ci/deps/travis-36-locale.yaml

@@ -16,7 +16,6 @@ dependencies:
   - blosc=1.14.3
   - python-blosc
   - fastparquet=0.3.2
-  - gcsfs=0.2.2
   - html5lib
   - ipython
   - jinja2
@@ -33,7 +32,6 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs=0.3.0
   - scipy
   - sqlalchemy=1.1.4
   - xarray=0.10

ci/deps/travis-36-slow.yaml

@@ -13,6 +13,7 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
+  - fsspec>=0.7.4
   - html5lib
   - lxml
   - matplotlib
@@ -25,7 +26,7 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - scipy
   - sqlalchemy
   - xlrd

ci/deps/travis-37.yaml

@@ -13,12 +13,13 @@ dependencies:
 
   # pandas dependencies
   - botocore>=1.11
+  - fsspec>=0.7.4
   - numpy
   - python-dateutil
   - nomkl
   - pyarrow
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - tabulate
   - pyreadstat
   - pip

doc/redirects.csv

@@ -269,11 +269,11 @@ generated/pandas.core.resample.Resampler.std,../reference/api/pandas.core.resamp
 generated/pandas.core.resample.Resampler.sum,../reference/api/pandas.core.resample.Resampler.sum
 generated/pandas.core.resample.Resampler.transform,../reference/api/pandas.core.resample.Resampler.transform
 generated/pandas.core.resample.Resampler.var,../reference/api/pandas.core.resample.Resampler.var
-generated/pandas.core.window.EWM.corr,../reference/api/pandas.core.window.EWM.corr
-generated/pandas.core.window.EWM.cov,../reference/api/pandas.core.window.EWM.cov
-generated/pandas.core.window.EWM.mean,../reference/api/pandas.core.window.EWM.mean
-generated/pandas.core.window.EWM.std,../reference/api/pandas.core.window.EWM.std
-generated/pandas.core.window.EWM.var,../reference/api/pandas.core.window.EWM.var
+generated/pandas.core.window.ExponentialMovingWindow.corr,../reference/api/pandas.core.window.ExponentialMovingWindow.corr
+generated/pandas.core.window.ExponentialMovingWindow.cov,../reference/api/pandas.core.window.ExponentialMovingWindow.cov
+generated/pandas.core.window.ExponentialMovingWindow.mean,../reference/api/pandas.core.window.ExponentialMovingWindow.mean
+generated/pandas.core.window.ExponentialMovingWindow.std,../reference/api/pandas.core.window.ExponentialMovingWindow.std
+generated/pandas.core.window.ExponentialMovingWindow.var,../reference/api/pandas.core.window.ExponentialMovingWindow.var
 generated/pandas.core.window.Expanding.aggregate,../reference/api/pandas.core.window.Expanding.aggregate
 generated/pandas.core.window.Expanding.apply,../reference/api/pandas.core.window.Expanding.apply
 generated/pandas.core.window.Expanding.corr,../reference/api/pandas.core.window.Expanding.corr

doc/source/getting_started/install.rst

@@ -267,8 +267,9 @@ SQLAlchemy                1.1.4              SQL support for databases other tha
 SciPy                     0.19.0             Miscellaneous statistical functions
 XLsxWriter                0.9.8              Excel writing
 blosc                                        Compression for HDF5
+fsspec                    0.7.4              Handling files aside from local and HTTP
 fastparquet               0.3.2              Parquet reading / writing
-gcsfs                     0.2.2              Google Cloud Storage access
+gcsfs                     0.6.0              Google Cloud Storage access
 html5lib                                     HTML parser for read_html (see :ref:`note <optional_html>`)
 lxml                      3.8.0              HTML parser for read_html (see :ref:`note <optional_html>`)
 matplotlib                2.2.2              Visualization
@@ -282,7 +283,7 @@ pyreadstat                                   SPSS files (.sav) reading
 pytables                  3.4.3              HDF5 reading / writing
 pyxlsb                    1.0.6              Reading for xlsb files
 qtpy                                         Clipboard I/O
-s3fs                      0.3.0              Amazon S3 access
+s3fs                      0.4.0              Amazon S3 access
 tabulate                  0.8.3              Printing in Markdown-friendly format (see `tabulate`_)
 xarray                    0.8.2              pandas-like API for N-dimensional data
 xclip                                        Clipboard I/O on linux

doc/source/reference/window.rst

@@ -8,7 +8,7 @@ Window
 
 Rolling objects are returned by ``.rolling`` calls: :func:`pandas.DataFrame.rolling`, :func:`pandas.Series.rolling`, etc.
 Expanding objects are returned by ``.expanding`` calls: :func:`pandas.DataFrame.expanding`, :func:`pandas.Series.expanding`, etc.
-EWM objects are returned by ``.ewm`` calls: :func:`pandas.DataFrame.ewm`, :func:`pandas.Series.ewm`, etc.
+ExponentialMovingWindow objects are returned by ``.ewm`` calls: :func:`pandas.DataFrame.ewm`, :func:`pandas.Series.ewm`, etc.
 
 Standard moving window functions
 --------------------------------
@@ -69,11 +69,11 @@ Exponentially-weighted moving window functions
 .. autosummary::
    :toctree: api/
 
-   EWM.mean
-   EWM.std
-   EWM.var
-   EWM.corr
-   EWM.cov
+   ExponentialMovingWindow.mean
+   ExponentialMovingWindow.std
+   ExponentialMovingWindow.var
+   ExponentialMovingWindow.corr
+   ExponentialMovingWindow.cov
 
 Window indexer
 --------------

doc/source/user_guide/advanced.rst

@@ -260,7 +260,9 @@ You don't have to specify all levels of the ``MultiIndex`` by passing only the
 first elements of the tuple. For example, you can use "partial" indexing to
 get all elements with ``bar`` in the first level as follows:
 
-df.loc['bar']
+.. ipython:: python
+
+   df.loc['bar']
 
 This is a shortcut for the slightly more verbose notation ``df.loc[('bar',),]`` (equivalent
 to ``df.loc['bar',]`` in this example).

doc/source/user_guide/computation.rst

@@ -230,7 +230,7 @@ see the :ref:`groupby docs <groupby.transform.window_resample>`.
    The API for window statistics is quite similar to the way one works with ``GroupBy`` objects, see the documentation :ref:`here <groupby>`.
 
 We work with ``rolling``, ``expanding`` and ``exponentially weighted`` data through the corresponding
-objects, :class:`~pandas.core.window.Rolling`, :class:`~pandas.core.window.Expanding` and :class:`~pandas.core.window.EWM`.
+objects, :class:`~pandas.core.window.Rolling`, :class:`~pandas.core.window.Expanding` and :class:`~pandas.core.window.ExponentialMovingWindow`.
 
 .. ipython:: python
 
@@ -777,7 +777,7 @@ columns by reshaping and indexing:
 Aggregation
 -----------
 
-Once the ``Rolling``, ``Expanding`` or ``EWM`` objects have been created, several methods are available to
+Once the ``Rolling``, ``Expanding`` or ``ExponentialMovingWindow`` objects have been created, several methods are available to
 perform multiple computations on the data. These operations are similar to the :ref:`aggregating API <basics.aggregate>`,
 :ref:`groupby API <groupby.aggregate>`, and :ref:`resample API <timeseries.aggregate>`.
 
@@ -971,7 +971,7 @@ Exponentially weighted windows
 
 A related set of functions are exponentially weighted versions of several of
 the above statistics. A similar interface to ``.rolling`` and ``.expanding`` is accessed
-through the ``.ewm`` method to receive an :class:`~EWM` object.
+through the ``.ewm`` method to receive an :class:`~ExponentialMovingWindow` object.
 A number of expanding EW (exponentially weighted)
 methods are provided:
 
@@ -980,11 +980,11 @@ methods are provided:
     :header: "Function", "Description"
     :widths: 20, 80
 
-    :meth:`~EWM.mean`, EW moving average
-    :meth:`~EWM.var`, EW moving variance
-    :meth:`~EWM.std`, EW moving standard deviation
-    :meth:`~EWM.corr`, EW moving correlation
-    :meth:`~EWM.cov`, EW moving covariance
+    :meth:`~ExponentialMovingWindow.mean`, EW moving average
+    :meth:`~ExponentialMovingWindow.var`, EW moving variance
+    :meth:`~ExponentialMovingWindow.std`, EW moving standard deviation
+    :meth:`~ExponentialMovingWindow.corr`, EW moving correlation
+    :meth:`~ExponentialMovingWindow.cov`, EW moving covariance
 
 In general, a weighted moving average is calculated as
 
@@ -1090,12 +1090,12 @@ Here is an example for a univariate time series:
    @savefig ewma_ex.png
    s.ewm(span=20).mean().plot(style='k')
 
-EWM has a ``min_periods`` argument, which has the same
+ExponentialMovingWindow has a ``min_periods`` argument, which has the same
 meaning it does for all the ``.expanding`` and ``.rolling`` methods:
 no output values will be set until at least ``min_periods`` non-null values
 are encountered in the (expanding) window.
 
-EWM also has an ``ignore_na`` argument, which determines how
+ExponentialMovingWindow also has an ``ignore_na`` argument, which determines how
 intermediate null values affect the calculation of the weights.
 When ``ignore_na=False`` (the default), weights are calculated based on absolute
 positions, so that intermediate null values affect the result.

doc/source/user_guide/cookbook.rst

@@ -1166,6 +1166,25 @@ Storing Attributes to a group node
    store.close()
    os.remove('test.h5')
 
+You can create or load a HDFStore in-memory  by passing the ``driver``
+parameter to PyTables. Changes are only written to disk when the HDFStore
+is closed.
+
+.. ipython:: python
+
+   store = pd.HDFStore('test.h5', 'w', diver='H5FD_CORE')
+
+   df = pd.DataFrame(np.random.randn(8, 3))
+   store['test'] = df
+
+   # only after closing the store, data is written to disk:
+   store.close()
+
+.. ipython:: python
+   :suppress:
+
+   os.remove('test.h5')
+
 .. _cookbook.binary:
 
 Binary files

doc/source/user_guide/timeseries.rst

@@ -235,6 +235,8 @@ inferred frequency upon creation:
 
     pd.DatetimeIndex(['2018-01-01', '2018-01-03', '2018-01-05'], freq='infer')
 
+.. _timeseries.converting.format:
+
 Providing a format argument
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -319,6 +321,12 @@ which can be specified. These are computed from the starting point specified by
    pd.to_datetime([1349720105100, 1349720105200, 1349720105300,
                    1349720105400, 1349720105500], unit='ms')
 
+.. note::
+
+   The ``unit`` parameter does not use the same strings as the ``format`` parameter
+   that was discussed :ref:`above<timeseries.converting.format>`). The
+   available units are listed on the documentation for :func:`pandas.to_datetime`.
+
 Constructing a :class:`Timestamp` or :class:`DatetimeIndex` with an epoch timestamp
 with the ``tz`` argument specified will currently localize the epoch timestamps to UTC
 first then convert the result to the specified time zone. However, this behavior

doc/source/whatsnew/v0.25.0.rst

@@ -1206,7 +1206,7 @@ Groupby/resample/rolling
 - Bug in :meth:`pandas.core.groupby.GroupBy.agg` where incorrect results are returned for uint64 columns. (:issue:`26310`)
 - Bug in :meth:`pandas.core.window.Rolling.median` and :meth:`pandas.core.window.Rolling.quantile` where MemoryError is raised with empty window (:issue:`26005`)
 - Bug in :meth:`pandas.core.window.Rolling.median` and :meth:`pandas.core.window.Rolling.quantile` where incorrect results are returned with ``closed='left'`` and ``closed='neither'`` (:issue:`26005`)
-- Improved :class:`pandas.core.window.Rolling`, :class:`pandas.core.window.Window` and :class:`pandas.core.window.EWM` functions to exclude nuisance columns from results instead of raising errors and raise a ``DataError`` only if all columns are nuisance (:issue:`12537`)
+- Improved :class:`pandas.core.window.Rolling`, :class:`pandas.core.window.Window` and :class:`pandas.core.window.ExponentialMovingWindow` functions to exclude nuisance columns from results instead of raising errors and raise a ``DataError`` only if all columns are nuisance (:issue:`12537`)
 - Bug in :meth:`pandas.core.window.Rolling.max` and :meth:`pandas.core.window.Rolling.min` where incorrect results are returned with an empty variable window (:issue:`26005`)
 - Raise a helpful exception when an unsupported weighted window function is used as an argument of :meth:`pandas.core.window.Window.aggregate` (:issue:`26597`)
 

doc/source/whatsnew/v1.1.0.rst

@@ -245,6 +245,22 @@ If needed you can adjust the bins with the argument ``offset`` (a Timedelta) tha
 
 For a full example, see: :ref:`timeseries.adjust-the-start-of-the-bins`.
 
+fsspec now used for filesystem handling
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For reading and writing to filesystems other than local and reading from HTTP(S),
+the optional dependency ``fsspec`` will be used to dispatch operations (:issue:`33452`).
+This will give unchanged
+functionality for S3 and GCS storage, which were already supported, but also add
+support for several other storage implementations such as `Azure Datalake and Blob`_,
+SSH, FTP, dropbox and github. For docs and capabilities, see the `fsspec docs`_.
+
+The existing capability to interface with S3 and GCS will be unaffected by this
+change, as ``fsspec`` will still bring in the same packages as before.
+
+.. _Azure Datalake and Blob: https://github.com/dask/adlfs
+
+.. _fsspec docs: https://filesystem-spec.readthedocs.io/en/latest/
 
 .. _whatsnew_110.enhancements.other:
 
@@ -292,11 +308,13 @@ Other enhancements
 - :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` now accept an ``errors`` argument (:issue:`22610`)
 - :meth:`groupby.transform` now allows ``func`` to be ``pad``, ``backfill`` and ``cumcount`` (:issue:`31269`).
 - :meth:`~pandas.io.json.read_json` now accepts `nrows` parameter. (:issue:`33916`).
+- :meth:`DataFrame.hist`, :meth:`Series.hist`, :meth:`core.groupby.DataFrameGroupBy.hist`, and :meth:`core.groupby.SeriesGroupBy.hist` have gained the ``legend`` argument. Set to True to show a legend in the histogram. (:issue:`6279`)
 - :func:`concat` and :meth:`~DataFrame.append` now preserve extension dtypes, for example
   combining a nullable integer column with a numpy integer column will no longer
   result in object dtype but preserve the integer dtype (:issue:`33607`, :issue:`34339`).
 - :meth:`~pandas.io.gbq.read_gbq` now allows to disable progress bar (:issue:`33360`).
 - :meth:`~pandas.io.gbq.read_gbq` now supports the ``max_results`` kwarg from ``pandas-gbq`` (:issue:`34639`).
+- :meth:`Dataframe.cov` and :meth:`Series.cov` now support a new parameter ddof to support delta degrees of freedom as in the corresponding numpy methods (:issue:`34611`).
 - :meth:`DataFrame.to_html` and :meth:`DataFrame.to_string`'s ``col_space`` parameter now accepts a list of dict to change only some specific columns' width (:issue:`28917`).
 
 .. ---------------------------------------------------------------------------
@@ -700,7 +718,9 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | fastparquet     | 0.3.2           |         |
 +-----------------+-----------------+---------+
-| gcsfs           | 0.2.2           |         |
+| fsspec          | 0.7.4           |         |
++-----------------+-----------------+---------+
+| gcsfs           | 0.6.0           |    X    |
 +-----------------+-----------------+---------+
 | lxml            | 3.8.0           |         |
 +-----------------+-----------------+---------+
@@ -716,7 +736,7 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | pytables        | 3.4.3           |    X    |
 +-----------------+-----------------+---------+
-| s3fs            | 0.3.0           |         |
+| s3fs            | 0.4.0           |    X    |
 +-----------------+-----------------+---------+
 | scipy           | 1.2.0           |    X    |
 +-----------------+-----------------+---------+
@@ -956,6 +976,7 @@ MultiIndex
         df.loc[(['b', 'a'], [2, 1]), :]
 
 - Bug in :meth:`MultiIndex.intersection` was not guaranteed to preserve order when ``sort=False``. (:issue:`31325`)
+- Bug in :meth:`DataFrame.truncate` was dropping :class:`MultiIndex` names. (:issue:`34564`)
 
 .. ipython:: python
 

environment.yml

@@ -98,7 +98,9 @@ dependencies:
 
   - pyqt>=5.9.2  # pandas.read_clipboard
   - pytables>=3.4.3  # pandas.read_hdf, DataFrame.to_hdf
-  - s3fs  # pandas.read_csv... when using 's3://...' path
+  - s3fs>=0.4.0  # file IO when using 's3://...' path
+  - fsspec>=0.7.4  # for generic remote file operations
+  - gcsfs>=0.6.0  # file IO when using 'gcs://...' path
   - sqlalchemy  # pandas.read_sql, DataFrame.to_sql
   - xarray  # DataFrame.to_xarray
   - cftime  # Needed for downstream xarray.CFTimeIndex test