Merge branch 'master' of https://github.com/pandas-dev/pandas into doc-decorator

Author: HH-MWB

asv_bench/benchmarks/frame_methods.py

@@ -564,7 +564,7 @@ def setup(self):
 
     def time_frame_get_dtype_counts(self):
         with warnings.catch_warnings(record=True):
-            self.df._data.get_dtype_counts()
+            self.df.dtypes.value_counts()
 
     def time_info(self):
         self.df.info()

doc/source/getting_started/intro_tutorials/10_text_data.rst

@@ -199,7 +199,7 @@ names in the ``Name`` column. By using pandas string methods, the
 
 Next, we need to get the corresponding location, preferably the index
 label, in the table for which the name length is the largest. The
-:meth:`~Series.idxmax`` method does exactly that. It is not a string method and is
+:meth:`~Series.idxmax` method does exactly that. It is not a string method and is
 applied to integers, so no ``str`` is used.
 
 .. ipython:: python

doc/source/user_guide/computation.rst

@@ -312,15 +312,35 @@ We provide a number of common statistical functions:
     :meth:`~Rolling.median`, Arithmetic median of values
     :meth:`~Rolling.min`, Minimum
     :meth:`~Rolling.max`, Maximum
-    :meth:`~Rolling.std`, Bessel-corrected sample standard deviation
-    :meth:`~Rolling.var`, Unbiased variance
+    :meth:`~Rolling.std`, Sample standard deviation
+    :meth:`~Rolling.var`, Sample variance
     :meth:`~Rolling.skew`, Sample skewness (3rd moment)
     :meth:`~Rolling.kurt`, Sample kurtosis (4th moment)
     :meth:`~Rolling.quantile`, Sample quantile (value at %)
     :meth:`~Rolling.apply`, Generic apply
     :meth:`~Rolling.cov`, Unbiased covariance (binary)
     :meth:`~Rolling.corr`, Correlation (binary)
 
+.. _computation.window_variance.caveats:
+
+.. note::
+
+   Please note that :meth:`~Rolling.std` and :meth:`~Rolling.var` use the sample
+   variance formula by default, i.e. the sum of squared differences is divided by
+   ``window_size - 1`` and not by ``window_size`` during averaging. In statistics,
+   we use sample when the dataset is drawn from a larger population that we
+   don't have access to. Using it implies that the data in our window is a
+   random sample from the population, and we are interested not in the variance
+   inside the specific window but in the variance of some general window that
+   our windows represent. In this situation, using the sample variance formula
+   results in an unbiased estimator and so is preferred.
+
+   Usually, we are instead interested in the variance of each window as we slide
+   it over the data, and in this case we should specify ``ddof=0`` when calling
+   these methods to use population variance instead of sample variance. Using
+   sample variance under the circumstances would result in a biased estimator
+   of the variable we are trying to determine.
+
 .. _stats.rolling_apply:
 
 Rolling apply
@@ -848,15 +868,22 @@ Method summary
     :meth:`~Expanding.median`, Arithmetic median of values
     :meth:`~Expanding.min`, Minimum
     :meth:`~Expanding.max`, Maximum
-    :meth:`~Expanding.std`, Unbiased standard deviation
-    :meth:`~Expanding.var`, Unbiased variance
+    :meth:`~Expanding.std`, Sample standard deviation
+    :meth:`~Expanding.var`, Sample variance
     :meth:`~Expanding.skew`, Unbiased skewness (3rd moment)
     :meth:`~Expanding.kurt`, Unbiased kurtosis (4th moment)
     :meth:`~Expanding.quantile`, Sample quantile (value at %)
     :meth:`~Expanding.apply`, Generic apply
     :meth:`~Expanding.cov`, Unbiased covariance (binary)
     :meth:`~Expanding.corr`, Correlation (binary)
 
+.. note::
+
+   Using sample variance formulas for :meth:`~Expanding.std` and
+   :meth:`~Expanding.var` comes with the same caveats as using them with rolling
+   windows. See :ref:`this section <computation.window_variance.caveats>` for more
+   information.
+
 .. currentmodule:: pandas
 
 Aside from not having a ``window`` parameter, these functions have the same

doc/source/whatsnew/v0.14.0.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0140:
 
-v0.14.0 (May 31 , 2014)
------------------------
+Version 0.14.0 (May 31 , 2014)
+------------------------------
 
 {{ header }}
 
@@ -321,7 +321,7 @@ Text parsing API changes
 
 .. _whatsnew_0140.groupby:
 
-Groupby API changes
+GroupBy API changes
 ~~~~~~~~~~~~~~~~~~~
 
 More consistent behavior for some groupby methods:
@@ -473,8 +473,8 @@ Some other enhancements to the sql functions include:
 
 .. _whatsnew_0140.slicers:
 
-Multiindexing using slicers
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Multi-indexing using slicers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In 0.14.0 we added a new way to slice MultiIndexed objects.
 You can slice a MultiIndex by providing multiple indexers.

doc/source/whatsnew/v0.14.1.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0141:
 
-v0.14.1 (July 11, 2014)
------------------------
+Version 0.14.1 (July 11, 2014)
+------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.15.0.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0150:
 
-v0.15.0 (October 18, 2014)
---------------------------
+Version 0.15.0 (October 18, 2014)
+---------------------------------
 
 {{ header }}
 
@@ -105,7 +105,7 @@ For full docs, see the :ref:`categorical introduction <categorical>` and the
 
 .. _whatsnew_0150.timedeltaindex:
 
-TimedeltaIndex/Scalar
+TimedeltaIndex/scalar
 ^^^^^^^^^^^^^^^^^^^^^
 
 We introduce a new scalar type ``Timedelta``, which is a subclass of ``datetime.timedelta``, and behaves in a similar manner,
@@ -247,8 +247,8 @@ Additionally :meth:`~pandas.DataFrame.memory_usage` is an available method for a
 
 .. _whatsnew_0150.dt:
 
-.dt accessor
-^^^^^^^^^^^^
+Series.dt accessor
+^^^^^^^^^^^^^^^^^^
 
 ``Series`` has gained an accessor to succinctly return datetime like properties for the *values* of the Series, if its a datetime/period like Series. (:issue:`7207`)
 This will return a Series, indexed like the existing Series. See the :ref:`docs <basics.dt_accessors>`
@@ -600,7 +600,7 @@ Rolling/expanding moments improvements
 
 .. _whatsnew_0150.sql:
 
-Improvements in the SQL io module
+Improvements in the SQL IO module
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 - Added support for a ``chunksize`` parameter to ``to_sql`` function. This allows DataFrame to be written in chunks and avoid packet-size overflow errors (:issue:`8062`).

doc/source/whatsnew/v0.15.1.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0151:
 
-v0.15.1 (November 9, 2014)
---------------------------
+Version 0.15.1 (November 9, 2014)
+---------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.15.2.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0152:
 
-v0.15.2 (December 12, 2014)
----------------------------
+Version 0.15.2 (December 12, 2014)
+----------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.16.0.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0160:
 
-v0.16.0 (March 22, 2015)
-------------------------
+Version 0.16.0 (March 22, 2015)
+-------------------------------
 
 {{ header }}
 
@@ -218,7 +218,7 @@ Backwards incompatible API changes
 
 .. _whatsnew_0160.api_breaking.timedelta:
 
-Changes in Timedelta
+Changes in timedelta
 ^^^^^^^^^^^^^^^^^^^^
 
 In v0.15.0 a new scalar type ``Timedelta`` was introduced, that is a

doc/source/whatsnew/v0.16.1.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0161:
 
-v0.16.1 (May 11, 2015)
-----------------------
+Version 0.16.1 (May 11, 2015)
+-----------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.16.2.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0162:
 
-v0.16.2 (June 12, 2015)
------------------------
+Version 0.16.2 (June 12, 2015)
+------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.17.0.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0170:
 
-v0.17.0 (October 9, 2015)
--------------------------
+Version 0.17.0 (October 9, 2015)
+--------------------------------
 
 {{ header }}
 
@@ -181,8 +181,8 @@ Each method signature only includes relevant arguments. Currently, these are lim
 Additional methods for ``dt`` accessor
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-strftime
-""""""""
+Series.dt.strftime
+""""""""""""""""""
 
 We are now supporting a ``Series.dt.strftime`` method for datetime-likes to generate a formatted string (:issue:`10110`). Examples:
 
@@ -202,8 +202,8 @@ We are now supporting a ``Series.dt.strftime`` method for datetime-likes to gene
 
 The string format is as the python standard library and details can be found `here <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
 
-total_seconds
-"""""""""""""
+Series.dt.total_seconds
+"""""""""""""""""""""""
 
 ``pd.Series`` of type ``timedelta64`` has new method ``.dt.total_seconds()`` returning the duration of the timedelta in seconds (:issue:`10817`)
 

doc/source/whatsnew/v0.17.1.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0171:
 
-v0.17.1 (November 21, 2015)
----------------------------
+Version 0.17.1 (November 21, 2015)
+----------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v1.1.0.rst

@@ -174,8 +174,8 @@ Other API changes
 - Added :meth:`DataFrame.value_counts` (:issue:`5377`)
 - :meth:`Groupby.groups` now returns an abbreviated representation when called on large dataframes (:issue:`1135`)
 - ``loc`` lookups with an object-dtype :class:`Index` and an integer key will now raise ``KeyError`` instead of ``TypeError`` when key is missing (:issue:`31905`)
-- Using a :func:`pandas.api.indexers.BaseIndexer` with ``std``, ``var``, ``count``, ``skew``, ``cov``, ``corr`` will now raise a ``NotImplementedError`` (:issue:`32865`)
-- Using a :func:`pandas.api.indexers.BaseIndexer` with ``min``, ``max`` will now return correct results for any monotonic :func:`pandas.api.indexers.BaseIndexer` descendant (:issue:`32865`)
+- Using a :func:`pandas.api.indexers.BaseIndexer` with ``skew``, ``cov``, ``corr`` will now raise a ``NotImplementedError`` (:issue:`32865`)
+- Using a :func:`pandas.api.indexers.BaseIndexer` with ``count``, ``min``, ``max`` will now return correct results for any monotonic :func:`pandas.api.indexers.BaseIndexer` descendant (:issue:`32865`)
 - Added a :func:`pandas.api.indexers.FixedForwardWindowIndexer` class to support forward-looking windows during ``rolling`` operations.
 -
 
@@ -458,6 +458,7 @@ Datetimelike
 - Bug in :class:`Timestamp` arithmetic when adding or subtracting a ``np.ndarray`` with ``timedelta64`` dtype (:issue:`33296`)
 - Bug in :meth:`DatetimeIndex.to_period` not infering the frequency when called with no arguments (:issue:`33358`)
 - Bug in :meth:`DatetimeIndex.tz_localize` incorrectly retaining ``freq`` in some cases where the original freq is no longer valid (:issue:`30511`)
+- Bug in :meth:`DatetimeIndex.intersection` losing ``freq`` and timezone in some cases (:issue:`33604`)
 
 Timedelta
 ^^^^^^^^^
@@ -525,6 +526,7 @@ Indexing
 - Bug in `Series.__getitem__` with an integer key and a :class:`MultiIndex` with leading integer level failing to raise ``KeyError`` if the key is not present in the first level (:issue:`33355`)
 - Bug in :meth:`DataFrame.iloc` when slicing a single column-:class:`DataFrame`` with ``ExtensionDtype`` (e.g. ``df.iloc[:, :1]``) returning an invalid result (:issue:`32957`)
 - Bug in :meth:`DatetimeIndex.insert` and :meth:`TimedeltaIndex.insert` causing index ``freq`` to be lost when setting an element into an empty :class:`Series` (:issue:33573`)
+- Bug in :meth:`Series.__setitem__` with an :class:`IntervalIndex` and a list-like key of integers (:issue:`33473`)
 
 Missing
 ^^^^^^^
@@ -576,6 +578,8 @@ I/O
 - Bug in :meth:`read_excel` did not correctly handle multiple embedded spaces in OpenDocument text cells. (:issue:`32207`)
 - Bug in :meth:`read_json` was raising ``TypeError`` when reading a list of booleans into a Series. (:issue:`31464`)
 - Bug in :func:`pandas.io.json.json_normalize` where location specified by `record_path` doesn't point to an array. (:issue:`26284`)
+- :func:`pandas.read_hdf` has a more explicit error message when loading an
+  unsupported HDF file (:issue:`9539`)
 
 Plotting
 ^^^^^^^^
@@ -598,6 +602,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrameGroupBy.agg` with dictionary input losing ``ExtensionArray`` dtypes (:issue:`32194`)
 - Bug in :meth:`DataFrame.resample` where an ``AmbiguousTimeError`` would be raised when the resulting timezone aware :class:`DatetimeIndex` had a DST transition at midnight (:issue:`25758`)
 - Bug in :meth:`DataFrame.groupby` where a ``ValueError`` would be raised when grouping by a categorical column with read-only categories and ``sort=False`` (:issue:`33410`)
+- Bug in :meth:`GroupBy.first` and :meth:`GroupBy.last` where None is not preserved in object dtype (:issue:`32800`)
 
 Reshaping
 ^^^^^^^^^

pandas/_libs/__init__.py

@@ -6,9 +6,11 @@
     "Timedelta",
     "Timestamp",
     "iNaT",
+    "Interval",
 ]
 
 
+from pandas._libs.interval import Interval
 from pandas._libs.tslibs import (
     NaT,
     NaTType,

pandas/_libs/groupby.pyx

@@ -893,7 +893,9 @@ def group_last(rank_t[:, :] out,
             for j in range(K):
                 val = values[i, j]
 
-                if not checknull(val):
+                # None should not be treated like other NA-like
+                # so that it won't be converted to nan
+                if not checknull(val) or val is None:
                     # NB: use _treat_as_na here once
                     #  conditional-nogil is available.
                     nobs[lab, j] += 1
@@ -986,7 +988,9 @@ def group_nth(rank_t[:, :] out,
             for j in range(K):
                 val = values[i, j]
 
-                if not checknull(val):
+                # None should not be treated like other NA-like
+                # so that it won't be converted to nan
+                if not checknull(val) or val is None:
                     # NB: use _treat_as_na here once
                     #  conditional-nogil is available.
                     nobs[lab, j] += 1

pandas/_libs/parsers.pyx

@@ -55,9 +55,7 @@ from pandas.core.dtypes.common import (
     is_bool_dtype, is_object_dtype,
     is_datetime64_dtype,
     pandas_dtype, is_extension_array_dtype)
-from pandas.core.arrays import Categorical
 from pandas.core.dtypes.concat import union_categoricals
-import pandas.io.common as icom
 
 from pandas.compat import _import_lzma, _get_lzma_file
 from pandas.errors import (ParserError, DtypeWarning,
@@ -1149,7 +1147,8 @@ cdef class TextReader:
 
             # Method accepts list of strings, not encoded ones.
             true_values = [x.decode() for x in self.true_values]
-            cat = Categorical._from_inferred_categories(
+            array_type = dtype.construct_array_type()
+            cat = array_type._from_inferred_categories(
                 cats, codes, dtype, true_values=true_values)
             return cat, na_count
 

pandas/_libs/tslibs/offsets.pyx

@@ -157,8 +157,8 @@ cdef _wrap_timedelta_result(result):
     """
     if PyDelta_Check(result):
         # convert Timedelta back to a Tick
-        from pandas.tseries.offsets import _delta_to_tick
-        return _delta_to_tick(result)
+        from pandas.tseries.offsets import delta_to_tick
+        return delta_to_tick(result)
 
     return result