Merge pull request #110 from pandas-dev/master

Sync Fork from Upstream Repo

Author: sthagen

asv_bench/benchmarks/io/parsers.py

@@ -2,7 +2,7 @@
 
 try:
     from pandas._libs.tslibs.parsing import (
-        _concat_date_cols,
+        concat_date_cols,
         _does_string_look_like_datetime,
     )
 except ImportError:
@@ -39,4 +39,4 @@ def setup(self, value, dim):
             )
 
     def time_check_concat(self, value, dim):
-        _concat_date_cols(self.object)
+        concat_date_cols(self.object)

asv_bench/benchmarks/rolling.py

@@ -150,19 +150,18 @@ def time_quantile(self, constructor, window, dtype, percentile, interpolation):
         self.roll.quantile(percentile, interpolation=interpolation)
 
 
-class PeakMemFixed:
-    def setup(self):
-        N = 10
-        arr = 100 * np.random.random(N)
-        self.roll = pd.Series(arr).rolling(10)
-
-    def peakmem_fixed(self):
-        # GH 25926
-        # This is to detect memory leaks in rolling operations.
-        # To save time this is only ran on one method.
-        # 6000 iterations is enough for most types of leaks to be detected
-        for x in range(6000):
-            self.roll.max()
+class PeakMemFixedWindowMinMax:
+
+    params = ["min", "max"]
+
+    def setup(self, operation):
+        N = int(1e6)
+        arr = np.random.random(N)
+        self.roll = pd.Series(arr).rolling(2)
+
+    def peakmem_fixed(self, operation):
+        for x in range(5):
+            getattr(self.roll, operation)()
 
 
 class ForwardWindowMethods:

ci/deps/azure-36-minimum_versions.yaml

@@ -1,6 +1,5 @@
 name: pandas-dev
 channels:
-  - defaults
   - conda-forge
 dependencies:
   - python=3.6.1
@@ -19,12 +18,12 @@ dependencies:
   - jinja2=2.8
   - numba=0.46.0
   - numexpr=2.6.2
-  - numpy=1.13.3
+  - numpy=1.15.4
   - openpyxl=2.5.7
   - pytables=3.4.3
   - python-dateutil=2.7.3
   - pytz=2017.2
-  - scipy=0.19.0
+  - scipy=1.2
   - xlrd=1.1.0
   - xlsxwriter=0.9.8
   - xlwt=1.2.0

ci/deps/azure-37-numpydev.yaml

@@ -14,8 +14,7 @@ dependencies:
   - pytz
   - pip
   - pip:
-    - cython==0.29.16
-    # GH#33507 cython 3.0a1 is causing TypeErrors 2020-04-13
+    - cython>=0.29.16
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"
     - "--pre"

ci/deps/azure-macos-36.yaml

@@ -19,7 +19,7 @@ dependencies:
   - matplotlib=2.2.3
   - nomkl
   - numexpr
-  - numpy=1.14
+  - numpy=1.15.4
   - openpyxl
   - pyarrow>=0.13.0
   - pytables

ci/setup_env.sh

@@ -128,7 +128,7 @@ conda list pandas
 echo "[Build extensions]"
 python setup.py build_ext -q -i -j2
 
-# XXX: Some of our environments end up with old versions of pip (10.x)
+# TODO: Some of our environments end up with old versions of pip (10.x)
 # Adding a new enough version of pip to the requirements explodes the
 # solve time. Just using pip to update itself.
 # - py35_macos

conda.recipe/meta.yaml

@@ -20,12 +20,12 @@ requirements:
     - cython
     - numpy
     - setuptools >=3.3
-    - python-dateutil >=2.5.0
+    - python-dateutil >=2.7.3
     - pytz
   run:
     - python {{ python }}
     - {{ pin_compatible('numpy') }}
-    - python-dateutil >=2.5.0
+    - python-dateutil >=2.7.3
     - pytz
 
 test:

doc/source/development/contributing.rst

@@ -581,7 +581,7 @@ do not make sudden changes to the code that could have the potential to break
 a lot of user code as a result, that is, we need it to be as *backwards compatible*
 as possible to avoid mass breakages.
 
-Additional standards are outlined on the `pandas code style guide <code_style>`_
+Additional standards are outlined on the :ref:`pandas code style guide <code_style>`
 
 Optional dependencies
 ---------------------

doc/source/getting_started/install.rst

@@ -220,7 +220,7 @@ Dependencies
 Package                                                          Minimum supported version
 ================================================================ ==========================
 `setuptools <https://setuptools.readthedocs.io/en/latest/>`__    24.2.0
-`NumPy <https://www.numpy.org>`__                                1.13.3
+`NumPy <https://www.numpy.org>`__                                1.15.4
 `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__ 2.7.3
 `pytz <https://pypi.org/project/pytz/>`__                        2017.2
 ================================================================ ==========================

doc/source/reference/general_utility_functions.rst

@@ -35,9 +35,12 @@ Exceptions and warnings
 .. autosummary::
    :toctree: api/
 
+   errors.AccessorRegistrationWarning
    errors.DtypeWarning
    errors.EmptyDataError
    errors.OutOfBoundsDatetime
+   errors.MergeError
+   errors.NumbaUtilError
    errors.ParserError
    errors.ParserWarning
    errors.PerformanceWarning

doc/source/reference/groupby.rst

@@ -36,8 +36,10 @@ Function application
 
    GroupBy.apply
    GroupBy.agg
-   GroupBy.aggregate
-   GroupBy.transform
+   SeriesGroupBy.aggregate
+   DataFrameGroupBy.aggregate
+   SeriesGroupBy.transform
+   DataFrameGroupBy.transform
    GroupBy.pipe
 
 Computations / descriptive stats

doc/source/user_guide/basics.rst

@@ -1781,6 +1781,31 @@ used to sort a pandas object by its index levels.
    # Series
    unsorted_df['three'].sort_index()
 
+.. _basics.sort_index_key:
+
+.. versionadded:: 1.1.0
+
+Sorting by index also supports a ``key`` parameter that takes a callable
+function to apply to the index being sorted. For `MultiIndex` objects,
+the key is applied per-level to the levels specified by `level`.
+
+.. ipython:: python
+
+   s1 = pd.DataFrame({
+       "a": ['B', 'a', 'C'],
+       "b": [1, 2, 3],
+       "c": [2, 3, 4]
+   }).set_index(list("ab"))
+   s1
+
+.. ipython:: python
+
+   s1.sort_index(level="a")
+   s1.sort_index(level="a", key=lambda idx: idx.str.lower())
+
+For information on key sorting by value, see :ref:`value sorting
+<basics.sort_value_key>`.
+
 .. _basics.sort_values:
 
 By values
@@ -1813,6 +1838,39 @@ argument:
    s.sort_values()
    s.sort_values(na_position='first')
 
+.. _basics.sort_value_key:
+
+.. versionadded:: 1.1.0
+
+Sorting also supports a ``key`` parameter that takes a callable function
+to apply to the values being sorted.
+
+.. ipython:: python
+
+   s1 = pd.Series(['B', 'a', 'C'])
+
+.. ipython:: python
+
+   s1.sort_values()
+   s1.sort_values(key=lambda x: x.str.lower())
+
+`key` will be given the :class:`Series` of values and should return a ``Series``
+or array of the same shape with the transformed values. For `DataFrame` objects,
+the key is applied per column, so the key should still expect a Series and return
+a Series, e.g.
+
+.. ipython:: python
+
+   df = pd.DataFrame({"a": ['B', 'a', 'C'], "b": [1, 2, 3]})
+
+.. ipython:: python
+
+   df.sort_values(by='a')
+   df.sort_values(by='a', key=lambda col: col.str.lower())
+
+The name or type of each column can be used to apply different functions to
+different columns.
+
 .. _basics.sort_indexes_and_values:
 
 By indexes and values

doc/source/user_guide/enhancingperf.rst

@@ -396,7 +396,7 @@ Consider the following toy example of doubling each observation:
    1000 loops, best of 3: 233 us per loop
 
    # Custom function with numba
-   In [7]: %timeit (df['col1_doubled'] = double_every_value_withnumba(df['a'].to_numpy())
+   In [7]: %timeit df['col1_doubled'] = double_every_value_withnumba(df['a'].to_numpy())
    1000 loops, best of 3: 145 us per loop
 
 Caveats
@@ -599,13 +599,6 @@ identifier.
 The ``inplace`` keyword determines whether this assignment will performed
 on the original ``DataFrame`` or return a copy with the new column.
 
-.. warning::
-
-   For backwards compatibility, ``inplace`` defaults to ``True`` if not
-   specified. This will change in a future version of pandas - if your
-   code depends on an inplace assignment you should update to explicitly
-   set ``inplace=True``.
-
 .. ipython:: python
 
    df = pd.DataFrame(dict(a=range(5), b=range(5, 10)))
@@ -614,7 +607,7 @@ on the original ``DataFrame`` or return a copy with the new column.
    df.eval('a = 1', inplace=True)
    df
 
-When ``inplace`` is set to ``False``, a copy of the ``DataFrame`` with the
+When ``inplace`` is set to ``False``, the default, a copy of the ``DataFrame`` with the
 new or modified columns is returned and the original frame is unchanged.
 
 .. ipython:: python
@@ -653,11 +646,6 @@ whether the query modifies the original frame.
    df.query('a > 2', inplace=True)
    df
 
-.. warning::
-
-   Unlike with ``eval``, the default value for ``inplace`` for ``query``
-   is ``False``.  This is consistent with prior versions of pandas.
-
 Local variables
 ~~~~~~~~~~~~~~~
 

doc/source/user_guide/indexing.rst

@@ -881,7 +881,7 @@ The operators are: ``|`` for ``or``, ``&`` for ``and``, and ``~`` for ``not``.
 These **must** be grouped by using parentheses, since by default Python will
 evaluate an expression such as ``df['A'] > 2 & df['B'] < 3`` as
 ``df['A'] > (2 & df['B']) < 3``, while the desired evaluation order is
-``(df['A > 2) & (df['B'] < 3)``.
+``(df['A'] > 2) & (df['B'] < 3)``.
 
 Using a boolean vector to index a Series works exactly as in a NumPy ndarray:
 

doc/source/user_guide/reshaping.rst

@@ -471,16 +471,24 @@ If ``crosstab`` receives only two Series, it will provide a frequency table.
 
     pd.crosstab(df['A'], df['B'])
 
-Any input passed containing ``Categorical`` data will have **all** of its
-categories included in the cross-tabulation, even if the actual data does
-not contain any instances of a particular category.
+``crosstab`` can also be implemented
+to ``Categorical`` data.
 
 .. ipython:: python
 
     foo = pd.Categorical(['a', 'b'], categories=['a', 'b', 'c'])
     bar = pd.Categorical(['d', 'e'], categories=['d', 'e', 'f'])
     pd.crosstab(foo, bar)
 
+If you want to include **all** of data categories even if the actual data does
+not contain any instances of a particular category, you should set ``dropna=False``.
+
+For example:
+
+.. ipython:: python
+
+    pd.crosstab(foo, bar, dropna=False)
+
 Normalization
 ~~~~~~~~~~~~~