Merge branch 'master' into PR_TOOL_MERGE_PR_19714

Author: jreback

ci/lint.sh

@@ -111,6 +111,15 @@ if [ "$LINT" ]; then
         RET=1
     fi
 
+    # Check for the following code in the extension array base tests
+    # tm.assert_frame_equal
+    # tm.assert_series_equal
+    grep -r -E --include '*.py' --exclude base.py 'tm.assert_(series|frame)_equal' pandas/tests/extension/base
+
+    if [ $? = "0" ]; then
+        RET=1
+    fi
+
     echo "Check for invalid testing DONE"
 
     # Check for imports from pandas.core.common instead

doc/source/whatsnew/v0.23.0.txt

@@ -295,8 +295,10 @@ Other Enhancements
 - ``IntervalIndex.astype`` now supports conversions between subtypes when passed an ``IntervalDtype`` (:issue:`19197`)
 - :class:`IntervalIndex` and its associated constructor methods (``from_arrays``, ``from_breaks``, ``from_tuples``) have gained a ``dtype`` parameter (:issue:`19262`)
 - Added :func:`SeriesGroupBy.is_monotonic_increasing` and :func:`SeriesGroupBy.is_monotonic_decreasing` (:issue:`17015`)
+- For subclassed ``DataFrames``, :func:`DataFrame.apply` will now preserve the ``Series`` subclass (if defined) when passing the data to the applied function (:issue:`19822`)
 - :func:`DataFrame.from_dict` now accepts a ``columns`` argument that can be used to specify the column names when ``orient='index'`` is used (:issue:`18529`)
 
+
 .. _whatsnew_0230.api_breaking:
 
 Backwards incompatible API changes
@@ -730,7 +732,8 @@ Categorical
   ``self`` but in a different order (:issue:`19551`)
 - Bug in :meth:`Index.astype` with a categorical dtype where the resultant index is not converted to a :class:`CategoricalIndex` for all types of index (:issue:`18630`)
 - Bug in :meth:`Series.astype` and ``Categorical.astype()`` where an existing categorical data does not get updated (:issue:`10696`, :issue:`18593`)
-- Bug in :class:`Index` constructor with ``dtype=CategoricalDtype(...)`` where ``categories`` and ``ordered`` are not maintained (:issue:`19032`)
+- Bug in :class:`Index` constructor with ``dtype=CategoricalDtype(...)`` where ``categories`` and ``ordered`` are not maintained (issue:`19032`)
+- Bug in :class:`Series` constructor with scalar and ``dtype=CategoricalDtype(...)`` where ``categories`` and ``ordered`` are not maintained (issue:`19565`)
 
 Datetimelike
 ^^^^^^^^^^^^
@@ -909,3 +912,4 @@ Other
 ^^^^^
 
 - Improved error message when attempting to use a Python keyword as an identifier in a ``numexpr`` backed query (:issue:`18221`)
+- Bug in accessing a :func:`pandas.get_option`, which raised ``KeyError`` rather than ``OptionError`` when looking up a non-existant option key in some cases (:issue:`19789`)

pandas/core/apply.py

@@ -162,7 +162,7 @@ def apply_empty_result(self):
                 pass
 
         if reduce:
-            return Series(np.nan, index=self.agg_axis)
+            return self.obj._constructor_sliced(np.nan, index=self.agg_axis)
         else:
             return self.obj.copy()
 
@@ -175,11 +175,13 @@ def apply_raw(self):
             result = np.apply_along_axis(self.f, self.axis, self.values)
 
         # TODO: mixed type case
-        from pandas import DataFrame, Series
         if result.ndim == 2:
-            return DataFrame(result, index=self.index, columns=self.columns)
+            return self.obj._constructor(result,
+                                         index=self.index,
+                                         columns=self.columns)
         else:
-            return Series(result, index=self.agg_axis)
+            return self.obj._constructor_sliced(result,
+                                                index=self.agg_axis)
 
     def apply_broadcast(self, target):
         result_values = np.empty_like(target.values)
@@ -232,7 +234,7 @@ def apply_standard(self):
                                           axis=self.axis,
                                           dummy=dummy,
                                           labels=labels)
-                return Series(result, index=labels)
+                return self.obj._constructor_sliced(result, index=labels)
             except Exception:
                 pass
 
@@ -291,8 +293,7 @@ def wrap_results(self):
             return self.wrap_results_for_axis()
 
         # dict of scalars
-        from pandas import Series
-        result = Series(results)
+        result = self.obj._constructor_sliced(results)
         result.index = self.res_index
 
         return result
@@ -379,7 +380,6 @@ def wrap_results_for_axis(self):
         # we have a non-series and don't want inference
         elif not isinstance(results[0], ABCSeries):
             from pandas import Series
-
             result = Series(results)
             result.index = self.res_index
 

pandas/core/config.py

@@ -196,7 +196,10 @@ def __getattr__(self, key):
         if prefix:
             prefix += "."
         prefix += key
-        v = object.__getattribute__(self, "d")[key]
+        try:
+            v = object.__getattribute__(self, "d")[key]
+        except KeyError:
+            raise OptionError("No such option")
         if isinstance(v, dict):
             return DictWrapper(v, prefix)
         else:

pandas/core/dtypes/cast.py

@@ -1178,7 +1178,7 @@ def construct_1d_arraylike_from_scalar(value, length, dtype):
         subarr = DatetimeIndex([value] * length, dtype=dtype)
     elif is_categorical_dtype(dtype):
         from pandas import Categorical
-        subarr = Categorical([value] * length)
+        subarr = Categorical([value] * length, dtype=dtype)
     else:
         if not isinstance(dtype, (np.dtype, type(np.dtype))):
             dtype = dtype.dtype

pandas/tests/dtypes/test_cast.py

@@ -22,7 +22,8 @@
     maybe_convert_string_to_object,
     maybe_convert_scalar,
     find_common_type,
-    construct_1d_object_array_from_listlike)
+    construct_1d_object_array_from_listlike,
+    construct_1d_arraylike_from_scalar)
 from pandas.core.dtypes.dtypes import (
     CategoricalDtype,
     DatetimeTZDtype,
@@ -422,3 +423,15 @@ def test_cast_1d_array(self, datum1, datum2):
     @pytest.mark.parametrize('val', [1, 2., None])
     def test_cast_1d_array_invalid_scalar(self, val):
         pytest.raises(TypeError, construct_1d_object_array_from_listlike, val)
+
+    def test_cast_1d_arraylike_from_scalar_categorical(self):
+        # GH 19565 - Categorical result from scalar did not maintain categories
+        # and ordering of the passed dtype
+        cats = ['a', 'b', 'c']
+        cat_type = CategoricalDtype(categories=cats, ordered=False)
+        expected = pd.Categorical(['a', 'a'], categories=cats)
+        result = construct_1d_arraylike_from_scalar('a', len(expected),
+                                                    cat_type)
+        tm.assert_categorical_equal(result, expected,
+                                    check_category_order=True,
+                                    check_dtype=True)

pandas/tests/extension/base/__init__.py

@@ -31,6 +31,14 @@ class TestMyDtype(BaseDtypeTests):
 Your class ``TestDtype`` will inherit all the tests defined on
 ``BaseDtypeTests``. pytest's fixture discover will supply your ``dtype``
 wherever the test requires it. You're free to implement additional tests.
+
+All the tests in these modules use ``self.assert_frame_equal`` or
+``self.assert_series_equal`` for dataframe or series comparisons. By default,
+they use the usual ``pandas.testing.assert_frame_equal`` and
+``pandas.testing.assert_series_equal``. You can override the checks used
+by defining the staticmethods ``assert_frame_equal`` and
+``assert_series_equal`` on your base test class.
+
 """
 from .casting import BaseCastingTests  # noqa
 from .constructors import BaseConstructorsTests  # noqa

pandas/tests/extension/base/base.py

@@ -0,0 +1,6 @@
+import pandas.util.testing as tm
+
+
+class BaseExtensionTests(object):
+    assert_series_equal = staticmethod(tm.assert_series_equal)
+    assert_frame_equal = staticmethod(tm.assert_frame_equal)

pandas/tests/extension/base/casting.py

@@ -1,8 +1,10 @@
 import pandas as pd
 from pandas.core.internals import ObjectBlock
 
+from .base import BaseExtensionTests
 
-class BaseCastingTests(object):
+
+class BaseCastingTests(BaseExtensionTests):
     """Casting to and from ExtensionDtypes"""
 
     def test_astype_object_series(self, all_data):

pandas/tests/extension/base/constructors.py

@@ -4,8 +4,10 @@
 import pandas.util.testing as tm
 from pandas.core.internals import ExtensionBlock
 
+from .base import BaseExtensionTests
 
-class BaseConstructorsTests(object):
+
+class BaseConstructorsTests(BaseExtensionTests):
 
     def test_series_constructor(self, data):
         result = pd.Series(data)

pandas/tests/extension/base/dtype.py

@@ -1,8 +1,10 @@
 import numpy as np
 import pandas as pd
 
+from .base import BaseExtensionTests
 
-class BaseDtypeTests(object):
+
+class BaseDtypeTests(BaseExtensionTests):
     """Base class for ExtensionDtype classes"""
 
     def test_name(self, dtype):

pandas/tests/extension/base/getitem.py

@@ -1,73 +1,74 @@
 import numpy as np
 
 import pandas as pd
-import pandas.util.testing as tm
 
+from .base import BaseExtensionTests
 
-class BaseGetitemTests(object):
+
+class BaseGetitemTests(BaseExtensionTests):
     """Tests for ExtensionArray.__getitem__."""
 
     def test_iloc_series(self, data):
         ser = pd.Series(data)
         result = ser.iloc[:4]
         expected = pd.Series(data[:4])
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
         result = ser.iloc[[0, 1, 2, 3]]
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
     def test_iloc_frame(self, data):
         df = pd.DataFrame({"A": data, 'B': np.arange(len(data))})
         expected = pd.DataFrame({"A": data[:4]})
 
         # slice -> frame
         result = df.iloc[:4, [0]]
-        tm.assert_frame_equal(result, expected)
+        self.assert_frame_equal(result, expected)
 
         # sequence -> frame
         result = df.iloc[[0, 1, 2, 3], [0]]
-        tm.assert_frame_equal(result, expected)
+        self.assert_frame_equal(result, expected)
 
         expected = pd.Series(data[:4], name='A')
 
         # slice -> series
         result = df.iloc[:4, 0]
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
         # sequence -> series
         result = df.iloc[:4, 0]
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
     def test_loc_series(self, data):
         ser = pd.Series(data)
         result = ser.loc[:3]
         expected = pd.Series(data[:4])
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
         result = ser.loc[[0, 1, 2, 3]]
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
     def test_loc_frame(self, data):
         df = pd.DataFrame({"A": data, 'B': np.arange(len(data))})
         expected = pd.DataFrame({"A": data[:4]})
 
         # slice -> frame
         result = df.loc[:3, ['A']]
-        tm.assert_frame_equal(result, expected)
+        self.assert_frame_equal(result, expected)
 
         # sequence -> frame
         result = df.loc[[0, 1, 2, 3], ['A']]
-        tm.assert_frame_equal(result, expected)
+        self.assert_frame_equal(result, expected)
 
         expected = pd.Series(data[:4], name='A')
 
         # slice -> series
         result = df.loc[:3, 'A']
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
         # sequence -> series
         result = df.loc[:3, 'A']
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
     def test_getitem_scalar(self, data):
         result = data[0]

pandas/tests/extension/base/interface.py

@@ -5,8 +5,10 @@
 from pandas.core.dtypes.common import is_extension_array_dtype
 from pandas.core.dtypes.dtypes import ExtensionDtype
 
+from .base import BaseExtensionTests
 
-class BaseInterfaceTests(object):
+
+class BaseInterfaceTests(BaseExtensionTests):
     """Tests that the basic interface is satisfied."""
     # ------------------------------------------------------------------------
     # Interface

pandas/tests/extension/base/methods.py

@@ -2,10 +2,11 @@
 import numpy as np
 
 import pandas as pd
-import pandas.util.testing as tm
 
+from .base import BaseExtensionTests
 
-class BaseMethodsTests(object):
+
+class BaseMethodsTests(BaseExtensionTests):
     """Various Series and DataFrame methods."""
 
     @pytest.mark.parametrize('dropna', [True, False])
@@ -19,13 +20,13 @@ def test_value_counts(self, all_data, dropna):
         result = pd.Series(all_data).value_counts(dropna=dropna).sort_index()
         expected = pd.Series(other).value_counts(dropna=dropna).sort_index()
 
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
     def test_count(self, data_missing):
         df = pd.DataFrame({"A": data_missing})
         result = df.count(axis='columns')
         expected = pd.Series([0, 1])
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
     def test_apply_simple_series(self, data):
         result = pd.Series(data).apply(id)

pandas/tests/extension/base/missing.py

@@ -3,8 +3,10 @@
 import pandas as pd
 import pandas.util.testing as tm
 
+from .base import BaseExtensionTests
 
-class BaseMissingTests(object):
+
+class BaseMissingTests(BaseExtensionTests):
     def test_isna(self, data_missing):
         if data_missing._can_hold_na:
             expected = np.array([True, False])
@@ -16,30 +18,30 @@ def test_isna(self, data_missing):
 
         result = pd.Series(data_missing).isna()
         expected = pd.Series(expected)
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
     def test_dropna_series(self, data_missing):
         ser = pd.Series(data_missing)
         result = ser.dropna()
         expected = ser.iloc[[1]]
-        tm.assert_series_equal(result, expected)
+        self.assert_series_equal(result, expected)
 
     def test_dropna_frame(self, data_missing):
         df = pd.DataFrame({"A": data_missing})
 
         # defaults
         result = df.dropna()
         expected = df.iloc[[1]]
-        tm.assert_frame_equal(result, expected)
+        self.assert_frame_equal(result, expected)
 
         # axis = 1
         result = df.dropna(axis='columns')
         expected = pd.DataFrame(index=[0, 1])
-        tm.assert_frame_equal(result, expected)
+        self.assert_frame_equal(result, expected)
 
         # multiple
         df = pd.DataFrame({"A": data_missing,
                            "B": [1, np.nan]})
         result = df.dropna()
         expected = df.iloc[:0]
-        tm.assert_frame_equal(result, expected)
+        self.assert_frame_equal(result, expected)