Merge branch 'master' of https://github.com/pandas-dev/pandas into less24024b

Author: jbrockmendel

azure-pipelines.yml

@@ -43,6 +43,7 @@ jobs:
       ci/incremental/install_miniconda.sh
       ci/incremental/setup_conda_environment.sh
     displayName: 'Set up environment'
+    condition: true
 
   # Do not require pandas
   - script: |

ci/code_checks.sh

@@ -158,7 +158,12 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     # RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check that no file in the repo contains tailing whitespaces' ; echo $MSG
-    invgrep --exclude="*.svg" -RI "\s$" *
+    set -o pipefail
+    if [[ "$AZURE" == "true" ]]; then
+        ! grep -n --exclude="*.svg" -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
+    else
+        ! grep -n --exclude="*.svg" -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
+    fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi
 

doc/source/whatsnew/v0.24.0.rst

@@ -1318,6 +1318,7 @@ Datetimelike
 - Bug in :class:`DatetimeIndex` where constructing a :class:`DatetimeIndex` from a :class:`Categorical` or :class:`CategoricalIndex` would incorrectly drop timezone information (:issue:`18664`)
 - Bug in :class:`DatetimeIndex` and :class:`TimedeltaIndex` where indexing with ``Ellipsis`` would incorrectly lose the index's ``freq`` attribute (:issue:`21282`)
 - Clarified error message produced when passing an incorrect ``freq`` argument to :class:`DatetimeIndex` with ``NaT`` as the first entry in the passed data (:issue:`11587`)
+- Bug in :func:`to_datetime` where ``box`` and ``utc`` arguments were ignored when passing a :class:`DataFrame` or ``dict`` of unit mappings (:issue:`23760`)
 
 Timedelta
 ^^^^^^^^^
@@ -1361,6 +1362,7 @@ Timezones
 - Bug in :class:`DatetimeIndex` constructor where ``NaT`` and ``dateutil.tz.tzlocal`` would raise an ``OutOfBoundsDatetime`` error (:issue:`23807`)
 - Bug in :meth:`DatetimeIndex.tz_localize` and :meth:`Timestamp.tz_localize` with ``dateutil.tz.tzlocal`` near a DST transition that would return an incorrectly localized datetime (:issue:`23807`)
 - Bug in :class:`Timestamp` constructor where a ``dateutil.tz.tzutc`` timezone passed with a ``datetime.datetime`` argument would be converted to a ``pytz.UTC`` timezone (:issue:`23807`)
+- Bug in :func:`to_datetime` where ``utc=True`` was not respected when specifying a ``unit`` and ``errors='ignore'`` (:issue:`23758`)
 
 Offsets
 ^^^^^^^

pandas/core/arrays/categorical.py

@@ -341,9 +341,6 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
             elif categories is not None or ordered is not None:
                 raise ValueError("Cannot specify both `dtype` and `categories`"
                                  " or `ordered`.")
-
-            categories = dtype.categories
-
         elif is_categorical(values):
             # If no "dtype" was passed, use the one from "values", but honor
             # the "ordered" and "categories" arguments
@@ -355,19 +352,17 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
             if (isinstance(values, (ABCSeries, ABCIndexClass)) and
                isinstance(values._values, type(self))):
                 values = values._values.codes.copy()
-                if categories is None:
-                    categories = dtype.categories
                 fastpath = True
-
         else:
             # If dtype=None and values is not categorical, create a new dtype
             dtype = CategoricalDtype(categories, ordered)
 
-        # At this point, dtype is always a CategoricalDtype
+        # At this point, dtype is always a CategoricalDtype and you should not
+        # use categories and ordered seperately.
         # if dtype.categories is None, we are inferring
 
         if fastpath:
-            self._codes = coerce_indexer_dtype(values, categories)
+            self._codes = coerce_indexer_dtype(values, dtype.categories)
             self._dtype = self._dtype.update_dtype(dtype)
             return
 
@@ -379,7 +374,6 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
         if is_categorical_dtype(values):
             if dtype.categories is None:
                 dtype = CategoricalDtype(values.categories, dtype.ordered)
-
         elif not isinstance(values, (ABCIndexClass, ABCSeries)):
             # sanitize_array coerces np.nan to a string under certain versions
             # of numpy

pandas/core/tools/datetimes.py

@@ -204,7 +204,12 @@ def _convert_listlike_datetimes(arg, box, format, name=None, tz=None,
         if box:
             if errors == 'ignore':
                 from pandas import Index
-                return Index(result, name=name)
+                result = Index(result, name=name)
+                # GH 23758: We may still need to localize the result with tz
+                try:
+                    return result.tz_localize(tz)
+                except AttributeError:
+                    return result
 
             return DatetimeIndex(result, tz=tz, name=name)
         return result
@@ -572,7 +577,7 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
             values = convert_listlike(arg._values, True, format)
             result = Series(values, index=arg.index, name=arg.name)
     elif isinstance(arg, (ABCDataFrame, compat.MutableMapping)):
-        result = _assemble_from_unit_mappings(arg, errors=errors)
+        result = _assemble_from_unit_mappings(arg, errors, box, tz)
     elif isinstance(arg, ABCIndexClass):
         cache_array = _maybe_cache(arg, format, cache, convert_listlike)
         if not cache_array.empty:
@@ -618,7 +623,7 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
              }
 
 
-def _assemble_from_unit_mappings(arg, errors):
+def _assemble_from_unit_mappings(arg, errors, box, tz):
     """
     assemble the unit specified fields from the arg (DataFrame)
     Return a Series for actual parsing
@@ -631,6 +636,11 @@ def _assemble_from_unit_mappings(arg, errors):
         - If 'raise', then invalid parsing will raise an exception
         - If 'coerce', then invalid parsing will be set as NaT
         - If 'ignore', then invalid parsing will return the input
+    box : boolean
+
+        - If True, return a DatetimeIndex
+        - If False, return an array
+    tz : None or 'utc'
 
     Returns
     -------
@@ -683,7 +693,7 @@ def coerce(values):
               coerce(arg[unit_rev['month']]) * 100 +
               coerce(arg[unit_rev['day']]))
     try:
-        values = to_datetime(values, format='%Y%m%d', errors=errors)
+        values = to_datetime(values, format='%Y%m%d', errors=errors, utc=tz)
     except (TypeError, ValueError) as e:
         raise ValueError("cannot assemble the "
                          "datetimes: {error}".format(error=e))
@@ -698,7 +708,8 @@ def coerce(values):
             except (TypeError, ValueError) as e:
                 raise ValueError("cannot assemble the datetimes [{value}]: "
                                  "{error}".format(value=value, error=e))
-
+    if not box:
+        return values.values
     return values
 
 

pandas/tests/indexes/datetimes/test_tools.py

@@ -949,6 +949,33 @@ def test_dataframe_dtypes(self, cache):
         with pytest.raises(ValueError):
             to_datetime(df, cache=cache)
 
+    def test_dataframe_box_false(self):
+        # GH 23760
+        df = pd.DataFrame({'year': [2015, 2016],
+                           'month': [2, 3],
+                           'day': [4, 5]})
+        result = pd.to_datetime(df, box=False)
+        expected = np.array(['2015-02-04', '2016-03-05'],
+                            dtype='datetime64[ns]')
+        tm.assert_numpy_array_equal(result, expected)
+
+    def test_dataframe_utc_true(self):
+        # GH 23760
+        df = pd.DataFrame({'year': [2015, 2016],
+                           'month': [2, 3],
+                           'day': [4, 5]})
+        result = pd.to_datetime(df, utc=True)
+        expected = pd.Series(np.array(['2015-02-04', '2016-03-05'],
+                             dtype='datetime64[ns]')).dt.tz_localize('UTC')
+        tm.assert_series_equal(result, expected)
+
+    def test_to_datetime_errors_ignore_utc_true(self):
+        # GH 23758
+        result = pd.to_datetime([1], unit='s', box=True, utc=True,
+                                errors='ignore')
+        expected = DatetimeIndex(['1970-01-01 00:00:01'], tz='UTC')
+        tm.assert_index_equal(result, expected)
+
 
 class TestToDatetimeMisc(object):
     def test_to_datetime_barely_out_of_bounds(self):

pandas/tests/indexing/multiindex/test_getitem.py

@@ -4,8 +4,9 @@
 from pandas.compat import lrange, range, u, zip
 
 import pandas as pd
-from pandas import DataFrame, Index, MultiIndex, Series, date_range
+from pandas import DataFrame, Index, MultiIndex, Series
 import pandas.core.common as com
+from pandas.core.indexing import IndexingError
 from pandas.util import testing as tm
 
 
@@ -27,29 +28,6 @@ def test_series_getitem_multiindex(access_method, level1_value, expected):
     tm.assert_series_equal(result, expected)
 
 
-def test_series_getitem_multiindex_xs():
-    # GH6258
-    dt = list(date_range('20130903', periods=3))
-    idx = MultiIndex.from_product([list('AB'), dt])
-    s = Series([1, 3, 4, 1, 3, 4], index=idx)
-
-    result = s.xs('20130903', level=1)
-    expected = Series([1, 1], index=list('AB'))
-    tm.assert_series_equal(result, expected)
-
-
-def test_series_getitem_multiindex_xs_by_label():
-    # GH5684
-    idx = MultiIndex.from_tuples([('a', 'one'), ('a', 'two'), ('b', 'one'),
-                                  ('b', 'two')])
-    s = Series([1, 2, 3, 4], index=idx)
-    s.index.set_names(['L1', 'L2'], inplace=True)
-    result = s.xs('one', level='L2')
-    expected = Series([1, 3], index=['a', 'b'])
-    expected.index.set_names(['L1'], inplace=True)
-    tm.assert_series_equal(result, expected)
-
-
 @pytest.mark.parametrize('level0_value', ['D', 'A'])
 def test_getitem_duplicates_multiindex(level0_value):
     # GH 5725 the 'A' happens to be a valid Timestamp so the doesn't raise
@@ -145,63 +123,83 @@ def test_getitem_duplicates_multiindex_non_scalar_type_object():
 
 
 def test_getitem_simple(multiindex_dataframe_random_data):
-    frame = multiindex_dataframe_random_data
-    df = frame.T
-
-    col = df['foo', 'one']
-    tm.assert_almost_equal(col.values, df.values[:, 0])
-    msg = r"\('foo', 'four'\)"
-    with pytest.raises(KeyError, match=msg):
-        df[('foo', 'four')]
-    msg = "'foobar'"
-    with pytest.raises(KeyError, match=msg):
-        df['foobar']
+    df = multiindex_dataframe_random_data.T
+    expected = df.values[:, 0]
+    result = df['foo', 'one'].values
+    tm.assert_almost_equal(result, expected)
 
 
-@pytest.mark.filterwarnings("ignore:\\n.ix:DeprecationWarning")
-def test_series_getitem(multiindex_year_month_day_dataframe_random_data):
-    ymd = multiindex_year_month_day_dataframe_random_data
-    s = ymd['A']
-
-    result = s[2000, 3]
+@pytest.mark.parametrize('indexer,msg', [
+    (lambda df: df[('foo', 'four')], r"\('foo', 'four'\)"),
+    (lambda df: df['foobar'], "'foobar'")
+])
+def test_getitem_simple_key_error(
+        multiindex_dataframe_random_data, indexer, msg):
+    df = multiindex_dataframe_random_data.T
+    with pytest.raises(KeyError, match=msg):
+        indexer(df)
 
-    # TODO(wesm): unused?
-    # result2 = s.loc[2000, 3]
 
+@pytest.mark.parametrize('indexer', [
+    lambda s: s[2000, 3],
+    lambda s: s.loc[2000, 3]
+])
+def test_series_getitem(
+        multiindex_year_month_day_dataframe_random_data, indexer):
+    s = multiindex_year_month_day_dataframe_random_data['A']
     expected = s.reindex(s.index[42:65])
     expected.index = expected.index.droplevel(0).droplevel(0)
+
+    result = indexer(s)
     tm.assert_series_equal(result, expected)
 
-    result = s[2000, 3, 10]
-    expected = s[49]
+
+@pytest.mark.parametrize('indexer', [
+    lambda s: s[2000, 3, 10],
+    lambda s: s.loc[2000, 3, 10]
+])
+def test_series_getitem_returns_scalar(
+        multiindex_year_month_day_dataframe_random_data, indexer):
+    s = multiindex_year_month_day_dataframe_random_data['A']
+    expected = s.iloc[49]
+
+    result = indexer(s)
     assert result == expected
 
-    # fancy
+
+@pytest.mark.filterwarnings("ignore:\\n.ix:DeprecationWarning")
+@pytest.mark.parametrize('indexer', [
+    lambda s: s.loc[[(2000, 3, 10), (2000, 3, 13)]],
+    lambda s: s.ix[[(2000, 3, 10), (2000, 3, 13)]]
+])
+def test_series_getitem_fancy(
+        multiindex_year_month_day_dataframe_random_data, indexer):
+    s = multiindex_year_month_day_dataframe_random_data['A']
     expected = s.reindex(s.index[49:51])
-    result = s.loc[[(2000, 3, 10), (2000, 3, 13)]]
-    tm.assert_series_equal(result, expected)
 
-    result = s.ix[[(2000, 3, 10), (2000, 3, 13)]]
+    result = indexer(s)
     tm.assert_series_equal(result, expected)
 
-    # key error
-    msg = "356"
-    with pytest.raises(KeyError, match=msg):
-        s.__getitem__((2000, 3, 4))
-
 
-def test_series_getitem_corner(
-        multiindex_year_month_day_dataframe_random_data):
-    ymd = multiindex_year_month_day_dataframe_random_data
-    s = ymd['A']
+@pytest.mark.parametrize('indexer,error,msg', [
+    (lambda s: s.__getitem__((2000, 3, 4)), KeyError, '356'),
+    (lambda s: s[(2000, 3, 4)], KeyError, '356'),
+    (lambda s: s.loc[(2000, 3, 4)], IndexingError, 'Too many indexers'),
+    (lambda s: s.__getitem__(len(s)), IndexError, 'index out of bounds'),
+    (lambda s: s[len(s)], IndexError, 'index out of bounds'),
+    (lambda s: s.iloc[len(s)], IndexError,
+     'single positional indexer is out-of-bounds')
+])
+def test_series_getitem_indexing_errors(
+        multiindex_year_month_day_dataframe_random_data, indexer, error, msg):
+    s = multiindex_year_month_day_dataframe_random_data['A']
+    with pytest.raises(error, match=msg):
+        indexer(s)
 
-    # don't segfault, GH #495
-    # out of bounds access
-    msg = "index out of bounds"
-    with pytest.raises(IndexError, match=msg):
-        s.__getitem__(len(ymd))
 
-    # generator
+def test_series_getitem_corner_generator(
+        multiindex_year_month_day_dataframe_random_data):
+    s = multiindex_year_month_day_dataframe_random_data['A']
     result = s[(x > 0 for x in s)]
     expected = s[s > 0]
     tm.assert_series_equal(result, expected)

pandas/tests/indexing/multiindex/test_xs.py

@@ -3,7 +3,7 @@
 
 from pandas.compat import lrange, product as cart_product
 
-from pandas import DataFrame, Index, MultiIndex, concat
+from pandas import DataFrame, Index, MultiIndex, Series, concat, date_range
 import pandas.core.common as com
 from pandas.util import testing as tm
 
@@ -211,3 +211,27 @@ def test_xs_level_series_slice_not_implemented(
     msg = r'\(2000, slice\(3, 4, None\)\)'
     with pytest.raises(TypeError, match=msg):
         s[2000, 3:4]
+
+
+def test_series_getitem_multiindex_xs():
+    # GH6258
+    dt = list(date_range('20130903', periods=3))
+    idx = MultiIndex.from_product([list('AB'), dt])
+    s = Series([1, 3, 4, 1, 3, 4], index=idx)
+    expected = Series([1, 1], index=list('AB'))
+
+    result = s.xs('20130903', level=1)
+    tm.assert_series_equal(result, expected)
+
+
+def test_series_getitem_multiindex_xs_by_label():
+    # GH5684
+    idx = MultiIndex.from_tuples([('a', 'one'), ('a', 'two'), ('b', 'one'),
+                                  ('b', 'two')])
+    s = Series([1, 2, 3, 4], index=idx)
+    s.index.set_names(['L1', 'L2'], inplace=True)
+    expected = Series([1, 3], index=['a', 'b'])
+    expected.index.set_names(['L1'], inplace=True)
+
+    result = s.xs('one', level='L2')
+    tm.assert_series_equal(result, expected)