From 822cbd6fa4e201df8b7d0985809bcd06d3efba7e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 31 Dec 2019 18:38:10 -0800 Subject: [PATCH 1/4] REF: share join methods --- pandas/core/indexes/datetimelike.py | 63 ++++++++++++++++------------- pandas/core/indexes/datetimes.py | 12 ------ pandas/core/indexes/timedeltas.py | 12 +----- 3 files changed, 37 insertions(+), 50 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 76814403af385..f77d775a8b61d 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -6,7 +6,7 @@ import numpy as np -from pandas._libs import NaT, iNaT, lib +from pandas._libs import NaT, iNaT, join as libjoin, lib from pandas._libs.algos import unique_deltas from pandas.compat.numpy import function as nv from pandas.errors import AbstractMethodError @@ -72,6 +72,31 @@ def method(self, other): return method +def _join_i8_wrapper(joinf, with_indexers: bool = True): + """ + Create the join wrapper methods. + """ + + @staticmethod + def wrapper(left, right): + if isinstance(left, (np.ndarray, ABCIndex, ABCSeries, DatetimeLikeArrayMixin)): + left = left.view("i8") + if isinstance(right, (np.ndarray, ABCIndex, ABCSeries, DatetimeLikeArrayMixin)): + right = right.view("i8") + results = joinf(left, right) + if with_indexers: + # dtype should be timedelta64[ns] for TimedeltaIndex + # and datetime64[ns] for DatetimeIndex + dtype = left.dtype.base + + join_index, left_indexer, right_indexer = results + join_index = join_index.view(dtype) + return join_index, left_indexer, right_indexer + return results + + return wrapper + + class DatetimeIndexOpsMixin(ExtensionOpsMixin): """ Common ops mixin to support a unified interface datetimelike Index. @@ -208,32 +233,6 @@ def equals(self, other): return np.array_equal(self.asi8, other.asi8) - @staticmethod - def _join_i8_wrapper(joinf, dtype, with_indexers=True): - """ - Create the join wrapper methods. - """ - from pandas.core.arrays.datetimelike import DatetimeLikeArrayMixin - - @staticmethod - def wrapper(left, right): - if isinstance( - left, (np.ndarray, ABCIndex, ABCSeries, DatetimeLikeArrayMixin) - ): - left = left.view("i8") - if isinstance( - right, (np.ndarray, ABCIndex, ABCSeries, DatetimeLikeArrayMixin) - ): - right = right.view("i8") - results = joinf(left, right) - if with_indexers: - join_index, left_indexer, right_indexer = results - join_index = join_index.view(dtype) - return join_index, left_indexer, right_indexer - return results - - return wrapper - def _ensure_localized( self, arg, ambiguous="raise", nonexistent="raise", from_utc=False ): @@ -853,6 +852,16 @@ def _can_fast_union(self, other) -> bool: # this will raise return False + # -------------------------------------------------------------------- + # Join Methods + + _inner_indexer = _join_i8_wrapper(libjoin.inner_join_indexer) + _outer_indexer = _join_i8_wrapper(libjoin.outer_join_indexer) + _left_indexer = _join_i8_wrapper(libjoin.left_join_indexer) + _left_indexer_unique = _join_i8_wrapper( + libjoin.left_join_indexer_unique, with_indexers=False + ) + def wrap_arithmetic_op(self, other, result): if result is NotImplemented: diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 53d2ed22cd631..07bbdf25512ca 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -5,7 +5,6 @@ import numpy as np from pandas._libs import NaT, Timestamp, index as libindex, lib, tslib as libts -import pandas._libs.join as libjoin from pandas._libs.tslibs import ccalendar, fields, parsing, timezones from pandas.util._decorators import Appender, Substitution, cache_readonly @@ -32,7 +31,6 @@ import pandas.core.common as com from pandas.core.indexes.base import Index, maybe_extract_name from pandas.core.indexes.datetimelike import ( - DatetimeIndexOpsMixin, DatetimelikeDelegateMixin, DatetimeTimedeltaMixin, ) @@ -197,16 +195,6 @@ class DatetimeIndex(DatetimeTimedeltaMixin, DatetimeDelegateMixin): _typ = "datetimeindex" _join_precedence = 10 - def _join_i8_wrapper(joinf, **kwargs): - return DatetimeIndexOpsMixin._join_i8_wrapper(joinf, dtype="M8[ns]", **kwargs) - - _inner_indexer = _join_i8_wrapper(libjoin.inner_join_indexer) - _outer_indexer = _join_i8_wrapper(libjoin.outer_join_indexer) - _left_indexer = _join_i8_wrapper(libjoin.left_join_indexer) - _left_indexer_unique = _join_i8_wrapper( - libjoin.left_join_indexer_unique, with_indexers=False - ) - _engine_type = libindex.DatetimeEngine _supports_partial_string_indexing = True diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index 65c3ece6000fc..ed1e4f26806af 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -3,7 +3,7 @@ import numpy as np -from pandas._libs import NaT, Timedelta, index as libindex, join as libjoin, lib +from pandas._libs import NaT, Timedelta, index as libindex, lib from pandas.util._decorators import Appender, Substitution from pandas.core.dtypes.common import ( @@ -123,16 +123,6 @@ class TimedeltaIndex( _typ = "timedeltaindex" _join_precedence = 10 - def _join_i8_wrapper(joinf, **kwargs): - return DatetimeIndexOpsMixin._join_i8_wrapper(joinf, dtype="m8[ns]", **kwargs) - - _inner_indexer = _join_i8_wrapper(libjoin.inner_join_indexer) - _outer_indexer = _join_i8_wrapper(libjoin.outer_join_indexer) - _left_indexer = _join_i8_wrapper(libjoin.left_join_indexer) - _left_indexer_unique = _join_i8_wrapper( - libjoin.left_join_indexer_unique, with_indexers=False - ) - _engine_type = libindex.TimedeltaEngine _comparables = ["name", "freq"] From 94df9e99f0ff9cae97623517b38991e65d3257f3 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 31 Dec 2019 18:55:28 -0800 Subject: [PATCH 2/4] REF: share join methods for DTI/TDI --- pandas/core/indexes/datetimelike.py | 61 ++++++++++++++++++++++++++++- pandas/core/indexes/datetimes.py | 49 ----------------------- pandas/core/indexes/timedeltas.py | 20 ---------- 3 files changed, 60 insertions(+), 70 deletions(-) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index f77d775a8b61d..53642d223215d 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -8,6 +8,7 @@ from pandas._libs import NaT, iNaT, join as libjoin, lib from pandas._libs.algos import unique_deltas +from pandas._libs.tslibs import timezones from pandas.compat.numpy import function as nv from pandas.errors import AbstractMethodError from pandas.util._decorators import Appender, cache_readonly @@ -77,7 +78,6 @@ def _join_i8_wrapper(joinf, with_indexers: bool = True): Create the join wrapper methods. """ - @staticmethod def wrapper(left, right): if isinstance(left, (np.ndarray, ABCIndex, ABCSeries, DatetimeLikeArrayMixin)): left = left.view("i8") @@ -854,6 +854,7 @@ def _can_fast_union(self, other) -> bool: # -------------------------------------------------------------------- # Join Methods + _join_precedence = 10 _inner_indexer = _join_i8_wrapper(libjoin.inner_join_indexer) _outer_indexer = _join_i8_wrapper(libjoin.outer_join_indexer) @@ -862,6 +863,64 @@ def _can_fast_union(self, other) -> bool: libjoin.left_join_indexer_unique, with_indexers=False ) + def join( + self, other, how: str = "left", level=None, return_indexers=False, sort=False + ): + """ + See Index.join + """ + if self._is_convertible_to_index_for_join(other): + try: + other = type(self)(other) + except (TypeError, ValueError): + pass + + this, other = self._maybe_utc_convert(other) + return Index.join( + this, + other, + how=how, + level=level, + return_indexers=return_indexers, + sort=sort, + ) + + def _maybe_utc_convert(self, other): + this = self + if not hasattr(self, "tz"): + return this, other + + if isinstance(other, type(self)): + if self.tz is not None: + if other.tz is None: + raise TypeError("Cannot join tz-naive with tz-aware DatetimeIndex") + elif other.tz is not None: + raise TypeError("Cannot join tz-naive with tz-aware DatetimeIndex") + + if not timezones.tz_compare(self.tz, other.tz): + this = self.tz_convert("UTC") + other = other.tz_convert("UTC") + return this, other + + @classmethod + def _is_convertible_to_index_for_join(cls, other: Index) -> bool: + """ + return a boolean whether I can attempt conversion to a + DatetimeIndex/TimedeltaIndex + """ + if isinstance(other, cls): + return False + elif len(other) > 0 and other.inferred_type not in ( + "floating", + "mixed-integer", + "integer", + "integer-na", + "mixed-integer-float", + "mixed", + ): + return True + return False + def wrap_arithmetic_op(self, other, result): if result is NotImplemented: diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 07bbdf25512ca..fafa9e95a5963 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -193,7 +193,6 @@ class DatetimeIndex(DatetimeTimedeltaMixin, DatetimeDelegateMixin): """ _typ = "datetimeindex" - _join_precedence = 10 _engine_type = libindex.DatetimeEngine _supports_partial_string_indexing = True @@ -633,54 +632,6 @@ def snap(self, freq="S"): # we know it conforms; skip check return DatetimeIndex._simple_new(snapped, name=self.name, tz=self.tz, freq=freq) - def join( - self, other, how: str = "left", level=None, return_indexers=False, sort=False - ): - """ - See Index.join - """ - if ( - not isinstance(other, DatetimeIndex) - and len(other) > 0 - and other.inferred_type - not in ( - "floating", - "integer", - "integer-na", - "mixed-integer", - "mixed-integer-float", - "mixed", - ) - ): - try: - other = DatetimeIndex(other) - except (TypeError, ValueError): - pass - - this, other = self._maybe_utc_convert(other) - return Index.join( - this, - other, - how=how, - level=level, - return_indexers=return_indexers, - sort=sort, - ) - - def _maybe_utc_convert(self, other): - this = self - if isinstance(other, DatetimeIndex): - if self.tz is not None: - if other.tz is None: - raise TypeError("Cannot join tz-naive with tz-aware DatetimeIndex") - elif other.tz is not None: - raise TypeError("Cannot join tz-naive with tz-aware DatetimeIndex") - - if not timezones.tz_compare(self.tz, other.tz): - this = self.tz_convert("UTC") - other = other.tz_convert("UTC") - return this, other - def _wrap_joined_index(self, joined, other): name = get_op_result_name(self, other) if ( diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index ed1e4f26806af..27d2ac8f24402 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -121,7 +121,6 @@ class TimedeltaIndex( """ _typ = "timedeltaindex" - _join_precedence = 10 _engine_type = libindex.TimedeltaEngine @@ -284,25 +283,6 @@ def _union(self, other, sort): result._set_freq("infer") return result - def join(self, other, how="left", level=None, return_indexers=False, sort=False): - """ - See Index.join - """ - if _is_convertible_to_index(other): - try: - other = TimedeltaIndex(other) - except (TypeError, ValueError): - pass - - return Index.join( - self, - other, - how=how, - level=level, - return_indexers=return_indexers, - sort=sort, - ) - def _wrap_joined_index(self, joined, other): name = get_op_result_name(self, other) if ( From 218bcac3e353d1a9feb7dda31091cc80856981bf Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 31 Dec 2019 18:58:48 -0800 Subject: [PATCH 3/4] remove unused func --- pandas/core/indexes/timedeltas.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index 27d2ac8f24402..e6790d092778f 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -539,24 +539,6 @@ def delete(self, loc): TimedeltaIndex._add_datetimelike_methods() -def _is_convertible_to_index(other) -> bool: - """ - return a boolean whether I can attempt conversion to a TimedeltaIndex - """ - if isinstance(other, TimedeltaIndex): - return True - elif len(other) > 0 and other.inferred_type not in ( - "floating", - "mixed-integer", - "integer", - "integer-na", - "mixed-integer-float", - "mixed", - ): - return True - return False - - def timedelta_range( start=None, end=None, periods=None, freq=None, name=None, closed=None ) -> TimedeltaIndex: From cf1dcc9dddd82867d6cbec781f27a7f2ed90d01a Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 31 Dec 2019 19:36:19 -0800 Subject: [PATCH 4/4] restore staticmethod --- pandas/core/indexes/datetimelike.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py index 53642d223215d..6a49f9f670aab 100644 --- a/pandas/core/indexes/datetimelike.py +++ b/pandas/core/indexes/datetimelike.py @@ -78,11 +78,13 @@ def _join_i8_wrapper(joinf, with_indexers: bool = True): Create the join wrapper methods. """ + @staticmethod # type: ignore def wrapper(left, right): if isinstance(left, (np.ndarray, ABCIndex, ABCSeries, DatetimeLikeArrayMixin)): left = left.view("i8") if isinstance(right, (np.ndarray, ABCIndex, ABCSeries, DatetimeLikeArrayMixin)): right = right.view("i8") + results = joinf(left, right) if with_indexers: # dtype should be timedelta64[ns] for TimedeltaIndex