-
-
Notifications
You must be signed in to change notification settings - Fork 18.6k
BUG: lib.infer_dtype with mixed-freq Periods #41526
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
jreback
merged 3 commits into
pandas-dev:master
from
jbrockmendel:ref-maybe_convert_objects-2
May 18, 2021
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1186,6 +1186,7 @@ cdef class Seen: | |
bint coerce_numeric # coerce data to numeric | ||
bint timedelta_ # seen_timedelta | ||
bint datetimetz_ # seen_datetimetz | ||
bint period_ # seen_period | ||
|
||
def __cinit__(self, bint coerce_numeric=False): | ||
""" | ||
|
@@ -1210,6 +1211,7 @@ cdef class Seen: | |
self.datetime_ = False | ||
self.timedelta_ = False | ||
self.datetimetz_ = False | ||
self.period_ = False | ||
self.coerce_numeric = coerce_numeric | ||
|
||
cdef inline bint check_uint64_conflict(self) except -1: | ||
|
@@ -1996,18 +1998,35 @@ cpdef bint is_time_array(ndarray values, bint skipna=False): | |
return validator.validate(values) | ||
|
||
|
||
cdef class PeriodValidator(TemporalValidator): | ||
cdef inline bint is_value_typed(self, object value) except -1: | ||
return is_period_object(value) | ||
cdef bint is_period_array(ndarray[object] values): | ||
""" | ||
Is this an ndarray of Period objects (or NaT) with a single `freq`? | ||
""" | ||
cdef: | ||
Py_ssize_t i, n = len(values) | ||
int dtype_code = -10000 # i.e. c_FreqGroup.FR_UND | ||
object val | ||
|
||
cdef inline bint is_valid_null(self, object value) except -1: | ||
return checknull_with_nat(value) | ||
if len(values) == 0: | ||
return False | ||
|
||
for val in values: | ||
if is_period_object(val): | ||
if dtype_code == -10000: | ||
dtype_code = val._dtype._dtype_code | ||
elif dtype_code != val._dtype._dtype_code: | ||
# mismatched freqs | ||
return False | ||
elif checknull_with_nat(val): | ||
pass | ||
else: | ||
# Not a Period or NaT-like | ||
return False | ||
|
||
cpdef bint is_period_array(ndarray values): | ||
cdef: | ||
PeriodValidator validator = PeriodValidator(len(values), skipna=True) | ||
return validator.validate(values) | ||
if dtype_code == -10000: | ||
# we saw all-NaTs, no actual Periods | ||
return False | ||
return True | ||
|
||
|
||
cdef class IntervalValidator(Validator): | ||
|
@@ -2249,9 +2268,13 @@ def maybe_convert_numeric( | |
|
||
@cython.boundscheck(False) | ||
@cython.wraparound(False) | ||
def maybe_convert_objects(ndarray[object] objects, bint try_float=False, | ||
bint safe=False, bint convert_datetime=False, | ||
def maybe_convert_objects(ndarray[object] objects, | ||
*, | ||
bint try_float=False, | ||
bint safe=False, | ||
bint convert_datetime=False, | ||
bint convert_timedelta=False, | ||
bint convert_period=False, | ||
bint convert_to_nullable_integer=False) -> "ArrayLike": | ||
""" | ||
Type inference function-- convert object array to proper dtype | ||
|
@@ -2272,6 +2295,9 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=False, | |
convert_timedelta : bool, default False | ||
If an array-like object contains only timedelta values or NaT is | ||
encountered, whether to convert and return an array of m8[ns] dtype. | ||
convert_period : bool, default False | ||
If an array-like object contains only (homogeneous-freq) Period values | ||
or NaT, whether to convert and return a PeriodArray. | ||
convert_to_nullable_integer : bool, default False | ||
If an array-like object contains only integer values (and NaN) is | ||
encountered, whether to convert and return an IntegerArray. | ||
|
@@ -2292,7 +2318,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=False, | |
int64_t[:] itimedeltas | ||
Seen seen = Seen() | ||
object val | ||
float64_t fval, fnan | ||
float64_t fval, fnan = np.nan | ||
|
||
n = len(objects) | ||
|
||
|
@@ -2311,8 +2337,6 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=False, | |
timedeltas = np.empty(n, dtype='m8[ns]') | ||
itimedeltas = timedeltas.view(np.int64) | ||
|
||
fnan = np.nan | ||
|
||
for i in range(n): | ||
val = objects[i] | ||
if itemsize_max != -1: | ||
|
@@ -2330,7 +2354,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=False, | |
idatetimes[i] = NPY_NAT | ||
if convert_timedelta: | ||
itimedeltas[i] = NPY_NAT | ||
if not (convert_datetime or convert_timedelta): | ||
if not (convert_datetime or convert_timedelta or convert_period): | ||
seen.object_ = True | ||
break | ||
elif val is np.nan: | ||
|
@@ -2343,14 +2367,6 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=False, | |
elif util.is_float_object(val): | ||
floats[i] = complexes[i] = val | ||
seen.float_ = True | ||
elif util.is_datetime64_object(val): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. unrelated to the rest of the PR, this chunk of code is just redundant with a chunk below |
||
if convert_datetime: | ||
idatetimes[i] = convert_to_tsobject( | ||
val, None, None, 0, 0).value | ||
seen.datetime_ = True | ||
else: | ||
seen.object_ = True | ||
break | ||
elif is_timedelta(val): | ||
if convert_timedelta: | ||
itimedeltas[i] = convert_to_timedelta64(val, "ns").view("i8") | ||
|
@@ -2396,6 +2412,13 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=False, | |
else: | ||
seen.object_ = True | ||
break | ||
elif is_period_object(val): | ||
if convert_period: | ||
seen.period_ = True | ||
break | ||
else: | ||
seen.object_ = True | ||
break | ||
elif try_float and not isinstance(val, str): | ||
# this will convert Decimal objects | ||
try: | ||
|
@@ -2419,6 +2442,14 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=False, | |
return dti._data | ||
seen.object_ = True | ||
|
||
if seen.period_: | ||
if is_period_array(objects): | ||
from pandas import PeriodIndex | ||
pi = PeriodIndex(objects) | ||
|
||
# unbox to PeriodArray | ||
return pi._data | ||
|
||
if not seen.object_: | ||
result = None | ||
if not safe: | ||
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
can you not keep the current PeriodValidator format itself? (e.g. just put this function in validate). this is breaking the pattern.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this is the same pattern we use for is_datetime_with_singletz_array (and will end up using for is_interval_array in an upcoming PR)