Skip to content

CLN refactor rest of core #37586

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 38 commits into from
Jan 20, 2021
Merged
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
3f275aa
refactor rest of core
MarcoGorelli Nov 2, 2020
4a6c153
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 4, 2020
9db6e6c
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 6, 2020
8794ceb
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 7, 2020
803f49b
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 8, 2020
4511a0c
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 9, 2020
9c3598b
apply some suggestions
MarcoGorelli Nov 9, 2020
873d430
logicalnot -> ~
MarcoGorelli Nov 9, 2020
31d29af
parens
MarcoGorelli Nov 9, 2020
7a51410
parens
MarcoGorelli Nov 9, 2020
a31e89d
parens
MarcoGorelli Nov 9, 2020
a8c8c9c
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 23, 2020
5e053d4
factor out _set_join_index
MarcoGorelli Nov 23, 2020
5f0a278
:label: type
MarcoGorelli Nov 23, 2020
142e837
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Nov 29, 2020
8e18225
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 6, 2020
f181cb0
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 9, 2020
9383755
coverage, remove putmask
MarcoGorelli Dec 9, 2020
fe0e0ba
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 16, 2020
0d10a5b
pass dtype to maybe_castable directly
MarcoGorelli Dec 16, 2020
938b2d5
revert sourcery's change of moving message closer to usage
MarcoGorelli Dec 16, 2020
7fda4e7
avoid duplicated check
MarcoGorelli Dec 16, 2020
c36bf8d
remove redundant check
MarcoGorelli Dec 16, 2020
2c2868e
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 22, 2020
c6d3233
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Dec 26, 2020
1f782fd
Dtype -> DtypeObj
MarcoGorelli Dec 26, 2020
b315629
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 3, 2021
84207d9
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 17, 2021
3f79955
remove redundant parens
MarcoGorelli Jan 17, 2021
9888383
use set to check for membership
MarcoGorelli Jan 17, 2021
0cd394f
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 17, 2021
2ec839e
Merge remote-tracking branch 'upstream/master' into refactor-rest-of-…
MarcoGorelli Jan 18, 2021
5125e5f
revert change in generic.py
MarcoGorelli Jan 18, 2021
d693f58
revert _set_join_index refactoring
MarcoGorelli Jan 18, 2021
d4e3bd6
deduplicate from merge
MarcoGorelli Jan 18, 2021
5559352
:art:
MarcoGorelli Jan 18, 2021
4f14b81
some reversions
MarcoGorelli Jan 18, 2021
c03d876
revert maybe_castable
MarcoGorelli Jan 18, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions pandas/core/aggregation.py
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,6 @@ def validate_func_kwargs(
>>> validate_func_kwargs({'one': 'min', 'two': 'max'})
(['one', 'two'], ['min', 'max'])
"""
no_arg_message = "Must provide 'func' or named aggregation **kwargs."
tuple_given_message = "func is expected but received {} in **kwargs."
columns = list(kwargs)
func = []
Expand All @@ -396,6 +395,7 @@ def validate_func_kwargs(
raise TypeError(tuple_given_message.format(type(col_func).__name__))
func.append(col_func)
if not columns:
no_arg_message = "Must provide 'func' or named aggregation **kwargs."
raise TypeError(no_arg_message)
return columns, func

Expand Down Expand Up @@ -497,14 +497,14 @@ def transform_dict_like(
try:
results[name] = transform(colg, how, 0, *args, **kwargs)
except Exception as err:
if (
str(err) == "Function did not transform"
or str(err) == "No transform functions were provided"
):
if str(err) in [
"Function did not transform",
"No transform functions were provided",
]:
raise err

# combine results
if len(results) == 0:
if not results:
raise ValueError("Transform function failed")
return concat(results, axis=1)

Expand Down
25 changes: 8 additions & 17 deletions pandas/core/algorithms.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,8 @@ def _ensure_data(
values = extract_array(values, extract_numpy=True)

# we check some simple dtypes first
if is_object_dtype(dtype):
if is_object_dtype(dtype) or (is_object_dtype(values) and (dtype is None)):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you remove the parens around dtype is None as confusing (and actually put around the entire expression and format)

return ensure_object(np.asarray(values)), np.dtype("object")
elif is_object_dtype(values) and dtype is None:
return ensure_object(np.asarray(values)), np.dtype("object")

try:
if is_bool_dtype(values) or is_bool_dtype(dtype):
# we are actually coercing to uint64
Expand Down Expand Up @@ -150,12 +147,10 @@ def _ensure_data(
from pandas import PeriodIndex

values = PeriodIndex(values)._data
dtype = values.dtype
elif is_timedelta64_dtype(values.dtype) or is_timedelta64_dtype(dtype):
from pandas import TimedeltaIndex

values = TimedeltaIndex(values)._data
dtype = values.dtype
else:
# Datetime
if values.ndim > 1 and is_datetime64_ns_dtype(values.dtype):
Expand All @@ -169,8 +164,7 @@ def _ensure_data(
from pandas import DatetimeIndex

values = DatetimeIndex(values)._data
dtype = values.dtype

dtype = values.dtype
return values.asi8, dtype

elif is_categorical_dtype(values.dtype) and (
Expand Down Expand Up @@ -875,10 +869,9 @@ def value_counts_arraylike(values, dropna: bool):
keys, counts = f(values, dropna)

mask = isna(values)
if not dropna and mask.any():
if not isna(keys).any():
keys = np.insert(keys, 0, np.NaN)
counts = np.insert(counts, 0, mask.sum())
if not dropna and mask.any() and not isna(keys).any():
keys = np.insert(keys, 0, np.NaN)
counts = np.insert(counts, 0, mask.sum())

keys = _reconstruct_data(keys, original.dtype, original)

Expand Down Expand Up @@ -1741,9 +1734,8 @@ def take_nd(
dtype, fill_value = arr.dtype, arr.dtype.type()

flip_order = False
if arr.ndim == 2:
if arr.flags.f_contiguous:
flip_order = True
if arr.ndim == 2 and arr.flags.f_contiguous:
flip_order = True

if flip_order:
arr = arr.T
Expand Down Expand Up @@ -1915,8 +1907,7 @@ def searchsorted(arr, value, side="left", sorter=None) -> np.ndarray:
# and `value` is a pd.Timestamp, we may need to convert value
arr = ensure_wrapped_if_datetimelike(arr)

result = arr.searchsorted(value, side=side, sorter=sorter)
return result
return arr.searchsorted(value, side=side, sorter=sorter)


# ---- #
Expand Down
7 changes: 4 additions & 3 deletions pandas/core/apply.py
Original file line number Diff line number Diff line change
Expand Up @@ -369,9 +369,10 @@ def wrap_results_for_axis(
else:
raise

if not isinstance(results[0], ABCSeries):
if len(result.index) == len(self.res_columns):
result.index = self.res_columns
if not isinstance(results[0], ABCSeries) and len(result.index) == len(
self.res_columns
):
result.index = self.res_columns

if len(result.columns) == len(res_index):
result.columns = res_index
Expand Down
3 changes: 1 addition & 2 deletions pandas/core/array_algos/replace.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,7 @@ def _check_comparison_types(
if is_scalar(result) and isinstance(a, np.ndarray):
type_names = [type(a).__name__, type(b).__name__]

if isinstance(a, np.ndarray):
type_names[0] = f"ndarray(dtype={a.dtype})"
type_names[0] = f"ndarray(dtype={a.dtype})"

raise TypeError(
f"Cannot compare types {repr(type_names[0])} and {repr(type_names[1])}"
Expand Down
13 changes: 5 additions & 8 deletions pandas/core/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,10 +321,9 @@ def _try_aggregate_string_function(self, arg: str, *args, **kwargs):
return f

f = getattr(np, arg, None)
if f is not None:
if hasattr(self, "__array__"):
# in particular exclude Window
return f(self, *args, **kwargs)
if f is not None and hasattr(self, "__array__"):
# in particular exclude Window
return f(self, *args, **kwargs)

raise AttributeError(
f"'{arg}' is not a valid function for '{type(self).__name__}' object"
Expand Down Expand Up @@ -1040,15 +1039,14 @@ def value_counts(
1.0 1
dtype: int64
"""
result = value_counts(
return value_counts(
self,
sort=sort,
ascending=ascending,
normalize=normalize,
bins=bins,
dropna=dropna,
)
return result

def unique(self):
values = self._values
Expand Down Expand Up @@ -1311,8 +1309,7 @@ def drop_duplicates(self, keep="first"):
duplicated = self.duplicated(keep=keep)
# pandas\core\base.py:1507: error: Value of type "IndexOpsMixin" is not
# indexable [index]
result = self[np.logical_not(duplicated)] # type: ignore[index]
return result
return self[~duplicated] # type: ignore[index]

def duplicated(self, keep="first"):
return duplicated(self._values, keep=keep)
13 changes: 8 additions & 5 deletions pandas/core/construction.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,8 +343,7 @@ def array(
elif is_timedelta64_ns_dtype(dtype):
return TimedeltaArray._from_sequence(data, dtype=dtype, copy=copy)

result = PandasArray._from_sequence(data, dtype=dtype, copy=copy)
return result
return PandasArray._from_sequence(data, dtype=dtype, copy=copy)


def extract_array(obj: object, extract_numpy: bool = False) -> Union[Any, ArrayLike]:
Expand Down Expand Up @@ -584,9 +583,13 @@ def _try_cast(arr, dtype: Optional[DtypeObj], copy: bool, raise_cast_failure: bo
Otherwise an object array is returned.
"""
# perf shortcut as this is the most common case
if isinstance(arr, np.ndarray):
if maybe_castable(arr) and not copy and dtype is None:
return arr
if (
isinstance(arr, np.ndarray)
and maybe_castable(arr.dtype)
and not copy
and dtype is None
):
return arr

if isinstance(dtype, ExtensionDtype) and (dtype.kind != "M" or is_sparse(dtype)):
# create an extension array from its dtype
Expand Down
12 changes: 5 additions & 7 deletions pandas/core/dtypes/cast.py
Original file line number Diff line number Diff line change
Expand Up @@ -1294,20 +1294,18 @@ def convert_dtypes(
return inferred_dtype


def maybe_castable(arr: np.ndarray) -> bool:
def maybe_castable(dtype: DtypeObj) -> bool:
# return False to force a non-fastpath

assert isinstance(arr, np.ndarray) # GH 37024

# check datetime64[ns]/timedelta64[ns] are valid
# otherwise try to coerce
kind = arr.dtype.kind
kind = dtype.kind
if kind == "M":
return is_datetime64_ns_dtype(arr.dtype)
return is_datetime64_ns_dtype(dtype)
elif kind == "m":
return is_timedelta64_ns_dtype(arr.dtype)
return is_timedelta64_ns_dtype(dtype)

return arr.dtype.name not in POSSIBLY_CAST_DTYPES
return dtype.name not in POSSIBLY_CAST_DTYPES


def maybe_infer_to_datetimelike(
Expand Down
8 changes: 4 additions & 4 deletions pandas/core/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -746,7 +746,7 @@ def _repr_fits_horizontal_(self, ignore_width: bool = False) -> bool:
# and to_string on entire frame may be expensive
d = self

if not (max_rows is None): # unlimited rows
if max_rows is not None: # unlimited rows
# min of two, where one may be None
d = d.iloc[: min(max_rows, len(d))]
else:
Expand Down Expand Up @@ -1983,10 +1983,10 @@ def to_records(
np.asarray(self.iloc[:, i]) for i in range(len(self.columns))
]

count = 0
index_names = list(self.index.names)

if isinstance(self.index, MultiIndex):
count = 0
for i, n in enumerate(index_names):
if n is None:
index_names[i] = f"level_{count}"
Expand Down Expand Up @@ -3288,7 +3288,7 @@ def _set_value(self, index, col, value, takeable: bool = False):
takeable : interpret the index/col as indexers, default False
"""
try:
if takeable is True:
if takeable:
series = self._ixs(col, axis=1)
series._set_value(index, value, takeable=True)
return
Expand Down Expand Up @@ -4994,7 +4994,7 @@ class max type

multi_col = isinstance(self.columns, MultiIndex)
for i, (lev, lab) in reversed(list(enumerate(to_insert))):
if not (level is None or i in level):
if level is not None and i not in level:
continue
name = names[i]
if multi_col:
Expand Down
Loading