From 4e63c805b3f060903f88bd6858df1cdeaeecfc45 Mon Sep 17 00:00:00 2001 From: Gregory Rome Date: Mon, 6 May 2019 19:27:03 -0500 Subject: [PATCH 1/8] Add type annotations to io.pytables --- mypy.ini | 3 --- pandas/io/pytables.py | 15 ++++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/mypy.ini b/mypy.ini index 63ccb6eb994ba..1117fec18ec78 100644 --- a/mypy.ini +++ b/mypy.ini @@ -68,9 +68,6 @@ ignore_errors=True [mypy-pandas.core.window] ignore_errors=True -[mypy-pandas.io.pytables] -ignore_errors=True - [mypy-pandas.util._doctools] ignore_errors=True diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 18479b3420419..d0d751f4e2337 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -10,6 +10,7 @@ import os import re import time +from typing import Optional, List, Type, Union import warnings import numpy as np @@ -2285,9 +2286,9 @@ class Fixed(StringMixin): parent : my parent HDFStore group : the group node where the table resides """ - pandas_kind = None - obj_type = None - ndim = None + pandas_kind = None # type: Optional[str] + obj_type = None # type: Optional[Type[Union[DataFrame, Series]]] + ndim = None # type: Optional[int] is_table = False def __init__(self, parent, group, encoding=None, errors='strict', @@ -2447,7 +2448,7 @@ class GenericFixed(Fixed): """ a generified fixed version """ _index_type_map = {DatetimeIndex: 'datetime', PeriodIndex: 'period'} _reverse_index_map = {v: k for k, v in _index_type_map.items()} - attributes = [] + attributes = [] # type: List # indexer helpders def _class_to_alias(self, cls): @@ -3040,7 +3041,7 @@ class Table(Fixed): """ pandas_kind = 'wide_table' - table_type = None + table_type = None # type: Optional[str] levels = 1 is_table = True is_shape_reversed = False @@ -3861,7 +3862,7 @@ class LegacyTable(Table): IndexCol(name='index', axis=1, pos=0), IndexCol(name='column', axis=2, pos=1, index_kind='columns_kind'), DataCol(name='fields', cname='values', kind_attr='fields', pos=2) - ] + ] # type: Optional[List[IndexCol]] table_type = 'legacy' ndim = 3 @@ -4116,7 +4117,7 @@ class AppendableFrameTable(AppendableTable): pandas_kind = 'frame_table' table_type = 'appendable_frame' ndim = 2 - obj_type = DataFrame + obj_type = DataFrame # type: Type[Union[DataFrame, Series]] @property def is_transposed(self): From d2c1c7dac83b0bea20e56b5c637ec5d921d35783 Mon Sep 17 00:00:00 2001 From: Gregory Rome Date: Thu, 9 May 2019 13:20:53 -0500 Subject: [PATCH 2/8] Start correcting mypy errors in core.window --- mypy.ini | 6 ------ pandas/core/window.py | 9 +++++---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/mypy.ini b/mypy.ini index 09305b5eb7c8b..ef053db10e720 100644 --- a/mypy.ini +++ b/mypy.ini @@ -50,11 +50,5 @@ ignore_errors=True [mypy-pandas.core.util.hashing] ignore_errors=True -[mypy-pandas.core.window] -ignore_errors=True - -[mypy-pandas.util._doctools] -ignore_errors=True - [mypy-pandas.util.testing] ignore_errors=True diff --git a/pandas/core/window.py b/pandas/core/window.py index 2d7fdbeffbccc..39bf074c3167f 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -5,6 +5,7 @@ from collections import defaultdict from datetime import timedelta from textwrap import dedent +from typing import Set import warnings import numpy as np @@ -42,7 +43,7 @@ class _Window(PandasObject, SelectionMixin): _attributes = ['window', 'min_periods', 'center', 'win_type', 'axis', 'on', 'closed'] - exclusions = set() + exclusions = set() # type: Set def __init__(self, obj, window=None, min_periods=None, center=False, win_type=None, axis=0, on=None, closed=None, @@ -305,10 +306,10 @@ def _center_window(self, result, window): result = np.copy(result[tuple(lead_indexer)]) return result - def aggregate(self, arg, *args, **kwargs): - result, how = self._aggregate(arg, *args, **kwargs) + def aggregate(self, func, *args, **kwargs): + result, how = self._aggregate(func, *args, **kwargs) if result is None: - return self.apply(arg, raw=False, args=args, kwargs=kwargs) + return self.apply(func, raw=False, args=args, kwargs=kwargs) return result agg = aggregate From 54cd8f44ae6441aaac399772d647e5bccb3e1559 Mon Sep 17 00:00:00 2001 From: Gregory Rome Date: Mon, 13 May 2019 08:33:34 -0500 Subject: [PATCH 3/8] Complete misc type fixes --- mypy.ini | 24 ------------------------ pandas/core/base.py | 5 +++++ pandas/core/generic.py | 4 ++-- pandas/core/indexing.py | 5 +++-- pandas/core/internals/blocks.py | 8 ++++++-- pandas/core/reshape/merge.py | 3 ++- pandas/core/reshape/reshape.py | 3 ++- pandas/core/util/hashing.py | 3 ++- pandas/core/window.py | 6 +++++- pandas/io/pytables.py | 2 +- pandas/util/testing.py | 2 +- 11 files changed, 29 insertions(+), 36 deletions(-) diff --git a/mypy.ini b/mypy.ini index ef053db10e720..b905627ef51e8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -23,32 +23,8 @@ ignore_errors=True [mypy-pandas.core.indexes.timedeltas] ignore_errors=True -[mypy-pandas.core.indexing] -ignore_errors=True - -[mypy-pandas.core.internals.blocks] -ignore_errors=True - [mypy-pandas.core.ops] ignore_errors=True -[mypy-pandas.core.panel] -ignore_errors=True - [mypy-pandas.core.resample] ignore_errors=True - -[mypy-pandas.core.reshape.merge] -ignore_errors=True - -[mypy-pandas.core.reshape.reshape] -ignore_errors=True - -[mypy-pandas.core.series] -ignore_errors=True - -[mypy-pandas.core.util.hashing] -ignore_errors=True - -[mypy-pandas.util.testing] -ignore_errors=True diff --git a/pandas/core/base.py b/pandas/core/base.py index 5bccaeef66a82..2d7ea0bb6f3e6 100644 --- a/pandas/core/base.py +++ b/pandas/core/base.py @@ -4,6 +4,7 @@ import builtins from collections import OrderedDict import textwrap +from typing import no_type_check import warnings import numpy as np @@ -1572,5 +1573,9 @@ def duplicated(self, keep='first'): # ---------------------------------------------------------------------- # abstracts + # Removed from type checking because there is currently no good way to + # handle multiple inheritance with incompatible signatures with mypy + # https://github.com/python/mypy/issues/2125 + @no_type_check def _update_inplace(self, result, **kwargs): raise AbstractMethodError(self) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 876465d96e6fe..c67bc6eb60e56 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -6,7 +6,7 @@ import operator import pickle from textwrap import dedent -from typing import FrozenSet, List, Set +from typing import Callable, FrozenSet, List, Set import warnings import weakref @@ -3680,7 +3680,7 @@ class animal locomotion result._set_is_copy(self, copy=not result._is_view) return result - _xs = xs + _xs = xs # type: Callable def select(self, crit, axis=0): """ diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 65123a8f0f5a7..5ab27606af238 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -1,4 +1,5 @@ import textwrap +from typing import Optional, Type import warnings import numpy as np @@ -88,8 +89,8 @@ class IndexingError(Exception): class _NDFrameIndexer(_NDFrameIndexerBase): - _valid_types = None - _exception = KeyError + _valid_types = None # type: Optional[str] + _exception = KeyError # type: Type[Exception] axis = None def __call__(self, axis=None): diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 519975f34fc5e..a85ad5cb88137 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -7,7 +7,8 @@ import numpy as np -from pandas._libs import internals as libinternals, lib, tslib, tslibs +from pandas._libs import lib, tslib, tslibs +import pandas._libs.internals as libinternals from pandas._libs.tslibs import Timedelta, conversion, is_null_datetimelike from pandas.util._validators import validate_bool_kwarg @@ -2051,12 +2052,15 @@ def get_values(self, dtype=None): class DatetimeBlock(DatetimeLikeBlockMixin, Block): __slots__ = () is_datetime = True - _can_hold_na = True def __init__(self, values, placement, ndim=None): values = self._maybe_coerce_values(values) super().__init__(values, placement=placement, ndim=ndim) + @property + def _can_hold_na(self): + return True + def _maybe_coerce_values(self, values): """Input validation for values passed to __init__. Ensure that we have datetime64ns, coercing if necessary. diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index 39b955ea7cbe1..a1e6b670ac1a3 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -8,7 +8,8 @@ import numpy as np -from pandas._libs import hashtable as libhashtable, join as libjoin, lib +from pandas._libs import hashtable as libhashtable, lib +import pandas._libs.join as libjoin from pandas.errors import MergeError from pandas.util._decorators import Appender, Substitution diff --git a/pandas/core/reshape/reshape.py b/pandas/core/reshape/reshape.py index bd695e2d0d83d..c59f9ffc48055 100644 --- a/pandas/core/reshape/reshape.py +++ b/pandas/core/reshape/reshape.py @@ -3,7 +3,8 @@ import numpy as np -from pandas._libs import algos as _algos, reshape as _reshape +import pandas._libs.algos as _algos +import pandas._libs.reshape as _reshape from pandas._libs.sparse import IntIndex from pandas.core.dtypes.cast import maybe_promote diff --git a/pandas/core/util/hashing.py b/pandas/core/util/hashing.py index 29fc1e3671a83..93074f5afa2b3 100644 --- a/pandas/core/util/hashing.py +++ b/pandas/core/util/hashing.py @@ -5,7 +5,8 @@ import numpy as np -from pandas._libs import hashing, tslibs +import pandas._libs.hashing as hashing +import pandas._libs.tslibs as tslibs from pandas.core.dtypes.cast import infer_dtype_from_scalar from pandas.core.dtypes.common import ( diff --git a/pandas/core/window.py b/pandas/core/window.py index 39bf074c3167f..10c4789c98dac 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -5,7 +5,7 @@ from collections import defaultdict from datetime import timedelta from textwrap import dedent -from typing import Set +from typing import Set, no_type_check import warnings import numpy as np @@ -789,6 +789,10 @@ def __init__(self, obj, *args, **kwargs): corr = GroupByMixin._dispatch('corr', other=None, pairwise=None) cov = GroupByMixin._dispatch('cov', other=None, pairwise=None) + # Removed from type checking because there is currently no good way to + # handle multiple inheritance with mypy + # https://github.com/python/mypy/issues/2125 + @no_type_check def _apply(self, func, name, window=None, center=None, check_minp=None, **kwargs): """ diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index e3efc94b06148..3f42cc1ed25ef 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -10,7 +10,7 @@ import os import re import time -from typing import Optional, List, Type, Union +from typing import List, Optional, Type, Union import warnings import numpy as np diff --git a/pandas/util/testing.py b/pandas/util/testing.py index b7fa15984f294..cd22b9d96e059 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -21,7 +21,7 @@ from pandas._config.localization import ( # noqa:F401 can_set_locale, get_locales, set_locale) -from pandas._libs import testing as _testing +import pandas._libs.testing as _testing from pandas.compat import lmap, raise_with_traceback from pandas.core.dtypes.common import ( From 553167800a402ab3129efa2462ee0af07d1add26 Mon Sep 17 00:00:00 2001 From: Gregory Rome Date: Mon, 13 May 2019 11:19:49 -0500 Subject: [PATCH 4/8] Fix import sort on test_groupby.py --- pandas/tests/groupby/test_groupby.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/tests/groupby/test_groupby.py b/pandas/tests/groupby/test_groupby.py index 354242995d3c3..39d3ffdb286ab 100644 --- a/pandas/tests/groupby/test_groupby.py +++ b/pandas/tests/groupby/test_groupby.py @@ -10,8 +10,7 @@ import pandas as pd from pandas import ( - DataFrame, Index, MultiIndex, Series, Timestamp, date_range, - read_csv) + DataFrame, Index, MultiIndex, Series, Timestamp, date_range, read_csv) import pandas.core.common as com import pandas.util.testing as tm from pandas.util.testing import ( From 34a3fb45b3e7bd98f0118782b5a1f91e24afe79d Mon Sep 17 00:00:00 2001 From: Gregory Rome Date: Mon, 13 May 2019 13:41:14 -0500 Subject: [PATCH 5/8] Update after code reviews --- pandas/core/base.py | 7 +------ pandas/core/indexing.py | 4 ++-- pandas/io/pytables.py | 6 +++--- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/pandas/core/base.py b/pandas/core/base.py index 2d7ea0bb6f3e6..301af59eadb0e 100644 --- a/pandas/core/base.py +++ b/pandas/core/base.py @@ -4,7 +4,6 @@ import builtins from collections import OrderedDict import textwrap -from typing import no_type_check import warnings import numpy as np @@ -1573,9 +1572,5 @@ def duplicated(self, keep='first'): # ---------------------------------------------------------------------- # abstracts - # Removed from type checking because there is currently no good way to - # handle multiple inheritance with incompatible signatures with mypy - # https://github.com/python/mypy/issues/2125 - @no_type_check - def _update_inplace(self, result, **kwargs): + def _update_inplace(self, result, verify_is_copy=True, **kwargs): raise AbstractMethodError(self) diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index 5ab27606af238..ce8666ca2066e 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -1,5 +1,5 @@ import textwrap -from typing import Optional, Type +from typing import Type import warnings import numpy as np @@ -89,7 +89,7 @@ class IndexingError(Exception): class _NDFrameIndexer(_NDFrameIndexerBase): - _valid_types = None # type: Optional[str] + _valid_types = None # type: str _exception = KeyError # type: Type[Exception] axis = None diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 3f42cc1ed25ef..0bcabaf5931b1 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -2298,9 +2298,9 @@ class Fixed(StringMixin): parent : my parent HDFStore group : the group node where the table resides """ - pandas_kind = None # type: Optional[str] - obj_type = None # type: Optional[Type[Union[DataFrame, Series]]] - ndim = None # type: Optional[int] + pandas_kind = None # type: str + obj_type = None # type: Type[Union[DataFrame, Series]] + ndim = None # type: int is_table = False def __init__(self, parent, group, encoding=None, errors='strict', From 2eeff8cb9eee425e8dfb195944bdbd137b5b2b9c Mon Sep 17 00:00:00 2001 From: Gregory Rome Date: Mon, 13 May 2019 16:06:33 -0500 Subject: [PATCH 6/8] Whatsnew, second rev indexing, window and pytables --- doc/source/whatsnew/v0.25.0.rst | 1 + pandas/core/indexing.py | 3 +-- pandas/core/window.py | 8 ++------ pandas/io/pytables.py | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index dacd433f112a5..f443a68b1efd7 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -217,6 +217,7 @@ Other API Changes - Comparing :class:`Timestamp` with unsupported objects now returns :py:obj:`NotImplemented` instead of raising ``TypeError``. This implies that unsupported rich comparisons are delegated to the other object, and are now consistent with Python 3 behavior for ``datetime`` objects (:issue:`24011`) - Bug in :meth:`DatetimeIndex.snap` which didn't preserving the ``name`` of the input :class:`Index` (:issue:`25575`) - The ``arg`` argument in :meth:`pandas.core.groupby.DataFrameGroupBy.agg` has been renamed to ``func`` (:issue:`26089`) +- The ``arg`` argument in :meth:`pandas.core.window._Window.aggregate` has been renamed to ``func`` (:issue:`26372`) .. _whatsnew_0250.deprecations: diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py index ce8666ca2066e..eb210224f6ec3 100755 --- a/pandas/core/indexing.py +++ b/pandas/core/indexing.py @@ -1,5 +1,4 @@ import textwrap -from typing import Type import warnings import numpy as np @@ -90,7 +89,7 @@ class IndexingError(Exception): class _NDFrameIndexer(_NDFrameIndexerBase): _valid_types = None # type: str - _exception = KeyError # type: Type[Exception] + _exception = Exception axis = None def __call__(self, axis=None): diff --git a/pandas/core/window.py b/pandas/core/window.py index 10c4789c98dac..cbf0c110a4631 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -5,7 +5,7 @@ from collections import defaultdict from datetime import timedelta from textwrap import dedent -from typing import Set, no_type_check +from typing import Set import warnings import numpy as np @@ -789,11 +789,7 @@ def __init__(self, obj, *args, **kwargs): corr = GroupByMixin._dispatch('corr', other=None, pairwise=None) cov = GroupByMixin._dispatch('cov', other=None, pairwise=None) - # Removed from type checking because there is currently no good way to - # handle multiple inheritance with mypy - # https://github.com/python/mypy/issues/2125 - @no_type_check - def _apply(self, func, name, window=None, center=None, + def _apply(self, func, name=None, window=None, center=None, check_minp=None, **kwargs): """ Dispatch to apply; we are stripping all of the _apply kwargs and diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 0bcabaf5931b1..4176acb652c46 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -3053,7 +3053,7 @@ class Table(Fixed): """ pandas_kind = 'wide_table' - table_type = None # type: Optional[str] + table_type = None # type: str levels = 1 is_table = True is_shape_reversed = False From 7a06e6ee887a963446949f346d1e407045c959f5 Mon Sep 17 00:00:00 2001 From: Gregory Rome Date: Mon, 13 May 2019 16:08:13 -0500 Subject: [PATCH 7/8] Annotate Set in core.window --- pandas/core/window.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/window.py b/pandas/core/window.py index cbf0c110a4631..9ec1902cb4937 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -43,7 +43,7 @@ class _Window(PandasObject, SelectionMixin): _attributes = ['window', 'min_periods', 'center', 'win_type', 'axis', 'on', 'closed'] - exclusions = set() # type: Set + exclusions = set() # type: Set[str] def __init__(self, obj, window=None, min_periods=None, center=False, win_type=None, axis=0, on=None, closed=None, From 26efef57091a340591845287b21d9b72574d90e4 Mon Sep 17 00:00:00 2001 From: Gregory Rome Date: Mon, 13 May 2019 21:29:31 -0500 Subject: [PATCH 8/8] Define attributes as List[str] --- pandas/io/pytables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 4176acb652c46..3194804b4efc2 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -2460,7 +2460,7 @@ class GenericFixed(Fixed): """ a generified fixed version """ _index_type_map = {DatetimeIndex: 'datetime', PeriodIndex: 'period'} _reverse_index_map = {v: k for k, v in _index_type_map.items()} - attributes = [] # type: List + attributes = [] # type: List[str] # indexer helpders def _class_to_alias(self, cls):