Description
Pandas version checks
-
I have checked that this issue has not already been reported.
-
I have confirmed this bug exists on the latest version of pandas.
-
I have confirmed this bug exists on the main branch of pandas.
Reproducible Example
In [1]: import pandas as pd
In [2]: pd.__version__
Out[2]: '2.2.0'
In [3]: i = pd.Index(['a', 'b', 'c', None], dtype='category')
In [4]: i
Out[4]: CategoricalIndex(['a', 'b', 'c', nan], categories=['a', 'b', 'c'], ordered=False, dtype='category')
In [5]: i.difference(['1'])
Out[5]: CategoricalIndex(['a', 'b', 'c', nan], categories=['a', 'b', 'c'], ordered=False, dtype='category')
In [6]: i.difference(['1', None])
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:3802, in Index.get_loc(self, key)
3801 try:
-> 3802 return self._engine.get_loc(casted_key)
3803 except KeyError as err:
File index.pyx:153, in pandas._libs.index.IndexEngine.get_loc()
File index.pyx:182, in pandas._libs.index.IndexEngine.get_loc()
File pandas/_libs/hashtable_class_helper.pxi:7081, in pandas._libs.hashtable.PyObjectHashTable.get_item()
File pandas/_libs/hashtable_class_helper.pxi:7089, in pandas._libs.hashtable.PyObjectHashTable.get_item()
KeyError: nan
The above exception was the direct cause of the following exception:
KeyError Traceback (most recent call last)
Cell In[6], line 1
----> 1 i.difference(['1', None])
File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:3661, in Index.difference(self, other, sort)
3658 return result.sort_values()
3659 return result
-> 3661 result = self._difference(other, sort=sort)
3662 return self._wrap_difference_result(other, result)
File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:3667, in Index._difference(self, other, sort)
3664 def _difference(self, other, sort):
3665 # overridden by RangeIndex
3666 other = other.unique()
-> 3667 the_diff = self[other.get_indexer_for(self) == -1]
3668 the_diff = the_diff if self.is_unique else the_diff.unique()
3669 the_diff = _maybe_try_sort(the_diff, sort)
File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:6181, in Index.get_indexer_for(self, target)
6163 """
6164 Guaranteed return of an indexer even when non-unique.
6165
(...)
6178 array([0, 2])
6179 """
6180 if self._index_as_unique:
-> 6181 return self.get_indexer(target)
6182 indexer, _ = self.get_indexer_non_unique(target)
6183 return indexer
File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:3923, in Index.get_indexer(self, target, method, limit, tolerance)
3917 indexer = algos.take_nd(categories_indexer, target.codes, fill_value=-1)
3919 if (not self._is_multi and self.hasnans) and target.hasnans:
3920 # Exclude MultiIndex because hasnans raises NotImplementedError
3921 # we should only get here if we are unique, so loc is an integer
3922 # GH#41934
-> 3923 loc = self.get_loc(np.nan)
3924 mask = target.isna()
3925 indexer[mask] = loc
File /nvme/0/pgali/envs/cudfdev/lib/python3.10/site-packages/pandas/core/indexes/base.py:3809, in Index.get_loc(self, key)
3804 if isinstance(casted_key, slice) or (
3805 isinstance(casted_key, abc.Iterable)
3806 and any(isinstance(x, slice) for x in casted_key)
3807 ):
3808 raise InvalidIndexError(key)
-> 3809 raise KeyError(key) from err
3810 except TypeError:
3811 # If we have a listlike key, _check_indexing_error will raise
3812 # InvalidIndexError. Otherwise we fall through and re-raise
3813 # the TypeError.
3814 self._check_indexing_error(key)
KeyError: nanIssue Description
There seems to be a KeyError in pandas-2.2, but in previous versions of pandas there was no error.
Expected Behavior
No error.
Installed Versions
INSTALLED VERSIONS
commit : fd3f571
python : 3.10.13.final.0
python-bits : 64
OS : Linux
OS-release : 5.15.0-91-generic
Version : #101-Ubuntu SMP Tue Nov 14 13:30:08 UTC 2023
machine : x86_64
processor : x86_64
byteorder : little
LC_ALL : None
LANG : en_US.UTF-8
LOCALE : en_US.UTF-8
pandas : 2.2.0
numpy : 1.26.4
pytz : 2024.1
dateutil : 2.8.2
setuptools : 69.0.3
pip : 24.0
Cython : 3.0.8
pytest : 7.4.4
hypothesis : 6.98.3
sphinx : 6.2.1
blosc : None
feather : None
xlsxwriter : None
lxml.etree : None
html5lib : None
pymysql : None
psycopg2 : None
jinja2 : 3.1.3
IPython : 8.21.0
pandas_datareader : None
adbc-driver-postgresql: None
adbc-driver-sqlite : None
bs4 : 4.12.3
bottleneck : None
dataframe-api-compat : None
fastparquet : None
fsspec : 2024.2.0
gcsfs : None
matplotlib : None
numba : 0.59.0
numexpr : None
odfpy : None
openpyxl : None
pandas_gbq : None
pyarrow : 14.0.1
pyreadstat : None
python-calamine : None
pyxlsb : None
s3fs : 2024.2.0
scipy : 1.12.0
sqlalchemy : 2.0.25
tables : None
tabulate : 0.9.0
xarray : None
xlrd : None
zstandard : None
tzdata : 2023.4
qtpy : None
pyqt5 : None