Merge branch 'main' into issue2

Author: ahmedibrhm

.github/workflows/32-bit-linux.yml

@@ -12,6 +12,9 @@ on:
     paths-ignore:
       - "doc/**"
 
+permissions:
+  contents: read
+
 jobs:
   pytest:
     runs-on: ubuntu-latest

.github/workflows/assign.yml

@@ -3,8 +3,14 @@ on:
   issue_comment:
     types: created
 
+permissions:
+  contents: read
+
 jobs:
   issue_assign:
+    permissions:
+      issues: write
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
     - if: github.event.comment.body == 'take'

.github/workflows/asv-bot.yml

@@ -9,8 +9,15 @@ env:
   ENV_FILE: environment.yml
   COMMENT: ${{github.event.comment.body}}
 
+permissions:
+  contents: read
+
 jobs:
   autotune:
+    permissions:
+      contents: read
+      issues: write
+      pull-requests: write
     name: "Run benchmarks"
     # TODO: Support more benchmarking options later, against different branches, against self, etc
     if: startsWith(github.event.comment.body, '@github-actions benchmark')

.github/workflows/autoupdate-pre-commit-config.yml

@@ -5,8 +5,14 @@ on:
     - cron: "0 7 1 * *" # At 07:00 on 1st of every month.
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   update-pre-commit:
+    permissions:
+      contents: write  # for technote-space/create-pr-action to push code
+      pull-requests: write  # for technote-space/create-pr-action to create a PR
     if: github.repository_owner == 'pandas-dev'
     name: Autoupdate pre-commit config
     runs-on: ubuntu-latest

.github/workflows/code-checks.yml

@@ -14,6 +14,9 @@ env:
   ENV_FILE: environment.yml
   PANDAS_CI: 1
 
+permissions:
+  contents: read
+
 jobs:
   pre_commit:
     name: pre-commit

.github/workflows/docbuild-and-upload.yml

@@ -14,6 +14,9 @@ env:
   ENV_FILE: environment.yml
   PANDAS_CI: 1
 
+permissions:
+  contents: read
+
 jobs:
   web_and_docs:
     name: Doc Build and Upload

.github/workflows/macos-windows.yml

@@ -18,6 +18,9 @@ env:
   PATTERN: "not slow and not db and not network and not single_cpu"
 
 
+permissions:
+  contents: read
+
 jobs:
   pytest:
     defaults:

.github/workflows/python-dev.yml

@@ -27,6 +27,9 @@ env:
   COVERAGE: true
   PYTEST_TARGET: pandas
 
+permissions:
+  contents: read
+
 jobs:
   build:
     if: false # Comment this line out to "unfreeze"

.github/workflows/sdist.yml

@@ -13,6 +13,9 @@ on:
     paths-ignore:
       - "doc/**"
 
+permissions:
+  contents: read
+
 jobs:
   build:
     if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}

.github/workflows/stale-pr.yml

@@ -4,8 +4,13 @@ on:
   # * is a special character in YAML so you have to quote this string
   - cron: "0 0 * * *"
 
+permissions:
+  contents: read
+
 jobs:
   stale:
+    permissions:
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
     - uses: actions/stale@v4

.github/workflows/ubuntu.yml

@@ -15,6 +15,9 @@ on:
 env:
   PANDAS_CI: 1
 
+permissions:
+  contents: read
+
 jobs:
   pytest:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -93,7 +93,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - pyright@1.1.253
+        - pyright@1.1.258
 -   repo: local
     hooks:
     -   id: pyright_reportGeneralTypeIssues

README.md

@@ -169,4 +169,4 @@ Or maybe through using pandas you have an idea of your own or are looking for so
 
 Feel free to ask questions on the [mailing list](https://groups.google.com/forum/?fromgroups#!forum/pydata) or on [Gitter](https://gitter.im/pydata/pandas).
 
-As contributors and maintainers to this project, you are expected to abide by pandas' code of conduct. More information can be found at: [Contributor Code of Conduct](https://github.com/pandas-dev/pandas/blob/main/.github/CODE_OF_CONDUCT.md)
+As contributors and maintainers to this project, you are expected to abide by pandas' code of conduct. More information can be found at: [Contributor Code of Conduct](https://github.com/pandas-dev/.github/blob/master/CODE_OF_CONDUCT.md)

asv_bench/benchmarks/tslibs/strftime.py renamed to asv_bench/benchmarks/strftime.py

@@ -26,11 +26,14 @@ Exceptions and warnings
 
    errors.AbstractMethodError
    errors.AccessorRegistrationWarning
+   errors.AttributeConflictWarning
+   errors.ClosedFileError
    errors.CSSWarning
    errors.DataError
    errors.DtypeWarning
    errors.DuplicateLabelError
    errors.EmptyDataError
+   errors.IncompatibilityWarning
    errors.IndexingError
    errors.InvalidIndexError
    errors.IntCastingNaNError
@@ -44,6 +47,7 @@ Exceptions and warnings
    errors.ParserError
    errors.ParserWarning
    errors.PerformanceWarning
+   errors.PossibleDataLossError
    errors.PyperclipException
    errors.PyperclipWindowsException
    errors.SettingWithCopyError

doc/source/reference/testing.rst

@@ -1001,6 +1001,8 @@ Groupby/resample/rolling
 - Bug in :meth:`.DataFrameGroupBy.describe` and :meth:`.SeriesGroupBy.describe` produces inconsistent results for empty datasets (:issue:`41575`)
 - Bug in :meth:`DataFrame.resample` reduction methods when used with ``on`` would attempt to aggregate the provided column (:issue:`47079`)
 - Bug in :meth:`DataFrame.groupby` and :meth:`Series.groupby` would not respect ``dropna=False`` when the input DataFrame/Series had a NaN values in a :class:`MultiIndex` (:issue:`46783`)
+- Bug in :meth:`DataFrameGroupBy.resample` raises ``KeyError`` when getting the result from a key list which misses the resample key (:issue:`47362`)
+-
 
 Reshaping
 ^^^^^^^^^

doc/source/whatsnew/v1.5.0.rst

@@ -73,7 +73,6 @@
 from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import (
-    DT64NS_DTYPE,
     is_all_strings,
     is_categorical_dtype,
     is_datetime64_any_dtype,
@@ -1103,6 +1102,7 @@ def _add_datetimelike_scalar(self, other) -> DatetimeArray:
         self = cast("TimedeltaArray", self)
 
         from pandas.core.arrays import DatetimeArray
+        from pandas.core.arrays.datetimes import tz_to_dtype
 
         assert other is not NaT
         other = Timestamp(other)
@@ -1113,10 +1113,17 @@ def _add_datetimelike_scalar(self, other) -> DatetimeArray:
             # Preserve our resolution
             return DatetimeArray._simple_new(result, dtype=result.dtype)
 
+        if self._reso != other._reso:
+            raise NotImplementedError(
+                "Addition between TimedeltaArray and Timestamp with mis-matched "
+                "resolutions is not yet supported."
+            )
+
         i8 = self.asi8
         result = checked_add_with_arr(i8, other.value, arr_mask=self._isnan)
-        dtype = DatetimeTZDtype(tz=other.tz) if other.tz else DT64NS_DTYPE
-        return DatetimeArray(result, dtype=dtype, freq=self.freq)
+        dtype = tz_to_dtype(tz=other.tz, unit=self._unit)
+        res_values = result.view(f"M8[{self._unit}]")
+        return DatetimeArray._simple_new(res_values, dtype=dtype, freq=self.freq)
 
     @final
     def _add_datetime_arraylike(self, other) -> DatetimeArray:

pandas/core/arrays/datetimelike.py

@@ -91,22 +91,23 @@
 _midnight = time(0, 0)
 
 
-def tz_to_dtype(tz):
+def tz_to_dtype(tz: tzinfo | None, unit: str = "ns"):
     """
     Return a datetime64[ns] dtype appropriate for the given timezone.
 
     Parameters
     ----------
     tz : tzinfo or None
+    unit : str, default "ns"
 
     Returns
     -------
     np.dtype or Datetime64TZDType
     """
     if tz is None:
-        return DT64NS_DTYPE
+        return np.dtype(f"M8[{unit}]")
     else:
-        return DatetimeTZDtype(tz=tz)
+        return DatetimeTZDtype(tz=tz, unit=unit)
 
 
 def _field_accessor(name: str, field: str, docstring=None):
@@ -800,7 +801,7 @@ def tz_convert(self, tz) -> DatetimeArray:
             )
 
         # No conversion since timestamps are all UTC to begin with
-        dtype = tz_to_dtype(tz)
+        dtype = tz_to_dtype(tz, unit=self._unit)
         return self._simple_new(self._ndarray, dtype=dtype, freq=self.freq)
 
     @dtl.ravel_compat
@@ -965,10 +966,14 @@ def tz_localize(self, tz, ambiguous="raise", nonexistent="raise") -> DatetimeArr
             # Convert to UTC
 
             new_dates = tzconversion.tz_localize_to_utc(
-                self.asi8, tz, ambiguous=ambiguous, nonexistent=nonexistent
+                self.asi8,
+                tz,
+                ambiguous=ambiguous,
+                nonexistent=nonexistent,
+                reso=self._reso,
             )
-        new_dates = new_dates.view(DT64NS_DTYPE)
-        dtype = tz_to_dtype(tz)
+        new_dates = new_dates.view(f"M8[{self._unit}]")
+        dtype = tz_to_dtype(tz, unit=self._unit)
 
         freq = None
         if timezones.is_utc(tz) or (len(self) == 1 and not isna(new_dates[0])):

pandas/core/arrays/datetimes.py

@@ -398,7 +398,7 @@ def __mul__(self, other) -> TimedeltaArray:
             freq = None
             if self.freq is not None and not isna(other):
                 freq = self.freq * other
-            return type(self)(result, freq=freq)
+            return type(self)._simple_new(result, dtype=result.dtype, freq=freq)
 
         if not hasattr(other, "dtype"):
             # list, tuple
@@ -412,13 +412,14 @@ def __mul__(self, other) -> TimedeltaArray:
             # this multiplication will succeed only if all elements of other
             #  are int or float scalars, so we will end up with
             #  timedelta64[ns]-dtyped result
-            result = [self[n] * other[n] for n in range(len(self))]
+            arr = self._ndarray
+            result = [arr[n] * other[n] for n in range(len(self))]
             result = np.array(result)
-            return type(self)(result)
+            return type(self)._simple_new(result, dtype=result.dtype)
 
         # numpy will accept float or int dtype, raise TypeError for others
         result = self._ndarray * other
-        return type(self)(result)
+        return type(self)._simple_new(result, dtype=result.dtype)
 
     __rmul__ = __mul__
 
@@ -446,7 +447,8 @@ def __truediv__(self, other):
             if self.freq is not None:
                 # Tick division is not implemented, so operate on Timedelta
                 freq = self.freq.delta / other
-            return type(self)(result, freq=freq)
+                freq = to_offset(freq)
+            return type(self)._simple_new(result, dtype=result.dtype, freq=freq)
 
         if not hasattr(other, "dtype"):
             # e.g. list, tuple
@@ -462,6 +464,7 @@ def __truediv__(self, other):
         elif is_object_dtype(other.dtype):
             # We operate on raveled arrays to avoid problems in inference
             #  on NaT
+            # TODO: tests with non-nano
             srav = self.ravel()
             orav = other.ravel()
             result_list = [srav[n] / orav[n] for n in range(len(srav))]
@@ -488,7 +491,7 @@ def __truediv__(self, other):
 
         else:
             result = self._ndarray / other
-            return type(self)(result)
+            return type(self)._simple_new(result, dtype=result.dtype)
 
     @unpack_zerodim_and_defer("__rtruediv__")
     def __rtruediv__(self, other):

pandas/core/arrays/timedeltas.py

@@ -404,9 +404,12 @@ def _outer_indexer(
     # associated code in pandas 2.0.
     _is_backward_compat_public_numeric_index: bool = False
 
-    _engine_type: type[libindex.IndexEngine] | type[
-        libindex.ExtensionEngine
-    ] = libindex.ObjectEngine
+    @property
+    def _engine_type(
+        self,
+    ) -> type[libindex.IndexEngine] | type[libindex.ExtensionEngine]:
+        return libindex.ObjectEngine
+
     # whether we support partial string indexing. Overridden
     # in DatetimeIndex and PeriodIndex
     _supports_partial_string_indexing = False

pandas/core/indexes/base.py

@@ -192,7 +192,7 @@ def _should_fallback_to_positional(self) -> bool:
     _values: Categorical
 
     @property
-    def _engine_type(self):
+    def _engine_type(self) -> type[libindex.IndexEngine]:
         # self.codes can have dtype int8, int16, int32 or int64, so we need
         # to return the corresponding engine type (libindex.Int8Engine, etc.).
         return {

pandas/core/indexes/category.py

@@ -252,9 +252,12 @@ class DatetimeIndex(DatetimeTimedeltaMixin):
     _typ = "datetimeindex"
 
     _data_cls = DatetimeArray
-    _engine_type = libindex.DatetimeEngine
     _supports_partial_string_indexing = True
 
+    @property
+    def _engine_type(self) -> type[libindex.DatetimeEngine]:
+        return libindex.DatetimeEngine
+
     _data: DatetimeArray
     inferred_freq: str | None
     tz: tzinfo | None

pandas/core/indexes/datetimes.py

@@ -106,7 +106,7 @@ class NumericIndex(Index):
     }
 
     @property
-    def _engine_type(self):
+    def _engine_type(self) -> type[libindex.IndexEngine]:
         # error: Invalid index type "Union[dtype[Any], ExtensionDtype]" for
         # "Dict[dtype[Any], Type[IndexEngine]]"; expected type "dtype[Any]"
         return self._engine_types[self.dtype]  # type: ignore[index]
@@ -373,10 +373,13 @@ class Int64Index(IntegerIndex):
     __doc__ = _num_index_shared_docs["class_descr"] % _index_descr_args
 
     _typ = "int64index"
-    _engine_type = libindex.Int64Engine
     _default_dtype = np.dtype(np.int64)
     _dtype_validation_metadata = (is_signed_integer_dtype, "signed integer")
 
+    @property
+    def _engine_type(self) -> type[libindex.Int64Engine]:
+        return libindex.Int64Engine
+
 
 class UInt64Index(IntegerIndex):
     _index_descr_args = {
@@ -388,10 +391,13 @@ class UInt64Index(IntegerIndex):
     __doc__ = _num_index_shared_docs["class_descr"] % _index_descr_args
 
     _typ = "uint64index"
-    _engine_type = libindex.UInt64Engine
     _default_dtype = np.dtype(np.uint64)
     _dtype_validation_metadata = (is_unsigned_integer_dtype, "unsigned integer")
 
+    @property
+    def _engine_type(self) -> type[libindex.UInt64Engine]:
+        return libindex.UInt64Engine
+
 
 class Float64Index(NumericIndex):
     _index_descr_args = {