Merge remote-tracking branch 'upstream/main' into test_numpy_complex2

Author: MichaelTiemannOSC

doc/source/whatsnew/v2.2.0.rst

@@ -227,6 +227,7 @@ MultiIndex
 
 I/O
 ^^^
+- Bug in :func:`read_csv` where ``on_bad_lines="warn"`` would write to ``stderr`` instead of raise a Python warning. This now yields a :class:`.errors.ParserWarning` (:issue:`54296`)
 - Bug in :func:`read_excel`, with ``engine="xlrd"`` (``xls`` files) erroring when file contains NaNs/Infs (:issue:`54564`)
 
 Period

pandas/_libs/parsers.pyx

@@ -6,7 +6,6 @@ from csv import (
     QUOTE_NONE,
     QUOTE_NONNUMERIC,
 )
-import sys
 import time
 import warnings
 
@@ -880,9 +879,15 @@ cdef class TextReader:
 
     cdef _check_tokenize_status(self, int status):
         if self.parser.warn_msg != NULL:
-            print(PyUnicode_DecodeUTF8(
-                self.parser.warn_msg, strlen(self.parser.warn_msg),
-                self.encoding_errors), file=sys.stderr)
+            warnings.warn(
+                PyUnicode_DecodeUTF8(
+                    self.parser.warn_msg,
+                    strlen(self.parser.warn_msg),
+                    self.encoding_errors
+                ),
+                ParserWarning,
+                stacklevel=find_stack_level()
+            )
             free(self.parser.warn_msg)
             self.parser.warn_msg = NULL
 

pandas/compat/__init__.py

@@ -30,6 +30,7 @@
     pa_version_under9p0,
     pa_version_under11p0,
     pa_version_under13p0,
+    pa_version_under14p0,
 )
 
 if TYPE_CHECKING:
@@ -186,6 +187,7 @@ def get_bz2_file() -> type[pandas.compat.compressors.BZ2File]:
     "pa_version_under9p0",
     "pa_version_under11p0",
     "pa_version_under13p0",
+    "pa_version_under14p0",
     "IS64",
     "ISMUSL",
     "PY310",

pandas/compat/pyarrow.py

@@ -15,6 +15,7 @@
     pa_version_under11p0 = _palv < Version("11.0.0")
     pa_version_under12p0 = _palv < Version("12.0.0")
     pa_version_under13p0 = _palv < Version("13.0.0")
+    pa_version_under14p0 = _palv < Version("14.0.0")
 except ImportError:
     pa_version_under7p0 = True
     pa_version_under8p0 = True
@@ -23,3 +24,4 @@
     pa_version_under11p0 = True
     pa_version_under12p0 = True
     pa_version_under13p0 = True
+    pa_version_under14p0 = True

pandas/core/internals/managers.py

@@ -969,6 +969,10 @@ def fast_xs(self, loc: int) -> SingleBlockManager:
         n = len(self)
 
         if isinstance(dtype, ExtensionDtype):
+            # TODO: use object dtype as workaround for non-performant
+            #  EA.__setitem__ methods. (primarily ArrowExtensionArray.__setitem__
+            #  when iteratively setting individual values)
+            #  https://github.com/pandas-dev/pandas/pull/54508#issuecomment-1675827918
             result = np.empty(n, dtype=object)
         else:
             result = np.empty(n, dtype=dtype)

pandas/io/clipboard/__init__.py

@@ -17,9 +17,12 @@
 On Windows, no additional modules are needed.
 On Mac, the pyobjc module is used, falling back to the pbcopy and pbpaste cli
     commands. (These commands should come with OS X.).
-On Linux, install xclip or xsel via package manager. For example, in Debian:
+On Linux, install xclip, xsel, or wl-clipboard (for "wayland" sessions) via
+package manager.
+For example, in Debian:
     sudo apt-get install xclip
     sudo apt-get install xsel
+    sudo apt-get install wl-clipboard
 
 Otherwise on Linux, you will need the PyQt5 modules installed.
 
@@ -28,20 +31,19 @@
 Cygwin is currently not supported.
 
 Security Note: This module runs programs with these names:
-    - which
-    - where
     - pbcopy
     - pbpaste
     - xclip
     - xsel
+    - wl-copy/wl-paste
     - klipper
     - qdbus
 A malicious user could rename or add programs with these names, tricking
 Pyperclip into running them with whatever permissions the Python process has.
 
 """
 
-__version__ = "1.7.0"
+__version__ = "1.8.2"
 
 
 import contextlib
@@ -55,7 +57,7 @@
 )
 import os
 import platform
-from shutil import which
+from shutil import which as _executable_exists
 import subprocess
 import time
 import warnings
@@ -74,25 +76,14 @@
 EXCEPT_MSG = """
     Pyperclip could not find a copy/paste mechanism for your system.
     For more information, please visit
-    https://pyperclip.readthedocs.io/en/latest/#not-implemented-error
+    https://pyperclip.readthedocs.io/en/latest/index.html#not-implemented-error
     """
 
 ENCODING = "utf-8"
 
-# The "which" unix command finds where a command is.
-if platform.system() == "Windows":
-    WHICH_CMD = "where"
-else:
-    WHICH_CMD = "which"
 
-
-def _executable_exists(name):
-    return (
-        subprocess.call(
-            [WHICH_CMD, name], stdout=subprocess.PIPE, stderr=subprocess.PIPE
-        )
-        == 0
-    )
+class PyperclipTimeoutException(PyperclipException):
+    pass
 
 
 def _stringifyText(text) -> str:
@@ -229,6 +220,32 @@ def paste_xsel(primary=False):
     return copy_xsel, paste_xsel
 
 
+def init_wl_clipboard():
+    PRIMARY_SELECTION = "-p"
+
+    def copy_wl(text, primary=False):
+        text = _stringifyText(text)  # Converts non-str values to str.
+        args = ["wl-copy"]
+        if primary:
+            args.append(PRIMARY_SELECTION)
+        if not text:
+            args.append("--clear")
+            subprocess.check_call(args, close_fds=True)
+        else:
+            p = subprocess.Popen(args, stdin=subprocess.PIPE, close_fds=True)
+            p.communicate(input=text.encode(ENCODING))
+
+    def paste_wl(primary=False):
+        args = ["wl-paste", "-n"]
+        if primary:
+            args.append(PRIMARY_SELECTION)
+        p = subprocess.Popen(args, stdout=subprocess.PIPE, close_fds=True)
+        stdout, _stderr = p.communicate()
+        return stdout.decode(ENCODING)
+
+    return copy_wl, paste_wl
+
+
 def init_klipper_clipboard():
     def copy_klipper(text):
         text = _stringifyText(text)  # Converts non-str values to str.
@@ -534,7 +551,7 @@ def determine_clipboard():
         return init_windows_clipboard()
 
     if platform.system() == "Linux":
-        if which("wslconfig.exe"):
+        if _executable_exists("wslconfig.exe"):
             return init_wsl_clipboard()
 
     # Setup for the macOS platform:
@@ -549,6 +566,8 @@ def determine_clipboard():
 
     # Setup for the LINUX platform:
     if HAS_DISPLAY:
+        if os.environ.get("WAYLAND_DISPLAY") and _executable_exists("wl-copy"):
+            return init_wl_clipboard()
         if _executable_exists("xsel"):
             return init_xsel_clipboard()
         if _executable_exists("xclip"):
@@ -602,6 +621,7 @@ def set_clipboard(clipboard):
         "qt": init_qt_clipboard,  # TODO - split this into 'qtpy', 'pyqt4', and 'pyqt5'
         "xclip": init_xclip_clipboard,
         "xsel": init_xsel_clipboard,
+        "wl-clipboard": init_wl_clipboard,
         "klipper": init_klipper_clipboard,
         "windows": init_windows_clipboard,
         "no": init_no_clipboard,
@@ -671,7 +691,56 @@ def is_available() -> bool:
 copy, paste = lazy_load_stub_copy, lazy_load_stub_paste
 
 
-__all__ = ["copy", "paste", "set_clipboard", "determine_clipboard"]
+def waitForPaste(timeout=None):
+    """This function call blocks until a non-empty text string exists on the
+    clipboard. It returns this text.
+
+    This function raises PyperclipTimeoutException if timeout was set to
+    a number of seconds that has elapsed without non-empty text being put on
+    the clipboard."""
+    startTime = time.time()
+    while True:
+        clipboardText = paste()
+        if clipboardText != "":
+            return clipboardText
+        time.sleep(0.01)
+
+        if timeout is not None and time.time() > startTime + timeout:
+            raise PyperclipTimeoutException(
+                "waitForPaste() timed out after " + str(timeout) + " seconds."
+            )
+
+
+def waitForNewPaste(timeout=None):
+    """This function call blocks until a new text string exists on the
+    clipboard that is different from the text that was there when the function
+    was first called. It returns this text.
+
+    This function raises PyperclipTimeoutException if timeout was set to
+    a number of seconds that has elapsed without non-empty text being put on
+    the clipboard."""
+    startTime = time.time()
+    originalText = paste()
+    while True:
+        currentText = paste()
+        if currentText != originalText:
+            return currentText
+        time.sleep(0.01)
+
+        if timeout is not None and time.time() > startTime + timeout:
+            raise PyperclipTimeoutException(
+                "waitForNewPaste() timed out after " + str(timeout) + " seconds."
+            )
+
+
+__all__ = [
+    "copy",
+    "paste",
+    "waitForPaste",
+    "waitForNewPaste",
+    "set_clipboard",
+    "determine_clipboard",
+]
 
 # pandas aliases
 clipboard_get = paste

pandas/io/parsers/python_parser.py

@@ -13,23 +13,25 @@
 import csv
 from io import StringIO
 import re
-import sys
 from typing import (
     IO,
     TYPE_CHECKING,
     DefaultDict,
     Literal,
     cast,
 )
+import warnings
 
 import numpy as np
 
 from pandas._libs import lib
 from pandas.errors import (
     EmptyDataError,
     ParserError,
+    ParserWarning,
 )
 from pandas.util._decorators import cache_readonly
+from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import (
     is_bool_dtype,
@@ -778,8 +780,11 @@ def _alert_malformed(self, msg: str, row_num: int) -> None:
         if self.on_bad_lines == self.BadLineHandleMethod.ERROR:
             raise ParserError(msg)
         if self.on_bad_lines == self.BadLineHandleMethod.WARN:
-            base = f"Skipping line {row_num}: "
-            sys.stderr.write(base + msg + "\n")
+            warnings.warn(
+                f"Skipping line {row_num}: {msg}\n",
+                ParserWarning,
+                stacklevel=find_stack_level(),
+            )
 
     def _next_iter_line(self, row_num: int) -> list[Scalar] | None:
         """

pandas/tests/extension/test_arrow.py

@@ -40,6 +40,7 @@
     pa_version_under9p0,
     pa_version_under11p0,
     pa_version_under13p0,
+    pa_version_under14p0,
 )
 
 from pandas.core.dtypes.dtypes import (
@@ -927,7 +928,7 @@ def _is_temporal_supported(self, opname, pa_dtype):
                 or (
                     opname
                     in ("__truediv__", "__rtruediv__", "__floordiv__", "__rfloordiv__")
-                    and not pa_version_under13p0
+                    and not pa_version_under14p0
                 )
             )
             and pa.types.is_duration(pa_dtype)