WIP/CI: Debug ResourceWarnings unclosed io.BufferedRandom

.github/workflows/python-dev.yml

@@ -50,7 +50,7 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
         pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         pip install git+https://github.com/nedbat/coveragepy.git
-        pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov
+        pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov psutil
         pip list
 
     - name: Build Pandas

ci/deps/actions-38-db-min.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - psutil
 
   # required
   - numpy<1.20  # GH#39541 compat for pyarrow<3

ci/deps/actions-38-db.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - pytest-cov>=2.10.1  # this is only needed in the coverage build, ref: GH 35737
+  - psutil
 
   # pandas dependencies
   - aiobotocore<2.0.0

ci/deps/actions-38-locale.yaml

@@ -11,6 +11,7 @@ dependencies:
   - pytest-xdist>=1.31
   - pytest-asyncio>=0.12.0
   - hypothesis>=5.5.3
+  - psutil
 
   # pandas dependencies
   - beautifulsoup4

ci/deps/actions-38-locale_slow.yaml

@@ -11,6 +11,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - psutil
 
   # pandas dependencies
   - beautifulsoup4=4.8.2

ci/deps/actions-38-slow.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - psutil
 
   # pandas dependencies
   - beautifulsoup4

ci/deps/actions-38.yaml

@@ -11,6 +11,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - psutil
 
   # pandas dependencies
   - numpy

ci/deps/actions-39-numpydev.yaml

@@ -9,6 +9,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - psutil
 
   # pandas dependencies
   - python-dateutil

ci/deps/actions-39-slow.yaml

@@ -11,6 +11,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - psutil
 
   # pandas dependencies
   - beautifulsoup4

ci/deps/actions-39.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - psutil
 
   # pandas dependencies
   - beautifulsoup4

ci/deps/azure-macos-38.yaml

@@ -10,6 +10,7 @@ dependencies:
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - pytest-azurepipelines
+  - psutil
 
   # pandas dependencies
   - beautifulsoup4

ci/deps/azure-windows-38.yaml

@@ -11,6 +11,7 @@ dependencies:
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - pytest-azurepipelines
+  - psutil
 
   # pandas dependencies
   - blosc

ci/deps/azure-windows-39.yaml

@@ -11,6 +11,7 @@ dependencies:
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
   - pytest-azurepipelines
+  - psutil
 
   # pandas dependencies
   - beautifulsoup4

ci/deps/circle-38-arm64.yaml

@@ -19,3 +19,4 @@ dependencies:
   - flask
   - pip:
     - moto
+    - psutil

pandas/_testing/_warnings.py

@@ -155,16 +155,23 @@ def _assert_caught_no_extra_warnings(
                 #  suppress these
                 continue
 
-            extra_warnings.append(
-                (
-                    actual_warning.category.__name__,
-                    actual_warning.message,
-                    actual_warning.filename,
-                    actual_warning.lineno,
-                )
-            )
+            warning_data = [
+                actual_warning.category.__name__,
+                actual_warning.message,
+                actual_warning.filename,
+                actual_warning.lineno,
+            ]
+
+            if actual_warning.category == ResourceWarning:
+                import psutil
+
+                proc = psutil.Process()
+                flist = proc.open_files()
+                warning_data.append(flist)
+
+            extra_warnings.append(tuple(warning_data))
 
-    if extra_warnings:
+    if extra_warnings and any(data[0] == ResourceWarning for data in extra_warnings):
         raise AssertionError(f"Caused unexpected warning(s): {repr(extra_warnings)}")
 
 

pandas/conftest.py

@@ -17,8 +17,6 @@
 - Dtypes
 - Misc
 """
-# pyright: reportUntypedFunctionDecorator = false
-
 from collections import abc
 from datetime import (
     date,
@@ -31,13 +29,17 @@
 import operator
 import os
 
+# pyright: reportUntypedFunctionDecorator = false
+import sys
+
 from dateutil.tz import (
     tzlocal,
     tzutc,
 )
 import hypothesis
 from hypothesis import strategies as st
 import numpy as np
+import psutil
 import pytest
 from pytz import (
     FixedOffset,
@@ -199,6 +201,18 @@ def add_imports(doctest_namespace):
     doctest_namespace["pd"] = pd
 
 
+@pytest.fixture(autouse=True)
+def check_bufferedrandom_resourcewarning():
+    proc = psutil.Process()
+    with tm.assert_produces_warning(None):
+        yield
+        # sys.stderr for xdist
+        # https://github.com/pytest-dev/pytest/issues/1693#issuecomment-233282644
+        open_files = proc.open_files()
+        if open_files:
+            print(open_files, flush=True, file=sys.stderr)
+
+
 # ----------------------------------------------------------------
 # Common arguments
 # ----------------------------------------------------------------

pandas/tests/io/formats/test_printing.py

@@ -1,5 +1,4 @@
 import numpy as np
-import pytest
 
 import pandas._config.config as cf
 
@@ -120,16 +119,8 @@ def test_ambiguous_width(self):
 
 
 class TestTableSchemaRepr:
-    @classmethod
-    def setup_class(cls):
-        pytest.importorskip("IPython")
-
-        from IPython.core.interactiveshell import InteractiveShell
-
-        cls.display_formatter = InteractiveShell.instance().display_formatter
-
-    def test_publishes(self):
-
+    def test_publishes(self, ip):
+        ipython = ip.instance(config=ip.config)
         df = pd.DataFrame({"A": [1, 2]})
         objects = [df["A"], df, df]  # dataframe / series
         expected_keys = [
@@ -140,13 +131,13 @@ def test_publishes(self):
         opt = pd.option_context("display.html.table_schema", True)
         for obj, expected in zip(objects, expected_keys):
             with opt:
-                formatted = self.display_formatter.format(obj)
+                formatted = ipython.display_formatter.format(obj)
             assert set(formatted[0].keys()) == expected
 
         with_latex = pd.option_context("display.latex.repr", True)
 
         with opt, with_latex:
-            formatted = self.display_formatter.format(obj)
+            formatted = ipython.display_formatter.format(obj)
 
         expected = {
             "text/plain",
@@ -156,7 +147,7 @@ def test_publishes(self):
         }
         assert set(formatted[0].keys()) == expected
 
-    def test_publishes_not_implemented(self):
+    def test_publishes_not_implemented(self, ip):
         # column MultiIndex
         # GH 15996
         midx = pd.MultiIndex.from_product([["A", "B"], ["a", "b", "c"]])
@@ -165,7 +156,7 @@ def test_publishes_not_implemented(self):
         opt = pd.option_context("display.html.table_schema", True)
 
         with opt:
-            formatted = self.display_formatter.format(df)
+            formatted = ip.instance(config=ip.config).display_formatter.format(df)
 
         expected = {"text/plain", "text/html"}
         assert set(formatted[0].keys()) == expected
@@ -184,9 +175,9 @@ def test_config_default_off(self):
 
         assert result is None
 
-    def test_enable_data_resource_formatter(self):
+    def test_enable_data_resource_formatter(self, ip):
         # GH 10491
-        formatters = self.display_formatter.formatters
+        formatters = ip.instance(config=ip.config).display_formatter.formatters
         mimetype = "application/vnd.dataresource+json"
 
         with pd.option_context("display.html.table_schema", True):
@@ -202,4 +193,4 @@ def test_enable_data_resource_formatter(self):
             assert "application/vnd.dataresource+json" in formatters
             assert formatters[mimetype].enabled
             # smoke test that it works
-            self.display_formatter.format(cf)
+            ip.instance(config=ip.config).display_formatter.format(cf)