REGR: Notebook (html) repr of DataFrame no longer follows min_rows/max_rows settings (#37363)

* TST: add failing test

* FIX: fix display logic

* TST: add test cases

* TYP: type max_rows on top of the method

Author: ivanovmg

pandas/io/formats/format.py

@@ -639,20 +639,31 @@ def _calc_max_cols_fitted(self) -> Optional[int]:
 
     def _calc_max_rows_fitted(self) -> Optional[int]:
         """Number of rows with data fitting the screen."""
-        if not self._is_in_terminal():
-            return self.max_rows
+        max_rows: Optional[int]
 
-        _, height = get_terminal_size()
-        if self.max_rows == 0:
-            # rows available to fill with actual data
-            return height - self._get_number_of_auxillary_rows()
+        if self._is_in_terminal():
+            _, height = get_terminal_size()
+            if self.max_rows == 0:
+                # rows available to fill with actual data
+                return height - self._get_number_of_auxillary_rows()
 
-        max_rows: Optional[int]
-        if self._is_screen_short(height):
-            max_rows = height
+            if self._is_screen_short(height):
+                max_rows = height
+            else:
+                max_rows = self.max_rows
         else:
             max_rows = self.max_rows
 
+        return self._adjust_max_rows(max_rows)
+
+    def _adjust_max_rows(self, max_rows: Optional[int]) -> Optional[int]:
+        """Adjust max_rows using display logic.
+
+        See description here:
+        https://pandas.pydata.org/docs/dev/user_guide/options.html#frequently-used-options
+
+        GH #37359
+        """
         if max_rows:
             if (len(self.frame) > max_rows) and self.min_rows:
                 # if truncated, set max_rows showed to min_rows

pandas/tests/io/formats/test_format.py

@@ -2029,6 +2029,35 @@ def test_period(self):
         )
         assert str(df) == exp
 
+    @pytest.mark.parametrize(
+        "length, max_rows, min_rows, expected",
+        [
+            (10, 10, 10, 10),
+            (10, 10, None, 10),
+            (10, 8, None, 8),
+            (20, 30, 10, 30),  # max_rows > len(frame), hence max_rows
+            (50, 30, 10, 10),  # max_rows < len(frame), hence min_rows
+            (100, 60, 10, 10),  # same
+            (60, 60, 10, 60),  # edge case
+            (61, 60, 10, 10),  # edge case
+        ],
+    )
+    def test_max_rows_fitted(self, length, min_rows, max_rows, expected):
+        """Check that display logic is correct.
+
+        GH #37359
+
+        See description here:
+        https://pandas.pydata.org/docs/dev/user_guide/options.html#frequently-used-options
+        """
+        formatter = fmt.DataFrameFormatter(
+            DataFrame(np.random.rand(length, 3)),
+            max_rows=max_rows,
+            min_rows=min_rows,
+        )
+        result = formatter.max_rows_fitted
+        assert result == expected
+
 
 def gen_series_formatting():
     s1 = Series(["a"] * 100)