diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst
index c35473b852eb9..d8f94608b0b7b 100644
--- a/doc/source/whatsnew/v2.2.0.rst
+++ b/doc/source/whatsnew/v2.2.0.rst
@@ -166,8 +166,7 @@ MultiIndex
 
 I/O
 ^^^
--
--
+- Bug in :func:`read_excel`, with ``engine="xlrd"`` (``xls`` files) erroring when file contains NaNs/Infs (:issue:`54564`)
 
 Period
 ^^^^^^
diff --git a/pandas/io/excel/_xlrd.py b/pandas/io/excel/_xlrd.py
index c68a0ab516e05..a444970792e6e 100644
--- a/pandas/io/excel/_xlrd.py
+++ b/pandas/io/excel/_xlrd.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from datetime import time
+import math
 from typing import TYPE_CHECKING
 
 import numpy as np
@@ -120,9 +121,11 @@ def _parse_cell(cell_contents, cell_typ):
             elif cell_typ == XL_CELL_NUMBER:
                 # GH5394 - Excel 'numbers' are always floats
                 # it's a minimal perf hit and less surprising
-                val = int(cell_contents)
-                if val == cell_contents:
-                    cell_contents = val
+                if math.isfinite(cell_contents):
+                    # GH54564 - don't attempt to convert NaN/Inf
+                    val = int(cell_contents)
+                    if val == cell_contents:
+                        cell_contents = val
             return cell_contents
 
         data = []
diff --git a/pandas/tests/io/data/excel/test6.xls b/pandas/tests/io/data/excel/test6.xls
new file mode 100644
index 0000000000000..e43a1a67510d8
Binary files /dev/null and b/pandas/tests/io/data/excel/test6.xls differ
diff --git a/pandas/tests/io/excel/test_xlrd.py b/pandas/tests/io/excel/test_xlrd.py
index 509029861715e..6d5008ca9ee68 100644
--- a/pandas/tests/io/excel/test_xlrd.py
+++ b/pandas/tests/io/excel/test_xlrd.py
@@ -1,5 +1,6 @@
 import io
 
+import numpy as np
 import pytest
 
 import pandas as pd
@@ -44,6 +45,17 @@ def test_read_xlsx_fails(datapath):
         pd.read_excel(path, engine="xlrd")
 
 
+def test_nan_in_xls(datapath):
+    # GH 54564
+    path = datapath("io", "data", "excel", "test6.xls")
+
+    expected = pd.DataFrame({0: np.r_[0, 2].astype("int64"), 1: np.r_[1, np.nan]})
+
+    result = pd.read_excel(path, header=None)
+
+    tm.assert_frame_equal(result, expected)
+
+
 @pytest.mark.parametrize(
     "file_header",
     [