diff --git a/pandas/core/frame.py b/pandas/core/frame.py
index 5d802e8a6a77f..3b69dae785534 100644
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -2892,14 +2892,17 @@ def _get_value(self, index, col, takeable: bool = False):
         engine = self.index._engine
 
         try:
-            return engine.get_value(series._values, index)
+            if isinstance(series._values, np.ndarray):
+                # i.e. not EA, we can use engine
+                return engine.get_value(series._values, index)
+            else:
+                loc = series.index.get_loc(index)
+                return series._values[loc]
         except KeyError:
             # GH 20629
             if self.index.nlevels > 1:
                 # partial indexing forbidden
                 raise
-        except (TypeError, ValueError):
-            pass
 
         # we cannot handle direct indexing
         # use positional
diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py
index 722fe152e6a85..075380f410220 100755
--- a/pandas/core/indexing.py
+++ b/pandas/core/indexing.py
@@ -2101,8 +2101,7 @@ def __setitem__(self, key, value):
         if len(key) != self.ndim:
             raise ValueError("Not enough indexers for scalar access (setting)!")
         key = list(self._convert_key(key, is_setter=True))
-        key.append(value)
-        self.obj._set_value(*key, takeable=self._takeable)
+        self.obj._set_value(*key, value=value, takeable=self._takeable)
 
 
 @Appender(IndexingMixin.at.__doc__)
diff --git a/pandas/core/series.py b/pandas/core/series.py
index e4883c4f0c38d..2af41393078cb 100644
--- a/pandas/core/series.py
+++ b/pandas/core/series.py
@@ -1060,9 +1060,12 @@ def _set_value(self, label, value, takeable: bool = False):
         try:
             if takeable:
                 self._values[label] = value
-            else:
+            elif isinstance(self._values, np.ndarray):
+                # i.e. not EA, so we can use _engine
                 self.index._engine.set_value(self._values, label, value)
-        except (KeyError, TypeError):
+            else:
+                self.loc[label] = value
+        except KeyError:
 
             # set using a non-recursive method
             self.loc[label] = value