API: Deprecate regex=True default in Series.str.replace

doc/source/user_guide/text.rst

@@ -262,25 +262,30 @@ i.e., from the end of the string to the beginning of the string:
                    '', np.nan, 'CABA', 'dog', 'cat'],
                   dtype="string")
    s3
-   s3.str.replace('^.a|dog', 'XX-XX ', case=False)
+   s3.str.replace('^.a|dog', 'XX-XX ', case=False, regex=True)
 
-Some caution must be taken to keep regular expressions in mind! For example, the
-following code will cause trouble because of the regular expression meaning of
-``$``:
+Some caution must be taken when dealing with regular expressions! The current behavior
+is to treat single character patterns as literal strings, even when ``regex`` is set
+to ``True``. (This behavior is deprecated and will be removed in a future version so
+that the ``regex`` keyword is always respected.) For example, the following code will
+cause trouble because of the regular expression meaning of ``$``:
 
 .. ipython:: python
 
    # Consider the following badly formatted financial data
    dollars = pd.Series(['12', '-$10', '$10,000'], dtype="string")
 
-   # This does what you'd naively expect:
-   dollars.str.replace('$', '')
+   # Here $ is treated as a literal character
+   dollars.str.replace('$', '', regex=True)
 
-   # But this doesn't:
-   dollars.str.replace('-$', '-')
+   # But here it is not
+   dollars.str.replace('-$', '-', regex=True)
 
    # We need to escape the special character (for >1 len patterns)
-   dollars.str.replace(r'-\$', '-')
+   dollars.str.replace(r'-\$', '-', regex=True)
+
+   # Or set regex equal to False
+   dollars.str.replace('-$', '-', regex=False)
 
 If you do want literal replacement of a string (equivalent to
 :meth:`str.replace`), you can set the optional ``regex`` parameter to
@@ -290,7 +295,7 @@ and ``repl`` must be strings:
 .. ipython:: python
 
     # These lines are equivalent
-    dollars.str.replace(r'-\$', '-')
+    dollars.str.replace(r'-\$', '-', regex=True)
     dollars.str.replace('-$', '-', regex=False)
 
 The ``replace`` method can also take a callable as replacement. It is called
@@ -306,7 +311,7 @@ positional argument (a regex object) and return a string.
        return m.group(0)[::-1]
 
    pd.Series(['foo 123', 'bar baz', np.nan],
-             dtype="string").str.replace(pat, repl)
+             dtype="string").str.replace(pat, repl, regex=True)
 
    # Using regex groups
    pat = r"(?P<one>\w+) (?P<two>\w+) (?P<three>\w+)"
@@ -315,7 +320,7 @@ positional argument (a regex object) and return a string.
        return m.group('two').swapcase()
 
    pd.Series(['Foo Bar Baz', np.nan],
-             dtype="string").str.replace(pat, repl)
+             dtype="string").str.replace(pat, repl, regex=True)
 
 The ``replace`` method also accepts a compiled regular expression object
 from :func:`re.compile` as a pattern. All flags should be included in the
@@ -325,7 +330,7 @@ compiled regular expression object.
 
    import re
    regex_pat = re.compile(r'^.a|dog', flags=re.IGNORECASE)
-   s3.str.replace(regex_pat, 'XX-XX ')
+   s3.str.replace(regex_pat, 'XX-XX ', regex=True)
 
 Including a ``flags`` argument when calling ``replace`` with a compiled
 regular expression object will raise a ``ValueError``.

pandas/core/reshape/melt.py

@@ -483,7 +483,7 @@ def melt_stub(df, stub: str, i, j, value_vars, sep: str):
             var_name=j,
         )
         newdf[j] = Categorical(newdf[j])
-        newdf[j] = newdf[j].str.replace(re.escape(stub + sep), "")
+        newdf[j] = newdf[j].str.replace(re.escape(stub + sep), "", regex=True)
 
         # GH17627 Cast numerics suffixes to int/float
         newdf[j] = to_numeric(newdf[j], errors="ignore")

pandas/core/strings.py

@@ -2837,7 +2837,20 @@ def fullmatch(self, pat, case=True, flags=0, na=np.nan):
 
     @copy(str_replace)
     @forbid_nonstring_types(["bytes"])
-    def replace(self, pat, repl, n=-1, case=None, flags=0, regex=True):
+    def replace(self, pat, repl, n=-1, case=None, flags=0, regex=None):
+        if regex is None:
+            if (
+                isinstance(pat, str)
+                and len(pat) > 1
+                and any(c in pat for c in ".+*|^$[](){}\\")
+            ):
+                # warn only in cases where regex behavior would differ from literal
+                msg = (
+                    "The default value of regex will change from "
+                    "True to False in a future version."
+                )
+                warnings.warn(msg, FutureWarning, stacklevel=3)
+            regex = True
         result = str_replace(
             self._parent, pat, repl, n=n, case=case, flags=flags, regex=regex
         )

pandas/tests/series/methods/test_replace.py

@@ -449,3 +449,9 @@ def test_replace_with_compiled_regex(self):
         result = s.replace({regex: "z"}, regex=True)
         expected = pd.Series(["z", "b", "c"])
         tm.assert_series_equal(result, expected)
+
+    def test_str_replace_regex_default_raises_warning(self):
+        # https://github.com/pandas-dev/pandas/pull/24809
+        s = pd.Series(["a", "b", "c"])
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            s.str.replace("^.$", "")

pandas/tests/test_strings.py

@@ -995,11 +995,11 @@ def test_casemethods(self):
     def test_replace(self):
         values = Series(["fooBAD__barBAD", np.nan])
 
-        result = values.str.replace("BAD[_]*", "")
+        result = values.str.replace("BAD[_]*", "", regex=True)
         exp = Series(["foobar", np.nan])
         tm.assert_series_equal(result, exp)
 
-        result = values.str.replace("BAD[_]*", "", n=1)
+        result = values.str.replace("BAD[_]*", "", n=1, regex=True)
         exp = Series(["foobarBAD", np.nan])
         tm.assert_series_equal(result, exp)
 
@@ -1008,15 +1008,17 @@ def test_replace(self):
             ["aBAD", np.nan, "bBAD", True, datetime.today(), "fooBAD", None, 1, 2.0]
         )
 
-        rs = Series(mixed).str.replace("BAD[_]*", "")
+        rs = Series(mixed).str.replace("BAD[_]*", "", regex=True)
         xp = Series(["a", np.nan, "b", np.nan, np.nan, "foo", np.nan, np.nan, np.nan])
         assert isinstance(rs, Series)
         tm.assert_almost_equal(rs, xp)
 
         # flags + unicode
         values = Series([b"abcd,\xc3\xa0".decode("utf-8")])
         exp = Series([b"abcd, \xc3\xa0".decode("utf-8")])
-        result = values.str.replace(r"(?<=\w),(?=\w)", ", ", flags=re.UNICODE)
+        result = values.str.replace(
+            r"(?<=\w),(?=\w)", ", ", flags=re.UNICODE, regex=True
+        )
         tm.assert_series_equal(result, exp)
 
         # GH 13438
@@ -1034,7 +1036,7 @@ def test_replace_callable(self):
 
         # test with callable
         repl = lambda m: m.group(0).swapcase()
-        result = values.str.replace("[a-z][A-Z]{2}", repl, n=2)
+        result = values.str.replace("[a-z][A-Z]{2}", repl, n=2, regex=True)
         exp = Series(["foObaD__baRbaD", np.nan])
         tm.assert_series_equal(result, exp)
 
@@ -1060,7 +1062,7 @@ def test_replace_callable(self):
         values = Series(["Foo Bar Baz", np.nan])
         pat = r"(?P<first>\w+) (?P<middle>\w+) (?P<last>\w+)"
         repl = lambda m: m.group("middle").swapcase()
-        result = values.str.replace(pat, repl)
+        result = values.str.replace(pat, repl, regex=True)
         exp = Series(["bAR", np.nan])
         tm.assert_series_equal(result, exp)
 
@@ -1070,11 +1072,11 @@ def test_replace_compiled_regex(self):
 
         # test with compiled regex
         pat = re.compile(r"BAD[_]*")
-        result = values.str.replace(pat, "")
+        result = values.str.replace(pat, "", regex=True)
         exp = Series(["foobar", np.nan])
         tm.assert_series_equal(result, exp)
 
-        result = values.str.replace(pat, "", n=1)
+        result = values.str.replace(pat, "", n=1, regex=True)
         exp = Series(["foobarBAD", np.nan])
         tm.assert_series_equal(result, exp)
 
@@ -1083,7 +1085,7 @@ def test_replace_compiled_regex(self):
             ["aBAD", np.nan, "bBAD", True, datetime.today(), "fooBAD", None, 1, 2.0]
         )
 
-        rs = Series(mixed).str.replace(pat, "")
+        rs = Series(mixed).str.replace(pat, "", regex=True)
         xp = Series(["a", np.nan, "b", np.nan, np.nan, "foo", np.nan, np.nan, np.nan])
         assert isinstance(rs, Series)
         tm.assert_almost_equal(rs, xp)
@@ -1121,7 +1123,7 @@ def test_replace_literal(self):
         # GH16808 literal replace (regex=False vs regex=True)
         values = Series(["f.o", "foo", np.nan])
         exp = Series(["bao", "bao", np.nan])
-        result = values.str.replace("f.", "ba")
+        result = values.str.replace("f.", "ba", regex=True)
         tm.assert_series_equal(result, exp)
 
         exp = Series(["bao", "foo", np.nan])
@@ -3331,7 +3333,7 @@ def test_replace_moar(self):
         )
         tm.assert_series_equal(result, expected)
 
-        result = s.str.replace("^.a|dog", "XX-XX ", case=False)
+        result = s.str.replace("^.a|dog", "XX-XX ", case=False, regex=True)
         expected = Series(
             [
                 "A",