BUG: str regex \w, \d, \s and \b only matching ASCII for pyarrow-backed strings

@@ -997,6 +997,8 @@ Strings - Bug in :meth:`Series.str.normalize`, :meth:`Series.str.zfill` and their :class:`Index` counterparts with PyArrow-backed string dtypes raising ``ArrowInvalid`` or returning a ``null`` dtype when the object was empty, the values were all missing, or the missing values were grouped into a chunk of their own (:issue:`66822`) - Bug in addition and multiplication on length-zero ``string``-dtype :class:`Series` and :class:`Index` returning ``object`` dtype instead of the string dtype, and raising ``TypeError`` when the other operand was length-zero with a different string storage or ``object`` dtype (:issue:`40624`) - Bug in :meth:`Series.str.match` and :meth:`Index.str.match` raising for regex flags other than ``re.IGNORECASE`` (e.g. ``re.MULTILINE``, ``re.DOTALL``, ``re.ASCII``), whether passed via ``flags`` or carried by a compiled ``re.Pattern`` (:issue:`63108`, :issue:`66348`)+- Bug in :meth:`Series.str.contains`, :meth:`Series.str.match`, :meth:`Series.str.fullmatch`, :meth:`Series.str.replace` and :meth:`Series.str.count` for PyArrow-backed :class:`StringDtype` where ``\w``, ``\d``, ``\s`` and ``\b`` (and their negations) in a regular expression only matched ASCII characters (:issue:`70770`)+- Bug in :meth:`Series.str.contains`, :meth:`Series.str.match`, :meth:`Series.str.fullmatch`, :meth:`Series.str.replace` and :meth:`Series.str.count` for PyArrow-backed :class:`StringDtype` where a lookaround or backreference inside a repeat such as ``(?:a(?=b))+`` raised an error (:issue:`70770`) - Bug in :meth:`Series.str.match` and :meth:`Index.str.match` where passing ``flags=0`` was not equivalent to omitting ``flags``, raising for patterns using PyArrow-only regex syntax such as ``\p{L}`` or an inline flag such as ``(?i)`` (:issue:`63108`) - Bug in :meth:`Series.str.match`, :meth:`Series.str.fullmatch`, :meth:`Series.str.contains`, :meth:`Series.str.count`, :meth:`Series.str.extract`, :meth:`Series.str.replace` and their :class:`Index` counterparts with :class:`ArrowDtype` raising instead of honoring ``flags`` such as ``re.ASCII``, or the flags carried by a compiled ``re.Pattern`` (:issue:`66348`) - Bug in :meth:`Series.str.partition` and :meth:`Series.str.split` with ``expand=True`` raising for :class:`ArrowDtype` when the :class:`Series` was empty or held only null values; an empty :class:`Series` now expands to no columns and an all-null one to a single all-NA column, matching object dtype (:issue:`63602`)

@@ -25,6 +25,7 @@  if TYPE_CHECKING:     from collections.abc import Callable+    from types import ModuleType      from pandas._typing import Scalar @@ -50,12 +51,24 @@ def _apply_elementwise(self, func: Callable[..., Any]) -> list[list[Any]]:         raise NotImplementedError      @staticmethod-    def _has_unsupported_regex(pat: str | re.Pattern[str]) -> bool:+    def _re_parser() -> ModuleType:+        try:+            # error: Module "re" has no attribute "_parser"+            from re import _parser  # type: ignore[attr-defined]+        except Exception as err:+            raise type(err)(+                "Incompatible version for regex; you will need to upgrade pandas "+                "or downgrade Python"+            ) from err+        return _parser++    @staticmethod+    def _regex_nodes(pat: str | re.Pattern[str]) -> list[tuple[Any, Any]]:         """-        Determine if regex pattern contains features not supported by RE2 / pyarrow.+        Parse a regex pattern into a flat list of its (op, arg) nodes. -        This includes lookaround (lookahead or lookbehind) assertions and-        backreferences.+        Nodes nested in groups, branches, repeats and character classes are+        included.          Parameters         ----------@@ -64,48 +77,72 @@ def _has_unsupported_regex(pat: str | re.Pattern[str]) -> bool:          Returns         --------        bool-            Whether `pat` contains a lookahead or lookbehind.+        list[tuple[Any, Any]]+            The nodes of `pat`, or an empty list if Python's ``re`` cannot+            parse it.         """-        try:-            # error: Module "re" has no attribute "_parser"-            from re import _parser  # type: ignore[attr-defined]+        _parser = ArrowStringArrayMixin._re_parser()+        repeats = {_parser.MAX_REPEAT, _parser.MIN_REPEAT, _parser.POSSESSIVE_REPEAT} -            regex_parser = _parser.parse-        except Exception as err:-            raise type(err)(-                "Incompatible version for regex; you will need to upgrade pandas "-                "or downgrade Python"-            ) from err--        def has_unsupported_code(tokens):-            # For certain op codes we need to recurse.+        def walk(tokens):             for op_code, argument in tokens:-                if (-                    (-                        op_code == _parser.SUBPATTERN-                        and has_unsupported_code(argument[3])-                    )-                    or (-                        op_code == _parser.BRANCH-                        and any(has_unsupported_code(tokens) for tokens in argument[1])-                    )-                    or (-                        op_code-                        in [_parser.ASSERT_NOT, _parser.ASSERT, _parser.GROUPREF]-                    )-                ):-                    return True-            return False+                yield op_code, argument+                if op_code == _parser.SUBPATTERN:+                    yield from walk(argument[3])+                elif op_code == _parser.BRANCH:+                    for branch in argument[1]:+                        yield from walk(branch)+                elif op_code in repeats:+                    yield from walk(argument[2])+                elif op_code in (_parser.ATOMIC_GROUP, _parser.IN):+                    yield from walk(argument)          str_pat = pat.pattern if isinstance(pat, re.Pattern) else pat         try:-            tokens = regex_parser(str_pat)+            tokens = _parser.parse(str_pat)         except re.error:             # Pattern not valid for Python's re (e.g. RE2 syntax like \x{...} or \p)             # Let the pyarrow backend handle it.+            return []+        return list(walk(tokens))++    def _needs_python_regex(self, pat: str | re.Pattern[str]) -> bool:+        """+        Whether a regex operation must use Python's ``re`` instead of pyarrow.++        This is the case for features not supported by RE2 / pyarrow (lookaround+        assertions and backreferences), and for ``\\w``, ``\\d``, ``\\s`` and+        ``\\b`` (or their negations) on non-ASCII data, since RE2 only matches+        them against ASCII characters while Python's ``re`` is Unicode-aware.+        """+        _parser = self._re_parser()+        unsupported = {_parser.ASSERT, _parser.ASSERT_NOT, _parser.GROUPREF}+        unicode_sensitive = {+            (_parser.CATEGORY, _parser.CATEGORY_WORD),+            (_parser.CATEGORY, _parser.CATEGORY_NOT_WORD),+            (_parser.CATEGORY, _parser.CATEGORY_DIGIT),+            (_parser.CATEGORY, _parser.CATEGORY_NOT_DIGIT),+            (_parser.CATEGORY, _parser.CATEGORY_SPACE),+            (_parser.CATEGORY, _parser.CATEGORY_NOT_SPACE),+            (_parser.AT, _parser.AT_BOUNDARY),+            (_parser.AT, _parser.AT_NON_BOUNDARY),+        }++        nodes = self._regex_nodes(pat)+        if any(op_code in unsupported for op_code, _ in nodes):+            return True+        if not any(+            (op_code, argument) in unicode_sensitive+            for op_code, argument in nodes+            if op_code in (_parser.CATEGORY, _parser.AT)+        ):             return False-        return has_unsupported_code(tokens)+        # all-ASCII data keeps the faster pyarrow path+        return self._has_non_ascii()++    def _has_non_ascii(self) -> bool:+        # min_count=0 so that an empty or all-null array reports False+        return not pc.all(pc.string_is_ascii(self._pa_array), min_count=0).as_py()      @staticmethod     def _is_re_pattern_with_flags(pat: str | re.Pattern[str]) -> bool:@@ -494,9 +531,7 @@ def _str_fullmatch(         return ArrowStringArrayMixin._str_match(self, pat, case, flags, na)      def _str_find(self, sub: str, start: int = 0, end: int | None = None):-        # min_count=0 so that an empty or all-null array reports True instead of-        #  null, keeping it on the pyarrow path below-        if not pc.all(pc.string_is_ascii(self._pa_array), min_count=0).as_py():+        if self._has_non_ascii():             # GH#64123 - pc.find_substring returns byte offsets instead of             # character offsets for multi-byte UTF-8 characters, so we fall back             # to Python str.find which correctly returns character offsets.

@@ -461,7 +461,7 @@ def _str_contains(         if (             flags             or self._is_re_pattern_with_flags(pat)-            or (regex and self._has_unsupported_regex(pat))+            or (regex and self._needs_python_regex(pat))             # a compiled pattern is not a valid literal; defer to the object path             or (not regex and isinstance(pat, re.Pattern))         ):@@ -482,7 +482,7 @@ def _str_match(         if (             flags             or self._is_re_pattern_with_flags(pat)-            or self._has_unsupported_regex(pat)+            or self._needs_python_regex(pat)         ):             return super()._str_match(pat, case, flags, na) @@ -499,7 +499,7 @@ def _str_fullmatch(         if (             flags             or self._is_re_pattern_with_flags(pat)-            or self._has_unsupported_regex(pat)+            or self._needs_python_regex(pat)         ):             return super()._str_fullmatch(pat, case, flags, na) @@ -524,7 +524,7 @@ def _str_replace(                 # https://docs.python.org/3/library/re.html                 isinstance(repl, str) and r"\g<" in repl             )-            or (regex and self._has_unsupported_regex(pat))+            or (regex and self._needs_python_regex(pat))         ):             return super()._str_replace(pat, repl, n, case, flags, regex) @@ -545,7 +545,7 @@ def _str_count(self, pat: str | re.Pattern[str], flags: int = 0):         if (             flags             or self._is_re_pattern_with_flags(pat)-            or self._has_unsupported_regex(pat)+            or self._needs_python_regex(pat)         ):             return super()._str_count(pat, flags) 

@@ -1018,6 +1018,7 @@ def test_replace_regex_single_character(regex, any_string_dtype):         (r"(?<=a)b", ["aa", "ax", "ba", "bb"]),         (r"a(?!b)", ["xx", "ab", "bx", "bb"]),         (r"(?<!b)a", ["xx", "xb", "ba", "bb"]),+        (r"(?:a(?=b))+", ["aa", "xb", "ba", "bb"]),         ("ab", ["aa", "x", "ba", "bb"]),     ], )@@ -1037,6 +1038,33 @@ def test_replace_lookarounds(any_string_dtype, pat, expected_data):     tm.assert_series_equal(result, expected)  +@pytest.mark.parametrize(+    "pat, repl, expected_data",+    [+        (+            r"[^\w\s]",+            "",+            ["café crème", "MüllerLüdenscheidt", "東京 2024", "a\xa0 b"],+        ),+        (r"\s+", " ", ["café crème!", "Müller-Lüdenscheidt", "東京 2024", "a b"]),+        (+            r"\b東",+            "T",+            ["café crème!", "Müller-Lüdenscheidt", "T京 2024", "a\xa0 b"],+        ),+    ],+)+def test_replace_unicode_regex_classes(any_string_dtype, pat, repl, expected_data):+    # GH#70770+    ser = pd.Series(+        ["café crème!", "Müller-Lüdenscheidt", "東京 2024", "a\xa0 b"],+        dtype=any_string_dtype,+    )+    result = ser.str.replace(pat, repl, regex=True)+    expected = pd.Series(expected_data, dtype=any_string_dtype)+    tm.assert_series_equal(result, expected)++ def test_replace_end_of_string(any_string_dtype):     # https://github.com/pandas-dev/pandas/pull/63613     ser = pd.Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)@@ -1627,6 +1655,31 @@ def test_fullmatch_lookarounds(any_string_dtype, pat):     tm.assert_series_equal(result, expected)  +@pytest.mark.parametrize(+    "method, pat, expected_data",+    [+        ("contains", r"\W", [False, True, True, False]),+        ("contains", r"\d", [False, False, True, True]),+        ("match", r"\w+\s", [False, False, True, False]),+        ("fullmatch", r"\w+", [True, False, False, True]),+        ("fullmatch", r"\w+(?:\s\w+)*", [True, False, True, True]),+    ],+)+def test_match_methods_unicode_regex_classes(+    any_string_dtype, method, pat, expected_data+):+    # GH#70770+    expected_dtype = (+        np.bool_ if is_object_or_nan_string_dtype(any_string_dtype) else "boolean"+    )+    ser = pd.Series(+        ["naïve", "Müller-Lüdenscheidt", "東京 2024", "١٢٣"], dtype=any_string_dtype+    )+    result = getattr(ser.str, method)(pat)+    expected = pd.Series(expected_data, dtype=expected_dtype)+    tm.assert_series_equal(result, expected)++ def test_fullmatch_end_of_string(any_string_dtype):     # https://github.com/pandas-dev/pandas/pull/63613     expected_dtype = (

@@ -11,7 +11,6 @@  import pandas as pd import pandas._testing as tm-from pandas.core.arrays._arrow_string_mixins import ArrowStringArrayMixin from pandas.core.strings.accessor import StringMethods from pandas.tests.strings import is_object_or_nan_string_dtype @@ -131,6 +130,28 @@ def test_count_compiled_pattern_with_flags(any_string_dtype):     tm.assert_series_equal(result, expected)  +@pytest.mark.parametrize(+    "pat, expected_data",+    [+        (r"\w", [4, 6, 2, 3, 2]),+        (r"\s", [0, 1, 0, 0, 1]),+        (r"\d", [0, 0, 0, 3, 0]),+        (r"\b", [2, 4, 2, 2, 4]),+    ],+)+def test_count_unicode_regex_classes(any_string_dtype, pat, expected_data):+    # GH#70770+    expected_dtype = (+        "int64" if is_object_or_nan_string_dtype(any_string_dtype) else "Int64"+    )+    ser = pd.Series(+        ["café", "été nœl", "東京", "١٢٣", "a\xa0b"], dtype=any_string_dtype+    )+    result = ser.str.count(pat)+    expected = pd.Series(expected_data, dtype=expected_dtype)+    tm.assert_series_equal(result, expected)++ def test_repeat(any_string_dtype):     ser = pd.Series(["a", "b", np.nan, "c", np.nan, "d"], dtype=any_string_dtype) @@ -992,9 +1013,10 @@ def test_setitem_with_different_string_storage():         (r"[(?=)]", False),         (r"(?#(?=comment)", False),         (r"(test # (?=comment))", True),-        (r"(?=test)+", False),-        (r"(?=test)*", False),-        (r"(?=test)?", False),+        (r"(?=test)+", True),+        (r"(?=test)*", True),+        (r"(?=test)?", True),+        (r"(?:a(?=b))+", True),         (r"abc|(?=test)", True),         (r"^(?=test)$", True),         # backreferences@@ -1003,6 +1025,40 @@ def test_setitem_with_different_string_storage():         (r"\b(?P<word>\w+)\s+(?P=word)\b", True),     ], )-def test_has_regex_unsupported_code(pat, expected):+def test_needs_python_regex_unsupported_code(pat, expected):     # https://github.com/pandas-dev/pandas/issues/60833-    assert ArrowStringArrayMixin._has_unsupported_regex(pat) == expected+    pytest.importorskip("pyarrow")+    arr = pd.array(["abc"], dtype=pd.StringDtype("pyarrow"))+    assert arr._needs_python_regex(pat) == expected+++@pytest.mark.parametrize(+    "pat, expected",+    [+        (r"\w+", True),+        (r"\W", True),+        (r"\d{3}", True),+        (r"\D", True),+        (r"\s+", True),+        (r"\S", True),+        (r"\bword\b", True),+        (r"\B", True),+        (r"[^\w\s]", True),+        (r"[\w-]", True),+        (r"(?:x\S)*", True),+        (r"(foo|bar\d)", True),+        (r"(?>\w)", True),+        (r"[a-z0-9_]+", False),+        (r"\\w", False),+        (r"abc", False),+        (r"", False),+    ],+)+def test_needs_python_regex_unicode_classes(pat, expected):+    # GH#70770+    pytest.importorskip("pyarrow")+    arr = pd.array(["é"], dtype=pd.StringDtype("pyarrow"))+    assert arr._needs_python_regex(pat) == expected+    # all-ASCII data keeps the pyarrow path+    ascii_arr = pd.array(["e", None], dtype=pd.StringDtype("pyarrow"))+    assert not ascii_arr._needs_python_regex(pat)