⚡️ Speed up function should_use_regex by 16%

[codeflash-ai]

📄 16% (0.16x) speedup for should_use_regex in pandas/core/array_algos/replace.py

⏱️ Runtime : 30.2 milliseconds → 25.9 milliseconds (best of 44 runs)

📝 Explanation and details

The optimization achieves a 16% speedup by eliminating redundant regex compilation operations and introducing strategic short-circuits in the should_use_regex function.

Key Optimizations Applied:

1. Fast-path for compiled Pattern objects: When to_replace is already a compiled regex Pattern, the optimized code directly accesses to_replace.pattern instead of calling is_re_compilable and re.compile again. This eliminates expensive redundant operations for Pattern inputs.

2. Single compilation strategy: For non-Pattern inputs, the original code called re.compile twice - once in is_re_compilable and again for the empty pattern check. The optimized version compiles once and reuses the result, reducing compilation overhead by ~50% for string inputs.

3. Early termination logic: Added if not regex or not is_re_compilable(to_replace): return False to short-circuit when conditions aren't met, avoiding unnecessary compilation in the main function.

4. Fast-path in is_re_compilable: Added if isinstance(obj, Pattern): return True to immediately return for already-compiled patterns without attempting recompilation.

Performance Impact Analysis:

The line profiler shows the most significant gains occur when processing compiled Pattern objects. For example:

• test_basic_compiled_pattern_regex_false: 2.89μs → 860ns (236% faster)
• test_large_compiled_pattern: 13.9μs → 296ns (4584% faster)

These dramatic improvements happen because the optimization eliminates the expensive re.compile call entirely for Pattern inputs, which were the biggest bottleneck in the original implementation (82% of total time was spent in is_re_compilable).

Workload Benefits:
The optimization is particularly effective for workloads involving pre-compiled regex patterns or mixed input types, where avoiding redundant compilation provides substantial performance gains while maintaining identical functionality.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 10093 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

import re
# function to test
from typing import Any

# imports
import pytest  # used for our unit tests
from pandas.core.array_algos.replace import should_use_regex

# unit tests

# -----------------------------
# Basic Test Cases
# -----------------------------

def test_basic_string_regex_true():
    # Should use regex for a valid regex string when regex=True
    codeflash_output = should_use_regex(True, r"\d+") # 49.2μs -> 49.6μs (0.839% slower)

def test_basic_string_regex_false():
    # Should NOT use regex for a valid regex string when regex=False
    codeflash_output = should_use_regex(False, r"\d+") # 640ns -> 702ns (8.83% slower)

def test_basic_literal_string():
    # Should NOT use regex for a literal string when regex=True
    codeflash_output = should_use_regex(True, "hello") # 36.7μs -> 37.2μs (1.40% slower)

def test_basic_literal_string_regex_false():
    # Should NOT use regex for a literal string when regex=False
    codeflash_output = should_use_regex(False, "hello") # 687ns -> 659ns (4.25% faster)

def test_basic_compiled_pattern_regex_false():
    # Should use regex if to_replace is a compiled pattern, regardless of regex param
    pattern = re.compile(r"\w+")
    codeflash_output = should_use_regex(False, pattern) # 2.89μs -> 860ns (236% faster)

def test_basic_compiled_pattern_regex_true():
    # Should use regex if to_replace is a compiled pattern and regex=True
    pattern = re.compile(r"\w+")
    codeflash_output = should_use_regex(True, pattern) # 2.70μs -> 890ns (203% faster)

# -----------------------------
# Edge Test Cases
# -----------------------------

def test_empty_string_regex_true():
    # Should NOT use regex for empty string even if regex=True
    codeflash_output = should_use_regex(True, "") # 2.25μs -> 2.32μs (3.06% slower)

def test_empty_string_regex_false():
    # Should NOT use regex for empty string if regex=False
    codeflash_output = should_use_regex(False, "") # 657ns -> 670ns (1.94% slower)

def test_empty_compiled_pattern():
    # Should NOT use regex for compiled empty pattern
    pattern = re.compile("")
    codeflash_output = should_use_regex(True, pattern) # 2.77μs -> 908ns (205% faster)
    codeflash_output = should_use_regex(False, pattern) # 1.23μs -> 339ns (263% faster)

def test_non_string_non_pattern():
    # Should NOT use regex for integer input
    codeflash_output = should_use_regex(True, 123) # 3.59μs -> 3.51μs (2.37% faster)
    codeflash_output = should_use_regex(False, 123) # 395ns -> 348ns (13.5% faster)

def test_bytes_input():
    # Should NOT use regex for bytes input
    codeflash_output = should_use_regex(True, b"abc") # 37.5μs -> 37.2μs (0.682% faster)
    codeflash_output = should_use_regex(False, b"abc") # 456ns -> 417ns (9.35% faster)

def test_none_input():
    # Should NOT use regex for None input
    codeflash_output = should_use_regex(True, None) # 3.34μs -> 3.25μs (2.70% faster)
    codeflash_output = should_use_regex(False, None) # 381ns -> 310ns (22.9% faster)

def test_list_input():
    # Should NOT use regex for list input
    codeflash_output = should_use_regex(True, ["abc", "def"]) # 2.73μs -> 2.79μs (2.26% slower)
    codeflash_output = should_use_regex(False, ["abc", "def"]) # 359ns -> 332ns (8.13% faster)

def test_dict_input():
    # Should NOT use regex for dict input
    codeflash_output = should_use_regex(True, {"a": 1}) # 2.71μs -> 2.66μs (1.73% faster)
    codeflash_output = should_use_regex(False, {"a": 1}) # 360ns -> 336ns (7.14% faster)

def test_object_input():
    # Should NOT use regex for arbitrary object input
    class Dummy: pass
    codeflash_output = should_use_regex(True, Dummy()) # 3.84μs -> 3.73μs (3.06% faster)
    codeflash_output = should_use_regex(False, Dummy()) # 341ns -> 343ns (0.583% slower)

def test_pattern_with_only_spaces():
    # Should use regex for a string of spaces if regex=True
    # Spaces are a valid regex, but not empty
    codeflash_output = should_use_regex(True, "   ") # 37.2μs -> 36.7μs (1.35% faster)

def test_pattern_with_newline():
    # Should use regex for a string containing newline
    codeflash_output = should_use_regex(True, "\n") # 29.9μs -> 29.5μs (1.31% faster)

def test_pattern_with_null_char():
    # Should use regex for a string containing null char
    codeflash_output = should_use_regex(True, "\x00") # 28.7μs -> 28.7μs (0.063% faster)


def test_large_string_regex_true():
    # Should use regex for a very large regex string if regex=True
    large_pattern = "a" * 1000
    codeflash_output = should_use_regex(True, large_pattern) # 2.86μs -> 2.72μs (5.33% faster)

def test_large_string_regex_false():
    # Should NOT use regex for a very large string if regex=False
    large_pattern = "a" * 1000
    codeflash_output = should_use_regex(False, large_pattern) # 662ns -> 766ns (13.6% slower)

def test_large_compiled_pattern():
    # Should use regex for a very large compiled pattern
    pattern = re.compile("a" * 1000)
    codeflash_output = should_use_regex(True, pattern) # 15.6μs -> 958ns (1524% faster)
    codeflash_output = should_use_regex(False, pattern) # 13.9μs -> 296ns (4584% faster)

def test_many_patterns_in_list():
    # Should NOT use regex for a list of many patterns
    patterns = [r"\d+"] * 1000
    codeflash_output = should_use_regex(True, patterns) # 2.93μs -> 3.01μs (2.53% slower)

def test_many_patterns_in_tuple():
    # Should NOT use regex for a tuple of many patterns
    patterns = tuple([r"\d+"] * 1000)
    codeflash_output = should_use_regex(True, patterns) # 5.12μs -> 5.05μs (1.39% faster)

def test_large_unicode_string():
    # Should use regex for a large unicode string if regex=True
    large_unicode = "𝔘" * 1000
    codeflash_output = should_use_regex(True, large_unicode) # 764μs -> 750μs (1.89% faster)

def test_large_bytes_input():
    # Should NOT use regex for large bytes input
    large_bytes = b"x" * 1000
    codeflash_output = should_use_regex(True, large_bytes) # 743μs -> 722μs (2.93% faster)

def test_large_dict_input():
    # Should NOT use regex for large dict input
    large_dict = {str(i): i for i in range(1000)}
    codeflash_output = should_use_regex(True, large_dict) # 3.08μs -> 3.02μs (1.92% faster)

# -----------------------------
# Additional Robustness Cases
# -----------------------------

def test_pattern_with_escaped_characters():
    # Should use regex for a string with escaped regex chars
    codeflash_output = should_use_regex(True, r"\n\t\r") # 37.7μs -> 36.9μs (2.31% faster)

def test_pattern_with_non_ascii():
    # Should use regex for a non-ascii pattern string
    codeflash_output = should_use_regex(True, "café") # 33.6μs -> 33.9μs (0.656% slower)

def test_pattern_with_regex_flag():
    # Should use regex for a compiled pattern with flags
    pattern = re.compile(r"\w+", re.IGNORECASE)
    codeflash_output = should_use_regex(True, pattern) # 2.92μs -> 946ns (209% faster)

def test_pattern_with_group():
    # Should use regex for a string with group
    codeflash_output = should_use_regex(True, r"(abc)") # 44.8μs -> 44.3μs (1.18% faster)

def test_pattern_with_alternation():
    # Should use regex for a string with alternation
    codeflash_output = should_use_regex(True, r"abc|def") # 50.3μs -> 50.0μs (0.538% faster)

def test_pattern_with_quantifier():
    # Should use regex for a string with quantifier
    codeflash_output = should_use_regex(True, r"a{2,5}") # 37.7μs -> 37.6μs (0.413% faster)

def test_pattern_with_start_end_anchors():
    # Should use regex for a string with start/end anchors
    codeflash_output = should_use_regex(True, r"^abc$") # 33.1μs -> 32.2μs (2.73% faster)

def test_pattern_with_empty_group():
    # Should use regex for a string with empty group
    codeflash_output = should_use_regex(True, r"()") # 34.6μs -> 34.7μs (0.544% slower)

def test_pattern_with_comment():
    # Should use regex for a string with regex comment
    codeflash_output = should_use_regex(True, r"(?#this is a comment)") # 28.1μs -> 28.2μs (0.302% slower)

def test_pattern_with_lookahead():
    # Should use regex for a string with lookahead
    codeflash_output = should_use_regex(True, r"abc(?=def)") # 41.8μs -> 40.7μs (2.92% faster)

def test_pattern_with_lookbehind():
    # Should use regex for a string with lookbehind
    codeflash_output = should_use_regex(True, r"(?<=abc)def") # 42.5μs -> 41.3μs (2.81% faster)

def test_pattern_with_non_capturing_group():
    # Should use regex for a string with non-capturing group
    codeflash_output = should_use_regex(True, r"(?:abc)") # 40.4μs -> 38.8μs (4.26% faster)

def test_pattern_with_named_group():
    # Should use regex for a string with named group
    codeflash_output = should_use_regex(True, r"(?P<name>abc)") # 47.7μs -> 47.0μs (1.46% faster)

def test_pattern_with_unicode_flag():
    # Should use regex for a compiled pattern with unicode flag
    pattern = re.compile(r"\w+", re.UNICODE)
    codeflash_output = should_use_regex(True, pattern) # 2.83μs -> 942ns (200% faster)

def test_pattern_with_dotall_flag():
    # Should use regex for a compiled pattern with DOTALL flag
    pattern = re.compile(r".*", re.DOTALL)
    codeflash_output = should_use_regex(True, pattern) # 2.99μs -> 885ns (238% faster)

def test_pattern_with_multiline_flag():
    # Should use regex for a compiled pattern with MULTILINE flag
    pattern = re.compile(r"^abc$", re.MULTILINE)
    codeflash_output = should_use_regex(True, pattern) # 2.94μs -> 851ns (245% faster)

def test_pattern_with_ignorecase_flag():
    # Should use regex for a compiled pattern with IGNORECASE flag
    pattern = re.compile(r"abc", re.IGNORECASE)
    codeflash_output = should_use_regex(True, pattern) # 2.80μs -> 857ns (227% faster)

def test_pattern_with_verbose_flag():
    # Should use regex for a compiled pattern with VERBOSE flag
    pattern = re.compile(r"abc", re.VERBOSE)
    codeflash_output = should_use_regex(True, pattern) # 2.88μs -> 920ns (212% faster)

def test_pattern_with_multiple_flags():
    # Should use regex for a compiled pattern with multiple flags
    pattern = re.compile(r"abc", re.IGNORECASE | re.MULTILINE | re.DOTALL)
    codeflash_output = should_use_regex(True, pattern) # 2.79μs -> 909ns (207% faster)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.
#------------------------------------------------
import re
# function to test
from typing import Any

# imports
import pytest  # used for our unit tests
from pandas.core.array_algos.replace import should_use_regex

# unit tests

# --------- BASIC TEST CASES ---------

def test_basic_regex_true_with_string_pattern():
    # Should use regex if flag is True and to_replace is a compilable string
    codeflash_output = should_use_regex(True, r"\d+") # 2.68μs -> 2.58μs (3.80% faster)

def test_basic_regex_false_with_string_pattern():
    # Should not use regex if flag is False, even if to_replace is a regex string
    codeflash_output = should_use_regex(False, r"\d+") # 658ns -> 689ns (4.50% slower)

def test_basic_regex_true_with_compiled_pattern():
    # Should use regex if to_replace is a compiled pattern, regardless of flag
    pattern = re.compile(r"\d+")
    codeflash_output = should_use_regex(False, pattern) # 2.85μs -> 862ns (231% faster)
    codeflash_output = should_use_regex(True, pattern) # 1.26μs -> 293ns (329% faster)

def test_basic_regex_false_with_non_regex_string():
    # Should not use regex if to_replace is a plain string and flag is False
    codeflash_output = should_use_regex(False, "abc") # 601ns -> 643ns (6.53% slower)

def test_basic_regex_true_with_non_regex_string():
    # Should use regex if flag is True and to_replace is a non-regex string
    codeflash_output = should_use_regex(True, "abc") # 37.6μs -> 37.7μs (0.244% slower)

def test_basic_regex_true_with_integer_string():
    # Should use regex if flag is True and to_replace is a string that can be compiled
    codeflash_output = should_use_regex(True, "123") # 32.3μs -> 32.7μs (1.15% slower)

def test_basic_regex_false_with_integer_string():
    # Should not use regex if flag is False, regardless of compilability
    codeflash_output = should_use_regex(False, "123") # 666ns -> 622ns (7.07% faster)

def test_basic_regex_false_with_integer():
    # Should not use regex if to_replace is not compilable (int)
    codeflash_output = should_use_regex(True, 123) # 3.47μs -> 3.44μs (0.726% faster)
    codeflash_output = should_use_regex(False, 123) # 382ns -> 366ns (4.37% faster)

def test_basic_regex_false_with_none():
    # Should not use regex if to_replace is None
    codeflash_output = should_use_regex(True, None) # 3.29μs -> 3.10μs (6.23% faster)
    codeflash_output = should_use_regex(False, None) # 366ns -> 330ns (10.9% faster)

# --------- EDGE TEST CASES ---------

def test_edge_regex_true_with_empty_string():
    # Should not use regex if to_replace is empty string, even if flag is True
    codeflash_output = should_use_regex(True, "") # 2.30μs -> 2.25μs (2.13% faster)
    codeflash_output = should_use_regex(False, "") # 376ns -> 337ns (11.6% faster)

def test_edge_regex_true_with_empty_compiled_pattern():
    # Should not use regex if to_replace is compiled empty pattern
    empty_pat = re.compile("")
    codeflash_output = should_use_regex(True, empty_pat) # 2.84μs -> 878ns (223% faster)
    codeflash_output = should_use_regex(False, empty_pat) # 1.31μs -> 332ns (295% faster)

def test_edge_regex_true_with_whitespace_string():
    # Should use regex if flag is True and string is whitespace (compilable)
    codeflash_output = should_use_regex(True, " ") # 33.3μs -> 33.4μs (0.398% slower)

def test_edge_regex_false_with_whitespace_string():
    # Should not use regex if flag is False and string is whitespace
    codeflash_output = should_use_regex(False, " ") # 604ns -> 645ns (6.36% slower)

def test_edge_regex_true_with_non_string_non_compilable():
    # Should not use regex if to_replace is list, dict, set, etc.
    codeflash_output = should_use_regex(True, []) # 2.88μs -> 2.90μs (0.621% slower)
    codeflash_output = should_use_regex(False, {}) # 380ns -> 324ns (17.3% faster)
    codeflash_output = should_use_regex(True, set()) # 1.47μs -> 1.50μs (1.60% slower)

def test_edge_regex_true_with_bytes_pattern():
    # Should not use regex if to_replace is bytes, even if flag is True
    codeflash_output = should_use_regex(True, b"\d+") # 46.6μs -> 47.1μs (1.16% slower)

def test_edge_regex_true_with_float():
    # Should not use regex if to_replace is float, even if flag is True
    codeflash_output = should_use_regex(True, 1.23) # 3.53μs -> 3.42μs (3.34% faster)

def test_edge_regex_true_with_object():
    # Should not use regex if to_replace is an arbitrary object
    class Dummy: pass
    codeflash_output = should_use_regex(True, Dummy()) # 3.50μs -> 3.39μs (3.07% faster)

def test_edge_regex_true_with_string_that_is_not_regex():
    # Should use regex if flag is True and string is compilable, even if not a regex
    codeflash_output = should_use_regex(True, "not_a_regex") # 42.7μs -> 42.4μs (0.745% faster)

def test_edge_regex_true_with_string_that_looks_like_regex_but_flag_false():
    # Should not use regex if flag is False, even if string looks like regex
    codeflash_output = should_use_regex(False, "^abc$") # 652ns -> 649ns (0.462% faster)

def test_edge_regex_true_with_pattern_with_only_special_chars():
    # Should use regex if flag is True and string is compilable
    codeflash_output = should_use_regex(True, ".*") # 36.6μs -> 36.1μs (1.42% faster)

def test_edge_regex_true_with_pattern_with_escaped_chars():
    # Should use regex if flag is True and string is compilable
    codeflash_output = should_use_regex(True, r"\n\t") # 35.0μs -> 33.2μs (5.43% faster)

def test_edge_regex_true_with_pattern_with_unicode_chars():
    # Should use regex if flag is True and string is compilable
    codeflash_output = should_use_regex(True, r"\u1234") # 34.1μs -> 34.0μs (0.250% faster)

def test_edge_regex_true_with_pattern_with_newline():
    # Should use regex if flag is True and string contains newlines
    codeflash_output = should_use_regex(True, "abc\ndef") # 34.9μs -> 34.7μs (0.735% faster)

def test_edge_regex_true_with_pattern_with_null_char():
    # Should use regex if flag is True and string contains null char
    codeflash_output = should_use_regex(True, "abc\0def") # 34.9μs -> 34.4μs (1.54% faster)

def test_edge_regex_true_with_pattern_with_long_string():
    # Should use regex if flag is True and string is long but compilable
    long_str = "a" * 100
    codeflash_output = should_use_regex(True, long_str) # 99.8μs -> 97.7μs (2.22% faster)

def test_edge_regex_true_with_pattern_with_non_ascii():
    # Should use regex if flag is True and string contains non-ascii chars
    codeflash_output = should_use_regex(True, "你好") # 30.1μs -> 30.2μs (0.612% slower)

def test_edge_regex_true_with_pattern_with_surrogate_pair():
    # Should use regex if flag is True and string contains surrogate pairs
    codeflash_output = should_use_regex(True, "\udc80\udc81") # 29.8μs -> 29.8μs (0.050% slower)

def test_edge_regex_true_with_pattern_with_control_chars():
    # Should use regex if flag is True and string contains control chars
    codeflash_output = should_use_regex(True, "\x01\x02") # 30.0μs -> 29.4μs (1.76% faster)

def test_edge_regex_true_with_pattern_with_tab_char():
    # Should use regex if flag is True and string contains tab
    codeflash_output = should_use_regex(True, "\t") # 27.8μs -> 27.7μs (0.397% faster)

# --------- LARGE SCALE TEST CASES ---------

def test_large_scale_many_different_patterns():
    # Test should_use_regex with many different patterns
    for i in range(1000):
        s = f"pattern_{i}"
        codeflash_output = should_use_regex(True, s) # 19.4ms -> 19.0ms (1.98% faster)
        codeflash_output = should_use_regex(False, s)

def test_large_scale_many_empty_patterns():
    # Test should_use_regex with many empty patterns
    for _ in range(1000):
        codeflash_output = should_use_regex(True, "") # 819μs -> 835μs (1.94% slower)
        codeflash_output = should_use_regex(False, "")

def test_large_scale_many_compiled_patterns():
    # Test should_use_regex with many compiled patterns
    for i in range(1000):
        pat = re.compile(f"pattern_{i}")
        codeflash_output = should_use_regex(True, pat) # 1.41ms -> 289μs (387% faster)
        codeflash_output = should_use_regex(False, pat)

def test_large_scale_many_non_compilable_objects():
    # Test should_use_regex with many non-compilable objects
    for i in range(1000):
        obj = [i]
        codeflash_output = should_use_regex(True, obj) # 814μs -> 828μs (1.80% slower)
        codeflash_output = should_use_regex(False, obj)


def test_large_scale_long_pattern_string():
    # Test should_use_regex with a very long pattern string
    long_pattern = "a" * 1000
    codeflash_output = should_use_regex(True, long_pattern) # 749μs -> 717μs (4.43% faster)
    codeflash_output = should_use_regex(False, long_pattern) # 488ns -> 433ns (12.7% faster)

def test_large_scale_long_pattern_compiled():
    # Test should_use_regex with a very long compiled pattern
    long_pattern = re.compile("a" * 1000)
    codeflash_output = should_use_regex(True, long_pattern) # 15.4μs -> 994ns (1447% faster)
    codeflash_output = should_use_regex(False, long_pattern) # 13.8μs -> 291ns (4650% faster)

def test_large_scale_long_pattern_empty_compiled():
    # Test should_use_regex with many compiled empty patterns
    for _ in range(1000):
        empty_pat = re.compile("")
        codeflash_output = should_use_regex(True, empty_pat) # 1.04ms -> 241μs (329% faster)
        codeflash_output = should_use_regex(False, empty_pat)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

To edit these changes git checkout codeflash/optimize-should_use_regex-mho8r9f1 and push.