Fix f/t-string edge cases

Author: dflook

src/python_minifier/f_string.py

@@ -62,7 +62,7 @@ def complete_debug_specifier(self, partial_specifier_candidates, value_node):
     def _generate_candidates_with_processor(self, prefix, str_processor):
         """Generate f-string candidates using the given prefix and string processor function."""
         candidates = []
-        
+
         for quote in self.allowed_quotes:
             quote_candidates = ['']
             debug_specifier_candidates = []
@@ -99,7 +99,7 @@ def _generate_candidates_with_processor(self, prefix, str_processor):
                     raise RuntimeError('Unexpected JoinedStr value')
 
             candidates += [prefix + quote + x + quote for x in quote_candidates]
-        
+
         return candidates
 
     def candidates(self):
@@ -134,7 +134,33 @@ def _contains_literal_backslashes(self):
 
 
     def str_for(self, s, quote):
-        return s.replace('{', '{{').replace('}', '}}')
+        # Escape null bytes and other characters that can't appear in Python source
+        escaped = ''
+        is_multiline = len(quote) == 3  # Triple-quoted strings
+
+        for c in s:
+            if c == '\0':
+                escaped += '\\x00'
+            elif c == '\n' and not is_multiline:
+                # Only escape newlines in single-quoted strings
+                escaped += '\\n'
+            elif c == '\r':
+                # Always escape carriage returns because Python normalizes them during parsing
+                # This prevents semantic changes (\\r -> \\n) in multiline strings
+                escaped += '\\r'
+            elif c == '\t':
+                # Always escape tabs for consistency (though not strictly necessary in multiline)
+                escaped += '\\t'
+            elif c == '{':
+                escaped += '{{'
+            elif c == '}':
+                escaped += '}}'
+            elif ord(c) < 32 and c not in '\n\r\t':
+                # Escape other control characters
+                escaped += f'\\x{ord(c):02x}'
+            else:
+                escaped += c
+        return escaped
 
 
 class OuterFString(FString):
@@ -316,7 +342,9 @@ def _literals(self):
             if literal == '':
                 literal += self.current_quote
 
-            if c == '\n':
+            if c == '\0':
+                literal += '\\x00'
+            elif c == '\n':
                 literal += '\\n'
             elif c == '\r':
                 literal += '\\r'
@@ -333,7 +361,7 @@ def __str__(self):
         if self._s == '':
             return str(min(self.allowed_quotes, key=len)) * 2
 
-        if '\0' in self._s or ('\\' in self._s and not self.pep701):
+        if '\\' in self._s and not self.pep701:
             raise ValueError('Impossible to represent a character in f-string expression part')
 
         if not self.pep701 and ('\n' in self._s or '\r' in self._s):
@@ -391,14 +419,35 @@ def candidates(self):
         return candidates
 
     def str_for(self, s):
-        # For Python 3.12+ raw f-string regression (fixed in 3.14rc2), we need to escape backslashes
-        # in format specs so they round-trip correctly
-        if (3, 12) <= sys.version_info < (3, 14) and '\\' in s:
-            # In Python 3.12-3.13, format specs need backslashes escaped
-            escaped = s.replace('\\', '\\\\')
-        else:
-            escaped = s
-        return escaped.replace('{', '{{').replace('}', '}}')
+        # Special handling for problematic format spec characters that can cause parsing issues
+        # If the format spec contains only braces, it's likely an invalid test case
+
+        # Escape null bytes and other unprintable characters
+        escaped = ''
+        for c in s:
+            if c == '\0':
+                escaped += '\\x00'
+            elif c == '{':
+                escaped += '{{'
+            elif c == '}':
+                escaped += '}}'
+            elif c == '\\':
+                # For Python 3.12+ raw f-string regression (fixed in 3.14rc2), we need to escape backslashes
+                # in format specs so they round-trip correctly
+                if (3, 12) <= sys.version_info < (3, 14):
+                    escaped += '\\\\'
+                else:
+                    escaped += c
+            elif c == '\r':
+                # Always escape carriage returns because Python normalizes them to newlines during parsing
+                # This prevents AST mismatches (\r -> \n normalization)
+                escaped += '\\r'
+            elif ord(c) < 32 and c not in '\t\n':
+                # Escape other control characters except tab, newline
+                escaped += f'\\x{ord(c):02x}'
+            else:
+                escaped += c
+        return escaped
 
 
 class Bytes(object):
@@ -449,7 +498,24 @@ def _literals(self):
 
             if literal == '':
                 literal = 'b' + self.current_quote
-            literal += chr(b)
+
+            # Handle special characters that need escaping
+            if b == 0:  # null byte
+                literal += '\\x00'
+            elif b == ord('\\'):  # backslash
+                literal += '\\\\'
+            elif b == ord('\n'):  # newline
+                literal += '\\n'
+            elif b == ord('\r'):  # carriage return
+                literal += '\\r'
+            elif b == ord('\t'):  # tab
+                literal += '\\t'
+            elif len(self.current_quote) == 1 and b == ord(self.current_quote):  # single quote character
+                literal += '\\' + self.current_quote
+            elif 32 <= b <= 126:  # printable ASCII
+                literal += chr(b)
+            else:  # other non-printable characters
+                literal += f'\\x{b:02x}'
 
         if literal:
             literal += self.current_quote
@@ -459,8 +525,6 @@ def __str__(self):
         if self._b == b'':
             return 'b' + str(min(self.allowed_quotes, key=len)) * 2
 
-        if b'\0' in self._b or b'\\' in self._b:
-            raise ValueError('Impossible to represent a %r character in f-string expression part')
 
         if b'\n' in self._b or b'\r' in self._b:
             if '"""' not in self.allowed_quotes and "'''" not in self.allowed_quotes:

src/python_minifier/t_string.py

@@ -71,27 +71,40 @@ def candidates(self):
         """Generate all possible representations"""
         actual_candidates = []
 
+        # Normal t-string candidates
+        actual_candidates.extend(self._generate_candidates_with_processor('t', self.str_for))
+
+        # Raw t-string candidates (if we detect backslashes)
+        if self._contains_literal_backslashes():
+            actual_candidates.extend(self._generate_candidates_with_processor('rt', self.raw_str_for))
+
+        return filter(self.is_correct_ast, actual_candidates)
+
+    def _generate_candidates_with_processor(self, prefix, str_processor):
+        """Generate t-string candidates using the given prefix and string processor function."""
+        candidates = []
+
         for quote in self.allowed_quotes:
-            candidates = ['']
+            quote_candidates = ['']
             debug_specifier_candidates = []
 
             for v in self.node.values:
                 if is_constant_node(v, ast.Constant) and isinstance(v.value, str):
                     # String literal part - check for debug specifiers
 
                     # Could this be used as a debug specifier?
-                    if len(candidates) < 10:
+                    if len(quote_candidates) < 10:
                         import re
                         debug_specifier = re.match(r'.*=\s*$', v.value)
                         if debug_specifier:
                             # Maybe! Save for potential debug specifier completion
                             try:
-                                debug_specifier_candidates = [x + '{' + v.value for x in candidates]
+                                debug_specifier_candidates = [x + '{' + v.value for x in quote_candidates]
                             except Exception:
                                 continue
 
                     try:
-                        candidates = [x + self.str_for(v.value, quote) for x in candidates]
+                        quote_candidates = [x + str_processor(v.value, quote) for x in quote_candidates]
                     except Exception:
                         continue
 
@@ -103,17 +116,17 @@ def candidates(self):
 
                         # Regular interpolation processing
                         interpolation_candidates = InterpolationValue(v).get_candidates()
-                        candidates = [x + y for x in candidates for y in interpolation_candidates] + completed
+                        quote_candidates = [x + y for x in quote_candidates for y in interpolation_candidates] + completed
 
                         debug_specifier_candidates = []
                     except Exception:
                         continue
                 else:
                     raise RuntimeError('Unexpected TemplateStr value: %r' % v)
 
-            actual_candidates.extend(['t' + quote + x + quote for x in candidates])
+            candidates.extend([prefix + quote + x + quote for x in quote_candidates])
 
-        return filter(self.is_correct_ast, actual_candidates)
+        return candidates
 
     def str_for(self, s, quote):
         """Convert string literal to properly escaped form"""
@@ -125,6 +138,24 @@ def str_for(self, s, quote):
             return '\\\n'
         return mini_s
 
+    def raw_str_for(self, s):
+        """
+        Generate string representation for raw t-strings.
+        Don't escape backslashes like MiniString does.
+        """
+        return s.replace('{', '{{').replace('}', '}}')
+
+    def _contains_literal_backslashes(self):
+        """
+        Check if this t-string contains literal backslashes in constant values.
+        This indicates it may need to be a raw t-string.
+        """
+        for node in ast.walk(self.node):
+            if is_constant_node(node, ast.Str):
+                if '\\' in node.s:
+                    return True
+        return False
+
     def __str__(self):
         """Generate the shortest valid t-string representation"""
         if len(self.node.values) == 0:
@@ -195,20 +226,27 @@ def get_candidates(self):
                 format_candidates = python_minifier.f_string.OuterFString(
                     self.node.format_spec, pep701=True
                 ).candidates()
-                # Remove the f prefix and quotes to get just the format part
+                # Remove the f/rf prefix and quotes to get just the format part
                 format_parts = []
                 for fmt in format_candidates:
-                    if fmt.startswith('f'):
+                    # Handle both f"..." and rf"..." patterns
+                    if fmt.startswith('rf'):
+                        # Remove rf prefix and outer quotes
+                        inner = fmt[2:]
+                    elif fmt.startswith('f'):
                         # Remove f prefix and outer quotes
                         inner = fmt[1:]
-                        if (inner.startswith('"') and inner.endswith('"')) or \
-                           (inner.startswith("'") and inner.endswith("'")):
-                            format_parts.append(inner[1:-1])
-                        elif (inner.startswith('"""') and inner.endswith('"""')) or \
-                             (inner.startswith("'''") and inner.endswith("'''")):
-                            format_parts.append(inner[3:-3])
-                        else:
-                            format_parts.append(inner)
+                    else:
+                        continue
+
+                    if (inner.startswith('"') and inner.endswith('"')) or \
+                       (inner.startswith("'") and inner.endswith("'")):
+                        format_parts.append(inner[1:-1])
+                    elif (inner.startswith('"""') and inner.endswith('"""')) or \
+                         (inner.startswith("'''") and inner.endswith("'''")):
+                        format_parts.append(inner[3:-3])
+                    else:
+                        format_parts.append(inner)
 
                 if format_parts:
                     self._append(format_parts)