PEP 822: d-string draft implementation

Lib/test/test_dstring.py

@@ -0,0 +1,104 @@
+import unittest
+
+
+_dstring_prefixes = "d db df dt dr drb drf drt".split()
+_dstring_prefixes += [p.upper() for p in _dstring_prefixes]
+
+
+def d(s):
+    # Helper function to evaluate d-strings.
+    if '"""' in s:
+        return eval(f"d'''{s}'''")
+    else:
+        return eval(f'd"""{s}"""')
+
+
+class DStringTestCase(unittest.TestCase):
+    def assertAllRaise(self, exception_type, regex, error_strings):
+        for str in error_strings:
+            with self.subTest(str=str):
+                with self.assertRaisesRegex(exception_type, regex) as cm:
+                    eval(str)
+
+    def test_single_quote(self):
+        exprs = [
+            f"{p}'hello, world'" for p in _dstring_prefixes
+        ] + [
+            f'{p}"hello, world"' for p in _dstring_prefixes
+        ]
+        self.assertAllRaise(SyntaxError, "d-string must be triple-quoted", exprs)
+
+    def test_empty_dstring(self):
+        exprs = [
+            f"{p}''''''" for p in _dstring_prefixes
+        ] + [
+            f'{p}""""""' for p in _dstring_prefixes
+        ]
+        self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs)
+
+        for prefix in _dstring_prefixes:
+            expr = f"{prefix}'''\n'''"
+            expr2 = f'{prefix}"""\n"""'
+            with self.subTest(expr=expr):
+                v = eval(expr)
+                v2 = eval(expr2)
+                if 't' in prefix.lower():
+                    self.assertEqual(v.strings, ("",))
+                    self.assertEqual(v2.strings, ("",))
+                elif 'b' in prefix.lower():
+                    self.assertEqual(v, b"")
+                    self.assertEqual(v2, b"")
+                else:
+                    self.assertEqual(v, "")
+                    self.assertEqual(v2, "")
+
+    def test_dedent(self):
+        # Basic dedent - remove common leading whitespace
+        result = d("""
+    hello
+    world
+    """)
+        self.assertEqual(result, "hello\nworld\n")
+
+        # Dedent with varying indentation
+        result = d("""
+     line1
+       line2
+    line3
+      """)
+        self.assertEqual(result, " line1\n   line2\nline3\n  ")
+
+        # Dedent with tabs
+        result = d("""
+\thello
+\tworld
+\t""")
+        self.assertEqual(result, "hello\nworld\n")
+
+        # Mixed spaces and tabs (using common leading whitespace)
+        result = d("""
+\t\t    hello
+\t\t    world
+\t\t  """)
+        self.assertEqual(result, "  hello\n  world\n")
+
+        # Empty lines do not affect the calculation of common leading whitespace
+        result = d("""
+    hello
+
+    world
+    """)
+        self.assertEqual(result, "hello\n\nworld\n")
+
+        # Lines with only whitespace also have their indentation removed.
+        result = d("""
+    hello
+  \n\
+      \n\
+    world
+    """)
+        self.assertEqual(result, "hello\n\n  \nworld\n")
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_tokenize.py

@@ -3420,7 +3420,7 @@ def determine_valid_prefixes():
         # some uppercase-only prefix is added.
         for letter in itertools.chain(string.ascii_lowercase, string.ascii_uppercase):
             try:
-                eval(f'{letter}""')
+                eval(f'{letter}"""\n"""')  # d-string needs multiline
                 single_char_valid_prefixes.add(letter.lower())
             except SyntaxError:
                 pass
@@ -3444,7 +3444,7 @@ def determine_valid_prefixes():
                             # because it's a valid expression: not ""
                             continue
                         try:
-                            eval(f'{p}""')
+                            eval(f'{p}"""\n"""')  # d-string needs multiline
 
                             # No syntax error, so p is a valid string
                             # prefix.

Lib/tokenize.py

@@ -86,7 +86,8 @@ def _all_string_prefixes():
     # The valid string prefixes. Only contain the lower case versions,
     #  and don't contain any permutations (include 'fr', but not
     #  'rf'). The various permutations will be generated.
-    _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'br', 'fr', 'tr']
+    _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'd', 'br', 'fr', 'tr',
+                              'bd', 'rd', 'fd', 'td', 'brd', 'frd', 'trd']
     # if we add binary f-strings, add: ['fb', 'fbr']
     result = {''}
     for prefix in _valid_string_prefixes:

Objects/unicodeobject.c

@@ -13480,8 +13480,8 @@ of all lines in the [src, end).
 It returns the length of the common leading whitespace and sets `output` to
 point to the beginning of the common leading whitespace if length > 0.
 */
-static Py_ssize_t
-search_longest_common_leading_whitespace(
+Py_ssize_t
+_Py_search_longest_common_leading_whitespace(
     const char *const src,
     const char *const end,
     const char **output)
@@ -13576,7 +13576,7 @@ _PyUnicode_Dedent(PyObject *unicode)
     // [whitespace_start, whitespace_start + whitespace_len)
     // describes the current longest common leading whitespace
     const char *whitespace_start = NULL;
-    Py_ssize_t whitespace_len = search_longest_common_leading_whitespace(
+    Py_ssize_t whitespace_len = _Py_search_longest_common_leading_whitespace(
         src, end, &whitespace_start);
 
     if (whitespace_len == 0) {