gh-153569: Add tokenizer source and cursor primitives

Lib/test/test_capi/test_tokenizer.py

@@ -0,0 +1,17 @@
+import unittest
+from test.support import import_helper
+
+
+_testinternalcapi = import_helper.import_module("_testinternalcapi")
+
+
+class TokenizerTests(unittest.TestCase):
+    def test_source(self):
+        _testinternalcapi.test_tokenizer_source()
+
+    def test_cursor(self):
+        _testinternalcapi.test_tokenizer_cursor()
+
+
+if __name__ == "__main__":
+    unittest.main()

Makefile.pre.in

@@ -398,8 +398,10 @@ TOKENIZER_OBJS=		\
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
 		Parser/lexer/string.o \
+		Parser/tokenizer/cursor.o \
 		Parser/tokenizer/file_tokenizer.o \
 		Parser/tokenizer/readline_tokenizer.o \
+		Parser/tokenizer/source.o \
 		Parser/tokenizer/string_tokenizer.o \
 		Parser/tokenizer/utf8_tokenizer.o \
 		Parser/tokenizer/helpers.o
@@ -414,6 +416,8 @@ TOKENIZER_HEADERS= \
 		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \
+		Parser/tokenizer/cursor.h \
+		Parser/tokenizer/source.h \
 		Parser/tokenizer/tokenizer.h \
 		Parser/tokenizer/helpers.h
 
@@ -3479,7 +3483,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
 MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
 MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
 MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
 MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
 MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
 

Modules/Setup.stdlib.in

@@ -172,7 +172,7 @@
 @MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
-@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c
+@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tokenizer.c _testinternalcapi/tuple.c
 @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c
 @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c

Modules/_testinternalcapi.c

@@ -3420,6 +3420,9 @@ module_exec(PyObject *module)
     if (_PyTestInternalCapi_Init_CriticalSection(module) < 0) {
         return 1;
     }
+    if (_PyTestInternalCapi_Init_Tokenizer(module) < 0) {
+        return 1;
+    }
     if (_PyTestInternalCapi_Init_Tuple(module) < 0) {
         return 1;
     }

Modules/_testinternalcapi/parts.h

@@ -15,6 +15,7 @@ int _PyTestInternalCapi_Init_PyTime(PyObject *module);
 int _PyTestInternalCapi_Init_Set(PyObject *module);
 int _PyTestInternalCapi_Init_Complex(PyObject *module);
 int _PyTestInternalCapi_Init_CriticalSection(PyObject *module);
+int _PyTestInternalCapi_Init_Tokenizer(PyObject *module);
 int _PyTestInternalCapi_Init_Tuple(PyObject *module);
 
 #endif // Py_TESTINTERNALCAPI_PARTS_H

Modules/_testinternalcapi/tokenizer.c

@@ -0,0 +1,309 @@
+#include "parts.h"
+
+#include "../../Parser/tokenizer/cursor.h"
+
+static int
+check(int condition, const char *message)
+{
+    if (condition) {
+        return 0;
+    }
+    PyErr_SetString(PyExc_AssertionError, message);
+    return -1;
+}
+
+static int
+check_system_error(int failed, const char *message)
+{
+    if (!failed || !PyErr_ExceptionMatches(PyExc_SystemError)) {
+        PyErr_SetString(PyExc_AssertionError, message);
+        return -1;
+    }
+    PyErr_Clear();
+    return 0;
+}
+
+static int
+same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right)
+{
+    return left->source == right->source &&
+        left->pos == right->pos &&
+        left->line_start == right->line_start &&
+        left->line_end == right->line_end &&
+        left->lineno == right->lineno;
+}
+
+static PyObject *
+test_tokenizer_source(PyObject *Py_UNUSED(module),
+                      PyObject *Py_UNUSED(args))
+{
+    _PyTok_SourceText source;
+    _PyTok_SourceInit(&source);
+
+    _PyTok_Loc loc;
+    _PyTok_Line line;
+    if (check(_PyTok_SourceLocation(
+                  &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0,
+              "cannot locate empty source") < 0 ||
+            check(loc.lineno == 1 && loc.byte_col == 0,
+                  "wrong empty source location") < 0 ||
+            check(_PyTok_SourceLine(&source, 1, &line) == 0,
+                  "cannot find empty source line") < 0 ||
+            check(line.start == 0 && line.end == 0,
+                  "wrong empty source line") < 0 ||
+            check_system_error(
+                _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
+                "accepted empty source line") < 0 ||
+            check_system_error(
+                _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
+                "accepted multiple source lines") < 0 ||
+            check_system_error(
+                _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
+                "accepted missing implicit newline") < 0) {
+        goto error;
+    }
+
+    if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0,
+              "wrong first source offset") < 0 ||
+            check(_PyTok_SourceAppendLine(
+                      &source, "\xce\xb2\n", 3, 1) == 6,
+                  "wrong second source offset") < 0 ||
+            check(_PyTok_SourceAppendLine(
+                      &source, "nul\0x\n", 6, 0) == 9,
+                  "wrong third source offset") < 0) {
+        goto error;
+    }
+
+    int marker_line = 257;
+    int final_line = 300;
+    _PyTok_Off marker_start = -1;
+    for (int lineno = 4; lineno <= final_line; lineno++) {
+        const char *text = lineno == marker_line ? "marker\n" : "x\n";
+        Py_ssize_t len = (Py_ssize_t)strlen(text);
+        _PyTok_Off start = _PyTok_SourceAppendLine(
+            &source, text, len, lineno == final_line);
+        if (start < 0) {
+            goto error;
+        }
+        if (lineno == marker_line) {
+            marker_start = start;
+        }
+    }
+
+    if (check(source.nlines == final_line, "wrong source line count") < 0 ||
+            check(_PyTok_SourceLine(&source, marker_line, &line) == 0,
+                  "cannot find late source line") < 0 ||
+            check(line.start == marker_start &&
+                      line.end == marker_start + 7,
+                  "wrong late source line") < 0 ||
+            check(!line.implicit_newline && !line.contains_nul,
+                  "wrong late source flags") < 0 ||
+            check(_PyTok_SourceLine(&source, 2, &line) == 0,
+                  "cannot find second source line") < 0 ||
+            check(line.start == 6 && line.end == 9 &&
+                      line.implicit_newline && !line.contains_nul,
+                  "wrong second source line") < 0 ||
+            check(_PyTok_SourceLine(&source, 3, &line) == 0,
+                  "cannot find third source line") < 0 ||
+            check(line.contains_nul, "missing null byte flag") < 0 ||
+            check(_PyTok_SourceLine(&source, final_line, &line) == 0,
+                  "cannot find final source line") < 0 ||
+            check(line.implicit_newline,
+                  "missing late implicit newline flag") < 0) {
+        goto error;
+    }
+
+    Py_ssize_t view_len;
+    const char *view = _PyTok_SourceSpanView(
+        &source, _PyTok_SpanFromBounds(6, 8), &view_len);
+    if (check(view != NULL && view_len == 2 &&
+                  memcmp(view, "\xce\xb2", 2) == 0,
+              "wrong source span view") < 0 ||
+            check(_PyTok_SourceLocation(
+                      &source, marker_start,
+                      _PYTOK_AFFINITY_LEFT, &loc) == 0,
+                  "cannot locate left line boundary") < 0 ||
+            check(loc.lineno == marker_line - 1 && loc.byte_col == 2,
+                  "wrong left boundary location") < 0 ||
+            check(_PyTok_SourceLocation(
+                      &source, marker_start,
+                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
+                  "cannot locate right line boundary") < 0 ||
+            check(loc.lineno == marker_line && loc.byte_col == 0,
+                  "wrong right boundary location") < 0 ||
+            check(_PyTok_SourceLocation(
+                      &source, marker_start + 1,
+                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
+                  "cannot locate late source byte") < 0 ||
+            check(loc.lineno == marker_line && loc.byte_col == 1,
+                  "wrong late source location") < 0) {
+        goto error;
+    }
+
+    if (check(_PyTok_SourceLocation(
+                  &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0,
+              "cannot locate left EOF") < 0 ||
+            check(loc.lineno == final_line && loc.byte_col == 2,
+                  "wrong left EOF location") < 0 ||
+            check(_PyTok_SourceLocation(
+                      &source, source.len,
+                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
+                  "cannot locate right EOF") < 0 ||
+            check(loc.lineno == final_line + 1 && loc.byte_col == 0,
+                  "wrong right EOF location") < 0 ||
+            check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0,
+                  "cannot find virtual EOF line") < 0 ||
+            check(line.start == source.len && line.end == source.len,
+                  "wrong virtual EOF line") < 0) {
+        goto error;
+    }
+
+    view = _PyTok_SourceSpanView(
+        &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len);
+    if (check_system_error(view == NULL, "accepted invalid source span") < 0 ||
+            check_system_error(
+                _PyTok_SourceLocation(
+                    &source, source.len + 1,
+                    _PYTOK_AFFINITY_RIGHT, &loc) < 0,
+                "accepted invalid source offset") < 0 ||
+            check_system_error(
+                _PyTok_SourceLine(&source, final_line + 2, &line) < 0,
+                "accepted invalid source line") < 0) {
+        goto error;
+    }
+
+    _PyTok_SourceClear(&source);
+    _PyTok_SourceInit(&source);
+    if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
+            check_system_error(
+                _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
+                "appended after unterminated source line") < 0 ||
+            check(_PyTok_SourceLocation(
+                      &source, source.len,
+                      _PYTOK_AFFINITY_RIGHT, &loc) == 0,
+                  "cannot locate unterminated EOF") < 0 ||
+            check(loc.lineno == 1 && loc.byte_col == 4,
+                  "wrong unterminated EOF location") < 0) {
+        goto error;
+    }
+
+    _PyTok_SourceClear(&source);
+    Py_RETURN_NONE;
+
+error:
+    _PyTok_SourceClear(&source);
+    return NULL;
+}
+
+static PyObject *
+test_tokenizer_cursor(PyObject *Py_UNUSED(module),
+                      PyObject *Py_UNUSED(args))
+{
+    _PyTok_SourceText source;
+    _PyTok_SourceInit(&source);
+    if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
+            _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) {
+        goto error;
+    }
+
+    _PyTok_Cursor cursor;
+    _PyTok_CursorInit(&cursor, &source);
+    if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 ||
+            check(cursor.lineno == 3 && cursor.pos == source.len,
+                  "wrong cursor at virtual EOF") < 0 ||
+            _PyTok_CursorSetLine(&cursor, 1) < 0) {
+        goto error;
+    }
+
+    char large[BUFSIZ + 1];
+    memset(large, 'z', sizeof(large));
+    large[sizeof(large) - 1] = '\n';
+    if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) {
+        goto error;
+    }
+
+    if (check(_PyTok_CursorPeek(&cursor, 0) == 'a',
+              "wrong cursor peek after relocation") < 0 ||
+            check(_PyTok_CursorPeek(&cursor, 1) == 'b',
+                  "wrong distant cursor peek") < 0 ||
+            check(_PyTok_CursorAdvance(&cursor) == 'a',
+                  "wrong first cursor byte") < 0 ||
+            check(_PyTok_CursorAdvance(&cursor) == 'b',
+                  "wrong second cursor byte") < 0 ||
+            check(_PyTok_CursorAdvance(&cursor) == '\n',
+                  "wrong final cursor byte") < 0 ||
+            check(_PyTok_CursorAdvance(&cursor) == EOF,
+                  "cursor advanced past line") < 0 ||
+            check(_PyTok_CursorSetOffset(&cursor, 2) == 0,
+                  "cannot seek cursor offset") < 0 ||
+            check(_PyTok_CursorAdvance(&cursor) == '\n',
+                  "wrong cursor byte after seek") < 0 ||
+            check(_PyTok_CursorSetOffset(&cursor, 3) == 0,
+                  "cannot seek line boundary") < 0 ||
+            check(cursor.lineno == 2 && cursor.line_start == 3 &&
+                      _PyTok_CursorAdvance(&cursor) == 'c',
+                  "wrong cursor at line boundary") < 0 ||
+            check(_PyTok_CursorSetLine(&cursor, 3) == 0,
+                  "cannot advance cursor to final line") < 0 ||
+            check(cursor.line_start == 6 &&
+                      _PyTok_CursorAdvance(&cursor) == 'z',
+                  "wrong cursor byte on final line") < 0) {
+        goto error;
+    }
+
+    _PyTok_Cursor saved = cursor;
+    if (check_system_error(
+            _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0,
+            "accepted invalid cursor offset") < 0 ||
+            check(same_cursor(&cursor, &saved),
+                  "invalid offset changed cursor") < 0 ||
+            check_system_error(
+                _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0,
+                "accepted invalid cursor line") < 0 ||
+            check(same_cursor(&cursor, &saved),
+                  "invalid line changed cursor") < 0 ||
+            check(_PyTok_CursorSetOffset(&cursor, source.len) == 0,
+                  "cannot set cursor to EOF") < 0 ||
+            check(cursor.lineno == 4 && cursor.pos == source.len,
+                  "wrong cursor at EOF") < 0) {
+        goto error;
+    }
+
+#if SIZEOF_VOID_P > 4
+    char byte = 0;
+    _PyTok_SourceText huge_source = {
+        .bytes = &byte,
+        .len = (_PyTok_Off)INT_MAX + 1,
+    };
+    _PyTok_Cursor huge_cursor = {
+        .source = &huge_source,
+        .pos = INT_MAX,
+        .line_end = (_PyTok_Off)INT_MAX + 1,
+        .lineno = 1,
+    };
+    if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF &&
+                  huge_cursor.pos == INT_MAX,
+              "cursor advanced past maximum column") < 0) {
+        goto error;
+    }
+#endif
+
+    _PyTok_SourceClear(&source);
+    Py_RETURN_NONE;
+
+error:
+    _PyTok_SourceClear(&source);
+    return NULL;
+}
+
+static PyMethodDef test_methods[] = {
+    {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS},
+    {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS},
+    {NULL},
+};
+
+int
+_PyTestInternalCapi_Init_Tokenizer(PyObject *module)
+{
+    return PyModule_AddFunctions(module, test_methods);
+}

PCbuild/_testinternalcapi.vcxproj

@@ -100,6 +100,7 @@
     <ClCompile Include="..\Modules\_testinternalcapi\set.c" />
     <ClCompile Include="..\Modules\_testinternalcapi\complex.c" />
     <ClCompile Include="..\Modules\_testinternalcapi\interpreter.c" />
+    <ClCompile Include="..\Modules\_testinternalcapi\tokenizer.c" />
     <ClCompile Include="..\Modules\_testinternalcapi\tuple.c" />
   </ItemGroup>
   <ItemGroup>