From 230bc5d037a5ed54cb30858bf06f5c91c8426604 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 30 Aug 2026 13:36:50 +0100 Subject: [PATCH 1/2] gh-153569: bound tokenizer input storage File and readline tokenizers keep every byte they read because the lexer owns buffer growth and repairs its pointers after reallocations. Long inputs therefore grow tokenizer memory with the entire source. Let the reader reuse a bounded input window when no token or formatted string needs older bytes. Track the absolute offset of that window and save pointer offsets only when backing storage moves. --- Lib/test/test_tokenize.py | 25 +++++ Parser/lexer/buffer.c | 82 +++++++--------- Parser/lexer/buffer.h | 18 +++- Parser/lexer/state.c | 1 - Parser/lexer/state.h | 4 +- Parser/lexer/string.c | 8 +- Parser/tokenizer/reader.c | 148 +++++++++++++++++++++-------- Parser/tokenizer/reader_internal.h | 2 + 8 files changed, 190 insertions(+), 98 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 7e02191db86be5..0b5bdcdeda3c19 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2427,6 +2427,31 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self): (token.ENDMARKER, "", (1, 0), (1, 0), ""), ) + def test_fstring_offsets_survive_buffer_reallocation(self): + padding = " " * 9000 + expression_line = ")=:>{2}}\n" + physical_lines = [ + 'f"""\n', + "{(\n", + padding + "1\n", + expression_line, + '"""\n', + ] + source = "".join(physical_lines) + chunks = iter([ + "".join(physical_lines[:2]), + "".join(physical_lines[2:4]), + physical_lines[4], + "", + ]) + + expected = self._get_tokens(source, extra_tokens=True) + tokens = list(tokenize._generate_tokens_from_c_tokenizer( + chunks.__next__, + extra_tokens=True, + )) + self.assertEqual(tokens, expected) + def test_extra_tokens_relaxes_lexer_errors(self): cases = [ ( diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c index cd6885a7d01040..9c39544ca7c479 100644 --- a/Parser/lexer/buffer.c +++ b/Parser/lexer/buffer.c @@ -1,62 +1,46 @@ #include "Python.h" -#include "errcode.h" - +#include "buffer.h" #include "state.h" -/* Traverse and remember all f-string buffers, in order to be able to restore - them after reallocating tok->buf */ void -_PyLexer_remember_fstring_buffers(struct tok_state *tok) +_PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base, + _PyLexer_BufferPointers *pointers) { - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); + pointers->buf_from_base = tok->buf - base; + pointers->cur_from_buf = tok->cur - tok->buf; + pointers->inp_from_buf = tok->inp - tok->buf; + pointers->start_from_buf = tok->start == NULL + ? -1 : tok->start - tok->buf; + pointers->line_start_from_buf = tok->line_start == NULL + ? -1 : tok->line_start - tok->buf; + pointers->multi_line_start_from_buf = tok->multi_line_start == NULL + ? -1 : tok->multi_line_start - tok->buf; + for (int index = tok->tok_mode_stack_index; index > 0; --index) { + tokenizer_mode *mode = &tok->tok_mode_stack[index]; mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; - mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf; + mode->multi_line_start_offset = mode->multi_line_start == NULL + ? -1 : mode->multi_line_start - tok->buf; } } -/* Traverse and restore all f-string buffers after reallocating tok->buf */ void -_PyLexer_restore_fstring_buffers(struct tok_state *tok) -{ - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); - mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset; - mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset; - } -} - -int -_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) +_PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base, + const _PyLexer_BufferPointers *pointers) { - Py_ssize_t cur = tok->cur - tok->buf; - Py_ssize_t oldsize = tok->inp - tok->buf; - Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1); - if (newsize > tok->end - tok->buf) { - char *newbuf = tok->buf; - Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf; - Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf; - Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf; - _PyLexer_remember_fstring_buffers(tok); - newbuf = (char *)PyMem_Realloc(newbuf, newsize); - if (newbuf == NULL) { - tok->done = E_NOMEM; - return 0; - } - tok->buf = newbuf; - tok->cur = tok->buf + cur; - tok->inp = tok->buf + oldsize; - tok->end = tok->buf + newsize; - tok->start = start < 0 ? NULL : tok->buf + start; - tok->line_start = line_start < 0 ? NULL : tok->buf + line_start; - tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start; - _PyLexer_restore_fstring_buffers(tok); + tok->buf = base + pointers->buf_from_base; + tok->cur = tok->buf + pointers->cur_from_buf; + tok->inp = tok->buf + pointers->inp_from_buf; + tok->start = pointers->start_from_buf < 0 + ? NULL : tok->buf + pointers->start_from_buf; + tok->line_start = pointers->line_start_from_buf < 0 + ? NULL : tok->buf + pointers->line_start_from_buf; + tok->multi_line_start = pointers->multi_line_start_from_buf < 0 + ? NULL : tok->buf + pointers->multi_line_start_from_buf; + for (int index = tok->tok_mode_stack_index; index > 0; --index) { + tokenizer_mode *mode = &tok->tok_mode_stack[index]; + mode->start = mode->start_offset < 0 + ? NULL : tok->buf + mode->start_offset; + mode->multi_line_start = mode->multi_line_start_offset < 0 + ? NULL : tok->buf + mode->multi_line_start_offset; } - return 1; } diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h index bb218162ff4845..285da124226d50 100644 --- a/Parser/lexer/buffer.h +++ b/Parser/lexer/buffer.h @@ -3,8 +3,20 @@ #include "pyport.h" -void _PyLexer_remember_fstring_buffers(struct tok_state *tok); -void _PyLexer_restore_fstring_buffers(struct tok_state *tok); -int _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size); +struct tok_state; + +typedef struct { + Py_ssize_t buf_from_base; + Py_ssize_t cur_from_buf; + Py_ssize_t inp_from_buf; + Py_ssize_t start_from_buf; + Py_ssize_t line_start_from_buf; + Py_ssize_t multi_line_start_from_buf; +} _PyLexer_BufferPointers; + +void _PyLexer_SaveBufferPointers( + struct tok_state *, const char *, _PyLexer_BufferPointers *); +void _PyLexer_RestoreBufferPointers( + struct tok_state *, char *, const _PyLexer_BufferPointers *); #endif diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 2a6408bef927a3..e9829c60fafa12 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -26,7 +26,6 @@ _PyTokenizer_tok_new(void) tok->interactive_src_start = NULL; tok->interactive_src_end = NULL; tok->start = NULL; - tok->end = NULL; tok->done = E_OK; tok->fp = NULL; tok->tabsize = TABSIZE; diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 0824785195491e..55ddc015e21a44 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -67,15 +67,15 @@ typedef struct _tokenizer_mode { /* Tokenizer state */ struct tok_state { - /* Input state; buf <= cur <= inp <= end */ + /* Input state; buf <= cur <= inp */ /* NB an entire line is held in the buffer */ char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ + _PyTok_Off buf_offset; /* Logical offset of buf[0]. */ int fp_interactive; /* If the file descriptor is interactive */ char *interactive_src_start; /* The start of the source parsed so far in interactive mode */ char *interactive_src_end; /* The end of the source parsed so far in interactive mode */ - const char *end; /* End of input buffer if buf != NULL */ const char *start; /* Start of current token if not NULL */ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index d67c48f7f678ed..fc0299c5c7c592 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -125,7 +125,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) { assert(tok->cur != NULL); - Py_ssize_t size = strlen(tok->cur); + Py_ssize_t size = cur == 0 + ? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur); tokenizer_mode *tok_mode = TOK_GET_MODE(tok); switch (cur) { @@ -142,7 +143,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) goto error; } tok_mode->last_expr_buffer = new_buffer; - strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size); + memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, + tok->cur, size); tok_mode->last_expr_size += size; break; case '{': @@ -155,7 +157,7 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) } tok_mode->last_expr_size = size; tok_mode->last_expr_end = -1; - strncpy(tok_mode->last_expr_buffer, tok->cur, size); + memcpy(tok_mode->last_expr_buffer, tok->cur, size); break; case '}': case '!': diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 82b824f56374fc..68c8da2186ede9 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -13,6 +13,12 @@ # include #endif +static inline int +reader_is_streaming(_PyTok_ReaderKind kind) +{ + return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE; +} + void _PyTok_ReaderFree(struct tok_state *tok) { @@ -28,10 +34,10 @@ _PyTok_ReaderFree(struct tok_state *tok) } PyMem_Free(reader->file_buffer); PyMem_Free(reader->decoded); - if (reader->kind != _PYTOK_READER_PREPARED) { + if (reader_is_streaming(reader->kind)) { PyMem_Free(tok->buf); - tok->buf = NULL; } + tok->buf = NULL; PyMem_Free(reader); tok->reader = NULL; } @@ -60,6 +66,26 @@ reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed) return 0; } +static int +reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed) +{ + _PyTok_Reader *reader = tok->reader; + if (needed <= reader->input_buffer_cap) { + return 0; + } + assert(tok->buf != NULL); + assert(tok->cur >= tok->buf && tok->cur <= tok->inp); + assert(tok->inp - tok->buf <= reader->input_buffer_cap); + _PyLexer_BufferPointers pointers; + _PyLexer_SaveBufferPointers(tok, tok->buf, &pointers); + if (reserve_buffer( + &tok->buf, &reader->input_buffer_cap, needed) < 0) { + return -1; + } + _PyLexer_RestoreBufferPointers(tok, tok->buf, &pointers); + return 0; +} + static int append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len) { @@ -529,19 +555,31 @@ reader_next(struct tok_state *tok, _PyTok_Chunk *chunk) Py_UNREACHABLE(); } +static void +reset_streaming_buffer(struct tok_state *tok) +{ + assert(tok->buf != NULL); + assert(tok->cur >= tok->buf && tok->cur <= tok->inp); + Py_ssize_t consumed = tok->inp - tok->buf; + assert(tok->buf_offset <= PY_SSIZE_T_MAX - consumed); + tok->buf_offset += consumed; + tok->cur = tok->inp = tok->buf; +} + int _PyTok_ReaderUnderflow(struct tok_state *tok) { - int prepared = tok->reader->kind == _PYTOK_READER_PREPARED; + _PyTok_ReaderKind kind = tok->reader->kind; + int prepared = kind == _PYTOK_READER_PREPARED; + int streaming = reader_is_streaming(kind); int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok); - if (reset_buffer && tok->reader->kind != _PYTOK_READER_INTERACTIVE) { - tok->cur = tok->inp = tok->buf; - } - _PyTok_Chunk chunk; _PyTok_ReadResult result = reader_next(tok, &chunk); if (result != _PYTOK_READ_LINE) { + if (reset_buffer && streaming) { + reset_streaming_buffer(tok); + } if (result == _PYTOK_READ_EOF) { tok->done = E_EOF; } @@ -558,34 +596,68 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) ? E_NOMEM : E_ERROR; } } - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && + if (kind == _PYTOK_READER_INTERACTIVE && result != _PYTOK_READ_STOPPED) { PySys_WriteStderr("\n"); } return 0; } - Py_ssize_t copy_len = chunk.len; - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && + Py_ssize_t scan_len = chunk.len; + if (kind == _PYTOK_READER_INTERACTIVE && chunk.implicit_newline) { - copy_len--; + scan_len--; } - if (reset_buffer && tok->reader->kind == _PYTOK_READER_INTERACTIVE) { - tok->cur = tok->inp = tok->buf; - } - if (!prepared && !_PyLexer_tok_reserve_buf(tok, copy_len + 1)) { - _PyTok_ChunkClear(&chunk); - tok->input_error = 1; - return 0; + if (streaming) { + if (reset_buffer) { + reset_streaming_buffer(tok); + } + Py_ssize_t used = tok->inp - tok->buf; + int overflow = scan_len > PY_SSIZE_T_MAX - used - 1 || + tok->buf_offset > PY_SSIZE_T_MAX - used - scan_len; + if (overflow) { + PyErr_NoMemory(); + } + if (overflow || reserve_input_buffer(tok, used + scan_len + 1) < 0) { + _PyTok_ChunkClear(&chunk); + tok->done = E_NOMEM; + tok->input_error = 1; + return 0; + } + memcpy(tok->inp, chunk.data, (size_t)scan_len); + tok->inp += scan_len; + *tok->inp = '\0'; } - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && - _PyTok_SourceAppendLine(&tok->source, chunk.data, chunk.len, - chunk.implicit_newline) < 0) { - _PyTok_ChunkClear(&chunk); - tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) - ? E_NOMEM : E_ERROR; - tok->input_error = 1; - return 0; + else if (!prepared) { + int source_will_grow = + chunk.len > tok->source.cap - tok->source.len - 1; + _PyLexer_BufferPointers pointers; + if (!reset_buffer && source_will_grow) { + _PyLexer_SaveBufferPointers( + tok, tok->source.bytes, &pointers); + } + _PyTok_Off source_start = _PyTok_SourceAppendLine( + &tok->source, chunk.data, chunk.len, + chunk.implicit_newline); + if (source_start < 0) { + _PyTok_ChunkClear(&chunk); + tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) + ? E_NOMEM : E_ERROR; + tok->input_error = 1; + return 0; + } + if (reset_buffer) { + tok->buf = tok->cur = tok->source.bytes + source_start; + tok->buf_offset = source_start; + tok->line_start = tok->buf; + tok->start = NULL; + tok->multi_line_start = NULL; + } + else if (source_will_grow) { + _PyLexer_RestoreBufferPointers( + tok, tok->source.bytes, &pointers); + } + tok->inp = tok->source.bytes + source_start + scan_len; } if (tok->fp_interactive) { tok->interactive_src_start = tok->source.bytes; @@ -594,14 +666,10 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) if (prepared) { if (tok->start == NULL) { tok->buf = tok->cur; + tok->buf_offset = chunk.data - tok->source.bytes; } tok->inp = chunk.data + chunk.len; } - else { - memcpy(tok->inp, chunk.data, (size_t)copy_len); - tok->inp += copy_len; - *tok->inp = '\0'; - } tok->implicit_newline = chunk.implicit_newline; if (!prepared && tok->tok_mode_stack_index && @@ -611,7 +679,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) return 0; } ADVANCE_LINENO(); - if (tok->reader->kind == _PYTOK_READER_FILE && + if (kind == _PYTOK_READER_FILE && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) { _PyTok_ChunkClear(&chunk); @@ -639,14 +707,15 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind) if (kind == _PYTOK_READER_PREPARED) { return tok; } - tok->buf = PyMem_Malloc(BUFSIZ); - if (tok->buf == NULL) { - PyErr_NoMemory(); - _PyTokenizer_Free(tok); - return NULL; + if (reader_is_streaming(kind)) { + if (reserve_buffer( + &tok->buf, &tok->reader->input_buffer_cap, BUFSIZ) < 0) { + _PyTokenizer_Free(tok); + return NULL; + } + tok->cur = tok->inp = tok->buf; + tok->buf[0] = '\0'; } - tok->cur = tok->inp = tok->buf; - tok->end = tok->buf + BUFSIZ; return tok; } @@ -664,7 +733,6 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input, return NULL; } tok->buf = tok->cur = tok->inp = tok->str; - tok->end = tok->buf; return tok; } diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 121d0f96f6698a..49a6f04ec60af2 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -44,6 +44,8 @@ typedef struct _PyTok_Reader { PyObject *decoder; const char *nextprompt; + Py_ssize_t input_buffer_cap; + char *file_buffer; Py_ssize_t file_buffer_cap; _PyTok_Chunk prefetched_lines[2]; From f134e816fff8dd8adad43180aba9156f4f6eee8f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 30 Aug 2026 13:36:50 +0100 Subject: [PATCH 2/2] gh-153569: return tokenizer tokens as source spans Tokenizer results expose pointers into the active input buffer. That ties every consumer to the buffer lifetime and prevents the reader from reusing older storage. Return logical source spans with their start and end locations instead. Pegen and `_tokenize` materialize a short-lived view before requesting another token, and the tokenizer no longer keeps a second end pointer for the last token. --- Parser/lexer/lexer.c | 14 ++++++----- Parser/lexer/lexer.h | 21 ++++++++++++++++ Parser/lexer/state.c | 51 +++++++++++++++++++++------------------ Parser/lexer/state.h | 9 +++---- Parser/pegen.c | 41 +++++++++++++++++-------------- Parser/tokenizer/source.h | 3 ++- Python/Python-tokenize.c | 45 ++++++++++++++++++++-------------- 7 files changed, 113 insertions(+), 71 deletions(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index a96362c8961023..f96b31b9d2f38a 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -12,8 +12,6 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) -#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\ - _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end)) /* Spaces in this constant are treated as "zero or more spaces or tabs" when tokenizing. */ @@ -360,21 +358,25 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str && !(tok->cur > ignore_end && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0])))); + int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT; + int start_col_offset = is_type_ignore + ? ignore_end_col_offset : current_starting_col_offset; + p_end = tok->cur; if (is_type_ignore) { p_start = ignore_end; - p_end = tok->cur; /* If this type ignore is the only thing on the line, consume the newline also. */ if (blankline) { tok_nextc(tok); tok->atbol = 1; } - return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset); } else { p_start = type_start; - p_end = tok->cur; - return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset); } + _PyLexer_token_setup(tok, token, type, p_start, p_end); + token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset}; + token->end_loc = (_PyTok_Loc){tok->lineno, tok->col_offset}; + return type; } } if (tok->tok_extra_tokens) { diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index 1d97ac57b745b0..040935a7e68913 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -7,4 +7,25 @@ int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur); int _PyTokenizer_Get(struct tok_state *, struct token *); +/* The view points into the current input window. The next + _PyTokenizer_Get() call may discard it. */ +static inline const char * +_PyToken_TextView(const struct tok_state *tok, const struct token *token, + Py_ssize_t *length) +{ + assert(length != NULL); + if (token->span.start < 0) { + assert(token->span.start == -1 && token->span.end == -1); + *length = 0; + return ""; + } + assert(_PyTok_SpanIsValid(token->span)); + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + assert(token->span.start >= tok->buf_offset); + assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf); + *length = token->span.end - token->span.start; + return tok->buf + (token->span.start - tok->buf_offset); +} + #endif diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index e9829c60fafa12..d82a7d0f296bac 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -100,41 +100,46 @@ _PyToken_Free(struct token *token) { void _PyToken_Init(struct token *token) { +#ifdef Py_DEBUG + token->span = (_PyTok_Span){-1, -1}; + token->start_loc = (_PyTok_Loc){-1, -1}; + token->end_loc = (_PyTok_Loc){-1, -1}; +#endif token->metadata = NULL; } -int -_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, - int end_col_offset, const char *start, const char *end) +static inline _PyTok_Span +buffer_span(const struct tok_state *tok, const char *start, const char *end) { - token->level = tok->level; - token->lineno = token->end_lineno = tok->lineno; - token->col_offset = col_offset; - token->end_col_offset = end_col_offset; - token->start = start; - token->end = end; - return type; + if (start == NULL) { + assert(end == NULL); + return (_PyTok_Span){-1, -1}; + } + assert(end != NULL); + const char *base = tok->buf; + assert(base != NULL); + assert(tok->inp >= base); + Py_ssize_t start_offset = start - base; + Py_ssize_t end_offset = end - base; + assert(start_offset >= 0 && start_offset <= end_offset); + assert(end_offset <= tok->inp - base); + assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset); + return _PyTok_SpanFromBounds( + tok->buf_offset + start_offset, tok->buf_offset + end_offset); } int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end) { - assert((start == NULL && end == NULL) || (start != NULL && end != NULL)); token->level = tok->level; - if (ISSTRINGLIT(type)) { - token->lineno = tok->first_lineno; - } - else { - token->lineno = tok->lineno; - } - token->end_lineno = tok->lineno; - token->col_offset = token->end_col_offset = -1; - token->start = start; - token->end = end; + token->span = buffer_span(tok, start, end); + int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno; + token->start_loc = (_PyTok_Loc){lineno, -1}; + token->end_loc = (_PyTok_Loc){tok->lineno, -1}; if (start != NULL && end != NULL) { - token->col_offset = tok->starting_col_offset; - token->end_col_offset = tok->col_offset; + token->start_loc.byte_col = tok->starting_col_offset; + token->end_loc.byte_col = tok->col_offset; } return type; } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 55ddc015e21a44..496962fd0484f0 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -23,8 +23,9 @@ enum interactive_underflow_t { struct token { int level; - int lineno, col_offset, end_lineno, end_col_offset; - const char *start, *end; + _PyTok_Span span; + _PyTok_Loc start_loc; + _PyTok_Loc end_loc; PyObject *metadata; }; @@ -69,7 +70,7 @@ typedef struct _tokenizer_mode { struct tok_state { /* Input state; buf <= cur <= inp */ /* NB an entire line is held in the buffer */ - char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */ + char *buf; /* Owned for file/readline input; source-backed otherwise. */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ _PyTok_Off buf_offset; /* Logical offset of buf[0]. */ @@ -128,8 +129,6 @@ struct tok_state { #endif }; -int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, - int end_col_offset, const char *start, const char *end); int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); struct tok_state *_PyTokenizer_tok_new(void); diff --git a/Parser/pegen.c b/Parser/pegen.c index fcec810037e98d..d86dd22444e6a7 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -171,18 +171,17 @@ growable_comment_array_deallocate(growable_comment_array *arr) { } static int -_get_keyword_or_name_type(Parser *p, struct token *new_token) +_get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length) { - Py_ssize_t name_len = new_token->end_col_offset - new_token->col_offset; - assert(name_len > 0); + assert(length > 0); - if (name_len >= p->n_keyword_lists || - p->keywords[name_len] == NULL || - p->keywords[name_len]->type == -1) { + if (length >= p->n_keyword_lists || + p->keywords[length] == NULL || + p->keywords[length]->type == -1) { return NAME; } - for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) { - if (strncmp(k->str, new_token->start, (size_t)name_len) == 0) { + for (KeywordToken *k = p->keywords[length]; k != NULL && k->type != -1; k++) { + if (memcmp(k->str, text, (size_t)length) == 0) { return k->type; } } @@ -193,8 +192,11 @@ static int initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) { assert(parser_token != NULL); - parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type; - parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start); + Py_ssize_t length; + const char *text = _PyToken_TextView(p->tok, new_token, &length); + parser_token->type = token_type == NAME + ? _get_keyword_or_name_type(p, text, length) : token_type; + parser_token->bytes = PyBytes_FromStringAndSize(text, length); if (parser_token->bytes == NULL) { return -1; } @@ -214,12 +216,14 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to } parser_token->level = new_token->level; - parser_token->lineno = new_token->lineno; - parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset - : new_token->col_offset; - parser_token->end_lineno = new_token->end_lineno; - parser_token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_col_offset - : new_token->end_col_offset; + parser_token->lineno = new_token->start_loc.lineno; + parser_token->col_offset = p->tok->lineno == p->starting_lineno + ? p->starting_col_offset + new_token->start_loc.byte_col + : new_token->start_loc.byte_col; + parser_token->end_lineno = new_token->end_loc.lineno; + parser_token->end_col_offset = p->tok->lineno == p->starting_lineno + ? p->starting_col_offset + new_token->end_loc.byte_col + : new_token->end_loc.byte_col; p->fill += 1; @@ -261,13 +265,14 @@ _PyPegen_fill_token(Parser *p) // Record and skip '# type: ignore' comments while (type == TYPE_IGNORE) { - Py_ssize_t len = new_token.end_col_offset - new_token.col_offset; + Py_ssize_t len; + const char *text = _PyToken_TextView(p->tok, &new_token, &len); char *tag = PyMem_Malloc((size_t)len + 1); if (tag == NULL) { PyErr_NoMemory(); goto error; } - strncpy(tag, new_token.start, (size_t)len); + memcpy(tag, text, (size_t)len); tag[len] = '\0'; // Ownership of tag passes to the growable array if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) { diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index b42ecda1b31aa5..363475ff9015e3 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -5,7 +5,8 @@ typedef Py_ssize_t _PyTok_Off; -/* Half-open byte offsets into a _PyTok_SourceText. */ +/* Spans use half-open logical byte offsets into decoded input. Their backing + storage may retain only the current input window. */ typedef struct { _PyTok_Off start; _PyTok_Off end; diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 762b7b3e4c8d71..71f236b08d93c8 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -203,14 +203,19 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si } static void -_get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_start, - PyObject *line, int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno, +_get_col_offsets(tokenizeriterobject *it, const struct token *token, + const char *token_start, const char *line_start, + PyObject *line, int line_changed, Py_ssize_t *col_offset, Py_ssize_t *end_col_offset) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); + const char *token_end = token_start == NULL + ? NULL : token_start + token->span.end - token->span.start; + Py_ssize_t lineno = token->start_loc.lineno; + Py_ssize_t end_lineno = token->end_loc.lineno; Py_ssize_t byte_offset = -1; - if (token.start != NULL && token.start >= line_start) { - byte_offset = token.start - line_start; + if (token_start != NULL && token_start >= line_start) { + byte_offset = token_start - line_start; if (line_changed) { *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); it->byte_col_offset_diff = byte_offset - *col_offset; @@ -220,15 +225,13 @@ _get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_s } } - if (token.end != NULL && token.end >= it->tok->line_start) { - Py_ssize_t end_byte_offset = token.end - it->tok->line_start; + if (token_end != NULL && token_end >= it->tok->line_start) { + Py_ssize_t end_byte_offset = token_end - it->tok->line_start; if (lineno == end_lineno) { - // If the whole token is at the same line, we can just use the token.start - // buffer for figuring out the new column offset, since using line is not - // performant for very long lines. + // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); *end_col_offset = *col_offset + token_col_offset; - it->byte_col_offset_diff += token.end - token.start - token_col_offset; + it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset); @@ -263,12 +266,17 @@ tokenizeriter_next(PyObject *op) it->done = 1; goto exit; } - PyObject *str = NULL; - if (token.start == NULL || token.end == NULL) { + const char *token_start = NULL; + PyObject *str; + if (token.span.start < 0) { + assert(token.span.start == -1 && token.span.end == -1); str = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } else { - str = PyUnicode_FromStringAndSize(token.start, token.end - token.start); + Py_ssize_t token_length; + token_start = _PyToken_TextView( + it->tok, &token, &token_length); + str = PyUnicode_FromStringAndSize(token_start, token_length); } if (str == NULL) { goto exit; @@ -297,12 +305,12 @@ tokenizeriter_next(PyObject *op) goto exit; } - Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno; - Py_ssize_t end_lineno = it->tok->lineno; + Py_ssize_t lineno = token.start_loc.lineno; + Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - _get_col_offsets(it, token, line_start, line, line_changed, - lineno, end_lineno, &col_offset, &end_col_offset); + _get_col_offsets(it, &token, token_start, line_start, line, line_changed, + &col_offset, &end_col_offset); if (it->tok->tok_extra_tokens) { if (is_trailing_token) { @@ -317,7 +325,8 @@ tokenizeriter_next(PyObject *op) else if (type == NEWLINE) { Py_DECREF(str); if (!it->tok->implicit_newline) { - if (it->tok->start[0] == '\r') { + assert(token_start != NULL); + if (token_start[0] == '\r') { str = PyUnicode_FromString("\r\n"); } else { str = PyUnicode_FromString("\n");