From 71f341c445617704d514f0532a4aedd8122f3c7e Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 13 Nov 2026 16:19:29 +0900 Subject: [PATCH] Add interpreter runtime hooks at Python literal decoding boundaries Keep decoder failures across diagnostic backtracking and dispatch literal warnings in grammar action order. Preserve format-spec exception provenance and retain standalone parser behavior. Assisted-by: Codex:gpt-6.1-sol --- crates/ruff_python_parser/src/error.rs | 8 + crates/ruff_python_parser/src/lib.rs | 56 ++ .../src/parser/expression.rs | 124 ++++- crates/ruff_python_parser/src/parser/mod.rs | 57 ++ crates/ruff_python_parser/src/string.rs | 53 +- .../ruff_python_parser/tests/unicode_names.rs | 520 ++++++++++++++++++ 6 files changed, 800 insertions(+), 18 deletions(-) create mode 100644 crates/ruff_python_parser/tests/unicode_names.rs diff --git a/crates/ruff_python_parser/src/error.rs b/crates/ruff_python_parser/src/error.rs index 9a20cc90bda93..10fb5b92b9edb 100644 --- a/crates/ruff_python_parser/src/error.rs +++ b/crates/ruff_python_parser/src/error.rs @@ -823,6 +823,10 @@ pub enum LexicalErrorType { escaped_end_quote: bool, detected_line: u32, }, + /// The interpreter could not load the Unicode name provider for a `\N` escape. + UnicodeNameProviderUnavailable, + /// A format-spec decoder failure propagates its original exception, not `SyntaxError`. + FormatSpecDecodeError(Box), /// An escape sequence in a string literal cannot be decoded. `start` and `end` are the /// inclusive positions of the escape sequence in the string content, where a non-ASCII /// character counts as its ten-character `\U` escape. @@ -949,6 +953,8 @@ impl LexicalErrorType { pub fn is_tokenizer_error(&self) -> bool { match self { Self::StringError + | Self::FormatSpecDecodeError(_) + | Self::UnicodeNameProviderUnavailable | Self::UnicodeEscapeError { .. } | Self::BytesEscapeError { .. } | Self::InvalidByteLiteral @@ -1006,6 +1012,8 @@ impl std::fmt::Display for LexicalErrorType { Self::InvalidByteLiteral => { write!(f, "bytes can only contain ASCII literal characters") } + Self::FormatSpecDecodeError(error) => std::fmt::Display::fmt(error, f), + Self::UnicodeNameProviderUnavailable => write!(f, "(unicode error) {}", crate::UnicodeNameProviderUnavailable), Self::UnicodeEscapeError { kind, start, end } => write!( f, "(unicode error) 'unicodeescape' codec can't decode bytes in position {start}-{end}: {kind}" diff --git a/crates/ruff_python_parser/src/lib.rs b/crates/ruff_python_parser/src/lib.rs index 8e3aa126e03da..9d7887e1cc4d0 100644 --- a/crates/ruff_python_parser/src/lib.rs +++ b/crates/ruff_python_parser/src/lib.rs @@ -70,6 +70,51 @@ pub use crate::error::{ }; pub use crate::parser::ParseOptions; +/// A per-parse Unicode character-name provider for embedded interpreters. +/// +/// Preparation runs when a non-raw, non-bytes literal actually decodes `\N`, before +/// validating its braces or name. Implementations should cache preparation for this parse, +/// including failure: parser recovery may decode a literal more than once. No provider is retained after parsing. +/// Names are passed as UTF-8 bytes, without case folding or normalization. +/// Like numeric escapes, surrogate code points are represented by the replacement character +/// in the scalar-only AST; this interface does not add surrogate-preserving strings. +pub trait UnicodeNameProvider: std::fmt::Debug { + /// Load the database, or report the interpreter's Unicode name import failure. + fn prepare(&self) -> Result<(), UnicodeNameProviderUnavailable>; + /// Resolve one character name after preparation, excluding named sequences. + fn lookup(&self, name: &[u8]) -> Option; +} + +/// Interpreter effects executed at string-decoding boundaries. +/// +/// A runtime is borrowed only for this parse. Diagnostic reparses should share the same +/// session cache so that module preparation and emitted warnings are not repeated. +pub trait ParseRuntime: UnicodeNameProvider { + /// Called after a literal (or f/t-string literal fragment) has decoded successfully. + /// `source` is the undecoded content of that literal, without its quote delimiters. + /// An error aborts interpreter effects and survives grammar backtracking. + fn literal_decoded( + &self, + _source: &str, + _flags: ruff_python_ast::AnyStringFlags, + _range: ruff_text_size::TextRange, + ) -> Result<(), ParseError> { + Ok(()) + } +} + +/// The interpreter could not load its Unicode character-name database. +#[derive(Debug, Clone, Copy)] +pub struct UnicodeNameProviderUnavailable; + +impl std::fmt::Display for UnicodeNameProviderUnavailable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("\\N escapes not supported (can't load unicodedata module)") + } +} + +impl std::error::Error for UnicodeNameProviderUnavailable {} + use crate::parser::Parser; use ruff_python_ast::token::{Token, TokenFlags, TokenKind, Tokens}; @@ -291,6 +336,17 @@ pub fn parse_unchecked(source: &str, options: ParseOptions) -> Parsed { Parser::new(source, options).parse() } +/// Parse with interpreter-owned Unicode names and literal-warning dispatch. +/// Decoder and runtime callback failures abort interpreter effects even when grammar +/// diagnostics rewind. Existing parsing APIs use the built-in database without callbacks. +pub fn parse_unchecked_with_runtime( + source: &str, + options: ParseOptions, + provider: &dyn ParseRuntime, +) -> Parsed { + Parser::new(source, options).with_runtime(provider).parse() +} + /// Parse the given Python source code using the specified [`PySourceType`]. pub fn parse_unchecked_source(source: &str, source_type: PySourceType) -> Parsed { // SAFETY: Safe because `PySourceType` always parses to a `ModModule` diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 25f74cd8ea4f0..b654a6971da22 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -13,7 +13,8 @@ use ruff_python_ast::{ use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use crate::error::{ - ComprehensionUnpackingKind, FStringKind, StarTupleKind, UnparenthesizedNamedExprKind, + ComprehensionUnpackingKind, FStringKind, LexicalError, LexicalErrorType, StarTupleKind, + UnparenthesizedNamedExprKind, }; use crate::parser::progress::ParserProgress; use crate::parser::{FunctionKind, IpyEscapeContext, Parser, helpers}; @@ -416,6 +417,32 @@ impl<'src> Parser<'src> { { return; } + if self + .runtime_for_literal(self.current_token_range()) + .is_some() + { + // invalid_expression uses expression_without_invalid on its right-hand side. + // Do not recursively run invalid-expression actions before committing its error. + let checkpoint = self.checkpoint(); + self.parse_conditional_expression_without_comma_check(ExpressionContext::default()); + self.rewind(checkpoint); + + // If the missing-comma action declines to raise (outside brackets, or NAME + // STRING), CPython next tries invalid_legacy_expression: NAME !'(' star_expressions. + // That rule evaluates its operand even for names other than print and exec. + if let Expr::Name(name) = &first.expr + && !first.is_parenthesized + && !self.at(TokenKind::Lpar) + && (self.bracket_level() == 0 + || self.at(TokenKind::String) + || matches!(name.id.as_str(), "print" | "exec")) + && self.decoder_abort.is_none() + { + let checkpoint = self.checkpoint(); + self.parse_expression_list(ExpressionContext::starred_bitwise_or()); + self.rewind(checkpoint); + } + } if let Expr::Name(name) = &first.expr && !first.is_parenthesized { @@ -444,7 +471,11 @@ impl<'src> Parser<'src> { // [a b +] let checkpoint = self.checkpoint(); let errors_before = self.errors.len(); - let second = self.parse_conditional_expression_or_higher(); + let second = if self.runtime.is_some() { + self.parse_conditional_expression_without_comma_check(ExpressionContext::default()) + } else { + self.parse_conditional_expression_or_higher() + }; let second_end = self.complete_prefix_end(&second.expr, errors_before); self.rewind(checkpoint); let Some(second_end) = second_end else { @@ -2020,9 +2051,13 @@ impl<'src> Parser<'src> { let value = self.bump_string_value(); - match parse_string_literal(value, flags, range) { - Ok(string) => string, + match parse_string_literal(value, flags, range, self.runtime_for_literal(range)) { + Ok(string) => { + self.literal_decoded(value, flags, range); + string + } Err(error) => { + self.record_decoder_abort(&error); let location = error.location(); self.add_error(ParseErrorType::Lexical(error.into_error()), location); @@ -2068,11 +2103,12 @@ impl<'src> Parser<'src> { kind: InterpolatedStringKind, ) -> InterpolatedStringData { let start = self.node_start(); + let errors_before = self.errors.len(); let mut flags = self.tokens.current_flags().as_any_string_flags(); self.bump(kind.start_token()); let outer_escape_error = self.interpolated_string_escape_error.take(); - let elements = self.parse_interpolated_string_elements( + let mut elements = self.parse_interpolated_string_elements( flags, InterpolatedStringElementsKind::Regular(kind), kind, @@ -2082,6 +2118,12 @@ impl<'src> Parser<'src> { if !self.expect(kind.end_token()) { flags = flags.with_unclosed(true); } + if self.runtime.is_some() + && self.errors.len() == errors_before + && !self.tokens.has_tokenizer_error_in(self.node_range(start)) + { + self.decode_interpolated_literals(&mut elements, flags); + } // test_err interpolated_string_escape_error // f'a\N b{x}' // t'{x}\x1' 'c' @@ -2099,6 +2141,35 @@ impl<'src> Parser<'src> { } } + fn decode_interpolated_literals( + &mut self, + elements: &mut ast::InterpolatedStringElements, + flags: ast::AnyStringFlags, + ) { + for element in elements { + if let InterpolatedStringElement::Literal(literal) = element { + let range = literal.range; + let source = self.src_text(range); + match parse_interpolated_string_literal_element( + source, + flags, + range, + self.runtime_for_literal(range), + ) { + Ok(decoded) => { + self.literal_decoded(source, flags, range); + *literal = decoded; + } + Err(error) => { + self.record_decoder_abort(&error); + self.interpolated_string_escape_error + .get_or_insert(error.into_error()); + } + } + } + } + } + /// Check `range` for comment tokens, report an `UnsupportedSyntaxError` for each one found, /// and return whether any comments were found. fn check_fstring_comments(&mut self, range: TextRange) -> bool { @@ -2148,9 +2219,45 @@ impl<'src> Parser<'src> { let range = parser.current_token_range(); let value = parser.current_token_text(); parser.bump(middle_token_kind); - InterpolatedStringElement::Literal( - parse_interpolated_string_literal_element(value, flags, range) + if parser.runtime.is_some() + && matches!(elements_kind, InterpolatedStringElementsKind::Regular(_)) + { + // The joined-string semantic action runs only after its grammar + // succeeds. Expression literals are evaluated independently. + InterpolatedStringElement::Literal( + ast::InterpolatedStringLiteralElement { + value: "".into(), + range, + node_index: AtomicNodeIndex::NONE, + }, + ) + } else { + InterpolatedStringElement::Literal( + parse_interpolated_string_literal_element( + value, + flags, + range, + parser.runtime_for_literal(range), + ) + .inspect(|_| { + parser.literal_decoded(value, flags, range); + }) .unwrap_or_else(|lex_error| { + let lex_error = if parser.runtime_for_literal(range).is_some() + && matches!( + elements_kind, + InterpolatedStringElementsKind::FormatSpec(_) + ) { + LexicalError::new( + LexicalErrorType::FormatSpecDecodeError(Box::new( + lex_error.into_error(), + )), + range, + ) + } else { + lex_error + }; + parser.record_decoder_abort(&lex_error); // test_err invalid_fstring_literal_element // f'hello \N{INVALID} world' // f"""hello \N{INVALID} world""" @@ -2163,7 +2270,8 @@ impl<'src> Parser<'src> { node_index: AtomicNodeIndex::NONE, } }), - ) + ) + } } // `Invalid` tokens are created when there's a lexical error, so // we ignore it here to avoid creating unexpected token errors diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index c6d4fc9faf186..0dd5bf8db563d 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -64,6 +64,9 @@ impl NameInterner { #[derive(Debug)] pub(crate) struct Parser<'src> { source: &'src str, + runtime: Option<&'src dyn crate::ParseRuntime>, + /// Interpreter decoder failures cannot be undone by grammar backtracking. + decoder_abort: Option, /// Token source for the parser that skips over any non-trivia token. tokens: TokenSource<'src>, @@ -162,6 +165,8 @@ impl<'src> Parser<'src> { Parser { options, source, + runtime: None, + decoder_abort: None, errors: Vec::new(), deferred_starred_unpacking_error: None, unsupported_syntax_errors: Vec::new(), @@ -186,6 +191,49 @@ impl<'src> Parser<'src> { } } + /// Recovery after a committed syntax error must not load interpreter modules while + /// decoding later literals. The built-in database remains available for recovery ASTs. + fn runtime_for_literal(&self, range: TextRange) -> Option<&'src dyn crate::ParseRuntime> { + let runtime = self.runtime?; + if self.decoder_abort.is_none() + && !self + .tokens + .has_tokenizer_error_in(TextRange::new(self.start_offset, range.start())) + && self.errors.is_empty() + { + Some(runtime) + } else { + None + } + } + + fn literal_decoded( + &mut self, + source: &str, + flags: ruff_python_ast::AnyStringFlags, + range: TextRange, + ) { + if let Some(runtime) = self.runtime_for_literal(range) + && let Err(error) = runtime.literal_decoded(source, flags, range) + { + self.decoder_abort.get_or_insert(error); + } + } + + fn record_decoder_abort(&mut self, error: &crate::error::LexicalError) { + if self.runtime_for_literal(error.location()).is_some() { + self.decoder_abort.get_or_insert_with(|| ParseError { + error: ParseErrorType::Lexical(error.error().clone()), + location: error.location(), + }); + } + } + + pub(crate) fn with_runtime(mut self, provider: &'src dyn crate::ParseRuntime) -> Self { + self.runtime = Some(provider); + self + } + /// Consumes the [`Parser`] and returns the parsed [`Parsed`]. pub(crate) fn parse(mut self) -> Parsed { let syntax = self.maybe_grow_stack(|parser| match parser.options.mode { @@ -263,6 +311,15 @@ impl<'src> Parser<'src> { TokenKind::EndOfFile, "Parser should be at the end of the file." ); + if let Some(error) = self.decoder_abort { + let (tokens, _) = self.tokens.finish(); + return Parsed { + syntax, + tokens: Tokens::new(tokens), + errors: vec![error], + unsupported_syntax_errors: self.unsupported_syntax_errors, + }; + } // TODO consider re-integrating lexical error handling into the parser? let mut parse_errors = self.errors; if !parse_errors.is_empty() diff --git a/crates/ruff_python_parser/src/string.rs b/crates/ruff_python_parser/src/string.rs index 020fbd77e095d..bf18ea1b8bae4 100644 --- a/crates/ruff_python_parser/src/string.rs +++ b/crates/ruff_python_parser/src/string.rs @@ -86,6 +86,7 @@ enum EscapedChar { } struct StringParser<'src> { + runtime: Option<&'src dyn crate::ParseRuntime>, /// The raw content of the string e.g., the `foo` part in `"foo"`. source: &'src str, /// Current position of the parser in the source. @@ -99,9 +100,16 @@ struct StringParser<'src> { } impl<'src> StringParser<'src> { - fn new(source: &'src str, flags: AnyStringFlags, offset: TextSize, range: TextRange) -> Self { + fn new( + source: &'src str, + flags: AnyStringFlags, + offset: TextSize, + range: TextRange, + runtime: Option<&'src dyn crate::ParseRuntime>, + ) -> Self { Self { source, + runtime, cursor: 0, flags, offset, @@ -256,6 +264,11 @@ impl<'src> StringParser<'src> { } fn parse_unicode_name(&mut self, escape_start: usize) -> Result { + if let Some(provider) = self.runtime { + provider.prepare().map_err(|_| { + LexicalError::new(LexicalErrorType::UnicodeNameProviderUnavailable, self.range) + })?; + } if self.peek_char() != Some('{') { return Err(self.escape_error( UnicodeEscapeErrorKind::MalformedName, @@ -279,16 +292,26 @@ impl<'src> StringParser<'src> { )); } + let provider = self.runtime; let name_and_ending = self.skip_bytes(close_idx + 1); let name = &name_and_ending[..name_and_ending.len() - 1]; - unicode_names2::character(name).ok_or_else(|| { - self.escape_error( - UnicodeEscapeErrorKind::UnknownName, - escape_start, - self.cursor, - ) - }) + let codepoint = match provider { + Some(provider) => provider.lookup(name.as_bytes()), + None => unicode_names2::character(name).map(u32::from), + }; + codepoint + .and_then(|codepoint| match codepoint { + 0xD800..=0xDFFF => Some(char::REPLACEMENT_CHARACTER), + _ => char::from_u32(codepoint), + }) + .ok_or_else(|| { + self.escape_error( + UnicodeEscapeErrorKind::UnknownName, + escape_start, + self.cursor, + ) + }) } /// Parse an escaped character, returning the new character. @@ -567,16 +590,26 @@ pub(crate) fn parse_string_literal( source: &str, flags: AnyStringFlags, range: TextRange, + runtime: Option<&dyn crate::ParseRuntime>, ) -> Result { - StringParser::new(source, flags, range.start() + flags.opener_len(), range).parse() + StringParser::new( + source, + flags, + range.start() + flags.opener_len(), + range, + runtime, + ) + .parse() } pub(crate) fn parse_interpolated_string_literal_element( source: &str, flags: AnyStringFlags, range: TextRange, + runtime: Option<&dyn crate::ParseRuntime>, ) -> Result { - StringParser::new(source, flags, range.start(), range).parse_interpolated_string_middle() + StringParser::new(source, flags, range.start(), range, runtime) + .parse_interpolated_string_middle() } #[cfg(test)] diff --git a/crates/ruff_python_parser/tests/unicode_names.rs b/crates/ruff_python_parser/tests/unicode_names.rs new file mode 100644 index 0000000000000..8c5061f61922c --- /dev/null +++ b/crates/ruff_python_parser/tests/unicode_names.rs @@ -0,0 +1,520 @@ +use std::cell::RefCell; + +use ruff_python_ast::{Expr, Mod}; +use ruff_python_parser::{ + Mode, ParseOptions, ParseRuntime, UnicodeNameProvider, UnicodeNameProviderUnavailable, + parse_unchecked_with_runtime, +}; + +#[derive(Debug)] +struct Provider<'a> { + events: &'a RefCell>, + available: bool, +} + +impl ParseRuntime for Provider<'_> {} + +impl UnicodeNameProvider for Provider<'_> { + fn prepare(&self) -> Result<(), UnicodeNameProviderUnavailable> { + self.events.borrow_mut().push("prepare".into()); + if self.available { + Ok(()) + } else { + Err(UnicodeNameProviderUnavailable) + } + } + + fn lookup(&self, name: &[u8]) -> Option { + self.events + .borrow_mut() + .push(String::from_utf8_lossy(name).into_owned()); + (name == b"CUSTOM NAME").then_some(u32::from('Z')) + } +} + +#[test] +fn named_escape_uses_borrowed_provider() { + let events = RefCell::default(); + let provider = Provider { + events: &events, + available: true, + }; + let parsed = parse_unchecked_with_runtime( + r"'\N{CUSTOM NAME}'", + ParseOptions::from(Mode::Expression), + &provider, + ); + assert!(parsed.errors().is_empty()); + let Mod::Expression(module) = parsed.syntax() else { + panic!("expected expression") + }; + let Expr::StringLiteral(literal) = module.body.as_ref() else { + panic!("expected string") + }; + assert_eq!(literal.value.to_str(), "Z"); + assert_eq!(*events.borrow(), ["prepare", "CUSTOM NAME"]); +} + +#[test] +fn only_decoded_named_escapes_prepare_provider() { + for source in [ + r"r'\N{A}'", + r"b'\N{A}'", + r"'\\N{A}'", + "# \\N{A}\npass", + r"'\u0041'", + ] { + let events = RefCell::default(); + let provider = Provider { + events: &events, + available: false, + }; + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &provider); + assert!( + parsed.errors().is_empty(), + "{source}: {:?}", + parsed.errors() + ); + assert!(events.borrow().is_empty(), "{source}"); + } +} + +#[test] +fn preparation_precedes_name_validation() { + for source in [ + r"'\N'", + r"'\N{}'", + r"'\N{missing'", + r"'\N{CUSTOM NAME}'", + r"f'\N{CUSTOM NAME}'", + r"t'\N{CUSTOM NAME}'", + ] { + let events = RefCell::default(); + let provider = Provider { + events: &events, + available: false, + }; + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &provider); + assert_eq!( + parsed.errors()[0].error.to_string(), + "(unicode error) \\N escapes not supported (can't load unicodedata module)", + "{source}" + ); + assert!(events.borrow().iter().all(|event| event == "prepare")); + assert!(!events.borrow().is_empty()); + } +} + +#[test] +fn provider_error_obeys_syntax_error_order() { + for (source, provider_first) in [(r"x = ; '\N{A}'", false), (r"'\N{A}'; x = ;", true)] { + let events = RefCell::default(); + let provider = Provider { + events: &events, + available: false, + }; + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &provider); + assert_eq!( + parsed.errors()[0] + .error + .to_string() + .contains("can't load unicodedata"), + provider_first, + "{source}" + ); + } +} + +#[test] +fn syntax_error_does_not_import_during_recovery() { + let events = RefCell::default(); + let provider = Provider { + events: &events, + available: false, + }; + let parsed = parse_unchecked_with_runtime(r"x = ; '\N{A}'", Mode::Module.into(), &provider); + assert!(!parsed.errors().is_empty()); + assert!(events.borrow().is_empty()); +} + +#[test] +fn invalid_grammar_still_prepares_named_escapes() { + for source in [ + r"(a '\N{A}')", + r"pass if '\N{A}' else 1", + r"case '\N{A}': pass", + ] { + let events = RefCell::default(); + let provider = Provider { + events: &events, + available: false, + }; + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &provider); + assert!( + !events.borrow().is_empty(), + "{source}: {:?}", + parsed.errors() + ); + assert!( + parsed.errors()[0] + .error + .to_string() + .contains("can't load unicodedata"), + "{source}: {:?}", + parsed.errors() + ); + } +} + +#[derive(Debug)] +struct WarningRuntime<'a>(Provider<'a>); + +impl UnicodeNameProvider for WarningRuntime<'_> { + fn prepare(&self) -> Result<(), UnicodeNameProviderUnavailable> { + self.0.prepare() + } + fn lookup(&self, name: &[u8]) -> Option { + self.0.lookup(name) + } +} + +impl ParseRuntime for WarningRuntime<'_> { + fn literal_decoded( + &self, + source: &str, + _flags: ruff_python_ast::AnyStringFlags, + range: ruff_text_size::TextRange, + ) -> Result<(), ruff_python_parser::ParseError> { + self.0.events.borrow_mut().push(format!("decoded:{source}")); + if source == r"\q" { + Err(ruff_python_parser::ParseError { + error: ruff_python_parser::ParseErrorType::OtherError("warning-as-error".into()), + location: range, + }) + } else { + Ok(()) + } + } +} + +#[test] +fn decoder_failure_precedes_warning_in_same_literal() { + for source in [r"'\q\N{A}'", r"f'\q\N{A}'", r"t'\q\N{A}'"] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert!( + parsed.errors()[0] + .error + .to_string() + .contains("can't load unicodedata"), + "{source}: {:?}", + parsed.errors() + ); + assert_eq!(*events.borrow(), ["prepare"]); + } +} + +#[test] +fn earlier_literal_warning_aborts_before_later_import() { + for source in [r"'\q' '\N{A}'", r"'\q'; '\N{A}'"] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert_eq!(parsed.errors()[0].error.to_string(), "warning-as-error"); + assert_eq!(*events.borrow(), ["decoded:\\q"]); + } +} + +#[test] +fn decoder_errors_survive_invalid_grammar_backtracking() { + for (source, reason) in [ + (r"(a '\N{UNKNOWN}')", "unknown Unicode character name"), + (r"(a '\N{}')", "malformed \\N"), + (r"(a '\xg')", "truncated \\x"), + ] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: true, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert!( + parsed.errors()[0].error.to_string().contains(reason), + "{source}: {:?}", + parsed.errors() + ); + } +} + +#[test] +fn invalid_interpolated_grammar_does_not_decode_outer_fragments() { + for source in [ + r"f'\N{A}{'", + r"f'\N{A}}'", + r"f'\N{A}{1+}'", + r"f'{1+}\N{A}'", + r"t'\N{A}{1+}'", + r"f'\q{1+}'", + ] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert!(!parsed.errors().is_empty()); + assert!( + events.borrow().is_empty(), + "{source}: {:?}", + events.borrow() + ); + } +} + +#[test] +fn inner_expression_decoding_precedes_outer_fragments() { + for source in [ + r#"f'{"\N{A}"}{1+}'"#, + r#"f'\N{A}{"\N{B}"}'"#, + r#"f'\q{"\N{B}"}'"#, + ] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert!( + parsed.errors()[0] + .error + .to_string() + .contains("can't load unicodedata"), + "{source}: {:?}", + parsed.errors() + ); + assert_eq!(*events.borrow(), ["prepare"]); + assert_eq!( + &source[parsed.errors()[0].location], + if source.contains('B') { + r#""\N{B}""# + } else { + r#""\N{A}""# + } + ); + } +} + +#[test] +fn tokenizer_failure_before_literal_does_not_import() { + for source in [r"$; '\N{A}'", r" '\N{A}'"] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert!(!parsed.errors().is_empty()); + assert!( + events.borrow().is_empty(), + "{source}: {:?}", + events.borrow() + ); + } +} + +#[test] +fn decoder_failure_precedes_later_tokenizer_failure() { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(r"'\N{A}'; $", Mode::Module.into(), &runtime); + assert!( + parsed.errors()[0] + .error + .to_string() + .contains("can't load unicodedata") + ); + assert_eq!(*events.borrow(), ["prepare"]); +} + +#[test] +fn format_spec_decode_is_immediate_and_preserves_exception_kind() { + for source in [ + r"f'{1:\N{A}}'", + r"f'{1:\N{A}'", + r"f'{1:\N{A}}{'", + r"f'{1:\N{A}{2+}}'", + r"t'{1:\N{A}}'", + ] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert!( + matches!( + parsed.errors()[0].error, + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::FormatSpecDecodeError(_) + ) + ), + "{source}: {:?}", + parsed.errors() + ); + assert_eq!(*events.borrow(), ["prepare"]); + } +} + +#[test] +fn invalid_format_spec_expression_stops_later_import() { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(r"f'{1:{2+}\N{A}}'", Mode::Module.into(), &runtime); + assert!(!parsed.errors().is_empty()); + assert!(events.borrow().is_empty()); +} + +#[derive(Debug)] +struct BuiltinRuntime; +impl UnicodeNameProvider for BuiltinRuntime { + fn prepare(&self) -> Result<(), UnicodeNameProviderUnavailable> { + Ok(()) + } + fn lookup(&self, name: &[u8]) -> Option { + std::str::from_utf8(name) + .ok() + .and_then(unicode_names2::character) + .map(u32::from) + } +} +impl ParseRuntime for BuiltinRuntime {} + +#[test] +fn runtime_preserves_valid_ast_and_tokens() { + for source in [ + r"'\N{LATIN CAPITAL LETTER A}'", + r"f'\N{LATIN CAPITAL LETTER A}{x!r:>3}'", + r"f'left{{{x:>3}}}right'", + r"f'{x:\N{LATIN CAPITAL LETTER A}}'", + r#"f'\N{LATIN CAPITAL LETTER A}{f"nested{x}"}'"#, + r"t'left{x:\N{LATIN CAPITAL LETTER A}}right'", + r"rf'\N{{name}} {x}'", + ] { + let plain = ruff_python_parser::parse_unchecked(source, Mode::Module.into()); + let runtime = parse_unchecked_with_runtime(source, Mode::Module.into(), &BuiltinRuntime); + assert!( + runtime.errors().is_empty(), + "{source}: {:?}", + runtime.errors() + ); + assert_eq!(plain.syntax(), runtime.syntax(), "{source}"); + assert_eq!(plain.tokens(), runtime.tokens(), "{source}"); + } +} + +#[test] +fn committed_grammar_errors_do_not_decode_recovery_literals() { + for source in [r"def f(a '\N{A}'): pass", r"def '\N{A}'", r"import '\N{A}'"] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert!(!parsed.errors().is_empty()); + assert!( + events.borrow().is_empty(), + "{source}: {:?}; {:?}", + events.borrow(), + parsed.errors() + ); + } +} + +#[test] +fn expression_grammar_can_decode_before_invalid_syntax() { + for source in [ + r"a '\N{A}'", + r"f(a '\N{A}')", + r"x = 1 '\N{A}'", + r"for '\N{A}' in x: pass", + ] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert!( + parsed.errors()[0] + .error + .to_string() + .contains("can't load unicodedata"), + "{source}: {:?}", + parsed.errors() + ); + assert_eq!(*events.borrow(), ["prepare"]); + } +} + +#[test] +fn invalid_expression_probe_is_bounded_by_grammar() { + for (source, imports) in [ + (r"(a b '\N{A}')", false), + (r"a b '\N{A}'", true), + (r"def f(a=1 '\N{A}'): pass", true), + (r"def f(a: int '\N{A}'): pass", true), + ] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert_eq!( + events.borrow().contains(&"prepare".to_owned()), + imports, + "{source}: {:?}", + parsed.errors() + ); + } +} + +#[test] +fn legacy_probe_evaluates_star_expression_list() { + for (source, imports) in [ + (r"a 1, '\N{A}'", true), + (r"a 1, *x, '\N{A}'", true), + (r"print 1, *x, '\N{A}'", true), + (r"(a 1, *x, '\N{A}')", false), + (r"print 1, '\N{A}'", true), + (r"(print 1, '\N{A}')", true), + (r"(a 1, '\N{A}')", false), + ] { + let events = RefCell::default(); + let runtime = WarningRuntime(Provider { + events: &events, + available: false, + }); + let parsed = parse_unchecked_with_runtime(source, Mode::Module.into(), &runtime); + assert_eq!( + events.borrow().contains(&"prepare".to_owned()), + imports, + "{source}: {:?}", + parsed.errors() + ); + } +}