Add per-parse runtime hooks for Unicode names and literal diagnostics

crates/ruff_python_parser/src/error.rs

@@ -823,6 +823,10 @@ pub enum LexicalErrorType {
         escaped_end_quote: bool,
         detected_line: u32,
     },
+    /// The interpreter could not load the Unicode name provider for a `\N` escape.
+    UnicodeNameProviderUnavailable,
+    /// A format-spec decoder failure propagates its original exception, not `SyntaxError`.
+    FormatSpecDecodeError(Box<LexicalErrorType>),
     /// An escape sequence in a string literal cannot be decoded. `start` and `end` are the
     /// inclusive positions of the escape sequence in the string content, where a non-ASCII
     /// character counts as its ten-character `\U` escape.
@@ -949,6 +953,8 @@ impl LexicalErrorType {
     pub fn is_tokenizer_error(&self) -> bool {
         match self {
             Self::StringError
+            | Self::FormatSpecDecodeError(_)
+            | Self::UnicodeNameProviderUnavailable
             | Self::UnicodeEscapeError { .. }
             | Self::BytesEscapeError { .. }
             | Self::InvalidByteLiteral
@@ -1006,6 +1012,8 @@ impl std::fmt::Display for LexicalErrorType {
             Self::InvalidByteLiteral => {
                 write!(f, "bytes can only contain ASCII literal characters")
             }
+            Self::FormatSpecDecodeError(error) => std::fmt::Display::fmt(error, f),
+            Self::UnicodeNameProviderUnavailable => write!(f, "(unicode error) {}", crate::UnicodeNameProviderUnavailable),
             Self::UnicodeEscapeError { kind, start, end } => write!(
                 f,
                 "(unicode error) 'unicodeescape' codec can't decode bytes in position {start}-{end}: {kind}"

crates/ruff_python_parser/src/lib.rs

@@ -70,6 +70,51 @@ pub use crate::error::{
 };
 pub use crate::parser::ParseOptions;
 
+/// A per-parse Unicode character-name provider for embedded interpreters.
+///
+/// Preparation runs when a non-raw, non-bytes literal actually decodes `\N`, before
+/// validating its braces or name. Implementations should cache preparation for this parse,
+/// including failure: parser recovery may decode a literal more than once. No provider is retained after parsing.
+/// Names are passed as UTF-8 bytes, without case folding or normalization.
+/// Like numeric escapes, surrogate code points are represented by the replacement character
+/// in the scalar-only AST; this interface does not add surrogate-preserving strings.
+pub trait UnicodeNameProvider: std::fmt::Debug {
+    /// Load the database, or report the interpreter's Unicode name import failure.
+    fn prepare(&self) -> Result<(), UnicodeNameProviderUnavailable>;
+    /// Resolve one character name after preparation, excluding named sequences.
+    fn lookup(&self, name: &[u8]) -> Option<u32>;
+}
+
+/// Interpreter effects executed at string-decoding boundaries.
+///
+/// A runtime is borrowed only for this parse. Diagnostic reparses should share the same
+/// session cache so that module preparation and emitted warnings are not repeated.
+pub trait ParseRuntime: UnicodeNameProvider {
+    /// Called after a literal (or f/t-string literal fragment) has decoded successfully.
+    /// `source` is the undecoded content of that literal, without its quote delimiters.
+    /// An error aborts interpreter effects and survives grammar backtracking.
+    fn literal_decoded(
+        &self,
+        _source: &str,
+        _flags: ruff_python_ast::AnyStringFlags,
+        _range: ruff_text_size::TextRange,
+    ) -> Result<(), ParseError> {
+        Ok(())
+    }
+}
+
+/// The interpreter could not load its Unicode character-name database.
+#[derive(Debug, Clone, Copy)]
+pub struct UnicodeNameProviderUnavailable;
+
+impl std::fmt::Display for UnicodeNameProviderUnavailable {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("\\N escapes not supported (can't load unicodedata module)")
+    }
+}
+
+impl std::error::Error for UnicodeNameProviderUnavailable {}
+
 use crate::parser::Parser;
 
 use ruff_python_ast::token::{Token, TokenFlags, TokenKind, Tokens};
@@ -291,6 +336,17 @@ pub fn parse_unchecked(source: &str, options: ParseOptions) -> Parsed<Mod> {
     Parser::new(source, options).parse()
 }
 
+/// Parse with interpreter-owned Unicode names and literal-warning dispatch.
+/// Decoder and runtime callback failures abort interpreter effects even when grammar
+/// diagnostics rewind. Existing parsing APIs use the built-in database without callbacks.
+pub fn parse_unchecked_with_runtime(
+    source: &str,
+    options: ParseOptions,
+    provider: &dyn ParseRuntime,
+) -> Parsed<Mod> {
+    Parser::new(source, options).with_runtime(provider).parse()
+}
+
 /// Parse the given Python source code using the specified [`PySourceType`].
 pub fn parse_unchecked_source(source: &str, source_type: PySourceType) -> Parsed<ModModule> {
     // SAFETY: Safe because `PySourceType` always parses to a `ModModule`

crates/ruff_python_parser/src/parser/expression.rs

@@ -13,7 +13,8 @@ use ruff_python_ast::{
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 
 use crate::error::{
-    ComprehensionUnpackingKind, FStringKind, StarTupleKind, UnparenthesizedNamedExprKind,
+    ComprehensionUnpackingKind, FStringKind, LexicalError, LexicalErrorType, StarTupleKind,
+    UnparenthesizedNamedExprKind,
 };
 use crate::parser::progress::ParserProgress;
 use crate::parser::{FunctionKind, IpyEscapeContext, Parser, helpers};
@@ -416,6 +417,32 @@ impl<'src> Parser<'src> {
         {
             return;
         }
+        if self
+            .runtime_for_literal(self.current_token_range())
+            .is_some()
+        {
+            // invalid_expression uses expression_without_invalid on its right-hand side.
+            // Do not recursively run invalid-expression actions before committing its error.
+            let checkpoint = self.checkpoint();
+            self.parse_conditional_expression_without_comma_check(ExpressionContext::default());
+            self.rewind(checkpoint);
+
+            // If the missing-comma action declines to raise (outside brackets, or NAME
+            // STRING), CPython next tries invalid_legacy_expression: NAME !'(' star_expressions.
+            // That rule evaluates its operand even for names other than print and exec.
+            if let Expr::Name(name) = &first.expr
+                && !first.is_parenthesized
+                && !self.at(TokenKind::Lpar)
+                && (self.bracket_level() == 0
+                    || self.at(TokenKind::String)
+                    || matches!(name.id.as_str(), "print" | "exec"))
+                && self.decoder_abort.is_none()
+            {
+                let checkpoint = self.checkpoint();
+                self.parse_expression_list(ExpressionContext::starred_bitwise_or());
+                self.rewind(checkpoint);
+            }
+        }
         if let Expr::Name(name) = &first.expr
             && !first.is_parenthesized
         {
@@ -444,7 +471,11 @@ impl<'src> Parser<'src> {
         // [a b +]
         let checkpoint = self.checkpoint();
         let errors_before = self.errors.len();
-        let second = self.parse_conditional_expression_or_higher();
+        let second = if self.runtime.is_some() {
+            self.parse_conditional_expression_without_comma_check(ExpressionContext::default())
+        } else {
+            self.parse_conditional_expression_or_higher()
+        };
         let second_end = self.complete_prefix_end(&second.expr, errors_before);
         self.rewind(checkpoint);
         let Some(second_end) = second_end else {
@@ -2020,9 +2051,13 @@ impl<'src> Parser<'src> {
 
         let value = self.bump_string_value();
 
-        match parse_string_literal(value, flags, range) {
-            Ok(string) => string,
+        match parse_string_literal(value, flags, range, self.runtime_for_literal(range)) {
+            Ok(string) => {
+                self.literal_decoded(value, flags, range);
+                string
+            }
             Err(error) => {
+                self.record_decoder_abort(&error);
                 let location = error.location();
                 self.add_error(ParseErrorType::Lexical(error.into_error()), location);
 
@@ -2068,11 +2103,12 @@ impl<'src> Parser<'src> {
         kind: InterpolatedStringKind,
     ) -> InterpolatedStringData {
         let start = self.node_start();
+        let errors_before = self.errors.len();
         let mut flags = self.tokens.current_flags().as_any_string_flags();
 
         self.bump(kind.start_token());
         let outer_escape_error = self.interpolated_string_escape_error.take();
-        let elements = self.parse_interpolated_string_elements(
+        let mut elements = self.parse_interpolated_string_elements(
             flags,
             InterpolatedStringElementsKind::Regular(kind),
             kind,
@@ -2082,6 +2118,12 @@ impl<'src> Parser<'src> {
         if !self.expect(kind.end_token()) {
             flags = flags.with_unclosed(true);
         }
+        if self.runtime.is_some()
+            && self.errors.len() == errors_before
+            && !self.tokens.has_tokenizer_error_in(self.node_range(start))
+        {
+            self.decode_interpolated_literals(&mut elements, flags);
+        }
         // test_err interpolated_string_escape_error
         // f'a\N b{x}'
         // t'{x}\x1' 'c'
@@ -2099,6 +2141,35 @@ impl<'src> Parser<'src> {
         }
     }
 
+    fn decode_interpolated_literals(
+        &mut self,
+        elements: &mut ast::InterpolatedStringElements,
+        flags: ast::AnyStringFlags,
+    ) {
+        for element in elements {
+            if let InterpolatedStringElement::Literal(literal) = element {
+                let range = literal.range;
+                let source = self.src_text(range);
+                match parse_interpolated_string_literal_element(
+                    source,
+                    flags,
+                    range,
+                    self.runtime_for_literal(range),
+                ) {
+                    Ok(decoded) => {
+                        self.literal_decoded(source, flags, range);
+                        *literal = decoded;
+                    }
+                    Err(error) => {
+                        self.record_decoder_abort(&error);
+                        self.interpolated_string_escape_error
+                            .get_or_insert(error.into_error());
+                    }
+                }
+            }
+        }
+    }
+
     /// Check `range` for comment tokens, report an `UnsupportedSyntaxError` for each one found,
     /// and return whether any comments were found.
     fn check_fstring_comments(&mut self, range: TextRange) -> bool {
@@ -2148,9 +2219,45 @@ impl<'src> Parser<'src> {
                         let range = parser.current_token_range();
                         let value = parser.current_token_text();
                         parser.bump(middle_token_kind);
-                        InterpolatedStringElement::Literal(
-                            parse_interpolated_string_literal_element(value, flags, range)
+                        if parser.runtime.is_some()
+                            && matches!(elements_kind, InterpolatedStringElementsKind::Regular(_))
+                        {
+                            // The joined-string semantic action runs only after its grammar
+                            // succeeds. Expression literals are evaluated independently.
+                            InterpolatedStringElement::Literal(
+                                ast::InterpolatedStringLiteralElement {
+                                    value: "".into(),
+                                    range,
+                                    node_index: AtomicNodeIndex::NONE,
+                                },
+                            )
+                        } else {
+                            InterpolatedStringElement::Literal(
+                                parse_interpolated_string_literal_element(
+                                    value,
+                                    flags,
+                                    range,
+                                    parser.runtime_for_literal(range),
+                                )
+                                .inspect(|_| {
+                                    parser.literal_decoded(value, flags, range);
+                                })
                                 .unwrap_or_else(|lex_error| {
+                                    let lex_error = if parser.runtime_for_literal(range).is_some()
+                                        && matches!(
+                                            elements_kind,
+                                            InterpolatedStringElementsKind::FormatSpec(_)
+                                        ) {
+                                        LexicalError::new(
+                                            LexicalErrorType::FormatSpecDecodeError(Box::new(
+                                                lex_error.into_error(),
+                                            )),
+                                            range,
+                                        )
+                                    } else {
+                                        lex_error
+                                    };
+                                    parser.record_decoder_abort(&lex_error);
                                     // test_err invalid_fstring_literal_element
                                     // f'hello \N{INVALID} world'
                                     // f"""hello \N{INVALID} world"""
@@ -2163,7 +2270,8 @@ impl<'src> Parser<'src> {
                                         node_index: AtomicNodeIndex::NONE,
                                     }
                                 }),
-                        )
+                            )
+                        }
                     }
                     // `Invalid` tokens are created when there's a lexical error, so
                     // we ignore it here to avoid creating unexpected token errors

crates/ruff_python_parser/src/parser/mod.rs

@@ -64,6 +64,9 @@ impl NameInterner {
 #[derive(Debug)]
 pub(crate) struct Parser<'src> {
     source: &'src str,
+    runtime: Option<&'src dyn crate::ParseRuntime>,
+    /// Interpreter decoder failures cannot be undone by grammar backtracking.
+    decoder_abort: Option<ParseError>,
 
     /// Token source for the parser that skips over any non-trivia token.
     tokens: TokenSource<'src>,
@@ -162,6 +165,8 @@ impl<'src> Parser<'src> {
         Parser {
             options,
             source,
+            runtime: None,
+            decoder_abort: None,
             errors: Vec::new(),
             deferred_starred_unpacking_error: None,
             unsupported_syntax_errors: Vec::new(),
@@ -186,6 +191,49 @@ impl<'src> Parser<'src> {
         }
     }
 
+    /// Recovery after a committed syntax error must not load interpreter modules while
+    /// decoding later literals. The built-in database remains available for recovery ASTs.
+    fn runtime_for_literal(&self, range: TextRange) -> Option<&'src dyn crate::ParseRuntime> {
+        let runtime = self.runtime?;
+        if self.decoder_abort.is_none()
+            && !self
+                .tokens
+                .has_tokenizer_error_in(TextRange::new(self.start_offset, range.start()))
+            && self.errors.is_empty()
+        {
+            Some(runtime)
+        } else {
+            None
+        }
+    }
+
+    fn literal_decoded(
+        &mut self,
+        source: &str,
+        flags: ruff_python_ast::AnyStringFlags,
+        range: TextRange,
+    ) {
+        if let Some(runtime) = self.runtime_for_literal(range)
+            && let Err(error) = runtime.literal_decoded(source, flags, range)
+        {
+            self.decoder_abort.get_or_insert(error);
+        }
+    }
+
+    fn record_decoder_abort(&mut self, error: &crate::error::LexicalError) {
+        if self.runtime_for_literal(error.location()).is_some() {
+            self.decoder_abort.get_or_insert_with(|| ParseError {
+                error: ParseErrorType::Lexical(error.error().clone()),
+                location: error.location(),
+            });
+        }
+    }
+
+    pub(crate) fn with_runtime(mut self, provider: &'src dyn crate::ParseRuntime) -> Self {
+        self.runtime = Some(provider);
+        self
+    }
+
     /// Consumes the [`Parser`] and returns the parsed [`Parsed`].
     pub(crate) fn parse(mut self) -> Parsed<Mod> {
         let syntax = self.maybe_grow_stack(|parser| match parser.options.mode {
@@ -263,6 +311,15 @@ impl<'src> Parser<'src> {
             TokenKind::EndOfFile,
             "Parser should be at the end of the file."
         );
+        if let Some(error) = self.decoder_abort {
+            let (tokens, _) = self.tokens.finish();
+            return Parsed {
+                syntax,
+                tokens: Tokens::new(tokens),
+                errors: vec![error],
+                unsupported_syntax_errors: self.unsupported_syntax_errors,
+            };
+        }
         // TODO consider re-integrating lexical error handling into the parser?
         let mut parse_errors = self.errors;
         if !parse_errors.is_empty()