From 7b551fc2d772511cf0d13f9af9fe5585b5ad5210 Mon Sep 17 00:00:00 2001 From: Sam Gammon Date: Mon, 16 Nov 2026 12:26:20 -0800 Subject: [PATCH 1/7] Resolve variable-access names lazily from tagged bytecode Drop the per-code-unit variable-access side table added in #11. It duplicated names the code unit already holds and locations the Bytecode DSL already records: every name occurrence was copied into the code unit, marshaled (UTF-8 name plus five ints), re-interned on load, and kept on the heap whether or not a tool was attached. The ReadVariableTag/WriteVariableTag node object now resolves names on demand from the tagged region's bytecode (TagTreeNode#getBytecodeNode and the enter/return bytecode indices). The access is the last matching load or store outside nested variable tags. Its operands identify the name: frame slots map to varnames, cellvars and freevars; LocalAccessor operands map through the locals table info; cell instructions carry an index; and global/name instructions carry the name constant. Identifier locations come from the tag's own source layers. Fused two-target unpacking now gets one identifier layer per operand, so per-name ranges are preserved. Variable tags no longer overwrite the compiler's currentLocation, which is used for syntax error positions and loop-header trace lines. Bump the bytecode cache version to 37 for the marshal format change. Co-Authored-By: Claude Opus 5.5 --- docs/user/Tooling.md | 4 +- .../test/runtime/VariableAccessTests.java | 63 +++++- .../modules/MarshalModuleBuiltins.java | 18 +- .../bytecode_dsl/BytecodeDSLCompiler.java | 2 +- .../bytecode_dsl/RootNodeCompiler.java | 87 ++++---- .../bytecode_dsl/BytecodeDSLCodeUnit.java | 22 +- .../bytecode_dsl/PBytecodeDSLRootNode.java | 9 +- .../VariableAccessDescriptor.java | 196 ++++++++++++++++-- 8 files changed, 288 insertions(+), 113 deletions(-) diff --git a/docs/user/Tooling.md b/docs/user/Tooling.md index e498423ef9..ecbb141a9e 100644 --- a/docs/user/Tooling.md +++ b/docs/user/Tooling.md @@ -49,7 +49,9 @@ use Truffle's UTF-16 columns, including for non-BMP identifiers. For unexecuted source, attach a source-section load listener before calling the instrument `Env.parse(Source)` API. Retain the returned call target while collecting facts. Reading node objects and their name/location descriptors does not require an entered Python context or guest -values, and supports concurrent readers. Code-unit serialization preserves this metadata. +values, and supports concurrent readers. Names are resolved on demand from the tagged load and store +instructions and the code object's existing name tables, so code units carry no extra metadata and +unmarshaled code reports the same names and locations. This interface describes compiler-emitted accesses, rather than a complete source symbol index. It covers name expressions, assignment and augmented-assignment targets, loop targets and diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java index eda760b817..8f04d9395f 100644 --- a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java @@ -57,7 +57,6 @@ import org.graalvm.polyglot.Context; import org.junit.Test; -import com.oracle.graal.python.builtins.objects.code.PCode; import com.oracle.truffle.api.CallTarget; import com.oracle.truffle.api.instrumentation.InstrumentableNode; import com.oracle.truffle.api.instrumentation.SourceSectionFilter; @@ -169,6 +168,56 @@ def method(self, __private): } } + @Test + public void scopesAndNestedAccesses() throws IOException { + try (Context context = Context.newBuilder("python").allowAllAccess(true).build()) { + AnalysisInstrument instrument = context.getEngine().getInstruments().get(ID).lookup(AnalysisInstrument.class); + context.initialize("python"); + context.enter(); + try { + Analysis analysis = instrument.analyze(Source.newBuilder("python", """ + counter = 0 + def bump(): + global counter + counter += 1 + class Holder: + attr = counter + def method(self): + return __class__ + def outer(): + cell = 1 + def inner(): + nonlocal cell + cell += 1 + first = (second := cell) + return inner, first, second + """, "scopes.py").build()); + List accesses = analysis.accesses(); + // module and class bodies: name stores/loads + assertTrue(accesses.contains(new Access("", "counter", true, 1, 1, "counter"))); + assertTrue(accesses.stream().anyMatch(a -> a.name().equals("attr") && a.write() && a.line() == 6 && a.column() == 5)); + assertTrue(accesses.stream().anyMatch(a -> a.name().equals("counter") && !a.write() && a.line() == 6 && a.column() == 12)); + // explicit global, augmented + assertTrue(accesses.contains(new Access("bump", "counter", true, 4, 5, "counter"))); + assertTrue(accesses.contains(new Access("bump", "counter", false, 4, 5, "counter"))); + // free variable read + assertTrue(accesses.contains(new Access("method", "__class__", false, 8, 16, "__class__"))); + // cell store in the defining scope, nonlocal augmented store and load + assertTrue(accesses.contains(new Access("outer", "cell", true, 10, 5, "cell"))); + assertTrue(accesses.contains(new Access("inner", "cell", true, 13, 9, "cell"))); + assertTrue(accesses.contains(new Access("inner", "cell", false, 13, 9, "cell"))); + // a nested walrus store is its own access, not part of the enclosing store + List line14 = accesses.stream().filter(a -> a.line() == 14).toList(); + assertEquals(3, line14.size()); + assertTrue(line14.contains(new Access("outer", "first", true, 14, 5, "first"))); + assertTrue(line14.contains(new Access("outer", "second", true, 14, 14, "second"))); + assertTrue(line14.contains(new Access("outer", "cell", false, 14, 24, "cell"))); + } finally { + context.leave(); + } + } + } + @Test public void executedTagsPreservePythonBehavior() throws IOException { try (Context context = Context.newBuilder("python").allowAllAccess(true).build()) { @@ -208,7 +257,14 @@ def serialized(arg): return result """, "serialized.py").build(); Analysis analysis = instrument.analyzeSerialized(source); - assertEquals(6, analysis.accesses().stream().filter(a -> a.root().equals("serialized")).count()); + List accesses = analysis.accesses().stream().filter(a -> a.root().equals("serialized")).toList(); + assertEquals(6, accesses.size()); + // The original and the unmarshaled code report the same names and locations, which + // are derived from the serialized bytecode and its source information. + for (Access access : accesses) { + assertEquals(2, accesses.stream().filter(a -> a.name().equals(access.name()) && a.write() == access.write() && a.line() == access.line() && + a.column() == access.column()).count()); + } } finally { context.leave(); } @@ -309,9 +365,6 @@ private Analysis analyze(Source source, boolean execute, boolean serialized) thr cloned = types.FunctionType(marshal.loads(marshal.dumps(serialized.__code__)), globals()) assert cloned(42) == 42 """, "roundtrip.py").build()).call(); - PCode original = (PCode) env.parse(Source.newBuilder("python", "serialized.__code__", "original-code.py").build()).call(); - PCode cloned = (PCode) env.parse(Source.newBuilder("python", "cloned.__code__", "cloned-code.py").build()).call(); - assertEquals(original.getCodeUnit().variableAccesses, cloned.getCodeUnit().variableAccesses); } } finally { binding.dispose(); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java index a76a352a27..14b8795283 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java @@ -115,7 +115,6 @@ import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.StringLiterals; import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit; -import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit.VariableAccess; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNodeGen; import com.oracle.graal.python.nodes.call.special.LookupAndCallBinaryNode; @@ -1429,16 +1428,10 @@ private BytecodeDSLCodeUnit readBytecodeDSLCodeUnit() { int yieldFromGeneratorIndex = readInt(); int instrumentationDataIndex = readInt(); int maxProfileCEventStackSize = readInt(); - int accessCount = readSize(); - List variableAccesses = new ArrayList<>(accessCount); - for (int i = 0; i < accessCount; i++) { - variableAccesses.add(new VariableAccess(readString(true).toJavaStringUncached(), readInt(), readInt(), readInt(), readInt(), readInt() != 0)); - } BytecodeSupplier provider = new BytecodeSupplier(serialized, bytecodeFile, sourceReference, bytecodeOffset, bytecodeSize, cacheKey); return new BytecodeDSLCodeUnit(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, constants, - startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, variableAccesses, - provider); + startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, provider); } private void writeCodeUnit(BytecodeDSLCodeUnit code) throws IOException { @@ -1474,15 +1467,6 @@ private void writeBytecodeDSLCodeUnit(BytecodeDSLCodeUnit code) throws IOExcepti writeInt(code.yieldFromGeneratorIndex); writeInt(code.instrumentationDataIndex); writeInt(code.maxProfileCEventStackSize); - writeInt(code.variableAccesses.size()); - for (VariableAccess access : code.variableAccesses) { - writeString(PythonUtils.toTruffleStringUncached(access.name())); - writeInt(access.startLine()); - writeInt(access.startColumn()); - writeInt(access.endLine()); - writeInt(access.endColumn()); - writeInt(access.write() ? 1 : 0); - } } private PCode readCode() { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java index 407f553b30..47c9be0845 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java @@ -71,7 +71,7 @@ public class BytecodeDSLCompiler { * Bump this with every Truffle import change because serialized GraalPy bytecode effectively * depends on Truffle bytecode internals. */ - public static final int BYTECODE_VERSION = 36; + public static final int BYTECODE_VERSION = 37; public static final record BytecodeDSLCompilerResult(PBytecodeDSLRootNode rootNode, BytecodeDSLCodeUnit codeUnit) { } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java index 8b42358512..ebee32806e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java @@ -82,7 +82,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; @@ -107,7 +106,6 @@ import com.oracle.graal.python.lib.PyObjectRichCompareBool; import com.oracle.graal.python.nodes.StringLiterals; import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit; -import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit.VariableAccess; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNodeGen; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNodeGen.Builder; @@ -217,7 +215,6 @@ public final class RootNodeCompiler implements BaseBytecodeDSLVisitor variableAccesses = new LinkedHashSet<>(); private final HashMap varnames; private final HashMap cellvars; private final HashMap freevars; @@ -595,7 +592,6 @@ flags, orderedTruffleStringArray(names), yieldFromGenerator != null ? yieldFromGenerator.getLocalIndex() : -1, instrumentationDataLocal.getLocalIndex(), maxProfileCEventStackSize, - List.copyOf(variableAccesses), new BytecodeSupplier(nodes)); ctx.codeUnits.put(key, codeUnit); } @@ -1911,51 +1907,66 @@ private SourceRange variableSourceRange(SourceRange range) { } } - private boolean recordVariableAccess(ExprTy.Name node, boolean write) { - SourceRange range = variableSourceRange(node); - if (node.id.startsWith(".") || range == SourceRange.ARTIFICIAL_RANGE || range.startLine < 1 || range.startColumn < 0 || - range.endLine < range.startLine || range.endColumn < 1 || !ctx.source.hasCharacters()) { - return false; + /** + * Returns the identifier range of a source-level variable access in Truffle (UTF-16) columns, + * or {@code null} if the access has no usable source token, e.g. compiler-generated names and + * AST-created code without valid positions. Such accesses are not tagged. + */ + private SourceRange variableAccessRange(ExprTy.Name node) { + if (node.id.startsWith(".")) { + return null; } - try { - ctx.source.createSection(range.startLine, range.startColumn + 1, range.endLine, range.endColumn); - } catch (IllegalArgumentException e) { - // AST-created code and incomplete source ranges need not identify a source token. - return false; + SourceRange range = variableSourceRange(node); + if (range == SourceRange.ARTIFICIAL_RANGE || range.startLine < 1 || range.startColumn < 0 || range.endLine < range.startLine || + range.endLine == range.startLine && range.endColumn <= range.startColumn) { + return null; } - variableAccesses.add(new VariableAccess(maybeMangle(node.id), range.startLine, range.startColumn, range.endLine, range.endColumn, write)); - return true; + return range; } private boolean beginVariableAccess(ExprTy.Name node, boolean write, Builder b) { - if (!recordVariableAccess(node, write)) { + SourceRange range = variableAccessRange(node); + if (range == null) { return false; } - beginVariableTag(variableSourceRange(node), write, b); + beginVariableTag(write, b, range); return true; } private boolean beginDirectVariableWrite(ExprTy target, BytecodeLocal local, Builder b) { - if (!isTemporaryLocal(local) && target instanceof ExprTy.Name name && recordVariableAccess(name, true)) { - beginVariableTag(variableSourceRange(target), true, b); - return true; + if (!isTemporaryLocal(local) && target instanceof ExprTy.Name name) { + return beginVariableAccess(name, true, b); } return false; } - private void beginVariableTag(SourceRange accessRange, boolean write, Builder b) { - beginSourceSection(accessRange, b); - // Bytecode DSL stack walks select the innermost tag. Keep its program location at the - // statement (or expression root), while the outer source layer identifies the access. + /** + * Opens a variable tag. The names are not recorded here: instrumentation resolves them lazily + * from the operands of the tagged instructions (see {@code VariableAccessDescriptor}). The + * source layers are, from most to least concrete: the enclosing statement (or expression + * root), then one identifier range per variable operand of the tagged store or load. + *

+ * Bytecode DSL stack walks select the innermost tag, so its program location stays at the + * statement to keep debugger caller locations stable; the identifier layers locate the access. + */ + private void beginVariableTag(boolean write, Builder b, SourceRange... accessRanges) { + for (int i = accessRanges.length - 1; i >= 0; i--) { + beginSourceSectionInner(b, accessRanges[i]); + } SourceRange statement = statementLocations.peek(); - beginSourceSection(variableSourceRange(statement != null ? statement : getRootSourceRange(startNode)), b); + beginSourceSectionInner(b, variableSourceRange(statement != null ? statement : getRootSourceRange(startNode))); b.beginTag(write ? WriteVariableTag.class : ReadVariableTag.class); } - private void endVariableTag(boolean write, Builder b) { + private void endVariableTag(boolean write, int accessCount, Builder b) { b.endTag(write ? WriteVariableTag.class : ReadVariableTag.class); - endSourceSection(b); - endSourceSection(b); + for (int i = 0; i <= accessCount; i++) { + endSourceSection(b); + } + } + + private void endVariableTag(boolean write, Builder b) { + endVariableTag(write, 1, b); } private void endDirectVariableWrite(boolean tagged, Builder b) { @@ -3866,21 +3877,19 @@ private void visitIterableAssign(ExprTy[] nodes) { if (nodes.length == 2 && containsNoStarred(nodes)) { BytecodeLocal target1 = allowFastLocalLookup ? beginTemporaryLocalOrGetLocal(nodes[0], b) : beginTemporaryLocal(); BytecodeLocal target2 = allowFastLocalLookup ? beginTemporaryLocalOrGetLocal(nodes[1], b) : beginTemporaryLocal(); - boolean tagged = false; - if (!isTemporaryLocal(target1)) { - tagged |= recordVariableAccess((ExprTy.Name) nodes[0], true); - } - if (!isTemporaryLocal(target2)) { - tagged |= recordVariableAccess((ExprTy.Name) nodes[1], true); - } + // One instruction stores both targets; give each operand its own identifier + // layer. Temporary targets are tagged when they are stored to the real target. + SourceRange range1 = isTemporaryLocal(target1) ? null : variableAccessRange((ExprTy.Name) nodes[0]); + SourceRange range2 = isTemporaryLocal(target2) ? null : variableAccessRange((ExprTy.Name) nodes[1]); + boolean tagged = range1 != null || range2 != null; if (tagged) { - beginVariableTag(variableSourceRange(nodes[0]).withEnd(variableSourceRange(nodes[1])), true, b); + beginVariableTag(true, b, range1 != null ? range1 : SourceRange.ARTIFICIAL_RANGE, range2 != null ? range2 : SourceRange.ARTIFICIAL_RANGE); } b.beginUnpackToLocals2(target1, target2); generateValue.run(); b.endUnpackToLocals2(); if (tagged) { - endVariableTag(true, b); + endVariableTag(true, 2, b); } storeTemporaryLocalToTarget(target1, nodes[0], b, allowFastLocalLookup); storeTemporaryLocalToTarget(target2, nodes[1], b, allowFastLocalLookup); @@ -4040,7 +4049,7 @@ public Void visit(ExprTy.Name node) { beginStoreLocal(node.id, b); beginAugAssign(); if (tagged) { - recordVariableAccess(node, false); + // Shares the enclosing write tag's statement and identifier layers. b.beginTag(ReadVariableTag.class); } emitReadLocal(node.id, b); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java index e5dd910c2d..fc8c32e0ea 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java @@ -78,21 +78,6 @@ public final class BytecodeDSLCodeUnit { @CompilationFinal(dimensions = 1) public final TruffleString[] freevars; @CompilationFinal(dimensions = 1) public final Object[] constants; - /** Source-level variable accesses, independent of bytecode indices and live frames. */ - public final List variableAccesses; - - public record VariableAccess(String name, int startLine, int startColumn, int endLine, int endColumn, boolean write) { - public boolean isWithin(SourceSection section) { - return section.isAvailable() && - (startLine > section.getStartLine() || startLine == section.getStartLine() && startColumn + 1 >= section.getStartColumn()) && - (endLine < section.getEndLine() || endLine == section.getEndLine() && endColumn <= section.getEndColumn()); - } - - public SourceSection getSourceSection(Source source) { - return source.createSection(startLine, startColumn + 1, endLine, endColumn); - } - } - public final int startLine; public final int startColumn; public final int endLine; @@ -107,8 +92,7 @@ public SourceSection getSourceSection(Source source) { public BytecodeDSLCodeUnit(TruffleString name, TruffleString qualname, int argCount, int kwOnlyArgCount, int positionalOnlyArgCount, int flags, TruffleString[] names, TruffleString[] varnames, TruffleString[] cellvars, TruffleString[] freevars, Object[] constants, int startLine, int startColumn, int endLine, int endColumn, - int classcellIndex, int selfIndex, int yieldFromGeneratorIndex, int instrumentationDataIndex, int maxProfileCEventStackSize, List variableAccesses, - BytecodeSupplier supplier) { + int classcellIndex, int selfIndex, int yieldFromGeneratorIndex, int instrumentationDataIndex, int maxProfileCEventStackSize, BytecodeSupplier supplier) { assert isInterned(name); this.name = name; assert qualname == null || isInterned(qualname); @@ -136,7 +120,6 @@ public BytecodeDSLCodeUnit(TruffleString name, TruffleString qualname, int argCo this.yieldFromGeneratorIndex = yieldFromGeneratorIndex; this.instrumentationDataIndex = instrumentationDataIndex; this.maxProfileCEventStackSize = maxProfileCEventStackSize; - this.variableAccesses = List.copyOf(variableAccesses); } public abstract static class BytecodeSupplier { @@ -148,8 +131,7 @@ public abstract static class BytecodeSupplier { public BytecodeDSLCodeUnit withFlags(int flags) { return new BytecodeDSLCodeUnit(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, constants, - startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, variableAccesses, - supplier); + startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, supplier); } @TruffleBoundary diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java index a820013afe..c24c21bddb 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java @@ -288,8 +288,6 @@ import com.oracle.truffle.api.bytecode.BytecodeLocation; import com.oracle.truffle.api.bytecode.BytecodeNode; import com.oracle.truffle.api.bytecode.TagTreeNode; -import com.oracle.truffle.api.instrumentation.StandardTags.ReadVariableTag; -import com.oracle.truffle.api.instrumentation.StandardTags.WriteVariableTag; import com.oracle.truffle.api.bytecode.BytecodeRootNode; import com.oracle.truffle.api.bytecode.ConstantOperand; import com.oracle.truffle.api.bytecode.ContinuationResult; @@ -504,12 +502,7 @@ public Object getInstrumentationNodeObject(TagTreeNode node) { if (co == null) { return null; } - if (node.hasTag(ReadVariableTag.class)) { - return VariableAccessDescriptor.create(ReadVariableTag.NAME, false, node.getSourceSections(), co.variableAccesses); - } else if (node.hasTag(WriteVariableTag.class)) { - return VariableAccessDescriptor.create(WriteVariableTag.NAME, true, node.getSourceSections(), co.variableAccesses); - } - return null; + return VariableAccessDescriptor.create(node, co); } public final PythonLanguage getLanguage() { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java index 780a4561d0..53593d9e76 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java @@ -44,8 +44,16 @@ import java.util.ArrayList; import java.util.List; -import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit.VariableAccess; +import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.bytecode.BytecodeNode; +import com.oracle.truffle.api.bytecode.Instruction; +import com.oracle.truffle.api.bytecode.Instruction.Argument; +import com.oracle.truffle.api.bytecode.LocalAccessor; +import com.oracle.truffle.api.bytecode.TagTree; +import com.oracle.truffle.api.bytecode.TagTreeNode; +import com.oracle.truffle.api.instrumentation.StandardTags.ReadVariableTag; +import com.oracle.truffle.api.instrumentation.StandardTags.WriteVariableTag; import com.oracle.truffle.api.interop.InteropLibrary; import com.oracle.truffle.api.interop.InvalidArrayIndexException; import com.oracle.truffle.api.interop.TruffleObject; @@ -54,8 +62,18 @@ import com.oracle.truffle.api.library.ExportLibrary; import com.oracle.truffle.api.library.ExportMessage; import com.oracle.truffle.api.source.SourceSection; +import com.oracle.truffle.api.strings.TruffleString; -/** Immutable compiler metadata; safe to query without a language context or the GIL. */ +/** + * Node object of a {@link ReadVariableTag} or {@link WriteVariableTag} region. Nothing is recorded + * for it at compile time: the names are resolved on demand from the operands of the load or store + * instruction in the tag's bytecode range, against the names the code unit already holds + * ({@code varnames}, {@code cellvars}, {@code freevars} and name constants). The identifier + * locations are the tag's source layers emitted by the compiler: the enclosing statement first, + * then one identifier range per variable operand. + *

+ * Instances are immutable and safe to create and query without a language context or the GIL. + */ @ExportLibrary(InteropLibrary.class) public final class VariableAccessDescriptor implements TruffleObject { private final String property; @@ -67,38 +85,172 @@ private VariableAccessDescriptor(String property, Object names) { } @TruffleBoundary - public static Object create(String property, boolean write, SourceSection[] sections, List accesses) { - SourceSection section = null; - for (SourceSection candidate : sections) { - if (candidate == null || !candidate.isAvailable()) { + public static Object create(TagTreeNode node, BytecodeDSLCodeUnit code) { + boolean write; + if (node.hasTag(WriteVariableTag.class)) { + write = true; + } else if (node.hasTag(ReadVariableTag.class)) { + write = false; + } else { + return null; + } + BytecodeNode bytecode = node.getBytecodeNode(); + if (bytecode == null) { + return null; + } + List nested = new ArrayList<>(); + collectNestedVariableTags(node, nested); + + // Operands are emitted before the operation that consumes them, so the access is the last + // matching instruction of this region outside nested variable tags. Cell stores receive + // the cell from the region's first local load instead of an index operand. + Instruction access = null; + Instruction firstLocalLoad = null; + int returnIndex = node.getReturnBytecodeIndex(); + for (int bci = node.getEnterBytecodeIndex(); bci <= returnIndex;) { + TagTree skip = enclosing(nested, bci); + if (skip != null) { + bci = bytecode.getInstruction(skip.getReturnBytecodeIndex()).getNextBytecodeIndex(); continue; } - for (VariableAccess access : accesses) { - if (access.write() == write && access.isWithin(candidate) && (section == null || isWithin(candidate, section))) { - section = candidate; - break; - } + Instruction instruction = bytecode.getInstruction(bci); + String name = baseName(instruction); + if (firstLocalLoad == null && name.equals("load.local")) { + firstLocalLoad = instruction; + } + if (isAccess(name, write)) { + access = instruction; + } + bci = instruction.getNextBytecodeIndex(); + } + if (access == null) { + return null; + } + + String[] accessNames = resolveNames(code, bytecode, access, baseName(access), firstLocalLoad); + SourceSection[] sections = node.getSourceSections(); + List result = new ArrayList<>(accessNames.length); + for (int i = 0; i < accessNames.length; i++) { + if (accessNames[i] != null) { + // Layer 0 is the statement; identifier layers follow in operand order. + SourceSection section = sections != null && i + 1 < sections.length ? sections[i + 1] : null; + result.add(new VariableName(accessNames[i], section)); } } - if (section == null) { + if (result.isEmpty()) { return null; } - List names = new ArrayList<>(); - for (VariableAccess access : accesses) { - if (access.write() == write && access.isWithin(section)) { - names.add(new VariableName(access.name(), access.getSourceSection(section.getSource()))); + Object value = result.size() == 1 ? result.get(0) : new NameArray(result.toArray()); + return new VariableAccessDescriptor(write ? WriteVariableTag.NAME : ReadVariableTag.NAME, value); + } + + private static void collectNestedVariableTags(TagTree tree, List result) { + for (TagTree child : tree.getTreeChildren()) { + if (child.hasTag(ReadVariableTag.class) || child.hasTag(WriteVariableTag.class)) { + result.add(child); + } else { + collectNestedVariableTags(child, result); + } + } + } + + private static TagTree enclosing(List trees, int bci) { + for (TagTree tree : trees) { + if (bci >= tree.getEnterBytecodeIndex() && bci <= tree.getReturnBytecodeIndex()) { + return tree; } } - if (names.isEmpty()) { + return null; + } + + /** The instruction name without quickening or boxing-elimination suffixes. */ + private static String baseName(Instruction instruction) { + String name = instruction.getName(); + int quickening = name.indexOf('$'); + return quickening < 0 ? name : name.substring(0, quickening); + } + + private static boolean isAccess(String name, boolean write) { + if (write) { + return switch (name) { + case "store.local", "c.StoreCell", "c.WriteGlobal", "c.WriteName", "c.ForIterate", "c.UnpackToLocals2" -> true; + default -> false; + }; + } else { + return switch (name) { + case "load.local", "c.LoadCell", "c.LoadFromDictOrCell", "c.ReadGlobal", "c.LoadFromDictOrGlobals", "c.ReadName" -> true; + default -> false; + }; + } + } + + /** One entry per variable operand of the access; {@code null} for compiler temporaries. */ + private static String[] resolveNames(BytecodeDSLCodeUnit code, BytecodeNode bytecode, Instruction access, String name, Instruction firstLocalLoad) { + switch (name) { + case "c.StoreCell": + return firstLocalLoad == null ? new String[0] : resolveNames(code, bytecode, firstLocalLoad, "load.local", null); + case "c.LoadCell": + case "c.LoadFromDictOrCell": + for (Argument argument : access.getArguments()) { + Integer index = intArgument(argument); + if (index != null) { + return new String[]{cellOrFreeName(code, index)}; + } + } + return new String[0]; + case "c.WriteGlobal": + case "c.WriteName": + case "c.ReadGlobal": + case "c.LoadFromDictOrGlobals": + case "c.ReadName": + for (Argument argument : access.getArguments()) { + if (argument.getKind() == Argument.Kind.CONSTANT && argument.asConstant() instanceof TruffleString constant) { + return new String[]{constant.toJavaStringUncached()}; + } + } + return new String[0]; + default: + // load.local and store.local encode the frame slot; c.ForIterate and + // c.UnpackToLocals2 take local accessors, whose info is the varnames index. + List names = new ArrayList<>(2); + for (Argument argument : access.getArguments()) { + if (argument.getKind() == Argument.Kind.LOCAL_OFFSET) { + names.add(slotName(code, argument.asLocalOffset())); + } else if (argument.getKind() == Argument.Kind.CONSTANT && argument.asConstant() instanceof LocalAccessor local) { + names.add(local.getLocalInfo(bytecode) instanceof Integer index && index < code.varnames.length ? code.varnames[index].toJavaStringUncached() : null); + } + } + return names.toArray(new String[0]); + } + } + + private static Integer intArgument(Argument argument) { + return switch (argument.getKind()) { + case INTEGER -> argument.asInteger(); + case CONSTANT -> argument.asConstant() instanceof Integer i ? i : null; + default -> null; + }; + } + + /** Frame slot layout: varnames, then cellvars, then freevars, then compiler temporaries. */ + private static String slotName(BytecodeDSLCodeUnit code, int slot) { + if (slot < 0) { return null; + } else if (slot < code.varnames.length) { + return code.varnames[slot].toJavaStringUncached(); } - Object value = names.size() == 1 ? names.get(0) : new NameArray(names.toArray()); - return new VariableAccessDescriptor(property, value); + return cellOrFreeName(code, slot - code.varnames.length); } - private static boolean isWithin(SourceSection candidate, SourceSection outer) { - return (candidate.getStartLine() > outer.getStartLine() || candidate.getStartLine() == outer.getStartLine() && candidate.getStartColumn() >= outer.getStartColumn()) && - (candidate.getEndLine() < outer.getEndLine() || candidate.getEndLine() == outer.getEndLine() && candidate.getEndColumn() <= outer.getEndColumn()); + private static String cellOrFreeName(BytecodeDSLCodeUnit code, int index) { + if (index < 0) { + return null; + } else if (index < code.cellvars.length) { + return code.cellvars[index].toJavaStringUncached(); + } else if (index - code.cellvars.length < code.freevars.length) { + return code.freevars[index - code.cellvars.length].toJavaStringUncached(); + } + return null; } @ExportMessage From 89991986256facfbe88159231acea96c18673c44 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Mon, 2 Nov 2026 12:01:01 +0100 Subject: [PATCH 2/7] Fix JDK 21 download URLs for github jobs (cherry picked from commit 3fa9e560b70ea1486d8ec26d34233bf3373f61e8) --- .github/scripts/extract_matrix.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/extract_matrix.py b/.github/scripts/extract_matrix.py index 00c9683c12..cd82a92ffa 100644 --- a/.github/scripts/extract_matrix.py +++ b/.github/scripts/extract_matrix.py @@ -77,7 +77,7 @@ ) DOWNLOADS_LINKS = { - "GRADLE_JAVA_HOME": "https://download.oracle.com/java/{major_version}/latest/jdk-{major_version}_{os}-{arch_short}_bin{ext}", + "GRADLE_JAVA_HOME": "https://download.oracle.com/java/{major_version}/archive/jdk-{version}_{os}-{arch_short}_bin{ext}", "ECLIPSE": "https://www.eclipse.org/downloads/download.php?file=/eclipse/downloads/drops4/R-4.26-202211231800/eclipse-SDK-4.26-linux-gtk-x86_64.tar.gz" } @@ -234,6 +234,7 @@ def get_download_link(self, key: str, version: str) -> str: vars = { "major_version": major_version, + "version": version, "os": os, "arch": arch, "arch_short": arch_short, From 34e1446d878b5c5e4983b776046d9f65bcf1a00a Mon Sep 17 00:00:00 2001 From: Sam Gammon Date: Mon, 16 Nov 2026 13:45:42 -0800 Subject: [PATCH 3/7] Identify variable accesses by operation class, not instruction names Instruction names are debugging aids and can change at any time. Use InstructionDescriptor#getOperationClass() (GR-81626, backported in elide-dev/graal#241) to recognize the custom name operations, and recognize built-in local loads/stores by their local argument. The access is still the last match outside nested variable tags: store values can contain untagged local loads, such as the self/__class__ loads that super() generates. Pin the Graal imports to the backport commit. Co-Authored-By: Claude Opus 5.5 --- .../test/runtime/VariableAccessTests.java | 7 + .../VariableAccessDescriptor.java | 134 ++++++++++-------- mx.graalpython/suite.py | 4 +- 3 files changed, 83 insertions(+), 62 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java index 8f04d9395f..43cfe519d6 100644 --- a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java @@ -191,6 +191,10 @@ def inner(): cell += 1 first = (second := cell) return inner, first, second + class Child(Holder): + def method(self): + kept = super().method + return lambda: kept """, "scopes.py").build()); List accesses = analysis.accesses(); // module and class bodies: name stores/loads @@ -212,6 +216,9 @@ def inner(): assertTrue(line14.contains(new Access("outer", "first", true, 14, 5, "first"))); assertTrue(line14.contains(new Access("outer", "second", true, 14, 14, "second"))); assertTrue(line14.contains(new Access("outer", "cell", false, 14, 24, "cell"))); + // the store value's implicit loads (__class__, self for super()) are not the access + assertEquals(List.of(new Access("method", "kept", true, 18, 9, "kept"), new Access("method", "super", false, 18, 16, "super")), + accesses.stream().filter(a -> a.line() == 18).toList()); } finally { context.leave(); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java index 53593d9e76..06d93a9cc2 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java @@ -45,6 +45,16 @@ import java.util.List; import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.ForIterate; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.LoadCell; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.LoadFromDictOrCell; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.LoadFromDictOrGlobals; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.ReadName; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.StoreCell; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.UnpackToLocals2; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.WriteGlobal; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.WriteName; +import com.oracle.graal.python.nodes.frame.ReadGlobalOrBuiltinNode; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.bytecode.BytecodeNode; import com.oracle.truffle.api.bytecode.Instruction; @@ -102,10 +112,12 @@ public static Object create(TagTreeNode node, BytecodeDSLCodeUnit code) { collectNestedVariableTags(node, nested); // Operands are emitted before the operation that consumes them, so the access is the last - // matching instruction of this region outside nested variable tags. Cell stores receive - // the cell from the region's first local load instead of an index operand. + // load or store of this region outside nested variable tags: a custom name operation, or + // a built-in instruction with a local argument (load.local/store.local). Instructions are + // identified by operation class and arguments, never by their debug names. Cell stores + // receive the cell from the region's first built-in local load. Instruction access = null; - Instruction firstLocalLoad = null; + Instruction firstLocalAccess = null; int returnIndex = node.getReturnBytecodeIndex(); for (int bci = node.getEnterBytecodeIndex(); bci <= returnIndex;) { TagTree skip = enclosing(nested, bci); @@ -114,11 +126,15 @@ public static Object create(TagTreeNode node, BytecodeDSLCodeUnit code) { continue; } Instruction instruction = bytecode.getInstruction(bci); - String name = baseName(instruction); - if (firstLocalLoad == null && name.equals("load.local")) { - firstLocalLoad = instruction; - } - if (isAccess(name, write)) { + Class operation = instruction.getDescriptor().getOperationClass(); + if (operation == null) { + if (localSlot(instruction) >= 0) { + if (firstLocalAccess == null) { + firstLocalAccess = instruction; + } + access = instruction; + } + } else if (isAccessOperation(operation, write)) { access = instruction; } bci = instruction.getNextBytecodeIndex(); @@ -127,7 +143,7 @@ public static Object create(TagTreeNode node, BytecodeDSLCodeUnit code) { return null; } - String[] accessNames = resolveNames(code, bytecode, access, baseName(access), firstLocalLoad); + String[] accessNames = resolveNames(code, bytecode, access, firstLocalAccess); SourceSection[] sections = node.getSourceSections(); List result = new ArrayList<>(accessNames.length); for (int i = 0; i < accessNames.length; i++) { @@ -163,65 +179,63 @@ private static TagTree enclosing(List trees, int bci) { return null; } - /** The instruction name without quickening or boxing-elimination suffixes. */ - private static String baseName(Instruction instruction) { - String name = instruction.getName(); - int quickening = name.indexOf('$'); - return quickening < 0 ? name : name.substring(0, quickening); - } - - private static boolean isAccess(String name, boolean write) { + private static boolean isAccessOperation(Class operation, boolean write) { if (write) { - return switch (name) { - case "store.local", "c.StoreCell", "c.WriteGlobal", "c.WriteName", "c.ForIterate", "c.UnpackToLocals2" -> true; - default -> false; - }; + return operation == StoreCell.class || operation == WriteGlobal.class || operation == WriteName.class || operation == ForIterate.class || + operation == UnpackToLocals2.class; } else { - return switch (name) { - case "load.local", "c.LoadCell", "c.LoadFromDictOrCell", "c.ReadGlobal", "c.LoadFromDictOrGlobals", "c.ReadName" -> true; - default -> false; - }; + return operation == LoadCell.class || operation == LoadFromDictOrCell.class || operation == ReadGlobalOrBuiltinNode.class || + operation == LoadFromDictOrGlobals.class || operation == ReadName.class; } } + /** The frame slot of a built-in local load or store, or -1. */ + private static int localSlot(Instruction instruction) { + for (Argument argument : instruction.getArguments()) { + // Python disables block scoping, so local offsets and local indices coincide. + switch (argument.getKind()) { + case LOCAL_OFFSET: + return argument.asLocalOffset(); + case LOCAL_INDEX: + return argument.asLocalIndex(); + default: + break; + } + } + return -1; + } + /** One entry per variable operand of the access; {@code null} for compiler temporaries. */ - private static String[] resolveNames(BytecodeDSLCodeUnit code, BytecodeNode bytecode, Instruction access, String name, Instruction firstLocalLoad) { - switch (name) { - case "c.StoreCell": - return firstLocalLoad == null ? new String[0] : resolveNames(code, bytecode, firstLocalLoad, "load.local", null); - case "c.LoadCell": - case "c.LoadFromDictOrCell": - for (Argument argument : access.getArguments()) { - Integer index = intArgument(argument); - if (index != null) { - return new String[]{cellOrFreeName(code, index)}; - } + private static String[] resolveNames(BytecodeDSLCodeUnit code, BytecodeNode bytecode, Instruction access, Instruction firstLocalAccess) { + Class operation = access.getDescriptor().getOperationClass(); + if (operation == null) { + return new String[]{slotName(code, localSlot(access))}; + } else if (operation == StoreCell.class) { + return firstLocalAccess == null ? new String[0] : new String[]{slotName(code, localSlot(firstLocalAccess))}; + } else if (operation == LoadCell.class || operation == LoadFromDictOrCell.class) { + for (Argument argument : access.getArguments()) { + Integer index = intArgument(argument); + if (index != null) { + return new String[]{cellOrFreeName(code, index)}; } - return new String[0]; - case "c.WriteGlobal": - case "c.WriteName": - case "c.ReadGlobal": - case "c.LoadFromDictOrGlobals": - case "c.ReadName": - for (Argument argument : access.getArguments()) { - if (argument.getKind() == Argument.Kind.CONSTANT && argument.asConstant() instanceof TruffleString constant) { - return new String[]{constant.toJavaStringUncached()}; - } - } - return new String[0]; - default: - // load.local and store.local encode the frame slot; c.ForIterate and - // c.UnpackToLocals2 take local accessors, whose info is the varnames index. - List names = new ArrayList<>(2); - for (Argument argument : access.getArguments()) { - if (argument.getKind() == Argument.Kind.LOCAL_OFFSET) { - names.add(slotName(code, argument.asLocalOffset())); - } else if (argument.getKind() == Argument.Kind.CONSTANT && argument.asConstant() instanceof LocalAccessor local) { - names.add(local.getLocalInfo(bytecode) instanceof Integer index && index < code.varnames.length ? code.varnames[index].toJavaStringUncached() : null); - } - } - return names.toArray(new String[0]); + } + return new String[0]; + } + // Global and name operations carry the name; ForIterate and UnpackToLocals2 take local + // accessors, whose info is the varnames index. + List names = new ArrayList<>(2); + for (Argument argument : access.getArguments()) { + if (argument.getKind() != Argument.Kind.CONSTANT) { + continue; + } + Object constant = argument.asConstant(); + if (constant instanceof TruffleString name) { + return new String[]{name.toJavaStringUncached()}; + } else if (constant instanceof LocalAccessor local) { + names.add(local.getLocalInfo(bytecode) instanceof Integer index && index < code.varnames.length ? code.varnames[index].toJavaStringUncached() : null); + } } + return names.toArray(new String[0]); } private static Integer intArgument(Argument argument) { diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 4eb28ea78b..31c07e3482 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -53,7 +53,7 @@ }, { "name": "tools", - "version": "fe7f970d99b666853ebe7abd14f14328aec29118", + "version": "2a257154ffc9f3705be292cdacd6939fa862317f", "subdir": True, "urls": [ {"url": "https://github.com/elide-dev/graal", "kind": "git"}, @@ -61,7 +61,7 @@ }, { "name": "regex", - "version": "fe7f970d99b666853ebe7abd14f14328aec29118", + "version": "2a257154ffc9f3705be292cdacd6939fa862317f", "subdir": True, "urls": [ {"url": "https://github.com/elide-dev/graal", "kind": "git"}, From 420218a02d6f5363c8a4c2a98cbbbd6919f6858e Mon Sep 17 00:00:00 2001 From: Sam Gammon Date: Mon, 16 Nov 2026 14:16:08 -0800 Subject: [PATCH 4/7] Pin Graal imports to the merged getOperationClass() backport (elide-dev/graal#241) Co-Authored-By: Claude Opus 5.5 --- mx.graalpython/suite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 31c07e3482..25b3bfb37e 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -53,7 +53,7 @@ }, { "name": "tools", - "version": "2a257154ffc9f3705be292cdacd6939fa862317f", + "version": "67e12a237ab29b0f8dc665c9388a33a96719f4a0", "subdir": True, "urls": [ {"url": "https://github.com/elide-dev/graal", "kind": "git"}, @@ -61,7 +61,7 @@ }, { "name": "regex", - "version": "2a257154ffc9f3705be292cdacd6939fa862317f", + "version": "67e12a237ab29b0f8dc665c9388a33a96719f4a0", "subdir": True, "urls": [ {"url": "https://github.com/elide-dev/graal", "kind": "git"}, From 919d9f881599d12ad9463659259c39463c2c1fa4 Mon Sep 17 00:00:00 2001 From: Sam Gammon Date: Mon, 16 Nov 2026 14:41:32 -0800 Subject: [PATCH 5/7] Keep instruction locations unchanged inside variable tags #11 made the enclosing statement the innermost source section of every tagged variable access. Source sections exist whether or not tags are instrumented, so exception locations, traceback carets and line tables moved to the statement. This broke the tagged CPython tests test_traceback (test_caret_in_type_annotation, test_byte_offset_with_emojis_term_highlight) and test_compile.test_lineno_procedure_call. Open the statement and identifier layers outside the tag, and re-open the previously innermost range inside it. The tag's own location stays at the statement, which keeps debugger caller locations stable (PythonDebugTest), and the tagged instructions keep their previous location. The augmented-assignment read tag now uses the same layout. Co-Authored-By: Claude Opus 5.5 --- .../bytecode_dsl/RootNodeCompiler.java | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java index ebee32806e..58f7f87f14 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java @@ -256,6 +256,8 @@ public final class RootNodeCompiler implements BaseBytecodeDSLVisitor statementLocations = new ArrayDeque<>(); + /** Source ranges of the currently open source sections, innermost first. */ + private final Deque openSourceRanges = new ArrayDeque<>(); private SourceRange currentLocation; BytecodeLocal yieldFromGenerator; private int lastTracedLine; @@ -765,6 +767,7 @@ public BytecodeDSLCompilerResult compile() { public void reset() { statementLocations.clear(); + openSourceRanges.clear(); currentLocation = null; currentSaveExceptionLocal = null; prevSaveExceptionLocal = null; @@ -946,7 +949,8 @@ private static SourceRange getRootSourceRange(SSTNode node) { } } - private static void beginSourceSectionInner(Builder b, SourceRange sourceRange) { + private void beginSourceSectionInner(Builder b, SourceRange sourceRange) { + openSourceRanges.push(sourceRange); if (sourceRange.startLine >= 1 && sourceRange != SourceRange.ARTIFICIAL_RANGE) { if (sourceRange.startColumn >= 0 && sourceRange.endLine >= sourceRange.startLine && sourceRange.endColumn >= 0) { int startColumn = sourceRange.startColumn + 1; @@ -971,6 +975,7 @@ private static void beginSourceSectionInner(Builder b, SourceRange sourceRange) void endSourceSection(Builder b) { b.endSourceSection(); + openSourceRanges.pop(); } void beginStatement(SSTNode node, Builder b) { @@ -1942,23 +1947,29 @@ private boolean beginDirectVariableWrite(ExprTy target, BytecodeLocal local, Bui /** * Opens a variable tag. The names are not recorded here: instrumentation resolves them lazily - * from the operands of the tagged instructions (see {@code VariableAccessDescriptor}). The - * source layers are, from most to least concrete: the enclosing statement (or expression - * root), then one identifier range per variable operand of the tagged store or load. + * from the operands of the tagged instructions (see {@code VariableAccessDescriptor}). *

- * Bytecode DSL stack walks select the innermost tag, so its program location stays at the - * statement to keep debugger caller locations stable; the identifier layers locate the access. + * Layout, from outermost: one identifier range per variable operand of the tagged store or + * load (in reverse operand order), the enclosing statement, the tag, and then the source range + * that was innermost before the tag. The tag's own source sections are therefore, from most to + * least concrete, the statement followed by the identifiers in operand order. Bytecode DSL + * stack walks select the innermost tag, so debugger caller locations stay at the statement. + * The tagged instructions keep their previous innermost location, so exception locations, + * tracebacks and line tables are unaffected by these layers. */ private void beginVariableTag(boolean write, Builder b, SourceRange... accessRanges) { + SourceRange enclosing = openSourceRanges.isEmpty() ? SourceRange.ARTIFICIAL_RANGE : openSourceRanges.peek(); for (int i = accessRanges.length - 1; i >= 0; i--) { beginSourceSectionInner(b, accessRanges[i]); } SourceRange statement = statementLocations.peek(); beginSourceSectionInner(b, variableSourceRange(statement != null ? statement : getRootSourceRange(startNode))); b.beginTag(write ? WriteVariableTag.class : ReadVariableTag.class); + beginSourceSectionInner(b, enclosing); } private void endVariableTag(boolean write, int accessCount, Builder b) { + endSourceSection(b); b.endTag(write ? WriteVariableTag.class : ReadVariableTag.class); for (int i = 0; i <= accessCount; i++) { endSourceSection(b); @@ -4049,12 +4060,11 @@ public Void visit(ExprTy.Name node) { beginStoreLocal(node.id, b); beginAugAssign(); if (tagged) { - // Shares the enclosing write tag's statement and identifier layers. - b.beginTag(ReadVariableTag.class); + beginVariableTag(false, b, variableAccessRange(node)); } emitReadLocal(node.id, b); if (tagged) { - b.endTag(ReadVariableTag.class); + endVariableTag(false, b); } value.accept(StatementCompiler.this); endAugAssign(); @@ -4890,7 +4900,7 @@ public void endWrapWithDecorators(ExprTy[] decorators) { b.endTraceLineWithArgument(decorators[decorators.length - 1 - i].getSourceRange().startLine); b.endInstrumentCall(); endCallUnaryMethod(); - b.endSourceSection(); + endSourceSection(b); } } From e450cca760baffe65a2ae82b8c1bd72f5ecdd64d Mon Sep 17 00:00:00 2001 From: Sam Gammon Date: Mon, 16 Nov 2026 20:55:08 -0800 Subject: [PATCH 6/7] ci: sync ci/graal/common.json with the pinned Graal revision #11 moved the Graal imports to a revision that expects labsjdk with JVMCI 25.5 (and mx 7.88.0), but ci/graal/common.json still pinned JVMCI 25.4. On that JDK Truffle falls back to the non-optimizing runtime ("Graal expected a JVMCI release version of 25.5, but got 25.4"), so every junit test fails on the -Dpolyglot.engine.CompilationFailureAction option the gate sets. This went unnoticed while the JDK 21 bootstrap download failed before any test ran. Co-Authored-By: Claude Opus 5.5 --- ci/graal/common.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/graal/common.json b/ci/graal/common.json index d54f2650d4..efd7479bab 100644 --- a/ci/graal/common.json +++ b/ci/graal/common.json @@ -4,7 +4,7 @@ "Jsonnet files should not include this file directly but use ci/common.jsonnet instead." ], - "mx_version": "7.87.0", + "mx_version": "7.88.0", "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { @@ -49,12 +49,12 @@ "graalvm-ee-25-ea": {"name": "graalvm-jdk", "version": "25.0.0", "ea": "36", "platformspecific": true }, "oraclejdk-latest": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-25.0.4.1+1", "platformspecific": true, "extrabundles": ["static-libs"]}, - "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.4-b23", "platformspecific": true }, - "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.4-b23-debug", "platformspecific": true }, - "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.4-b23-sulong", "platformspecific": true }, - "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.4-b23", "platformspecific": true }, - "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.4-b23-debug", "platformspecific": true }, - "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.4-b23-sulong", "platformspecific": true } + "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.5-b24", "platformspecific": true }, + "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.5-b24-debug", "platformspecific": true }, + "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.5-b24-sulong", "platformspecific": true }, + "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.5-b24", "platformspecific": true }, + "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.5-b24-debug", "platformspecific": true }, + "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.5-b24-sulong", "platformspecific": true } }, "default_jdks" : ["labsjdk-ee-latest", "labsjdk-ce-latest"], From d07e2e83d3e1a4645364a52975eaf12c5906317b Mon Sep 17 00:00:00 2001 From: Sam Gammon Date: Mon, 16 Nov 2026 21:02:22 -0800 Subject: [PATCH 7/7] Apply mx 7.88 Eclipse formatting to TpSlotDescrGet The style gate's CodeFormatCheck with mx 7.88.0 indents the sealed class's permits continuation line. Co-Authored-By: Claude Opus 5.5 --- .../python/builtins/objects/type/slots/TpSlotDescrGet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/slots/TpSlotDescrGet.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/slots/TpSlotDescrGet.java index f7db105bcf..bea97f3247 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/slots/TpSlotDescrGet.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/slots/TpSlotDescrGet.java @@ -101,7 +101,7 @@ private TpSlotDescrGet() { } public abstract static sealed class TpSlotDescrGetBuiltin extends TpSlotBuiltin// - permits TpSlotDescrGetBuiltinComplex, TpSlotDescrGetBuiltinSimple { + permits TpSlotDescrGetBuiltinComplex, TpSlotDescrGetBuiltinSimple { static final BuiltinSlotWrapperSignature SIGNATURE = BuiltinSlotWrapperSignature.of(2, J_DOLLAR_SELF, "obj", "type"); static final PExternalFunctionWrapper WRAPPER = PExternalFunctionWrapper.DESCR_GET;