From 0289c6d2921e98620362ce42521555c5dfb434be Mon Sep 17 00:00:00 2001 From: Sam Gammon Date: Sun, 1 Nov 2026 12:06:17 -0800 Subject: [PATCH] Adopt SDK .23 variable-access instrumentation metadata --- docs/user/Tooling.md | 24 ++ .../python/test/debug/PythonDebugTest.java | 12 +- .../test/runtime/VariableAccessTests.java | 322 ++++++++++++++++++ .../modules/MarshalModuleBuiltins.java | 18 +- .../bytecode_dsl/BytecodeDSLCompiler.java | 2 +- .../bytecode_dsl/RootNodeCompiler.java | 119 +++++++ .../bytecode_dsl/BytecodeDSLCodeUnit.java | 22 +- .../bytecode_dsl/PBytecodeDSLRootNode.java | 18 + .../VariableAccessDescriptor.java | 195 +++++++++++ mx.graalpython/suite.py | 12 +- 10 files changed, 724 insertions(+), 20 deletions(-) create mode 100644 graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java create mode 100644 graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java diff --git a/docs/user/Tooling.md b/docs/user/Tooling.md index e0754c2af2..e498423ef9 100644 --- a/docs/user/Tooling.md +++ b/docs/user/Tooling.md @@ -36,6 +36,30 @@ You can inspect variables, set watch expressions, interactively evaluate code sn ![Chrome Inspector](./assets/Chrome_Inspector.png) +## Variable metadata for instruments + +The Elide fork's Bytecode DSL interpreter exposes source-level name reads and assignment targets +through Truffle's `ReadVariableTag` and `WriteVariableTag`. Node objects supply `readVariableName` +or `writeVariableName`, respectively. Each name is an immutable interop string with the source +location of that occurrence; operations that assign multiple names can return an array of these +strings. A tag's program location retains the enclosing statement or expression root for debugger +stack walks; use the name object's source location for the exact identifier range. Names reflect Python identifier normalization and private-name mangling. Source positions +use Truffle's UTF-16 columns, including for non-BMP identifiers. + +For unexecuted source, attach a source-section load listener before calling the instrument +`Env.parse(Source)` API. Retain the returned call target while collecting facts. Reading node +objects and their name/location descriptors does not require an entered Python context or guest +values, and supports concurrent readers. Code-unit serialization preserves this metadata. + +This interface describes compiler-emitted accesses, rather than a complete source symbol index. +It covers name expressions, assignment and augmented-assignment targets, loop targets and +comprehension targets. Attribute/subscript updates are not variable bindings; only the name +expressions used to compute their receivers and indices are tagged. Parameter declarations, +import/definition bindings, deletion, optimized-away expressions, and AST-created names without +valid source ranges are outside this initial coverage. A name's location is its access site, not +its declaration or proof that an import/reference resolves. Native Image tooling qualification +remains separate from JVM testing. + ## Profiling GraalPy provides three main profiling capabilities: CPU sampling, CPU tracing, and memory tracing. diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java index f4eb2173d3..a70691e4ca 100644 --- a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java @@ -413,6 +413,8 @@ def gen_fun(a, b): session.install(Breakpoint.newBuilder(DebuggerTester.getSourceImpl(source)).lineIs(2).build()); session.install(Breakpoint.newBuilder(DebuggerTester.getSourceImpl(source)).lineIs(4).build()); tester.startEval(source); + // SDK .23 includes the debugger's once-per-statement breakpoint semantics: resuming + // inside a yield does not hit its BEFORE breakpoint a second time. expectSuspended((SuspendedEvent event) -> { DebugStackFrame frame = event.getTopStackFrame(); @@ -420,22 +422,12 @@ def gen_fun(a, b): checkStack(frame, "gen_fun", "a", "1", "b", "2"); event.prepareContinue(); }); - expectSuspended((SuspendedEvent event) -> { - DebugStackFrame frame = event.getTopStackFrame(); - assertEquals(2, frame.getSourceSection().getStartLine()); - event.prepareContinue(); - }); expectSuspended((SuspendedEvent event) -> { DebugStackFrame frame = event.getTopStackFrame(); assertEquals(4, frame.getSourceSection().getStartLine()); checkStack(frame, "gen_fun", "a", "1", "b", "2", "c", "3"); event.prepareContinue(); }); - expectSuspended((SuspendedEvent event) -> { - DebugStackFrame frame = event.getTopStackFrame(); - assertEquals(4, frame.getSourceSection().getStartLine()); - event.prepareContinue(); - }); assertEquals("3", tester.expectDone()); } } diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java new file mode 100644 index 0000000000..eda760b817 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package com.oracle.graal.python.test.runtime; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.ref.Reference; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.graalvm.polyglot.Context; +import org.junit.Test; + +import com.oracle.graal.python.builtins.objects.code.PCode; +import com.oracle.truffle.api.CallTarget; +import com.oracle.truffle.api.instrumentation.InstrumentableNode; +import com.oracle.truffle.api.instrumentation.SourceSectionFilter; +import com.oracle.truffle.api.instrumentation.StandardTags.ReadVariableTag; +import com.oracle.truffle.api.instrumentation.StandardTags.WriteVariableTag; +import com.oracle.truffle.api.instrumentation.TruffleInstrument; +import com.oracle.truffle.api.interop.InteropException; +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.InvalidArrayIndexException; +import com.oracle.truffle.api.interop.UnknownIdentifierException; +import com.oracle.truffle.api.nodes.Node; +import com.oracle.truffle.api.source.Source; +import com.oracle.truffle.api.source.SourceSection; + +public class VariableAccessTests { + private static final String ID = "python-variable-access-tests"; + + private record Access(String root, String name, boolean write, int line, int column, String text) { + } + + @Test + public void unexecutedVariableAccesses() throws Exception { + for (boolean forceSections : new boolean[]{false, true}) { + for (boolean eagerNodes : new boolean[]{false, true}) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (Context context = Context.newBuilder("python").allowAllAccess(true).allowExperimentalOptions(true).option("python.ForceInitializeSourceSections", Boolean.toString( + forceSections)).option("python.EagerlyMaterializeInstrumentationNodes", Boolean.toString(eagerNodes)).out(output).err(output).build()) { + AnalysisInstrument instrument = context.getEngine().getInstruments().get(ID).lookup(AnalysisInstrument.class); + context.initialize("python"); + output.reset(); + Analysis analysis; + context.enter(); + try { + analysis = instrument.analyze(Source.newBuilder("python", """ + def inspect(outer): + inner = outer + inner += outer + left, right = outer + return inner, left, right + @must_not_execute() + def poison(arg: must_not_execute() = must_not_execute()): + return arg + import must_not_import + print('must not execute') + raise RuntimeError('must not execute') + """, "accesses.py").uri(URI.create("file:///accesses.py")).build()); + List accesses = analysis.accesses().stream().filter(a -> a.root().equals("inspect")).toList(); + assertEquals(11, accesses.size()); + assertTrue(accesses.contains(new Access("inspect", "inner", true, 2, 5, "inner"))); + assertTrue(accesses.contains(new Access("inspect", "outer", false, 2, 13, "outer"))); + assertTrue(accesses.contains(new Access("inspect", "inner", true, 3, 5, "inner"))); + assertTrue(accesses.contains(new Access("inspect", "inner", false, 3, 5, "inner"))); + assertTrue(accesses.contains(new Access("inspect", "left", true, 4, 5, "left"))); + assertTrue(accesses.contains(new Access("inspect", "right", true, 4, 11, "right"))); + Analysis edited = instrument.analyze(Source.newBuilder("python", """ + def changed(value): + return value + """, "accesses.py").uri(URI.create("file:///accesses.py")).build()); + assertEquals(List.of(new Access("changed", "value", false, 2, 12, "value")), edited.accesses()); + assertEquals("", output.toString()); + } finally { + context.leave(); + } + // Both node-object construction and interop reads must work without an entered + // context, concurrently with other metadata readers. + CompletableFuture> first = CompletableFuture.supplyAsync(analysis::accesses); + CompletableFuture> second = CompletableFuture.supplyAsync(analysis::accesses); + assertEquals(first.get(), second.get()); + } + } + } + } + + @Test + public void bindingNamesAndAssignmentForms() throws IOException { + try (Context context = Context.newBuilder("python").allowAllAccess(true).build()) { + AnalysisInstrument instrument = context.getEngine().getInstruments().get(ID).lookup(AnalysisInstrument.class); + context.initialize("python"); + context.enter(); + try { + Analysis analysis = instrument.analyze(Source.newBuilder("python", """ + def enclosing(captured): + def nested(obj, seq, index): + nonlocal captured + global shared + captured = shared + obj.attribute = seq[index] + return (local := captured) + return nested + class Example: + def method(self, __private): + K = __private + return K + """, "bindings.py").build()); + List accesses = analysis.accesses(); + assertTrue(accesses.stream().anyMatch(a -> a.root().equals("nested") && a.name().equals("captured") && a.write())); + assertTrue(accesses.stream().anyMatch(a -> a.name().equals("shared") && !a.write())); + assertTrue(accesses.stream().anyMatch(a -> a.name().equals("local") && a.write())); + assertFalse(accesses.stream().anyMatch(a -> a.name().equals("attribute"))); + assertTrue(accesses.stream().anyMatch(a -> a.name().equals("_Example__private") && a.text().equals("__private"))); + assertTrue(accesses.stream().anyMatch(a -> a.name().equals("K") && a.text().equals("K") && a.write())); + Analysis unicode = instrument.analyze(Source.newBuilder("python", + "def unicode(arg):\r\n emoji = '😀'; 𐐀 = arg; return 𐐀\r\n", "unicode.py").build()); + assertTrue(unicode.accesses().stream().anyMatch(a -> a.name().equals("𐐀") && a.text().equals("𐐀") && a.write())); + assertTrue(unicode.accesses().stream().anyMatch(a -> a.name().equals("𐐀") && a.text().equals("𐐀") && !a.write())); + } finally { + context.leave(); + } + } + } + + @Test + public void executedTagsPreservePythonBehavior() throws IOException { + try (Context context = Context.newBuilder("python").allowAllAccess(true).build()) { + AnalysisInstrument instrument = context.getEngine().getInstruments().get(ID).lookup(AnalysisInstrument.class); + context.initialize("python"); + context.enter(); + try { + Analysis analysis = instrument.analyzeExecuting(Source.newBuilder("python", """ + def run(pair): + left, right = pair + total = 0 + for value in (left, right): + total += value + return total + assert run((3, 4)) == 7 + assert [x + 1 for x in range(2)] == [1, 2] + """, "execution.py").build()); + assertTrue(analysis.accesses().stream().anyMatch(a -> a.name().equals("value") && a.write())); + assertTrue(analysis.accesses().stream().anyMatch(a -> a.name().equals("x") && a.write())); + assertFalse(analysis.accesses().stream().anyMatch(a -> a.name().startsWith("."))); + } finally { + context.leave(); + } + } + } + + @Test + public void serializedCodeRetainsMetadata() throws IOException { + try (Context context = Context.newBuilder("python").allowAllAccess(true).build()) { + AnalysisInstrument instrument = context.getEngine().getInstruments().get(ID).lookup(AnalysisInstrument.class); + context.initialize("python"); + context.enter(); + try { + Source source = Source.newBuilder("python", """ + def serialized(arg): + result = arg + return result + """, "serialized.py").build(); + Analysis analysis = instrument.analyzeSerialized(source); + assertEquals(6, analysis.accesses().stream().filter(a -> a.root().equals("serialized")).count()); + } finally { + context.leave(); + } + } + } + + private record Analysis(CallTarget target, List nodes) { + List accesses() { + List result = new ArrayList<>(); + InteropLibrary interop = InteropLibrary.getUncached(); + for (Node node : nodes) { + boolean write = ((InstrumentableNode) node).hasTag(WriteVariableTag.class); + String property = write ? WriteVariableTag.NAME : ReadVariableTag.NAME; + Object descriptor = ((InstrumentableNode) node).getNodeObject(); + assertNotNull(descriptor); + assertTrue(interop.hasMembers(descriptor)); + assertTrue(interop.isMemberReadable(descriptor, property)); + assertFalse(interop.isMemberModifiable(descriptor, property)); + assertFalse(interop.isMemberInsertable(descriptor, "other")); + assertThrows(UnknownIdentifierException.class, () -> interop.readMember(descriptor, "other")); + try { + Object members = interop.getMembers(descriptor); + assertEquals(1, interop.getArraySize(members)); + assertEquals(property, interop.asString(interop.readArrayElement(members, 0))); + assertFalse(interop.isArrayElementReadable(members, -1)); + assertThrows(InvalidArrayIndexException.class, () -> interop.readArrayElement(members, -1)); + Object names = interop.readMember(descriptor, property); + if (interop.hasArrayElements(names)) { + assertFalse(interop.isArrayElementReadable(names, -1)); + assertThrows(InvalidArrayIndexException.class, () -> interop.readArrayElement(names, interop.getArraySize(names))); + for (long i = 0; i < interop.getArraySize(names); i++) { + result.add(access(node, write, interop.readArrayElement(names, i), interop)); + } + } else { + result.add(access(node, write, names, interop)); + } + } catch (InteropException e) { + throw new AssertionError(e); + } + } + Reference.reachabilityFence(target); + return result; + } + + private static Access access(Node node, boolean write, Object name, InteropLibrary interop) throws InteropException { + assertTrue(interop.isString(name)); + assertTrue(interop.hasSourceLocation(name)); + SourceSection section = interop.getSourceLocation(name); + assertEquals(node.getSourceSection().getSource(), section.getSource()); + return new Access(node.getRootNode().getName(), interop.asString(name), write, section.getStartLine(), section.getStartColumn(), section.getSource().hasCharacters() + ? section.getCharacters().toString() + : ""); + } + } + + @TruffleInstrument.Registration(id = ID, services = AnalysisInstrument.class) + public static final class AnalysisInstrument extends TruffleInstrument { + private Env env; + + @Override + protected void onCreate(Env instrumentEnv) { + env = instrumentEnv; + env.registerService(this); + } + + Analysis analyze(Source source) throws IOException { + return analyze(source, false, false); + } + + Analysis analyzeSerialized(Source source) throws IOException { + return analyze(source, true, true); + } + + Analysis analyzeExecuting(Source source) throws IOException { + return analyze(source, true, false); + } + + private Analysis analyze(Source source, boolean execute, boolean serialized) throws IOException { + List nodes = new ArrayList<>(); + SourceSectionFilter filter = SourceSectionFilter.newBuilder().sourceIs(source).tagIs(ReadVariableTag.class, WriteVariableTag.class).build(); + if (serialized) { + // marshal reconstructs an unloaded Source named after the code's qualified + // name, rather than retaining the original in-memory Source identity. + filter = SourceSectionFilter.newBuilder().rootNameIs(name -> "serialized".equals(name)).tagIs(ReadVariableTag.class, WriteVariableTag.class).build(); + } + var binding = env.getInstrumenter().attachLoadSourceSectionListener(filter, event -> nodes.add(event.getNode()), true); + CallTarget target; + try { + target = env.parse(source); + if (execute) { + target.call(); + } + if (serialized) { + // This fixture intentionally executes harmless definitions and a cloned + // function. The poison-source test exercises the inert path separately. + env.parse(Source.newBuilder("python", """ + import marshal, types + cloned = types.FunctionType(marshal.loads(marshal.dumps(serialized.__code__)), globals()) + assert cloned(42) == 42 + """, "roundtrip.py").build()).call(); + PCode original = (PCode) env.parse(Source.newBuilder("python", "serialized.__code__", "original-code.py").build()).call(); + PCode cloned = (PCode) env.parse(Source.newBuilder("python", "cloned.__code__", "cloned-code.py").build()).call(); + assertEquals(original.getCodeUnit().variableAccesses, cloned.getCodeUnit().variableAccesses); + } + } finally { + binding.dispose(); + } + return new Analysis(target, List.copyOf(nodes)); + } + } +} diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java index 14b8795283..a76a352a27 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java @@ -115,6 +115,7 @@ import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.StringLiterals; import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit; +import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit.VariableAccess; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNodeGen; import com.oracle.graal.python.nodes.call.special.LookupAndCallBinaryNode; @@ -1428,10 +1429,16 @@ private BytecodeDSLCodeUnit readBytecodeDSLCodeUnit() { int yieldFromGeneratorIndex = readInt(); int instrumentationDataIndex = readInt(); int maxProfileCEventStackSize = readInt(); + int accessCount = readSize(); + List variableAccesses = new ArrayList<>(accessCount); + for (int i = 0; i < accessCount; i++) { + variableAccesses.add(new VariableAccess(readString(true).toJavaStringUncached(), readInt(), readInt(), readInt(), readInt(), readInt() != 0)); + } BytecodeSupplier provider = new BytecodeSupplier(serialized, bytecodeFile, sourceReference, bytecodeOffset, bytecodeSize, cacheKey); return new BytecodeDSLCodeUnit(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, constants, - startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, provider); + startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, variableAccesses, + provider); } private void writeCodeUnit(BytecodeDSLCodeUnit code) throws IOException { @@ -1467,6 +1474,15 @@ private void writeBytecodeDSLCodeUnit(BytecodeDSLCodeUnit code) throws IOExcepti writeInt(code.yieldFromGeneratorIndex); writeInt(code.instrumentationDataIndex); writeInt(code.maxProfileCEventStackSize); + writeInt(code.variableAccesses.size()); + for (VariableAccess access : code.variableAccesses) { + writeString(PythonUtils.toTruffleStringUncached(access.name())); + writeInt(access.startLine()); + writeInt(access.startColumn()); + writeInt(access.endLine()); + writeInt(access.endColumn()); + writeInt(access.write() ? 1 : 0); + } } private PCode readCode() { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java index 7b80fd9c5f..407f553b30 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java @@ -71,7 +71,7 @@ public class BytecodeDSLCompiler { * Bump this with every Truffle import change because serialized GraalPy bytecode effectively * depends on Truffle bytecode internals. */ - public static final int BYTECODE_VERSION = 35; + public static final int BYTECODE_VERSION = 36; public static final record BytecodeDSLCompilerResult(PBytecodeDSLRootNode rootNode, BytecodeDSLCodeUnit codeUnit) { } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java index 05c0b9d684..8b42358512 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java @@ -75,11 +75,14 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.util.ArrayList; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; @@ -104,6 +107,7 @@ import com.oracle.graal.python.lib.PyObjectRichCompareBool; import com.oracle.graal.python.nodes.StringLiterals; import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit; +import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit.VariableAccess; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNodeGen; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNodeGen.Builder; @@ -160,6 +164,8 @@ import com.oracle.truffle.api.bytecode.serialization.BytecodeSerializer; import com.oracle.truffle.api.debug.DebuggerTags; import com.oracle.truffle.api.instrumentation.StandardTags.StatementTag; +import com.oracle.truffle.api.instrumentation.StandardTags.ReadVariableTag; +import com.oracle.truffle.api.instrumentation.StandardTags.WriteVariableTag; import com.oracle.truffle.api.strings.TruffleString; /** @@ -211,6 +217,7 @@ public final class RootNodeCompiler implements BaseBytecodeDSLVisitor variableAccesses = new LinkedHashSet<>(); private final HashMap varnames; private final HashMap cellvars; private final HashMap freevars; @@ -251,6 +258,7 @@ public final class RootNodeCompiler implements BaseBytecodeDSLVisitor statementLocations = new ArrayDeque<>(); private SourceRange currentLocation; BytecodeLocal yieldFromGenerator; private int lastTracedLine; @@ -587,6 +595,7 @@ flags, orderedTruffleStringArray(names), yieldFromGenerator != null ? yieldFromGenerator.getLocalIndex() : -1, instrumentationDataLocal.getLocalIndex(), maxProfileCEventStackSize, + List.copyOf(variableAccesses), new BytecodeSupplier(nodes)); ctx.codeUnits.put(key, codeUnit); } @@ -759,6 +768,7 @@ public BytecodeDSLCompilerResult compile() { } public void reset() { + statementLocations.clear(); currentLocation = null; currentSaveExceptionLocal = null; prevSaveExceptionLocal = null; @@ -972,6 +982,7 @@ void beginStatement(SSTNode node, Builder b) { } void beginStatement(SourceRange sourceRange, Builder b) { + statementLocations.push(sourceRange); beginSourceSection(sourceRange, b); b.beginTag(StatementTag.class); b.beginBlock(); @@ -981,6 +992,7 @@ void endStatement(Builder b) { b.endBlock(); b.endTag(StatementTag.class); endSourceSection(b); + statementLocations.pop(); } void beginReturn(Builder b) { @@ -1408,9 +1420,11 @@ private void emitComprehension(ComprehensionTy[] generators, int index, Builder } else { b.emitClearTraceLine(); } + boolean tagged = beginDirectVariableWrite(comp.target, localValue, b); b.beginForIterate(localValue); b.emitLoadStackValue(iter); b.endForIterate(); + endDirectVariableWrite(tagged, b); b.endBlock(); b.beginBlock(); @@ -1879,6 +1893,77 @@ private BytecodeLocal getFastLocal(String name) { return null; } + private SourceRange variableSourceRange(SSTNode node) { + if (!ctx.source.hasCharacters()) { + return SourceRange.ARTIFICIAL_RANGE; + } + return variableSourceRange(node.getSourceRange()); + } + + private SourceRange variableSourceRange(SourceRange range) { + try { + // Parser columns count Unicode code points; Truffle source columns count UTF-16 units. + int start = Character.offsetByCodePoints(ctx.source.getCharacters(range.startLine), 0, range.startColumn); + int end = Character.offsetByCodePoints(ctx.source.getCharacters(range.endLine), 0, range.endColumn); + return new SourceRange(range.startLine, start, range.endLine, end); + } catch (IllegalArgumentException | IndexOutOfBoundsException e) { + return SourceRange.ARTIFICIAL_RANGE; + } + } + + private boolean recordVariableAccess(ExprTy.Name node, boolean write) { + SourceRange range = variableSourceRange(node); + if (node.id.startsWith(".") || range == SourceRange.ARTIFICIAL_RANGE || range.startLine < 1 || range.startColumn < 0 || + range.endLine < range.startLine || range.endColumn < 1 || !ctx.source.hasCharacters()) { + return false; + } + try { + ctx.source.createSection(range.startLine, range.startColumn + 1, range.endLine, range.endColumn); + } catch (IllegalArgumentException e) { + // AST-created code and incomplete source ranges need not identify a source token. + return false; + } + variableAccesses.add(new VariableAccess(maybeMangle(node.id), range.startLine, range.startColumn, range.endLine, range.endColumn, write)); + return true; + } + + private boolean beginVariableAccess(ExprTy.Name node, boolean write, Builder b) { + if (!recordVariableAccess(node, write)) { + return false; + } + beginVariableTag(variableSourceRange(node), write, b); + return true; + } + + private boolean beginDirectVariableWrite(ExprTy target, BytecodeLocal local, Builder b) { + if (!isTemporaryLocal(local) && target instanceof ExprTy.Name name && recordVariableAccess(name, true)) { + beginVariableTag(variableSourceRange(target), true, b); + return true; + } + return false; + } + + private void beginVariableTag(SourceRange accessRange, boolean write, Builder b) { + beginSourceSection(accessRange, b); + // Bytecode DSL stack walks select the innermost tag. Keep its program location at the + // statement (or expression root), while the outer source layer identifies the access. + SourceRange statement = statementLocations.peek(); + beginSourceSection(variableSourceRange(statement != null ? statement : getRootSourceRange(startNode)), b); + b.beginTag(write ? WriteVariableTag.class : ReadVariableTag.class); + } + + private void endVariableTag(boolean write, Builder b) { + b.endTag(write ? WriteVariableTag.class : ReadVariableTag.class); + endSourceSection(b); + endSourceSection(b); + } + + private void endDirectVariableWrite(boolean tagged, Builder b) { + if (tagged) { + endVariableTag(true, b); + } + } + private void emitReadLocal(String name, Builder b) { emitNameOperation(name, NameOperation.Read, b); } @@ -3097,7 +3182,11 @@ private void emitGetIter(ComprehensionTy[] generators) { public Void visit(ExprTy.Name node) { beginSourceSection(node, b); beginTraceLineChecked(b); + boolean tagged = beginVariableAccess(node, false, b); emitReadLocal(node.id, b); + if (tagged) { + endVariableTag(false, b); + } endTraceLineChecked(node, b); endSourceSection(b); return null; @@ -3724,9 +3813,13 @@ public class StoreVisitor implements BaseBytecodeDSLVisitor { public Void visit(ExprTy.Name node) { beginSourceSection(node, b); emitTraceLineChecked(node, b); + boolean tagged = beginVariableAccess(node, true, b); beginStoreLocal(node.id, b); generateValue.run(); endStoreLocal(node.id, b); + if (tagged) { + endVariableTag(true, b); + } endSourceSection(b); return null; } @@ -3773,9 +3866,22 @@ private void visitIterableAssign(ExprTy[] nodes) { if (nodes.length == 2 && containsNoStarred(nodes)) { BytecodeLocal target1 = allowFastLocalLookup ? beginTemporaryLocalOrGetLocal(nodes[0], b) : beginTemporaryLocal(); BytecodeLocal target2 = allowFastLocalLookup ? beginTemporaryLocalOrGetLocal(nodes[1], b) : beginTemporaryLocal(); + boolean tagged = false; + if (!isTemporaryLocal(target1)) { + tagged |= recordVariableAccess((ExprTy.Name) nodes[0], true); + } + if (!isTemporaryLocal(target2)) { + tagged |= recordVariableAccess((ExprTy.Name) nodes[1], true); + } + if (tagged) { + beginVariableTag(variableSourceRange(nodes[0]).withEnd(variableSourceRange(nodes[1])), true, b); + } b.beginUnpackToLocals2(target1, target2); generateValue.run(); b.endUnpackToLocals2(); + if (tagged) { + endVariableTag(true, b); + } storeTemporaryLocalToTarget(target1, nodes[0], b, allowFastLocalLookup); storeTemporaryLocalToTarget(target2, nodes[1], b, allowFastLocalLookup); endTemporaryLocal(target1); @@ -3930,12 +4036,23 @@ public Void visit(ExprTy.Name node) { beginSourceSection(node, b); emitTraceLineChecked(node, b); + boolean tagged = beginVariableAccess(node, true, b); beginStoreLocal(node.id, b); beginAugAssign(); + if (tagged) { + recordVariableAccess(node, false); + b.beginTag(ReadVariableTag.class); + } emitReadLocal(node.id, b); + if (tagged) { + b.endTag(ReadVariableTag.class); + } value.accept(StatementCompiler.this); endAugAssign(); endStoreLocal(node.id, b); + if (tagged) { + endVariableTag(true, b); + } endSourceSection(b); return null; @@ -4572,9 +4689,11 @@ public Void visit(StmtTy.For node) { // condition b.beginBlock(); b.emitTraceLineAtLoopHeader(currentLocation.startLine); + boolean tagged = beginDirectVariableWrite(node.target, value, b); b.beginForIterate(value); b.emitLoadStackValue(iter); b.endForIterate(); + endDirectVariableWrite(tagged, b); b.endBlock(); // body diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java index fc8c32e0ea..e5dd910c2d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java @@ -78,6 +78,21 @@ public final class BytecodeDSLCodeUnit { @CompilationFinal(dimensions = 1) public final TruffleString[] freevars; @CompilationFinal(dimensions = 1) public final Object[] constants; + /** Source-level variable accesses, independent of bytecode indices and live frames. */ + public final List variableAccesses; + + public record VariableAccess(String name, int startLine, int startColumn, int endLine, int endColumn, boolean write) { + public boolean isWithin(SourceSection section) { + return section.isAvailable() && + (startLine > section.getStartLine() || startLine == section.getStartLine() && startColumn + 1 >= section.getStartColumn()) && + (endLine < section.getEndLine() || endLine == section.getEndLine() && endColumn <= section.getEndColumn()); + } + + public SourceSection getSourceSection(Source source) { + return source.createSection(startLine, startColumn + 1, endLine, endColumn); + } + } + public final int startLine; public final int startColumn; public final int endLine; @@ -92,7 +107,8 @@ public final class BytecodeDSLCodeUnit { public BytecodeDSLCodeUnit(TruffleString name, TruffleString qualname, int argCount, int kwOnlyArgCount, int positionalOnlyArgCount, int flags, TruffleString[] names, TruffleString[] varnames, TruffleString[] cellvars, TruffleString[] freevars, Object[] constants, int startLine, int startColumn, int endLine, int endColumn, - int classcellIndex, int selfIndex, int yieldFromGeneratorIndex, int instrumentationDataIndex, int maxProfileCEventStackSize, BytecodeSupplier supplier) { + int classcellIndex, int selfIndex, int yieldFromGeneratorIndex, int instrumentationDataIndex, int maxProfileCEventStackSize, List variableAccesses, + BytecodeSupplier supplier) { assert isInterned(name); this.name = name; assert qualname == null || isInterned(qualname); @@ -120,6 +136,7 @@ public BytecodeDSLCodeUnit(TruffleString name, TruffleString qualname, int argCo this.yieldFromGeneratorIndex = yieldFromGeneratorIndex; this.instrumentationDataIndex = instrumentationDataIndex; this.maxProfileCEventStackSize = maxProfileCEventStackSize; + this.variableAccesses = List.copyOf(variableAccesses); } public abstract static class BytecodeSupplier { @@ -131,7 +148,8 @@ public abstract static class BytecodeSupplier { public BytecodeDSLCodeUnit withFlags(int flags) { return new BytecodeDSLCodeUnit(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, constants, - startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, supplier); + startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, variableAccesses, + supplier); } @TruffleBoundary diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java index d25e3fca4f..a820013afe 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java @@ -192,6 +192,7 @@ import com.oracle.graal.python.lib.PyTupleCheckNode; import com.oracle.graal.python.lib.RichCmpOp; import com.oracle.graal.python.nodes.BuiltinNames; +import com.oracle.graal.python.nodes.instrumentation.VariableAccessDescriptor; import com.oracle.graal.python.nodes.ErrorMessages; import com.oracle.graal.python.nodes.PGuards; import com.oracle.graal.python.nodes.PRaiseNode; @@ -286,6 +287,9 @@ import com.oracle.truffle.api.bytecode.BytecodeFrame; import com.oracle.truffle.api.bytecode.BytecodeLocation; import com.oracle.truffle.api.bytecode.BytecodeNode; +import com.oracle.truffle.api.bytecode.TagTreeNode; +import com.oracle.truffle.api.instrumentation.StandardTags.ReadVariableTag; +import com.oracle.truffle.api.instrumentation.StandardTags.WriteVariableTag; import com.oracle.truffle.api.bytecode.BytecodeRootNode; import com.oracle.truffle.api.bytecode.ConstantOperand; import com.oracle.truffle.api.bytecode.ContinuationResult; @@ -494,6 +498,20 @@ protected void prepareForInstrumentation(Set> materializedTags) { } } + @Override + @TruffleBoundary + public Object getInstrumentationNodeObject(TagTreeNode node) { + if (co == null) { + return null; + } + if (node.hasTag(ReadVariableTag.class)) { + return VariableAccessDescriptor.create(ReadVariableTag.NAME, false, node.getSourceSections(), co.variableAccesses); + } else if (node.hasTag(WriteVariableTag.class)) { + return VariableAccessDescriptor.create(WriteVariableTag.NAME, true, node.getSourceSections(), co.variableAccesses); + } + return null; + } + public final PythonLanguage getLanguage() { return getLanguage(PythonLanguage.class); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java new file mode 100644 index 0000000000..780a4561d0 --- /dev/null +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +// skip GIL +package com.oracle.graal.python.nodes.instrumentation; + +import java.util.ArrayList; +import java.util.List; + +import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit.VariableAccess; +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.InvalidArrayIndexException; +import com.oracle.truffle.api.interop.TruffleObject; +import com.oracle.truffle.api.interop.UnknownIdentifierException; +import com.oracle.truffle.api.interop.UnsupportedMessageException; +import com.oracle.truffle.api.library.ExportLibrary; +import com.oracle.truffle.api.library.ExportMessage; +import com.oracle.truffle.api.source.SourceSection; + +/** Immutable compiler metadata; safe to query without a language context or the GIL. */ +@ExportLibrary(InteropLibrary.class) +public final class VariableAccessDescriptor implements TruffleObject { + private final String property; + private final Object names; + + private VariableAccessDescriptor(String property, Object names) { + this.property = property; + this.names = names; + } + + @TruffleBoundary + public static Object create(String property, boolean write, SourceSection[] sections, List accesses) { + SourceSection section = null; + for (SourceSection candidate : sections) { + if (candidate == null || !candidate.isAvailable()) { + continue; + } + for (VariableAccess access : accesses) { + if (access.write() == write && access.isWithin(candidate) && (section == null || isWithin(candidate, section))) { + section = candidate; + break; + } + } + } + if (section == null) { + return null; + } + List names = new ArrayList<>(); + for (VariableAccess access : accesses) { + if (access.write() == write && access.isWithin(section)) { + names.add(new VariableName(access.name(), access.getSourceSection(section.getSource()))); + } + } + if (names.isEmpty()) { + return null; + } + Object value = names.size() == 1 ? names.get(0) : new NameArray(names.toArray()); + return new VariableAccessDescriptor(property, value); + } + + private static boolean isWithin(SourceSection candidate, SourceSection outer) { + return (candidate.getStartLine() > outer.getStartLine() || candidate.getStartLine() == outer.getStartLine() && candidate.getStartColumn() >= outer.getStartColumn()) && + (candidate.getEndLine() < outer.getEndLine() || candidate.getEndLine() == outer.getEndLine() && candidate.getEndColumn() <= outer.getEndColumn()); + } + + @ExportMessage + @SuppressWarnings("static-method") + boolean hasMembers() { + return true; + } + + @ExportMessage + Object getMembers(@SuppressWarnings("unused") boolean includeInternal) { + return new NameArray(new Object[]{property}); + } + + @ExportMessage + boolean isMemberReadable(String member) { + return property.equals(member); + } + + @ExportMessage + Object readMember(String member) throws UnknownIdentifierException { + if (!property.equals(member)) { + throw UnknownIdentifierException.create(member); + } + return names; + } + + @ExportLibrary(InteropLibrary.class) + static final class NameArray implements TruffleObject { + private final Object[] values; + + NameArray(Object[] values) { + this.values = values; + } + + @ExportMessage + @SuppressWarnings("static-method") + boolean hasArrayElements() { + return true; + } + + @ExportMessage + long getArraySize() { + return values.length; + } + + @ExportMessage + boolean isArrayElementReadable(long index) { + return index >= 0 && index < values.length; + } + + @ExportMessage + Object readArrayElement(long index) throws InvalidArrayIndexException { + if (!isArrayElementReadable(index)) { + throw InvalidArrayIndexException.create(index); + } + return values[(int) index]; + } + } + + @ExportLibrary(InteropLibrary.class) + static final class VariableName implements TruffleObject { + private final String name; + private final SourceSection section; + + VariableName(String name, SourceSection section) { + this.name = name; + this.section = section; + } + + @ExportMessage + @SuppressWarnings("static-method") + boolean isString() { + return true; + } + + @ExportMessage + String asString() { + return name; + } + + @ExportMessage + boolean hasSourceLocation() { + return section != null && section.isAvailable(); + } + + @ExportMessage + SourceSection getSourceLocation() throws UnsupportedMessageException { + if (!hasSourceLocation()) { + throw UnsupportedMessageException.create(); + } + return section; + } + } +} diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index dd88789ba6..4eb28ea78b 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -40,7 +40,7 @@ "versionFrom": "regex", "subdir": True, "urls": [ - {"url": "https://github.com/oracle/graal", "kind": "git"}, + {"url": "https://github.com/elide-dev/graal", "kind": "git"}, ] }, { @@ -48,23 +48,23 @@ "versionFrom": "truffle", "subdir": True, "urls": [ - {"url": "https://github.com/oracle/graal", "kind": "git"}, + {"url": "https://github.com/elide-dev/graal", "kind": "git"}, ] }, { "name": "tools", - "version": "5727d2bee28565f4d4db22770df1276de23d054e", + "version": "fe7f970d99b666853ebe7abd14f14328aec29118", "subdir": True, "urls": [ - {"url": "https://github.com/oracle/graal", "kind": "git"}, + {"url": "https://github.com/elide-dev/graal", "kind": "git"}, ], }, { "name": "regex", - "version": "5727d2bee28565f4d4db22770df1276de23d054e", + "version": "fe7f970d99b666853ebe7abd14f14328aec29118", "subdir": True, "urls": [ - {"url": "https://github.com/oracle/graal", "kind": "git"}, + {"url": "https://github.com/elide-dev/graal", "kind": "git"}, ] }, ],