diff --git a/.github/scripts/extract_matrix.py b/.github/scripts/extract_matrix.py index 00c9683c12..cd82a92ffa 100644 --- a/.github/scripts/extract_matrix.py +++ b/.github/scripts/extract_matrix.py @@ -77,7 +77,7 @@ ) DOWNLOADS_LINKS = { - "GRADLE_JAVA_HOME": "https://download.oracle.com/java/{major_version}/latest/jdk-{major_version}_{os}-{arch_short}_bin{ext}", + "GRADLE_JAVA_HOME": "https://download.oracle.com/java/{major_version}/archive/jdk-{version}_{os}-{arch_short}_bin{ext}", "ECLIPSE": "https://www.eclipse.org/downloads/download.php?file=/eclipse/downloads/drops4/R-4.26-202211231800/eclipse-SDK-4.26-linux-gtk-x86_64.tar.gz" } @@ -234,6 +234,7 @@ def get_download_link(self, key: str, version: str) -> str: vars = { "major_version": major_version, + "version": version, "os": os, "arch": arch, "arch_short": arch_short, diff --git a/ci/graal/common.json b/ci/graal/common.json index d54f2650d4..efd7479bab 100644 --- a/ci/graal/common.json +++ b/ci/graal/common.json @@ -4,7 +4,7 @@ "Jsonnet files should not include this file directly but use ci/common.jsonnet instead." ], - "mx_version": "7.87.0", + "mx_version": "7.88.0", "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { @@ -49,12 +49,12 @@ "graalvm-ee-25-ea": {"name": "graalvm-jdk", "version": "25.0.0", "ea": "36", "platformspecific": true }, "oraclejdk-latest": {"name": "jpg-jdk", "version": "25", "build_id": "jdk-25.0.4.1+1", "platformspecific": true, "extrabundles": ["static-libs"]}, - "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.4-b23", "platformspecific": true }, - "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.4-b23-debug", "platformspecific": true }, - "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.4-b23-sulong", "platformspecific": true }, - "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.4-b23", "platformspecific": true }, - "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.4-b23-debug", "platformspecific": true }, - "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.4-b23-sulong", "platformspecific": true } + "labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.5-b24", "platformspecific": true }, + "labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.5-b24-debug", "platformspecific": true }, + "labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-25.0.4.1.1+1-jvmci-25.5-b24-sulong", "platformspecific": true }, + "labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.5-b24", "platformspecific": true }, + "labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.5-b24-debug", "platformspecific": true }, + "labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-25.0.4.1.1+1-jvmci-25.5-b24-sulong", "platformspecific": true } }, "default_jdks" : ["labsjdk-ee-latest", "labsjdk-ce-latest"], diff --git a/docs/user/Tooling.md b/docs/user/Tooling.md index e498423ef9..ecbb141a9e 100644 --- a/docs/user/Tooling.md +++ b/docs/user/Tooling.md @@ -49,7 +49,9 @@ use Truffle's UTF-16 columns, including for non-BMP identifiers. For unexecuted source, attach a source-section load listener before calling the instrument `Env.parse(Source)` API. Retain the returned call target while collecting facts. Reading node objects and their name/location descriptors does not require an entered Python context or guest -values, and supports concurrent readers. Code-unit serialization preserves this metadata. +values, and supports concurrent readers. Names are resolved on demand from the tagged load and store +instructions and the code object's existing name tables, so code units carry no extra metadata and +unmarshaled code reports the same names and locations. This interface describes compiler-emitted accesses, rather than a complete source symbol index. It covers name expressions, assignment and augmented-assignment targets, loop targets and diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java index eda760b817..43cfe519d6 100644 --- a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java +++ b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/runtime/VariableAccessTests.java @@ -57,7 +57,6 @@ import org.graalvm.polyglot.Context; import org.junit.Test; -import com.oracle.graal.python.builtins.objects.code.PCode; import com.oracle.truffle.api.CallTarget; import com.oracle.truffle.api.instrumentation.InstrumentableNode; import com.oracle.truffle.api.instrumentation.SourceSectionFilter; @@ -169,6 +168,63 @@ def method(self, __private): } } + @Test + public void scopesAndNestedAccesses() throws IOException { + try (Context context = Context.newBuilder("python").allowAllAccess(true).build()) { + AnalysisInstrument instrument = context.getEngine().getInstruments().get(ID).lookup(AnalysisInstrument.class); + context.initialize("python"); + context.enter(); + try { + Analysis analysis = instrument.analyze(Source.newBuilder("python", """ + counter = 0 + def bump(): + global counter + counter += 1 + class Holder: + attr = counter + def method(self): + return __class__ + def outer(): + cell = 1 + def inner(): + nonlocal cell + cell += 1 + first = (second := cell) + return inner, first, second + class Child(Holder): + def method(self): + kept = super().method + return lambda: kept + """, "scopes.py").build()); + List accesses = analysis.accesses(); + // module and class bodies: name stores/loads + assertTrue(accesses.contains(new Access("", "counter", true, 1, 1, "counter"))); + assertTrue(accesses.stream().anyMatch(a -> a.name().equals("attr") && a.write() && a.line() == 6 && a.column() == 5)); + assertTrue(accesses.stream().anyMatch(a -> a.name().equals("counter") && !a.write() && a.line() == 6 && a.column() == 12)); + // explicit global, augmented + assertTrue(accesses.contains(new Access("bump", "counter", true, 4, 5, "counter"))); + assertTrue(accesses.contains(new Access("bump", "counter", false, 4, 5, "counter"))); + // free variable read + assertTrue(accesses.contains(new Access("method", "__class__", false, 8, 16, "__class__"))); + // cell store in the defining scope, nonlocal augmented store and load + assertTrue(accesses.contains(new Access("outer", "cell", true, 10, 5, "cell"))); + assertTrue(accesses.contains(new Access("inner", "cell", true, 13, 9, "cell"))); + assertTrue(accesses.contains(new Access("inner", "cell", false, 13, 9, "cell"))); + // a nested walrus store is its own access, not part of the enclosing store + List line14 = accesses.stream().filter(a -> a.line() == 14).toList(); + assertEquals(3, line14.size()); + assertTrue(line14.contains(new Access("outer", "first", true, 14, 5, "first"))); + assertTrue(line14.contains(new Access("outer", "second", true, 14, 14, "second"))); + assertTrue(line14.contains(new Access("outer", "cell", false, 14, 24, "cell"))); + // the store value's implicit loads (__class__, self for super()) are not the access + assertEquals(List.of(new Access("method", "kept", true, 18, 9, "kept"), new Access("method", "super", false, 18, 16, "super")), + accesses.stream().filter(a -> a.line() == 18).toList()); + } finally { + context.leave(); + } + } + } + @Test public void executedTagsPreservePythonBehavior() throws IOException { try (Context context = Context.newBuilder("python").allowAllAccess(true).build()) { @@ -208,7 +264,14 @@ def serialized(arg): return result """, "serialized.py").build(); Analysis analysis = instrument.analyzeSerialized(source); - assertEquals(6, analysis.accesses().stream().filter(a -> a.root().equals("serialized")).count()); + List accesses = analysis.accesses().stream().filter(a -> a.root().equals("serialized")).toList(); + assertEquals(6, accesses.size()); + // The original and the unmarshaled code report the same names and locations, which + // are derived from the serialized bytecode and its source information. + for (Access access : accesses) { + assertEquals(2, accesses.stream().filter(a -> a.name().equals(access.name()) && a.write() == access.write() && a.line() == access.line() && + a.column() == access.column()).count()); + } } finally { context.leave(); } @@ -309,9 +372,6 @@ private Analysis analyze(Source source, boolean execute, boolean serialized) thr cloned = types.FunctionType(marshal.loads(marshal.dumps(serialized.__code__)), globals()) assert cloned(42) == 42 """, "roundtrip.py").build()).call(); - PCode original = (PCode) env.parse(Source.newBuilder("python", "serialized.__code__", "original-code.py").build()).call(); - PCode cloned = (PCode) env.parse(Source.newBuilder("python", "cloned.__code__", "cloned-code.py").build()).call(); - assertEquals(original.getCodeUnit().variableAccesses, cloned.getCodeUnit().variableAccesses); } } finally { binding.dispose(); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java index a76a352a27..14b8795283 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java @@ -115,7 +115,6 @@ import com.oracle.graal.python.nodes.PRaiseNode; import com.oracle.graal.python.nodes.StringLiterals; import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit; -import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit.VariableAccess; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNodeGen; import com.oracle.graal.python.nodes.call.special.LookupAndCallBinaryNode; @@ -1429,16 +1428,10 @@ private BytecodeDSLCodeUnit readBytecodeDSLCodeUnit() { int yieldFromGeneratorIndex = readInt(); int instrumentationDataIndex = readInt(); int maxProfileCEventStackSize = readInt(); - int accessCount = readSize(); - List variableAccesses = new ArrayList<>(accessCount); - for (int i = 0; i < accessCount; i++) { - variableAccesses.add(new VariableAccess(readString(true).toJavaStringUncached(), readInt(), readInt(), readInt(), readInt(), readInt() != 0)); - } BytecodeSupplier provider = new BytecodeSupplier(serialized, bytecodeFile, sourceReference, bytecodeOffset, bytecodeSize, cacheKey); return new BytecodeDSLCodeUnit(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, constants, - startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, variableAccesses, - provider); + startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, provider); } private void writeCodeUnit(BytecodeDSLCodeUnit code) throws IOException { @@ -1474,15 +1467,6 @@ private void writeBytecodeDSLCodeUnit(BytecodeDSLCodeUnit code) throws IOExcepti writeInt(code.yieldFromGeneratorIndex); writeInt(code.instrumentationDataIndex); writeInt(code.maxProfileCEventStackSize); - writeInt(code.variableAccesses.size()); - for (VariableAccess access : code.variableAccesses) { - writeString(PythonUtils.toTruffleStringUncached(access.name())); - writeInt(access.startLine()); - writeInt(access.startColumn()); - writeInt(access.endLine()); - writeInt(access.endColumn()); - writeInt(access.write() ? 1 : 0); - } } private PCode readCode() { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/slots/TpSlotDescrGet.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/slots/TpSlotDescrGet.java index f7db105bcf..bea97f3247 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/slots/TpSlotDescrGet.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/slots/TpSlotDescrGet.java @@ -101,7 +101,7 @@ private TpSlotDescrGet() { } public abstract static sealed class TpSlotDescrGetBuiltin extends TpSlotBuiltin// - permits TpSlotDescrGetBuiltinComplex, TpSlotDescrGetBuiltinSimple { + permits TpSlotDescrGetBuiltinComplex, TpSlotDescrGetBuiltinSimple { static final BuiltinSlotWrapperSignature SIGNATURE = BuiltinSlotWrapperSignature.of(2, J_DOLLAR_SELF, "obj", "type"); static final PExternalFunctionWrapper WRAPPER = PExternalFunctionWrapper.DESCR_GET; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java index 407f553b30..47c9be0845 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java @@ -71,7 +71,7 @@ public class BytecodeDSLCompiler { * Bump this with every Truffle import change because serialized GraalPy bytecode effectively * depends on Truffle bytecode internals. */ - public static final int BYTECODE_VERSION = 36; + public static final int BYTECODE_VERSION = 37; public static final record BytecodeDSLCompilerResult(PBytecodeDSLRootNode rootNode, BytecodeDSLCodeUnit codeUnit) { } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java index 8b42358512..58f7f87f14 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java @@ -82,7 +82,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; @@ -107,7 +106,6 @@ import com.oracle.graal.python.lib.PyObjectRichCompareBool; import com.oracle.graal.python.nodes.StringLiterals; import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit; -import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit.VariableAccess; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNodeGen; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNodeGen.Builder; @@ -217,7 +215,6 @@ public final class RootNodeCompiler implements BaseBytecodeDSLVisitor variableAccesses = new LinkedHashSet<>(); private final HashMap varnames; private final HashMap cellvars; private final HashMap freevars; @@ -259,6 +256,8 @@ public final class RootNodeCompiler implements BaseBytecodeDSLVisitor statementLocations = new ArrayDeque<>(); + /** Source ranges of the currently open source sections, innermost first. */ + private final Deque openSourceRanges = new ArrayDeque<>(); private SourceRange currentLocation; BytecodeLocal yieldFromGenerator; private int lastTracedLine; @@ -595,7 +594,6 @@ flags, orderedTruffleStringArray(names), yieldFromGenerator != null ? yieldFromGenerator.getLocalIndex() : -1, instrumentationDataLocal.getLocalIndex(), maxProfileCEventStackSize, - List.copyOf(variableAccesses), new BytecodeSupplier(nodes)); ctx.codeUnits.put(key, codeUnit); } @@ -769,6 +767,7 @@ public BytecodeDSLCompilerResult compile() { public void reset() { statementLocations.clear(); + openSourceRanges.clear(); currentLocation = null; currentSaveExceptionLocal = null; prevSaveExceptionLocal = null; @@ -950,7 +949,8 @@ private static SourceRange getRootSourceRange(SSTNode node) { } } - private static void beginSourceSectionInner(Builder b, SourceRange sourceRange) { + private void beginSourceSectionInner(Builder b, SourceRange sourceRange) { + openSourceRanges.push(sourceRange); if (sourceRange.startLine >= 1 && sourceRange != SourceRange.ARTIFICIAL_RANGE) { if (sourceRange.startColumn >= 0 && sourceRange.endLine >= sourceRange.startLine && sourceRange.endColumn >= 0) { int startColumn = sourceRange.startColumn + 1; @@ -975,6 +975,7 @@ private static void beginSourceSectionInner(Builder b, SourceRange sourceRange) void endSourceSection(Builder b) { b.endSourceSection(); + openSourceRanges.pop(); } void beginStatement(SSTNode node, Builder b) { @@ -1911,51 +1912,72 @@ private SourceRange variableSourceRange(SourceRange range) { } } - private boolean recordVariableAccess(ExprTy.Name node, boolean write) { - SourceRange range = variableSourceRange(node); - if (node.id.startsWith(".") || range == SourceRange.ARTIFICIAL_RANGE || range.startLine < 1 || range.startColumn < 0 || - range.endLine < range.startLine || range.endColumn < 1 || !ctx.source.hasCharacters()) { - return false; + /** + * Returns the identifier range of a source-level variable access in Truffle (UTF-16) columns, + * or {@code null} if the access has no usable source token, e.g. compiler-generated names and + * AST-created code without valid positions. Such accesses are not tagged. + */ + private SourceRange variableAccessRange(ExprTy.Name node) { + if (node.id.startsWith(".")) { + return null; } - try { - ctx.source.createSection(range.startLine, range.startColumn + 1, range.endLine, range.endColumn); - } catch (IllegalArgumentException e) { - // AST-created code and incomplete source ranges need not identify a source token. - return false; + SourceRange range = variableSourceRange(node); + if (range == SourceRange.ARTIFICIAL_RANGE || range.startLine < 1 || range.startColumn < 0 || range.endLine < range.startLine || + range.endLine == range.startLine && range.endColumn <= range.startColumn) { + return null; } - variableAccesses.add(new VariableAccess(maybeMangle(node.id), range.startLine, range.startColumn, range.endLine, range.endColumn, write)); - return true; + return range; } private boolean beginVariableAccess(ExprTy.Name node, boolean write, Builder b) { - if (!recordVariableAccess(node, write)) { + SourceRange range = variableAccessRange(node); + if (range == null) { return false; } - beginVariableTag(variableSourceRange(node), write, b); + beginVariableTag(write, b, range); return true; } private boolean beginDirectVariableWrite(ExprTy target, BytecodeLocal local, Builder b) { - if (!isTemporaryLocal(local) && target instanceof ExprTy.Name name && recordVariableAccess(name, true)) { - beginVariableTag(variableSourceRange(target), true, b); - return true; + if (!isTemporaryLocal(local) && target instanceof ExprTy.Name name) { + return beginVariableAccess(name, true, b); } return false; } - private void beginVariableTag(SourceRange accessRange, boolean write, Builder b) { - beginSourceSection(accessRange, b); - // Bytecode DSL stack walks select the innermost tag. Keep its program location at the - // statement (or expression root), while the outer source layer identifies the access. + /** + * Opens a variable tag. The names are not recorded here: instrumentation resolves them lazily + * from the operands of the tagged instructions (see {@code VariableAccessDescriptor}). + *

+ * Layout, from outermost: one identifier range per variable operand of the tagged store or + * load (in reverse operand order), the enclosing statement, the tag, and then the source range + * that was innermost before the tag. The tag's own source sections are therefore, from most to + * least concrete, the statement followed by the identifiers in operand order. Bytecode DSL + * stack walks select the innermost tag, so debugger caller locations stay at the statement. + * The tagged instructions keep their previous innermost location, so exception locations, + * tracebacks and line tables are unaffected by these layers. + */ + private void beginVariableTag(boolean write, Builder b, SourceRange... accessRanges) { + SourceRange enclosing = openSourceRanges.isEmpty() ? SourceRange.ARTIFICIAL_RANGE : openSourceRanges.peek(); + for (int i = accessRanges.length - 1; i >= 0; i--) { + beginSourceSectionInner(b, accessRanges[i]); + } SourceRange statement = statementLocations.peek(); - beginSourceSection(variableSourceRange(statement != null ? statement : getRootSourceRange(startNode)), b); + beginSourceSectionInner(b, variableSourceRange(statement != null ? statement : getRootSourceRange(startNode))); b.beginTag(write ? WriteVariableTag.class : ReadVariableTag.class); + beginSourceSectionInner(b, enclosing); } - private void endVariableTag(boolean write, Builder b) { - b.endTag(write ? WriteVariableTag.class : ReadVariableTag.class); - endSourceSection(b); + private void endVariableTag(boolean write, int accessCount, Builder b) { endSourceSection(b); + b.endTag(write ? WriteVariableTag.class : ReadVariableTag.class); + for (int i = 0; i <= accessCount; i++) { + endSourceSection(b); + } + } + + private void endVariableTag(boolean write, Builder b) { + endVariableTag(write, 1, b); } private void endDirectVariableWrite(boolean tagged, Builder b) { @@ -3866,21 +3888,19 @@ private void visitIterableAssign(ExprTy[] nodes) { if (nodes.length == 2 && containsNoStarred(nodes)) { BytecodeLocal target1 = allowFastLocalLookup ? beginTemporaryLocalOrGetLocal(nodes[0], b) : beginTemporaryLocal(); BytecodeLocal target2 = allowFastLocalLookup ? beginTemporaryLocalOrGetLocal(nodes[1], b) : beginTemporaryLocal(); - boolean tagged = false; - if (!isTemporaryLocal(target1)) { - tagged |= recordVariableAccess((ExprTy.Name) nodes[0], true); - } - if (!isTemporaryLocal(target2)) { - tagged |= recordVariableAccess((ExprTy.Name) nodes[1], true); - } + // One instruction stores both targets; give each operand its own identifier + // layer. Temporary targets are tagged when they are stored to the real target. + SourceRange range1 = isTemporaryLocal(target1) ? null : variableAccessRange((ExprTy.Name) nodes[0]); + SourceRange range2 = isTemporaryLocal(target2) ? null : variableAccessRange((ExprTy.Name) nodes[1]); + boolean tagged = range1 != null || range2 != null; if (tagged) { - beginVariableTag(variableSourceRange(nodes[0]).withEnd(variableSourceRange(nodes[1])), true, b); + beginVariableTag(true, b, range1 != null ? range1 : SourceRange.ARTIFICIAL_RANGE, range2 != null ? range2 : SourceRange.ARTIFICIAL_RANGE); } b.beginUnpackToLocals2(target1, target2); generateValue.run(); b.endUnpackToLocals2(); if (tagged) { - endVariableTag(true, b); + endVariableTag(true, 2, b); } storeTemporaryLocalToTarget(target1, nodes[0], b, allowFastLocalLookup); storeTemporaryLocalToTarget(target2, nodes[1], b, allowFastLocalLookup); @@ -4040,12 +4060,11 @@ public Void visit(ExprTy.Name node) { beginStoreLocal(node.id, b); beginAugAssign(); if (tagged) { - recordVariableAccess(node, false); - b.beginTag(ReadVariableTag.class); + beginVariableTag(false, b, variableAccessRange(node)); } emitReadLocal(node.id, b); if (tagged) { - b.endTag(ReadVariableTag.class); + endVariableTag(false, b); } value.accept(StatementCompiler.this); endAugAssign(); @@ -4881,7 +4900,7 @@ public void endWrapWithDecorators(ExprTy[] decorators) { b.endTraceLineWithArgument(decorators[decorators.length - 1 - i].getSourceRange().startLine); b.endInstrumentCall(); endCallUnaryMethod(); - b.endSourceSection(); + endSourceSection(b); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java index e5dd910c2d..fc8c32e0ea 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/BytecodeDSLCodeUnit.java @@ -78,21 +78,6 @@ public final class BytecodeDSLCodeUnit { @CompilationFinal(dimensions = 1) public final TruffleString[] freevars; @CompilationFinal(dimensions = 1) public final Object[] constants; - /** Source-level variable accesses, independent of bytecode indices and live frames. */ - public final List variableAccesses; - - public record VariableAccess(String name, int startLine, int startColumn, int endLine, int endColumn, boolean write) { - public boolean isWithin(SourceSection section) { - return section.isAvailable() && - (startLine > section.getStartLine() || startLine == section.getStartLine() && startColumn + 1 >= section.getStartColumn()) && - (endLine < section.getEndLine() || endLine == section.getEndLine() && endColumn <= section.getEndColumn()); - } - - public SourceSection getSourceSection(Source source) { - return source.createSection(startLine, startColumn + 1, endLine, endColumn); - } - } - public final int startLine; public final int startColumn; public final int endLine; @@ -107,8 +92,7 @@ public SourceSection getSourceSection(Source source) { public BytecodeDSLCodeUnit(TruffleString name, TruffleString qualname, int argCount, int kwOnlyArgCount, int positionalOnlyArgCount, int flags, TruffleString[] names, TruffleString[] varnames, TruffleString[] cellvars, TruffleString[] freevars, Object[] constants, int startLine, int startColumn, int endLine, int endColumn, - int classcellIndex, int selfIndex, int yieldFromGeneratorIndex, int instrumentationDataIndex, int maxProfileCEventStackSize, List variableAccesses, - BytecodeSupplier supplier) { + int classcellIndex, int selfIndex, int yieldFromGeneratorIndex, int instrumentationDataIndex, int maxProfileCEventStackSize, BytecodeSupplier supplier) { assert isInterned(name); this.name = name; assert qualname == null || isInterned(qualname); @@ -136,7 +120,6 @@ public BytecodeDSLCodeUnit(TruffleString name, TruffleString qualname, int argCo this.yieldFromGeneratorIndex = yieldFromGeneratorIndex; this.instrumentationDataIndex = instrumentationDataIndex; this.maxProfileCEventStackSize = maxProfileCEventStackSize; - this.variableAccesses = List.copyOf(variableAccesses); } public abstract static class BytecodeSupplier { @@ -148,8 +131,7 @@ public abstract static class BytecodeSupplier { public BytecodeDSLCodeUnit withFlags(int flags) { return new BytecodeDSLCodeUnit(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, constants, - startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, variableAccesses, - supplier); + startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, supplier); } @TruffleBoundary diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java index a820013afe..c24c21bddb 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java @@ -288,8 +288,6 @@ import com.oracle.truffle.api.bytecode.BytecodeLocation; import com.oracle.truffle.api.bytecode.BytecodeNode; import com.oracle.truffle.api.bytecode.TagTreeNode; -import com.oracle.truffle.api.instrumentation.StandardTags.ReadVariableTag; -import com.oracle.truffle.api.instrumentation.StandardTags.WriteVariableTag; import com.oracle.truffle.api.bytecode.BytecodeRootNode; import com.oracle.truffle.api.bytecode.ConstantOperand; import com.oracle.truffle.api.bytecode.ContinuationResult; @@ -504,12 +502,7 @@ public Object getInstrumentationNodeObject(TagTreeNode node) { if (co == null) { return null; } - if (node.hasTag(ReadVariableTag.class)) { - return VariableAccessDescriptor.create(ReadVariableTag.NAME, false, node.getSourceSections(), co.variableAccesses); - } else if (node.hasTag(WriteVariableTag.class)) { - return VariableAccessDescriptor.create(WriteVariableTag.NAME, true, node.getSourceSections(), co.variableAccesses); - } - return null; + return VariableAccessDescriptor.create(node, co); } public final PythonLanguage getLanguage() { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java index 780a4561d0..06d93a9cc2 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/instrumentation/VariableAccessDescriptor.java @@ -44,8 +44,26 @@ import java.util.ArrayList; import java.util.List; -import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit.VariableAccess; +import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.ForIterate; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.LoadCell; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.LoadFromDictOrCell; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.LoadFromDictOrGlobals; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.ReadName; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.StoreCell; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.UnpackToLocals2; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.WriteGlobal; +import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode.WriteName; +import com.oracle.graal.python.nodes.frame.ReadGlobalOrBuiltinNode; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.bytecode.BytecodeNode; +import com.oracle.truffle.api.bytecode.Instruction; +import com.oracle.truffle.api.bytecode.Instruction.Argument; +import com.oracle.truffle.api.bytecode.LocalAccessor; +import com.oracle.truffle.api.bytecode.TagTree; +import com.oracle.truffle.api.bytecode.TagTreeNode; +import com.oracle.truffle.api.instrumentation.StandardTags.ReadVariableTag; +import com.oracle.truffle.api.instrumentation.StandardTags.WriteVariableTag; import com.oracle.truffle.api.interop.InteropLibrary; import com.oracle.truffle.api.interop.InvalidArrayIndexException; import com.oracle.truffle.api.interop.TruffleObject; @@ -54,8 +72,18 @@ import com.oracle.truffle.api.library.ExportLibrary; import com.oracle.truffle.api.library.ExportMessage; import com.oracle.truffle.api.source.SourceSection; +import com.oracle.truffle.api.strings.TruffleString; -/** Immutable compiler metadata; safe to query without a language context or the GIL. */ +/** + * Node object of a {@link ReadVariableTag} or {@link WriteVariableTag} region. Nothing is recorded + * for it at compile time: the names are resolved on demand from the operands of the load or store + * instruction in the tag's bytecode range, against the names the code unit already holds + * ({@code varnames}, {@code cellvars}, {@code freevars} and name constants). The identifier + * locations are the tag's source layers emitted by the compiler: the enclosing statement first, + * then one identifier range per variable operand. + *

+ * Instances are immutable and safe to create and query without a language context or the GIL. + */ @ExportLibrary(InteropLibrary.class) public final class VariableAccessDescriptor implements TruffleObject { private final String property; @@ -67,38 +95,176 @@ private VariableAccessDescriptor(String property, Object names) { } @TruffleBoundary - public static Object create(String property, boolean write, SourceSection[] sections, List accesses) { - SourceSection section = null; - for (SourceSection candidate : sections) { - if (candidate == null || !candidate.isAvailable()) { + public static Object create(TagTreeNode node, BytecodeDSLCodeUnit code) { + boolean write; + if (node.hasTag(WriteVariableTag.class)) { + write = true; + } else if (node.hasTag(ReadVariableTag.class)) { + write = false; + } else { + return null; + } + BytecodeNode bytecode = node.getBytecodeNode(); + if (bytecode == null) { + return null; + } + List nested = new ArrayList<>(); + collectNestedVariableTags(node, nested); + + // Operands are emitted before the operation that consumes them, so the access is the last + // load or store of this region outside nested variable tags: a custom name operation, or + // a built-in instruction with a local argument (load.local/store.local). Instructions are + // identified by operation class and arguments, never by their debug names. Cell stores + // receive the cell from the region's first built-in local load. + Instruction access = null; + Instruction firstLocalAccess = null; + int returnIndex = node.getReturnBytecodeIndex(); + for (int bci = node.getEnterBytecodeIndex(); bci <= returnIndex;) { + TagTree skip = enclosing(nested, bci); + if (skip != null) { + bci = bytecode.getInstruction(skip.getReturnBytecodeIndex()).getNextBytecodeIndex(); continue; } - for (VariableAccess access : accesses) { - if (access.write() == write && access.isWithin(candidate) && (section == null || isWithin(candidate, section))) { - section = candidate; - break; + Instruction instruction = bytecode.getInstruction(bci); + Class operation = instruction.getDescriptor().getOperationClass(); + if (operation == null) { + if (localSlot(instruction) >= 0) { + if (firstLocalAccess == null) { + firstLocalAccess = instruction; + } + access = instruction; } + } else if (isAccessOperation(operation, write)) { + access = instruction; } + bci = instruction.getNextBytecodeIndex(); } - if (section == null) { + if (access == null) { return null; } - List names = new ArrayList<>(); - for (VariableAccess access : accesses) { - if (access.write() == write && access.isWithin(section)) { - names.add(new VariableName(access.name(), access.getSourceSection(section.getSource()))); + + String[] accessNames = resolveNames(code, bytecode, access, firstLocalAccess); + SourceSection[] sections = node.getSourceSections(); + List result = new ArrayList<>(accessNames.length); + for (int i = 0; i < accessNames.length; i++) { + if (accessNames[i] != null) { + // Layer 0 is the statement; identifier layers follow in operand order. + SourceSection section = sections != null && i + 1 < sections.length ? sections[i + 1] : null; + result.add(new VariableName(accessNames[i], section)); } } - if (names.isEmpty()) { + if (result.isEmpty()) { return null; } - Object value = names.size() == 1 ? names.get(0) : new NameArray(names.toArray()); - return new VariableAccessDescriptor(property, value); + Object value = result.size() == 1 ? result.get(0) : new NameArray(result.toArray()); + return new VariableAccessDescriptor(write ? WriteVariableTag.NAME : ReadVariableTag.NAME, value); } - private static boolean isWithin(SourceSection candidate, SourceSection outer) { - return (candidate.getStartLine() > outer.getStartLine() || candidate.getStartLine() == outer.getStartLine() && candidate.getStartColumn() >= outer.getStartColumn()) && - (candidate.getEndLine() < outer.getEndLine() || candidate.getEndLine() == outer.getEndLine() && candidate.getEndColumn() <= outer.getEndColumn()); + private static void collectNestedVariableTags(TagTree tree, List result) { + for (TagTree child : tree.getTreeChildren()) { + if (child.hasTag(ReadVariableTag.class) || child.hasTag(WriteVariableTag.class)) { + result.add(child); + } else { + collectNestedVariableTags(child, result); + } + } + } + + private static TagTree enclosing(List trees, int bci) { + for (TagTree tree : trees) { + if (bci >= tree.getEnterBytecodeIndex() && bci <= tree.getReturnBytecodeIndex()) { + return tree; + } + } + return null; + } + + private static boolean isAccessOperation(Class operation, boolean write) { + if (write) { + return operation == StoreCell.class || operation == WriteGlobal.class || operation == WriteName.class || operation == ForIterate.class || + operation == UnpackToLocals2.class; + } else { + return operation == LoadCell.class || operation == LoadFromDictOrCell.class || operation == ReadGlobalOrBuiltinNode.class || + operation == LoadFromDictOrGlobals.class || operation == ReadName.class; + } + } + + /** The frame slot of a built-in local load or store, or -1. */ + private static int localSlot(Instruction instruction) { + for (Argument argument : instruction.getArguments()) { + // Python disables block scoping, so local offsets and local indices coincide. + switch (argument.getKind()) { + case LOCAL_OFFSET: + return argument.asLocalOffset(); + case LOCAL_INDEX: + return argument.asLocalIndex(); + default: + break; + } + } + return -1; + } + + /** One entry per variable operand of the access; {@code null} for compiler temporaries. */ + private static String[] resolveNames(BytecodeDSLCodeUnit code, BytecodeNode bytecode, Instruction access, Instruction firstLocalAccess) { + Class operation = access.getDescriptor().getOperationClass(); + if (operation == null) { + return new String[]{slotName(code, localSlot(access))}; + } else if (operation == StoreCell.class) { + return firstLocalAccess == null ? new String[0] : new String[]{slotName(code, localSlot(firstLocalAccess))}; + } else if (operation == LoadCell.class || operation == LoadFromDictOrCell.class) { + for (Argument argument : access.getArguments()) { + Integer index = intArgument(argument); + if (index != null) { + return new String[]{cellOrFreeName(code, index)}; + } + } + return new String[0]; + } + // Global and name operations carry the name; ForIterate and UnpackToLocals2 take local + // accessors, whose info is the varnames index. + List names = new ArrayList<>(2); + for (Argument argument : access.getArguments()) { + if (argument.getKind() != Argument.Kind.CONSTANT) { + continue; + } + Object constant = argument.asConstant(); + if (constant instanceof TruffleString name) { + return new String[]{name.toJavaStringUncached()}; + } else if (constant instanceof LocalAccessor local) { + names.add(local.getLocalInfo(bytecode) instanceof Integer index && index < code.varnames.length ? code.varnames[index].toJavaStringUncached() : null); + } + } + return names.toArray(new String[0]); + } + + private static Integer intArgument(Argument argument) { + return switch (argument.getKind()) { + case INTEGER -> argument.asInteger(); + case CONSTANT -> argument.asConstant() instanceof Integer i ? i : null; + default -> null; + }; + } + + /** Frame slot layout: varnames, then cellvars, then freevars, then compiler temporaries. */ + private static String slotName(BytecodeDSLCodeUnit code, int slot) { + if (slot < 0) { + return null; + } else if (slot < code.varnames.length) { + return code.varnames[slot].toJavaStringUncached(); + } + return cellOrFreeName(code, slot - code.varnames.length); + } + + private static String cellOrFreeName(BytecodeDSLCodeUnit code, int index) { + if (index < 0) { + return null; + } else if (index < code.cellvars.length) { + return code.cellvars[index].toJavaStringUncached(); + } else if (index - code.cellvars.length < code.freevars.length) { + return code.freevars[index - code.cellvars.length].toJavaStringUncached(); + } + return null; } @ExportMessage diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 4eb28ea78b..25b3bfb37e 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -53,7 +53,7 @@ }, { "name": "tools", - "version": "fe7f970d99b666853ebe7abd14f14328aec29118", + "version": "67e12a237ab29b0f8dc665c9388a33a96719f4a0", "subdir": True, "urls": [ {"url": "https://github.com/elide-dev/graal", "kind": "git"}, @@ -61,7 +61,7 @@ }, { "name": "regex", - "version": "fe7f970d99b666853ebe7abd14f14328aec29118", + "version": "67e12a237ab29b0f8dc665c9388a33a96719f4a0", "subdir": True, "urls": [ {"url": "https://github.com/elide-dev/graal", "kind": "git"},