Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/user/Tooling.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ use Truffle's UTF-16 columns, including for non-BMP identifiers.
For unexecuted source, attach a source-section load listener before calling the instrument
`Env.parse(Source)` API. Retain the returned call target while collecting facts. Reading node
objects and their name/location descriptors does not require an entered Python context or guest
values, and supports concurrent readers. Code-unit serialization preserves this metadata.
values, and supports concurrent readers. Names are resolved on demand from the tagged load and store
instructions and the code object's existing name tables, so code units carry no extra metadata and
unmarshaled code reports the same names and locations.

This interface describes compiler-emitted accesses, rather than a complete source symbol index.
It covers name expressions, assignment and augmented-assignment targets, loop targets and
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@
import org.graalvm.polyglot.Context;
import org.junit.Test;

import com.oracle.graal.python.builtins.objects.code.PCode;
import com.oracle.truffle.api.CallTarget;
import com.oracle.truffle.api.instrumentation.InstrumentableNode;
import com.oracle.truffle.api.instrumentation.SourceSectionFilter;
Expand Down Expand Up @@ -169,6 +168,63 @@ def method(self, __private):
}
}

@Test
public void scopesAndNestedAccesses() throws IOException {
try (Context context = Context.newBuilder("python").allowAllAccess(true).build()) {
AnalysisInstrument instrument = context.getEngine().getInstruments().get(ID).lookup(AnalysisInstrument.class);
context.initialize("python");
context.enter();
try {
Analysis analysis = instrument.analyze(Source.newBuilder("python", """
counter = 0
def bump():
global counter
counter += 1
class Holder:
attr = counter
def method(self):
return __class__
def outer():
cell = 1
def inner():
nonlocal cell
cell += 1
first = (second := cell)
return inner, first, second
class Child(Holder):
def method(self):
kept = super().method
return lambda: kept
""", "scopes.py").build());
List<Access> accesses = analysis.accesses();
// module and class bodies: name stores/loads
assertTrue(accesses.contains(new Access("<module>", "counter", true, 1, 1, "counter")));
assertTrue(accesses.stream().anyMatch(a -> a.name().equals("attr") && a.write() && a.line() == 6 && a.column() == 5));
assertTrue(accesses.stream().anyMatch(a -> a.name().equals("counter") && !a.write() && a.line() == 6 && a.column() == 12));
// explicit global, augmented
assertTrue(accesses.contains(new Access("bump", "counter", true, 4, 5, "counter")));
assertTrue(accesses.contains(new Access("bump", "counter", false, 4, 5, "counter")));
// free variable read
assertTrue(accesses.contains(new Access("method", "__class__", false, 8, 16, "__class__")));
// cell store in the defining scope, nonlocal augmented store and load
assertTrue(accesses.contains(new Access("outer", "cell", true, 10, 5, "cell")));
assertTrue(accesses.contains(new Access("inner", "cell", true, 13, 9, "cell")));
assertTrue(accesses.contains(new Access("inner", "cell", false, 13, 9, "cell")));
// a nested walrus store is its own access, not part of the enclosing store
List<Access> line14 = accesses.stream().filter(a -> a.line() == 14).toList();
assertEquals(3, line14.size());
assertTrue(line14.contains(new Access("outer", "first", true, 14, 5, "first")));
assertTrue(line14.contains(new Access("outer", "second", true, 14, 14, "second")));
assertTrue(line14.contains(new Access("outer", "cell", false, 14, 24, "cell")));
// the store value's implicit loads (__class__, self for super()) are not the access
assertEquals(List.of(new Access("method", "kept", true, 18, 9, "kept"), new Access("method", "super", false, 18, 16, "super")),
accesses.stream().filter(a -> a.line() == 18).toList());
} finally {
context.leave();
}
}
}

@Test
public void executedTagsPreservePythonBehavior() throws IOException {
try (Context context = Context.newBuilder("python").allowAllAccess(true).build()) {
Expand Down Expand Up @@ -208,7 +264,14 @@ def serialized(arg):
return result
""", "serialized.py").build();
Analysis analysis = instrument.analyzeSerialized(source);
assertEquals(6, analysis.accesses().stream().filter(a -> a.root().equals("serialized")).count());
List<Access> accesses = analysis.accesses().stream().filter(a -> a.root().equals("serialized")).toList();
assertEquals(6, accesses.size());
// The original and the unmarshaled code report the same names and locations, which
// are derived from the serialized bytecode and its source information.
for (Access access : accesses) {
assertEquals(2, accesses.stream().filter(a -> a.name().equals(access.name()) && a.write() == access.write() && a.line() == access.line() &&
a.column() == access.column()).count());
}
} finally {
context.leave();
}
Expand Down Expand Up @@ -309,9 +372,6 @@ private Analysis analyze(Source source, boolean execute, boolean serialized) thr
cloned = types.FunctionType(marshal.loads(marshal.dumps(serialized.__code__)), globals())
assert cloned(42) == 42
""", "roundtrip.py").build()).call();
PCode original = (PCode) env.parse(Source.newBuilder("python", "serialized.__code__", "original-code.py").build()).call();
PCode cloned = (PCode) env.parse(Source.newBuilder("python", "cloned.__code__", "cloned-code.py").build()).call();
assertEquals(original.getCodeUnit().variableAccesses, cloned.getCodeUnit().variableAccesses);
}
} finally {
binding.dispose();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,6 @@
import com.oracle.graal.python.nodes.PRaiseNode;
import com.oracle.graal.python.nodes.StringLiterals;
import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit;
import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit.VariableAccess;
import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode;
import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNodeGen;
import com.oracle.graal.python.nodes.call.special.LookupAndCallBinaryNode;
Expand Down Expand Up @@ -1429,16 +1428,10 @@ private BytecodeDSLCodeUnit readBytecodeDSLCodeUnit() {
int yieldFromGeneratorIndex = readInt();
int instrumentationDataIndex = readInt();
int maxProfileCEventStackSize = readInt();
int accessCount = readSize();
List<VariableAccess> variableAccesses = new ArrayList<>(accessCount);
for (int i = 0; i < accessCount; i++) {
variableAccesses.add(new VariableAccess(readString(true).toJavaStringUncached(), readInt(), readInt(), readInt(), readInt(), readInt() != 0));
}

BytecodeSupplier provider = new BytecodeSupplier(serialized, bytecodeFile, sourceReference, bytecodeOffset, bytecodeSize, cacheKey);
return new BytecodeDSLCodeUnit(name, qualname, argCount, kwOnlyArgCount, positionalOnlyArgCount, flags, names, varnames, cellvars, freevars, constants,
startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, variableAccesses,
provider);
startLine, startColumn, endLine, endColumn, classcellIndex, selfIndex, yieldFromGeneratorIndex, instrumentationDataIndex, maxProfileCEventStackSize, provider);
}

private void writeCodeUnit(BytecodeDSLCodeUnit code) throws IOException {
Expand Down Expand Up @@ -1474,15 +1467,6 @@ private void writeBytecodeDSLCodeUnit(BytecodeDSLCodeUnit code) throws IOExcepti
writeInt(code.yieldFromGeneratorIndex);
writeInt(code.instrumentationDataIndex);
writeInt(code.maxProfileCEventStackSize);
writeInt(code.variableAccesses.size());
for (VariableAccess access : code.variableAccesses) {
writeString(PythonUtils.toTruffleStringUncached(access.name()));
writeInt(access.startLine());
writeInt(access.startColumn());
writeInt(access.endLine());
writeInt(access.endColumn());
writeInt(access.write() ? 1 : 0);
}
}

private PCode readCode() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public class BytecodeDSLCompiler {
* Bump this with every Truffle import change because serialized GraalPy bytecode effectively
* depends on Truffle bytecode internals.
*/
public static final int BYTECODE_VERSION = 38;
public static final int BYTECODE_VERSION = 39;

public static final record BytecodeDSLCompilerResult(PBytecodeDSLRootNode rootNode, BytecodeDSLCodeUnit codeUnit) {
}
Expand Down
Loading
Loading