Skip to content

Commit fc38d93

Browse files
msimacekmsimacek
committed
Unify gi_frame and the actual frame
1 parent 4dce2de commit fc38d93

File tree

11 files changed

+188
-105
lines changed

11 files changed

+188
-105
lines changed

‎graalpython/com.oracle.graal.python.test/src/tests/test_generators.py‎

Lines changed: 66 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# Copyright (C) 1996-2017 Python Software Foundation
33
#
44
# Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
5+
import itertools
56
import sys
67
import os
78
import unittest
@@ -509,42 +510,89 @@ def b():
509510
assert gen_b.gi_yieldfrom is None
510511

511512

513+
def _test_generator_frame(checks):
514+
def gen():
515+
a = 1
516+
yield
517+
b = 2
518+
yield
519+
c = 3
520+
521+
g = gen()
522+
f = None
523+
if checks[0]:
524+
f = g.gi_frame
525+
assert f
526+
assert f.f_back is None
527+
assert f.f_globals == globals()
528+
assert f.f_locals == {}
529+
assert f.f_lineno == gen.__code__.co_firstlineno
530+
next(g)
531+
if checks[1]:
532+
if not f:
533+
f = g.gi_frame
534+
assert f
535+
assert f.f_back is None
536+
assert f.f_globals == globals()
537+
assert f.f_locals == {'a': 1}
538+
assert f.f_lineno == gen.__code__.co_firstlineno + 2
539+
assert f is g.gi_frame
540+
next(g)
541+
if checks[2]:
542+
if not f:
543+
f = g.gi_frame
544+
assert f
545+
assert f.f_back is None
546+
assert f.f_globals == globals()
547+
assert f.f_locals == {'a': 1, 'b': 2}
548+
assert f.f_lineno == gen.__code__.co_firstlineno + 4
549+
assert f is g.gi_frame
550+
try:
551+
next(g)
552+
assert False
553+
except StopIteration:
554+
pass
555+
if f:
556+
assert f.f_back is sys._getframe()
557+
assert f.f_globals is globals()
558+
assert f.f_locals == {'a': 1, 'b': 2, 'c': 3}
559+
# TODO GR-61955
560+
if sys.implementation.name != "graalpy" or not __graalpython__.is_bytecode_dsl_interpreter:
561+
assert f.f_lineno == gen.__code__.co_firstlineno + 5
562+
assert not g.gi_frame
563+
564+
512565
def test_generator_frame():
566+
# Generte a whole matrix of possibilities of where the frame gets created vs re-synced
567+
for checks in itertools.product([False, True], repeat=3):
568+
_test_generator_frame(checks)
569+
570+
571+
def test_generator_frame_from_getframe():
513572
def gen():
514573
a = 1
515574
yield sys._getframe()
516575
b = 2
517576

518577
g = gen()
519-
assert not g.gi_running
520-
assert not g.gi_suspended
521-
assert g.gi_frame
522-
assert g.gi_frame.f_back is None
523-
assert g.gi_frame.f_globals == globals()
524-
assert g.gi_frame.f_locals == {}
525-
assert g.gi_frame.f_lineno == gen.__code__.co_firstlineno
526578
f = next(g)
527579
assert f
528-
assert not g.gi_running
529-
assert g.gi_suspended
530-
assert f is g.gi_frame
531580
assert f.f_back is None
532581
assert f.f_globals == globals()
533-
assert f.f_locals == {'a': 1}
582+
assert f.f_locals == {'a': 1}
534583
assert f.f_lineno == gen.__code__.co_firstlineno + 2
584+
assert f is g.gi_frame
535585
try:
536586
next(g)
537587
assert False
538588
except StopIteration:
539589
pass
540-
assert not g.gi_running
541-
assert not g.gi_suspended
542-
assert not g.gi_frame
543-
# CPython is inconsistent here that the backref is set for a generator that finished. Let's assume it's just an implementation accident
544-
# assert f.f_back is None
590+
assert f.f_back is sys._getframe()
545591
assert f.f_globals is globals()
546-
assert f.f_locals == {'a': 1, 'b': 2}
547-
assert f.f_lineno == gen.__code__.co_firstlineno + 3
592+
assert f.f_locals == {'a': 1, 'b': 2}
593+
# TODO GR-61955
594+
if sys.implementation.name != "graalpy" or not __graalpython__.is_bytecode_dsl_interpreter:
595+
assert f.f_lineno == gen.__code__.co_firstlineno + 3
548596

549597

550598
def test_generator_frame_in_running_generator():

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/frame/PFrame.java‎

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ public static final class Reference {
163163
// by a callee frame to inform the caller that it should materialize itself when it returns.
164164
private boolean escaped = false;
165165

166-
private final Reference callerInfo;
166+
private Reference callerInfo;
167167

168168
public Reference(RootNode rootNode, Reference callerInfo) {
169169
this.rootNode = rootNode;
@@ -174,9 +174,8 @@ public RootNode getRootNode() {
174174
return rootNode;
175175
}
176176

177-
public void setBackref(PFrame.Reference backref) {
178-
assert pyFrame != null : "setBackref should only be called when the PFrame escaped";
179-
pyFrame.setBackref(backref);
177+
public void setCallerInfo(Reference callerInfo) {
178+
this.callerInfo = callerInfo;
180179
}
181180

182181
public void markAsEscaped() {
@@ -307,25 +306,20 @@ public Thread getThread() {
307306
return thread;
308307
}
309308

310-
public PFrame.Reference getBackref() {
311-
return backref;
312-
}
313-
314-
public void setBackref(PFrame.Reference backref) {
315-
// GR-41914
316-
// @formatter:off
317-
// assert this.backref == null || this.backref == backref : "setBackref tried to set a backref different to the one that was previously attached";
318-
// @formatter:on
319-
this.backref = backref;
320-
}
321-
322309
public void setLine(int line) {
323310
if (lockLine) {
324311
return;
325312
}
326313
this.line = line;
327314
}
328315

316+
public void resetLine() {
317+
if (lockLine) {
318+
return;
319+
}
320+
this.line = UNINITIALIZED_LINE;
321+
}
322+
329323
public void setLineLock(int line) {
330324
this.line = line;
331325
this.lockLine = true;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/CommonGeneratorBuiltins.java‎

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import com.oracle.graal.python.builtins.objects.exception.PBaseException;
6262
import com.oracle.graal.python.builtins.objects.exception.PrepareExceptionNode;
6363
import com.oracle.graal.python.builtins.objects.frame.PFrame;
64+
import com.oracle.graal.python.builtins.objects.function.PArguments;
6465
import com.oracle.graal.python.builtins.objects.traceback.PTraceback;
6566
import com.oracle.graal.python.builtins.objects.type.slots.TpSlotIterNext.TpIterNextBuiltin;
6667
import com.oracle.graal.python.lib.IteratorExhausted;
@@ -103,6 +104,7 @@
103104
import com.oracle.truffle.api.nodes.DirectCallNode;
104105
import com.oracle.truffle.api.nodes.IndirectCallNode;
105106
import com.oracle.truffle.api.nodes.Node;
107+
import com.oracle.truffle.api.nodes.RootNode;
106108
import com.oracle.truffle.api.profiles.InlinedBranchProfile;
107109
import com.oracle.truffle.api.profiles.InlinedConditionProfile;
108110

@@ -183,19 +185,26 @@ static Object cachedBytecodeDSL(VirtualFrame frame, Node inliningTarget, PGenera
183185
* generator root.
184186
*/
185187
MaterializedFrame generatorFrame = self.getGeneratorFrame();
186-
Object[] arguments = new Object[]{generatorFrame, sendValue};
188+
Object[] callArguments = new Object[]{generatorFrame, sendValue};
189+
Object[] generatorArguments = generatorFrame.getArguments();
190+
PArguments.setCallerFrameInfo(generatorArguments, null);
191+
PArguments.setException(generatorArguments, null);
187192
if (frame == null) {
188193
PythonContext context = PythonContext.get(inliningTarget);
189194
PythonThreadState threadState = context.getThreadState(context.getLanguage(inliningTarget));
190-
Object state = IndirectCalleeContext.enter(threadState, generatorFrame.getArguments());
195+
Object state = IndirectCalleeContext.enter(threadState, generatorArguments);
196+
PFrame.Reference ref = PArguments.getCurrentFrameInfo(generatorArguments);
197+
ref.setCallerInfo(PArguments.getCallerFrameInfo(generatorArguments));
191198
try {
192-
generatorResult = callNode.call(arguments);
199+
generatorResult = callNode.call(callArguments);
193200
} finally {
194201
IndirectCalleeContext.exit(threadState, state);
195202
}
196203
} else {
197-
callContext.executePrepareCall(frame, generatorFrame.getArguments(), rootNode.getCallerFlags());
198-
generatorResult = callNode.call(arguments);
204+
callContext.executePrepareCall(frame, generatorArguments, rootNode.getCallerFlags());
205+
PFrame.Reference ref = PArguments.getCurrentFrameInfo(generatorArguments);
206+
ref.setCallerInfo(PArguments.getCallerFrameInfo(generatorArguments));
207+
generatorResult = callNode.call(callArguments);
199208
}
200209
} catch (PException e) {
201210
throw handleException(self, inliningTarget, errorProfile, raiseNode, e);
@@ -250,19 +259,26 @@ static Object genericBytecodeDSL(VirtualFrame frame, Node inliningTarget, PGener
250259
// See the cached specialization for notes about the arguments handling
251260
PRootNode rootNode = PGenerator.unwrapContinuationRoot((ContinuationRootNode) callTarget.getRootNode());
252261
MaterializedFrame generatorFrame = self.getGeneratorFrame();
253-
Object[] arguments = new Object[]{generatorFrame, sendValue};
262+
Object[] callArguments = new Object[]{generatorFrame, sendValue};
263+
Object[] generatorArguments = generatorFrame.getArguments();
264+
PArguments.setCallerFrameInfo(generatorArguments, null);
265+
PArguments.setException(generatorArguments, null);
254266
if (frame == null) {
255267
PythonContext context = PythonContext.get(inliningTarget);
256268
PythonThreadState threadState = context.getThreadState(context.getLanguage(inliningTarget));
257-
Object state = IndirectCalleeContext.enter(threadState, generatorFrame.getArguments());
269+
Object state = IndirectCalleeContext.enter(threadState, generatorArguments);
270+
PFrame.Reference ref = PArguments.getCurrentFrameInfo(generatorArguments);
271+
ref.setCallerInfo(PArguments.getCallerFrameInfo(generatorArguments));
258272
try {
259-
generatorResult = callNode.call(callTarget, arguments);
273+
generatorResult = callNode.call(callTarget, callArguments);
260274
} finally {
261275
IndirectCalleeContext.exit(threadState, state);
262276
}
263277
} else {
264-
callContext.executePrepareCall(frame, generatorFrame.getArguments(), rootNode.getCallerFlags());
265-
generatorResult = callNode.call(callTarget, arguments);
278+
callContext.executePrepareCall(frame, generatorArguments, rootNode.getCallerFlags());
279+
PFrame.Reference ref = PArguments.getCurrentFrameInfo(generatorArguments);
280+
ref.setCallerInfo(PArguments.getCallerFrameInfo(generatorArguments));
281+
generatorResult = callNode.call(callTarget, callArguments);
266282
}
267283
} catch (PException e) {
268284
throw handleException(self, inliningTarget, errorProfile, raiseNode, e);
@@ -388,9 +404,16 @@ static Object sendThrow(VirtualFrame frame, PGenerator self, Object typ, Object
388404
// Instead, we throw the exception here and fake entering the generator by adding
389405
// its frame to the traceback manually.
390406
self.markAsFinished();
391-
Node location = self.getCurrentCallTarget().getRootNode();
407+
Node location;
408+
RootNode rootNode = self.getCurrentCallTarget().getRootNode();
409+
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
410+
location = self.getBytecodeNode();
411+
} else {
412+
location = rootNode;
413+
}
392414
MaterializedFrame generatorFrame = self.getGeneratorFrame();
393-
PFrame pFrame = MaterializeFrameNode.materializeGeneratorFrame(location, generatorFrame, PFrame.Reference.EMPTY);
415+
PFrame.Reference ref = new PFrame.Reference(rootNode, PFrame.Reference.EMPTY);
416+
PFrame pFrame = MaterializeFrameNode.materializeGeneratorFrame(location, generatorFrame, self.getGlobals(), ref);
394417
FrameInfo info = (FrameInfo) generatorFrame.getFrameDescriptor().getInfo();
395418
pFrame.setLine(info.getFirstLineNumber());
396419
Object existingTracebackObj = getTracebackNode.execute(inliningTarget, instance);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/GeneratorBuiltins.java‎

Lines changed: 41 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import com.oracle.graal.python.builtins.objects.PNone;
4646
import com.oracle.graal.python.builtins.objects.PythonAbstractObject;
4747
import com.oracle.graal.python.builtins.objects.frame.PFrame;
48+
import com.oracle.graal.python.builtins.objects.function.PArguments;
4849
import com.oracle.graal.python.builtins.objects.str.StringNodes;
4950
import com.oracle.graal.python.builtins.objects.str.StringUtils.SimpleTruffleStringFormatNode;
5051
import com.oracle.graal.python.builtins.objects.type.TpSlots;
@@ -54,9 +55,11 @@
5455
import com.oracle.graal.python.nodes.bytecode.BytecodeFrameInfo;
5556
import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLFrameInfo;
5657
import com.oracle.graal.python.nodes.frame.MaterializeFrameNode;
58+
import com.oracle.graal.python.nodes.frame.ReadFrameNode;
5759
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
5860
import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode;
5961
import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
62+
import com.oracle.graal.python.runtime.CallerFlags;
6063
import com.oracle.graal.python.runtime.PythonOptions;
6164
import com.oracle.graal.python.runtime.object.PFactory;
6265
import com.oracle.truffle.api.bytecode.BytecodeLocation;
@@ -184,32 +187,57 @@ static Object setRunning(@SuppressWarnings("unused") PGenerator self, @SuppressW
184187
public abstract static class GetFrameNode extends PythonUnaryBuiltinNode {
185188

186189
@Specialization
187-
static Object getFrame(PGenerator self) {
190+
static Object getFrame(VirtualFrame frame, PGenerator self,
191+
@Cached ReadFrameNode readFrameNode) {
188192
if (self.isFinished()) {
189193
return PNone.NONE;
190194
} else {
195+
MaterializedFrame generatorFrame = self.getGeneratorFrame();
196+
PFrame.Reference currentRef = PArguments.getCurrentFrameInfo(generatorFrame.getArguments());
197+
if (self.isRunning()) {
198+
PFrame pyFrame = readFrameNode.getFrameForReference(frame, currentRef, 0, CallerFlags.NEEDS_PFRAME);
199+
pyFrame.getRef().markAsEscaped();
200+
return pyFrame;
201+
}
202+
PFrame pyFrame = currentRef.getPyFrame();
203+
if (currentRef.isEscaped() && currentRef.getPyFrame() != null) {
204+
// If it's escaped, we synced at every exit
205+
return pyFrame;
206+
}
191207
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
192-
MaterializedFrame generatorFrame = self.getGeneratorFrame();
193208
BytecodeDSLFrameInfo info = (BytecodeDSLFrameInfo) generatorFrame.getFrameDescriptor().getInfo();
194-
PFrame frame = MaterializeFrameNode.materializeGeneratorFrame(self.getBytecodeNode(), generatorFrame, PFrame.Reference.EMPTY);
209+
if (pyFrame == null) {
210+
pyFrame = MaterializeFrameNode.materializeGeneratorFrame(self.getBytecodeNode(), generatorFrame, self.getGlobals(), currentRef);
211+
currentRef.setPyFrame(pyFrame);
212+
}
195213
BytecodeLocation location = self.getCurrentLocation();
196214
if (location != null) {
197-
int bci = location.getBytecodeIndex();
198-
frame.setBci(bci);
199-
frame.setLine(info.getRootNode().bciToLine(bci, location.getBytecodeNode()));
215+
pyFrame.setBci(location.getBytecodeIndex());
216+
pyFrame.setLocation(location.getBytecodeNode());
217+
pyFrame.resetLine();
200218
} else {
201-
frame.setBci(0);
219+
pyFrame.setBci(0);
220+
pyFrame.setLine(info.getRootNode().getFirstLineno());
202221
}
203-
return frame;
204222
} else {
205-
MaterializedFrame generatorFrame = self.getGeneratorFrame();
206223
BytecodeFrameInfo info = (BytecodeFrameInfo) generatorFrame.getFrameDescriptor().getInfo();
207-
PFrame frame = MaterializeFrameNode.materializeGeneratorFrame(info.getRootNode(), generatorFrame, PFrame.Reference.EMPTY);
224+
if (pyFrame == null) {
225+
pyFrame = MaterializeFrameNode.materializeGeneratorFrame(info.getRootNode(), generatorFrame, self.getGlobals(), currentRef);
226+
currentRef.setPyFrame(pyFrame);
227+
}
208228
int bci = self.getBci();
209-
frame.setBci(bci);
210-
frame.setLine(info.getRootNode().bciToLine(bci));
211-
return frame;
229+
if (bci >= 0) {
230+
pyFrame.setBci(bci);
231+
pyFrame.resetLine();
232+
} else {
233+
pyFrame.setBci(0);
234+
pyFrame.setLine(info.getRootNode().getFirstLineno());
235+
}
212236
}
237+
pyFrame.setLastCallerFlags(CallerFlags.ALL_FRAME_FLAGS);
238+
assert currentRef.getPyFrame() == pyFrame;
239+
currentRef.markAsEscaped();
240+
return pyFrame;
213241
}
214242
}
215243
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/PGenerator.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,10 @@ public MaterializedFrame getGeneratorFrame() {
277277
return frame;
278278
}
279279

280+
public PythonObject getGlobals() {
281+
return globals;
282+
}
283+
280284
public static Object getSendValue(Object[] arguments) {
281285
return PArguments.getArgument(arguments, 1);
282286
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeGeneratorRootNode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,8 @@ public Object executeOSR(VirtualFrame osrFrame, int target, Object interpreterSt
145145

146146
@Override
147147
public Object execute(VirtualFrame frame) {
148-
calleeContext.enter(frame);
149148
MaterializedFrame generatorFrame = PGenerator.getGeneratorFrame(frame);
149+
ExecutionContext.CalleeContext.enterGenerator(frame, generatorFrame);
150150
/*
151151
* Using the materialized frame as stack would be bad for compiled performance, so we copy
152152
* the stack slots back to the virtual frame and use that as the stack. The values are

0 commit comments

Comments
 (0)