Skip to content

Commit 80d8d73

Browse files
markshannonmarkshannon
committed
Reduce the work done to spill and reload the stack around calls
1 parent c375992 commit 80d8d73

19 files changed

Lines changed: 3434 additions & 1495 deletions

‎Include/internal/pycore_interpframe.h‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *
152152
// visited is GC bookkeeping for the current stack walk, not frame state.
153153
dest->visited = 0;
154154
#ifdef Py_DEBUG
155+
dest->stackpointer_valid = src->stackpointer_valid;
155156
dest->lltrace = src->lltrace;
156157
#endif
157158
for (int i = 0; i < stacktop; i++) {
@@ -207,6 +208,7 @@ _PyFrame_Initialize(
207208
frame->owner = FRAME_OWNED_BY_THREAD;
208209
frame->visited = 0;
209210
#ifdef Py_DEBUG
211+
frame->stackpointer_valid = 1;
210212
frame->lltrace = 0;
211213
#endif
212214

@@ -230,24 +232,37 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
230232
static inline _PyStackRef*
231233
_PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
232234
{
235+
return frame->stackpointer;
236+
}
237+
238+
static inline void
239+
_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
240+
{
241+
frame->stackpointer = stack_pointer;
242+
}
243+
244+
static inline void
245+
_PyFrame_StackPointerValidate(_PyInterpreterFrame *frame)
246+
{
247+
#ifdef Py_DEBUG
248+
/* Avoid bloating the JIT code */
233249
#ifndef _Py_JIT
234-
assert(frame->stackpointer != NULL);
250+
assert(frame->stackpointer_valid == 0);
235251
#endif
236-
_PyStackRef *sp = frame->stackpointer;
237-
#ifndef NDEBUG
238-
frame->stackpointer = NULL;
252+
frame->stackpointer_valid = 1;
239253
#endif
240-
return sp;
241254
}
242255

243256
static inline void
244-
_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
257+
_PyFrame_StackPointerInvalidate(_PyInterpreterFrame *frame)
245258
{
259+
#ifdef Py_DEBUG
246260
/* Avoid bloating the JIT code */
247261
#ifndef _Py_JIT
248-
assert(frame->stackpointer == NULL);
262+
assert(frame->stackpointer_valid == 1);
263+
#endif
264+
frame->stackpointer_valid = 0;
249265
#endif
250-
frame->stackpointer = stack_pointer;
251266
}
252267

253268
/* Determine whether a frame is incomplete.
@@ -397,6 +412,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
397412
frame->owner = FRAME_OWNED_BY_THREAD;
398413
frame->visited = 0;
399414
#ifdef Py_DEBUG
415+
frame->stackpointer_valid = 1;
400416
frame->lltrace = 0;
401417
#endif
402418
frame->return_offset = 0;

‎Include/internal/pycore_interpframe_structs.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ struct _PyInterpreterFrame {
4444
char owner;
4545
#ifdef Py_DEBUG
4646
uint8_t visited:1;
47-
uint8_t lltrace:7;
47+
uint8_t stackpointer_valid:1;
48+
uint8_t lltrace:6;
4849
#else
4950
uint8_t visited;
5051
#endif

‎Include/internal/pycore_opcode_metadata.h‎

Lines changed: 6 additions & 6 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_optimizer.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ int _Py_uop_analyze_and_optimize(
312312
_PyUOpInstruction *input, int trace_len, int curr_stackentries,
313313
_PyUOpInstruction *output, _PyBloomFilter *dependencies);
314314

315-
extern PyTypeObject _PyUOpExecutor_Type;
315+
PyAPI_DATA(PyTypeObject) _PyUOpExecutor_Type;
316316

317317

318318
#define UOP_FORMAT_TARGET 0

‎Include/internal/pycore_uop_metadata.h‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/test/test_generated_cases.py‎

Lines changed: 41 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -369,8 +369,9 @@ def test_sync_sp(self):
369369
stack_pointer += -1;
370370
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
371371
_PyFrame_SetStackPointer(frame, stack_pointer);
372+
_PyFrame_StackPointerValidate(frame);
372373
escaping_call();
373-
stack_pointer = _PyFrame_GetStackPointer(frame);
374+
_PyFrame_StackPointerInvalidate(frame);
374375
res = Py_None;
375376
stack_pointer[0] = res;
376377
stack_pointer += 1;
@@ -392,8 +393,9 @@ def test_sync_sp(self):
392393
res = Py_None;
393394
stack_pointer[-1] = res;
394395
_PyFrame_SetStackPointer(frame, stack_pointer);
396+
_PyFrame_StackPointerValidate(frame);
395397
escaping_call();
396-
stack_pointer = _PyFrame_GetStackPointer(frame);
398+
_PyFrame_StackPointerInvalidate(frame);
397399
DISPATCH();
398400
}
399401
"""
@@ -626,18 +628,20 @@ def test_macro_instruction(self):
626628
uint16_t counter = read_u16(&this_instr[1].cache);
627629
(void)counter;
628630
_PyFrame_SetStackPointer(frame, stack_pointer);
631+
_PyFrame_StackPointerValidate(frame);
629632
op1(left, right);
630-
stack_pointer = _PyFrame_GetStackPointer(frame);
633+
_PyFrame_StackPointerInvalidate(frame);
631634
}
632635
/* Skip 2 cache entries */
633636
// OP2
634637
{
635638
arg2 = stack_pointer[-3];
636639
uint32_t extra = read_u32(&this_instr[4].cache);
637640
(void)extra;
638-
_PyFrame_SetStackPointer(frame, stack_pointer);
641+
assert(stack_pointer == _PyFrame_GetStackPointer(frame));
642+
_PyFrame_StackPointerValidate(frame);
639643
res = op2(arg2, left, right);
640-
stack_pointer = _PyFrame_GetStackPointer(frame);
644+
_PyFrame_StackPointerInvalidate(frame);
641645
}
642646
stack_pointer[-3] = res;
643647
stack_pointer += -2;
@@ -662,8 +666,9 @@ def test_macro_instruction(self):
662666
uint16_t counter = read_u16(&this_instr[1].cache);
663667
(void)counter;
664668
_PyFrame_SetStackPointer(frame, stack_pointer);
669+
_PyFrame_StackPointerValidate(frame);
665670
op1(left, right);
666-
stack_pointer = _PyFrame_GetStackPointer(frame);
671+
_PyFrame_StackPointerInvalidate(frame);
667672
DISPATCH();
668673
}
669674
@@ -685,8 +690,9 @@ def test_macro_instruction(self):
685690
left = stack_pointer[-2];
686691
arg2 = stack_pointer[-3];
687692
_PyFrame_SetStackPointer(frame, stack_pointer);
693+
_PyFrame_StackPointerValidate(frame);
688694
res = op3(arg2, left, right);
689-
stack_pointer = _PyFrame_GetStackPointer(frame);
695+
_PyFrame_StackPointerInvalidate(frame);
690696
stack_pointer[-3] = res;
691697
stack_pointer += -2;
692698
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
@@ -1475,8 +1481,10 @@ def test_stack_save_reload(self):
14751481
next_instr += 1;
14761482
INSTRUCTION_STATS(BALANCED);
14771483
_PyFrame_SetStackPointer(frame, stack_pointer);
1484+
_PyFrame_StackPointerValidate(frame);
14781485
code();
14791486
stack_pointer = _PyFrame_GetStackPointer(frame);
1487+
_PyFrame_StackPointerInvalidate(frame);
14801488
DISPATCH();
14811489
}
14821490
"""
@@ -1500,6 +1508,10 @@ def test_stack_save_reload_paired(self):
15001508
frame->instr_ptr = next_instr;
15011509
next_instr += 1;
15021510
INSTRUCTION_STATS(BALANCED);
1511+
_PyFrame_SetStackPointer(frame, stack_pointer);
1512+
_PyFrame_StackPointerValidate(frame);
1513+
stack_pointer = _PyFrame_GetStackPointer(frame);
1514+
_PyFrame_StackPointerInvalidate(frame);
15031515
DISPATCH();
15041516
}
15051517
"""
@@ -1522,13 +1534,13 @@ def test_stack_reload_only(self):
15221534
frame->instr_ptr = next_instr;
15231535
next_instr += 1;
15241536
INSTRUCTION_STATS(BALANCED);
1525-
_PyFrame_SetStackPointer(frame, stack_pointer);
1537+
// Explicit stack reload
15261538
stack_pointer = _PyFrame_GetStackPointer(frame);
1539+
assert(frame->stackpointer_valid == 0);
15271540
DISPATCH();
15281541
}
15291542
"""
1530-
with self.assertRaises(SyntaxError):
1531-
self.run_cases_test(input, output)
1543+
self.run_cases_test(input, output)
15321544

15331545
def test_stack_save_only(self):
15341546

@@ -1614,16 +1626,19 @@ def test_escaping_call_next_to_cmacro(self):
16141626
INSTRUCTION_STATS(OP);
16151627
#ifdef Py_GIL_DISABLED
16161628
_PyFrame_SetStackPointer(frame, stack_pointer);
1629+
_PyFrame_StackPointerValidate(frame);
16171630
escaping_call();
1618-
stack_pointer = _PyFrame_GetStackPointer(frame);
1631+
_PyFrame_StackPointerInvalidate(frame);
16191632
#else
16201633
_PyFrame_SetStackPointer(frame, stack_pointer);
1634+
_PyFrame_StackPointerValidate(frame);
16211635
another_escaping_call();
1622-
stack_pointer = _PyFrame_GetStackPointer(frame);
1636+
_PyFrame_StackPointerInvalidate(frame);
16231637
#endif
1624-
_PyFrame_SetStackPointer(frame, stack_pointer);
1638+
assert(stack_pointer == _PyFrame_GetStackPointer(frame));
1639+
_PyFrame_StackPointerValidate(frame);
16251640
yet_another_escaping_call();
1626-
stack_pointer = _PyFrame_GetStackPointer(frame);
1641+
_PyFrame_StackPointerInvalidate(frame);
16271642
DISPATCH();
16281643
}
16291644
"""
@@ -1735,8 +1750,9 @@ def test_complex_label(self):
17351750
LABEL(my_label)
17361751
{
17371752
_PyFrame_SetStackPointer(frame, stack_pointer);
1753+
_PyFrame_StackPointerValidate(frame);
17381754
do_thing();
1739-
stack_pointer = _PyFrame_GetStackPointer(frame);
1755+
_PyFrame_StackPointerInvalidate(frame);
17401756
if (complex) {
17411757
JUMP_TO_LABEL(other_label);
17421758
}
@@ -1762,12 +1778,14 @@ def test_spilled_label(self):
17621778
LABEL(one)
17631779
{
17641780
stack_pointer = _PyFrame_GetStackPointer(frame);
1781+
_PyFrame_StackPointerInvalidate(frame);
17651782
JUMP_TO_LABEL(two);
17661783
}
17671784
17681785
LABEL(two)
17691786
{
17701787
_PyFrame_SetStackPointer(frame, stack_pointer);
1788+
_PyFrame_StackPointerValidate(frame);
17711789
JUMP_TO_LABEL(one);
17721790
}
17731791
"""
@@ -1817,16 +1835,18 @@ def test_multiple_labels(self):
18171835
LABEL(my_label_1)
18181836
{
18191837
_PyFrame_SetStackPointer(frame, stack_pointer);
1838+
_PyFrame_StackPointerValidate(frame);
18201839
do_thing1();
1821-
stack_pointer = _PyFrame_GetStackPointer(frame);
1840+
_PyFrame_StackPointerInvalidate(frame);
18221841
JUMP_TO_LABEL(my_label_2);
18231842
}
18241843
18251844
LABEL(my_label_2)
18261845
{
18271846
_PyFrame_SetStackPointer(frame, stack_pointer);
1847+
_PyFrame_StackPointerValidate(frame);
18281848
do_thing2();
1829-
stack_pointer = _PyFrame_GetStackPointer(frame);
1849+
_PyFrame_StackPointerInvalidate(frame);
18301850
JUMP_TO_LABEL(my_label_1);
18311851
}
18321852
"""
@@ -1878,14 +1898,16 @@ def test_reassigning_dead_inputs(self):
18781898
_PyStackRef in;
18791899
in = stack_pointer[-1];
18801900
_PyFrame_SetStackPointer(frame, stack_pointer);
1901+
_PyFrame_StackPointerValidate(frame);
18811902
temp = use(in);
1882-
stack_pointer = _PyFrame_GetStackPointer(frame);
1903+
_PyFrame_StackPointerInvalidate(frame);
18831904
in = temp;
18841905
stack_pointer += -1;
18851906
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
18861907
_PyFrame_SetStackPointer(frame, stack_pointer);
1908+
_PyFrame_StackPointerValidate(frame);
18871909
PyStackRef_CLOSE(in);
1888-
stack_pointer = _PyFrame_GetStackPointer(frame);
1910+
_PyFrame_StackPointerInvalidate(frame);
18891911
DISPATCH();
18901912
}
18911913
"""

‎Modules/_testinternalcapi/interpreter.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
8787
entry.frame.return_offset = 0;
8888
#ifdef Py_DEBUG
8989
entry.frame.lltrace = 0;
90+
entry.frame.stackpointer_valid = 1;
9091
#endif
9192
/* Push frame */
9293
entry.frame.previous = tstate->current_frame;
@@ -117,6 +118,7 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
117118
next_instr = frame->instr_ptr;
118119
monitor_throw(tstate, frame, next_instr);
119120
stack_pointer = _PyFrame_GetStackPointer(frame);
121+
_PyFrame_StackPointerInvalidate(frame);
120122
#if _Py_TAIL_CALL_INTERP
121123
# if Py_STATS
122124
return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode);

0 commit comments

Comments
 (0)