bpo-45923: Handle call events in bytecode (GH-30364)

* Add a RESUME instruction to handle "call" events.

Author: markshannon

Doc/library/dis.rst

@@ -1206,6 +1206,20 @@ All of the following opcodes use their arguments.
    .. versionadded:: 3.11
 
 
+.. opcode:: RESUME (where)
+
+    A no-op. Performs internal tracing, debugging and optimization checks.
+
+    The ``where`` operand marks where the ``RESUME`` occurs:
+
+    * ``0`` The start of a function
+    * ``1`` After a ``yield`` expression
+    * ``2`` After a ``yield from`` expression
+    * ``3`` After an ``await`` expression
+
+   .. versionadded:: 3.11
+
+
 .. opcode:: HAVE_ARGUMENT
 
    This is not really an opcode.  It identifies the dividing line between

Include/opcode.h

@@ -379,16 +379,21 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE)
 #     Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP)
 #     Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes)
+#     Python 3.11a4 3474 (Add RESUME opcode)
+
+#     Python 3.12 will start with magic number 3500
 
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually
 # due to the addition of new opcodes).
 #
+# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n.
+#
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3473).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3474).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/importlib/_bootstrap_external.py

@@ -178,6 +178,7 @@ def jabs_op(name, op):
 hasfree.append(148)
 def_op('COPY_FREE_VARS', 149)
 
+def_op('RESUME', 151)
 def_op('MATCH_CLASS', 152)
 
 def_op('FORMAT_VALUE', 155)

Lib/opcode.py

@@ -367,7 +367,7 @@ def test_co_positions_artificial_instructions(self):
             # get assigned the first_lineno but they don't have other positions.
             # There is no easy way of inferring them at that stage, so for now
             # we don't support it.
-            self.assertTrue(positions.count(None) in [0, 4])
+            self.assertIn(positions.count(None), [0, 3, 4])
 
             if not any(positions):
                 artificial_instructions.append(instr)
@@ -378,6 +378,7 @@ def test_co_positions_artificial_instructions(self):
                 for instruction in artificial_instructions
             ],
             [
+                ('RESUME', 0),
                 ("PUSH_EXC_INFO", None),
                 ("LOAD_CONST", None), # artificial 'None'
                 ("STORE_NAME", "e"),  # XX: we know the location for this
@@ -419,7 +420,9 @@ def test_co_positions_empty_linetable(self):
         def func():
             x = 1
         new_code = func.__code__.replace(co_linetable=b'')
-        for line, end_line, column, end_column in new_code.co_positions():
+        positions = new_code.co_positions()
+        next(positions) # Skip RESUME at start
+        for line, end_line, column, end_column in positions:
             self.assertIsNone(line)
             self.assertEqual(end_line, new_code.co_firstlineno + 1)
 
@@ -428,7 +431,9 @@ def test_co_positions_empty_endlinetable(self):
         def func():
             x = 1
         new_code = func.__code__.replace(co_endlinetable=b'')
-        for line, end_line, column, end_column in new_code.co_positions():
+        positions = new_code.co_positions()
+        next(positions) # Skip RESUME at start
+        for line, end_line, column, end_column in positions:
             self.assertEqual(line, new_code.co_firstlineno + 1)
             self.assertIsNone(end_line)
 
@@ -437,7 +442,9 @@ def test_co_positions_empty_columntable(self):
         def func():
             x = 1
         new_code = func.__code__.replace(co_columntable=b'')
-        for line, end_line, column, end_column in new_code.co_positions():
+        positions = new_code.co_positions()
+        next(positions) # Skip RESUME at start
+        for line, end_line, column, end_column in positions:
             self.assertEqual(line, new_code.co_firstlineno + 1)
             self.assertEqual(end_line, new_code.co_firstlineno + 1)
             self.assertIsNone(column)

Lib/test/test_code.py

@@ -158,7 +158,7 @@ def test_leading_newlines(self):
         s256 = "".join(["\n"] * 256 + ["spam"])
         co = compile(s256, 'fn', 'exec')
         self.assertEqual(co.co_firstlineno, 1)
-        self.assertEqual(list(co.co_lines()), [(0, 8, 257)])
+        self.assertEqual(list(co.co_lines()), [(0, 2, None), (2, 10, 257)])
 
     def test_literals_with_leading_zeroes(self):
         for arg in ["077787", "0xj", "0x.", "0e",  "090000000000000",
@@ -759,7 +759,7 @@ def unused_block_while_else():
 
         for func in funcs:
             opcodes = list(dis.get_instructions(func))
-            self.assertLessEqual(len(opcodes), 3)
+            self.assertLessEqual(len(opcodes), 4)
             self.assertEqual('LOAD_CONST', opcodes[-2].opname)
             self.assertEqual(None, opcodes[-2].argval)
             self.assertEqual('RETURN_VALUE', opcodes[-1].opname)
@@ -778,10 +778,10 @@ def continue_in_while():
         # Check that we did not raise but we also don't generate bytecode
         for func in funcs:
             opcodes = list(dis.get_instructions(func))
-            self.assertEqual(2, len(opcodes))
-            self.assertEqual('LOAD_CONST', opcodes[0].opname)
-            self.assertEqual(None, opcodes[0].argval)
-            self.assertEqual('RETURN_VALUE', opcodes[1].opname)
+            self.assertEqual(3, len(opcodes))
+            self.assertEqual('LOAD_CONST', opcodes[1].opname)
+            self.assertEqual(None, opcodes[1].argval)
+            self.assertEqual('RETURN_VALUE', opcodes[2].opname)
 
     def test_consts_in_conditionals(self):
         def and_true(x):
@@ -802,9 +802,9 @@ def or_false(x):
         for func in funcs:
             with self.subTest(func=func):
                 opcodes = list(dis.get_instructions(func))
-                self.assertEqual(2, len(opcodes))
-                self.assertIn('LOAD_', opcodes[0].opname)
-                self.assertEqual('RETURN_VALUE', opcodes[1].opname)
+                self.assertLessEqual(len(opcodes), 3)
+                self.assertIn('LOAD_', opcodes[-2].opname)
+                self.assertEqual('RETURN_VALUE', opcodes[-1].opname)
 
     def test_imported_load_method(self):
         sources = [
@@ -906,7 +906,7 @@ def load_attr():
                 o.
                 a
             )
-        load_attr_lines = [ 2, 3, 1 ]
+        load_attr_lines = [ 0, 2, 3, 1 ]
 
         def load_method():
             return (
@@ -915,7 +915,7 @@ def load_method():
                     0
                 )
             )
-        load_method_lines = [ 2, 3, 4, 3, 1 ]
+        load_method_lines = [ 0, 2, 3, 4, 3, 1 ]
 
         def store_attr():
             (
@@ -924,7 +924,7 @@ def store_attr():
             ) = (
                 v
             )
-        store_attr_lines = [ 5, 2, 3 ]
+        store_attr_lines = [ 0, 5, 2, 3 ]
 
         def aug_store_attr():
             (
@@ -933,7 +933,7 @@ def aug_store_attr():
             ) += (
                 v
             )
-        aug_store_attr_lines = [ 2, 3, 5, 1, 3 ]
+        aug_store_attr_lines = [ 0, 2, 3, 5, 1, 3 ]
 
         funcs = [ load_attr, load_method, store_attr, aug_store_attr]
         func_lines = [ load_attr_lines, load_method_lines,
@@ -942,7 +942,8 @@ def aug_store_attr():
         for func, lines in zip(funcs, func_lines, strict=True):
             with self.subTest(func=func):
                 code_lines = [ line-func.__code__.co_firstlineno
-                              for (_, _, line) in func.__code__.co_lines() ]
+                              for (_, _, line) in func.__code__.co_lines()
+                              if line is not None ]
                 self.assertEqual(lines, code_lines)
 
     def test_line_number_genexp(self):
@@ -966,7 +967,7 @@ async def test(aseq):
             async for i in aseq:
                 body
 
-        expected_lines = [None, 1, 2, 1]
+        expected_lines = [None, 0, 1, 2, 1]
         code_lines = [ None if line is None else line-test.__code__.co_firstlineno
                       for (_, _, line) in test.__code__.co_lines() ]
         self.assertEqual(expected_lines, code_lines)