gh-76785: Fixes for test.support.interpreters (gh-112982)

This involves a number of changes for PEP 734.

Author: ericsnowcurrently

.github/CODEOWNERS

@@ -78,6 +78,11 @@ Python/traceback.c            @iritkatriel
 **/*importlib/resources/*      @jaraco @warsaw @FFY00
 **/importlib/metadata/*       @jaraco @warsaw
 
+# Subinterpreters
+Lib/test/support/interpreters/**  @ericsnowcurrently
+Modules/_xx*interp*module.c   @ericsnowcurrently
+Lib/test/test_interpreters/**  @ericsnowcurrently
+
 # Dates and times
 **/*datetime*                 @pganssle @abalkin
 **/*str*time*                 @pganssle @abalkin
@@ -148,7 +153,15 @@ Doc/c-api/stable.rst          @encukou
 **/*itertools*                @rhettinger
 **/*collections*              @rhettinger
 **/*random*                   @rhettinger
-**/*queue*                    @rhettinger
+Doc/**/*queue*                @rhettinger
+PCbuild/**/*queue*            @rhettinger
+Modules/_queuemodule.c        @rhettinger
+Lib/*queue*.py                @rhettinger
+Lib/asyncio/*queue*.py        @rhettinger
+Lib/multiprocessing/*queue*.py  @rhettinger
+Lib/test/*queue*.py           @rhettinger
+Lib/test_asyncio/*queue*.py   @rhettinger
+Lib/test_multiprocessing/*queue*.py  @rhettinger
 **/*bisect*                   @rhettinger
 **/*heapq*                    @rhettinger
 **/*functools*                @rhettinger

Include/internal/pycore_crossinterp.h

@@ -11,6 +11,13 @@ extern "C" {
 #include "pycore_lock.h"            // PyMutex
 #include "pycore_pyerrors.h"
 
+/**************/
+/* exceptions */
+/**************/
+
+PyAPI_DATA(PyObject *) PyExc_InterpreterError;
+PyAPI_DATA(PyObject *) PyExc_InterpreterNotFoundError;
+
 
 /***************************/
 /* cross-interpreter calls */
@@ -160,6 +167,9 @@ struct _xi_state {
 extern PyStatus _PyXI_Init(PyInterpreterState *interp);
 extern void _PyXI_Fini(PyInterpreterState *interp);
 
+extern PyStatus _PyXI_InitTypes(PyInterpreterState *interp);
+extern void _PyXI_FiniTypes(PyInterpreterState *interp);
+
 
 /***************************/
 /* short-term data sharing */

Include/internal/pycore_interp.h

@@ -250,9 +250,9 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst
 // Export for the _xxinterpchannels module.
 PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t);
 
-extern int _PyInterpreterState_IDInitref(PyInterpreterState *);
-extern int _PyInterpreterState_IDIncref(PyInterpreterState *);
-extern void _PyInterpreterState_IDDecref(PyInterpreterState *);
+PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *);
+PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *);
+PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *);
 
 extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp);
 

Lib/test/support/interpreters/__init__.py

@@ -0,0 +1,160 @@
+"""Subinterpreters High Level Module."""
+
+import threading
+import weakref
+import _xxsubinterpreters as _interpreters
+
+# aliases:
+from _xxsubinterpreters import (
+    InterpreterError, InterpreterNotFoundError,
+    is_shareable,
+)
+
+
+__all__ = [
+    'get_current', 'get_main', 'create', 'list_all', 'is_shareable',
+    'Interpreter',
+    'InterpreterError', 'InterpreterNotFoundError', 'ExecFailure',
+    'create_queue', 'Queue', 'QueueEmpty', 'QueueFull',
+]
+
+
+_queuemod = None
+
+def __getattr__(name):
+    if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'):
+        global create_queue, Queue, QueueEmpty, QueueFull
+        ns = globals()
+        from .queues import (
+            create as create_queue,
+            Queue, QueueEmpty, QueueFull,
+        )
+        return ns[name]
+    else:
+        raise AttributeError(name)
+
+
+class ExecFailure(RuntimeError):
+
+    def __init__(self, excinfo):
+        msg = excinfo.formatted
+        if not msg:
+            if excinfo.type and snapshot.msg:
+                msg = f'{snapshot.type.__name__}: {snapshot.msg}'
+            else:
+                msg = snapshot.type.__name__ or snapshot.msg
+        super().__init__(msg)
+        self.snapshot = excinfo
+
+
+def create():
+    """Return a new (idle) Python interpreter."""
+    id = _interpreters.create(isolated=True)
+    return Interpreter(id)
+
+
+def list_all():
+    """Return all existing interpreters."""
+    return [Interpreter(id) for id in _interpreters.list_all()]
+
+
+def get_current():
+    """Return the currently running interpreter."""
+    id = _interpreters.get_current()
+    return Interpreter(id)
+
+
+def get_main():
+    """Return the main interpreter."""
+    id = _interpreters.get_main()
+    return Interpreter(id)
+
+
+_known = weakref.WeakValueDictionary()
+
+class Interpreter:
+    """A single Python interpreter."""
+
+    def __new__(cls, id, /):
+        # There is only one instance for any given ID.
+        if not isinstance(id, int):
+            raise TypeError(f'id must be an int, got {id!r}')
+        id = int(id)
+        try:
+            self = _known[id]
+            assert hasattr(self, '_ownsref')
+        except KeyError:
+            # This may raise InterpreterNotFoundError:
+            _interpreters._incref(id)
+            try:
+                self = super().__new__(cls)
+                self._id = id
+                self._ownsref = True
+            except BaseException:
+                _interpreters._deccref(id)
+                raise
+            _known[id] = self
+        return self
+
+    def __repr__(self):
+        return f'{type(self).__name__}({self.id})'
+
+    def __hash__(self):
+        return hash(self._id)
+
+    def __del__(self):
+        self._decref()
+
+    def _decref(self):
+        if not self._ownsref:
+            return
+        self._ownsref = False
+        try:
+            _interpreters._decref(self.id)
+        except InterpreterNotFoundError:
+            pass
+
+    @property
+    def id(self):
+        return self._id
+
+    def is_running(self):
+        """Return whether or not the identified interpreter is running."""
+        return _interpreters.is_running(self._id)
+
+    def close(self):
+        """Finalize and destroy the interpreter.
+
+        Attempting to destroy the current interpreter results
+        in a RuntimeError.
+        """
+        return _interpreters.destroy(self._id)
+
+    def exec_sync(self, code, /, channels=None):
+        """Run the given source code in the interpreter.
+
+        This is essentially the same as calling the builtin "exec"
+        with this interpreter, using the __dict__ of its __main__
+        module as both globals and locals.
+
+        There is no return value.
+
+        If the code raises an unhandled exception then an ExecFailure
+        is raised, which summarizes the unhandled exception.  The actual
+        exception is discarded because objects cannot be shared between
+        interpreters.
+
+        This blocks the current Python thread until done.  During
+        that time, the previous interpreter is allowed to run
+        in other threads.
+        """
+        excinfo = _interpreters.exec(self._id, code, channels)
+        if excinfo is not None:
+            raise ExecFailure(excinfo)
+
+    def run(self, code, /, channels=None):
+        def task():
+            self.exec_sync(code, channels=channels)
+        t = threading.Thread(target=task)
+        t.start()
+        return t

Lib/test/support/interpreters.py Lib/test/support/interpreters/channels.pyLib/test/support/interpreters.py renamed to Lib/test/support/interpreters/channels.py

@@ -1,135 +1,23 @@
-"""Subinterpreters High Level Module."""
+"""Cross-interpreter Channels High Level Module."""
 
 import time
-import _xxsubinterpreters as _interpreters
 import _xxinterpchannels as _channels
 
 # aliases:
-from _xxsubinterpreters import is_shareable
 from _xxinterpchannels import (
     ChannelError, ChannelNotFoundError, ChannelClosedError,
     ChannelEmptyError, ChannelNotEmptyError,
 )
 
 
 __all__ = [
-    'Interpreter', 'get_current', 'get_main', 'create', 'list_all',
-    'RunFailedError',
+    'create', 'list_all',
     'SendChannel', 'RecvChannel',
-    'create_channel', 'list_all_channels', 'is_shareable',
-    'ChannelError', 'ChannelNotFoundError',
-    'ChannelEmptyError',
-    ]
+    'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError',
+]
 
 
-class RunFailedError(RuntimeError):
-
-    def __init__(self, excinfo):
-        msg = excinfo.formatted
-        if not msg:
-            if excinfo.type and snapshot.msg:
-                msg = f'{snapshot.type.__name__}: {snapshot.msg}'
-            else:
-                msg = snapshot.type.__name__ or snapshot.msg
-        super().__init__(msg)
-        self.snapshot = excinfo
-
-
-def create(*, isolated=True):
-    """Return a new (idle) Python interpreter."""
-    id = _interpreters.create(isolated=isolated)
-    return Interpreter(id, isolated=isolated)
-
-
-def list_all():
-    """Return all existing interpreters."""
-    return [Interpreter(id) for id in _interpreters.list_all()]
-
-
-def get_current():
-    """Return the currently running interpreter."""
-    id = _interpreters.get_current()
-    return Interpreter(id)
-
-
-def get_main():
-    """Return the main interpreter."""
-    id = _interpreters.get_main()
-    return Interpreter(id)
-
-
-class Interpreter:
-    """A single Python interpreter."""
-
-    def __init__(self, id, *, isolated=None):
-        if not isinstance(id, (int, _interpreters.InterpreterID)):
-            raise TypeError(f'id must be an int, got {id!r}')
-        self._id = id
-        self._isolated = isolated
-
-    def __repr__(self):
-        data = dict(id=int(self._id), isolated=self._isolated)
-        kwargs = (f'{k}={v!r}' for k, v in data.items())
-        return f'{type(self).__name__}({", ".join(kwargs)})'
-
-    def __hash__(self):
-        return hash(self._id)
-
-    def __eq__(self, other):
-        if not isinstance(other, Interpreter):
-            return NotImplemented
-        else:
-            return other._id == self._id
-
-    @property
-    def id(self):
-        return self._id
-
-    @property
-    def isolated(self):
-        if self._isolated is None:
-            # XXX The low-level function has not been added yet.
-            # See bpo-....
-            self._isolated = _interpreters.is_isolated(self._id)
-        return self._isolated
-
-    def is_running(self):
-        """Return whether or not the identified interpreter is running."""
-        return _interpreters.is_running(self._id)
-
-    def close(self):
-        """Finalize and destroy the interpreter.
-
-        Attempting to destroy the current interpreter results
-        in a RuntimeError.
-        """
-        return _interpreters.destroy(self._id)
-
-    # XXX Rename "run" to "exec"?
-    def run(self, src_str, /, channels=None):
-        """Run the given source code in the interpreter.
-
-        This is essentially the same as calling the builtin "exec"
-        with this interpreter, using the __dict__ of its __main__
-        module as both globals and locals.
-
-        There is no return value.
-
-        If the code raises an unhandled exception then a RunFailedError
-        is raised, which summarizes the unhandled exception.  The actual
-        exception is discarded because objects cannot be shared between
-        interpreters.
-
-        This blocks the current Python thread until done.  During
-        that time, the previous interpreter is allowed to run
-        in other threads.
-        """
-        excinfo = _interpreters.exec(self._id, src_str, channels)
-        if excinfo is not None:
-            raise RunFailedError(excinfo)
-
-
-def create_channel():
+def create():
     """Return (recv, send) for a new cross-interpreter channel.
 
     The channel may be used to pass data safely between interpreters.
@@ -139,7 +27,7 @@ def create_channel():
     return recv, send
 
 
-def list_all_channels():
+def list_all():
     """Return a list of (recv, send) for all open channels."""
     return [(RecvChannel(cid), SendChannel(cid))
             for cid in _channels.list_all()]