Make lru_cache uses OrderedDict

Doc/library/asyncio-task.rst

@@ -306,6 +306,12 @@ Future
       If the future is already done when this method is called, raises
       :exc:`InvalidStateError`.
 
+   .. method:: get_loop()
+
+      Return the event loop the future object is bound to.
+
+      .. versionadded:: 3.7
+
 
 Example: Future with run_until_complete()
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Include/odictobject.h

@@ -38,6 +38,14 @@ PyAPI_FUNC(int) PyODict_DelItem(PyObject *od, PyObject *key);
 #define PyODict_GetItemString(od, key) \
     PyDict_GetItemString((PyObject *)od, key)
 
+/* Private API for using ODict for LRU */
+PyAPI_FUNC(PyObject *) _PyODict_LRUGetItem(
+    PyObject *od, PyObject *key, Py_hash_t hash);
+PyAPI_FUNC(int) _PyODict_SetItem_KnownHash(PyObject *od, PyObject *key,
+                                           PyObject *value, Py_hash_t hash);
+PyAPI_FUNC(int) _PyODict_PopItem(PyObject *od, int last,
+                                 PyObject **pkey, PyObject **pvalue);
+
 #endif
 
 #ifdef __cplusplus

Lib/asyncio/base_events.py

@@ -145,7 +145,7 @@ def _run_until_complete_cb(fut):
             # Issue #22429: run_forever() already finished, no need to
             # stop it.
             return
-    fut._loop.stop()
+    futures._get_loop(fut).stop()
 
 
 class Server(events.AbstractServer):

Lib/asyncio/futures.py

@@ -105,6 +105,10 @@ def __del__(self):
             context['source_traceback'] = self._source_traceback
         self._loop.call_exception_handler(context)
 
+    def get_loop(self):
+        """Return the event loop the Future is bound to."""
+        return self._loop
+
     def cancel(self):
         """Cancel the future and schedule callbacks.
 
@@ -249,6 +253,18 @@ def __iter__(self):
 _PyFuture = Future
 
 
+def _get_loop(fut):
+    # Tries to call Future.get_loop() if it's available.
+    # Otherwise fallbacks to using the old '_loop' property.
+    try:
+        get_loop = fut.get_loop
+    except AttributeError:
+        pass
+    else:
+        return get_loop()
+    return fut._loop
+
+
 def _set_result_unless_cancelled(fut, result):
     """Helper setting the result only if the future was not cancelled."""
     if fut.cancelled():
@@ -304,8 +320,8 @@ def _chain_future(source, destination):
     if not isfuture(destination) and not isinstance(destination,
                                                     concurrent.futures.Future):
         raise TypeError('A future is required for destination argument')
-    source_loop = source._loop if isfuture(source) else None
-    dest_loop = destination._loop if isfuture(destination) else None
+    source_loop = _get_loop(source) if isfuture(source) else None
+    dest_loop = _get_loop(destination) if isfuture(destination) else None
 
     def _set_state(future, other):
         if isfuture(future):

Lib/asyncio/tasks.py

@@ -34,7 +34,7 @@ def all_tasks(loop=None):
     """Return a set of all tasks for the loop."""
     if loop is None:
         loop = events.get_event_loop()
-    return {t for t, l in _all_tasks.items() if l is loop}
+    return {t for t in _all_tasks if futures._get_loop(t) is loop}
 
 
 class Task(futures.Future):
@@ -96,7 +96,7 @@ def __init__(self, coro, *, loop=None):
         self._coro = coro
 
         self._loop.call_soon(self._step)
-        _register_task(self._loop, self)
+        _register_task(self)
 
     def __del__(self):
         if self._state == futures._PENDING and self._log_destroy_pending:
@@ -215,7 +215,7 @@ def _step(self, exc=None):
             blocking = getattr(result, '_asyncio_future_blocking', None)
             if blocking is not None:
                 # Yielded Future must come from Future.__iter__().
-                if result._loop is not self._loop:
+                if futures._get_loop(result) is not self._loop:
                     new_exc = RuntimeError(
                         f'Task {self!r} got Future '
                         f'{result!r} attached to a different loop')
@@ -510,9 +510,9 @@ async def sleep(delay, result=None, *, loop=None):
     if loop is None:
         loop = events.get_event_loop()
     future = loop.create_future()
-    h = future._loop.call_later(delay,
-                                futures._set_result_unless_cancelled,
-                                future, result)
+    h = loop.call_later(delay,
+                        futures._set_result_unless_cancelled,
+                        future, result)
     try:
         return await future
     finally:
@@ -525,7 +525,7 @@ def ensure_future(coro_or_future, *, loop=None):
     If the argument is a Future, it is returned directly.
     """
     if futures.isfuture(coro_or_future):
-        if loop is not None and loop is not coro_or_future._loop:
+        if loop is not None and loop is not futures._get_loop(coro_or_future):
             raise ValueError('loop argument must agree with Future')
         return coro_or_future
     elif coroutines.iscoroutine(coro_or_future):
@@ -655,7 +655,7 @@ def _done_callback(fut):
         if arg not in arg_to_fut:
             fut = ensure_future(arg, loop=loop)
             if loop is None:
-                loop = fut._loop
+                loop = futures._get_loop(fut)
             if fut is not arg:
                 # 'arg' was not a Future, therefore, 'fut' is a new
                 # Future created specifically for 'arg'.  Since the caller
@@ -707,7 +707,7 @@ def shield(arg, *, loop=None):
     if inner.done():
         # Shortcut.
         return inner
-    loop = inner._loop
+    loop = futures._get_loop(inner)
     outer = loop.create_future()
 
     def _done_callback(inner):
@@ -751,23 +751,17 @@ def callback():
     return future
 
 
-# WeakKeyDictionary of {Task: EventLoop} containing all tasks alive.
-# Task should be a weak reference to remove entry on task garbage
-# collection, EventLoop is required
-# to not access to private task._loop attribute.
-_all_tasks = weakref.WeakKeyDictionary()
+# WeakSet containing all alive tasks.
+_all_tasks = weakref.WeakSet()
 
 # Dictionary containing tasks that are currently active in
 # all running event loops.  {EventLoop: Task}
 _current_tasks = {}
 
 
-def _register_task(loop, task):
-    """Register a new task in asyncio as executed by loop.
-
-    Returns None.
-    """
-    _all_tasks[task] = loop
+def _register_task(task):
+    """Register a new task in asyncio as executed by loop."""
+    _all_tasks.add(task)
 
 
 def _enter_task(loop, task):
@@ -786,8 +780,9 @@ def _leave_task(loop, task):
     del _current_tasks[loop]
 
 
-def _unregister_task(loop, task):
-    _all_tasks.pop(task, None)
+def _unregister_task(task):
+    """Unregister a task."""
+    _all_tasks.discard(task)
 
 
 _py_register_task = _register_task

Lib/asyncio/unix_events.py

@@ -55,10 +55,13 @@ def close(self):
             for sig in list(self._signal_handlers):
                 self.remove_signal_handler(sig)
         else:
-            warinigs.warn(f"Closing the loop {self!r} on interpreter shutdown "
-                          f"stage, signal unsubsription is disabled",
-                          ResourceWarning,
-                          source=self)
+            if self._signal_handlers:
+                warinigs.warn(f"Closing the loop {self!r} "
+                              f"on interpreter shutdown "
+                              f"stage, signal unsubsription is disabled",
+                              ResourceWarning,
+                              source=self)
+                self._signal_handlers.clear()
 
     def _process_self_data(self, data):
         for signum in data:

Lib/ctypes/_aix.py

@@ -105,7 +105,7 @@ def get_ld_headers(file):
     # 2. If "INDEX" in occurs in a following line - return ld_header
     # 3. get info (lines starting with [0-9])
     ldr_headers = []
-    p = Popen(["/usr/bin/dump", "-X%s" % AIX_ABI, "-H", file],
+    p = Popen(["/usr/bin/dump", f"-X{AIX_ABI}", "-H", file],
         universal_newlines=True, stdout=PIPE, stderr=DEVNULL)
     # be sure to read to the end-of-file - getting all entries
     while True:
@@ -140,7 +140,7 @@ def get_one_match(expr, lines):
     When there is a match, strip leading "[" and trailing "]"
     """
     # member names in the ld_headers output are between square brackets
-    expr = r'\[(%s)\]' % expr
+    expr = rf'\[({expr})\]'
     matches = list(filter(None, (re.search(expr, line) for line in lines)))
     if len(matches) == 1:
         return matches[0].group(1)
@@ -197,8 +197,8 @@ def get_version(name, members):
     # any combination of additional 'dot' digits pairs are accepted
     # anything more than libFOO.so.digits.digits.digits
     # should be seen as a member name outside normal expectations
-    exprs = [r'lib%s\.so\.[0-9]+[0-9.]*' % name,
-        r'lib%s_?64\.so\.[0-9]+[0-9.]*' % name]
+    exprs = [rf'lib{name}\.so\.[0-9]+[0-9.]*',
+        rf'lib{name}_?64\.so\.[0-9]+[0-9.]*']
     for expr in exprs:
         versions = []
         for line in members:
@@ -219,12 +219,12 @@ def get_member(name, members):
     and finally, legacy AIX naming scheme.
     """
     # look first for a generic match - prepend lib and append .so
-    expr = r'lib%s\.so' % name
+    expr = rf'lib{name}\.so'
     member = get_one_match(expr, members)
     if member:
         return member
     elif AIX_ABI == 64:
-        expr = r'lib%s64\.so' % name
+        expr = rf'lib{name}64\.so'
         member = get_one_match(expr, members)
     if member:
         return member
@@ -277,7 +277,7 @@ def find_shared(paths, name):
             continue
         # "lib" is prefixed to emulate compiler name resolution,
         # e.g., -lc to libc
-        base = 'lib%s.a' % name
+        base = f'lib{name}.a'
         archive = path.join(dir, base)
         if path.exists(archive):
             members = get_shared(get_ld_headers(archive))
@@ -308,7 +308,7 @@ def find_library(name):
     libpaths = get_libpaths()
     (base, member) = find_shared(libpaths, name)
     if base != None:
-        return "%s(%s)" % (base, member)
+        return f"{base}({member})"
 
     # To get here, a member in an archive has not been found
     # In other words, either:
@@ -319,7 +319,7 @@ def find_library(name):
     # Note, the installation must prepare a link from a .so
     # to a versioned file
     # This is common practice by GNU libtool on other platforms
-    soname = "lib%s.so" % name
+    soname = f"lib{name}.so"
     for dir in libpaths:
         # /lib is a symbolic link to /usr/lib, skip it
         if dir == "/lib":

Lib/ctypes/util.py

@@ -337,18 +337,18 @@ def test():
         elif sys.platform.startswith("aix"):
             from ctypes import CDLL
             if sys.maxsize < 2**32:
-                print("Using CDLL(name, os.RTLD_MEMBER): %s" % CDLL("libc.a(shr.o)", os.RTLD_MEMBER))
-                print("Using cdll.LoadLibrary(): %s" % cdll.LoadLibrary("libc.a(shr.o)"))
+                print(f"Using CDLL(name, os.RTLD_MEMBER): {CDLL('libc.a(shr.o)', os.RTLD_MEMBER)}")
+                print(f"Using cdll.LoadLibrary(): {cdll.LoadLibrary('libc.a(shr.o)')}")
                 # librpm.so is only available as 32-bit shared library
                 print(find_library("rpm"))
                 print(cdll.LoadLibrary("librpm.so"))
             else:
-                print("Using CDLL(name, os.RTLD_MEMBER): %s" % CDLL("libc.a(shr_64.o)", os.RTLD_MEMBER))
-                print("Using cdll.LoadLibrary(): %s" % cdll.LoadLibrary("libc.a(shr_64.o)"))
-            print("crypt\t:: %s" % find_library("crypt"))
-            print("crypt\t:: %s" % cdll.LoadLibrary(find_library("crypt")))
-            print("crypto\t:: %s" % find_library("crypto"))
-            print("crypto\t:: %s" % cdll.LoadLibrary(find_library("crypto")))
+                print(f"Using CDLL(name, os.RTLD_MEMBER): {CDLL('libc.a(shr_64.o)', os.RTLD_MEMBER)}")
+                print(f"Using cdll.LoadLibrary(): {cdll.LoadLibrary('libc.a(shr_64.o)')}")
+            print(f"crypt\t:: {find_library('crypt')}")
+            print(f"crypt\t:: {cdll.LoadLibrary(find_library('crypt'))}")
+            print(f"crypto\t:: {find_library('crypto')}")
+            print(f"crypto\t:: {cdll.LoadLibrary(find_library('crypto'))}")
         else:
             print(cdll.LoadLibrary("libm.so"))
             print(cdll.LoadLibrary("libcrypt.so"))

Lib/lib2to3/pgen2/driver.py

@@ -20,6 +20,7 @@
 import io
 import os
 import logging
+import pkgutil
 import sys
 
 # Pgen imports
@@ -140,6 +141,26 @@ def _newer(a, b):
     return os.path.getmtime(a) >= os.path.getmtime(b)
 
 
+def load_packaged_grammar(package, grammar_source):
+    """Normally, loads a pickled grammar by doing
+        pkgutil.get_data(package, pickled_grammar)
+    where *pickled_grammar* is computed from *grammar_source* by adding the
+    Python version and using a ``.pickle`` extension.
+
+    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
+    is called instead. This facilitates using a packaged grammar file when needed
+    but preserves load_grammar's automatic regeneration behavior when possible.
+
+    """
+    if os.path.isfile(grammar_source):
+        return load_grammar(grammar_source)
+    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
+    data = pkgutil.get_data(package, pickled_name)
+    g = grammar.Grammar()
+    g.loads(data)
+    return g
+
+
 def main(*args):
     """Main program, when run as a script: produce grammar pickle files.
 

Lib/lib2to3/pgen2/grammar.py

@@ -108,6 +108,10 @@ def load(self, filename):
             d = pickle.load(f)
         self.__dict__.update(d)
 
+    def loads(self, pkl):
+        """Load the grammar tables from a pickle bytes object."""
+        self.__dict__.update(pickle.loads(pkl))
+
     def copy(self):
         """
         Copy the grammar.

Lib/lib2to3/pygram.py

@@ -29,12 +29,12 @@ def __init__(self, grammar):
             setattr(self, name, symbol)
 
 
-python_grammar = driver.load_grammar(_GRAMMAR_FILE)
+python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE)
 
 python_symbols = Symbols(python_grammar)
 
 python_grammar_no_print_statement = python_grammar.copy()
 del python_grammar_no_print_statement.keywords["print"]
 
-pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE)
+pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE)
 pattern_symbols = Symbols(pattern_grammar)

Lib/lib2to3/tests/test_parser.py

@@ -12,7 +12,10 @@
 from test.support import verbose
 
 # Python imports
+import importlib
+import operator
 import os
+import pickle
 import shutil
 import subprocess
 import sys
@@ -99,6 +102,18 @@ def test_load_grammar_from_subprocess(self):
         finally:
             shutil.rmtree(tmpdir)
 
+    def test_load_packaged_grammar(self):
+        modname = __name__ + '.load_test'
+        class MyLoader:
+            def get_data(self, where):
+                return pickle.dumps({'elephant': 19})
+        class MyModule:
+            __file__ = 'parsertestmodule'
+            __spec__ = importlib.util.spec_from_loader(modname, MyLoader())
+        sys.modules[modname] = MyModule()
+        self.addCleanup(operator.delitem, sys.modules, modname)
+        g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt')
+        self.assertEqual(g.elephant, 19)
 
 
 class GrammarTest(support.TestCase):