Close #19466: Clear the frames of daemon threads earlier during the Python

shutdown to call objects destructors. So "unclosed file" resource warnings are
now corretly emitted for daemon threads.

Author: vstinner

Lib/test/test_threading.py

@@ -617,6 +617,52 @@ def test_BoundedSemaphore_limit(self):
                 t.join()
             self.assertRaises(ValueError, bs.release)
 
+    def test_locals_at_exit(self):
+        # Issue #19466: thread locals must not be deleted before destructors
+        # are called
+        rc, out, err = assert_python_ok("-c", """if 1:
+            import threading
+
+            class Atexit:
+                def __del__(self):
+                    print("thread_dict.atexit = %r" % thread_dict.atexit)
+
+            thread_dict = threading.local()
+            thread_dict.atexit = "atexit"
+
+            atexit = Atexit()
+        """)
+        self.assertEqual(out.rstrip(), b"thread_dict.atexit = 'atexit'")
+
+    def test_warnings_at_exit(self):
+        # Issue #19466: try to call most destructors at Python shutdown before
+        # destroying Python thread states
+        filename = __file__
+        rc, out, err = assert_python_ok("-Wd", "-c", """if 1:
+            import time
+            import threading
+
+            def open_sleep():
+                # a warning will be emitted when the open file will be
+                # destroyed (without being explicitly closed) while the daemon
+                # thread is destroyed
+                fileobj = open(%a, 'rb')
+                start_event.set()
+                time.sleep(60.0)
+
+            start_event = threading.Event()
+
+            thread = threading.Thread(target=open_sleep)
+            thread.daemon = True
+            thread.start()
+
+            # wait until the thread started
+            start_event.wait()
+        """ % filename)
+        self.assertRegex(err.rstrip(),
+                         b"^sys:1: ResourceWarning: unclosed file ")
+
+
 class ThreadJoinOnShutdown(BaseTestCase):
 
     def _run_and_join(self, script):
@@ -701,6 +747,10 @@ def test_4_daemon_threads(self):
             import sys
             import time
             import threading
+            import warnings
+
+            # ignore "unclosed file ..." warnings
+            warnings.filterwarnings('ignore', '', ResourceWarning)
 
             thread_has_run = set()
 

Misc/NEWS

@@ -10,6 +10,10 @@ Projected release date: 2013-11-24
 Core and Builtins
 -----------------
 
+- Issue #19466: Clear the frames of daemon threads earlier during the
+  Python shutdown to call objects destructors. So "unclosed file" resource
+  warnings are now corretly emitted for daemon threads.
+
 - Issue #19514: Deduplicate some _Py_IDENTIFIER declarations.
   Patch by Andrei Dorian Duma.
 

Python/pythonrun.c

@@ -576,11 +576,13 @@ Py_Finalize(void)
     _Py_Finalizing = tstate;
     initialized = 0;
 
-    /* Flush stdout+stderr */
-    flush_std_files();
-
-    /* Disable signal handling */
-    PyOS_FiniInterrupts();
+    /* Destroy the state of all threads except of the current thread: in
+       practice, only daemon threads should still be alive. Clear frames of
+       other threads to call objects destructor. Destructors will be called in
+       the current Python thread. Since _Py_Finalizing has been set, no other
+       Python threads can lock the GIL at this point (if they try, they will
+       exit immediatly). */
+    _PyThreadState_DeleteExcept(tstate);
 
     /* Collect garbage.  This may call finalizers; it's nice to call these
      * before all modules are destroyed.
@@ -595,13 +597,21 @@ Py_Finalize(void)
      * XXX I haven't seen a real-life report of either of these.
      */
     PyGC_Collect();
+
 #ifdef COUNT_ALLOCS
     /* With COUNT_ALLOCS, it helps to run GC multiple times:
        each collection might release some types from the type
        list, so they become garbage. */
     while (PyGC_Collect() > 0)
         /* nothing */;
 #endif
+
+    /* Flush stdout+stderr */
+    flush_std_files();
+
+    /* Disable signal handling */
+    PyOS_FiniInterrupts();
+
     /* Destroy all modules */
     PyImport_Cleanup();