bpo-34582: Adds JUnit XML output for regression tests (GH-9210)

Author: zooba

.vsts/linux-pr.yml

@@ -70,6 +70,15 @@ steps:
   displayName: 'Run patchcheck.py'
   condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
 
-- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
   displayName: 'Tests'
   condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: PublishTestResults@2
+  displayName: 'Publish Test Results'
+  inputs:
+    testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+    mergeTestResults: true
+    testRunTitle: '$(system.pullRequest.targetBranch)-linux'
+    platform: linux
+  condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))

.vsts/macos-pr.yml

@@ -50,6 +50,15 @@ steps:
   displayName: 'Display build info'
   condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
 
-- script: make buildbottest TESTOPTS="-j4 -uall,-cpu"
+- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
   displayName: 'Tests'
   condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: PublishTestResults@2
+  displayName: 'Publish Test Results'
+  inputs:
+    testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+    mergeTestResults: true
+    testRunTitle: '$(system.pullRequest.targetBranch)-macOS'
+    platform: macOS
+  condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))

.vsts/windows-pr.yml

@@ -54,8 +54,17 @@ steps:
   displayName: 'Display build info'
   condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
 
-- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml"
   displayName: 'Tests'
   env:
     PREFIX: $(Py_OutDir)\$(outDirSuffix)
   condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: PublishTestResults@2
+  displayName: 'Publish Test Results'
+  inputs:
+    testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml'
+    mergeTestResults: true
+    testRunTitle: '$(System.PullRequest.TargetBranch)-$(outDirSuffix)'
+    platform: $(outDirSuffix)
+  condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))

Lib/test/eintrdata/eintr_tester.py

@@ -52,7 +52,8 @@ def setUpClass(cls):
 
         # Issue #25277: Use faulthandler to try to debug a hang on FreeBSD
         if hasattr(faulthandler, 'dump_traceback_later'):
-            faulthandler.dump_traceback_later(10 * 60, exit=True)
+            faulthandler.dump_traceback_later(10 * 60, exit=True,
+                                              file=sys.__stderr__)
 
     @classmethod
     def stop_alarm(cls):

Lib/test/libregrtest/cmdline.py

@@ -268,6 +268,10 @@ def _create_parser():
                        help='if a test file alters the environment, mark '
                             'the test as failed')
 
+    group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
+                       help='writes JUnit-style XML results to the specified '
+                            'file')
+
     return parser
 
 

Lib/test/libregrtest/main.py

@@ -100,8 +100,11 @@ def __init__(self):
         self.next_single_test = None
         self.next_single_filename = None
 
+        # used by --junit-xml
+        self.testsuite_xml = None
+
     def accumulate_result(self, test, result):
-        ok, test_time = result
+        ok, test_time, xml_data = result
         if ok not in (CHILD_ERROR, INTERRUPTED):
             self.test_times.append((test_time, test))
         if ok == PASSED:
@@ -118,6 +121,15 @@ def accumulate_result(self, test, result):
         elif ok != INTERRUPTED:
             raise ValueError("invalid test result: %r" % ok)
 
+        if xml_data:
+            import xml.etree.ElementTree as ET
+            for e in xml_data:
+                try:
+                    self.testsuite_xml.append(ET.fromstring(e))
+                except ET.ParseError:
+                    print(xml_data, file=sys.__stderr__)
+                    raise
+
     def display_progress(self, test_index, test):
         if self.ns.quiet:
             return
@@ -164,6 +176,9 @@ def parse_args(self, kwargs):
                       file=sys.stderr)
                 ns.findleaks = False
 
+        if ns.xmlpath:
+            support.junit_xml_list = self.testsuite_xml = []
+
         # Strip .py extensions.
         removepy(ns.args)
 
@@ -384,7 +399,7 @@ def run_tests_sequential(self):
                     result = runtest(self.ns, test)
                 except KeyboardInterrupt:
                     self.interrupted = True
-                    self.accumulate_result(test, (INTERRUPTED, None))
+                    self.accumulate_result(test, (INTERRUPTED, None, None))
                     break
                 else:
                     self.accumulate_result(test, result)
@@ -508,6 +523,31 @@ def finalize(self):
         if self.ns.runleaks:
             os.system("leaks %d" % os.getpid())
 
+    def save_xml_result(self):
+        if not self.ns.xmlpath and not self.testsuite_xml:
+            return
+
+        import xml.etree.ElementTree as ET
+        root = ET.Element("testsuites")
+
+        # Manually count the totals for the overall summary
+        totals = {'tests': 0, 'errors': 0, 'failures': 0}
+        for suite in self.testsuite_xml:
+            root.append(suite)
+            for k in totals:
+                try:
+                    totals[k] += int(suite.get(k, 0))
+                except ValueError:
+                    pass
+
+        for k, v in totals.items():
+            root.set(k, str(v))
+
+        xmlpath = os.path.join(support.SAVEDCWD, self.ns.xmlpath)
+        with open(xmlpath, 'wb') as f:
+            for s in ET.tostringlist(root):
+                f.write(s)
+
     def main(self, tests=None, **kwargs):
         global TEMPDIR
 
@@ -570,6 +610,9 @@ def _main(self, tests, kwargs):
             self.rerun_failed_tests()
 
         self.finalize()
+
+        self.save_xml_result()
+
         if self.bad:
             sys.exit(2)
         if self.interrupted:

Lib/test/libregrtest/runtest.py

@@ -85,15 +85,18 @@ def runtest(ns, test):
     ns -- regrtest namespace of options
     test -- the name of the test
 
-    Returns the tuple (result, test_time), where result is one of the
-    constants:
+    Returns the tuple (result, test_time, xml_data), where result is one
+    of the constants:
 
         INTERRUPTED      KeyboardInterrupt when run under -j
         RESOURCE_DENIED  test skipped because resource denied
         SKIPPED          test skipped for some other reason
         ENV_CHANGED      test failed because it changed the execution environment
         FAILED           test failed
         PASSED           test passed
+
+    If ns.xmlpath is not None, xml_data is a list containing each
+    generated testsuite element.
     """
 
     output_on_failure = ns.verbose3
@@ -106,22 +109,13 @@ def runtest(ns, test):
         # reset the environment_altered flag to detect if a test altered
         # the environment
         support.environment_altered = False
+        support.junit_xml_list = xml_list = [] if ns.xmlpath else None
         if ns.failfast:
             support.failfast = True
         if output_on_failure:
             support.verbose = True
 
-            # Reuse the same instance to all calls to runtest(). Some
-            # tests keep a reference to sys.stdout or sys.stderr
-            # (eg. test_argparse).
-            if runtest.stringio is None:
-                stream = io.StringIO()
-                runtest.stringio = stream
-            else:
-                stream = runtest.stringio
-                stream.seek(0)
-                stream.truncate()
-
+            stream = io.StringIO()
             orig_stdout = sys.stdout
             orig_stderr = sys.stderr
             try:
@@ -138,12 +132,18 @@ def runtest(ns, test):
         else:
             support.verbose = ns.verbose  # Tell tests to be moderately quiet
             result = runtest_inner(ns, test, display_failure=not ns.verbose)
-        return result
+
+        if xml_list:
+            import xml.etree.ElementTree as ET
+            xml_data = [ET.tostring(x).decode('us-ascii') for x in xml_list]
+        else:
+            xml_data = None
+        return result + (xml_data,)
     finally:
         if use_timeout:
             faulthandler.cancel_dump_traceback_later()
         cleanup_test_droppings(test, ns.verbose)
-runtest.stringio = None
+        support.junit_xml_list = None
 
 
 def post_test_cleanup():

Lib/test/libregrtest/runtest_mp.py

@@ -67,7 +67,7 @@ def run_tests_worker(worker_args):
     try:
         result = runtest(ns, testname)
     except KeyboardInterrupt:
-        result = INTERRUPTED, ''
+        result = INTERRUPTED, '', None
     except BaseException as e:
         traceback.print_exc()
         result = CHILD_ERROR, str(e)
@@ -122,7 +122,7 @@ def _runtest(self):
             self.current_test = None
 
         if retcode != 0:
-            result = (CHILD_ERROR, "Exit code %s" % retcode)
+            result = (CHILD_ERROR, "Exit code %s" % retcode, None)
             self.output.put((test, stdout.rstrip(), stderr.rstrip(),
                              result))
             return False
@@ -133,6 +133,7 @@ def _runtest(self):
             return True
 
         result = json.loads(result)
+        assert len(result) == 3, f"Invalid result tuple: {result!r}"
         self.output.put((test, stdout.rstrip(), stderr.rstrip(),
                          result))
         return False
@@ -195,7 +196,7 @@ def get_running(workers):
             regrtest.accumulate_result(test, result)
 
             # Display progress
-            ok, test_time = result
+            ok, test_time, xml_data = result
             text = format_test_result(test, ok)
             if (ok not in (CHILD_ERROR, INTERRUPTED)
                 and test_time >= PROGRESS_MIN_TIME

Lib/test/support/__init__.py

@@ -6,13 +6,15 @@
 import asyncio.events
 import collections.abc
 import contextlib
+import datetime
 import errno
 import faulthandler
 import fnmatch
 import functools
 import gc
 import importlib
 import importlib.util
+import io
 import logging.handlers
 import nntplib
 import os
@@ -34,6 +36,8 @@
 import urllib.error
 import warnings
 
+from .testresult import get_test_runner
+
 try:
     import multiprocessing.process
 except ImportError:
@@ -295,6 +299,7 @@ def get_attribute(obj, name):
 max_memuse = 0           # Disable bigmem tests (they will still be run with
                          # small sizes, to make sure they work.)
 real_max_memuse = 0
+junit_xml_list = None    # list of testsuite XML elements
 failfast = False
 
 # _original_stdout is meant to hold stdout at the time regrtest began.
@@ -1891,13 +1896,16 @@ def _filter_suite(suite, pred):
 
 def _run_suite(suite):
     """Run tests from a unittest.TestSuite-derived class."""
-    if verbose:
-        runner = unittest.TextTestRunner(sys.stdout, verbosity=2,
-                                         failfast=failfast)
-    else:
-        runner = BasicTestRunner()
+    runner = get_test_runner(sys.stdout, verbosity=verbose)
+
+    # TODO: Remove this before merging (here for easy comparison with old impl)
+    #runner = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=failfast)
 
     result = runner.run(suite)
+
+    if junit_xml_list is not None:
+        junit_xml_list.append(result.get_xml_element())
+
     if not result.wasSuccessful():
         if len(result.errors) == 1 and not result.failures:
             err = result.errors[0][1]