1berry/cpython
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bpo-25894: Always report skipped and failed subtests separately (GH-28082)

Browse Source 
* In default mode output separate characters for skipped and failed subtests.
* In verbose mode output separate lines (including description) for skipped
   and failed subtests.
* In verbose mode output test description for errors in test cleanup.
This commit is contained in:
Serhiy Storchaka 2021-09-10 18:55:05 +03:00 committed by GitHub
parent ab327f2929
commit f0f29f328d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 253 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
Lib/unittest/runner.py
View File

@ -5,6 +5,7 @@ import time
import warnings
from . import result
from .case import _SubTest
from .signals import registerResult
__unittest = True
@ -40,6 +41,7 @@ class TextTestResult(result.TestResult):
self.showAll = verbosity > 1
self.dots = verbosity == 1
self.descriptions = descriptions
self._newline = True
def getDescription(self, test):
doc_first_line = test.shortDescription()
@ -54,11 +56,39 @@ class TextTestResult(result.TestResult):
self.stream.write(self.getDescription(test))
self.stream.write(" ... ")
self.stream.flush()
self._newline = False
def _write_status(self, test, status):
is_subtest = isinstance(test, _SubTest)
if is_subtest or self._newline:
if not self._newline:
self.stream.writeln()
if is_subtest:
self.stream.write(" ")
self.stream.write(self.getDescription(test))
self.stream.write(" ... ")
self.stream.writeln(status)
self._newline = True
def addSubTest(self, test, subtest, err):
if err is not None:
if self.showAll:
if issubclass(err[0], subtest.failureException):
self._write_status(subtest, "FAIL")
else:
self._write_status(subtest, "ERROR")
elif self.dots:
if issubclass(err[0], subtest.failureException):
self.stream.write('F')
else:
self.stream.write('E')
self.stream.flush()
super(TextTestResult, self).addSubTest(test, subtest, err)
def addSuccess(self, test):
super(TextTestResult, self).addSuccess(test)
if self.showAll:
self.stream.writeln("ok")
self._write_status(test, "ok")
elif self.dots:
self.stream.write('.')
self.stream.flush()
@ -66,7 +96,7 @@ class TextTestResult(result.TestResult):
def addError(self, test, err):
super(TextTestResult, self).addError(test, err)
if self.showAll:
self.stream.writeln("ERROR")
self._write_status(test, "ERROR")
elif self.dots:
self.stream.write('E')
self.stream.flush()
@ -74,7 +104,7 @@ class TextTestResult(result.TestResult):
def addFailure(self, test, err):
super(TextTestResult, self).addFailure(test, err)
if self.showAll:
self.stream.writeln("FAIL")
self._write_status(test, "FAIL")
elif self.dots:
self.stream.write('F')
self.stream.flush()
@ -82,7 +112,7 @@ class TextTestResult(result.TestResult):
def addSkip(self, test, reason):
super(TextTestResult, self).addSkip(test, reason)
if self.showAll:
self.stream.writeln("skipped {0!r}".format(reason))
self._write_status(test, "skipped {0!r}".format(reason))
elif self.dots:
self.stream.write("s")
self.stream.flush()

324
Lib/unittest/test/test_result.py
View File

@ -305,115 +305,6 @@ class Test_TestResult(unittest.TestCase):
self.assertIs(test_case, subtest)
self.assertIn("some recognizable failure", formatted_exc)
def testGetDescriptionWithoutDocstring(self):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self),
'testGetDescriptionWithoutDocstring (' + __name__ +
'.Test_TestResult)')
def testGetSubTestDescriptionWithoutDocstring(self):
with self.subTest(foo=1, bar=2):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
'.Test_TestResult) (foo=1, bar=2)')
with self.subTest('some message'):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
'.Test_TestResult) [some message]')
def testGetSubTestDescriptionWithoutDocstringAndParams(self):
with self.subTest():
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstringAndParams '
'(' + __name__ + '.Test_TestResult) (<subtest>)')
def testGetSubTestDescriptionForFalsyValues(self):
expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TestResult) [%s]'
result = unittest.TextTestResult(None, True, 1)
for arg in [0, None, []]:
with self.subTest(arg):
self.assertEqual(
result.getDescription(self._subtest),
expected % (__name__, arg)
)
def testGetNestedSubTestDescriptionWithoutDocstring(self):
with self.subTest(foo=1):
with self.subTest(baz=2, bar=3):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetNestedSubTestDescriptionWithoutDocstring '
'(' + __name__ + '.Test_TestResult) (baz=2, bar=3, foo=1)')
def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
with self.subTest(foo=1, bar=2):
with self.subTest(baz=3, bar=4):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetDuplicatedNestedSubTestDescriptionWithoutDocstring '
'(' + __name__ + '.Test_TestResult) (baz=3, bar=4, foo=1)')
@unittest.skipIf(sys.flags.optimize >= 2,
"Docstrings are omitted with -O2 and above")
def testGetDescriptionWithOneLineDocstring(self):
"""Tests getDescription() for a method with a docstring."""
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self),
('testGetDescriptionWithOneLineDocstring '
'(' + __name__ + '.Test_TestResult)\n'
'Tests getDescription() for a method with a docstring.'))
@unittest.skipIf(sys.flags.optimize >= 2,
"Docstrings are omitted with -O2 and above")
def testGetSubTestDescriptionWithOneLineDocstring(self):
"""Tests getDescription() for a method with a docstring."""
result = unittest.TextTestResult(None, True, 1)
with self.subTest(foo=1, bar=2):
self.assertEqual(
result.getDescription(self._subtest),
('testGetSubTestDescriptionWithOneLineDocstring '
'(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
'Tests getDescription() for a method with a docstring.'))
@unittest.skipIf(sys.flags.optimize >= 2,
"Docstrings are omitted with -O2 and above")
def testGetDescriptionWithMultiLineDocstring(self):
"""Tests getDescription() for a method with a longer docstring.
The second line of the docstring.
"""
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self),
('testGetDescriptionWithMultiLineDocstring '
'(' + __name__ + '.Test_TestResult)\n'
'Tests getDescription() for a method with a longer '
'docstring.'))
@unittest.skipIf(sys.flags.optimize >= 2,
"Docstrings are omitted with -O2 and above")
def testGetSubTestDescriptionWithMultiLineDocstring(self):
"""Tests getDescription() for a method with a longer docstring.
The second line of the docstring.
"""
result = unittest.TextTestResult(None, True, 1)
with self.subTest(foo=1, bar=2):
self.assertEqual(
result.getDescription(self._subtest),
('testGetSubTestDescriptionWithMultiLineDocstring '
'(' + __name__ + '.Test_TestResult) (foo=1, bar=2)\n'
'Tests getDescription() for a method with a longer '
'docstring.'))
def testStackFrameTrimming(self):
class Frame(object):
class tb_frame(object):
@ -450,6 +341,221 @@ class Test_TestResult(unittest.TestCase):
result = runner.run(test)
class Test_TextTestResult(unittest.TestCase):
maxDiff = None
def testGetDescriptionWithoutDocstring(self):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self),
'testGetDescriptionWithoutDocstring (' + __name__ +
'.Test_TextTestResult)')
def testGetSubTestDescriptionWithoutDocstring(self):
with self.subTest(foo=1, bar=2):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
'.Test_TextTestResult) (foo=1, bar=2)')
with self.subTest('some message'):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstring (' + __name__ +
'.Test_TextTestResult) [some message]')
def testGetSubTestDescriptionWithoutDocstringAndParams(self):
with self.subTest():
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetSubTestDescriptionWithoutDocstringAndParams '
'(' + __name__ + '.Test_TextTestResult) (<subtest>)')
def testGetSubTestDescriptionForFalsyValues(self):
expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TextTestResult) [%s]'
result = unittest.TextTestResult(None, True, 1)
for arg in [0, None, []]:
with self.subTest(arg):
self.assertEqual(
result.getDescription(self._subtest),
expected % (__name__, arg)
)
def testGetNestedSubTestDescriptionWithoutDocstring(self):
with self.subTest(foo=1):
with self.subTest(baz=2, bar=3):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetNestedSubTestDescriptionWithoutDocstring '
'(' + __name__ + '.Test_TextTestResult) (baz=2, bar=3, foo=1)')
def testGetDuplicatedNestedSubTestDescriptionWithoutDocstring(self):
with self.subTest(foo=1, bar=2):
with self.subTest(baz=3, bar=4):
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self._subtest),
'testGetDuplicatedNestedSubTestDescriptionWithoutDocstring '
'(' + __name__ + '.Test_TextTestResult) (baz=3, bar=4, foo=1)')
@unittest.skipIf(sys.flags.optimize >= 2,
"Docstrings are omitted with -O2 and above")
def testGetDescriptionWithOneLineDocstring(self):
"""Tests getDescription() for a method with a docstring."""
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self),
('testGetDescriptionWithOneLineDocstring '
'(' + __name__ + '.Test_TextTestResult)\n'
'Tests getDescription() for a method with a docstring.'))
@unittest.skipIf(sys.flags.optimize >= 2,
"Docstrings are omitted with -O2 and above")
def testGetSubTestDescriptionWithOneLineDocstring(self):
"""Tests getDescription() for a method with a docstring."""
result = unittest.TextTestResult(None, True, 1)
with self.subTest(foo=1, bar=2):
self.assertEqual(
result.getDescription(self._subtest),
('testGetSubTestDescriptionWithOneLineDocstring '
'(' + __name__ + '.Test_TextTestResult) (foo=1, bar=2)\n'
'Tests getDescription() for a method with a docstring.'))
@unittest.skipIf(sys.flags.optimize >= 2,
"Docstrings are omitted with -O2 and above")
def testGetDescriptionWithMultiLineDocstring(self):
"""Tests getDescription() for a method with a longer docstring.
The second line of the docstring.
"""
result = unittest.TextTestResult(None, True, 1)
self.assertEqual(
result.getDescription(self),
('testGetDescriptionWithMultiLineDocstring '
'(' + __name__ + '.Test_TextTestResult)\n'
'Tests getDescription() for a method with a longer '
'docstring.'))
@unittest.skipIf(sys.flags.optimize >= 2,
"Docstrings are omitted with -O2 and above")
def testGetSubTestDescriptionWithMultiLineDocstring(self):
"""Tests getDescription() for a method with a longer docstring.
The second line of the docstring.
"""
result = unittest.TextTestResult(None, True, 1)
with self.subTest(foo=1, bar=2):
self.assertEqual(
result.getDescription(self._subtest),
('testGetSubTestDescriptionWithMultiLineDocstring '
'(' + __name__ + '.Test_TextTestResult) (foo=1, bar=2)\n'
'Tests getDescription() for a method with a longer '
'docstring.'))
class Test(unittest.TestCase):
def testSuccess(self):
pass
def testSkip(self):
self.skipTest('skip')
def testFail(self):
self.fail('fail')
def testError(self):
raise Exception('error')
def testSubTestSuccess(self):
with self.subTest('one', a=1):
pass
with self.subTest('two', b=2):
pass
def testSubTestMixed(self):
with self.subTest('success', a=1):
pass
with self.subTest('skip', b=2):
self.skipTest('skip')
with self.subTest('fail', c=3):
self.fail('fail')
with self.subTest('error', d=4):
raise Exception('error')
tearDownError = None
def tearDown(self):
if self.tearDownError is not None:
raise self.tearDownError
def _run_test(self, test_name, verbosity, tearDownError=None):
stream = io.StringIO()
stream = unittest.runner._WritelnDecorator(stream)
result = unittest.TextTestResult(stream, True, verbosity)
test = self.Test(test_name)
test.tearDownError = tearDownError
test.run(result)
return stream.getvalue()
def testDotsOutput(self):
self.assertEqual(self._run_test('testSuccess', 1), '.')
self.assertEqual(self._run_test('testSkip', 1), 's')
self.assertEqual(self._run_test('testFail', 1), 'F')
self.assertEqual(self._run_test('testError', 1), 'E')
def testLongOutput(self):
classname = f'{__name__}.{self.Test.__qualname__}'
self.assertEqual(self._run_test('testSuccess', 2),
f'testSuccess ({classname}) ... ok\n')
self.assertEqual(self._run_test('testSkip', 2),
f"testSkip ({classname}) ... skipped 'skip'\n")
self.assertEqual(self._run_test('testFail', 2),
f'testFail ({classname}) ... FAIL\n')
self.assertEqual(self._run_test('testError', 2),
f'testError ({classname}) ... ERROR\n')
def testDotsOutputSubTestSuccess(self):
self.assertEqual(self._run_test('testSubTestSuccess', 1), '.')
def testLongOutputSubTestSuccess(self):
classname = f'{__name__}.{self.Test.__qualname__}'
self.assertEqual(self._run_test('testSubTestSuccess', 2),
f'testSubTestSuccess ({classname}) ... ok\n')
def testDotsOutputSubTestMixed(self):
self.assertEqual(self._run_test('testSubTestMixed', 1), 'sFE')
def testLongOutputSubTestMixed(self):
classname = f'{__name__}.{self.Test.__qualname__}'
self.assertEqual(self._run_test('testSubTestMixed', 2),
f'testSubTestMixed ({classname}) ... \n'
f" testSubTestMixed ({classname}) [skip] (b=2) ... skipped 'skip'\n"
f' testSubTestMixed ({classname}) [fail] (c=3) ... FAIL\n'
f' testSubTestMixed ({classname}) [error] (d=4) ... ERROR\n')
def testDotsOutputTearDownFail(self):
out = self._run_test('testSuccess', 1, AssertionError('fail'))
self.assertEqual(out, 'F')
out = self._run_test('testError', 1, AssertionError('fail'))
self.assertEqual(out, 'EF')
out = self._run_test('testFail', 1, Exception('error'))
self.assertEqual(out, 'FE')
out = self._run_test('testSkip', 1, AssertionError('fail'))
self.assertEqual(out, 'sF')
def testLongOutputTearDownFail(self):
classname = f'{__name__}.{self.Test.__qualname__}'
out = self._run_test('testSuccess', 2, AssertionError('fail'))
self.assertEqual(out,
f'testSuccess ({classname}) ... FAIL\n')
out = self._run_test('testError', 2, AssertionError('fail'))
self.assertEqual(out,
f'testError ({classname}) ... ERROR\n'
f'testError ({classname}) ... FAIL\n')
out = self._run_test('testFail', 2, Exception('error'))
self.assertEqual(out,
f'testFail ({classname}) ... FAIL\n'
f'testFail ({classname}) ... ERROR\n')
out = self._run_test('testSkip', 2, AssertionError('fail'))
self.assertEqual(out,
f"testSkip ({classname}) ... skipped 'skip'\n"
f'testSkip ({classname}) ... FAIL\n')
classDict = dict(unittest.TestResult.__dict__)
for m in ('addSkip', 'addExpectedFailure', 'addUnexpectedSuccess',
'__init__'):

4
Misc/NEWS.d/next/Library/2021-09-05-13-15-08.bpo-25894.zjbi2f.rst Normal file
View File

@ -0,0 +1,4 @@
:mod:`unittest` now always reports skipped and failed subtests separately:
separate characters in default mode and separate lines in verbose mode. Also
the test description is now output for errors in test method, class and
module cleanups.