| diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py |
| index 538ff05..e7f2013 100644 |
| |
| |
| @@ -263,7 +263,9 @@ def _create_parser(): |
| help='only write the name of test cases that will be run' |
| ' , don\'t execute them') |
| group.add_argument('-P', '--pgo', dest='pgo', action='store_true', |
| - help='enable Profile Guided Optimization training') |
| + help='enable Profile Guided Optimization (PGO) training') |
| + group.add_argument('--pgo-extended', action='store_true', |
| + help='enable extended PGO training (slower training)') |
| group.add_argument('--fail-env-changed', action='store_true', |
| help='if a test file alters the environment, mark ' |
| 'the test as failed') |
| @@ -339,6 +341,8 @@ def _parse_args(args, **kwargs): |
| parser.error("-G/--failfast needs either -v or -W") |
| if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): |
| parser.error("--pgo/-v don't go together!") |
| + if ns.pgo_extended: |
| + ns.pgo = True # pgo_extended implies pgo |
| |
| if ns.nowindows: |
| print("Warning: the --nowindows (-n) option is deprecated. " |
| diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py |
| index b6d05f6..524dbfa 100644 |
| |
| |
| @@ -17,6 +17,7 @@ from test.libregrtest.runtest import ( |
| INTERRUPTED, CHILD_ERROR, TEST_DID_NOT_RUN, |
| PROGRESS_MIN_TIME, format_test_result) |
| from test.libregrtest.setup import setup_tests |
| +from test.libregrtest.pgo import setup_pgo_tests |
| from test.libregrtest.utils import removepy, count, format_duration, printlist |
| from test import support |
| try: |
| @@ -214,6 +215,10 @@ class Regrtest: |
| |
| removepy(self.tests) |
| |
| + if self.ns.pgo: |
| + # add default PGO tests if no tests are specified |
| + setup_pgo_tests(self.ns) |
| + |
| stdtests = STDTESTS[:] |
| nottests = NOTTESTS.copy() |
| if self.ns.exclude: |
| @@ -601,6 +606,7 @@ class Regrtest: |
| input("Press any key to continue...") |
| |
| support.PGO = self.ns.pgo |
| + support.PGO_EXTENDED = self.ns.pgo_extended |
| |
| setup_tests(self.ns) |
| |
| diff --git a/Lib/test/libregrtest/pgo.py b/Lib/test/libregrtest/pgo.py |
| new file mode 100644 |
| index 0000000..379ff05 |
| |
| |
| @@ -0,0 +1,55 @@ |
| +# Set of tests run by default if --pgo is specified. The tests below were |
| +# chosen based on the following criteria: either they exercise a commonly used |
| +# C extension module or type, or they run some relatively typical Python code. |
| +# Long running tests should be avoided because the PGO instrumented executable |
| +# runs slowly. |
| +PGO_TESTS = [ |
| + 'test_array', |
| + 'test_base64', |
| + 'test_binascii', |
| + 'test_binop', |
| + 'test_bisect', |
| + 'test_bytes', |
| + 'test_bz2', |
| + 'test_cmath', |
| + 'test_codecs', |
| + 'test_collections', |
| + 'test_complex', |
| + 'test_dataclasses', |
| + 'test_datetime', |
| + 'test_decimal', |
| + 'test_difflib', |
| + 'test_embed', |
| + 'test_float', |
| + 'test_fstring', |
| + 'test_functools', |
| + 'test_generators', |
| + 'test_hashlib', |
| + 'test_heapq', |
| + 'test_int', |
| + 'test_itertools', |
| + 'test_json', |
| + 'test_long', |
| + 'test_lzma', |
| + 'test_math', |
| + 'test_memoryview', |
| + 'test_operator', |
| + 'test_ordered_dict', |
| + 'test_pickle', |
| + 'test_pprint', |
| + 'test_re', |
| + 'test_set', |
| + 'test_sqlite', |
| + 'test_statistics', |
| + 'test_struct', |
| + 'test_tabnanny', |
| + 'test_time', |
| + 'test_unicode', |
| + 'test_xml_etree', |
| + 'test_xml_etree_c', |
| +] |
| + |
| +def setup_pgo_tests(ns): |
| + if not ns.args and not ns.pgo_extended: |
| + # run default set of tests for PGO training |
| + ns.args = PGO_TESTS[:] |
| diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py |
| index 764057a..468ee46 100644 |
| |
| |
| @@ -2039,6 +2039,7 @@ class AbstractPickleTests(unittest.TestCase): |
| |
| FRAME_SIZE_TARGET = 64 * 1024 |
| |
| + @support.skip_if_pgo_task |
| def check_frame_opcodes(self, pickled): |
| """ |
| Check the arguments of FRAME opcodes in a protocol 4+ pickle. |
| @@ -2059,6 +2060,7 @@ class AbstractPickleTests(unittest.TestCase): |
| frame_size = len(pickled) - last_pos - frame_opcode_size |
| self.assertEqual(frame_size, last_arg) |
| |
| + @support.skip_if_pgo_task |
| def test_framing_many_objects(self): |
| obj = list(range(10**5)) |
| for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): |
| diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py |
| index 66c0fed..e80a819 100644 |
| |
| |
| @@ -953,6 +953,10 @@ SAVEDCWD = os.getcwd() |
| # useful for PGO |
| PGO = False |
| |
| +# Set by libregrtest/main.py if we are running the extended (time consuming) |
| +# PGO task. If this is True, PGO is also True. |
| +PGO_EXTENDED = False |
| + |
| @contextlib.contextmanager |
| def temp_dir(path=None, quiet=False): |
| """Return a context manager that creates a temporary directory. |
| @@ -2442,6 +2446,11 @@ def skip_unless_xattr(test): |
| msg = "no non-broken extended attribute support" |
| return test if ok else unittest.skip(msg)(test) |
| |
| +def skip_if_pgo_task(test): |
| + """Skip decorator for tests not run in (non-extended) PGO task""" |
| + ok = not PGO or PGO_EXTENDED |
| + msg = "Not run for (non-extended) PGO task" |
| + return test if ok else unittest.skip(msg)(test) |
| |
| def fs_is_case_insensitive(directory): |
| """Detects if the file system for the specified directory is case-insensitive.""" |
| diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py |
| index f340f23..ebb151c 100644 |
| |
| |
| @@ -654,6 +654,7 @@ class BZ2CompressorTest(BaseTest): |
| data += bz2c.flush() |
| self.assertEqual(ext_decompress(data), self.TEXT) |
| |
| + @support.skip_if_pgo_task |
| @bigmemtest(size=_4G + 100, memuse=2) |
| def testCompress4G(self, size): |
| # "Test BZ2Compressor.compress()/flush() with >4GiB input" |
| @@ -712,6 +713,7 @@ class BZ2DecompressorTest(BaseTest): |
| self.assertRaises(EOFError, bz2d.decompress, b"anything") |
| self.assertRaises(EOFError, bz2d.decompress, b"") |
| |
| + @support.skip_if_pgo_task |
| @bigmemtest(size=_4G + 100, memuse=3.3) |
| def testDecompress4G(self, size): |
| # "Test BZ2Decompressor.decompress() with >4GiB input" |
| diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py |
| index 9317951..8c1d016 100644 |
| |
| |
| @@ -2023,6 +2023,7 @@ class RegressionTests(unittest.TestCase): |
| self.assertRaises(AssertionError, list, cycle(gen1())) |
| self.assertEqual(hist, [0,1]) |
| |
| + @support.skip_if_pgo_task |
| def test_long_chain_of_empty_iterables(self): |
| # Make sure itertools.chain doesn't run into recursion limits when |
| # dealing with long chains of empty iterables. Even with a high |
| diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py |
| index 3dc2c1e..117de0a 100644 |
| |
| |
| @@ -333,6 +333,7 @@ class CompressorDecompressorTestCase(unittest.TestCase): |
| |
| # Test with inputs larger than 4GiB. |
| |
| + @support.skip_if_pgo_task |
| @bigmemtest(size=_4G + 100, memuse=2) |
| def test_compressor_bigmem(self, size): |
| lzc = LZMACompressor() |
| @@ -344,6 +345,7 @@ class CompressorDecompressorTestCase(unittest.TestCase): |
| finally: |
| ddata = None |
| |
| + @support.skip_if_pgo_task |
| @bigmemtest(size=_4G + 100, memuse=3) |
| def test_decompressor_bigmem(self, size): |
| lzd = LZMADecompressor() |
| diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py |
| index 5347bb1..9d83217 100644 |
| |
| |
| @@ -6,6 +6,7 @@ Note: test_regrtest cannot be run twice in parallel. |
| |
| import contextlib |
| import faulthandler |
| +import glob |
| import io |
| import os.path |
| import platform |
| @@ -532,6 +533,31 @@ class BaseTestCase(unittest.TestCase): |
| return proc.stdout |
| |
| |
| +class CheckActualTests(BaseTestCase): |
| + """ |
| + Check that regrtest appears to find the expected set of tests. |
| + """ |
| + |
| + def test_finds_expected_number_of_tests(self): |
| + args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests'] |
| + output = self.run_python(args) |
| + rough_number_of_tests_found = len(output.splitlines()) |
| + actual_testsuite_glob = os.path.join(os.path.dirname(__file__), |
| + 'test*.py') |
| + rough_counted_test_py_files = len(glob.glob(actual_testsuite_glob)) |
| + # We're not trying to duplicate test finding logic in here, |
| + # just give a rough estimate of how many there should be and |
| + # be near that. This is a regression test to prevent mishaps |
| + # such as https://bugs.python.org/issue37667 in the future. |
| + # If you need to change the values in here during some |
| + # mythical future test suite reorganization, don't go |
| + # overboard with logic and keep that goal in mind. |
| + self.assertGreater(rough_number_of_tests_found, |
| + rough_counted_test_py_files*9//10, |
| + msg='Unexpectedly low number of tests found in:\n' |
| + f'{", ".join(output.splitlines())}') |
| + |
| + |
| class ProgramsTestCase(BaseTestCase): |
| """ |
| Test various ways to run the Python test suite. Use options close |
| diff --git a/Makefile.pre.in b/Makefile.pre.in |
| index b452289..cc428ac 100644 |
| |
| |
| @@ -247,9 +247,10 @@ TCLTK_INCLUDES= @TCLTK_INCLUDES@ |
| TCLTK_LIBS= @TCLTK_LIBS@ |
| |
| # The task to run while instrumented when building the profile-opt target. |
| -# We exclude unittests with -x that take a rediculious amount of time to |
| -# run in the instrumented training build or do not provide much value. |
| -PROFILE_TASK=-m test.regrtest --pgo |
| +# To speed up profile generation, we don't run the full unit test suite |
| +# by default. The default is "-m test --pgo". To run more tests, use |
| +# PROFILE_TASK="-m test --pgo-extended" |
| +PROFILE_TASK= @PROFILE_TASK@ |
| |
| # report files for gcov / lcov coverage report |
| COVERAGE_INFO= $(abs_builddir)/coverage.info |
| diff --git a/configure.ac b/configure.ac |
| index c071ec3..816fc5a 100644 |
| |
| |
| @@ -1308,6 +1308,14 @@ else |
| DEF_MAKE_RULE="all" |
| fi |
| |
| +AC_ARG_VAR(PROFILE_TASK, Python args for PGO generation task) |
| +AC_MSG_CHECKING(PROFILE_TASK) |
| +if test -z "$PROFILE_TASK" |
| +then |
| + PROFILE_TASK='-m test --pgo' |
| +fi |
| +AC_MSG_RESULT($PROFILE_TASK) |
| + |
| # Make llvm-relatec checks work on systems where llvm tools are not installed with their |
| # normal names in the default $PATH (ie: Ubuntu). They exist under the |
| # non-suffixed name in their versioned llvm directory. |