From be0e6ad5ebfeb340f553c5656092230304237d36 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 1 May 2022 21:13:13 -0500 Subject: [PATCH] More progress toward test-runner feature parity - Added internal tests, which can run tests inside other source files, allowing access to "private" functions and data Note this required a special bit of handling our defining and later undefining test configurations to not polute the namespace of the source file, since it can end up with test cases from different suites/configuration namespaces. - Removed unnecessary/unused permutation argument to generated test functions. - Some cleanup to progress output of test.py. --- runners/test_runner.c | 6 +- runners/test_runner.h | 4 +- scripts/test_.py | 238 ++++++++++++++++++++++++------------------ 3 files changed, 140 insertions(+), 108 deletions(-) diff --git a/runners/test_runner.c b/runners/test_runner.c index 4d65f633..7fbf6ec6 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -218,7 +218,7 @@ static void test_case_permcount( test_define_geometry(&test_geometries[geom_perm]); if (case_->filter) { - if (!case_->filter(case_perm)) { + if (!case_->filter()) { continue; } } @@ -495,7 +495,7 @@ static void run(void) { // filter? if (test_suites[i]->cases[j]->filter) { - if (!test_suites[i]->cases[j]->filter(case_perm)) { + if (!test_suites[i]->cases[j]->filter()) { printf("skipped %s#%zu\n", test_suites[i]->cases[j]->id, perm); @@ -538,7 +538,7 @@ static void run(void) { // run the test printf("running %s#%zu\n", test_suites[i]->cases[j]->id, perm); - test_suites[i]->cases[j]->run(&cfg, case_perm); + test_suites[i]->cases[j]->run(&cfg); printf("finished %s#%zu\n", test_suites[i]->cases[j]->id, perm); diff --git a/runners/test_runner.h b/runners/test_runner.h index c413a85a..addb898e 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -24,8 +24,8 @@ struct test_case { const test_define_t *const *defines; const uint8_t *define_map; - bool (*filter)(uint32_t perm); - void (*run)(struct lfs_config *cfg, uint32_t perm); + bool (*filter)(void); + void (*run)(struct lfs_config *cfg); }; struct test_suite { diff --git a/scripts/test_.py b/scripts/test_.py index 55e12722..6e94f901 100755 --- a/scripts/test_.py +++ b/scripts/test_.py @@ -71,13 +71,15 @@ class TestCase: self.if_lineno = config.pop('if_lineno', None) self.code = config.pop('code') self.code_lineno = config.pop('code_lineno', None) + self.in_ = config.pop('in', + config.pop('suite_in', None)) self.normal = config.pop('normal', - config.pop('suite_normal', True)) + config.pop('suite_normal', True)) self.reentrant = config.pop('reentrant', - config.pop('suite_reentrant', False)) + config.pop('suite_reentrant', False)) self.valgrind = config.pop('valgrind', - config.pop('suite_valgrind', True)) + config.pop('suite_valgrind', True)) # figure out defines and build possible permutations self.defines = set() @@ -171,6 +173,7 @@ class TestSuite: # a couple of these we just forward to all cases defines = config.pop('defines', {}) + in_ = config.pop('in', None) normal = config.pop('normal', True) reentrant = config.pop('reentrant', False) valgrind = config.pop('valgrind', True) @@ -184,6 +187,7 @@ class TestSuite: if 'lineno' in case else ''), 'suite': self.name, 'suite_defines': defines, + 'suite_in': in_, 'suite_normal': normal, 'suite_reentrant': reentrant, 'suite_valgrind': valgrind, @@ -264,54 +268,15 @@ def compile(**args): 'LFS_TRACE_(__VA_ARGS__, "")') f.writeln() - if not args.get('source'): - # write test suite prologue - f.writeln('%s' % SUITE_PROLOGUE.strip()) - f.writeln() - if suite.code is not None: - if suite.code_lineno is not None: - f.writeln('#line %d "%s"' - % (suite.code_lineno, suite.path)) - f.write(suite.code) - if suite.code_lineno is not None: - f.writeln('#line %d "%s"' - % (f.lineno+1, args['output'])) - f.writeln() - - for i, define in enumerate(sorted(suite.defines)): - f.writeln('#ifndef %s' % define) - f.writeln('#define %-24s test_define(%d)' % (define, i)) - f.writeln('#endif') - f.writeln() - - for case in suite.cases: - # create case defines - if case.defines: - f.writeln('const test_define_t *const ' - '__test__%s__%s__defines[] = {' - % (suite.name, case.name)) - for permutation in case.permutations: - f.writeln(4*' '+'(const test_define_t[]){%s},' - % ', '.join(str(v) for _, v in sorted( - permutation.items()))) - f.writeln('};') - f.writeln() - - f.writeln('const uint8_t ' - '__test__%s__%s__define_map[] = {' - % (suite.name, case.name)) - f.writeln(4*' '+'%s,' - % ', '.join( - str(sorted(case.defines).index(k)) - if k in case.defines else '0xff' - for k in sorted(suite.defines))) - f.writeln('};') - f.writeln() - + # write out generated functions, this can end up in different + # files depending on the "in" attribute + # + # note it's up to the specific generated file to declare + # the test defines + def write_case_functions(f, suite, case): # create case filter function if suite.if_ is not None or case.if_ is not None: - f.writeln('bool __test__%s__%s__filter(' - '__attribute__((unused)) uint32_t perm) {' + f.writeln('bool __test__%s__%s__filter(void) {' % (suite.name, case.name)) if suite.if_ is not None: if suite.if_lineno is not None: @@ -341,8 +306,7 @@ def compile(**args): # create case run function f.writeln('void __test__%s__%s__run(' - '__attribute__((unused)) struct lfs_config *cfg, ' - '__attribute__((unused)) uint32_t perm) {' + '__attribute__((unused)) struct lfs_config *cfg) {' % (suite.name, case.name)) if CASE_PROLOGUE.strip(): f.writeln(4*' '+'%s' @@ -363,6 +327,65 @@ def compile(**args): f.writeln('}') f.writeln() + if not args.get('source'): + # write test suite prologue + f.writeln('%s' % SUITE_PROLOGUE.strip()) + f.writeln() + if suite.code is not None: + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (suite.code_lineno, suite.path)) + f.write(suite.code) + if suite.code_lineno is not None: + f.writeln('#line %d "%s"' + % (f.lineno+1, args['output'])) + f.writeln() + + if suite.defines: + for i, define in enumerate(sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s test_define(%d)' + % (define, i)) + f.writeln('#endif') + f.writeln() + + for case in suite.cases: + # create case defines + if case.defines: + f.writeln('const test_define_t *const ' + '__test__%s__%s__defines[] = {' + % (suite.name, case.name)) + for permutation in case.permutations: + f.writeln(4*' '+'(const test_define_t[]){%s},' + % ', '.join(str(v) for _, v in sorted( + permutation.items()))) + f.writeln('};') + f.writeln() + + f.writeln('const uint8_t ' + '__test__%s__%s__define_map[] = {' + % (suite.name, case.name)) + f.writeln(4*' '+'%s,' + % ', '.join( + str(sorted(case.defines).index(k)) + if k in case.defines else '0xff' + for k in sorted(suite.defines))) + f.writeln('};') + f.writeln() + + # create case functions + if case.in_ is None: + write_case_functions(f, suite, case) + else: + if suite.if_ is not None or case.if_ is not None: + f.writeln('extern bool __test__%s__%s__filter(' + 'void);' + % (suite.name, case.name)) + f.writeln('extern void __test__%s__%s__run(' + 'struct lfs_config *cfg);' + % (suite.name, case.name)) + f.writeln() + # create case struct f.writeln('const struct test_case __test__%s__%s__case = {' % (suite.name, case.name)) @@ -433,6 +456,35 @@ def compile(**args): f.write(SUITE_PROLOGUE) f.writeln() + # write any internal tests + for suite in suites: + for case in suite.cases: + if case.in_ == args.get('source'): + # write defines, but note we need to undef any + # new defines since we're in someone else's file + if suite.defines: + for i, define in enumerate( + sorted(suite.defines)): + f.writeln('#ifndef %s' % define) + f.writeln('#define %-24s test_define(%d)' + % (define, i)) + f.writeln('#define __TEST__%s__NEEDS_UNDEF' + % define) + f.writeln('#endif') + f.writeln() + + write_case_functions(f, suite, case) + + if suite.defines: + for define in sorted(suite.defines): + f.writeln('#ifdef __TEST__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef __TEST__%s__NEEDS_UNDEF' + % define) + f.writeln('#undef %s' % define) + f.writeln('#endif') + f.writeln() + # add suite info to test_runner.c if args['source'] == 'runners/test_runner.c': f.writeln() @@ -738,21 +790,25 @@ def run_stage(name, runner_, **args): if not args.get('verbose'): sys.stdout.write('\r\x1b[K' - 'running \x1b[%dm%s:\x1b[m ' - '%d/%d suites, %d/%d cases, %d/%d perms%s ' + 'running \x1b[%dm%s:\x1b[m %s ' % (32 if not failures else 31, name, - sum(passed_suite_perms[k] == v - for k, v in expected_suite_perms.items()), - len(expected_suite_perms), - sum(passed_case_perms[k] == v - for k, v in expected_case_perms.items()), - len(expected_case_perms), - passed_perms, - expected_perms, - ', \x1b[31m%d/%d failures\x1b[m' - % (len(failures), expected_perms) - if failures else '')) + ', '.join(filter(None, [ + '%d/%d suites' % ( + sum(passed_suite_perms[k] == v + for k, v in expected_suite_perms.items()), + len(expected_suite_perms)) + if (not args.get('by_suites') + and not args.get('by_cases')) else None, + '%d/%d cases' % ( + sum(passed_case_perms[k] == v + for k, v in expected_case_perms.items()), + len(expected_case_perms)) + if not args.get('by_cases') else None, + '%d/%d perms' % (passed_perms, expected_perms), + '\x1b[31m%d/%d failures\x1b[m' + % (len(failures), expected_perms) + if failures else None])))) sys.stdout.flush() needs_newline = True except KeyboardInterrupt: @@ -791,45 +847,21 @@ def run(**args): expected = 0 passed = 0 failures = [] - if args.get('by_cases'): - for type in ['normal', 'reentrant', 'valgrind']: - for case in expected_case_perms.keys(): - expected_, passed_, failures_, killed = run_stage( - '%s %s' % (type, case), - runner_ + ['--%s' % type, case], - **args) - expected += expected_ - passed += passed_ - failures.extend(failures_) - if (failures and not args.get('keep_going')) or killed: - break - if (failures and not args.get('keep_going')) or killed: - break - elif args.get('by_suites'): - for type in ['normal', 'reentrant', 'valgrind']: - for suite in expected_suite_perms.keys(): - expected_, passed_, failures_, killed = run_stage( - '%s %s' % (type, suite), - runner_ + ['--%s' % type, suite], - **args) - expected += expected_ - passed += passed_ - failures.extend(failures_) - if (failures and not args.get('keep_going')) or killed: - break - if (failures and not args.get('keep_going')) or killed: - break - else: - for type in ['normal', 'reentrant', 'valgrind']: - expected_, passed_, failures_, killed = run_stage( - '%s tests' % type, - runner_ + ['--%s' % type], - **args) - expected += expected_ - passed += passed_ - failures.extend(failures_) - if (failures and not args.get('keep_going')) or killed: - break + for type, by in it.product( + ['normal', 'reentrant', 'valgrind'], + expected_case_perms.keys() if args.get('by_cases') + else expected_suite_perms.keys() if args.get('by_suites') + else [None]): + + expected_, passed_, failures_, killed = run_stage( + '%s %s' % (type, by or 'tests'), + runner_ + ['--%s' % type] + ([by] if by is not None else []), + **args) + expected += expected_ + passed += passed_ + failures.extend(failures_) + if (failures and not args.get('keep_going')) or killed: + break # show summary print()