mirror of
https://github.com/littlefs-project/littlefs.git
synced 2025-12-01 12:20:02 +00:00
Reworked test_runner/bench_runner to evaluate define permutations lazily
I wondered if walking in Python 2's footsteps was going to run into the same issues and sure enough, memory backed iterators became unweildy. The motivation for this change is that large ranges in tests, such as iterators over seeds or permutations, became prohibitively expensive to compile. This meant more iteration moving into tests with more steps to reproduce failures. This sort of defeats the purpuse of the test framework. The solution here is to move test permutation generation out of test.py and into the test runner itself. The allows defines to generate their values programmatically. This does conflict with the test frameworks support of sets of explicit permutations, but this is fixed by also moving these "permutation sets" down into the test runner. I guess it turns out the closer your representation matches your implementation the better everythign works. Additionally the define caching layer got a bit of tweaking. We can't precalculate the defines because of mutual recursion, but we can precalculate which define/permutation each define id maps to. This is necessary as otherwise figuring out each define's define-specific permutation would be prohibitively expensive.
This commit is contained in:
Christopher Haster
116
scripts/test.py
116
scripts/test.py
@ -93,13 +93,13 @@ class TestCase:
|
||||
def parse_define(v):
|
||||
# a define entry can be a list
|
||||
if isinstance(v, list):
|
||||
for v_ in v:
|
||||
yield from parse_define(v_)
|
||||
return sum((parse_define(v_) for v_ in v), [])
|
||||
# or a string
|
||||
elif isinstance(v, str):
|
||||
# which can be comma-separated values, with optional
|
||||
# range statements. This matches the runtime define parser in
|
||||
# the runner itself.
|
||||
vs = []
|
||||
for v_ in csplit(v):
|
||||
m = re.search(r'\brange\b\s*\('
|
||||
'(?P<start>[^,\s]*)'
|
||||
@ -115,25 +115,24 @@ class TestCase:
|
||||
if m.group('step') else 1)
|
||||
if m.lastindex <= 1:
|
||||
start, stop = 0, start
|
||||
for x in range(start, stop, step):
|
||||
yield from parse_define('%s(%d)%s' % (
|
||||
v_[:m.start()], x, v_[m.end():]))
|
||||
vs.append(range(start, stop, step))
|
||||
else:
|
||||
yield v_
|
||||
vs.append(v_)
|
||||
return vs
|
||||
# or a literal value
|
||||
elif isinstance(v, bool):
|
||||
yield 'true' if v else 'false'
|
||||
return ['true' if v else 'false']
|
||||
else:
|
||||
yield v
|
||||
return [v]
|
||||
|
||||
# build possible permutations
|
||||
for suite_defines_ in suite_defines:
|
||||
self.defines |= suite_defines_.keys()
|
||||
for defines_ in defines:
|
||||
self.defines |= defines_.keys()
|
||||
self.permutations.extend(dict(perm) for perm in it.product(*(
|
||||
[(k, v) for v in parse_define(vs)]
|
||||
for k, vs in sorted((suite_defines_ | defines_).items()))))
|
||||
self.permutations.append({
|
||||
k: parse_define(v)
|
||||
for k, v in (suite_defines_ | defines_).items()})
|
||||
|
||||
for k in config.keys():
|
||||
print('%swarning:%s in %s, found unused key %r' % (
|
||||
@ -316,34 +315,32 @@ def compile(test_paths, **args):
|
||||
# the test defines
|
||||
def write_case_functions(f, suite, case):
|
||||
# create case define functions
|
||||
if case.defines:
|
||||
# deduplicate defines by value to try to reduce the
|
||||
# number of functions we generate
|
||||
define_cbs = {}
|
||||
for i, defines in enumerate(case.permutations):
|
||||
for k, v in sorted(defines.items()):
|
||||
if v not in define_cbs:
|
||||
name = ('__test__%s__%s__%d'
|
||||
% (case.name, k, i))
|
||||
define_cbs[v] = name
|
||||
f.writeln('intmax_t %s('
|
||||
'__attribute__((unused)) '
|
||||
'void *data) {' % name)
|
||||
f.writeln(4*' '+'return %s;' % v)
|
||||
f.writeln('}')
|
||||
f.writeln()
|
||||
f.writeln('const test_define_t '
|
||||
'__test__%s__defines[]['
|
||||
'TEST_IMPLICIT_DEFINE_COUNT+%d] = {'
|
||||
% (case.name, len(suite.defines)))
|
||||
for defines in case.permutations:
|
||||
f.writeln(4*' '+'{')
|
||||
for k, v in sorted(defines.items()):
|
||||
f.writeln(8*' '+'[%-24s] = {%s, NULL},' % (
|
||||
k+'_i', define_cbs[v]))
|
||||
f.writeln(4*' '+'},')
|
||||
f.writeln('};')
|
||||
f.writeln()
|
||||
for i, permutation in enumerate(case.permutations):
|
||||
for k, vs in sorted(permutation.items()):
|
||||
f.writeln('intmax_t __test__%s__%s__%d('
|
||||
'__attribute__((unused)) void *data, '
|
||||
'size_t i) {'
|
||||
% (case.name, k, i))
|
||||
j = 0
|
||||
for v in vs:
|
||||
# generate range
|
||||
if isinstance(v, range):
|
||||
f.writeln(
|
||||
4*' '+'if (i < %d) '
|
||||
'return (i-%d)*%d + %d;'
|
||||
% (j+len(v), j, v.step, v.start))
|
||||
j += len(v)
|
||||
# translate index to define
|
||||
else:
|
||||
f.writeln(
|
||||
4*' '+'if (i == %d) '
|
||||
'return %s;'
|
||||
% (j, v))
|
||||
j += 1;
|
||||
|
||||
f.writeln(4*' '+'__builtin_unreachable();')
|
||||
f.writeln('}')
|
||||
f.writeln()
|
||||
|
||||
# create case filter function
|
||||
if suite.if_ is not None or case.if_ is not None:
|
||||
@ -397,11 +394,11 @@ def compile(test_paths, **args):
|
||||
if case.in_ is None:
|
||||
write_case_functions(f, suite, case)
|
||||
else:
|
||||
if case.defines:
|
||||
f.writeln('extern const test_define_t '
|
||||
'__test__%s__defines[]['
|
||||
'TEST_IMPLICIT_DEFINE_COUNT+%d];'
|
||||
% (case.name, len(suite.defines)))
|
||||
for i, permutation in enumerate(case.permutations):
|
||||
for k, vs in sorted(permutation.items()):
|
||||
f.writeln('extern intmax_t __test__%s__%s__%d('
|
||||
'void *data, size_t i);'
|
||||
% (case.name, k, i))
|
||||
if suite.if_ is not None or case.if_ is not None:
|
||||
f.writeln('extern bool __test__%s__filter('
|
||||
'void);'
|
||||
@ -429,13 +426,14 @@ def compile(test_paths, **args):
|
||||
if suite.defines:
|
||||
# create suite define names
|
||||
f.writeln(4*' '+'.define_names = (const char *const['
|
||||
'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % (
|
||||
len(suite.defines)))
|
||||
'TEST_IMPLICIT_DEFINE_COUNT+%d]){'
|
||||
% (len(suite.defines)))
|
||||
for k in sorted(suite.defines):
|
||||
f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k))
|
||||
f.writeln(4*' '+'},')
|
||||
f.writeln(4*' '+'.define_count = '
|
||||
'TEST_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines))
|
||||
'TEST_IMPLICIT_DEFINE_COUNT+%d,'
|
||||
% len(suite.defines))
|
||||
if suite.cases:
|
||||
f.writeln(4*' '+'.cases = (const struct test_case[]){')
|
||||
for case in suite.cases:
|
||||
@ -447,12 +445,26 @@ def compile(test_paths, **args):
|
||||
% (' | '.join(filter(None, [
|
||||
'TEST_REENTRANT' if case.reentrant else None]))
|
||||
or 0))
|
||||
f.writeln(12*' '+'.permutations = %d,'
|
||||
% len(case.permutations))
|
||||
if case.defines:
|
||||
f.writeln(12*' '+'.defines '
|
||||
'= (const test_define_t*)__test__%s__defines,'
|
||||
% (case.name))
|
||||
f.writeln(12*' '+'.defines = '
|
||||
'(const test_define_t*)(const test_define_t[]['
|
||||
'TEST_IMPLICIT_DEFINE_COUNT+%d]){'
|
||||
% (len(suite.defines)))
|
||||
for i, permutation in enumerate(case.permutations):
|
||||
f.writeln(16*' '+'{')
|
||||
for k, vs in sorted(permutation.items()):
|
||||
f.writeln(20*' '
|
||||
+'[%-24s] = {__test__%s__%s__%d, NULL, '
|
||||
'%d},'
|
||||
% (k+'_i', case.name, k, i,
|
||||
sum(len(v)
|
||||
if isinstance(v, range)
|
||||
else 1
|
||||
for v in vs)))
|
||||
f.writeln(16*' '+'},')
|
||||
f.writeln(12*' '+'},')
|
||||
f.writeln(12*' '+'.permutations = %d,'
|
||||
% len(case.permutations))
|
||||
if suite.if_ is not None or case.if_ is not None:
|
||||
f.writeln(12*' '+'.filter = __test__%s__filter,'
|
||||
% (case.name))
|
||||
|
||||
Reference in New Issue
Block a user