mirror/littlefs
3
0
Fork 0
mirror of https://github.com/littlefs-project/littlefs.git synced 2025-10-29 19:47:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reworked how multi-layered defines work in the test-runner

Browse Source 
In the test-runner, defines are parameterized constants (limited
to integers) that are generated from the test suite tomls resulting
in many permutations of each test.

In order to make this efficient, these defines are implemented as
multi-layered lookup tables, using per-layer/per-scope indirect
mappings. This lets the test-runner and test suites define their
own defines with compile-time indexes independently. It also makes
building of the lookup tables very efficient, since they can be
incrementally populated as we expand the test permutations.

The four current define layers and when we need to build them:

layer                           defines         predefine_map   define_map
user-provided overrides         per-run         per-run         per-suite
per-permutation defines         per-perm        per-case        per-perm
per-geometry defines            per-perm        compile-time    -
default defines                 compile-time    compile-time    -
This commit is contained in:
Christopher Haster 2022-04-24 23:34:28 -05:00
parent 64436933e2
commit 5812d2b5cf
5 changed files with 541 additions and 373 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -14,3 +14,4 @@ test.c
tests/*.toml.*
scripts/__pycache__
.gdb_history
runners/test_runner

5
Makefile
View File

@ -54,6 +54,9 @@ override CFLAGS += -I.
override CFLAGS += -std=c99 -Wall -pedantic
override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef
override TESTFLAGS_ += -b
# forward -j flag
override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS))
ifdef VERBOSE
override TESTFLAGS += -v
override CALLSFLAGS += -v
@ -88,8 +91,6 @@ endif
ifneq ($(OBJDUMP),objdump)
override STRUCTSFLAGS += --objdump-tool="$(OBJDUMP)"
endif
# forward -j flag
override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS))
# commands

391
runners/test_runner.c
View File

@ -6,106 +6,151 @@
#include <sys/types.h>
#include <errno.h>
// disk geometries
// test geometries
struct test_geometry {
const char *name;
const test_define_t *defines;
test_define_t defines[TEST_GEOMETRY_DEFINE_COUNT];
};
// Note this includes the default configuration for test pre-defines
#define TEST_GEOMETRY(name, read, prog, erase, count) \
{name, (const test_define_t[]){ \
/* READ_SIZE */ read, \
/* PROG_SIZE */ prog, \
/* BLOCK_SIZE */ erase, \
/* BLOCK_COUNT */ count, \
/* BLOCK_CYCLES */ -1, \
/* CACHE_SIZE */ (64 % (prog) == 0) ? 64 : (prog), \
/* LOOKAHEAD_SIZE */ 16, \
/* ERASE_VALUE */ 0xff, \
/* ERASE_CYCLES */ 0, \
/* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \
}}
const struct test_geometry test_geometries[] = {
// Made up geometry that works well for testing
TEST_GEOMETRY("test", 16, 16, 512, (1024*1024)/512),
// EEPROM/NVRAM
TEST_GEOMETRY("eeprom", 1, 1, 512, (1024*1024)/512),
// SD/eMMC
TEST_GEOMETRY("emmc", 512, 512, 512, (1024*1024)/512),
// NOR flash
TEST_GEOMETRY("nor", 1, 1, 4096, (1024*1024)/4096),
// NAND flash
TEST_GEOMETRY("nand", 4096, 4096, 32*1024, (1024*1024)/(32*1024)),
};
const size_t test_geometry_count = (
sizeof(test_geometries) / sizeof(test_geometries[0]));
const struct test_geometry test_geometries[TEST_GEOMETRY_COUNT]
= TEST_GEOMETRIES;
// test define lookup and management
const test_define_t *test_defines[3] = {NULL};
const uint8_t *test_define_maps[2] = {NULL};
#define TEST_DEFINE_LAYERS 4
const test_define_t *test_defines[TEST_DEFINE_LAYERS] = {
NULL,
NULL,
NULL,
(const test_define_t[TEST_DEFAULT_COUNT])TEST_DEFAULTS,
};
const uint8_t *test_predefine_maps[TEST_DEFINE_LAYERS] = {
NULL,
NULL,
(const uint8_t[TEST_PREDEFINE_COUNT])TEST_GEOMETRY_DEFINE_MAP,
(const uint8_t[TEST_PREDEFINE_COUNT])TEST_DEFAULT_MAP,
};
const uint8_t *test_define_maps[TEST_DEFINE_LAYERS] = {
NULL,
NULL,
NULL,
NULL,
};
uint8_t test_override_predefine_map[TEST_PREDEFINE_COUNT];
uint8_t test_override_define_map[256];
uint8_t test_case_predefine_map[TEST_PREDEFINE_COUNT];
const char *const *test_override_names;
size_t test_override_count;
const char *const test_predefine_names[TEST_PREDEFINE_COUNT]
= TEST_PREDEFINE_NAMES;
const char *const *test_define_names;
size_t test_define_count;
test_define_t test_predefine(size_t define) {
for (int i = 0; i < TEST_DEFINE_LAYERS; i++) {
if (test_defines[i]
&& test_predefine_maps[i]
&& test_predefine_maps[i][define] != 0xff) {
return test_defines[i][test_predefine_maps[i][define]];
}
}
fprintf(stderr, "error: undefined predefine %s\n",
test_predefine_names[define]);
assert(false);
exit(-1);
}
test_define_t test_define(size_t define) {
if (test_define_maps[0] && test_define_maps[0][define] != 0xff) {
return test_defines[0][test_define_maps[0][define]];
} else if (test_define_maps[1] && test_define_maps[1][define] != 0xff) {
return test_defines[1][test_define_maps[1][define]];
} else {
return test_defines[2][define];
for (int i = 0; i < TEST_DEFINE_LAYERS; i++) {
if (test_defines[i]
&& test_define_maps[i]
&& test_define_maps[i][define] != 0xff) {
return test_defines[i][test_define_maps[i][define]];
}
}
fprintf(stderr, "error: undefined define %s\n",
test_define_names[define]);
assert(false);
exit(-1);
}
static void test_define_geometry(const struct test_geometry *geometry) {
if (geometry) {
test_defines[2] = geometry->defines;
} else {
test_defines[2] = NULL;
}
}
static void test_define_case(const struct test_case *case_, size_t perm) {
if (case_ && case_->defines) {
test_defines[1] = case_->defines[perm];
test_define_maps[1] = case_->define_map;
} else {
test_defines[1] = NULL;
test_define_maps[1] = NULL;
}
test_defines[2] = geometry->defines;
}
static void test_define_overrides(
const struct test_suite *suite,
const char *const *override_names,
const test_define_t *override_defines,
size_t override_count) {
if (override_names && override_defines && override_count > 0) {
uint8_t *define_map = malloc(suite->define_count * sizeof(uint8_t));
memset(define_map, 0xff, suite->define_count * sizeof(bool));
test_defines[0] = override_defines;
test_override_names = override_names;
test_override_count = override_count;
// lookup each override in the suite defines, they may have a
// different index in each suite
for (size_t i = 0; i < override_count; i++) {
size_t j = 0;
for (; j < suite->define_count; j++) {
if (strcmp(override_names[i], suite->define_names[j]) == 0) {
break;
}
}
if (j < suite->define_count) {
define_map[j] = i;
// map any predefines
memset(test_override_predefine_map, 0xff, TEST_PREDEFINE_COUNT);
for (size_t i = 0; i < override_count; i++) {
for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) {
if (strcmp(override_names[i], test_predefine_names[j]) == 0) {
test_override_predefine_map[j] = i;
}
}
}
test_predefine_maps[0] = test_override_predefine_map;
}
test_defines[0] = override_defines;
test_define_maps[0] = define_map;
static void test_define_suite(const struct test_suite *suite) {
test_define_names = suite->define_names;
test_define_count = suite->define_count;
// map any defines
memset(test_override_define_map, 0xff, suite->define_count);
for (size_t i = 0; i < test_override_count; i++) {
for (size_t j = 0; j < suite->define_count; j++) {
if (strcmp(test_override_names[i], suite->define_names[j]) == 0) {
test_override_define_map[j] = i;
}
}
}
test_define_maps[0] = test_override_define_map;
}
static void test_define_case(
const struct test_suite *suite,
const struct test_case *case_) {
(void)suite;
// case_->define_map is already correct, but we need to do
// some fixup for the predefine map
test_define_maps[1] = case_->define_map;
memset(test_case_predefine_map, 0xff, TEST_PREDEFINE_COUNT);
for (size_t i = 0; i < test_define_count; i++) {
for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) {
if (strcmp(test_define_names[i], test_predefine_names[j]) == 0) {
test_case_predefine_map[j] = case_->define_map[i];
}
}
}
test_predefine_maps[1] = test_case_predefine_map;
}
static void test_define_perm(
const struct test_suite *suite,
const struct test_case *case_,
size_t perm) {
(void)suite;
if (case_->defines) {
test_defines[1] = case_->defines[perm];
} else {
test_defines[0] = NULL;
free((uint8_t *)test_define_maps[0]);
test_define_maps[0] = NULL;
test_defines[1] = NULL;
}
}
@ -120,11 +165,6 @@ static size_t test_skip = 0;
static size_t test_count = -1;
static size_t test_every = 1;
static const char **override_names = NULL;
static test_define_t *override_defines = NULL;
static size_t override_count = 0;
static size_t override_cap = 0;
static const char *test_persist = NULL;
FILE *test_trace = NULL;
@ -140,7 +180,7 @@ static bool test_case_skip(const struct test_case *case_) {
}
static bool test_perm_skip(size_t perm) {
size_t geom_perm = perm % test_geometry_count;
size_t geom_perm = perm % TEST_GEOMETRY_COUNT;
return (test_perm != (size_t)-1 && perm != test_perm)
|| (test_geometry && (strcmp(
test_geometries[geom_perm].name,
@ -153,7 +193,8 @@ static bool test_step_skip(size_t step) {
&& (step-test_skip) % test_every == 0);
}
static void test_case_sumpermutations(
static void test_case_permcount(
const struct test_suite *suite,
const struct test_case *case_,
size_t *perms,
size_t *filtered) {
@ -161,7 +202,7 @@ static void test_case_sumpermutations(
size_t filtered_ = 0;
for (size_t perm = 0;
perm < test_geometry_count
perm < TEST_GEOMETRY_COUNT
* case_->permutations;
perm++) {
if (test_perm_skip(perm)) {
@ -171,23 +212,18 @@ static void test_case_sumpermutations(
perms_ += 1;
// setup defines
size_t case_perm = perm / test_geometry_count;
size_t geom_perm = perm % test_geometry_count;
size_t case_perm = perm / TEST_GEOMETRY_COUNT;
size_t geom_perm = perm % TEST_GEOMETRY_COUNT;
test_define_perm(suite, case_, case_perm);
test_define_geometry(&test_geometries[geom_perm]);
test_define_case(case_, case_perm);
if (case_->filter) {
if (!case_->filter(case_perm)) {
test_define_geometry(NULL);
test_define_case(NULL, 0);
continue;
}
}
filtered_ += 1;
test_define_geometry(NULL);
test_define_case(NULL, 0);
}
*perms += perms_;
@ -208,21 +244,18 @@ static void summary(void) {
continue;
}
test_define_overrides(
test_suites[i],
override_names, override_defines, override_count);
test_define_suite(test_suites[i]);
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
if (test_case_skip(test_suites[i]->cases[j])) {
continue;
}
test_case_sumpermutations(test_suites[i]->cases[j],
test_define_case(test_suites[i], test_suites[i]->cases[j]);
test_case_permcount(test_suites[i], test_suites[i]->cases[j],
&perms, &filtered);
}
test_define_overrides(NULL, NULL, NULL, 0);
cases += test_suites[i]->case_count;
types |= test_suites[i]->types;
}
@ -243,16 +276,13 @@ static void summary(void) {
}
static void list_suites(void) {
printf("%-36s %-12s %7s %7s %11s\n",
"id", "suite", "types", "cases", "perms");
printf("%-36s %7s %7s %11s\n", "suite", "types", "cases", "perms");
for (size_t i = 0; i < test_suite_count; i++) {
if (test_suite_skip(test_suites[i])) {
continue;
}
test_define_overrides(
test_suites[i],
override_names, override_defines, override_count);
test_define_suite(test_suites[i]);
size_t perms = 0;
size_t filtered = 0;
@ -261,12 +291,11 @@ static void list_suites(void) {
continue;
}
test_case_sumpermutations(test_suites[i]->cases[j],
test_define_case(test_suites[i], test_suites[i]->cases[j]);
test_case_permcount(test_suites[i], test_suites[i]->cases[j],
&perms, &filtered);
}
test_define_overrides(NULL, NULL, NULL, 0);
char perm_buf[64];
sprintf(perm_buf, "%zu/%zu", filtered, perms);
char type_buf[64];
@ -274,9 +303,8 @@ static void list_suites(void) {
(test_suites[i]->types & TEST_NORMAL) ? "n" : "",
(test_suites[i]->types & TEST_REENTRANT) ? "r" : "",
(test_suites[i]->types & TEST_VALGRIND) ? "V" : "");
printf("%-36s %-12s %7s %7zu %11s\n",
printf("%-36s %7s %7zu %11s\n",
test_suites[i]->id,
test_suites[i]->name,
type_buf,
test_suites[i]->case_count,
perm_buf);
@ -284,25 +312,24 @@ static void list_suites(void) {
}
static void list_cases(void) {
printf("%-36s %-12s %-12s %7s %11s\n",
"id", "suite", "case", "types", "perms");
printf("%-36s %7s %11s\n", "case", "types", "perms");
for (size_t i = 0; i < test_suite_count; i++) {
if (test_suite_skip(test_suites[i])) {
continue;
}
test_define_overrides(
test_suites[i],
override_names, override_defines, override_count);
test_define_suite(test_suites[i]);
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
if (test_case_skip(test_suites[i]->cases[j])) {
continue;
}
test_define_case(test_suites[i], test_suites[i]->cases[j]);
size_t perms = 0;
size_t filtered = 0;
test_case_sumpermutations(test_suites[i]->cases[j],
test_case_permcount(test_suites[i], test_suites[i]->cases[j],
&perms, &filtered);
test_types_t types = test_suites[i]->cases[j]->types;
@ -313,20 +340,15 @@ static void list_cases(void) {
(types & TEST_NORMAL) ? "n" : "",
(types & TEST_REENTRANT) ? "r" : "",
(types & TEST_VALGRIND) ? "V" : "");
printf("%-36s %-12s %-12s %7s %11s\n",
printf("%-36s %7s %11s\n",
test_suites[i]->cases[j]->id,
test_suites[i]->name,
test_suites[i]->cases[j]->name,
type_buf,
perm_buf);
}
test_define_overrides(NULL, NULL, NULL, 0);
}
}
static void list_paths(void) {
printf("%-36s %-36s\n", "id", "path");
for (size_t i = 0; i < test_suite_count; i++) {
if (test_suite_skip(test_suites[i])) {
continue;
@ -345,23 +367,22 @@ static void list_paths(void) {
}
static void list_defines(void) {
printf("%-36s %s\n", "id", "defines");
for (size_t i = 0; i < test_suite_count; i++) {
if (test_suite_skip(test_suites[i])) {
continue;
}
test_define_overrides(
test_suites[i],
override_names, override_defines, override_count);
test_define_suite(test_suites[i]);
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
if (test_case_skip(test_suites[i]->cases[j])) {
continue;
}
test_define_case(test_suites[i], test_suites[i]->cases[j]);
for (size_t perm = 0;
perm < test_geometry_count
perm < TEST_GEOMETRY_COUNT
* test_suites[i]->cases[j]->permutations;
perm++) {
if (test_perm_skip(perm)) {
@ -369,39 +390,38 @@ static void list_defines(void) {
}
// setup defines
size_t case_perm = perm / test_geometry_count;
size_t geom_perm = perm % test_geometry_count;
size_t case_perm = perm / TEST_GEOMETRY_COUNT;
size_t geom_perm = perm % TEST_GEOMETRY_COUNT;
test_define_perm(test_suites[i],
test_suites[i]->cases[j], case_perm);
test_define_geometry(&test_geometries[geom_perm]);
test_define_case(test_suites[i]->cases[j], case_perm);
// print each define
// print the case
char id_buf[256];
sprintf(id_buf, "%s#%zu", test_suites[i]->cases[j]->id, perm);
printf("%-36s ", id_buf);
for (size_t k = 0; k < test_suites[i]->define_count; k++) {
if (k >= TEST_PREDEFINE_COUNT && (
!test_suites[i]->cases[j]->define_map
|| test_suites[i]->cases[j]->define_map[k]
== 0xff)) {
continue;
}
printf("%s=%jd ",
test_suites[i]->define_names[k],
test_define(k));
// special case for the current geometry
printf("GEOMETRY=%s ", test_geometries[geom_perm].name);
// print each define
for (size_t k = 0; k < test_suites[i]->define_count; k++) {
if (test_suites[i]->cases[j]->define_map
&& test_suites[i]->cases[j]->define_map[k]
!= 0xff) {
printf("%s=%jd ",
test_suites[i]->define_names[k],
test_define(k));
}
}
printf("\n");
}
}
test_define_overrides(NULL, NULL, NULL, 0);
}
}
static void list_geometries(void) {
printf("%-36s %7s %7s %7s %7s %7s\n",
"name", "read", "prog", "erase", "count", "size");
for (size_t i = 0; i < test_geometry_count; i++) {
for (size_t i = 0; i < TEST_GEOMETRY_COUNT; i++) {
if (test_geometry && strcmp(
test_geometries[i].name,
test_geometry) != 0) {
@ -410,16 +430,33 @@ static void list_geometries(void) {
test_define_geometry(&test_geometries[i]);
printf("%-36s %7ju %7ju %7ju %7ju %7ju\n",
test_geometries[i].name,
READ_SIZE,
PROG_SIZE,
BLOCK_SIZE,
BLOCK_COUNT,
BLOCK_SIZE*BLOCK_COUNT);
printf("%-36s ", test_geometries[i].name);
// print each define
for (size_t k = 0; k < TEST_PREDEFINE_COUNT; k++) {
if (test_predefine_maps[2][k] != 0xff) {
printf("%s=%jd ",
test_predefine_names[k],
test_predefine(k));
}
}
printf("\n");
}
}
static void list_defaults(void) {
printf("%-36s ", "defaults");
// print each define
for (size_t k = 0; k < TEST_PREDEFINE_COUNT; k++) {
if (test_predefine_maps[3][k] != 0xff) {
printf("%s=%jd ",
test_predefine_names[k],
test_predefine(k));
}
}
printf("\n");
}
static void run(void) {
size_t step = 0;
for (size_t i = 0; i < test_suite_count; i++) {
@ -427,23 +464,22 @@ static void run(void) {
continue;
}
test_define_overrides(
test_suites[i],
override_names, override_defines, override_count);
test_define_suite(test_suites[i]);
for (size_t j = 0; j < test_suites[i]->case_count; j++) {
if (test_case_skip(test_suites[i]->cases[j])) {
continue;
}
test_define_case(test_suites[i], test_suites[i]->cases[j]);
for (size_t perm = 0;
perm < test_geometry_count
perm < TEST_GEOMETRY_COUNT
* test_suites[i]->cases[j]->permutations;
perm++) {
if (test_perm_skip(perm)) {
continue;
}
if (test_step_skip(step)) {
step += 1;
continue;
@ -451,10 +487,11 @@ static void run(void) {
step += 1;
// setup defines
size_t case_perm = perm / test_geometry_count;
size_t geom_perm = perm % test_geometry_count;
size_t case_perm = perm / TEST_GEOMETRY_COUNT;
size_t geom_perm = perm % TEST_GEOMETRY_COUNT;
test_define_perm(test_suites[i],
test_suites[i]->cases[j], case_perm);
test_define_geometry(&test_geometries[geom_perm]);
test_define_case(test_suites[i]->cases[j], case_perm);
// filter?
if (test_suites[i]->cases[j]->filter) {
@ -462,8 +499,6 @@ static void run(void) {
printf("skipped %s#%zu\n",
test_suites[i]->cases[j]->id,
perm);
test_define_geometry(NULL);
test_define_case(NULL, 0);
continue;
}
}
@ -514,13 +549,8 @@ static void run(void) {
"could not destroy block device: %d\n", err);
exit(-1);
}
test_define_geometry(NULL);
test_define_case(NULL, 0);
}
}
test_define_overrides(NULL, NULL, NULL, 0);
}
}
@ -536,14 +566,15 @@ enum opt_flags {
OPT_LIST_PATHS = 1,
OPT_LIST_DEFINES = 2,
OPT_LIST_GEOMETRIES = 3,
OPT_LIST_DEFAULTS = 4,
OPT_DEFINE = 'D',
OPT_GEOMETRY = 'G',
OPT_NORMAL = 'n',
OPT_REENTRANT = 'r',
OPT_VALGRIND = 'V',
OPT_SKIP = 4,
OPT_COUNT = 5,
OPT_EVERY = 6,
OPT_SKIP = 5,
OPT_COUNT = 6,
OPT_EVERY = 7,
OPT_PERSIST = 'p',
OPT_TRACE = 't',
};
@ -558,6 +589,7 @@ const struct option long_opts[] = {
{"list-paths", no_argument, NULL, OPT_LIST_PATHS},
{"list-defines", no_argument, NULL, OPT_LIST_DEFINES},
{"list-geometries", no_argument, NULL, OPT_LIST_GEOMETRIES},
{"list-defaults", no_argument, NULL, OPT_LIST_DEFAULTS},
{"define", required_argument, NULL, OPT_DEFINE},
{"geometry", required_argument, NULL, OPT_GEOMETRY},
{"normal", no_argument, NULL, OPT_NORMAL},
@ -579,6 +611,7 @@ const char *const help_text[] = {
"List the path for each test case.",
"List the defines for each test permutation.",
"List the disk geometries used for testing.",
"List the default defines in this test-runner.",
"Override a test define.",
"Filter by geometry.",
"Filter for normal tests. Can be combined.",
@ -594,6 +627,11 @@ const char *const help_text[] = {
int main(int argc, char **argv) {
void (*op)(void) = run;
static const char **override_names = NULL;
static test_define_t *override_defines = NULL;
static size_t override_count = 0;
static size_t override_cap = 0;
// parse options
while (true) {
int c = getopt_long(argc, argv, short_opts, long_opts, NULL);
@ -673,8 +711,18 @@ int main(int argc, char **argv) {
case OPT_LIST_GEOMETRIES:
op = list_geometries;
break;
case OPT_LIST_DEFAULTS:
op = list_defaults;
break;
// configuration
case OPT_DEFINE: {
// special case for -DGEOMETRY=<name>, we treat this the same
// as --geometry=<name>
if (strncmp(optarg, "GEOMETRY=", strlen("GEOMETRY=")) == 0) {
test_geometry = &optarg[strlen("GEOMETRY=")];
break;
}
// realloc if necessary
override_count += 1;
if (override_count > override_cap) {
@ -816,6 +864,9 @@ getopt_done: ;
test_suite = suite;
}
// register overrides
test_define_overrides(override_names, override_defines, override_count);
// do the thing
op();

63
runners/test_runner.h
View File

@ -41,27 +41,68 @@ struct test_suite {
size_t case_count;
};
// TODO remove this indirection
extern const struct test_suite *test_suites[];
extern const size_t test_suite_count;
// access generated test defines
test_define_t test_predefine(size_t define);
test_define_t test_define(size_t define);
// a few preconfigured defines that control how tests run
#define READ_SIZE test_define(0)
#define PROG_SIZE test_define(1)
#define BLOCK_SIZE test_define(2)
#define BLOCK_COUNT test_define(3)
#define BLOCK_CYCLES test_define(4)
#define CACHE_SIZE test_define(5)
#define LOOKAHEAD_SIZE test_define(6)
#define ERASE_VALUE test_define(7)
#define ERASE_CYCLES test_define(8)
#define BADBLOCK_BEHAVIOR test_define(9)
#define READ_SIZE test_predefine(0)
#define PROG_SIZE test_predefine(1)
#define BLOCK_SIZE test_predefine(2)
#define BLOCK_COUNT test_predefine(3)
#define CACHE_SIZE test_predefine(4)
#define LOOKAHEAD_SIZE test_predefine(5)
#define BLOCK_CYCLES test_predefine(6)
#define ERASE_VALUE test_predefine(7)
#define ERASE_CYCLES test_predefine(8)
#define BADBLOCK_BEHAVIOR test_predefine(9)
#define TEST_PREDEFINE_NAMES { \
"READ_SIZE", \
"PROG_SIZE", \
"BLOCK_SIZE", \
"BLOCK_COUNT", \
"CACHE_SIZE", \
"LOOKAHEAD_SIZE", \
"BLOCK_CYCLES", \
"ERASE_VALUE", \
"ERASE_CYCLES", \
"BADBLOCK_BEHAVIOR", \
}
#define TEST_PREDEFINE_COUNT 10
// default predefines
#define TEST_DEFAULTS { \
/* LOOKAHEAD_SIZE */ 16, \
/* BLOCK_CYCLES */ -1, \
/* ERASE_VALUE */ 0xff, \
/* ERASE_CYCLES */ 0, \
/* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \
}
#define TEST_DEFAULT_COUNT 5
#define TEST_DEFAULT_MAP { \
0xff, 0xff, 0xff, 0xff, 0xff, 0, 1, 2, 3, 4 \
}
// test geometries
#define TEST_GEOMETRIES { \
/*geometry, read, write, erase, count, cache */ \
{"test", { 16, 16, 512, (1024*1024)/512, 64}}, \
{"eeprom", { 1, 1, 512, (1024*1024)/512, 64}}, \
{"emmc", { 512, 512, 512, (1024*1024)/512, 512}}, \
{"nor", { 1, 1, 4096, (1024*1024)/4096, 64}}, \
{"nand", {4096, 4096, 32*1024, (1024*1024)/(32*1024), 4096}}, \
}
#define TEST_GEOMETRY_COUNT 5
#define TEST_GEOMETRY_DEFINE_COUNT 5
#define TEST_GEOMETRY_DEFINE_MAP { \
0, 1, 2, 3, 4, 0xff, 0xff, 0xff, 0xff, 0xff \
}
#endif

454
scripts/test_.py
View File

@ -31,30 +31,21 @@ CASE_PROLOGUE = """
CASE_EPILOGUE = """
"""
TEST_PREDEFINES = [
'READ_SIZE',
'PROG_SIZE',
'BLOCK_SIZE',
'BLOCK_COUNT',
'BLOCK_CYCLES',
'CACHE_SIZE',
'LOOKAHEAD_SIZE',
'ERASE_VALUE',
'ERASE_CYCLES',
'BADBLOCK_BEHAVIOR',
]
# TODO
# def testpath(path):
# def testcase(path):
# def testperm(path):
def testpath(path):
path, *_ = path.split('#', 1)
return path
def testsuite(path):
name = os.path.basename(path)
if name.endswith('.toml'):
name = name[:-len('.toml')]
return name
suite = testpath(path)
suite = os.path.basename(suite)
if suite.endswith('.toml'):
suite = suite[:-len('.toml')]
return suite
def testcase(path):
_, case, *_ = path.split('#', 2)
return '%s#%s' % (testsuite(path), case)
# TODO move this out in other files
def openio(path, mode='r'):
@ -111,7 +102,7 @@ class TestSuite:
# create a TestSuite object from a toml file
def __init__(self, path, args={}):
self.name = testsuite(path)
self.path = path
self.path = testpath(path)
# load toml file and parse test cases
with open(self.path) as f:
@ -125,9 +116,9 @@ class TestSuite:
code_linenos = []
for i, line in enumerate(f):
match = re.match(
'(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])' +
'|(?P<if>if\s*=)'
'|(?P<code>code\s*=)',
'(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])'
'|' '(?P<if>if\s*=)'
'|' '(?P<code>code\s*=)',
line)
if match and match.group('case'):
case_linenos.append((i+1, match.group('name')))
@ -187,8 +178,8 @@ class TestSuite:
'suite_valgrind': valgrind,
**case}))
# combine pre-defines and per-case defines
self.defines = TEST_PREDEFINES + sorted(
# combine per-case defines
self.defines = sorted(
set.union(*(set(case.defines) for case in self.cases)))
# combine other per-case things
@ -225,216 +216,222 @@ def compile(**args):
% args['test_paths'])
sys.exit(-1)
# write out a test suite
# load our suite
suite = TestSuite(paths[0])
if 'output' in args:
with openio(args['output'], 'w') as f:
# redirect littlefs tracing
f.write('#define LFS_TRACE_(fmt, ...) do { \\\n')
f.write(8*' '+'extern FILE *test_trace; \\\n')
f.write(8*' '+'if (test_trace) { \\\n')
f.write(12*' '+'fprintf(test_trace, '
'"%s:%d:trace: " fmt "%s\\n", \\\n')
f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n')
f.write(8*' '+'} \\\n')
f.write(4*' '+'} while (0)\n')
f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n')
f.write('#define LFS_TESTBD_TRACE(...) '
'LFS_TRACE_(__VA_ARGS__, "")\n')
f.write('\n')
else:
# load all suites
suites = [TestSuite(path) for path in paths]
suites.sort(key=lambda s: s.name)
f.write('%s\n' % SUITE_PROLOGUE.strip())
f.write('\n')
# write generated test source
if 'output' in args:
with openio(args['output'], 'w') as f:
_write = f.write
def write(s):
f.lineno += s.count('\n')
_write(s)
def writeln(s=''):
f.lineno += s.count('\n') + 1
_write(s)
_write('\n')
f.lineno = 1
f.write = write
f.writeln = writeln
# redirect littlefs tracing
f.writeln('#define LFS_TRACE_(fmt, ...) do { \\')
f.writeln(8*' '+'extern FILE *test_trace; \\')
f.writeln(8*' '+'if (test_trace) { \\')
f.writeln(12*' '+'fprintf(test_trace, '
'"%s:%d:trace: " fmt "%s\\n", \\')
f.writeln(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\')
f.writeln(8*' '+'} \\')
f.writeln(4*' '+'} while (0)')
f.writeln('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")')
f.writeln('#define LFS_TESTBD_TRACE(...) '
'LFS_TRACE_(__VA_ARGS__, "")')
f.writeln()
if not args.get('source'):
# write test suite prologue
f.writeln('%s' % SUITE_PROLOGUE.strip())
f.writeln()
if suite.code is not None:
if suite.code_lineno is not None:
f.write('#line %d "%s"\n'
f.writeln('#line %d "%s"'
% (suite.code_lineno, suite.path))
f.write(suite.code)
f.write('\n')
if suite.code_lineno is not None:
f.writeln('#line %d "%s"'
% (f.lineno+1, args['output']))
f.writeln()
for i, define in it.islice(
enumerate(suite.defines),
len(TEST_PREDEFINES), None):
f.write('#define %-24s test_define(%d)\n' % (define, i))
f.write('\n')
for i, define in enumerate(suite.defines):
f.writeln('#ifndef %s' % define)
f.writeln('#define %-24s test_define(%d)' % (define, i))
f.writeln('#endif')
f.writeln()
for case in suite.cases:
# create case defines
if case.defines:
sorted_defines = sorted(case.defines.items())
for perm, defines in enumerate(
it.product(*(
[(k, v) for v in vs]
for k, vs in sorted_defines))):
f.write('const test_define_t '
'__test__%s__%s__%d__defines[] = {\n'
% (suite.name, case.name, perm))
for k, v in defines:
f.write(4*' '+'%s,\n' % v)
f.write('};\n')
f.write('\n')
f.write('const test_define_t *const '
'__test__%s__%s__defines[] = {\n'
f.writeln('const test_define_t *const '
'__test__%s__%s__defines[] = {'
% (suite.name, case.name))
for perm in range(case.permutations):
f.write(4*' '+'__test__%s__%s__%d__defines,\n'
% (suite.name, case.name, perm))
f.write('};\n')
f.write('\n')
for defines in it.product(*(
[(k, v) for v in vs]
for k, vs in sorted_defines)):
f.writeln(4*' '+'(const test_define_t[]){%s},'
% ', '.join('%s' % v for _, v in defines))
f.writeln('};')
f.writeln()
f.write('const uint8_t '
'__test__%s__%s__define_map[] = {\n'
f.writeln('const uint8_t '
'__test__%s__%s__define_map[] = {'
% (suite.name, case.name))
for k in suite.defines:
f.write(4*' '+'%s,\n'
% ([k for k, _ in sorted_defines].index(k)
if k in case.defines else '0xff'))
f.write('};\n')
f.write('\n')
f.writeln(4*' '+'%s,'
% ', '.join(
'%s' % [k for k, _ in sorted_defines].index(k)
if k in case.defines else '0xff'
for k in suite.defines))
f.writeln('};')
f.writeln()
# create case filter function
if suite.if_ is not None or case.if_ is not None:
f.write('bool __test__%s__%s__filter('
'__attribute__((unused)) uint32_t perm) {\n'
f.writeln('bool __test__%s__%s__filter('
'__attribute__((unused)) uint32_t perm) {'
% (suite.name, case.name))
if suite.if_ is not None:
f.write(4*' '+'#line %d "%s"\n'
% (suite.if_lineno, suite.path))
f.write(4*' '+'if (!(%s)) {\n' % suite.if_)
f.write(8*' '+'return false;\n')
f.write(4*' '+'}\n')
f.write('\n')
if suite.if_lineno is not None:
f.writeln(4*' '+'#line %d "%s"'
% (suite.if_lineno, suite.path))
f.writeln(4*' '+'if (!(%s)) {' % suite.if_)
if suite.if_lineno is not None:
f.writeln(4*' '+'#line %d "%s"'
% (f.lineno+1, args['output']))
f.writeln(8*' '+'return false;')
f.writeln(4*' '+'}')
f.writeln()
if case.if_ is not None:
f.write(4*' '+'#line %d "%s"\n'
% (case.if_lineno, suite.path))
f.write(4*' '+'if (!(%s)) {\n' % case.if_)
f.write(8*' '+'return false;\n')
f.write(4*' '+'}\n')
f.write('\n')
f.write(4*' '+'return true;\n')
f.write('}\n')
f.write('\n')
if case.if_lineno is not None:
f.writeln(4*' '+'#line %d "%s"'
% (case.if_lineno, suite.path))
f.writeln(4*' '+'if (!(%s)) {' % case.if_)
if case.if_lineno is not None:
f.writeln(4*' '+'#line %d "%s"'
% (f.lineno+1, args['output']))
f.writeln(8*' '+'return false;')
f.writeln(4*' '+'}')
f.writeln()
f.writeln(4*' '+'return true;')
f.writeln('}')
f.writeln()
# create case run function
f.write('void __test__%s__%s__run('
f.writeln('void __test__%s__%s__run('
'__attribute__((unused)) struct lfs_config *cfg, '
'__attribute__((unused)) uint32_t perm) {\n'
'__attribute__((unused)) uint32_t perm) {'
% (suite.name, case.name))
f.write(4*' '+'%s\n'
% CASE_PROLOGUE.strip().replace('\n', '\n'+4*' '))
f.write('\n')
f.write(4*' '+'// test case %s\n' % case.id())
if CASE_PROLOGUE.strip():
f.writeln(4*' '+'%s'
% CASE_PROLOGUE.strip().replace('\n', '\n'+4*' '))
f.writeln()
f.writeln(4*' '+'// test case %s' % case.id())
if case.code_lineno is not None:
f.write(4*' '+'#line %d "%s"\n'
f.writeln(4*' '+'#line %d "%s"'
% (case.code_lineno, suite.path))
f.write(case.code)
f.write('\n')
f.write(4*' '+'%s\n'
% CASE_EPILOGUE.strip().replace('\n', '\n'+4*' '))
f.write('}\n')
f.write('\n')
if case.code_lineno is not None:
f.writeln(4*' '+'#line %d "%s"'
% (f.lineno+1, args['output']))
if CASE_EPILOGUE.strip():
f.writeln()
f.writeln(4*' '+'%s'
% CASE_EPILOGUE.strip().replace('\n', '\n'+4*' '))
f.writeln('}')
f.writeln()
# create case struct
f.write('const struct test_case __test__%s__%s__case = {\n'
f.writeln('const struct test_case __test__%s__%s__case = {'
% (suite.name, case.name))
f.write(4*' '+'.id = "%s",\n' % case.id())
f.write(4*' '+'.name = "%s",\n' % case.name)
f.write(4*' '+'.path = "%s",\n' % case.path)
f.write(4*' '+'.types = %s,\n'
f.writeln(4*' '+'.id = "%s",' % case.id())
f.writeln(4*' '+'.name = "%s",' % case.name)
f.writeln(4*' '+'.path = "%s",' % case.path)
f.writeln(4*' '+'.types = %s,'
% ' | '.join(filter(None, [
'TEST_NORMAL' if case.normal else None,
'TEST_REENTRANT' if case.reentrant else None,
'TEST_VALGRIND' if case.valgrind else None])))
f.write(4*' '+'.permutations = %d,\n' % case.permutations)
f.writeln(4*' '+'.permutations = %d,' % case.permutations)
if case.defines:
f.write(4*' '+'.defines = __test__%s__%s__defines,\n'
f.writeln(4*' '+'.defines = __test__%s__%s__defines,'
% (suite.name, case.name))
f.write(4*' '+'.define_map = '
'__test__%s__%s__define_map,\n'
f.writeln(4*' '+'.define_map = '
'__test__%s__%s__define_map,'
% (suite.name, case.name))
if suite.if_ is not None or case.if_ is not None:
f.write(4*' '+'.filter = __test__%s__%s__filter,\n'
f.writeln(4*' '+'.filter = __test__%s__%s__filter,'
% (suite.name, case.name))
f.write(4*' '+'.run = __test__%s__%s__run,\n'
f.writeln(4*' '+'.run = __test__%s__%s__run,'
% (suite.name, case.name))
f.write('};\n')
f.write('\n')
f.writeln('};')
f.writeln()
# create suite define names
f.write('const char *const __test__%s__define_names[] = {\n'
f.writeln('const char *const __test__%s__define_names[] = {'
% suite.name)
for k in suite.defines:
f.write(4*' '+'"%s",\n' % k)
f.write('};\n')
f.write('\n')
f.writeln(4*' '+'"%s",' % k)
f.writeln('};')
f.writeln()
# create suite struct
f.write('const struct test_suite __test__%s__suite = {\n'
f.writeln('const struct test_suite __test__%s__suite = {'
% suite.name)
f.write(4*' '+'.id = "%s",\n' % suite.id())
f.write(4*' '+'.name = "%s",\n' % suite.name)
f.write(4*' '+'.path = "%s",\n' % suite.path)
f.write(4*' '+'.types = %s,\n'
f.writeln(4*' '+'.id = "%s",' % suite.id())
f.writeln(4*' '+'.name = "%s",' % suite.name)
f.writeln(4*' '+'.path = "%s",' % suite.path)
f.writeln(4*' '+'.types = %s,'
% ' | '.join(filter(None, [
'TEST_NORMAL' if suite.normal else None,
'TEST_REENTRANT' if suite.reentrant else None,
'TEST_VALGRIND' if suite.valgrind else None])))
f.write(4*' '+'.define_names = __test__%s__define_names,\n'
f.writeln(4*' '+'.define_names = __test__%s__define_names,'
% suite.name)
f.write(4*' '+'.define_count = %d,\n' % len(suite.defines))
f.write(4*' '+'.cases = (const struct test_case *const []){\n')
f.writeln(4*' '+'.define_count = %d,' % len(suite.defines))
f.writeln(4*' '+'.cases = (const struct test_case *const []){')
for case in suite.cases:
f.write(8*' '+'&__test__%s__%s__case,\n'
f.writeln(8*' '+'&__test__%s__%s__case,'
% (suite.name, case.name))
f.write(4*' '+'},\n')
f.write(4*' '+'.case_count = %d,\n' % len(suite.cases))
f.write('};\n')
f.write('\n')
else:
# load all suites
suites = [TestSuite(path) for path in paths]
suites.sort(key=lambda s: s.name)
# write out a test source
if 'output' in args:
with openio(args['output'], 'w') as f:
# redirect littlefs tracing
f.write('#define LFS_TRACE_(fmt, ...) do { \\\n')
f.write(8*' '+'extern FILE *test_trace; \\\n')
f.write(8*' '+'if (test_trace) { \\\n')
f.write(12*' '+'fprintf(test_trace, '
'"%s:%d:trace: " fmt "%s\\n", \\\n')
f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n')
f.write(8*' '+'} \\\n')
f.write(4*' '+'} while (0)\n')
f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n')
f.write('#define LFS_TESTBD_TRACE(...) '
'LFS_TRACE_(__VA_ARGS__, "")\n')
f.write('\n')
f.writeln(4*' '+'},')
f.writeln(4*' '+'.case_count = %d,' % len(suite.cases))
f.writeln('};')
f.writeln()
else:
# copy source
f.write('#line 1 "%s"\n' % args['source'])
f.writeln('#line 1 "%s"' % args['source'])
with open(args['source']) as sf:
shutil.copyfileobj(sf, f)
f.write('\n')
f.writeln()
f.write(SUITE_PROLOGUE)
f.write('\n')
f.writeln()
# add suite info to test_runner.c
if args['source'] == 'runners/test_runner.c':
f.write('\n')
f.writeln()
for suite in suites:
f.write('extern const struct test_suite '
'__test__%s__suite;\n' % suite.name)
f.write('const struct test_suite *test_suites[] = {\n')
f.writeln('extern const struct test_suite '
'__test__%s__suite;' % suite.name)
f.writeln('const struct test_suite *test_suites[] = {')
for suite in suites:
f.write(4*' '+'&__test__%s__suite,\n' % suite.name)
f.write('};\n')
f.write('const size_t test_suite_count = %d;\n'
f.writeln(4*' '+'&__test__%s__suite,' % suite.name)
f.writeln('};')
f.writeln('const size_t test_suite_count = %d;'
% len(suites))
def runner(**args):
@ -469,7 +466,7 @@ def list_(**args):
def find_cases(runner_, **args):
# first get suite/case/perm counts
# query from runner
cmd = runner_ + ['--list-cases']
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
@ -483,9 +480,8 @@ def find_cases(runner_, **args):
expected_perms = 0
total_perms = 0
pattern = re.compile(
'^(?P<id>(?P<suite>[^#]+)#[^ #]+) +'
'[^ ]+ +[^ ]+ +[^ ]+ +'
'(?P<filtered>[0-9]+)/(?P<perms>[0-9]+)$')
'^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
'[^\s]+\s+(?P<filtered>\d+)/(?P<perms>\d+)')
# skip the first line
next(proc.stdout)
for line in proc.stdout:
@ -509,11 +505,69 @@ def find_cases(runner_, **args):
expected_perms,
total_perms)
def find_paths(runner_, **args):
# query from runner
cmd = runner_ + ['--list-paths']
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd,
stdout=sp.PIPE,
stderr=sp.PIPE if not args.get('verbose') else None,
universal_newlines=True,
errors='replace')
paths = co.OrderedDict()
pattern = re.compile(
'^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
'(?P<path>[^:]+):(?P<lineno>\d+)')
# skip the first line
for line in proc.stdout:
m = pattern.match(line)
if m:
paths[m.group('id')] = (m.group('path'), int(m.group('lineno')))
proc.wait()
if proc.returncode != 0:
if not args.get('verbose'):
for line in proc.stderr:
sys.stdout.write(line)
sys.exit(-1)
return paths
def find_defines(runner_, **args):
# query from runner
cmd = runner_ + ['--list-defines']
if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd,
stdout=sp.PIPE,
stderr=sp.PIPE if not args.get('verbose') else None,
universal_newlines=True,
errors='replace')
defines = co.OrderedDict()
pattern = re.compile(
'^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
'(?P<defines>(?:\w+=\w+\s*)+)')
# skip the first line
for line in proc.stdout:
m = pattern.match(line)
if m:
defines[m.group('id')] = {k: v
for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))}
proc.wait()
if proc.returncode != 0:
if not args.get('verbose'):
for line in proc.stderr:
sys.stdout.write(line)
sys.exit(-1)
return defines
class TestFailure(Exception):
def __init__(self, id, returncode, stdout, assert_=None):
def __init__(self, id, returncode, output, assert_=None):
self.id = id
self.returncode = returncode
self.stdout = stdout
self.output = output
self.assert_ = assert_
def run_step(name, runner_, **args):
@ -531,7 +585,7 @@ def run_step(name, runner_, **args):
pattern = re.compile('^(?:'
'(?P<op>running|finished|skipped) '
'(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)'
'|' '(?P<path>[^:]+):(?P<lineno>[0-9]+):(?P<op_>assert):'
'|' '(?P<path>[^:]+):(?P<lineno>\d+):(?P<op_>assert):'
' *(?P<message>.*)' ')$')
locals = th.local()
# TODO use process group instead of this set?
@ -554,7 +608,7 @@ def run_step(name, runner_, **args):
children.add(proc)
last_id = None
last_stdout = []
last_output = []
last_assert = None
try:
while True:
@ -567,7 +621,7 @@ def run_step(name, runner_, **args):
raise
if not line:
break
last_stdout.append(line)
last_output.append(line)
if args.get('verbose'):
sys.stdout.write(line)
@ -577,7 +631,7 @@ def run_step(name, runner_, **args):
if op == 'running':
locals.seen_perms += 1
last_id = m.group('id')
last_stdout = []
last_output = []
last_assert = None
elif op == 'finished':
passed_suite_perms[m.group('suite')] += 1
@ -590,10 +644,11 @@ def run_step(name, runner_, **args):
m.group('path'),
int(m.group('lineno')),
m.group('message'))
# TODO why is kill _so_ much faster than terminate?
proc.kill()
# go ahead and kill the process, aborting takes a while
if args.get('keep_going'):
proc.kill()
except KeyboardInterrupt:
raise TestFailure(last_id, 1, last_stdout)
raise TestFailure(last_id, 1, last_output)
finally:
children.remove(proc)
mpty.close()
@ -603,7 +658,7 @@ def run_step(name, runner_, **args):
raise TestFailure(
last_id,
proc.returncode,
last_stdout,
last_output,
last_assert)
def run_job(runner, skip=None, every=None):
@ -636,7 +691,6 @@ def run_step(name, runner_, **args):
else:
# stop other tests
for child in children:
# TODO why is kill _so_ much faster than terminate?
child.kill()
break
@ -759,11 +813,28 @@ def run(**args):
print()
# print each failure
# TODO get line, defines, path
if failures:
# get some extra info from runner
runner_paths = find_paths(runner_, **args)
runner_defines = find_defines(runner_, **args)
for failure in failures:
# print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s failed'
# # TODO this should be the suite path and lineno
# % (failure.assert
# show summary of failure
path, lineno = runner_paths[testcase(failure.id)]
defines = runner_defines[failure.id]
print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed'
% (path, lineno, failure.id,
' (%s)' % ', '.join(
'%s=%s' % (k, v) for k, v in defines.items())
if defines else ''))
if failure.output:
output = failure.output
if failure.assert_ is not None:
output = output[:-1]
for line in output[-5:]:
sys.stdout.write(line)
if failure.assert_ is not None:
path, lineno, message = failure.assert_
@ -785,7 +856,8 @@ def main(**args):
or args.get('list_cases')
or args.get('list_paths')
or args.get('list_defines')
or args.get('list_geometries')):
or args.get('list_geometries')
or args.get('list_defaults')):
list_(**args)
else:
run(**args)
@ -816,6 +888,8 @@ if __name__ == "__main__":
help="List the defines for each test permutation.")
test_parser.add_argument('--list-geometries', action='store_true',
help="List the disk geometries used for testing.")
test_parser.add_argument('--list-defaults', action='store_true',
help="List the default defines in this test-runner.")
test_parser.add_argument('-D', '--define', action='append',
help="Override a test define.")
test_parser.add_argument('-G', '--geometry',