diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20f23911..697a9b6c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -414,9 +414,10 @@ jobs: valgrind --version - name: bench run: | - BENCHFLAGS="$BENCHFLAGS -o lfs.bench.csv" make bench + make bench # find bench results + make lfs.bench.csv ./scripts/summary.py lfs.bench.csv \ -bsuite \ -freaded=bench_readed \ diff --git a/Makefile b/Makefile index 70bc67f8..43858c65 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,7 @@ TEST_GCNO := $(TEST_TAC:%.t.a.c=%.t.a.gcno) TEST_GCDA := $(TEST_TAC:%.t.a.c=%.t.a.gcda) TEST_PERF := $(TEST_RUNNER:%=%.perf) TEST_TRACE := $(TEST_RUNNER:%=%.trace) +TEST_CSV := $(TEST_RUNNER:%=%.csv) BENCHES ?= $(wildcard benches/*.toml) BENCH_SRC ?= $(SRC) \ @@ -66,6 +67,7 @@ BENCH_GCNO := $(BENCH_BAC:%.b.a.c=%.b.a.gcno) BENCH_GCDA := $(BENCH_BAC:%.b.a.c=%.b.a.gcda) BENCH_PERF := $(BENCH_RUNNER:%=%.perf) BENCH_TRACE := $(BENCH_RUNNER:%=%.trace) +BENCH_CSV := $(BENCH_RUNNER:%=%.csv) CFLAGS += -fcallgraph-info=su CFLAGS += -g3 @@ -132,6 +134,12 @@ endif ifndef NO_PERFBD BENCHFLAGS += -t $(BENCH_TRACE) --trace-backtrace --trace-freq=100 endif +ifdef YES_TESTMARKS +TESTFLAGS += -o $(TEST_CSV) +endif +ifndef NO_BENCHMARKS +BENCHFLAGS += -o $(BENCH_CSV) +endif ifdef VERBOSE TESTFLAGS += -v TESTCFLAGS += -v @@ -376,6 +384,23 @@ test: test-runner test-list: test-runner ./scripts/test.py $(TEST_RUNNER) $(TESTFLAGS) -l +## Summarize the testmarks +.PHONY: testmarks +testmarks: SUMMARYFLAGS+=-spassed +testmarks: $(TEST_CSV) $(BUILDDIR)/lfs.test.csv + $(strip ./scripts/summary.py $(TEST_CSV) \ + -bsuite \ + -fpassed=test_passed \ + $(SUMMARYFLAGS)) + +## Compare testmarks against a previous run +.PHONY: testmarks-diff +testmarks-diff: $(TEST_CSV) + $(strip ./scripts/summary.py $^ \ + -bsuite \ + -fpassed=test_passed \ + $(SUMMARYFLAGS) -d $(BUILDDIR)/lfs.test.csv) + ## Build the bench-runner .PHONY: bench-runner build-bench ifdef YES_COV @@ -410,6 +435,28 @@ bench: bench-runner bench-list: bench-runner ./scripts/bench.py $(BENCH_RUNNER) $(BENCHFLAGS) -l +## Summarize the benchmarks +.PHONY: benchmarks +benchmarks: SUMMARYFLAGS+=-Serased -Sproged -Sreaded +benchmarks: $(BENCH_CSV) $(BUILDDIR)/lfs.bench.csv + $(strip ./scripts/summary.py $(BENCH_CSV) \ + -bsuite \ + -freaded=bench_readed \ + -fproged=bench_proged \ + -ferased=bench_erased \ + $(SUMMARYFLAGS)) + +## Compare benchmarks against a previous run +.PHONY: benchmarks-diff +benchmarks-diff: $(BENCH_CSV) + $(strip ./scripts/summary.py $^ \ + -bsuite \ + -freaded=bench_readed \ + -fproged=bench_proged \ + -ferased=bench_erased \ + $(SUMMARYFLAGS) -d $(BUILDDIR)/lfs.bench.csv) + + # rules -include $(DEP) @@ -450,6 +497,12 @@ $(BUILDDIR)/lfs.perfbd.csv: $(BENCH_TRACE) $(patsubst %,-F%,$(SRC)) \ -q $(PERFBDFLAGS) -o $@) +$(BUILDDIR)/lfs.test.csv: $(TEST_CSV) + cp $^ $@ + +$(BUILDDIR)/lfs.bench.csv: $(BENCH_CSV) + cp $^ $@ + $(BUILDDIR)/runners/test_runner: $(TEST_OBJ) $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ @@ -487,14 +540,15 @@ $(BUILDDIR)/%.b.c: %.c $(BENCHES) clean: rm -f $(BUILDDIR)/lfs rm -f $(BUILDDIR)/liblfs.a - $(strip rm -f \ - $(BUILDDIR)/lfs.code.csv \ - $(BUILDDIR)/lfs.data.csv \ - $(BUILDDIR)/lfs.stack.csv \ - $(BUILDDIR)/lfs.structs.csv \ - $(BUILDDIR)/lfs.cov.csv \ - $(BUILDDIR)/lfs.perf.csv \ - $(BUILDDIR)/lfs.perfbd.csv) + rm -f $(BUILDDIR)/lfs.code.csv + rm -f $(BUILDDIR)/lfs.data.csv + rm -f $(BUILDDIR)/lfs.stack.csv + rm -f $(BUILDDIR)/lfs.structs.csv + rm -f $(BUILDDIR)/lfs.cov.csv + rm -f $(BUILDDIR)/lfs.perf.csv + rm -f $(BUILDDIR)/lfs.perfbd.csv + rm -f $(BUILDDIR)/lfs.test.csv + rm -f $(BUILDDIR)/lfs.bench.csv rm -f $(OBJ) rm -f $(DEP) rm -f $(ASM) @@ -509,6 +563,7 @@ clean: rm -f $(TEST_GCDA) rm -f $(TEST_PERF) rm -f $(TEST_TRACE) + rm -f $(TEST_CSV) rm -f $(BENCH_RUNNER) rm -f $(BENCH_BC) rm -f $(BENCH_BAC) @@ -519,3 +574,4 @@ clean: rm -f $(BENCH_GCDA) rm -f $(BENCH_PERF) rm -f $(BENCH_TRACE) + rm -f $(BENCH_CSV)