mirror/littlefs
3
0
Fork 0
mirror of https://github.com/littlefs-project/littlefs.git synced 2025-10-29 19:47:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #497 from littlefs-project/crc-rework-2

Browse Source 
Forward-looking erase-state CRCs
This commit is contained in:
Christopher Haster 2023-04-26 01:15:59 -05:00 committed by GitHub
commit 6f074ebe31
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 2050 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
.github/workflows/test.yml vendored
View File

@ -473,6 +473,42 @@ jobs:
path: status
retention-days: 1
# run compatibility tests using the current master as the previous version
test-compat:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
if: ${{github.event_name == 'pull_request'}}
# checkout the current pr target into lfsp
- uses: actions/checkout@v2
if: ${{github.event_name == 'pull_request'}}
with:
ref: ${{github.event.pull_request.base.ref}}
path: lfsp
- name: install
if: ${{github.event_name == 'pull_request'}}
run: |
# need a few things
sudo apt-get update -qq
sudo apt-get install -qq gcc python3 python3-pip
pip3 install toml
gcc --version
python3 --version
# adjust prefix of lfsp
- name: changeprefix
if: ${{github.event_name == 'pull_request'}}
run: |
./scripts/changeprefix.py lfs lfsp lfsp/*.h lfsp/*.c
- name: test-compat
if: ${{github.event_name == 'pull_request'}}
run: |
TESTS=tests/test_compat.toml \
SRC="$(find . lfsp -name '*.c' -maxdepth 1 \
-and -not -name '*.t.*' \
-and -not -name '*.b.*')" \
CFLAGS="-DLFSP=lfsp/lfsp.h" \
make test
# self-host with littlefs-fuse for a fuzz-like test
fuse:
runs-on: ubuntu-22.04

27
Makefile
View File

@ -1,15 +1,5 @@
ifdef BUILDDIR
# bit of a hack, but we want to make sure BUILDDIR directory structure
# is correct before any commands
$(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \
$(BUILDDIR)/ \
$(BUILDDIR)/bd \
$(BUILDDIR)/runners \
$(BUILDDIR)/tests \
$(BUILDDIR)/benches))
endif
# overrideable build dir, default is in-place
BUILDDIR ?= .
# overridable target/src/tools/flags/etc
ifneq ($(wildcard test.c main.c),)
TARGET ?= $(BUILDDIR)/lfs
@ -163,6 +153,18 @@ TESTFLAGS += --perf-path="$(PERF)"
BENCHFLAGS += --perf-path="$(PERF)"
endif
# this is a bit of a hack, but we want to make sure the BUILDDIR
# directory structure is correct before we run any commands
ifneq ($(BUILDDIR),.)
$(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \
$(addprefix $(BUILDDIR)/,$(dir \
$(SRC) \
$(TESTS) \
$(TEST_SRC) \
$(BENCHES) \
$(BENCH_SRC)))))
endif
# commands
@ -514,6 +516,9 @@ $(BUILDDIR)/runners/bench_runner: $(BENCH_OBJ)
$(BUILDDIR)/%.o $(BUILDDIR)/%.ci: %.c
$(CC) -c -MMD $(CFLAGS) $< -o $(BUILDDIR)/$*.o
$(BUILDDIR)/%.o $(BUILDDIR)/%.ci: $(BUILDDIR)/%.c
$(CC) -c -MMD $(CFLAGS) $< -o $(BUILDDIR)/$*.o
$(BUILDDIR)/%.s: %.c
$(CC) -S $(CFLAGS) $< -o $@

101
SPEC.md
View File

@ -1,10 +1,10 @@
## littlefs technical specification
This is the technical specification of the little filesystem. This document
covers the technical details of how the littlefs is stored on disk for
introspection and tooling. This document assumes you are familiar with the
design of the littlefs, for more info on how littlefs works check
out [DESIGN.md](DESIGN.md).
This is the technical specification of the little filesystem with on-disk
version lfs2.1. This document covers the technical details of how the littlefs
is stored on disk for introspection and tooling. This document assumes you are
familiar with the design of the littlefs, for more info on how littlefs works
check out [DESIGN.md](DESIGN.md).
```
| | | .---._____
@ -133,12 +133,6 @@ tags XORed together, starting with `0xffffffff`.
'-------------------' '-------------------'
```
One last thing to note before we get into the details around tag encoding. Each
tag contains a valid bit used to indicate if the tag and containing commit is
valid. This valid bit is the first bit found in the tag and the commit and can
be used to tell if we've attempted to write to the remaining space in the
block.
Here's a more complete example of metadata block containing 4 entries:
```
@ -191,6 +185,53 @@ Here's a more complete example of metadata block containing 4 entries:
'---- most recent D
```
Two things to note before we get into the details around tag encoding:
1. Each tag contains a valid bit used to indicate if the tag and containing
commit is valid. After XORing, this bit should always be zero.
At the end of each commit, the valid bit of the previous tag is XORed
with the lowest bit in the type field of the CRC tag. This allows
the CRC tag to force the next commit to fail the valid bit test if it
has not yet been written to.
2. The valid bit alone is not enough info to know if the next commit has been
erased. We don't know the order bits will be programmed in a program block,
so it's possible that the next commit had an attempted program that left the
valid bit unchanged.
To ensure we only ever program erased bytes, each commit can contain an
optional forward-CRC (FCRC). An FCRC contains a checksum of some amount of
bytes in the next commit at the time it was erased.
```
.-------------------. \ \
| revision count | | |
|-------------------| | |
| metadata | | |
| | +---. +-- current commit
| | | | |
|-------------------| | | |
| FCRC ---|-. | |
|-------------------| / | | |
| CRC -----|-' /
|-------------------| |
| padding | | padding (does't need CRC)
| | |
|-------------------| \ | \
| erased? | +-' |
| | | | +-- next commit
| v | / |
| | /
| |
'-------------------'
```
If the FCRC is missing or the checksum does not match, we must assume a
commit was attempted but failed due to power-loss.
Note that end-of-block commits do not need an FCRC.
## Metadata tags
So in littlefs, 32-bit tags describe every type of metadata. And this means
@ -785,3 +826,41 @@ CRC fields:
are made about the contents.
---
#### `0x5ff` LFS_TYPE_FCRC
Added in lfs2.1, the optional FCRC tag contains a checksum of some amount of
bytes in the next commit at the time it was erased. This allows us to ensure
that we only ever program erased bytes, even if a previous commit failed due
to power-loss.
When programming a commit, the FCRC size must be at least as large as the
program block size. However, the program block is not saved on disk, and can
change between mounts, so the FCRC size on disk may be different than the
current program block size.
If the FCRC is missing or the checksum does not match, we must assume a
commit was attempted but failed due to power-loss.
Layout of the FCRC tag:
```
tag data
[-- 32 --][-- 32 --|-- 32 --]
[1|- 11 -| 10 | 10 ][-- 32 --|-- 32 --]
^ ^ ^ ^ ^- fcrc size ^- fcrc
| | | '- size (8)
| | '------ id (0x3ff)
| '------------ type (0x5ff)
'----------------- valid bit
```
FCRC fields:
1. **FCRC size (32-bits)** - Number of bytes after this commit's CRC tag's
padding to include in the FCRC.
2. **FCRC (32-bits)** - CRC of the bytes after this commit's CRC tag's padding
when erased. Like the CRC tag, this uses a CRC-32 with a polynomial of
`0x04c11db7` initialized with `0xffffffff`.
---

9
bd/lfs_emubd.c
View File

@ -584,13 +584,14 @@ lfs_emubd_swear_t lfs_emubd_wear(const struct lfs_config *cfg,
wear = 0;
}
LFS_EMUBD_TRACE("lfs_emubd_wear -> %"PRIu32, wear);
LFS_EMUBD_TRACE("lfs_emubd_wear -> %"PRIi32, wear);
return wear;
}
int lfs_emubd_setwear(const struct lfs_config *cfg,
lfs_block_t block, lfs_emubd_wear_t wear) {
LFS_EMUBD_TRACE("lfs_emubd_setwear(%p, %"PRIu32")", (void*)cfg, block);
LFS_EMUBD_TRACE("lfs_emubd_setwear(%p, %"PRIu32", %"PRIi32")",
(void*)cfg, block, wear);
lfs_emubd_t *bd = cfg->context;
// check if block is valid
@ -599,12 +600,12 @@ int lfs_emubd_setwear(const struct lfs_config *cfg,
// set the wear
lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]);
if (!b) {
LFS_EMUBD_TRACE("lfs_emubd_setwear -> %"PRIu32, LFS_ERR_NOMEM);
LFS_EMUBD_TRACE("lfs_emubd_setwear -> %d", LFS_ERR_NOMEM);
return LFS_ERR_NOMEM;
}
b->wear = wear;
LFS_EMUBD_TRACE("lfs_emubd_setwear -> %"PRIu32, 0);
LFS_EMUBD_TRACE("lfs_emubd_setwear -> %d", 0);
return 0;
}

426
lfs.c
View File

@ -135,14 +135,14 @@ static int lfs_bd_cmp(lfs_t *lfs,
uint8_t dat[8];
diff = lfs_min(size-i, sizeof(dat));
int res = lfs_bd_read(lfs,
int err = lfs_bd_read(lfs,
pcache, rcache, hint-i,
block, off+i, &dat, diff);
if (res) {
return res;
if (err) {
return err;
}
res = memcmp(dat, data + i, diff);
int res = memcmp(dat, data + i, diff);
if (res) {
return res < 0 ? LFS_CMP_LT : LFS_CMP_GT;
}
@ -151,6 +151,27 @@ static int lfs_bd_cmp(lfs_t *lfs,
return LFS_CMP_EQ;
}
static int lfs_bd_crc(lfs_t *lfs,
const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint,
lfs_block_t block, lfs_off_t off, lfs_size_t size, uint32_t *crc) {
lfs_size_t diff = 0;
for (lfs_off_t i = 0; i < size; i += diff) {
uint8_t dat[8];
diff = lfs_min(size-i, sizeof(dat));
int err = lfs_bd_read(lfs,
pcache, rcache, hint-i,
block, off+i, &dat, diff);
if (err) {
return err;
}
*crc = lfs_crc(*crc, &dat, diff);
}
return 0;
}
#ifndef LFS_READONLY
static int lfs_bd_flush(lfs_t *lfs,
lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) {
@ -325,6 +346,10 @@ static inline uint16_t lfs_tag_type1(lfs_tag_t tag) {
return (tag & 0x70000000) >> 20;
}
static inline uint16_t lfs_tag_type2(lfs_tag_t tag) {
return (tag & 0x78000000) >> 20;
}
static inline uint16_t lfs_tag_type3(lfs_tag_t tag) {
return (tag & 0x7ff00000) >> 20;
}
@ -386,12 +411,16 @@ static inline bool lfs_gstate_hasorphans(const lfs_gstate_t *a) {
}
static inline uint8_t lfs_gstate_getorphans(const lfs_gstate_t *a) {
return lfs_tag_size(a->tag);
return lfs_tag_size(a->tag) & 0x1ff;
}
static inline bool lfs_gstate_hasmove(const lfs_gstate_t *a) {
return lfs_tag_type1(a->tag);
}
static inline bool lfs_gstate_needssuperblock(const lfs_gstate_t *a) {
return lfs_tag_size(a->tag) >> 9;
}
#endif
static inline bool lfs_gstate_hasmovehere(const lfs_gstate_t *a,
@ -413,6 +442,24 @@ static inline void lfs_gstate_tole32(lfs_gstate_t *a) {
}
#endif
// operations on forward-CRCs used to track erased state
struct lfs_fcrc {
lfs_size_t size;
uint32_t crc;
};
static void lfs_fcrc_fromle32(struct lfs_fcrc *fcrc) {
fcrc->size = lfs_fromle32(fcrc->size);
fcrc->crc = lfs_fromle32(fcrc->crc);
}
#ifndef LFS_READONLY
static void lfs_fcrc_tole32(struct lfs_fcrc *fcrc) {
fcrc->size = lfs_tole32(fcrc->size);
fcrc->crc = lfs_tole32(fcrc->crc);
}
#endif
// other endianness operations
static void lfs_ctz_fromle32(struct lfs_ctz *ctz) {
ctz->head = lfs_fromle32(ctz->head);
@ -490,6 +537,7 @@ static int lfs_file_outline(lfs_t *lfs, lfs_file_t *file);
static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file);
static int lfs_fs_deorphan(lfs_t *lfs, bool powerloss);
static void lfs_fs_prepsuperblock(lfs_t *lfs, bool needssuperblock);
static int lfs_fs_preporphans(lfs_t *lfs, int8_t orphans);
static void lfs_fs_prepmove(lfs_t *lfs,
uint16_t id, const lfs_block_t pair[2]);
@ -1035,6 +1083,11 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
bool tempsplit = false;
lfs_stag_t tempbesttag = besttag;
// assume not erased until proven otherwise
bool maybeerased = false;
bool hasfcrc = false;
struct lfs_fcrc fcrc;
dir->rev = lfs_tole32(dir->rev);
uint32_t crc = lfs_crc(0xffffffff, &dir->rev, sizeof(dir->rev));
dir->rev = lfs_fromle32(dir->rev);
@ -1049,7 +1102,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
if (err) {
if (err == LFS_ERR_CORRUPT) {
// can't continue?
dir->erased = false;
break;
}
return err;
@ -1058,19 +1110,18 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
crc = lfs_crc(crc, &tag, sizeof(tag));
tag = lfs_frombe32(tag) ^ ptag;
// next commit not yet programmed or we're not in valid range
// next commit not yet programmed?
if (!lfs_tag_isvalid(tag)) {
dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC &&
dir->off % lfs->cfg->prog_size == 0);
maybeerased = true;
break;
// out of range?
} else if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) {
dir->erased = false;
break;
}
ptag = tag;
if (lfs_tag_type1(tag) == LFS_TYPE_CRC) {
if (lfs_tag_type2(tag) == LFS_TYPE_CCRC) {
// check the crc attr
uint32_t dcrc;
err = lfs_bd_read(lfs,
@ -1078,7 +1129,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
dir->pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc));
if (err) {
if (err == LFS_ERR_CORRUPT) {
dir->erased = false;
break;
}
return err;
@ -1086,7 +1136,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
dcrc = lfs_fromle32(dcrc);
if (crc != dcrc) {
dir->erased = false;
break;
}
@ -1113,21 +1162,19 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
continue;
}
// crc the entry first, hopefully leaving it in the cache
for (lfs_off_t j = sizeof(tag); j < lfs_tag_dsize(tag); j++) {
uint8_t dat;
err = lfs_bd_read(lfs,
NULL, &lfs->rcache, lfs->cfg->block_size,
dir->pair[0], off+j, &dat, 1);
if (err) {
if (err == LFS_ERR_CORRUPT) {
dir->erased = false;
break;
}
return err;
}
// fcrc is only valid when last tag was a crc
hasfcrc = false;
crc = lfs_crc(crc, &dat, 1);
// crc the entry first, hopefully leaving it in the cache
err = lfs_bd_crc(lfs,
NULL, &lfs->rcache, lfs->cfg->block_size,
dir->pair[0], off+sizeof(tag),
lfs_tag_dsize(tag)-sizeof(tag), &crc);
if (err) {
if (err == LFS_ERR_CORRUPT) {
break;
}
return err;
}
// directory modification tags?
@ -1154,12 +1201,24 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
dir->pair[0], off+sizeof(tag), &temptail, 8);
if (err) {
if (err == LFS_ERR_CORRUPT) {
dir->erased = false;
break;
}
return err;
}
lfs_pair_fromle32(temptail);
} else if (lfs_tag_type3(tag) == LFS_TYPE_FCRC) {
err = lfs_bd_read(lfs,
NULL, &lfs->rcache, lfs->cfg->block_size,
dir->pair[0], off+sizeof(tag),
&fcrc, sizeof(fcrc));
if (err) {
if (err == LFS_ERR_CORRUPT) {
break;
}
}
lfs_fcrc_fromle32(&fcrc);
hasfcrc = true;
}
// found a match for our fetcher?
@ -1168,7 +1227,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
dir->pair[0], off+sizeof(tag)});
if (res < 0) {
if (res == LFS_ERR_CORRUPT) {
dir->erased = false;
break;
}
return res;
@ -1190,35 +1248,54 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
}
}
// consider what we have good enough
if (dir->off > 0) {
// synthetic move
if (lfs_gstate_hasmovehere(&lfs->gdisk, dir->pair)) {
if (lfs_tag_id(lfs->gdisk.tag) == lfs_tag_id(besttag)) {
besttag |= 0x80000000;
} else if (besttag != -1 &&
lfs_tag_id(lfs->gdisk.tag) < lfs_tag_id(besttag)) {
besttag -= LFS_MKTAG(0, 1, 0);
}
// found no valid commits?
if (dir->off == 0) {
// try the other block?
lfs_pair_swap(dir->pair);
dir->rev = revs[(r+1)%2];
continue;
}
// did we end on a valid commit? we may have an erased block
dir->erased = false;
if (maybeerased && hasfcrc && dir->off % lfs->cfg->prog_size == 0) {
// check for an fcrc matching the next prog's erased state, if
// this failed most likely a previous prog was interrupted, we
// need a new erase
uint32_t fcrc_ = 0xffffffff;
int err = lfs_bd_crc(lfs,
NULL, &lfs->rcache, lfs->cfg->block_size,
dir->pair[0], dir->off, fcrc.size, &fcrc_);
if (err && err != LFS_ERR_CORRUPT) {
return err;
}
// found tag? or found best id?
if (id) {
*id = lfs_min(lfs_tag_id(besttag), dir->count);
}
// found beginning of erased part?
dir->erased = (fcrc_ == fcrc.crc);
}
if (lfs_tag_isvalid(besttag)) {
return besttag;
} else if (lfs_tag_id(besttag) < dir->count) {
return LFS_ERR_NOENT;
} else {
return 0;
// synthetic move
if (lfs_gstate_hasmovehere(&lfs->gdisk, dir->pair)) {
if (lfs_tag_id(lfs->gdisk.tag) == lfs_tag_id(besttag)) {
besttag |= 0x80000000;
} else if (besttag != -1 &&
lfs_tag_id(lfs->gdisk.tag) < lfs_tag_id(besttag)) {
besttag -= LFS_MKTAG(0, 1, 0);
}
}
// failed, try the other block?
lfs_pair_swap(dir->pair);
dir->rev = revs[(r+1)%2];
// found tag? or found best id?
if (id) {
*id = lfs_min(lfs_tag_id(besttag), dir->count);
}
if (lfs_tag_isvalid(besttag)) {
return besttag;
} else if (lfs_tag_id(besttag) < dir->count) {
return LFS_ERR_NOENT;
} else {
return 0;
}
}
LFS_ERROR("Corrupted dir pair at {0x%"PRIx32", 0x%"PRIx32"}",
@ -1492,9 +1569,15 @@ static int lfs_dir_commitattr(lfs_t *lfs, struct lfs_commit *commit,
#endif
#ifndef LFS_READONLY
static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
// align to program units
const lfs_off_t end = lfs_alignup(commit->off + 2*sizeof(uint32_t),
//
// this gets a bit complex as we have two types of crcs:
// - 5-word crc with fcrc to check following prog (middle of block)
// - 2-word crc with no following prog (end of block)
const lfs_off_t end = lfs_alignup(
lfs_min(commit->off + 5*sizeof(uint32_t), lfs->cfg->block_size),
lfs->cfg->prog_size);
lfs_off_t off1 = 0;
@ -1504,89 +1587,116 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
// padding is not crced, which lets fetches skip padding but
// makes committing a bit more complicated
while (commit->off < end) {
lfs_off_t off = commit->off + sizeof(lfs_tag_t);
lfs_off_t noff = lfs_min(end - off, 0x3fe) + off;
lfs_off_t noff = (
lfs_min(end - (commit->off+sizeof(lfs_tag_t)), 0x3fe)
+ (commit->off+sizeof(lfs_tag_t)));
// too large for crc tag? need padding commits
if (noff < end) {
noff = lfs_min(noff, end - 2*sizeof(uint32_t));
noff = lfs_min(noff, end - 5*sizeof(uint32_t));
}
// read erased state from next program unit
lfs_tag_t tag = 0xffffffff;
int err = lfs_bd_read(lfs,
NULL, &lfs->rcache, sizeof(tag),
commit->block, noff, &tag, sizeof(tag));
if (err && err != LFS_ERR_CORRUPT) {
return err;
// space for fcrc?
uint8_t eperturb = -1;
if (noff >= end && noff <= lfs->cfg->block_size - lfs->cfg->prog_size) {
// first read the leading byte, this always contains a bit
// we can perturb to avoid writes that don't change the fcrc
int err = lfs_bd_read(lfs,
NULL, &lfs->rcache, lfs->cfg->prog_size,
commit->block, noff, &eperturb, 1);
if (err && err != LFS_ERR_CORRUPT) {
return err;
}
// find the expected fcrc, don't bother avoiding a reread
// of the eperturb, it should still be in our cache
struct lfs_fcrc fcrc = {.size=lfs->cfg->prog_size, .crc=0xffffffff};
err = lfs_bd_crc(lfs,
NULL, &lfs->rcache, lfs->cfg->prog_size,
commit->block, noff, fcrc.size, &fcrc.crc);
if (err && err != LFS_ERR_CORRUPT) {
return err;
}
lfs_fcrc_tole32(&fcrc);
err = lfs_dir_commitattr(lfs, commit,
LFS_MKTAG(LFS_TYPE_FCRC, 0x3ff, sizeof(struct lfs_fcrc)),
&fcrc);
if (err) {
return err;
}
}
// build crc tag
bool reset = ~lfs_frombe32(tag) >> 31;
tag = LFS_MKTAG(LFS_TYPE_CRC + reset, 0x3ff, noff - off);
// build commit crc
struct {
lfs_tag_t tag;
uint32_t crc;
} ccrc;
lfs_tag_t ntag = LFS_MKTAG(
LFS_TYPE_CCRC + (((uint8_t)~eperturb) >> 7), 0x3ff,
noff - (commit->off+sizeof(lfs_tag_t)));
ccrc.tag = lfs_tobe32(ntag ^ commit->ptag);
commit->crc = lfs_crc(commit->crc, &ccrc.tag, sizeof(lfs_tag_t));
ccrc.crc = lfs_tole32(commit->crc);
// write out crc
uint32_t footer[2];
footer[0] = lfs_tobe32(tag ^ commit->ptag);
commit->crc = lfs_crc(commit->crc, &footer[0], sizeof(footer[0]));
footer[1] = lfs_tole32(commit->crc);
err = lfs_bd_prog(lfs,
int err = lfs_bd_prog(lfs,
&lfs->pcache, &lfs->rcache, false,
commit->block, commit->off, &footer, sizeof(footer));
commit->block, commit->off, &ccrc, sizeof(ccrc));
if (err) {
return err;
}
// keep track of non-padding checksum to verify
if (off1 == 0) {
off1 = commit->off + sizeof(uint32_t);
off1 = commit->off + sizeof(lfs_tag_t);
crc1 = commit->crc;
}
commit->off += sizeof(tag)+lfs_tag_size(tag);
commit->ptag = tag ^ ((lfs_tag_t)reset << 31);
commit->crc = 0xffffffff; // reset crc for next "commit"
commit->off = noff;
// perturb valid bit?
commit->ptag = ntag ^ ((0x80 & ~eperturb) << 24);
// reset crc for next commit
commit->crc = 0xffffffff;
// manually flush here since we don't prog the padding, this confuses
// the caching layer
if (noff >= end || noff >= lfs->pcache.off + lfs->cfg->cache_size) {
// flush buffers
int err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false);
if (err) {
return err;
}
}
}
// flush buffers
int err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false);
// successful commit, check checksums to make sure
//
// note that we don't need to check padding commits, worst
// case if they are corrupted we would have had to compact anyways
lfs_off_t off = commit->begin;
uint32_t crc = 0xffffffff;
int err = lfs_bd_crc(lfs,
NULL, &lfs->rcache, off1+sizeof(uint32_t),
commit->block, off, off1-off, &crc);
if (err) {
return err;
}
// successful commit, check checksums to make sure
lfs_off_t off = commit->begin;
lfs_off_t noff = off1;
while (off < end) {
uint32_t crc = 0xffffffff;
for (lfs_off_t i = off; i < noff+sizeof(uint32_t); i++) {
// check against written crc, may catch blocks that
// become readonly and match our commit size exactly
if (i == off1 && crc != crc1) {
return LFS_ERR_CORRUPT;
}
// check non-padding commits against known crc
if (crc != crc1) {
return LFS_ERR_CORRUPT;
}
// leave it up to caching to make this efficient
uint8_t dat;
err = lfs_bd_read(lfs,
NULL, &lfs->rcache, noff+sizeof(uint32_t)-i,
commit->block, i, &dat, 1);
if (err) {
return err;
}
// make sure to check crc in case we happen to pick
// up an unrelated crc (frozen block?)
err = lfs_bd_crc(lfs,
NULL, &lfs->rcache, sizeof(uint32_t),
commit->block, off1, sizeof(uint32_t), &crc);
if (err) {
return err;
}
crc = lfs_crc(crc, &dat, 1);
}
// detected write error?
if (crc != 0) {
return LFS_ERR_CORRUPT;
}
// skip padding
off = lfs_min(end - noff, 0x3fe) + noff;
if (off < end) {
off = lfs_min(off, end - 2*sizeof(uint32_t));
}
noff = off + sizeof(uint32_t);
if (crc != 0) {
return LFS_ERR_CORRUPT;
}
return 0;
@ -4153,12 +4263,29 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) {
uint16_t minor_version = (0xffff & (superblock.version >> 0));
if ((major_version != LFS_DISK_VERSION_MAJOR ||
minor_version > LFS_DISK_VERSION_MINOR)) {
LFS_ERROR("Invalid version v%"PRIu16".%"PRIu16,
major_version, minor_version);
LFS_ERROR("Invalid version "
"v%"PRIu16".%"PRIu16" != v%"PRIu16".%"PRIu16,
major_version, minor_version,
LFS_DISK_VERSION_MAJOR, LFS_DISK_VERSION_MINOR);
err = LFS_ERR_INVAL;
goto cleanup;
}
// found older minor version? set an in-device only bit in the
// gstate so we know we need to rewrite the superblock before
// the first write
if (minor_version < LFS_DISK_VERSION_MINOR) {
LFS_DEBUG("Found older minor version "
"v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16,
major_version, minor_version,
LFS_DISK_VERSION_MAJOR, LFS_DISK_VERSION_MINOR);
#ifndef LFS_READONLY
// note this bit is reserved on disk, so fetching more gstate
// will not interfere here
lfs_fs_prepsuperblock(lfs, true);
#endif
}
// check superblock configuration
if (superblock.name_max) {
if (superblock.name_max > lfs->name_max) {
@ -4432,9 +4559,17 @@ static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2],
}
#endif
#ifndef LFS_READONLY
static void lfs_fs_prepsuperblock(lfs_t *lfs, bool needssuperblock) {
lfs->gstate.tag = (lfs->gstate.tag & ~LFS_MKTAG(0, 0, 0x200))
| (uint32_t)needssuperblock << 9;
}
#endif
#ifndef LFS_READONLY
static int lfs_fs_preporphans(lfs_t *lfs, int8_t orphans) {
LFS_ASSERT(lfs_tag_size(lfs->gstate.tag) > 0 || orphans >= 0);
LFS_ASSERT(lfs_tag_size(lfs->gstate.tag) > 0x000 || orphans >= 0);
LFS_ASSERT(lfs_tag_size(lfs->gstate.tag) < 0x1ff || orphans <= 0);
lfs->gstate.tag += orphans;
lfs->gstate.tag = ((lfs->gstate.tag & ~LFS_MKTAG(0x800, 0, 0)) |
((uint32_t)lfs_gstate_hasorphans(&lfs->gstate) << 31));
@ -4453,6 +4588,45 @@ static void lfs_fs_prepmove(lfs_t *lfs,
}
#endif
#ifndef LFS_READONLY
static int lfs_fs_desuperblock(lfs_t *lfs) {
if (!lfs_gstate_needssuperblock(&lfs->gstate)) {
return 0;
}
LFS_DEBUG("Rewriting superblock {0x%"PRIx32", 0x%"PRIx32"}",
lfs->root[0],
lfs->root[1]);
lfs_mdir_t root;
int err = lfs_dir_fetch(lfs, &root, lfs->root);
if (err) {
return err;
}
// write a new superblock
lfs_superblock_t superblock = {
.version = LFS_DISK_VERSION,
.block_size = lfs->cfg->block_size,
.block_count = lfs->cfg->block_count,
.name_max = lfs->name_max,
.file_max = lfs->file_max,
.attr_max = lfs->attr_max,
};
lfs_superblock_tole32(&superblock);
err = lfs_dir_commit(lfs, &root, LFS_MKATTRS(
{LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)),
&superblock}));
if (err) {
return err;
}
lfs_fs_prepsuperblock(lfs, false);
return 0;
}
#endif
#ifndef LFS_READONLY
static int lfs_fs_demove(lfs_t *lfs) {
if (!lfs_gstate_hasmove(&lfs->gdisk)) {
@ -4465,6 +4639,10 @@ static int lfs_fs_demove(lfs_t *lfs) {
lfs->gdisk.pair[1],
lfs_tag_id(lfs->gdisk.tag));
// no other gstate is supported at this time, so if we found something else
// something most likely went wrong in gstate calculation
LFS_ASSERT(lfs_tag_type3(lfs->gdisk.tag) == LFS_TYPE_DELETE);
// fetch and delete the moved entry
lfs_mdir_t movedir;
int err = lfs_dir_fetch(lfs, &movedir, lfs->gdisk.pair);
@ -4493,7 +4671,6 @@ static int lfs_fs_deorphan(lfs_t *lfs, bool powerloss) {
int8_t found = 0;
restart:
// Check for orphans in two separate passes:
// - 1 for half-orphans (relocations)
// - 2 for full-orphans (removes/renames)
@ -4502,10 +4679,12 @@ restart:
// references to full-orphans, effectively hiding them from the deorphan
// search.
//
for (int pass = 0; pass < 2; pass++) {
int pass = 0;
while (pass < 2) {
// Fix any orphans
lfs_mdir_t pdir = {.split = true, .tail = {0, 1}};
lfs_mdir_t dir;
bool moreorphans = false;
// iterate over all directory directory entries
while (!lfs_pair_isnull(pdir.tail)) {
@ -4566,7 +4745,7 @@ restart:
// did our commit create more orphans?
if (state == LFS_OK_ORPHANED) {
goto restart;
moreorphans = true;
}
// refetch tail
@ -4602,7 +4781,7 @@ restart:
// did our commit create more orphans?
if (state == LFS_OK_ORPHANED) {
goto restart;
moreorphans = true;
}
// refetch tail
@ -4612,6 +4791,8 @@ restart:
pdir = dir;
}
pass = moreorphans ? 0 : pass+1;
}
// mark orphans as fixed
@ -4623,7 +4804,12 @@ restart:
#ifndef LFS_READONLY
static int lfs_fs_forceconsistency(lfs_t *lfs) {
int err = lfs_fs_demove(lfs);
int err = lfs_fs_desuperblock(lfs);
if (err) {
return err;
}
err = lfs_fs_demove(lfs);
if (err) {
return err;
}

6
lfs.h
View File

@ -21,14 +21,14 @@ extern "C"
// Software library version
// Major (top-nibble), incremented on backwards incompatible changes
// Minor (bottom-nibble), incremented on feature additions
#define LFS_VERSION 0x00020005
#define LFS_VERSION 0x00020006
#define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16))
#define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0))
// Version of On-disk data structures
// Major (top-nibble), incremented on backwards incompatible changes
// Minor (bottom-nibble), incremented on feature additions
#define LFS_DISK_VERSION 0x00020000
#define LFS_DISK_VERSION 0x00020001
#define LFS_DISK_VERSION_MAJOR (0xffff & (LFS_DISK_VERSION >> 16))
#define LFS_DISK_VERSION_MINOR (0xffff & (LFS_DISK_VERSION >> 0))
@ -112,6 +112,8 @@ enum lfs_type {
LFS_TYPE_SOFTTAIL = 0x600,
LFS_TYPE_HARDTAIL = 0x601,
LFS_TYPE_MOVESTATE = 0x7ff,
LFS_TYPE_CCRC = 0x500,
LFS_TYPE_FCRC = 0x5ff,
// internal chip sources
LFS_FROM_NOOP = 0x000,

5
scripts/changeprefix.py
View File

@ -107,7 +107,10 @@ def main(from_prefix, to_prefix, paths=[], *,
elif no_renames:
to_path = from_path
else:
to_path, _ = changeprefix(from_prefix, to_prefix, from_path)
to_path = os.path.join(
os.path.dirname(from_path),
changeprefix(from_prefix, to_prefix,
os.path.basename(from_path))[0])
# rename contents
changefile(from_prefix, to_prefix, from_path, to_path,

62
scripts/readmdir.py
View File

@ -24,6 +24,8 @@ TAG_TYPES = {
'gstate': (0x700, 0x700),
'movestate': (0x7ff, 0x7ff),
'crc': (0x700, 0x500),
'ccrc': (0x780, 0x500),
'fcrc': (0x7ff, 0x5ff),
}
class Tag:
@ -99,7 +101,16 @@ class Tag:
return struct.unpack('b', struct.pack('B', self.chunk))[0]
def is_(self, type):
return (self.type & TAG_TYPES[type][0]) == TAG_TYPES[type][1]
try:
if ' ' in type:
type1, type3 = type.split()
return (self.is_(type1) and
(self.type & ~TAG_TYPES[type1][0]) == int(type3, 0))
return self.type == int(type, 0)
except (ValueError, KeyError):
return (self.type & TAG_TYPES[type][0]) == TAG_TYPES[type][1]
def mkmask(self):
return Tag(
@ -109,14 +120,20 @@ class Tag:
def chid(self, nid):
ntag = Tag(self.type, nid, self.size)
if hasattr(self, 'off'): ntag.off = self.off
if hasattr(self, 'data'): ntag.data = self.data
if hasattr(self, 'crc'): ntag.crc = self.crc
if hasattr(self, 'off'): ntag.off = self.off
if hasattr(self, 'data'): ntag.data = self.data
if hasattr(self, 'ccrc'): ntag.crc = self.crc
if hasattr(self, 'erased'): ntag.erased = self.erased
return ntag
def typerepr(self):
if self.is_('crc') and getattr(self, 'crc', 0xffffffff) != 0xffffffff:
return 'crc (bad)'
if (self.is_('ccrc')
and getattr(self, 'ccrc', 0xffffffff) != 0xffffffff):
crc_status = ' (bad)'
elif self.is_('fcrc') and getattr(self, 'erased', False):
crc_status = ' (era)'
else:
crc_status = ''
reverse_types = {v: k for k, v in TAG_TYPES.items()}
for prefix in range(12):
@ -124,12 +141,12 @@ class Tag:
if (mask, self.type & mask) in reverse_types:
type = reverse_types[mask, self.type & mask]
if prefix > 0:
return '%s %#0*x' % (
type, prefix//4, self.type & ((1 << prefix)-1))
return '%s %#x%s' % (
type, self.type & ((1 << prefix)-1), crc_status)
else:
return type
return '%s%s' % (type, crc_status)
else:
return '%02x' % self.type
return '%02x%s' % (self.type, crc_status)
def idrepr(self):
return repr(self.id) if self.id != 0x3ff else '.'
@ -172,6 +189,8 @@ class MetadataPair:
self.rev, = struct.unpack('<I', block[0:4])
crc = binascii.crc32(block[0:4])
fcrctag = None
fcrcdata = None
# parse tags
corrupt = False
@ -182,11 +201,11 @@ class MetadataPair:
while len(block) - off >= 4:
ntag, = struct.unpack('>I', block[off:off+4])
tag = Tag(int(tag) ^ ntag)
tag = Tag((int(tag) ^ ntag) & 0x7fffffff)
tag.off = off + 4
tag.data = block[off+4:off+tag.dsize]
if tag.is_('crc'):
crc = binascii.crc32(block[off:off+4+4], crc)
if tag.is_('ccrc'):
crc = binascii.crc32(block[off:off+2*4], crc)
else:
crc = binascii.crc32(block[off:off+tag.dsize], crc)
tag.crc = crc
@ -194,16 +213,29 @@ class MetadataPair:
self.all_.append(tag)
if tag.is_('crc'):
if tag.is_('fcrc') and len(tag.data) == 8:
fcrctag = tag
fcrcdata = struct.unpack('<II', tag.data)
elif tag.is_('ccrc'):
# is valid commit?
if crc != 0xffffffff:
corrupt = True
if not corrupt:
self.log = self.all_.copy()
# end of commit?
if fcrcdata:
fcrcsize, fcrc = fcrcdata
fcrc_ = 0xffffffff ^ binascii.crc32(
block[off:off+fcrcsize])
if fcrc_ == fcrc:
fcrctag.erased = True
corrupt = True
# reset tag parsing
crc = 0
tag = Tag(int(tag) ^ ((tag.type & 1) << 31))
fcrctag = None
fcrcdata = None
# find active ids
self.ids = list(it.takewhile(
@ -280,7 +312,7 @@ class MetadataPair:
f.write('\n')
for tag in tags:
f.write("%08x: %08x %-13s %4s %4s" % (
f.write("%08x: %08x %-14s %3s %4s" % (
tag.off, tag,
tag.typerepr(), tag.idrepr(), tag.sizerepr()))
if truncate:

1360
tests/test_compat.toml Normal file
View File

File diff suppressed because it is too large Load Diff

182
tests/test_powerloss.toml Normal file
View File

@ -0,0 +1,182 @@
# There are already a number of tests that test general operations under
# power-loss (see the reentrant attribute). These tests are for explicitly
# testing specific corner cases.
# only a revision count
[cases.test_powerloss_only_rev]
code = '''
lfs_t lfs;
lfs_format(&lfs, cfg) => 0;
lfs_mount(&lfs, cfg) => 0;
lfs_mkdir(&lfs, "notebook") => 0;
lfs_file_t file;
lfs_file_open(&lfs, &file, "notebook/paper",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0;
char buffer[256];
strcpy(buffer, "hello");
lfs_size_t size = strlen("hello");
for (int i = 0; i < 5; i++) {
lfs_file_write(&lfs, &file, buffer, size) => size;
lfs_file_sync(&lfs, &file) => 0;
}
lfs_file_close(&lfs, &file) => 0;
char rbuffer[256];
lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
for (int i = 0; i < 5; i++) {
lfs_file_read(&lfs, &file, rbuffer, size) => size;
assert(memcmp(rbuffer, buffer, size) == 0);
}
lfs_file_close(&lfs, &file) => 0;
lfs_unmount(&lfs) => 0;
// get pair/rev count
lfs_mount(&lfs, cfg) => 0;
lfs_dir_t dir;
lfs_dir_open(&lfs, &dir, "notebook") => 0;
lfs_block_t pair[2] = {dir.m.pair[0], dir.m.pair[1]};
uint32_t rev = dir.m.rev;
lfs_dir_close(&lfs, &dir) => 0;
lfs_unmount(&lfs) => 0;
// write just the revision count
uint8_t bbuffer[BLOCK_SIZE];
cfg->read(cfg, pair[1], 0, bbuffer, BLOCK_SIZE) => 0;
memcpy(bbuffer, &(uint32_t){lfs_tole32(rev+1)}, sizeof(uint32_t));
cfg->erase(cfg, pair[1]) => 0;
cfg->prog(cfg, pair[1], 0, bbuffer, BLOCK_SIZE) => 0;
lfs_mount(&lfs, cfg) => 0;
// can read?
lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
for (int i = 0; i < 5; i++) {
lfs_file_read(&lfs, &file, rbuffer, size) => size;
assert(memcmp(rbuffer, buffer, size) == 0);
}
lfs_file_close(&lfs, &file) => 0;
// can write?
lfs_file_open(&lfs, &file, "notebook/paper",
LFS_O_WRONLY | LFS_O_APPEND) => 0;
strcpy(buffer, "goodbye");
size = strlen("goodbye");
for (int i = 0; i < 5; i++) {
lfs_file_write(&lfs, &file, buffer, size) => size;
lfs_file_sync(&lfs, &file) => 0;
}
lfs_file_close(&lfs, &file) => 0;
lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
strcpy(buffer, "hello");
size = strlen("hello");
for (int i = 0; i < 5; i++) {
lfs_file_read(&lfs, &file, rbuffer, size) => size;
assert(memcmp(rbuffer, buffer, size) == 0);
}
strcpy(buffer, "goodbye");
size = strlen("goodbye");
for (int i = 0; i < 5; i++) {
lfs_file_read(&lfs, &file, rbuffer, size) => size;
assert(memcmp(rbuffer, buffer, size) == 0);
}
lfs_file_close(&lfs, &file) => 0;
lfs_unmount(&lfs) => 0;
'''
# partial prog, may not be byte in order!
[cases.test_powerloss_partial_prog]
if = "PROG_SIZE < BLOCK_SIZE"
defines.BYTE_OFF = ["0", "PROG_SIZE-1", "PROG_SIZE/2"]
defines.BYTE_VALUE = [0x33, 0xcc]
in = "lfs.c"
code = '''
lfs_t lfs;
lfs_format(&lfs, cfg) => 0;
lfs_mount(&lfs, cfg) => 0;
lfs_mkdir(&lfs, "notebook") => 0;
lfs_file_t file;
lfs_file_open(&lfs, &file, "notebook/paper",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0;
char buffer[256];
strcpy(buffer, "hello");
lfs_size_t size = strlen("hello");
for (int i = 0; i < 5; i++) {
lfs_file_write(&lfs, &file, buffer, size) => size;
lfs_file_sync(&lfs, &file) => 0;
}
lfs_file_close(&lfs, &file) => 0;
char rbuffer[256];
lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
for (int i = 0; i < 5; i++) {
lfs_file_read(&lfs, &file, rbuffer, size) => size;
assert(memcmp(rbuffer, buffer, size) == 0);
}
lfs_file_close(&lfs, &file) => 0;
lfs_unmount(&lfs) => 0;
// imitate a partial prog, value should not matter, if littlefs
// doesn't notice the partial prog testbd will assert
// get offset to next prog
lfs_mount(&lfs, cfg) => 0;
lfs_dir_t dir;
lfs_dir_open(&lfs, &dir, "notebook") => 0;
lfs_block_t block = dir.m.pair[0];
lfs_off_t off = dir.m.off;
lfs_dir_close(&lfs, &dir) => 0;
lfs_unmount(&lfs) => 0;
// tweak byte
uint8_t bbuffer[BLOCK_SIZE];
cfg->read(cfg, block, 0, bbuffer, BLOCK_SIZE) => 0;
bbuffer[off + BYTE_OFF] = BYTE_VALUE;
cfg->erase(cfg, block) => 0;
cfg->prog(cfg, block, 0, bbuffer, BLOCK_SIZE) => 0;
lfs_mount(&lfs, cfg) => 0;
// can read?
lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
for (int i = 0; i < 5; i++) {
lfs_file_read(&lfs, &file, rbuffer, size) => size;
assert(memcmp(rbuffer, buffer, size) == 0);
}
lfs_file_close(&lfs, &file) => 0;
// can write?
lfs_file_open(&lfs, &file, "notebook/paper",
LFS_O_WRONLY | LFS_O_APPEND) => 0;
strcpy(buffer, "goodbye");
size = strlen("goodbye");
for (int i = 0; i < 5; i++) {
lfs_file_write(&lfs, &file, buffer, size) => size;
lfs_file_sync(&lfs, &file) => 0;
}
lfs_file_close(&lfs, &file) => 0;
lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
strcpy(buffer, "hello");
size = strlen("hello");
for (int i = 0; i < 5; i++) {
lfs_file_read(&lfs, &file, rbuffer, size) => size;
assert(memcmp(rbuffer, buffer, size) == 0);
}
strcpy(buffer, "goodbye");
size = strlen("goodbye");
for (int i = 0; i < 5; i++) {
lfs_file_read(&lfs, &file, rbuffer, size) => size;
assert(memcmp(rbuffer, buffer, size) == 0);
}
lfs_file_close(&lfs, &file) => 0;
lfs_unmount(&lfs) => 0;
'''