mirror of
https://github.com/hathach/tinyusb.git
synced 2026-08-18 02:53:35 +00:00
Merge pull request #3762 from hathach/claude/agents-workflows
Multi-agent dev/test harness: worker agents, workflows, /pre-pr gate, per-board HIL locks
This commit is contained in:
Ha Thach
GitHub
254
test/hil/board_lock.py
Executable file
254
test/hil/board_lock.py
Executable file
@ -0,0 +1,254 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Per-board advisory locks for the HIL rig.
|
||||
|
||||
Arbitrates board access between dev sessions and CI's hil_test.py without
|
||||
stopping the actions-runner. Locks are kernel flocks: the kernel releases
|
||||
them automatically when the holder process dies, and holders clear their
|
||||
lock-file record on release so records stay truthful (/tmp also clears on
|
||||
reboot).
|
||||
|
||||
Usage:
|
||||
board_lock.py hold BOARD [BOARD...] --reason TEXT
|
||||
board_lock.py hold --all [--config CONFIG.json] --reason TEXT
|
||||
board_lock.py release BOARD [BOARD...] | release --all
|
||||
board_lock.py status
|
||||
|
||||
A holder process holds ALL boards given in one `hold` call; releasing any of
|
||||
them kills that holder and releases all of its boards.
|
||||
"""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
|
||||
LOCK_DIR = '/tmp/tinyusb-hil-locks'
|
||||
|
||||
|
||||
def lock_path(board: str) -> str:
|
||||
return os.path.join(LOCK_DIR, f'{board}.lock')
|
||||
|
||||
|
||||
def boards_from_config(config: str) -> list:
|
||||
try:
|
||||
with open(config) as f:
|
||||
return [b['name'] for b in json.load(f)['boards']]
|
||||
except (OSError, ValueError, KeyError) as e:
|
||||
print(f'ERROR: cannot read board roster {config}: {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def read_info(board: str):
|
||||
try:
|
||||
with open(lock_path(board)) as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def is_locked(board: str) -> bool:
|
||||
"""True if the recorded holder process is still alive.
|
||||
|
||||
Deliberately never touches the flock: even a momentary probe lock would
|
||||
make a concurrent acquirer's LOCK_NB attempt fail spuriously. The flock
|
||||
taken by acquirers themselves stays the only authority."""
|
||||
info = read_info(board)
|
||||
pid = info.get('pid') if isinstance(info, dict) else None
|
||||
if not isinstance(pid, int) or pid <= 0:
|
||||
return False
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True # alive but owned by another user (e.g. the CI runner)
|
||||
return True
|
||||
|
||||
|
||||
def cmd_hold(boards, reason):
|
||||
os.makedirs(LOCK_DIR, exist_ok=True)
|
||||
# No pre-check: the holder's own LOCK_NB flock is the only authority — a
|
||||
# recorded pid may be stale or recycled (e.g. a live hil_test.py worker
|
||||
# that already released this board's flock but not its record).
|
||||
# The holder signals success through this pipe. A generic is_locked()
|
||||
# poll would be fooled by a RIVAL invocation's flock — only the holder
|
||||
# itself knows whether it won every board.
|
||||
r_fd, w_fd = os.pipe()
|
||||
pid = os.fork()
|
||||
if pid > 0:
|
||||
os.close(w_fd)
|
||||
os.waitpid(pid, 0) # reap intermediate child
|
||||
ready, _, _ = select.select([r_fd], [], [], 10)
|
||||
ok = bool(ready) and os.read(r_fd, 1) == b'1'
|
||||
os.close(r_fd)
|
||||
if ok:
|
||||
print(f'held: {", ".join(boards)}')
|
||||
return 0
|
||||
for b in boards:
|
||||
info = read_info(b)
|
||||
if info:
|
||||
print(f'ERROR: {b} locked: {info}', file=sys.stderr)
|
||||
print('ERROR: holder failed to acquire locks', file=sys.stderr)
|
||||
return 1
|
||||
# intermediate child: detach, then spawn the actual holder
|
||||
os.setsid()
|
||||
if os.fork() > 0:
|
||||
os._exit(0)
|
||||
# holder (grandchild): acquire all flocks, signal the parent, sleep until killed
|
||||
os.close(r_fd)
|
||||
# Keep the success pipe clear of fds 0-2: invoked with stdio closed,
|
||||
# os.pipe() can land there and the dup2 loop below would clobber it.
|
||||
if w_fd <= 2:
|
||||
w_fd = fcntl.fcntl(w_fd, fcntl.F_DUPFD, 3)
|
||||
# Detach stdio: a `hold` whose output is captured must see EOF when the
|
||||
# front-end exits — the immortal holder must not keep that pipe open.
|
||||
devnull = os.open(os.devnull, os.O_RDWR)
|
||||
for std_fd in (0, 1, 2):
|
||||
os.dup2(devnull, std_fd)
|
||||
if devnull > 2:
|
||||
os.close(devnull)
|
||||
try:
|
||||
handles = []
|
||||
for b in boards:
|
||||
# O_RDWR without O_TRUNC: never truncate before the flock is
|
||||
# held — a losing racer must not wipe the winner's holder info.
|
||||
fd = os.open(lock_path(b), os.O_RDWR | os.O_CREAT, 0o666)
|
||||
fh = os.fdopen(fd, 'r+')
|
||||
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
fh.truncate(0)
|
||||
fh.seek(0)
|
||||
json.dump({'pid': os.getpid(), 'reason': reason,
|
||||
'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh)
|
||||
fh.flush()
|
||||
handles.append(fh)
|
||||
except OSError:
|
||||
try:
|
||||
os.write(w_fd, b'0')
|
||||
except OSError:
|
||||
pass
|
||||
os._exit(1) # lost a race; parent reports the failure
|
||||
os.write(w_fd, b'1')
|
||||
os.close(w_fd)
|
||||
|
||||
def _bow_out(*_):
|
||||
# clear the records before dying so read_info/status stay truthful
|
||||
# (the kernel drops the flocks themselves on exit either way)
|
||||
for h in handles:
|
||||
try:
|
||||
h.truncate(0)
|
||||
except OSError:
|
||||
pass
|
||||
os._exit(0)
|
||||
|
||||
signal.signal(signal.SIGTERM, _bow_out)
|
||||
while True:
|
||||
signal.pause()
|
||||
|
||||
|
||||
def cmd_release(boards):
|
||||
rc = 0
|
||||
victims = set()
|
||||
for b in boards:
|
||||
try:
|
||||
fd = os.open(lock_path(b), os.O_RDWR)
|
||||
except OSError:
|
||||
continue # no lock file (or another user's): nothing we can release
|
||||
fh = os.fdopen(fd, 'r+')
|
||||
try:
|
||||
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
# flock genuinely held — never SIGTERM on a mere pid record: the
|
||||
# pid may be recycled, or a live worker that already moved on.
|
||||
fh.close()
|
||||
info = read_info(b) or {}
|
||||
pid = info.get('pid')
|
||||
if info.get('reason') == 'hil_test.py':
|
||||
print(f'ERROR: {b} is mid-test by hil_test.py (pid {pid}) — not killing a '
|
||||
'CI run; wait for it to finish', file=sys.stderr)
|
||||
rc = 1
|
||||
elif isinstance(pid, int) and pid > 0:
|
||||
victims.add(pid)
|
||||
else:
|
||||
print(f'ERROR: {b} is held but its record is unreadable', file=sys.stderr)
|
||||
rc = 1
|
||||
continue
|
||||
# flock was free: only a stale record remained — clear it
|
||||
try:
|
||||
fh.truncate(0)
|
||||
except OSError:
|
||||
pass
|
||||
fh.close()
|
||||
for holder in sorted(victims):
|
||||
try:
|
||||
os.kill(holder, signal.SIGTERM)
|
||||
print(f'released holder pid {holder}')
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
except PermissionError:
|
||||
print(f'ERROR: holder pid {holder} belongs to another user — cannot signal it',
|
||||
file=sys.stderr)
|
||||
rc = 1
|
||||
time.sleep(0.3)
|
||||
still = [b for b in boards if is_locked(b)]
|
||||
if still:
|
||||
print(f'ERROR: still locked: {", ".join(still)}', file=sys.stderr)
|
||||
return 1
|
||||
return rc
|
||||
|
||||
|
||||
def cmd_status():
|
||||
if not os.path.isdir(LOCK_DIR):
|
||||
print('no locks')
|
||||
return 0
|
||||
any_locked = False
|
||||
for fn in sorted(os.listdir(LOCK_DIR)):
|
||||
if not fn.endswith('.lock'):
|
||||
continue
|
||||
b = fn[:-5]
|
||||
if is_locked(b):
|
||||
any_locked = True
|
||||
print(f'{b}: {read_info(b)}')
|
||||
if not any_locked:
|
||||
print('no locks')
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = ap.add_subparsers(dest='cmd', required=True)
|
||||
p_hold = sub.add_parser('hold')
|
||||
p_hold.add_argument('boards', nargs='*')
|
||||
p_hold.add_argument('--all', action='store_true')
|
||||
p_hold.add_argument('--config',
|
||||
default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
'tinyusb.json'),
|
||||
help='board roster JSON (default: tinyusb.json beside this script)')
|
||||
p_hold.add_argument('--reason', required=True)
|
||||
p_rel = sub.add_parser('release')
|
||||
p_rel.add_argument('boards', nargs='*')
|
||||
p_rel.add_argument('--all', action='store_true')
|
||||
sub.add_parser('status')
|
||||
a = ap.parse_args()
|
||||
if a.cmd == 'hold':
|
||||
boards = boards_from_config(a.config) if a.all else a.boards
|
||||
if not boards:
|
||||
ap.error('no boards given (name boards or use --all)')
|
||||
sys.exit(cmd_hold(boards, a.reason))
|
||||
if a.cmd == 'release':
|
||||
if a.all:
|
||||
boards = ([fn[:-5] for fn in os.listdir(LOCK_DIR) if fn.endswith('.lock')]
|
||||
if os.path.isdir(LOCK_DIR) else [])
|
||||
else:
|
||||
boards = a.boards
|
||||
if not boards:
|
||||
ap.error('no boards given (name boards or use --all)')
|
||||
sys.exit(cmd_release(boards))
|
||||
sys.exit(cmd_status())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -58,6 +58,51 @@ import ctypes
|
||||
from pymtp import MTP
|
||||
import string
|
||||
|
||||
# --- per-board dev-session locks (see test/hil/board_lock.py) ------------
|
||||
BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks'
|
||||
|
||||
def acquire_board_lock(board_name):
|
||||
"""Take this board's flock for the duration of its flash+test.
|
||||
Returns an open file handle (keep it referenced; closing releases it),
|
||||
or None when HIL_NO_BOARD_LOCK=1 or the lock dir is unusable (fail-open:
|
||||
locking must never break a test run by itself).
|
||||
Raises RuntimeError only when another session holds the board."""
|
||||
import fcntl
|
||||
if os.environ.get('HIL_NO_BOARD_LOCK') == '1':
|
||||
return None # user-authorized bypass — see board_lock.py / hil skill
|
||||
try:
|
||||
os.makedirs(BOARD_LOCK_DIR, exist_ok=True)
|
||||
fd = os.open(os.path.join(BOARD_LOCK_DIR, f'{board_name}.lock'),
|
||||
os.O_RDWR | os.O_CREAT, 0o666)
|
||||
fh = os.fdopen(fd, 'r+')
|
||||
except OSError as e:
|
||||
# odd lock dir (perms, path collision): proceed unlocked, but say so —
|
||||
# a silent fail-open is indistinguishable from the intentional bypass
|
||||
print(f'warning: board lock unavailable for {board_name} ({e}); proceeding unlocked',
|
||||
flush=True)
|
||||
return None
|
||||
try:
|
||||
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
try:
|
||||
info = fh.read(500).strip()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
info = ''
|
||||
fh.close()
|
||||
raise RuntimeError(f'board locked: {info or "unknown holder"}')
|
||||
# announce ourselves so the other side's conflict message is truthful;
|
||||
# best-effort — the flock itself is already held
|
||||
try:
|
||||
fh.truncate(0)
|
||||
fh.seek(0)
|
||||
json.dump({'pid': os.getpid(), 'reason': 'hil_test.py',
|
||||
'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh)
|
||||
fh.flush()
|
||||
except OSError:
|
||||
pass
|
||||
return fh
|
||||
|
||||
|
||||
ENUM_TIMEOUT = 15
|
||||
|
||||
STATUS_OK = "\033[32mOK\033[0m"
|
||||
@ -1659,63 +1704,82 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]:
|
||||
name = board['name']
|
||||
flasher = board['flasher']
|
||||
|
||||
# default to all tests
|
||||
test_list = []
|
||||
try:
|
||||
_lock_fh = acquire_board_lock(name)
|
||||
except RuntimeError as e:
|
||||
log_line(f'{name:25} {STATUS_FAILED}: {e}')
|
||||
# visible report row so the ❌ matches the exit code; failed-tests stays
|
||||
# empty so a re-run repeats the whole board (no bogus -bt test filter)
|
||||
return name, 1, [], [(name, {'board-locked': 'fail'})]
|
||||
try:
|
||||
# default to all tests
|
||||
test_list = []
|
||||
|
||||
if name in board_test:
|
||||
test_list = board_test[name]
|
||||
elif len(test_only) > 0:
|
||||
# Explicit -t: filter against the board's capabilities so a device-only
|
||||
# board doesn't try to run host/dual tests (the test functions need a
|
||||
# `dev_attached` entry in the board config that won't exist).
|
||||
board_tests = board.get('tests', {})
|
||||
if 'only' in board_tests:
|
||||
allowed = set(board_tests['only'])
|
||||
test_list = [t for t in test_only if t in allowed]
|
||||
else:
|
||||
for t in test_only:
|
||||
category = t.split('/', 1)[0]
|
||||
if board_tests.get(category) is True:
|
||||
test_list.append(t)
|
||||
else:
|
||||
if 'tests' in board:
|
||||
board_tests = board['tests']
|
||||
if board_tests.get('device') is True:
|
||||
test_list += list(device_tests)
|
||||
if board_tests.get('dual') is True:
|
||||
test_list += dual_tests
|
||||
if board_tests.get('host') is True:
|
||||
test_list += host_test
|
||||
if name in board_test:
|
||||
test_list = board_test[name]
|
||||
elif len(test_only) > 0:
|
||||
# Explicit -t: filter against the board's capabilities so a device-only
|
||||
# board doesn't try to run host/dual tests (the test functions need a
|
||||
# `dev_attached` entry in the board config that won't exist).
|
||||
board_tests = board.get('tests', {})
|
||||
if 'only' in board_tests:
|
||||
test_list = board_tests['only']
|
||||
if 'skip' in board_tests:
|
||||
for skip in board_tests['skip']:
|
||||
if skip in test_list:
|
||||
test_list.remove(skip)
|
||||
log_line(f'{name:25} {skip:30} ... Skip')
|
||||
allowed = set(board_tests['only'])
|
||||
test_list = [t for t in test_only if t in allowed]
|
||||
else:
|
||||
for t in test_only:
|
||||
category = t.split('/', 1)[0]
|
||||
if board_tests.get(category) is True:
|
||||
test_list.append(t)
|
||||
else:
|
||||
if 'tests' in board:
|
||||
board_tests = board['tests']
|
||||
if board_tests.get('device') is True:
|
||||
test_list += list(device_tests)
|
||||
if board_tests.get('dual') is True:
|
||||
test_list += dual_tests
|
||||
if board_tests.get('host') is True:
|
||||
test_list += host_test
|
||||
if 'only' in board_tests:
|
||||
test_list = board_tests['only']
|
||||
if 'skip' in board_tests:
|
||||
for skip in board_tests['skip']:
|
||||
if skip in test_list:
|
||||
test_list.remove(skip)
|
||||
log_line(f'{name:25} {skip:30} ... Skip')
|
||||
|
||||
err_count = 0
|
||||
failed_tests = []
|
||||
rows = [] # list of (row_label, {example: status}) — one row per build variant
|
||||
variants = board.get('variant') or [{'name': name, 'flags': ''}]
|
||||
err_count = 0
|
||||
failed_tests = []
|
||||
rows = [] # list of (row_label, {example: status}) — one row per build variant
|
||||
variants = board.get('variant') or [{'name': name, 'flags': ''}]
|
||||
|
||||
for v in variants:
|
||||
vname = v['name']
|
||||
cells = {}
|
||||
for test in test_list:
|
||||
ec, status, metric = test_example(board, vname, test)
|
||||
err_count += ec
|
||||
cells[test] = metric if metric else status
|
||||
if ec > 0:
|
||||
failed_tests.append(test)
|
||||
rows.append((vname, cells))
|
||||
for v in variants:
|
||||
vname = v['name']
|
||||
cells = {}
|
||||
for test in test_list:
|
||||
ec, status, metric = test_example(board, vname, test)
|
||||
err_count += ec
|
||||
cells[test] = metric if metric else status
|
||||
if ec > 0:
|
||||
failed_tests.append(test)
|
||||
rows.append((vname, cells))
|
||||
|
||||
# flash board_test last to disable board's usb (skipped when --skip-flash is set);
|
||||
# this is teardown/park, not a test — not recorded in the report
|
||||
if not skip_flash:
|
||||
test_example(board, variants[0]['name'], 'device/board_test')
|
||||
# flash board_test last to disable board's usb (skipped when --skip-flash is set);
|
||||
# this is teardown/park, not a test — not recorded in the report
|
||||
if not skip_flash:
|
||||
test_example(board, variants[0]['name'], 'device/board_test')
|
||||
|
||||
return name, err_count, sorted(set(failed_tests)), rows
|
||||
return name, err_count, sorted(set(failed_tests)), rows
|
||||
finally:
|
||||
if _lock_fh:
|
||||
try:
|
||||
# clear our pid record before dropping the flock: this worker
|
||||
# process lives on (pool reuse), so a stale record would make
|
||||
# board_lock.py's pid-liveness checks report a freed board as
|
||||
# still locked for the rest of the run
|
||||
_lock_fh.truncate(0)
|
||||
except OSError:
|
||||
pass
|
||||
_lock_fh.close()
|
||||
|
||||
|
||||
REPORT_MD = 'hil_report.md'
|
||||
@ -1781,7 +1845,17 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str:
|
||||
pass # corrupt/old sidecar: start fresh
|
||||
|
||||
# merge this run: current cells override prior for boards/tests that ran
|
||||
for _, _, _, rows in mret:
|
||||
for name, _, _, rows in mret:
|
||||
if rows and not any('board-locked' in cells for _, cells in rows):
|
||||
# board ran for real this time: clear a stale lock-failure cell
|
||||
# (its row is keyed by board name; test rows may be variant names)
|
||||
stale = acc.get(name)
|
||||
if stale is not None:
|
||||
stale.pop('board-locked', None)
|
||||
if not stale:
|
||||
# variant-keyed boards never repopulate the board-name row —
|
||||
# drop it or it renders as a blank ghost row
|
||||
del acc[name]
|
||||
for row_label, cells in rows:
|
||||
acc.setdefault(row_label, {}).update(cells)
|
||||
|
||||
@ -1844,6 +1918,11 @@ def main() -> None:
|
||||
if len(boards) == 0:
|
||||
config_boards = [e for e in config['boards'] if e['name'] not in skip_boards]
|
||||
else:
|
||||
unknown = [b for b in boards if b not in {e['name'] for e in config['boards']}]
|
||||
if unknown:
|
||||
# exiting 0 with 'No tests were run.' would read as a green HIL run
|
||||
print(f'ERROR: board(s) not in {config_file.name}: {", ".join(unknown)}')
|
||||
sys.exit(1)
|
||||
config_boards = [e for e in config['boards'] if e['name'] in boards]
|
||||
|
||||
build_err = 0
|
||||
|
||||
Reference in New Issue
Block a user