Merge remote-tracking branch 'origin/master' into usbtest

# Conflicts:
#	.claude/skills/hil/SKILL.md
#	test/hil/hil_test.py

test/hil/board_lock.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""Per-board advisory locks for the HIL rig.
+
+Arbitrates board access between dev sessions and CI's hil_test.py without
+stopping the actions-runner. Locks are kernel flocks: the kernel releases
+them automatically when the holder process dies, and holders clear their
+lock-file record on release so records stay truthful (/tmp also clears on
+reboot).
+
+Usage:
+  board_lock.py hold BOARD [BOARD...] --reason TEXT
+  board_lock.py hold --all [--config CONFIG.json] --reason TEXT
+  board_lock.py release BOARD [BOARD...] | release --all
+  board_lock.py status
+
+A holder process holds ALL boards given in one `hold` call; releasing any of
+them kills that holder and releases all of its boards.
+"""
+import argparse
+import fcntl
+import json
+import os
+import select
+import signal
+import sys
+import time
+
+LOCK_DIR = '/tmp/tinyusb-hil-locks'
+
+
+def lock_path(board: str) -> str:
+    return os.path.join(LOCK_DIR, f'{board}.lock')
+
+
+def boards_from_config(config: str) -> list:
+    try:
+        with open(config) as f:
+            return [b['name'] for b in json.load(f)['boards']]
+    except (OSError, ValueError, KeyError) as e:
+        print(f'ERROR: cannot read board roster {config}: {e}', file=sys.stderr)
+        sys.exit(1)
+
+
+def read_info(board: str):
+    try:
+        with open(lock_path(board)) as f:
+            return json.load(f)
+    except (OSError, ValueError):
+        return None
+
+
+def is_locked(board: str) -> bool:
+    """True if the recorded holder process is still alive.
+
+    Deliberately never touches the flock: even a momentary probe lock would
+    make a concurrent acquirer's LOCK_NB attempt fail spuriously. The flock
+    taken by acquirers themselves stays the only authority."""
+    info = read_info(board)
+    pid = info.get('pid') if isinstance(info, dict) else None
+    if not isinstance(pid, int) or pid <= 0:
+        return False
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True  # alive but owned by another user (e.g. the CI runner)
+    return True
+
+
+def cmd_hold(boards, reason):
+    os.makedirs(LOCK_DIR, exist_ok=True)
+    # No pre-check: the holder's own LOCK_NB flock is the only authority — a
+    # recorded pid may be stale or recycled (e.g. a live hil_test.py worker
+    # that already released this board's flock but not its record).
+    # The holder signals success through this pipe. A generic is_locked()
+    # poll would be fooled by a RIVAL invocation's flock — only the holder
+    # itself knows whether it won every board.
+    r_fd, w_fd = os.pipe()
+    pid = os.fork()
+    if pid > 0:
+        os.close(w_fd)
+        os.waitpid(pid, 0)  # reap intermediate child
+        ready, _, _ = select.select([r_fd], [], [], 10)
+        ok = bool(ready) and os.read(r_fd, 1) == b'1'
+        os.close(r_fd)
+        if ok:
+            print(f'held: {", ".join(boards)}')
+            return 0
+        for b in boards:
+            info = read_info(b)
+            if info:
+                print(f'ERROR: {b} locked: {info}', file=sys.stderr)
+        print('ERROR: holder failed to acquire locks', file=sys.stderr)
+        return 1
+    # intermediate child: detach, then spawn the actual holder
+    os.setsid()
+    if os.fork() > 0:
+        os._exit(0)
+    # holder (grandchild): acquire all flocks, signal the parent, sleep until killed
+    os.close(r_fd)
+    # Keep the success pipe clear of fds 0-2: invoked with stdio closed,
+    # os.pipe() can land there and the dup2 loop below would clobber it.
+    if w_fd <= 2:
+        w_fd = fcntl.fcntl(w_fd, fcntl.F_DUPFD, 3)
+    # Detach stdio: a `hold` whose output is captured must see EOF when the
+    # front-end exits — the immortal holder must not keep that pipe open.
+    devnull = os.open(os.devnull, os.O_RDWR)
+    for std_fd in (0, 1, 2):
+        os.dup2(devnull, std_fd)
+    if devnull > 2:
+        os.close(devnull)
+    try:
+        handles = []
+        for b in boards:
+            # O_RDWR without O_TRUNC: never truncate before the flock is
+            # held — a losing racer must not wipe the winner's holder info.
+            fd = os.open(lock_path(b), os.O_RDWR | os.O_CREAT, 0o666)
+            fh = os.fdopen(fd, 'r+')
+            fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            fh.truncate(0)
+            fh.seek(0)
+            json.dump({'pid': os.getpid(), 'reason': reason,
+                       'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh)
+            fh.flush()
+            handles.append(fh)
+    except OSError:
+        try:
+            os.write(w_fd, b'0')
+        except OSError:
+            pass
+        os._exit(1)  # lost a race; parent reports the failure
+    os.write(w_fd, b'1')
+    os.close(w_fd)
+
+    def _bow_out(*_):
+        # clear the records before dying so read_info/status stay truthful
+        # (the kernel drops the flocks themselves on exit either way)
+        for h in handles:
+            try:
+                h.truncate(0)
+            except OSError:
+                pass
+        os._exit(0)
+
+    signal.signal(signal.SIGTERM, _bow_out)
+    while True:
+        signal.pause()
+
+
+def cmd_release(boards):
+    rc = 0
+    victims = set()
+    for b in boards:
+        try:
+            fd = os.open(lock_path(b), os.O_RDWR)
+        except OSError:
+            continue  # no lock file (or another user's): nothing we can release
+        fh = os.fdopen(fd, 'r+')
+        try:
+            fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError:
+            # flock genuinely held — never SIGTERM on a mere pid record: the
+            # pid may be recycled, or a live worker that already moved on.
+            fh.close()
+            info = read_info(b) or {}
+            pid = info.get('pid')
+            if info.get('reason') == 'hil_test.py':
+                print(f'ERROR: {b} is mid-test by hil_test.py (pid {pid}) — not killing a '
+                      'CI run; wait for it to finish', file=sys.stderr)
+                rc = 1
+            elif isinstance(pid, int) and pid > 0:
+                victims.add(pid)
+            else:
+                print(f'ERROR: {b} is held but its record is unreadable', file=sys.stderr)
+                rc = 1
+            continue
+        # flock was free: only a stale record remained — clear it
+        try:
+            fh.truncate(0)
+        except OSError:
+            pass
+        fh.close()
+    for holder in sorted(victims):
+        try:
+            os.kill(holder, signal.SIGTERM)
+            print(f'released holder pid {holder}')
+        except ProcessLookupError:
+            pass
+        except PermissionError:
+            print(f'ERROR: holder pid {holder} belongs to another user — cannot signal it',
+                  file=sys.stderr)
+            rc = 1
+    time.sleep(0.3)
+    still = [b for b in boards if is_locked(b)]
+    if still:
+        print(f'ERROR: still locked: {", ".join(still)}', file=sys.stderr)
+        return 1
+    return rc
+
+
+def cmd_status():
+    if not os.path.isdir(LOCK_DIR):
+        print('no locks')
+        return 0
+    any_locked = False
+    for fn in sorted(os.listdir(LOCK_DIR)):
+        if not fn.endswith('.lock'):
+            continue
+        b = fn[:-5]
+        if is_locked(b):
+            any_locked = True
+            print(f'{b}: {read_info(b)}')
+    if not any_locked:
+        print('no locks')
+    return 0
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = ap.add_subparsers(dest='cmd', required=True)
+    p_hold = sub.add_parser('hold')
+    p_hold.add_argument('boards', nargs='*')
+    p_hold.add_argument('--all', action='store_true')
+    p_hold.add_argument('--config',
+                        default=os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                             'tinyusb.json'),
+                        help='board roster JSON (default: tinyusb.json beside this script)')
+    p_hold.add_argument('--reason', required=True)
+    p_rel = sub.add_parser('release')
+    p_rel.add_argument('boards', nargs='*')
+    p_rel.add_argument('--all', action='store_true')
+    sub.add_parser('status')
+    a = ap.parse_args()
+    if a.cmd == 'hold':
+        boards = boards_from_config(a.config) if a.all else a.boards
+        if not boards:
+            ap.error('no boards given (name boards or use --all)')
+        sys.exit(cmd_hold(boards, a.reason))
+    if a.cmd == 'release':
+        if a.all:
+            boards = ([fn[:-5] for fn in os.listdir(LOCK_DIR) if fn.endswith('.lock')]
+                      if os.path.isdir(LOCK_DIR) else [])
+        else:
+            boards = a.boards
+        if not boards:
+            ap.error('no boards given (name boards or use --all)')
+        sys.exit(cmd_release(boards))
+    sys.exit(cmd_status())
+
+
+if __name__ == '__main__':
+    main()

test/hil/hfp.json

@@ -23,9 +23,8 @@
                 "device": true, "host": false, "dual": false
             },
             "flasher": {
-                "name": "jlink",
-                "uid": "770935966",
-                "args": "-device STM32F746NG"
+                "name": "stlink",
+                "uid": "0670FF515448787067122222"
             }
         },
         {

test/hil/hil_test.py

@@ -66,6 +66,51 @@ import ctypes
 from pymtp import MTP
 import string
 
+# --- per-board dev-session locks (see test/hil/board_lock.py) ------------
+BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks'
+
+def acquire_board_lock(board_name):
+    """Take this board's flock for the duration of its flash+test.
+    Returns an open file handle (keep it referenced; closing releases it),
+    or None when HIL_NO_BOARD_LOCK=1 or the lock dir is unusable (fail-open:
+    locking must never break a test run by itself).
+    Raises RuntimeError only when another session holds the board."""
+    import fcntl
+    if os.environ.get('HIL_NO_BOARD_LOCK') == '1':
+        return None  # user-authorized bypass — see board_lock.py / hil skill
+    try:
+        os.makedirs(BOARD_LOCK_DIR, exist_ok=True)
+        fd = os.open(os.path.join(BOARD_LOCK_DIR, f'{board_name}.lock'),
+                     os.O_RDWR | os.O_CREAT, 0o666)
+        fh = os.fdopen(fd, 'r+')
+    except OSError as e:
+        # odd lock dir (perms, path collision): proceed unlocked, but say so —
+        # a silent fail-open is indistinguishable from the intentional bypass
+        print(f'warning: board lock unavailable for {board_name} ({e}); proceeding unlocked',
+              flush=True)
+        return None
+    try:
+        fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        try:
+            info = fh.read(500).strip()
+        except (OSError, UnicodeDecodeError):
+            info = ''
+        fh.close()
+        raise RuntimeError(f'board locked: {info or "unknown holder"}')
+    # announce ourselves so the other side's conflict message is truthful;
+    # best-effort — the flock itself is already held
+    try:
+        fh.truncate(0)
+        fh.seek(0)
+        json.dump({'pid': os.getpid(), 'reason': 'hil_test.py',
+                   'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh)
+        fh.flush()
+    except OSError:
+        pass
+    return fh
+
+
 # Enumeration wait budget. The first attempt gets ENUM_TIMEOUT; retry attempts get the
 # shorter ENUM_TIMEOUT_RETRY - the board was just re-flashed again, and a device that is
 # going to enumerate shows up within a few seconds, so a failing test costs ~3-5x a
@@ -1885,77 +1930,96 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]:
     name = board['name']
     flasher = board['flasher']
 
-    # default to all tests
-    test_list = []
+    try:
+        _lock_fh = acquire_board_lock(name)
+    except RuntimeError as e:
+        log_line(f'{name:25} {STATUS_FAILED}: {e}')
+        # visible report row so the ❌ matches the exit code; failed-tests stays
+        # empty so a re-run repeats the whole board (no bogus -bt test filter)
+        return name, 1, [], [(name, {'board-locked': 'fail'})]
+    try:
+        # default to all tests
+        test_list = []
 
-    if name in board_test:
-        test_list = board_test[name]
-    elif len(test_only) > 0:
-        # Explicit -t: filter against the board's capabilities so a device-only
-        # board doesn't try to run host/dual tests (the test functions need a
-        # `dev_attached` entry in the board config that won't exist).
-        board_tests = board.get('tests', {})
-        if 'only' in board_tests:
-            allowed = set(board_tests['only'])
-            test_list = [t for t in test_only if t in allowed]
-        else:
-            for t in test_only:
-                category = t.split('/', 1)[0]
-                if board_tests.get(category) is True:
-                    test_list.append(t)
-    else:
-        if 'tests' in board:
-            board_tests = board['tests']
-            if board_tests.get('device') is True:
-                test_list += list(device_tests)
-            if board_tests.get('dual') is True:
-                test_list += dual_tests
-            if board_tests.get('host') is True:
-                test_list += host_test
+        if name in board_test:
+            test_list = board_test[name]
+        elif len(test_only) > 0:
+            # Explicit -t: filter against the board's capabilities so a device-only
+            # board doesn't try to run host/dual tests (the test functions need a
+            # `dev_attached` entry in the board config that won't exist).
+            board_tests = board.get('tests', {})
             if 'only' in board_tests:
-                test_list = board_tests['only']
-            if 'skip' in board_tests:
-                for skip in board_tests['skip']:
-                    if skip in test_list:
-                        test_list.remove(skip)
-                        log_line(f'{name:25} {skip:30} ... Skip')
+                allowed = set(board_tests['only'])
+                test_list = [t for t in test_only if t in allowed]
+            else:
+                for t in test_only:
+                    category = t.split('/', 1)[0]
+                    if board_tests.get(category) is True:
+                        test_list.append(t)
+        else:
+            if 'tests' in board:
+                board_tests = board['tests']
+                if board_tests.get('device') is True:
+                    test_list += list(device_tests)
+                if board_tests.get('dual') is True:
+                    test_list += dual_tests
+                if board_tests.get('host') is True:
+                    test_list += host_test
+                if 'only' in board_tests:
+                    test_list = board_tests['only']
+                if 'skip' in board_tests:
+                    for skip in board_tests['skip']:
+                        if skip in test_list:
+                            test_list.remove(skip)
+                            log_line(f'{name:25} {skip:30} ... Skip')
 
-    err_count = 0
-    failed_tests = []
-    rows = []  # list of (row_label, {example: status}) — one row per build variant
-    variants = board.get('variant') or [{'name': name, 'flags': ''}]
+        err_count = 0
+        failed_tests = []
+        rows = []  # list of (row_label, {example: status}) — one row per build variant
+        variants = board.get('variant') or [{'name': name, 'flags': ''}]
 
-    prev_last = None  # last test of the previous variant: the variant boundary is an adjacency too
-    for v in variants:
-        vname = v['name']
-        # Shuffle each (board, variant)'s run order — de-synchronizes the worker pool so
-        # usbtest batteries and flash churn spread across the timeline instead of convoying,
-        # and surfaces order-dependent bugs. Seeded for replay (HIL_SHUFFLE_SEED, logged by
-        # main). Unique per-example PIDs make any two different examples re-enumerate; only
-        # the variant boundary can repeat the same example (same PID) — swap it away.
-        run_list = list(test_list)
-        if shuffle_seed is not None and len(run_list) > 1:
-            random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list)
-            if run_list[0] == prev_last:
-                run_list[0], run_list[-1] = run_list[-1], run_list[0]
-            log_line(f'{vname:40} test order: {", ".join(t.rsplit("/", 1)[-1] for t in run_list)}')
-        if run_list:
-            prev_last = run_list[-1]
-        cells = {}
-        for test in run_list:
-            ec, status, metric = test_example(board, vname, test)
-            err_count += ec
-            cells[test] = metric if metric else status
-            if ec > 0:
-                failed_tests.append(test)
-        rows.append((vname, cells))
+        prev_last = None  # last test of the previous variant: the variant boundary is an adjacency too
+        for v in variants:
+            vname = v['name']
+            # Shuffle each (board, variant)'s run order — de-synchronizes the worker pool so
+            # usbtest batteries and flash churn spread across the timeline instead of convoying,
+            # and surfaces order-dependent bugs. Seeded for replay (HIL_SHUFFLE_SEED, logged by
+            # main). Unique per-example PIDs make any two different examples re-enumerate; only
+            # the variant boundary can repeat the same example (same PID) — swap it away.
+            run_list = list(test_list)
+            if shuffle_seed is not None and len(run_list) > 1:
+                random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list)
+                if run_list[0] == prev_last:
+                    run_list[0], run_list[-1] = run_list[-1], run_list[0]
+                log_line(f'{vname:40} test order: {", ".join(t.rsplit("/", 1)[-1] for t in run_list)}')
+            if run_list:
+                prev_last = run_list[-1]
+            cells = {}
+            for test in run_list:
+                ec, status, metric = test_example(board, vname, test)
+                err_count += ec
+                cells[test] = metric if metric else status
+                if ec > 0:
+                    failed_tests.append(test)
+            rows.append((vname, cells))
 
-    # flash board_test last to disable board's usb (skipped when --skip-flash is set);
-    # this is teardown/park, not a test — not recorded in the report
-    if not skip_flash:
-        test_example(board, variants[0]['name'], 'device/board_test')
+        # flash board_test last to disable board's usb (skipped when --skip-flash is set);
+        # this is teardown/park, not a test — not recorded in the report
+        if not skip_flash:
+            test_example(board, variants[0]['name'], 'device/board_test')
 
-    return name, err_count, sorted(set(failed_tests)), rows
+        return name, err_count, sorted(set(failed_tests)), rows
+    finally:
+        if _lock_fh:
+            try:
+                # clear our pid record before dropping the flock: this worker
+                # process lives on (pool reuse), so a stale record would make
+                # board_lock.py's pid-liveness checks report a freed board as
+                # still locked for the rest of the run
+                _lock_fh.truncate(0)
+            except OSError:
+                pass
+            _lock_fh.close()
 
 
 REPORT_MD = 'hil_report.md'
@@ -2035,7 +2099,17 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str:
             pass  # corrupt/old sidecar: start fresh
 
     # merge this run: current cells override prior for boards/tests that ran
-    for _, _, _, rows in mret:
+    for name, _, _, rows in mret:
+        if rows and not any('board-locked' in cells for _, cells in rows):
+            # board ran for real this time: clear a stale lock-failure cell
+            # (its row is keyed by board name; test rows may be variant names)
+            stale = acc.get(name)
+            if stale is not None:
+                stale.pop('board-locked', None)
+                if not stale:
+                    # variant-keyed boards never repopulate the board-name row —
+                    # drop it or it renders as a blank ghost row
+                    del acc[name]
         for row_label, cells in rows:
             acc.setdefault(row_label, {}).update(cells)
 
@@ -2098,6 +2172,11 @@ def main() -> None:
     if len(boards) == 0:
         config_boards = [e for e in config['boards'] if e['name'] not in skip_boards]
     else:
+        unknown = [b for b in boards if b not in {e['name'] for e in config['boards']}]
+        if unknown:
+            # exiting 0 with 'No tests were run.' would read as a green HIL run
+            print(f'ERROR: board(s) not in {config_file.name}: {", ".join(unknown)}')
+            sys.exit(1)
         config_boards = [e for e in config['boards'] if e['name'] in boards]
 
     build_err = 0