path: root/artifacts/etf-recovery-candidate/ops-drill2.sh

#!/bin/sh
# 受控替身演练 v2（对应复审条目 1-4）。禁止生产操作。
# 退出语义：FAILED>0 → exit 1（不再恒 exit0）。
# 注入点：fake install/rsync/sed/systemctl(stop,start,restart)/curl(health)。
# 覆盖：A1/A2 precheck 硬拦；B 成功部署+手动 rollback 验收；H 并发锁；
#      C install 失败、D rsync 失败、E env(sed) 失败、F restart/health 失败
#      → 自动回滚且环境+发布物与部署前逐字节一致；
#      I 停服后发现新任务 → 恢复旧服务不替换；RID 不覆盖既有备份。
#      RA restore 中 stop 失败且无法确认服务已停 → 拒绝写任何文件并保持
#         env/binary/dist 逐字节不变；RB1/RB2 停后的 DB 读取错误或 DB_PATH
#         缺失 → 不深层 exit，恢复旧服务且无任何 swap。
set -eu
HERE=$(dirname "$0")
T="$(cd "$HERE" && pwd)/ops-drill2"
PASSED=0; FAILED=0
ok()   { PASSED=$((PASSED+1)); printf 'PASS: %s\n' "$1"; }
fail() { FAILED=$((FAILED+1)); printf 'FAIL: %s\n' "$1"; }
fx() { if [ "$2" = "$3" ]; then ok "$1"; else fail "$1 (got '$2' want '$3')"; fi; }
startswith() { case "$2" in "$1"*) true ;; *) false ;; esac; }

rm -rf "$T"; mkdir -p "$T/bin"

cat > "$T/bin/fake-systemctl" <<'SH'
#!/bin/sh
action=""; unit=""
for a in "$@"; do
  case "$a" in --user|--*) continue ;; esac
  if [ -z "$action" ]; then action=$a; else unit=$a; fi
done
case "$action $unit" in
  "stop fake-service")
    [ "$FAKE_STOP_FAIL" = "1" ] && exit 1
    printf 'stop\n' >> "$STATE_DIR/.audit.log"
    rm -f "$STATE_DIR/.active"
    rm -f "$STATE_DIR/.stopped"
    : > "$STATE_DIR/.stopped"
    [ "$FAKE_HEALTH_ON_STOP" = "1" ] && : > "$STATE_DIR/.healthfail"
    if [ -n "${FAKE_STOP_HOOK:-}" ]; then "$FAKE_STOP_HOOK"; fi
    exit 0 ;;
  "start fake-service")
    # deploy's own start may be injected (FAKE_START_FAIL) ONCE — a later
    # start attempt (rollback path) must succeed so rollback stays verifiable
    if [ "$FAKE_START_FAIL" = "1" ] && [ ! -e "$STATE_DIR/.startfail.seen" ]; then
      : > "$STATE_DIR/.startfail.seen"; exit 1
    fi
    printf 'start\n' >> "$STATE_DIR/.audit.log"
    if [ "${DEPLOY_PHASE:-}" = "restore" ]; then
      rm -f "$STATE_DIR/.healthfail"
      rm -rf "$STATE_DIR/.fakehealth.recovered"; : > "$STATE_DIR/.fakehealth.recovered"
    fi
    : > "$STATE_DIR/.active"; rm -f "$STATE_DIR/.stopped"; exit 0 ;;
  "restart fake-service")
    [ "$FAKE_RESTART_FAIL" = "1" ] && exit 1
    printf 'restart\n' >> "$STATE_DIR/.audit.log"
    rm -f "$STATE_DIR/.healthfail"
    rm -rf "$STATE_DIR/.fakehealth.recovered"
    : > "$STATE_DIR/.fakehealth.recovered"
    : > "$STATE_DIR/.active"; exit 0 ;;
  "is-active fake-service")
    [ -e "$STATE_DIR/.active" ] && exit 0 || { echo inactive; exit 3; } ;;
  *) exit 1 ;;
esac
SH
cat > "$T/bin/fake-docker" <<'SH'
#!/bin/sh
case "$*" in *"image inspect"*) exit 0 ;; *) exit 1 ;; esac
SH
cat > "$T/bin/fake-curl" <<'SH'
#!/bin/sh
if [ -e "$STATE_DIR/.healthfail" ]; then exit 7; fi
if [ "$FAKE_HEALTH" = "bad" ]; then
  if [ -e "$STATE_DIR/.fakehealth.recovered" ]; then
    printf '{"status":"ok","ai_configured":true,"version":"0.1.0","worker_available":true}'
    exit 0
  fi
  if [ ! -e "$STATE_DIR/.healthfail" ]; then : > "$STATE_DIR/.healthfail"; fi
  printf '{"status":"bad","worker_available":true}'
  exit 0
fi
printf '{"status":"ok","ai_configured":true,"version":"0.1.0","worker_available":true}'
exit 0
SH
cat > "$T/bin/fake-install" <<'SH'
#!/bin/sh
if [ "$INJECT" = "install" ]; then echo "install: injected failure" >&2; exit 1; fi
m=0755; src=$1; dst=$2
if [ "$1" = "-m" ]; then m=$2; src=$3; dst=$4; fi
cat "$src" > "$dst" || exit 1
chmod "$m" "$dst" && exit 0
exit 1
SH
cat > "$T/bin/fake-cp" <<'SH'
#!/bin/sh
if [ "$INJECT" = "backup-cp" ]; then echo "cp: injected failure" >&2; exit 1; fi
exec /bin/cp "$@"
SH
cat > "$T/bin/fake-mv" <<'SH'
#!/bin/sh
if [ "$INJECT" = "backup-manifest" ]; then echo "mv: injected failure" >&2; exit 1; fi
exec /bin/mv "$@"
SH
cat > "$T/bin/fake-rsync" <<'SH'
#!/bin/sh
# backup-stage failures are injected separately (INJECT=backup-rsync);
# a plain rsync failure must fire only once, in the swap phase
if [ "$INJECT" = "backup-rsync" ]; then echo "rsync: injected failure" >&2; exit 1; fi
if [ "$INJECT" = "rsync" ] && [ "${DEPLOY_PHASE:-}" = "swap" ] && [ ! -e "$STATE_DIR/.rsyncfail.seen" ]; then
  : > "$STATE_DIR/.rsyncfail.seen"
  echo "rsync: injected failure" >&2; exit 1
fi
exec /usr/bin/rsync "$@"
SH
cat > "$T/bin/fake-sed" <<'SH'
#!/bin/sh
if [ "$INJECT" = "env" ]; then echo "sed: injected failure" >&2; exit 1; fi
exec /usr/bin/sed "$@"
SH
cat > "$T/bin/newwork.sh" <<SH
#!/bin/sh
python3 - "$T/prod.sqlite3" <<'PY'
import sqlite3, sys
con = sqlite3.connect(sys.argv[1])
con.execute("INSERT INTO runs VALUES ('running')")
con.commit()
PY
SH
chmod +x "$T/bin/"*

set_env_fixture() {
  rm -rf "$T/state" "$T/release" "$T/repo" "$T/backups"
  mkdir -p "$T/state" "$T/release/frontend" "$T/repo/server/target/release" "$T/repo/frontend/dist" "$T/backups"
  printf 'binary-live-prev' > "$T/release/strategy-lab-server"
  chmod 755 "$T/release/strategy-lab-server"
  printf '<html>prev-dist</html>' > "$T/release/frontend/index.html"
  printf 'binary-the-current-one' > "$T/repo/server/target/release/strategy-lab-server"
  chmod 755 "$T/repo/server/target/release/strategy-lab-server"
  printf '<html>candidate</html>' > "$T/repo/frontend/dist/index.html"
  python3 - "$T" <<'PY'
import sqlite3, sys
con = sqlite3.connect(sys.argv[1] + "/prod.sqlite3")
con.execute("DROP TABLE IF EXISTS datasets")
con.execute("DROP TABLE IF EXISTS runs")
con.execute("CREATE TABLE datasets (status TEXT)")
con.execute("CREATE TABLE runs (status TEXT)")
con.commit()
PY
  printf 'BIND=127.0.0.1:18789\nDB_PATH=%s/prod.sqlite3\nDATA_DIR=%s/data\nWORKER_IMAGE=strategy-lab-worker:local\n' "$T" "$T" > "$T/state/runtime.env"
  : > "$T/state/.active"
}

common_env() {
  STATE_DIR="$T/state" RELEASE_DIR="$T/release" BACKUP_ROOT="$T/backups" \
  REPO_ROOT="$T" SERVER_BIN_SRC="$T/repo/server/target/release/strategy-lab-server" \
  DIST_SRC="$T/repo/frontend/dist" SERVICE=fake-service \
  SYSTEMCTL_CMD="$T/bin/fake-systemctl" DOCKER_CMD="$T/bin/fake-docker" \
  CURL_CMD="$T/bin/fake-curl" RSYNC_CMD="$T/bin/fake-rsync" \
  INSTALL_CMD="$T/bin/fake-install" SED_CMD="$T/bin/fake-sed" \
  CP_CMD="$T/bin/fake-cp" MV_CMD="$T/bin/fake-mv" \
  export STATE_DIR RELEASE_DIR BACKUP_ROOT REPO_ROOT SERVER_BIN_SRC DIST_SRC \
         SERVICE SYSTEMCTL_CMD DOCKER_CMD CURL_CMD RSYNC_CMD INSTALL_CMD SED_CMD \
         CP_CMD MV_CMD
}

deploy_out() {
  common_env
  IMAGE_ID=$1; export IMAGE_ID
  ( cd "$HERE/ops" && ./deploy.sh ) 2>&1
}

# --- A1 datasets ---
set_env_fixture
python3 - "$T/prod.sqlite3" <<'PY'
import sqlite3, sys
con = sqlite3.connect(sys.argv[1]); con.execute("INSERT INTO datasets VALUES ('running')"); con.commit()
PY
out=$(deploy_out sha256:img0 || true)
printf '%s' "$out" | grep -q "in-flight datasets" && ok "A1 precheck blocks in-flight datasets" || fail "A1: $out"
fx "A1 env untouched" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
fx "A1 release untouched" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"

# --- A2 runs ---
set_env_fixture
python3 - "$T/prod.sqlite3" <<'PY'
import sqlite3, sys
con = sqlite3.connect(sys.argv[1]); con.execute("INSERT INTO runs VALUES ('running')"); con.commit()
PY
out=$(deploy_out sha256:img0 || true)
printf '%s' "$out" | grep -q "in-flight runs" && ok "A2 precheck blocks in-flight runs" || fail "A2: $out"

# --- I: new work discovered after stop ---
set_env_fixture
fx "I00 sanity" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
out=$(FAKE_STOP_HOOK="$T/bin/newwork.sh" deploy_out sha256:img-a || true)
printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "I race branch restores old service" || { fail "I: $out"; printf '%s' "$out" | tail -5; }
fx "I env untouched" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
fx "I release untouched" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
printf '%s' "$out" | grep -q "MANUAL CHECK" && fail "I old service restart must succeed" || ok "I old service restarted via fake systemctl start"

# --- B: happy deploy ---
set_env_fixture
out=$(deploy_out sha256:img-1) || { fail "B deploy died: $out"; }
printf '%s' "$out" | grep -q "phase 6/6 deploy" && ok "B deploy completed" || fail "B: $(printf '%s' "$out" | tail -4)"
n=$(ls "$T/backups" 2>/dev/null | wc -l)
fx "B exactly one RID" "$n" "1"
R1=$(ls "$T/backups")
fx "B env pins immutable ID" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=sha256:img-1"
fx "B binary swapped" "$(cat "$T/release/strategy-lab-server")" "binary-the-current-one"
fx "B dist swapped" "$(cat "$T/release/frontend/index.html")" "<html>candidate</html>"
[ -x "$T/release/strategy-lab-server" ] && ok "B binary executable" || fail "B not executable"
[ -f "$T/backups/$R1/release-manifest.txt" ] && ok "B manifest exists" || fail "B manifest missing"
[ -f "$T/backups/$R1/complete" ] && ok "B backup has complete marker" || fail "B complete marker missing"

# --- RID uniqueness: deploying the SAME image ID again is refused (no overwrite) ---
out=$(deploy_out sha256:img-1 || true)
printf '%s' "$out" | grep -q "already pins" && ok "RID: re-pin of same ID refused (no overwrite)" || fail "RID refusal: $out"
n=$(ls "$T/backups" 2>/dev/null | wc -l)
fx "RID: backup count still exactly one" "$n" "1"

# --- G: manual rollback.sh $RID verifies everything ---
out=$( ( common_env; cd "$HERE/ops" && ./rollback.sh "$R1" ) 2>&1 ) || { fail "G manual rollback died: $out"; }
printf '%s' "$out" | grep -q "restored .*env verified" && ok "G manual rollback verified" || fail "G: $out"
fx "G env = manifest previous image (pre-deploy state)" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
fx "G release restored to backup (pre-deploy) state" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev"
fx "G dist restored to pre-deploy backup" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
audit=$(cat "$T/state/.audit.log")
printf '%s\n' "$audit" | head -1 | grep -qx stop && ok "G rollback stops service FIRST (no writes under a running binary)" || fail "G audit first line not stop: $(printf '%s' "$audit")"
printf '%s\n' "$audit" | tail -1 | grep -qx start && ok "G rollback restarts service LAST" || fail "G audit last line not start: $(printf '%s' "$audit")"
[ -e "$T/state/.active" ] && ok "G manual rollback leaves service ACTIVE" || fail "G: service not active after manual rollback"

# --- H: concurrent deploy refused by deploy.lock ---
rm -f "$T/state/deploy.lock"
set_env_fixture
( common_env >/dev/null 2>&1
  exec 9>"$T/state/deploy.lock"; flock 9
  sleep 1.5 ) & HOLD_PID=$!
sleep 0.4
out2=$( ( common_env >/dev/null; IMAGE_ID=sha256:img-h; export IMAGE_ID; cd "$HERE/ops" && ./deploy.sh ) 2>&1 || true)
wait $HOLD_PID || true
printf '%s' "$out2" | grep -q "deploy.lock" && ok "H concurrent deploy refused" || fail "H: $out2"

# --- C: install failure inside swap ---
set_env_fixture
out=$(INJECT=install deploy_out sha256:img-c || true) || true
printf '%s' "$out" | grep -q "install .* failed" && ok "C install failure detected" || fail "C: $out"
printf '%s' "$out" | grep -q "restored artifacts verified (env + sha256)" && ok "C auto-rollback verified" || fail "C rollback: $(printf '%s' "$out" | tail -4)"
fx "C binary restored" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev"
fx "C dist restored" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
fx "C env restored" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
[ -f "$T/backups/$(ls "$T/backups")/release-manifest.txt" ] && ok "C manifest" || fail "C manifest"

# --- D: rsync failure, ONE-SHOT in the swap phase -> auto rollback healthy ---
set_env_fixture
out=$(INJECT=rsync deploy_out sha256:img-d) || rc_d=$?
[ "$out" = "0" ] || [ "${rc_d:-nonzero}" != "0" ] && ok "D deploy exits non-zero" || fail "D exit expected nonzero"
printf '%s' "$out" | grep -q "rsync .* failed" && ok "D rsync failure detected" || fail "D: $out"
printf '%s' "$out" | grep -q "restored artifacts verified (env + sha256)" && ok "D auto-rollback verified" || fail "D rollback: $(printf '%s' "$out" | tail -4)"
fx "D dist restored" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
fx "D binary restored" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev"
fx "D env restored" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
[ -e "$T/state/.active" ] && ok "D old service ACTIVE after rollback" || fail "D: old service not active"
[ -e "$T/backups/$(ls "$T/backups")/complete" ] && ok "D backup was complete before swap" || fail "D backup lacked complete marker"

# --- BD1/BD2/BD3: backup-stage failure (cp / rsync / manifest) ---
# Service must STAY active (backup happens before stop), artifacts must be
# byte-identical, exit non-zero, and NO complete backup may exist.
for inj in backup-cp backup-rsync backup-manifest; do
  set_env_fixture
  env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
  bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')
  dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')
  out=$(INJECT="$inj" deploy_out sha256:img-bd || true)
  [ -e "$T/state/.active" ] && ok "$inj: old service ACTIVE (no downtime)" || fail "$inj: old service down"
  fx "$inj: env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before"
  fx "$inj: binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before"
  fx "$inj: dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before"
  [ -e "$T/backups/$(ls "$T/backups")/complete" ] && fail "$inj: INCOMPLETE backup must not have complete marker" || ok "$inj: no complete-marker backup"
done

# --- E: env(sed) failure → env byte-identical ---
set_env_fixture
env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
out=$(INJECT=env deploy_out sha256:img-e || true)
env_after=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
fx "E env byte-identical" "$env_before" "$env_after"
printf '%s' "$out" | grep -q "restored artifacts verified" && ok "E auto-rollback verified (env unchanged byte-identical)" || fail "E: $(printf '%s' "$out" | tail -4)"
fx "E env back to old image" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"

# --- F2: restart failure ---
set_env_fixture
out=$(FAKE_START_FAIL=1 deploy_out sha256:img-f || true)
printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F start-service failure -> rollback" || fail "F1: $(printf '%s' "$out" | tail -4)"
fx "F binary identical" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev"
fx "F dist identical" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
fx "F env identical" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"

# --- F3: health failure (curl level + bad status JSON) ---
set_env_fixture
set_env_fixture 2>/dev/null; out=$(FAKE_HEALTH_ON_STOP=1 deploy_out sha256:img-g || true)
printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F health(unreach) -> rollback" || fail "F3: $(printf '%s' "$out" | tail -4)"
set_env_fixture
out=$(FAKE_HEALTH=bad deploy_out sha256:img-h2 || true)
printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F health(bad-json) -> rollback" || fail "F4: $(printf '%s' "$out" | tail -4)"
fx "F4 env identical after health failure" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"

# --- RA: restore must REFUSE to write files when stop cannot be confirmed ---
set_env_fixture
out=$(deploy_out sha256:img-ra) || { fail "RA setup deploy died: $out"; }
RID_RA=$(ls "$T/backups")
env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')
dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')
rc_ra=0
out=$( { common_env
  FAKE_STOP_FAIL=1; export FAKE_STOP_FAIL
  IMAGE_ID=sha256:img-ra2; export IMAGE_ID
  cd "$HERE/ops" && ./deploy.sh restore "$RID_RA"; } 2>&1 ) || rc_ra=$?
[ "$rc_ra" -ne 0 ] && ok "RA restore exits non-zero when stop fails" || fail "RA restore exited 0: $out"
printf '%s' "$out" | grep -q "refusing" && ok "RA explicit refusal message" || fail "RA msg: $out"
fx "RA env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before"
fx "RA binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before"
fx "RA dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before"
[ -e "$T/state/.active" ] && ok "RA service still ACTIVE (recoverable files preserved)" || fail "RA service not active"

# --- RB1: DB read error after stop -> old service restored, NO swap ---
set_env_fixture
env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')
dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')
cat > "$T/bin/breakdb.sh" <<SH
#!/bin/sh
rm -f "$T/prod.sqlite3"
SH
chmod +x "$T/bin/breakdb.sh"
rc_rb=0
out=$(FAKE_STOP_HOOK="$T/bin/breakdb.sh" deploy_out sha256:img-rb) 2>&1 || rc_rb=$?
[ "$rc_rb" -ne 0 ] && ok "RB1 deploy exits non-zero on DB read error" || fail "RB1 exited 0: $out"
printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "RB1 old service restored after DB read error" || { fail "RB1: $out"; printf '%s' "$out" | tail -5; }
[ -e "$T/state/.active" ] && ok "RB1 old service ACTIVE" || fail "RB1 service down"
fx "RB1 env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before"
fx "RB1 binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before"
fx "RB1 dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before"

# --- RB2: DB_PATH key missing after stop -> old service restored, NO swap ---
set_env_fixture
env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')
dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')
cat > "$T/bin/dropdbpath.sh" <<SH
#!/bin/sh
sed '/^DB_PATH=/d' "$T/state/runtime.env" > "$T/state/runtime.env.tmp"
mv "$T/state/runtime.env.tmp" "$T/state/runtime.env"
printf 'DB_PATH_REMOVED=1\n' >> "$T/state/runtime.env"
SH
chmod +x "$T/bin/dropdbpath.sh"
rc_rc=0
out=$( { FAKE_STOP_HOOK="$T/bin/dropdbpath.sh" deploy_out sha256:img-rc; } 2>&1 ) || rc_rc=$?
[ "$rc_rc" -ne 0 ] && ok "RB2 deploy exits non-zero on missing DB_PATH" || fail "RB2 exited 0: $out"
printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "RB2 old service restored after DB_PATH missing" || { fail "RB2: $out"; printf '%s' "$out" | tail -5; }
[ -e "$T/state/.active" ] && ok "RB2 old service ACTIVE" || fail "RB2 service down"
# the stop hook removed DB_PATH (that is the injected fault); assert the OTHER
# keys survived byte-for-byte — no swap touched runtime.env beyond that fault
fx "RB2 env rest unchanged" \
  "$(grep -E '^(BIND|DATA_DIR|WORKER_IMAGE)=' "$T/state/runtime.env")" \
  "BIND=127.0.0.1:18789
DATA_DIR=$T/data
WORKER_IMAGE=strategy-lab-worker:local"
fx "RB2 binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before"
fx "RB2 dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before"

echo "=== PASSED=$PASSED FAILED=$FAILED ==="
[ $FAILED -eq 0 ] || exit 1
exit 0