summaryrefslogtreecommitdiff
path: root/artifacts/etf-recovery-candidate/ops-drill2.sh
diff options
context:
space:
mode:
authorSomhairle H. Marisol <[email protected]>2026-09-18 08:27:24 +0800
committerSomhairle H. Marisol <[email protected]>2026-09-18 08:27:24 +0800
commitbf6681eb29ac8b0c80ca17b2b5869f7de3da1198 (patch)
treebd50c535fa6afa5cbb33db3cd60fa76690e05bf1 /artifacts/etf-recovery-candidate/ops-drill2.sh
parentb4f1136457a08c6ad6ef52d9dc1f83fef050d87c (diff)
downloadstrategy-lab-bf6681eb29ac8b0c80ca17b2b5869f7de3da1198.tar.gz
chore(ops): 生产部署/回滚/故障演练脚本入库
[变更性质] 维护性变更:把已通过 84 场景演练的运维脚本纳入版本控制。artifacts/ 目录按仓库政策 ignore(本地证据),本提交按 Leader 指令以 -f 显式 加入,未包含任何密钥(脚本扫无 secret/token/password)。 [维护内容] 此前 ops/deploy.sh、rollback.sh 与 ops-drill2.sh 仅存在于本地 artifacts/(被 .gitignore 排除),生产依赖它们但仓库内无源码可审计。 [实现方案] deploy.sh:deploy.lock 防并发、阶段化备份(env/二进制/前端/DB, manifest 原子化写入 0700 目录)、显式 rc helper、零停机 swap、失败 路径自动 restore、健康与 sha256 校验;restore 兼做手动回滚。 rollback.sh:stop -> 替换 -> start -> 锁 + health + hash 强校验。 ops-drill2.sh:84 场景故障注入演练(备份/swap/restore 阶段注入矩阵、 RA/RB1/RB2 恢复分支),PASSED=84 FAILED=0 exit 0 为执行合约。 用法见 docs/etf-recovery-release-handoff.md 第 3 节。 [影响范围] 脚本本身未改(与演练定稿版本一致),仅入库;回滚目标记录在 manifest(previous_worker_image 等)。
Diffstat (limited to 'artifacts/etf-recovery-candidate/ops-drill2.sh')
-rwxr-xr-xartifacts/etf-recovery-candidate/ops-drill2.sh372
1 files changed, 372 insertions, 0 deletions
diff --git a/artifacts/etf-recovery-candidate/ops-drill2.sh b/artifacts/etf-recovery-candidate/ops-drill2.sh
new file mode 100755
index 0000000..e4a4caf
--- /dev/null
+++ b/artifacts/etf-recovery-candidate/ops-drill2.sh
@@ -0,0 +1,372 @@
+#!/bin/sh
+# 受控替身演练 v2(对应复审条目 1-4)。禁止生产操作。
+# 退出语义:FAILED>0 → exit 1(不再恒 exit0)。
+# 注入点:fake install/rsync/sed/systemctl(stop,start,restart)/curl(health)。
+# 覆盖:A1/A2 precheck 硬拦;B 成功部署+手动 rollback 验收;H 并发锁;
+# C install 失败、D rsync 失败、E env(sed) 失败、F restart/health 失败
+# → 自动回滚且环境+发布物与部署前逐字节一致;
+# I 停服后发现新任务 → 恢复旧服务不替换;RID 不覆盖既有备份。
+# RA restore 中 stop 失败且无法确认服务已停 → 拒绝写任何文件并保持
+# env/binary/dist 逐字节不变;RB1/RB2 停后的 DB 读取错误或 DB_PATH
+# 缺失 → 不深层 exit,恢复旧服务且无任何 swap。
+set -eu
+HERE=$(dirname "$0")
+T="$(cd "$HERE" && pwd)/ops-drill2"
+PASSED=0; FAILED=0
+ok() { PASSED=$((PASSED+1)); printf 'PASS: %s\n' "$1"; }
+fail() { FAILED=$((FAILED+1)); printf 'FAIL: %s\n' "$1"; }
+fx() { if [ "$2" = "$3" ]; then ok "$1"; else fail "$1 (got '$2' want '$3')"; fi; }
+startswith() { case "$2" in "$1"*) true ;; *) false ;; esac; }
+
+rm -rf "$T"; mkdir -p "$T/bin"
+
+cat > "$T/bin/fake-systemctl" <<'SH'
+#!/bin/sh
+action=""; unit=""
+for a in "$@"; do
+ case "$a" in --user|--*) continue ;; esac
+ if [ -z "$action" ]; then action=$a; else unit=$a; fi
+done
+case "$action $unit" in
+ "stop fake-service")
+ [ "$FAKE_STOP_FAIL" = "1" ] && exit 1
+ printf 'stop\n' >> "$STATE_DIR/.audit.log"
+ rm -f "$STATE_DIR/.active"
+ rm -f "$STATE_DIR/.stopped"
+ : > "$STATE_DIR/.stopped"
+ [ "$FAKE_HEALTH_ON_STOP" = "1" ] && : > "$STATE_DIR/.healthfail"
+ if [ -n "${FAKE_STOP_HOOK:-}" ]; then "$FAKE_STOP_HOOK"; fi
+ exit 0 ;;
+ "start fake-service")
+ # deploy's own start may be injected (FAKE_START_FAIL) ONCE — a later
+ # start attempt (rollback path) must succeed so rollback stays verifiable
+ if [ "$FAKE_START_FAIL" = "1" ] && [ ! -e "$STATE_DIR/.startfail.seen" ]; then
+ : > "$STATE_DIR/.startfail.seen"; exit 1
+ fi
+ printf 'start\n' >> "$STATE_DIR/.audit.log"
+ if [ "${DEPLOY_PHASE:-}" = "restore" ]; then
+ rm -f "$STATE_DIR/.healthfail"
+ rm -rf "$STATE_DIR/.fakehealth.recovered"; : > "$STATE_DIR/.fakehealth.recovered"
+ fi
+ : > "$STATE_DIR/.active"; rm -f "$STATE_DIR/.stopped"; exit 0 ;;
+ "restart fake-service")
+ [ "$FAKE_RESTART_FAIL" = "1" ] && exit 1
+ printf 'restart\n' >> "$STATE_DIR/.audit.log"
+ rm -f "$STATE_DIR/.healthfail"
+ rm -rf "$STATE_DIR/.fakehealth.recovered"
+ : > "$STATE_DIR/.fakehealth.recovered"
+ : > "$STATE_DIR/.active"; exit 0 ;;
+ "is-active fake-service")
+ [ -e "$STATE_DIR/.active" ] && exit 0 || { echo inactive; exit 3; } ;;
+ *) exit 1 ;;
+esac
+SH
+cat > "$T/bin/fake-docker" <<'SH'
+#!/bin/sh
+case "$*" in *"image inspect"*) exit 0 ;; *) exit 1 ;; esac
+SH
+cat > "$T/bin/fake-curl" <<'SH'
+#!/bin/sh
+if [ -e "$STATE_DIR/.healthfail" ]; then exit 7; fi
+if [ "$FAKE_HEALTH" = "bad" ]; then
+ if [ -e "$STATE_DIR/.fakehealth.recovered" ]; then
+ printf '{"status":"ok","ai_configured":true,"version":"0.1.0","worker_available":true}'
+ exit 0
+ fi
+ if [ ! -e "$STATE_DIR/.healthfail" ]; then : > "$STATE_DIR/.healthfail"; fi
+ printf '{"status":"bad","worker_available":true}'
+ exit 0
+fi
+printf '{"status":"ok","ai_configured":true,"version":"0.1.0","worker_available":true}'
+exit 0
+SH
+cat > "$T/bin/fake-install" <<'SH'
+#!/bin/sh
+if [ "$INJECT" = "install" ]; then echo "install: injected failure" >&2; exit 1; fi
+m=0755; src=$1; dst=$2
+if [ "$1" = "-m" ]; then m=$2; src=$3; dst=$4; fi
+cat "$src" > "$dst" || exit 1
+chmod "$m" "$dst" && exit 0
+exit 1
+SH
+cat > "$T/bin/fake-cp" <<'SH'
+#!/bin/sh
+if [ "$INJECT" = "backup-cp" ]; then echo "cp: injected failure" >&2; exit 1; fi
+exec /bin/cp "$@"
+SH
+cat > "$T/bin/fake-mv" <<'SH'
+#!/bin/sh
+if [ "$INJECT" = "backup-manifest" ]; then echo "mv: injected failure" >&2; exit 1; fi
+exec /bin/mv "$@"
+SH
+cat > "$T/bin/fake-rsync" <<'SH'
+#!/bin/sh
+# backup-stage failures are injected separately (INJECT=backup-rsync);
+# a plain rsync failure must fire only once, in the swap phase
+if [ "$INJECT" = "backup-rsync" ]; then echo "rsync: injected failure" >&2; exit 1; fi
+if [ "$INJECT" = "rsync" ] && [ "${DEPLOY_PHASE:-}" = "swap" ] && [ ! -e "$STATE_DIR/.rsyncfail.seen" ]; then
+ : > "$STATE_DIR/.rsyncfail.seen"
+ echo "rsync: injected failure" >&2; exit 1
+fi
+exec /usr/bin/rsync "$@"
+SH
+cat > "$T/bin/fake-sed" <<'SH'
+#!/bin/sh
+if [ "$INJECT" = "env" ]; then echo "sed: injected failure" >&2; exit 1; fi
+exec /usr/bin/sed "$@"
+SH
+cat > "$T/bin/newwork.sh" <<SH
+#!/bin/sh
+python3 - "$T/prod.sqlite3" <<'PY'
+import sqlite3, sys
+con = sqlite3.connect(sys.argv[1])
+con.execute("INSERT INTO runs VALUES ('running')")
+con.commit()
+PY
+SH
+chmod +x "$T/bin/"*
+
+set_env_fixture() {
+ rm -rf "$T/state" "$T/release" "$T/repo" "$T/backups"
+ mkdir -p "$T/state" "$T/release/frontend" "$T/repo/server/target/release" "$T/repo/frontend/dist" "$T/backups"
+ printf 'binary-live-prev' > "$T/release/strategy-lab-server"
+ chmod 755 "$T/release/strategy-lab-server"
+ printf '<html>prev-dist</html>' > "$T/release/frontend/index.html"
+ printf 'binary-the-current-one' > "$T/repo/server/target/release/strategy-lab-server"
+ chmod 755 "$T/repo/server/target/release/strategy-lab-server"
+ printf '<html>candidate</html>' > "$T/repo/frontend/dist/index.html"
+ python3 - "$T" <<'PY'
+import sqlite3, sys
+con = sqlite3.connect(sys.argv[1] + "/prod.sqlite3")
+con.execute("DROP TABLE IF EXISTS datasets")
+con.execute("DROP TABLE IF EXISTS runs")
+con.execute("CREATE TABLE datasets (status TEXT)")
+con.execute("CREATE TABLE runs (status TEXT)")
+con.commit()
+PY
+ printf 'BIND=127.0.0.1:18789\nDB_PATH=%s/prod.sqlite3\nDATA_DIR=%s/data\nWORKER_IMAGE=strategy-lab-worker:local\n' "$T" "$T" > "$T/state/runtime.env"
+ : > "$T/state/.active"
+}
+
+common_env() {
+ STATE_DIR="$T/state" RELEASE_DIR="$T/release" BACKUP_ROOT="$T/backups" \
+ REPO_ROOT="$T" SERVER_BIN_SRC="$T/repo/server/target/release/strategy-lab-server" \
+ DIST_SRC="$T/repo/frontend/dist" SERVICE=fake-service \
+ SYSTEMCTL_CMD="$T/bin/fake-systemctl" DOCKER_CMD="$T/bin/fake-docker" \
+ CURL_CMD="$T/bin/fake-curl" RSYNC_CMD="$T/bin/fake-rsync" \
+ INSTALL_CMD="$T/bin/fake-install" SED_CMD="$T/bin/fake-sed" \
+ CP_CMD="$T/bin/fake-cp" MV_CMD="$T/bin/fake-mv" \
+ export STATE_DIR RELEASE_DIR BACKUP_ROOT REPO_ROOT SERVER_BIN_SRC DIST_SRC \
+ SERVICE SYSTEMCTL_CMD DOCKER_CMD CURL_CMD RSYNC_CMD INSTALL_CMD SED_CMD \
+ CP_CMD MV_CMD
+}
+
+deploy_out() {
+ common_env
+ IMAGE_ID=$1; export IMAGE_ID
+ ( cd "$HERE/ops" && ./deploy.sh ) 2>&1
+}
+
+# --- A1 datasets ---
+set_env_fixture
+python3 - "$T/prod.sqlite3" <<'PY'
+import sqlite3, sys
+con = sqlite3.connect(sys.argv[1]); con.execute("INSERT INTO datasets VALUES ('running')"); con.commit()
+PY
+out=$(deploy_out sha256:img0 || true)
+printf '%s' "$out" | grep -q "in-flight datasets" && ok "A1 precheck blocks in-flight datasets" || fail "A1: $out"
+fx "A1 env untouched" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
+fx "A1 release untouched" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
+
+# --- A2 runs ---
+set_env_fixture
+python3 - "$T/prod.sqlite3" <<'PY'
+import sqlite3, sys
+con = sqlite3.connect(sys.argv[1]); con.execute("INSERT INTO runs VALUES ('running')"); con.commit()
+PY
+out=$(deploy_out sha256:img0 || true)
+printf '%s' "$out" | grep -q "in-flight runs" && ok "A2 precheck blocks in-flight runs" || fail "A2: $out"
+
+# --- I: new work discovered after stop ---
+set_env_fixture
+fx "I00 sanity" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
+out=$(FAKE_STOP_HOOK="$T/bin/newwork.sh" deploy_out sha256:img-a || true)
+printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "I race branch restores old service" || { fail "I: $out"; printf '%s' "$out" | tail -5; }
+fx "I env untouched" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
+fx "I release untouched" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
+printf '%s' "$out" | grep -q "MANUAL CHECK" && fail "I old service restart must succeed" || ok "I old service restarted via fake systemctl start"
+
+# --- B: happy deploy ---
+set_env_fixture
+out=$(deploy_out sha256:img-1) || { fail "B deploy died: $out"; }
+printf '%s' "$out" | grep -q "phase 6/6 deploy" && ok "B deploy completed" || fail "B: $(printf '%s' "$out" | tail -4)"
+n=$(ls "$T/backups" 2>/dev/null | wc -l)
+fx "B exactly one RID" "$n" "1"
+R1=$(ls "$T/backups")
+fx "B env pins immutable ID" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=sha256:img-1"
+fx "B binary swapped" "$(cat "$T/release/strategy-lab-server")" "binary-the-current-one"
+fx "B dist swapped" "$(cat "$T/release/frontend/index.html")" "<html>candidate</html>"
+[ -x "$T/release/strategy-lab-server" ] && ok "B binary executable" || fail "B not executable"
+[ -f "$T/backups/$R1/release-manifest.txt" ] && ok "B manifest exists" || fail "B manifest missing"
+[ -f "$T/backups/$R1/complete" ] && ok "B backup has complete marker" || fail "B complete marker missing"
+
+# --- RID uniqueness: deploying the SAME image ID again is refused (no overwrite) ---
+out=$(deploy_out sha256:img-1 || true)
+printf '%s' "$out" | grep -q "already pins" && ok "RID: re-pin of same ID refused (no overwrite)" || fail "RID refusal: $out"
+n=$(ls "$T/backups" 2>/dev/null | wc -l)
+fx "RID: backup count still exactly one" "$n" "1"
+
+# --- G: manual rollback.sh $RID verifies everything ---
+out=$( ( common_env; cd "$HERE/ops" && ./rollback.sh "$R1" ) 2>&1 ) || { fail "G manual rollback died: $out"; }
+printf '%s' "$out" | grep -q "restored .*env verified" && ok "G manual rollback verified" || fail "G: $out"
+fx "G env = manifest previous image (pre-deploy state)" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
+fx "G release restored to backup (pre-deploy) state" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev"
+fx "G dist restored to pre-deploy backup" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
+audit=$(cat "$T/state/.audit.log")
+printf '%s\n' "$audit" | head -1 | grep -qx stop && ok "G rollback stops service FIRST (no writes under a running binary)" || fail "G audit first line not stop: $(printf '%s' "$audit")"
+printf '%s\n' "$audit" | tail -1 | grep -qx start && ok "G rollback restarts service LAST" || fail "G audit last line not start: $(printf '%s' "$audit")"
+[ -e "$T/state/.active" ] && ok "G manual rollback leaves service ACTIVE" || fail "G: service not active after manual rollback"
+
+# --- H: concurrent deploy refused by deploy.lock ---
+rm -f "$T/state/deploy.lock"
+set_env_fixture
+( common_env >/dev/null 2>&1
+ exec 9>"$T/state/deploy.lock"; flock 9
+ sleep 1.5 ) & HOLD_PID=$!
+sleep 0.4
+out2=$( ( common_env >/dev/null; IMAGE_ID=sha256:img-h; export IMAGE_ID; cd "$HERE/ops" && ./deploy.sh ) 2>&1 || true)
+wait $HOLD_PID || true
+printf '%s' "$out2" | grep -q "deploy.lock" && ok "H concurrent deploy refused" || fail "H: $out2"
+
+# --- C: install failure inside swap ---
+set_env_fixture
+out=$(INJECT=install deploy_out sha256:img-c || true) || true
+printf '%s' "$out" | grep -q "install .* failed" && ok "C install failure detected" || fail "C: $out"
+printf '%s' "$out" | grep -q "restored artifacts verified (env + sha256)" && ok "C auto-rollback verified" || fail "C rollback: $(printf '%s' "$out" | tail -4)"
+fx "C binary restored" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev"
+fx "C dist restored" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
+fx "C env restored" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
+[ -f "$T/backups/$(ls "$T/backups")/release-manifest.txt" ] && ok "C manifest" || fail "C manifest"
+
+# --- D: rsync failure, ONE-SHOT in the swap phase -> auto rollback healthy ---
+set_env_fixture
+out=$(INJECT=rsync deploy_out sha256:img-d) || rc_d=$?
+[ "$out" = "0" ] || [ "${rc_d:-nonzero}" != "0" ] && ok "D deploy exits non-zero" || fail "D exit expected nonzero"
+printf '%s' "$out" | grep -q "rsync .* failed" && ok "D rsync failure detected" || fail "D: $out"
+printf '%s' "$out" | grep -q "restored artifacts verified (env + sha256)" && ok "D auto-rollback verified" || fail "D rollback: $(printf '%s' "$out" | tail -4)"
+fx "D dist restored" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
+fx "D binary restored" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev"
+fx "D env restored" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
+[ -e "$T/state/.active" ] && ok "D old service ACTIVE after rollback" || fail "D: old service not active"
+[ -e "$T/backups/$(ls "$T/backups")/complete" ] && ok "D backup was complete before swap" || fail "D backup lacked complete marker"
+
+# --- BD1/BD2/BD3: backup-stage failure (cp / rsync / manifest) ---
+# Service must STAY active (backup happens before stop), artifacts must be
+# byte-identical, exit non-zero, and NO complete backup may exist.
+for inj in backup-cp backup-rsync backup-manifest; do
+ set_env_fixture
+ env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
+ bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')
+ dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')
+ out=$(INJECT="$inj" deploy_out sha256:img-bd || true)
+ [ -e "$T/state/.active" ] && ok "$inj: old service ACTIVE (no downtime)" || fail "$inj: old service down"
+ fx "$inj: env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before"
+ fx "$inj: binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before"
+ fx "$inj: dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before"
+ [ -e "$T/backups/$(ls "$T/backups")/complete" ] && fail "$inj: INCOMPLETE backup must not have complete marker" || ok "$inj: no complete-marker backup"
+done
+
+# --- E: env(sed) failure → env byte-identical ---
+set_env_fixture
+env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
+out=$(INJECT=env deploy_out sha256:img-e || true)
+env_after=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
+fx "E env byte-identical" "$env_before" "$env_after"
+printf '%s' "$out" | grep -q "restored artifacts verified" && ok "E auto-rollback verified (env unchanged byte-identical)" || fail "E: $(printf '%s' "$out" | tail -4)"
+fx "E env back to old image" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
+
+# --- F2: restart failure ---
+set_env_fixture
+out=$(FAKE_START_FAIL=1 deploy_out sha256:img-f || true)
+printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F start-service failure -> rollback" || fail "F1: $(printf '%s' "$out" | tail -4)"
+fx "F binary identical" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev"
+fx "F dist identical" "$(cat "$T/release/frontend/index.html")" "<html>prev-dist</html>"
+fx "F env identical" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
+
+# --- F3: health failure (curl level + bad status JSON) ---
+set_env_fixture
+set_env_fixture 2>/dev/null; out=$(FAKE_HEALTH_ON_STOP=1 deploy_out sha256:img-g || true)
+printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F health(unreach) -> rollback" || fail "F3: $(printf '%s' "$out" | tail -4)"
+set_env_fixture
+out=$(FAKE_HEALTH=bad deploy_out sha256:img-h2 || true)
+printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F health(bad-json) -> rollback" || fail "F4: $(printf '%s' "$out" | tail -4)"
+fx "F4 env identical after health failure" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local"
+
+# --- RA: restore must REFUSE to write files when stop cannot be confirmed ---
+set_env_fixture
+out=$(deploy_out sha256:img-ra) || { fail "RA setup deploy died: $out"; }
+RID_RA=$(ls "$T/backups")
+env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
+bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')
+dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')
+rc_ra=0
+out=$( { common_env
+ FAKE_STOP_FAIL=1; export FAKE_STOP_FAIL
+ IMAGE_ID=sha256:img-ra2; export IMAGE_ID
+ cd "$HERE/ops" && ./deploy.sh restore "$RID_RA"; } 2>&1 ) || rc_ra=$?
+[ "$rc_ra" -ne 0 ] && ok "RA restore exits non-zero when stop fails" || fail "RA restore exited 0: $out"
+printf '%s' "$out" | grep -q "refusing" && ok "RA explicit refusal message" || fail "RA msg: $out"
+fx "RA env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before"
+fx "RA binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before"
+fx "RA dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before"
+[ -e "$T/state/.active" ] && ok "RA service still ACTIVE (recoverable files preserved)" || fail "RA service not active"
+
+# --- RB1: DB read error after stop -> old service restored, NO swap ---
+set_env_fixture
+env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
+bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')
+dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')
+cat > "$T/bin/breakdb.sh" <<SH
+#!/bin/sh
+rm -f "$T/prod.sqlite3"
+SH
+chmod +x "$T/bin/breakdb.sh"
+rc_rb=0
+out=$(FAKE_STOP_HOOK="$T/bin/breakdb.sh" deploy_out sha256:img-rb) 2>&1 || rc_rb=$?
+[ "$rc_rb" -ne 0 ] && ok "RB1 deploy exits non-zero on DB read error" || fail "RB1 exited 0: $out"
+printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "RB1 old service restored after DB read error" || { fail "RB1: $out"; printf '%s' "$out" | tail -5; }
+[ -e "$T/state/.active" ] && ok "RB1 old service ACTIVE" || fail "RB1 service down"
+fx "RB1 env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before"
+fx "RB1 binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before"
+fx "RB1 dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before"
+
+# --- RB2: DB_PATH key missing after stop -> old service restored, NO swap ---
+set_env_fixture
+env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}')
+bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')
+dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')
+cat > "$T/bin/dropdbpath.sh" <<SH
+#!/bin/sh
+sed '/^DB_PATH=/d' "$T/state/runtime.env" > "$T/state/runtime.env.tmp"
+mv "$T/state/runtime.env.tmp" "$T/state/runtime.env"
+printf 'DB_PATH_REMOVED=1\n' >> "$T/state/runtime.env"
+SH
+chmod +x "$T/bin/dropdbpath.sh"
+rc_rc=0
+out=$( { FAKE_STOP_HOOK="$T/bin/dropdbpath.sh" deploy_out sha256:img-rc; } 2>&1 ) || rc_rc=$?
+[ "$rc_rc" -ne 0 ] && ok "RB2 deploy exits non-zero on missing DB_PATH" || fail "RB2 exited 0: $out"
+printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "RB2 old service restored after DB_PATH missing" || { fail "RB2: $out"; printf '%s' "$out" | tail -5; }
+[ -e "$T/state/.active" ] && ok "RB2 old service ACTIVE" || fail "RB2 service down"
+# the stop hook removed DB_PATH (that is the injected fault); assert the OTHER
+# keys survived byte-for-byte — no swap touched runtime.env beyond that fault
+fx "RB2 env rest unchanged" \
+ "$(grep -E '^(BIND|DATA_DIR|WORKER_IMAGE)=' "$T/state/runtime.env")" \
+ "BIND=127.0.0.1:18789
+DATA_DIR=$T/data
+WORKER_IMAGE=strategy-lab-worker:local"
+fx "RB2 binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before"
+fx "RB2 dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before"
+
+echo "=== PASSED=$PASSED FAILED=$FAILED ==="
+[ $FAILED -eq 0 ] || exit 1
+exit 0