#!/bin/sh # 受控替身演练 v2(对应复审条目 1-4)。禁止生产操作。 # 退出语义:FAILED>0 → exit 1(不再恒 exit0)。 # 注入点:fake install/rsync/sed/systemctl(stop,start,restart)/curl(health)。 # 覆盖:A1/A2 precheck 硬拦;B 成功部署+手动 rollback 验收;H 并发锁; # C install 失败、D rsync 失败、E env(sed) 失败、F restart/health 失败 # → 自动回滚且环境+发布物与部署前逐字节一致; # I 停服后发现新任务 → 恢复旧服务不替换;RID 不覆盖既有备份。 # RA restore 中 stop 失败且无法确认服务已停 → 拒绝写任何文件并保持 # env/binary/dist 逐字节不变;RB1/RB2 停后的 DB 读取错误或 DB_PATH # 缺失 → 不深层 exit,恢复旧服务且无任何 swap。 set -eu HERE=$(dirname "$0") T="$(cd "$HERE" && pwd)/ops-drill2" PASSED=0; FAILED=0 ok() { PASSED=$((PASSED+1)); printf 'PASS: %s\n' "$1"; } fail() { FAILED=$((FAILED+1)); printf 'FAIL: %s\n' "$1"; } fx() { if [ "$2" = "$3" ]; then ok "$1"; else fail "$1 (got '$2' want '$3')"; fi; } startswith() { case "$2" in "$1"*) true ;; *) false ;; esac; } rm -rf "$T"; mkdir -p "$T/bin" cat > "$T/bin/fake-systemctl" <<'SH' #!/bin/sh action=""; unit="" for a in "$@"; do case "$a" in --user|--*) continue ;; esac if [ -z "$action" ]; then action=$a; else unit=$a; fi done case "$action $unit" in "stop fake-service") [ "$FAKE_STOP_FAIL" = "1" ] && exit 1 printf 'stop\n' >> "$STATE_DIR/.audit.log" rm -f "$STATE_DIR/.active" rm -f "$STATE_DIR/.stopped" : > "$STATE_DIR/.stopped" [ "$FAKE_HEALTH_ON_STOP" = "1" ] && : > "$STATE_DIR/.healthfail" if [ -n "${FAKE_STOP_HOOK:-}" ]; then "$FAKE_STOP_HOOK"; fi exit 0 ;; "start fake-service") # deploy's own start may be injected (FAKE_START_FAIL) ONCE — a later # start attempt (rollback path) must succeed so rollback stays verifiable if [ "$FAKE_START_FAIL" = "1" ] && [ ! -e "$STATE_DIR/.startfail.seen" ]; then : > "$STATE_DIR/.startfail.seen"; exit 1 fi printf 'start\n' >> "$STATE_DIR/.audit.log" if [ "${DEPLOY_PHASE:-}" = "restore" ]; then rm -f "$STATE_DIR/.healthfail" rm -rf "$STATE_DIR/.fakehealth.recovered"; : > "$STATE_DIR/.fakehealth.recovered" fi : > "$STATE_DIR/.active"; rm -f "$STATE_DIR/.stopped"; exit 0 ;; "restart fake-service") [ "$FAKE_RESTART_FAIL" = "1" ] && exit 1 printf 'restart\n' >> "$STATE_DIR/.audit.log" rm -f "$STATE_DIR/.healthfail" rm -rf "$STATE_DIR/.fakehealth.recovered" : > "$STATE_DIR/.fakehealth.recovered" : > "$STATE_DIR/.active"; exit 0 ;; "is-active fake-service") [ -e "$STATE_DIR/.active" ] && exit 0 || { echo inactive; exit 3; } ;; *) exit 1 ;; esac SH cat > "$T/bin/fake-docker" <<'SH' #!/bin/sh case "$*" in *"image inspect"*) exit 0 ;; *) exit 1 ;; esac SH cat > "$T/bin/fake-curl" <<'SH' #!/bin/sh if [ -e "$STATE_DIR/.healthfail" ]; then exit 7; fi if [ "$FAKE_HEALTH" = "bad" ]; then if [ -e "$STATE_DIR/.fakehealth.recovered" ]; then printf '{"status":"ok","ai_configured":true,"version":"0.1.0","worker_available":true}' exit 0 fi if [ ! -e "$STATE_DIR/.healthfail" ]; then : > "$STATE_DIR/.healthfail"; fi printf '{"status":"bad","worker_available":true}' exit 0 fi printf '{"status":"ok","ai_configured":true,"version":"0.1.0","worker_available":true}' exit 0 SH cat > "$T/bin/fake-install" <<'SH' #!/bin/sh if [ "$INJECT" = "install" ]; then echo "install: injected failure" >&2; exit 1; fi m=0755; src=$1; dst=$2 if [ "$1" = "-m" ]; then m=$2; src=$3; dst=$4; fi cat "$src" > "$dst" || exit 1 chmod "$m" "$dst" && exit 0 exit 1 SH cat > "$T/bin/fake-cp" <<'SH' #!/bin/sh if [ "$INJECT" = "backup-cp" ]; then echo "cp: injected failure" >&2; exit 1; fi exec /bin/cp "$@" SH cat > "$T/bin/fake-mv" <<'SH' #!/bin/sh if [ "$INJECT" = "backup-manifest" ]; then echo "mv: injected failure" >&2; exit 1; fi exec /bin/mv "$@" SH cat > "$T/bin/fake-rsync" <<'SH' #!/bin/sh # backup-stage failures are injected separately (INJECT=backup-rsync); # a plain rsync failure must fire only once, in the swap phase if [ "$INJECT" = "backup-rsync" ]; then echo "rsync: injected failure" >&2; exit 1; fi if [ "$INJECT" = "rsync" ] && [ "${DEPLOY_PHASE:-}" = "swap" ] && [ ! -e "$STATE_DIR/.rsyncfail.seen" ]; then : > "$STATE_DIR/.rsyncfail.seen" echo "rsync: injected failure" >&2; exit 1 fi exec /usr/bin/rsync "$@" SH cat > "$T/bin/fake-sed" <<'SH' #!/bin/sh if [ "$INJECT" = "env" ]; then echo "sed: injected failure" >&2; exit 1; fi exec /usr/bin/sed "$@" SH cat > "$T/bin/newwork.sh" < "$T/release/strategy-lab-server" chmod 755 "$T/release/strategy-lab-server" printf 'prev-dist' > "$T/release/frontend/index.html" printf 'binary-the-current-one' > "$T/repo/server/target/release/strategy-lab-server" chmod 755 "$T/repo/server/target/release/strategy-lab-server" printf 'candidate' > "$T/repo/frontend/dist/index.html" python3 - "$T" <<'PY' import sqlite3, sys con = sqlite3.connect(sys.argv[1] + "/prod.sqlite3") con.execute("DROP TABLE IF EXISTS datasets") con.execute("DROP TABLE IF EXISTS runs") con.execute("CREATE TABLE datasets (status TEXT)") con.execute("CREATE TABLE runs (status TEXT)") con.commit() PY printf 'BIND=127.0.0.1:18789\nDB_PATH=%s/prod.sqlite3\nDATA_DIR=%s/data\nWORKER_IMAGE=strategy-lab-worker:local\n' "$T" "$T" > "$T/state/runtime.env" : > "$T/state/.active" } common_env() { STATE_DIR="$T/state" RELEASE_DIR="$T/release" BACKUP_ROOT="$T/backups" \ REPO_ROOT="$T" SERVER_BIN_SRC="$T/repo/server/target/release/strategy-lab-server" \ DIST_SRC="$T/repo/frontend/dist" SERVICE=fake-service \ SYSTEMCTL_CMD="$T/bin/fake-systemctl" DOCKER_CMD="$T/bin/fake-docker" \ CURL_CMD="$T/bin/fake-curl" RSYNC_CMD="$T/bin/fake-rsync" \ INSTALL_CMD="$T/bin/fake-install" SED_CMD="$T/bin/fake-sed" \ CP_CMD="$T/bin/fake-cp" MV_CMD="$T/bin/fake-mv" \ export STATE_DIR RELEASE_DIR BACKUP_ROOT REPO_ROOT SERVER_BIN_SRC DIST_SRC \ SERVICE SYSTEMCTL_CMD DOCKER_CMD CURL_CMD RSYNC_CMD INSTALL_CMD SED_CMD \ CP_CMD MV_CMD } deploy_out() { common_env IMAGE_ID=$1; export IMAGE_ID ( cd "$HERE/ops" && ./deploy.sh ) 2>&1 } # --- A1 datasets --- set_env_fixture python3 - "$T/prod.sqlite3" <<'PY' import sqlite3, sys con = sqlite3.connect(sys.argv[1]); con.execute("INSERT INTO datasets VALUES ('running')"); con.commit() PY out=$(deploy_out sha256:img0 || true) printf '%s' "$out" | grep -q "in-flight datasets" && ok "A1 precheck blocks in-flight datasets" || fail "A1: $out" fx "A1 env untouched" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" fx "A1 release untouched" "$(cat "$T/release/frontend/index.html")" "prev-dist" # --- A2 runs --- set_env_fixture python3 - "$T/prod.sqlite3" <<'PY' import sqlite3, sys con = sqlite3.connect(sys.argv[1]); con.execute("INSERT INTO runs VALUES ('running')"); con.commit() PY out=$(deploy_out sha256:img0 || true) printf '%s' "$out" | grep -q "in-flight runs" && ok "A2 precheck blocks in-flight runs" || fail "A2: $out" # --- I: new work discovered after stop --- set_env_fixture fx "I00 sanity" "$(cat "$T/release/frontend/index.html")" "prev-dist" out=$(FAKE_STOP_HOOK="$T/bin/newwork.sh" deploy_out sha256:img-a || true) printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "I race branch restores old service" || { fail "I: $out"; printf '%s' "$out" | tail -5; } fx "I env untouched" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" fx "I release untouched" "$(cat "$T/release/frontend/index.html")" "prev-dist" printf '%s' "$out" | grep -q "MANUAL CHECK" && fail "I old service restart must succeed" || ok "I old service restarted via fake systemctl start" # --- B: happy deploy --- set_env_fixture out=$(deploy_out sha256:img-1) || { fail "B deploy died: $out"; } printf '%s' "$out" | grep -q "phase 6/6 deploy" && ok "B deploy completed" || fail "B: $(printf '%s' "$out" | tail -4)" n=$(ls "$T/backups" 2>/dev/null | wc -l) fx "B exactly one RID" "$n" "1" R1=$(ls "$T/backups") fx "B env pins immutable ID" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=sha256:img-1" fx "B binary swapped" "$(cat "$T/release/strategy-lab-server")" "binary-the-current-one" fx "B dist swapped" "$(cat "$T/release/frontend/index.html")" "candidate" [ -x "$T/release/strategy-lab-server" ] && ok "B binary executable" || fail "B not executable" [ -f "$T/backups/$R1/release-manifest.txt" ] && ok "B manifest exists" || fail "B manifest missing" [ -f "$T/backups/$R1/complete" ] && ok "B backup has complete marker" || fail "B complete marker missing" # --- RID uniqueness: deploying the SAME image ID again is refused (no overwrite) --- out=$(deploy_out sha256:img-1 || true) printf '%s' "$out" | grep -q "already pins" && ok "RID: re-pin of same ID refused (no overwrite)" || fail "RID refusal: $out" n=$(ls "$T/backups" 2>/dev/null | wc -l) fx "RID: backup count still exactly one" "$n" "1" # --- G: manual rollback.sh $RID verifies everything --- out=$( ( common_env; cd "$HERE/ops" && ./rollback.sh "$R1" ) 2>&1 ) || { fail "G manual rollback died: $out"; } printf '%s' "$out" | grep -q "restored .*env verified" && ok "G manual rollback verified" || fail "G: $out" fx "G env = manifest previous image (pre-deploy state)" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" fx "G release restored to backup (pre-deploy) state" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev" fx "G dist restored to pre-deploy backup" "$(cat "$T/release/frontend/index.html")" "prev-dist" audit=$(cat "$T/state/.audit.log") printf '%s\n' "$audit" | head -1 | grep -qx stop && ok "G rollback stops service FIRST (no writes under a running binary)" || fail "G audit first line not stop: $(printf '%s' "$audit")" printf '%s\n' "$audit" | tail -1 | grep -qx start && ok "G rollback restarts service LAST" || fail "G audit last line not start: $(printf '%s' "$audit")" [ -e "$T/state/.active" ] && ok "G manual rollback leaves service ACTIVE" || fail "G: service not active after manual rollback" # --- H: concurrent deploy refused by deploy.lock --- rm -f "$T/state/deploy.lock" set_env_fixture ( common_env >/dev/null 2>&1 exec 9>"$T/state/deploy.lock"; flock 9 sleep 1.5 ) & HOLD_PID=$! sleep 0.4 out2=$( ( common_env >/dev/null; IMAGE_ID=sha256:img-h; export IMAGE_ID; cd "$HERE/ops" && ./deploy.sh ) 2>&1 || true) wait $HOLD_PID || true printf '%s' "$out2" | grep -q "deploy.lock" && ok "H concurrent deploy refused" || fail "H: $out2" # --- C: install failure inside swap --- set_env_fixture out=$(INJECT=install deploy_out sha256:img-c || true) || true printf '%s' "$out" | grep -q "install .* failed" && ok "C install failure detected" || fail "C: $out" printf '%s' "$out" | grep -q "restored artifacts verified (env + sha256)" && ok "C auto-rollback verified" || fail "C rollback: $(printf '%s' "$out" | tail -4)" fx "C binary restored" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev" fx "C dist restored" "$(cat "$T/release/frontend/index.html")" "prev-dist" fx "C env restored" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" [ -f "$T/backups/$(ls "$T/backups")/release-manifest.txt" ] && ok "C manifest" || fail "C manifest" # --- D: rsync failure, ONE-SHOT in the swap phase -> auto rollback healthy --- set_env_fixture out=$(INJECT=rsync deploy_out sha256:img-d) || rc_d=$? [ "$out" = "0" ] || [ "${rc_d:-nonzero}" != "0" ] && ok "D deploy exits non-zero" || fail "D exit expected nonzero" printf '%s' "$out" | grep -q "rsync .* failed" && ok "D rsync failure detected" || fail "D: $out" printf '%s' "$out" | grep -q "restored artifacts verified (env + sha256)" && ok "D auto-rollback verified" || fail "D rollback: $(printf '%s' "$out" | tail -4)" fx "D dist restored" "$(cat "$T/release/frontend/index.html")" "prev-dist" fx "D binary restored" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev" fx "D env restored" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" [ -e "$T/state/.active" ] && ok "D old service ACTIVE after rollback" || fail "D: old service not active" [ -e "$T/backups/$(ls "$T/backups")/complete" ] && ok "D backup was complete before swap" || fail "D backup lacked complete marker" # --- BD1/BD2/BD3: backup-stage failure (cp / rsync / manifest) --- # Service must STAY active (backup happens before stop), artifacts must be # byte-identical, exit non-zero, and NO complete backup may exist. for inj in backup-cp backup-rsync backup-manifest; do set_env_fixture env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}') dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}') out=$(INJECT="$inj" deploy_out sha256:img-bd || true) [ -e "$T/state/.active" ] && ok "$inj: old service ACTIVE (no downtime)" || fail "$inj: old service down" fx "$inj: env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before" fx "$inj: binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before" fx "$inj: dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before" [ -e "$T/backups/$(ls "$T/backups")/complete" ] && fail "$inj: INCOMPLETE backup must not have complete marker" || ok "$inj: no complete-marker backup" done # --- E: env(sed) failure → env byte-identical --- set_env_fixture env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') out=$(INJECT=env deploy_out sha256:img-e || true) env_after=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') fx "E env byte-identical" "$env_before" "$env_after" printf '%s' "$out" | grep -q "restored artifacts verified" && ok "E auto-rollback verified (env unchanged byte-identical)" || fail "E: $(printf '%s' "$out" | tail -4)" fx "E env back to old image" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" # --- F2: restart failure --- set_env_fixture out=$(FAKE_START_FAIL=1 deploy_out sha256:img-f || true) printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F start-service failure -> rollback" || fail "F1: $(printf '%s' "$out" | tail -4)" fx "F binary identical" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev" fx "F dist identical" "$(cat "$T/release/frontend/index.html")" "prev-dist" fx "F env identical" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" # --- F3: health failure (curl level + bad status JSON) --- set_env_fixture set_env_fixture 2>/dev/null; out=$(FAKE_HEALTH_ON_STOP=1 deploy_out sha256:img-g || true) printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F health(unreach) -> rollback" || fail "F3: $(printf '%s' "$out" | tail -4)" set_env_fixture out=$(FAKE_HEALTH=bad deploy_out sha256:img-h2 || true) printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F health(bad-json) -> rollback" || fail "F4: $(printf '%s' "$out" | tail -4)" fx "F4 env identical after health failure" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" # --- RA: restore must REFUSE to write files when stop cannot be confirmed --- set_env_fixture out=$(deploy_out sha256:img-ra) || { fail "RA setup deploy died: $out"; } RID_RA=$(ls "$T/backups") env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}') dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}') rc_ra=0 out=$( { common_env FAKE_STOP_FAIL=1; export FAKE_STOP_FAIL IMAGE_ID=sha256:img-ra2; export IMAGE_ID cd "$HERE/ops" && ./deploy.sh restore "$RID_RA"; } 2>&1 ) || rc_ra=$? [ "$rc_ra" -ne 0 ] && ok "RA restore exits non-zero when stop fails" || fail "RA restore exited 0: $out" printf '%s' "$out" | grep -q "refusing" && ok "RA explicit refusal message" || fail "RA msg: $out" fx "RA env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before" fx "RA binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before" fx "RA dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before" [ -e "$T/state/.active" ] && ok "RA service still ACTIVE (recoverable files preserved)" || fail "RA service not active" # --- RB1: DB read error after stop -> old service restored, NO swap --- set_env_fixture env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}') dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}') cat > "$T/bin/breakdb.sh" <&1 || rc_rb=$? [ "$rc_rb" -ne 0 ] && ok "RB1 deploy exits non-zero on DB read error" || fail "RB1 exited 0: $out" printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "RB1 old service restored after DB read error" || { fail "RB1: $out"; printf '%s' "$out" | tail -5; } [ -e "$T/state/.active" ] && ok "RB1 old service ACTIVE" || fail "RB1 service down" fx "RB1 env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before" fx "RB1 binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before" fx "RB1 dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before" # --- RB2: DB_PATH key missing after stop -> old service restored, NO swap --- set_env_fixture env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}') dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}') cat > "$T/bin/dropdbpath.sh" < "$T/state/runtime.env.tmp" mv "$T/state/runtime.env.tmp" "$T/state/runtime.env" printf 'DB_PATH_REMOVED=1\n' >> "$T/state/runtime.env" SH chmod +x "$T/bin/dropdbpath.sh" rc_rc=0 out=$( { FAKE_STOP_HOOK="$T/bin/dropdbpath.sh" deploy_out sha256:img-rc; } 2>&1 ) || rc_rc=$? [ "$rc_rc" -ne 0 ] && ok "RB2 deploy exits non-zero on missing DB_PATH" || fail "RB2 exited 0: $out" printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "RB2 old service restored after DB_PATH missing" || { fail "RB2: $out"; printf '%s' "$out" | tail -5; } [ -e "$T/state/.active" ] && ok "RB2 old service ACTIVE" || fail "RB2 service down" # the stop hook removed DB_PATH (that is the injected fault); assert the OTHER # keys survived byte-for-byte — no swap touched runtime.env beyond that fault fx "RB2 env rest unchanged" \ "$(grep -E '^(BIND|DATA_DIR|WORKER_IMAGE)=' "$T/state/runtime.env")" \ "BIND=127.0.0.1:18789 DATA_DIR=$T/data WORKER_IMAGE=strategy-lab-worker:local" fx "RB2 binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before" fx "RB2 dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before" echo "=== PASSED=$PASSED FAILED=$FAILED ===" [ $FAILED -eq 0 ] || exit 1 exit 0