From bf6681eb29ac8b0c80ca17b2b5869f7de3da1198 Mon Sep 17 00:00:00 2001 From: "Somhairle H. Marisol" Date: Fri, 18 Sep 2026 08:27:24 +0800 Subject: chore(ops): 生产部署/回滚/故障演练脚本入库 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [变更性质] 维护性变更：把已通过 84 场景演练的运维脚本纳入版本控制。artifacts/ 目录按仓库政策 ignore（本地证据），本提交按 Leader 指令以 -f 显式 加入，未包含任何密钥（脚本扫无 secret/token/password）。 [维护内容] 此前 ops/deploy.sh、rollback.sh 与 ops-drill2.sh 仅存在于本地 artifacts/（被 .gitignore 排除），生产依赖它们但仓库内无源码可审计。 [实现方案] deploy.sh：deploy.lock 防并发、阶段化备份（env/二进制/前端/DB， manifest 原子化写入 0700 目录）、显式 rc helper、零停机 swap、失败 路径自动 restore、健康与 sha256 校验；restore 兼做手动回滚。 rollback.sh：stop -> 替换 -> start -> 锁 + health + hash 强校验。 ops-drill2.sh：84 场景故障注入演练（备份/swap/restore 阶段注入矩阵、 RA/RB1/RB2 恢复分支），PASSED=84 FAILED=0 exit 0 为执行合约。 用法见 docs/etf-recovery-release-handoff.md 第 3 节。 [影响范围] 脚本本身未改（与演练定稿版本一致），仅入库；回滚目标记录在 manifest（previous_worker_image 等）。 --- artifacts/etf-recovery-candidate/ops-drill2.sh | 372 +++++++++++++++++++++++ artifacts/etf-recovery-candidate/ops/deploy.sh | 329 ++++++++++++++++++++ artifacts/etf-recovery-candidate/ops/rollback.sh | 102 +++++++ 3 files changed, 803 insertions(+) create mode 100755 artifacts/etf-recovery-candidate/ops-drill2.sh create mode 100755 artifacts/etf-recovery-candidate/ops/deploy.sh create mode 100755 artifacts/etf-recovery-candidate/ops/rollback.sh diff --git a/artifacts/etf-recovery-candidate/ops-drill2.sh b/artifacts/etf-recovery-candidate/ops-drill2.sh new file mode 100755 index 0000000..e4a4caf --- /dev/null +++ b/artifacts/etf-recovery-candidate/ops-drill2.sh @@ -0,0 +1,372 @@ +#!/bin/sh +# 受控替身演练 v2（对应复审条目 1-4）。禁止生产操作。 +# 退出语义：FAILED>0 → exit 1（不再恒 exit0）。 +# 注入点：fake install/rsync/sed/systemctl(stop,start,restart)/curl(health)。 +# 覆盖：A1/A2 precheck 硬拦；B 成功部署+手动 rollback 验收；H 并发锁； +# C install 失败、D rsync 失败、E env(sed) 失败、F restart/health 失败 +# → 自动回滚且环境+发布物与部署前逐字节一致； +# I 停服后发现新任务 → 恢复旧服务不替换；RID 不覆盖既有备份。 +# RA restore 中 stop 失败且无法确认服务已停 → 拒绝写任何文件并保持 +# env/binary/dist 逐字节不变；RB1/RB2 停后的 DB 读取错误或 DB_PATH +# 缺失 → 不深层 exit，恢复旧服务且无任何 swap。 +set -eu +HERE=$(dirname "$0") +T="$(cd "$HERE" && pwd)/ops-drill2" +PASSED=0; FAILED=0 +ok() { PASSED=$((PASSED+1)); printf 'PASS: %s\n' "$1"; } +fail() { FAILED=$((FAILED+1)); printf 'FAIL: %s\n' "$1"; } +fx() { if [ "$2" = "$3" ]; then ok "$1"; else fail "$1 (got '$2' want '$3')"; fi; } +startswith() { case "$2" in "$1"*) true ;; *) false ;; esac; } + +rm -rf "$T"; mkdir -p "$T/bin" + +cat > "$T/bin/fake-systemctl" <<'SH' +#!/bin/sh +action=""; unit="" +for a in "$@"; do + case "$a" in --user|--*) continue ;; esac + if [ -z "$action" ]; then action=$a; else unit=$a; fi +done +case "$action $unit" in + "stop fake-service") + [ "$FAKE_STOP_FAIL" = "1" ] && exit 1 + printf 'stop\n' >> "$STATE_DIR/.audit.log" + rm -f "$STATE_DIR/.active" + rm -f "$STATE_DIR/.stopped" + : > "$STATE_DIR/.stopped" + [ "$FAKE_HEALTH_ON_STOP" = "1" ] && : > "$STATE_DIR/.healthfail" + if [ -n "${FAKE_STOP_HOOK:-}" ]; then "$FAKE_STOP_HOOK"; fi + exit 0 ;; + "start fake-service") + # deploy's own start may be injected (FAKE_START_FAIL) ONCE — a later + # start attempt (rollback path) must succeed so rollback stays verifiable + if [ "$FAKE_START_FAIL" = "1" ] && [ ! -e "$STATE_DIR/.startfail.seen" ]; then + : > "$STATE_DIR/.startfail.seen"; exit 1 + fi + printf 'start\n' >> "$STATE_DIR/.audit.log" + if [ "${DEPLOY_PHASE:-}" = "restore" ]; then + rm -f "$STATE_DIR/.healthfail" + rm -rf "$STATE_DIR/.fakehealth.recovered"; : > "$STATE_DIR/.fakehealth.recovered" + fi + : > "$STATE_DIR/.active"; rm -f "$STATE_DIR/.stopped"; exit 0 ;; + "restart fake-service") + [ "$FAKE_RESTART_FAIL" = "1" ] && exit 1 + printf 'restart\n' >> "$STATE_DIR/.audit.log" + rm -f "$STATE_DIR/.healthfail" + rm -rf "$STATE_DIR/.fakehealth.recovered" + : > "$STATE_DIR/.fakehealth.recovered" + : > "$STATE_DIR/.active"; exit 0 ;; + "is-active fake-service") + [ -e "$STATE_DIR/.active" ] && exit 0 || { echo inactive; exit 3; } ;; + *) exit 1 ;; +esac +SH +cat > "$T/bin/fake-docker" <<'SH' +#!/bin/sh +case "$*" in *"image inspect"*) exit 0 ;; *) exit 1 ;; esac +SH +cat > "$T/bin/fake-curl" <<'SH' +#!/bin/sh +if [ -e "$STATE_DIR/.healthfail" ]; then exit 7; fi +if [ "$FAKE_HEALTH" = "bad" ]; then + if [ -e "$STATE_DIR/.fakehealth.recovered" ]; then + printf '{"status":"ok","ai_configured":true,"version":"0.1.0","worker_available":true}' + exit 0 + fi + if [ ! -e "$STATE_DIR/.healthfail" ]; then : > "$STATE_DIR/.healthfail"; fi + printf '{"status":"bad","worker_available":true}' + exit 0 +fi +printf '{"status":"ok","ai_configured":true,"version":"0.1.0","worker_available":true}' +exit 0 +SH +cat > "$T/bin/fake-install" <<'SH' +#!/bin/sh +if [ "$INJECT" = "install" ]; then echo "install: injected failure" >&2; exit 1; fi +m=0755; src=$1; dst=$2 +if [ "$1" = "-m" ]; then m=$2; src=$3; dst=$4; fi +cat "$src" > "$dst" || exit 1 +chmod "$m" "$dst" && exit 0 +exit 1 +SH +cat > "$T/bin/fake-cp" <<'SH' +#!/bin/sh +if [ "$INJECT" = "backup-cp" ]; then echo "cp: injected failure" >&2; exit 1; fi +exec /bin/cp "$@" +SH +cat > "$T/bin/fake-mv" <<'SH' +#!/bin/sh +if [ "$INJECT" = "backup-manifest" ]; then echo "mv: injected failure" >&2; exit 1; fi +exec /bin/mv "$@" +SH +cat > "$T/bin/fake-rsync" <<'SH' +#!/bin/sh +# backup-stage failures are injected separately (INJECT=backup-rsync); +# a plain rsync failure must fire only once, in the swap phase +if [ "$INJECT" = "backup-rsync" ]; then echo "rsync: injected failure" >&2; exit 1; fi +if [ "$INJECT" = "rsync" ] && [ "${DEPLOY_PHASE:-}" = "swap" ] && [ ! -e "$STATE_DIR/.rsyncfail.seen" ]; then + : > "$STATE_DIR/.rsyncfail.seen" + echo "rsync: injected failure" >&2; exit 1 +fi +exec /usr/bin/rsync "$@" +SH +cat > "$T/bin/fake-sed" <<'SH' +#!/bin/sh +if [ "$INJECT" = "env" ]; then echo "sed: injected failure" >&2; exit 1; fi +exec /usr/bin/sed "$@" +SH +cat > "$T/bin/newwork.sh" < "$T/release/strategy-lab-server" + chmod 755 "$T/release/strategy-lab-server" + printf 'prev-dist' > "$T/release/frontend/index.html" + printf 'binary-the-current-one' > "$T/repo/server/target/release/strategy-lab-server" + chmod 755 "$T/repo/server/target/release/strategy-lab-server" + printf 'candidate' > "$T/repo/frontend/dist/index.html" + python3 - "$T" <<'PY' +import sqlite3, sys +con = sqlite3.connect(sys.argv[1] + "/prod.sqlite3") +con.execute("DROP TABLE IF EXISTS datasets") +con.execute("DROP TABLE IF EXISTS runs") +con.execute("CREATE TABLE datasets (status TEXT)") +con.execute("CREATE TABLE runs (status TEXT)") +con.commit() +PY + printf 'BIND=127.0.0.1:18789\nDB_PATH=%s/prod.sqlite3\nDATA_DIR=%s/data\nWORKER_IMAGE=strategy-lab-worker:local\n' "$T" "$T" > "$T/state/runtime.env" + : > "$T/state/.active" +} + +common_env() { + STATE_DIR="$T/state" RELEASE_DIR="$T/release" BACKUP_ROOT="$T/backups" \ + REPO_ROOT="$T" SERVER_BIN_SRC="$T/repo/server/target/release/strategy-lab-server" \ + DIST_SRC="$T/repo/frontend/dist" SERVICE=fake-service \ + SYSTEMCTL_CMD="$T/bin/fake-systemctl" DOCKER_CMD="$T/bin/fake-docker" \ + CURL_CMD="$T/bin/fake-curl" RSYNC_CMD="$T/bin/fake-rsync" \ + INSTALL_CMD="$T/bin/fake-install" SED_CMD="$T/bin/fake-sed" \ + CP_CMD="$T/bin/fake-cp" MV_CMD="$T/bin/fake-mv" \ + export STATE_DIR RELEASE_DIR BACKUP_ROOT REPO_ROOT SERVER_BIN_SRC DIST_SRC \ + SERVICE SYSTEMCTL_CMD DOCKER_CMD CURL_CMD RSYNC_CMD INSTALL_CMD SED_CMD \ + CP_CMD MV_CMD +} + +deploy_out() { + common_env + IMAGE_ID=$1; export IMAGE_ID + ( cd "$HERE/ops" && ./deploy.sh ) 2>&1 +} + +# --- A1 datasets --- +set_env_fixture +python3 - "$T/prod.sqlite3" <<'PY' +import sqlite3, sys +con = sqlite3.connect(sys.argv[1]); con.execute("INSERT INTO datasets VALUES ('running')"); con.commit() +PY +out=$(deploy_out sha256:img0 || true) +printf '%s' "$out" | grep -q "in-flight datasets" && ok "A1 precheck blocks in-flight datasets" || fail "A1: $out" +fx "A1 env untouched" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" +fx "A1 release untouched" "$(cat "$T/release/frontend/index.html")" "prev-dist" + +# --- A2 runs --- +set_env_fixture +python3 - "$T/prod.sqlite3" <<'PY' +import sqlite3, sys +con = sqlite3.connect(sys.argv[1]); con.execute("INSERT INTO runs VALUES ('running')"); con.commit() +PY +out=$(deploy_out sha256:img0 || true) +printf '%s' "$out" | grep -q "in-flight runs" && ok "A2 precheck blocks in-flight runs" || fail "A2: $out" + +# --- I: new work discovered after stop --- +set_env_fixture +fx "I00 sanity" "$(cat "$T/release/frontend/index.html")" "prev-dist" +out=$(FAKE_STOP_HOOK="$T/bin/newwork.sh" deploy_out sha256:img-a || true) +printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "I race branch restores old service" || { fail "I: $out"; printf '%s' "$out" | tail -5; } +fx "I env untouched" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" +fx "I release untouched" "$(cat "$T/release/frontend/index.html")" "prev-dist" +printf '%s' "$out" | grep -q "MANUAL CHECK" && fail "I old service restart must succeed" || ok "I old service restarted via fake systemctl start" + +# --- B: happy deploy --- +set_env_fixture +out=$(deploy_out sha256:img-1) || { fail "B deploy died: $out"; } +printf '%s' "$out" | grep -q "phase 6/6 deploy" && ok "B deploy completed" || fail "B: $(printf '%s' "$out" | tail -4)" +n=$(ls "$T/backups" 2>/dev/null | wc -l) +fx "B exactly one RID" "$n" "1" +R1=$(ls "$T/backups") +fx "B env pins immutable ID" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=sha256:img-1" +fx "B binary swapped" "$(cat "$T/release/strategy-lab-server")" "binary-the-current-one" +fx "B dist swapped" "$(cat "$T/release/frontend/index.html")" "candidate" +[ -x "$T/release/strategy-lab-server" ] && ok "B binary executable" || fail "B not executable" +[ -f "$T/backups/$R1/release-manifest.txt" ] && ok "B manifest exists" || fail "B manifest missing" +[ -f "$T/backups/$R1/complete" ] && ok "B backup has complete marker" || fail "B complete marker missing" + +# --- RID uniqueness: deploying the SAME image ID again is refused (no overwrite) --- +out=$(deploy_out sha256:img-1 || true) +printf '%s' "$out" | grep -q "already pins" && ok "RID: re-pin of same ID refused (no overwrite)" || fail "RID refusal: $out" +n=$(ls "$T/backups" 2>/dev/null | wc -l) +fx "RID: backup count still exactly one" "$n" "1" + +# --- G: manual rollback.sh $RID verifies everything --- +out=$( ( common_env; cd "$HERE/ops" && ./rollback.sh "$R1" ) 2>&1 ) || { fail "G manual rollback died: $out"; } +printf '%s' "$out" | grep -q "restored .*env verified" && ok "G manual rollback verified" || fail "G: $out" +fx "G env = manifest previous image (pre-deploy state)" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" +fx "G release restored to backup (pre-deploy) state" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev" +fx "G dist restored to pre-deploy backup" "$(cat "$T/release/frontend/index.html")" "prev-dist" +audit=$(cat "$T/state/.audit.log") +printf '%s\n' "$audit" | head -1 | grep -qx stop && ok "G rollback stops service FIRST (no writes under a running binary)" || fail "G audit first line not stop: $(printf '%s' "$audit")" +printf '%s\n' "$audit" | tail -1 | grep -qx start && ok "G rollback restarts service LAST" || fail "G audit last line not start: $(printf '%s' "$audit")" +[ -e "$T/state/.active" ] && ok "G manual rollback leaves service ACTIVE" || fail "G: service not active after manual rollback" + +# --- H: concurrent deploy refused by deploy.lock --- +rm -f "$T/state/deploy.lock" +set_env_fixture +( common_env >/dev/null 2>&1 + exec 9>"$T/state/deploy.lock"; flock 9 + sleep 1.5 ) & HOLD_PID=$! +sleep 0.4 +out2=$( ( common_env >/dev/null; IMAGE_ID=sha256:img-h; export IMAGE_ID; cd "$HERE/ops" && ./deploy.sh ) 2>&1 || true) +wait $HOLD_PID || true +printf '%s' "$out2" | grep -q "deploy.lock" && ok "H concurrent deploy refused" || fail "H: $out2" + +# --- C: install failure inside swap --- +set_env_fixture +out=$(INJECT=install deploy_out sha256:img-c || true) || true +printf '%s' "$out" | grep -q "install .* failed" && ok "C install failure detected" || fail "C: $out" +printf '%s' "$out" | grep -q "restored artifacts verified (env + sha256)" && ok "C auto-rollback verified" || fail "C rollback: $(printf '%s' "$out" | tail -4)" +fx "C binary restored" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev" +fx "C dist restored" "$(cat "$T/release/frontend/index.html")" "prev-dist" +fx "C env restored" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" +[ -f "$T/backups/$(ls "$T/backups")/release-manifest.txt" ] && ok "C manifest" || fail "C manifest" + +# --- D: rsync failure, ONE-SHOT in the swap phase -> auto rollback healthy --- +set_env_fixture +out=$(INJECT=rsync deploy_out sha256:img-d) || rc_d=$? +[ "$out" = "0" ] || [ "${rc_d:-nonzero}" != "0" ] && ok "D deploy exits non-zero" || fail "D exit expected nonzero" +printf '%s' "$out" | grep -q "rsync .* failed" && ok "D rsync failure detected" || fail "D: $out" +printf '%s' "$out" | grep -q "restored artifacts verified (env + sha256)" && ok "D auto-rollback verified" || fail "D rollback: $(printf '%s' "$out" | tail -4)" +fx "D dist restored" "$(cat "$T/release/frontend/index.html")" "prev-dist" +fx "D binary restored" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev" +fx "D env restored" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" +[ -e "$T/state/.active" ] && ok "D old service ACTIVE after rollback" || fail "D: old service not active" +[ -e "$T/backups/$(ls "$T/backups")/complete" ] && ok "D backup was complete before swap" || fail "D backup lacked complete marker" + +# --- BD1/BD2/BD3: backup-stage failure (cp / rsync / manifest) --- +# Service must STAY active (backup happens before stop), artifacts must be +# byte-identical, exit non-zero, and NO complete backup may exist. +for inj in backup-cp backup-rsync backup-manifest; do + set_env_fixture + env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') + bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}') + dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}') + out=$(INJECT="$inj" deploy_out sha256:img-bd || true) + [ -e "$T/state/.active" ] && ok "$inj: old service ACTIVE (no downtime)" || fail "$inj: old service down" + fx "$inj: env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before" + fx "$inj: binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before" + fx "$inj: dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before" + [ -e "$T/backups/$(ls "$T/backups")/complete" ] && fail "$inj: INCOMPLETE backup must not have complete marker" || ok "$inj: no complete-marker backup" +done + +# --- E: env(sed) failure → env byte-identical --- +set_env_fixture +env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') +out=$(INJECT=env deploy_out sha256:img-e || true) +env_after=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') +fx "E env byte-identical" "$env_before" "$env_after" +printf '%s' "$out" | grep -q "restored artifacts verified" && ok "E auto-rollback verified (env unchanged byte-identical)" || fail "E: $(printf '%s' "$out" | tail -4)" +fx "E env back to old image" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" + +# --- F2: restart failure --- +set_env_fixture +out=$(FAKE_START_FAIL=1 deploy_out sha256:img-f || true) +printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F start-service failure -> rollback" || fail "F1: $(printf '%s' "$out" | tail -4)" +fx "F binary identical" "$(cat "$T/release/strategy-lab-server")" "binary-live-prev" +fx "F dist identical" "$(cat "$T/release/frontend/index.html")" "prev-dist" +fx "F env identical" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" + +# --- F3: health failure (curl level + bad status JSON) --- +set_env_fixture +set_env_fixture 2>/dev/null; out=$(FAKE_HEALTH_ON_STOP=1 deploy_out sha256:img-g || true) +printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F health(unreach) -> rollback" || fail "F3: $(printf '%s' "$out" | tail -4)" +set_env_fixture +out=$(FAKE_HEALTH=bad deploy_out sha256:img-h2 || true) +printf '%s' "$out" | grep -q "restored artifacts verified" && ok "F health(bad-json) -> rollback" || fail "F4: $(printf '%s' "$out" | tail -4)" +fx "F4 env identical after health failure" "$(grep '^WORKER_IMAGE=' "$T/state/runtime.env")" "WORKER_IMAGE=strategy-lab-worker:local" + +# --- RA: restore must REFUSE to write files when stop cannot be confirmed --- +set_env_fixture +out=$(deploy_out sha256:img-ra) || { fail "RA setup deploy died: $out"; } +RID_RA=$(ls "$T/backups") +env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') +bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}') +dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}') +rc_ra=0 +out=$( { common_env + FAKE_STOP_FAIL=1; export FAKE_STOP_FAIL + IMAGE_ID=sha256:img-ra2; export IMAGE_ID + cd "$HERE/ops" && ./deploy.sh restore "$RID_RA"; } 2>&1 ) || rc_ra=$? +[ "$rc_ra" -ne 0 ] && ok "RA restore exits non-zero when stop fails" || fail "RA restore exited 0: $out" +printf '%s' "$out" | grep -q "refusing" && ok "RA explicit refusal message" || fail "RA msg: $out" +fx "RA env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before" +fx "RA binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before" +fx "RA dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before" +[ -e "$T/state/.active" ] && ok "RA service still ACTIVE (recoverable files preserved)" || fail "RA service not active" + +# --- RB1: DB read error after stop -> old service restored, NO swap --- +set_env_fixture +env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') +bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}') +dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}') +cat > "$T/bin/breakdb.sh" <&1 || rc_rb=$? +[ "$rc_rb" -ne 0 ] && ok "RB1 deploy exits non-zero on DB read error" || fail "RB1 exited 0: $out" +printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "RB1 old service restored after DB read error" || { fail "RB1: $out"; printf '%s' "$out" | tail -5; } +[ -e "$T/state/.active" ] && ok "RB1 old service ACTIVE" || fail "RB1 service down" +fx "RB1 env unchanged" "$(sha256sum "$T/state/runtime.env" | awk '{print $1}')" "$env_before" +fx "RB1 binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before" +fx "RB1 dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before" + +# --- RB2: DB_PATH key missing after stop -> old service restored, NO swap --- +set_env_fixture +env_before=$(sha256sum "$T/state/runtime.env" | awk '{print $1}') +bin_before=$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}') +dist_before=$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}') +cat > "$T/bin/dropdbpath.sh" < "$T/state/runtime.env.tmp" +mv "$T/state/runtime.env.tmp" "$T/state/runtime.env" +printf 'DB_PATH_REMOVED=1\n' >> "$T/state/runtime.env" +SH +chmod +x "$T/bin/dropdbpath.sh" +rc_rc=0 +out=$( { FAKE_STOP_HOOK="$T/bin/dropdbpath.sh" deploy_out sha256:img-rc; } 2>&1 ) || rc_rc=$? +[ "$rc_rc" -ne 0 ] && ok "RB2 deploy exits non-zero on missing DB_PATH" || fail "RB2 exited 0: $out" +printf '%s' "$out" | grep -q "restored old service WITHOUT replacement" && ok "RB2 old service restored after DB_PATH missing" || { fail "RB2: $out"; printf '%s' "$out" | tail -5; } +[ -e "$T/state/.active" ] && ok "RB2 old service ACTIVE" || fail "RB2 service down" +# the stop hook removed DB_PATH (that is the injected fault); assert the OTHER +# keys survived byte-for-byte — no swap touched runtime.env beyond that fault +fx "RB2 env rest unchanged" \ + "$(grep -E '^(BIND|DATA_DIR|WORKER_IMAGE)=' "$T/state/runtime.env")" \ + "BIND=127.0.0.1:18789 +DATA_DIR=$T/data +WORKER_IMAGE=strategy-lab-worker:local" +fx "RB2 binary unchanged" "$(sha256sum "$T/release/strategy-lab-server" | awk '{print $1}')" "$bin_before" +fx "RB2 dist unchanged" "$(sha256sum "$T/release/frontend/index.html" | awk '{print $1}')" "$dist_before" + +echo "=== PASSED=$PASSED FAILED=$FAILED ===" +[ $FAILED -eq 0 ] || exit 1 +exit 0 diff --git a/artifacts/etf-recovery-candidate/ops/deploy.sh b/artifacts/etf-recovery-candidate/ops/deploy.sh new file mode 100755 index 0000000..822c90e --- /dev/null +++ b/artifacts/etf-recovery-candidate/ops/deploy.sh @@ -0,0 +1,329 @@ +#!/bin/sh +# ============================================================== +# Strategy Lab — 部署脚本 v2（供 leader 审查/在隔离替身环境演练） +# *** 本脚本不会由开发会话在生产执行 *** +# 针对 leader 复审意见(2026-09-17)的修正： +# R1 每个关键命令显式检查返回码，绝不依赖 set -e 在 && / if ! 上下文的不可靠性。 +# R2 竞态窗口：先 systemctl stop 服务 → 重新硬检查 in-flight（服务停止后不再产生 +# 新任务）→ 才 swap。停后发现新任务：立即恢复旧服务（发布物未被动过）并退出， +# 任务状态原样保留。 +# R3 restore 入口也取同一把锁；回滚后验证 ①服务 active ②/api/health JSON +# status=="ok" ③恢复后的发布物 sha256 == release-manifest.txt 里的旧哈希。 +# 备份在发布目录外、0700；单一 RELEASE_ID；同秒碰撞自动加后缀，绝不覆盖既有备份。 +# +# 受控替身（演练/审查可注入）： +# SYSTEMCTL_CMD/DOCKER_CMD/CURL_CMD/RSYNC_CMD/INSTALL_CMD 均可指向替身。 +# *不含生产破坏性操作，禁止在无 STAGE_DIR 环境下执行（见 require_prod_paths）。 +set -eu + +SYSTEMCTL_CMD=${SYSTEMCTL_CMD:-systemctl} +DOCKER_CMD=${DOCKER_CMD:-docker} +CURL_CMD=${CURL_CMD:-curl} +INSTALL_CMD=${INSTALL_CMD:-install} +SED_CMD=${SED_CMD:-sed} +RSYNC_CMD=${RSYNC_CMD:-rsync} +CP_CMD=${CP_CMD:-cp} +MV_CMD=${MV_CMD:-mv} + +STATE_DIR=${STATE_DIR:-/home/somhairle/.local/share/strategy-lab-production} +RELEASE_DIR=${RELEASE_DIR:-$STATE_DIR/release} +BACKUP_ROOT=${BACKUP_ROOT:-$STATE_DIR/release-backups} +REPO_ROOT=${REPO_ROOT:?REPO_ROOT must be set (absolute path to strategy-lab checkout)} +SERVICE=${SERVICE:-strategy-lab-production} +SERVER_BIN_SRC=${SERVER_BIN_SRC:-$REPO_ROOT/server/target/release/strategy-lab-server} +DIST_SRC=${DIST_SRC:-$REPO_ROOT/frontend/dist} +IMAGE_ID=${IMAGE_ID:?IMAGE_ID must be the immutable image ID, e.g. sha256:...} + +msg() { printf '[deploy] %s\n' "$1"; } +note() { printf '[deploy] NOTE: %s\n' "$1" >&2; } +die() { printf '[deploy] FATAL: %s\n' "$1" >&2; exit 1; } + +# ---------------- explicit-rc helpers (R1) ---------------- +run_cp() { "$CP_CMD" -p "$1" "$2" || { note "cp -p $1 -> $2 failed"; return 1; }; } +run_sed() { "$SED_CMD" -i "$@" || { note "sed env replacement failed"; return 1; }; } +run_install(){ "$INSTALL_CMD" -m 0755 "$1" "$2" || { note "install $1 -> $2 failed"; return 1; }; } +run_rsync() { "$RSYNC_CMD" "$@" || { note "rsync $* failed"; return 1; } } +run_verchown_mode() { + chmod "$1" "$2" || { note "chmod $1 $2 failed"; return 1; } +} +# manifest is written to a tmp file and atomically moved into place; a failed +# mv (=> incomplete manifest) never looks like a finished backup +fin_stage() { + "$MV_CMD" "$1" "$2" || { note "finalize $2 failed"; return 1; } +} + +db_scalar() { + # $1 = db path, $2 = sql (single scalar). sqlite3 if present, else python3 stdlib. + if command -v sqlite3 >/dev/null 2>&1; then + sqlite3 "$1" "$2" + else + python3 - "$1" "$2" <<'PY' || die "db_scalar failed on $1" +import sqlite3, sys +c = sqlite3.connect(sys.argv[1]) +row = c.execute(sys.argv[2]).fetchone() +print(row[0] if row is not None else 0) +PY + fi +} + +env_value() { + grep -E "^$1=" "$STATE_DIR/runtime.env" | head -1 | cut -d= -f2- +} + +service_active() { "$SYSTEMCTL_CMD" --user is-active "$SERVICE" >/dev/null 2>&1; } +service_stop() { "$SYSTEMCTL_CMD" --user stop "$SERVICE" || return 1; } +service_start() { "$SYSTEMCTL_CMD" --user start "$SERVICE" || { note "service start failed"; return 1; }; } + +inflight_datasets_sql() { + printf "SELECT COUNT(*) FROM datasets WHERE status IN ('pending','running')" +} +inflight_runs_sql() { + printf "SELECT COUNT(*) FROM runs WHERE status IN ('queued','running')" +} +# Returns: +# 0 = no in-flight work +# 1 = in-flight work present (or cannot be ruled out) +# 2 = environment/DB error (missing DB_PATH, unreadable DB) — caller MUST +# treat this as "unknown state", never as a clean pass. This function +# NEVER exits the shell: after a service stop, a deep die() here would +# bypass the restore branch and strand the old service in downtime. +check_no_inflight() { + DB_PATH=$(env_value DB_PATH) || { note "DB_PATH missing in runtime.env"; return 2; } + DB_PATH=${DB_PATH:-} + [ -n "$DB_PATH" ] || { note "DB_PATH empty in runtime.env"; return 2; } + [ -f "$DB_PATH" ] || { note "DB file missing: $DB_PATH"; return 2; } + if ! n=$(db_scalar "$DB_PATH" "$(inflight_datasets_sql)"); then + note "read datasets failed" + return 2 + fi + [ "$n" = "0" ] || { note "in-flight datasets: $n"; return 1; } + if ! n=$(db_scalar "$DB_PATH" "$(inflight_runs_sql)"); then + note "read runs failed" + return 2 + fi + [ "$n" = "0" ] || { note "in-flight runs: $n"; return 1; } + return 0 +} + +precheck_live() { + [ -n "$IMAGE_ID" ] || die "IMAGE_ID required" + "$DOCKER_CMD" image inspect "$IMAGE_ID" >/dev/null 2>&1 || die "image not present: $IMAGE_ID" + [ "$(env_value WORKER_IMAGE)" != "$IMAGE_ID" ] || die "runtime.env already pins $IMAGE_ID" + [ -x "$SERVER_BIN_SRC" ] || die "missing server binary: $SERVER_BIN_SRC" + [ -f "$DIST_SRC/index.html" ] || { die "missing frontend dist: $DIST_SRC"; return 1; } + service_active || die "$SERVICE not active; resolve before deploy (start it or inspect)" + check_no_inflight || die "in-flight work; deploy blocked" + msg "precheck(live) OK" +} + +precheck_stopped() { + service_active && { note "service still active after stop"; return 1; } + check_no_inflight || { note "new in-flight work discovered after stop"; return 2; } + msg "precheck(stopped) OK" +} + +release_unique_backup_dir() { + rid_base=$(date -u +%Y%m%d-%H%M%S) + rid="$rid_base" + sfx=a + while [ -e "$BACKUP_ROOT/$rid" ]; do + rid="$rid_base-$sfx" + sfx=$(python3 -c "import sys,random;print(chr(ord('a')+random.randrange(26)))" 2>/dev/null || printf '%s' "$sfx") + done + RELEASE_ID=$rid + printf '%s' "$rid" +} + +backup() { + release_unique_backup_dir >/dev/null 2>&1 || die "release id generation failed" + [ -n "$RELEASE_ID" ] || die "empty RELEASE_ID" + BACKUP_DIR=$BACKUP_ROOT/$RELEASE_ID + if [ -e "$BACKUP_DIR" ]; then + die "backup would overwrite existing $BACKUP_DIR (refusing)" + fi + mkdir -p "$BACKUP_DIR" || die "mkdir backup failed" + run_verchown_mode 700 "$BACKUP_ROOT" + run_verchown_mode 700 "$BACKUP_DIR" + run_cp "$STATE_DIR/runtime.env" "$BACKUP_DIR/runtime.env" || { mv "$BACKUP_DIR" "$BACKUP_DIR.broken.$RELEASE_ID" 2>/dev/null; die "backup: runtime.env copy failed"; } + run_verchown_mode 600 "$BACKUP_DIR/runtime.env" + run_rsync -a "$RELEASE_DIR/" "$BACKUP_DIR/release/" || { mv "$BACKUP_DIR" "$BACKUP_DIR.broken.$RELEASE_ID" 2>/dev/null; die "backup: release copy failed"; } + { + printf 'release_id=%s\ncreated_at=%s\n' "$RELEASE_ID" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + printf 'image_id_to_use=%s\nprevious_worker_image=%s\nprevious_server=%s\nprevious_frontend=%s\n' \ + "$IMAGE_ID" "$(env_value WORKER_IMAGE)" \ + "$RELEASE_DIR/strategy-lab-server" "$RELEASE_DIR/frontend" + sha256sum "$RELEASE_DIR/strategy-lab-server" 2>/dev/null | awk '{print "previous_server_sha256=" $1}' || printf 'previous_server_sha256=unknown\n' + } > "$BACKUP_DIR/release-manifest.txt.tmp" || { mv "$BACKUP_DIR" "$BACKUP_DIR.broken.$RELEASE_ID" 2>/dev/null; die "backup: manifest write failed"; } + # completeness check BEFORE the final marker: restore refuses backups + # without the 'complete' marker, so never mark anything half-copied. + [ -f "$BACKUP_DIR/runtime.env" ] && [ -f "$BACKUP_DIR/release/strategy-lab-server" ] \ + || { mv "$BACKUP_DIR" "$BACKUP_DIR.broken.$RELEASE_ID" 2>/dev/null; die "backup incomplete; refusing to mark"; } + grep -q '^previous_server_sha256=' "$BACKUP_DIR/release-manifest.txt.tmp" \ + || { mv "$BACKUP_DIR" "$BACKUP_DIR.broken.$RELEASE_ID" 2>/dev/null; die "backup manifest missing previous_server_sha256"; } + + fin_stage "$BACKUP_DIR/release-manifest.txt.tmp" "$BACKUP_DIR/release-manifest.txt" \ + || { mv "$BACKUP_DIR" "$BACKUP_DIR.broken.$RELEASE_ID" 2>/dev/null; die "backup: manifest finalize failed"; } + run_verchown_mode 600 "$BACKUP_DIR/release-manifest.txt" + : > "$BACKUP_DIR/complete" || die "backup marker failed" + run_verchown_mode 600 "$BACKUP_DIR/complete" + msg "backup $RELEASE_ID -> $BACKUP_DIR (complete)" +} + +set_worker_image_id() { + rc=0 + run_sed "s|^WORKER_IMAGE=.*|WORKER_IMAGE=${IMAGE_ID}|" "$STATE_DIR/runtime.env" || rc=$? + [ "$rc" -eq 0 ] || return "$rc" + run_verchown_mode 600 "$STATE_DIR/runtime.env" || return 1 + [ "$(env_value WORKER_IMAGE)" = "$IMAGE_ID" ] || { + note "runtime.env replacement verification failed" + return 1 + } + return 0 +} + +swap_release() { + run_install "$SERVER_BIN_SRC" "$RELEASE_DIR/strategy-lab-server" || return 1 + # --checksum: dist contents must win even when size+mtime collide (drill + # regression: same-size files with identical mtime were silently skipped) + run_rsync -a --delete --checksum "$DIST_SRC/" "$RELEASE_DIR/frontend/" || return 1 + return 0 +} + +health_ok() { + BIND=$(env_value BIND) || return 1 + body=$("$CURL_CMD" -fsS -m 5 "http://$BIND/api/health" 2>/dev/null) || return 1 + printf '%s' "$body" | grep -q '"status":"ok"' || return 1 + return 0 +} + +wait_health() { + i=1 + while [ $i -le 20 ]; do + if health_ok; then msg "health OK"; return 0; fi + sleep 1 + i=$((i + 1)) + done + return 1 +} + +restore_release_id() { + rid=${1:?usage: deploy.sh restore } + bd=$BACKUP_ROOT/$rid + DEPLOY_PHASE=restore; export DEPLOY_PHASE + [ -d "$bd" ] || die "no backup for release id: $rid" + [ -f "$bd/complete" ] || die "backup incomplete (no complete marker): $bd" + [ -f "$bd/runtime.env" ] || die "backup incomplete: runtime.env" + [ -f "$bd/release/strategy-lab-server" ] || die "backup incomplete: release" + # R3: restore takes the same deploy.lock unless the caller already holds it + if [ "${DEPLOY_LOCK_HELD:-0}" != "1" ]; then + exec 9>"$STATE_DIR/deploy.lock" + flock -n 9 || die "another deploy/rollback holds deploy.lock" + fi + # --checksum: dist contents must win even when size+mtime collide + # touch on-disk artifacts only while the (old/new) binary is not running: + # stop first, then replace files, then start again and verify health. + # If stop cannot be CONFIRMED (command failed or service still active), + # refuse to write ANY file: the running binary may still be using them. + # The restore is non-destructive here — the caller can retry after + # resolving the stop problem, and all existing files remain recoverable. + if ! "$SYSTEMCTL_CMD" --user stop "$SERVICE" 2>/dev/null; then + # stop reported failure: only safe to continue if service is verifiably down + if service_active; then + die "stop failed and $SERVICE still ACTIVE — refusing to restore files (nothing was written; backup $rid intact and recoverable)" + fi + note "stop reported failure but service verified inactive; continuing" + fi + service_active && die "could not confirm $SERVICE stopped — refusing to restore files (nothing was written; backup $rid intact and recoverable)" + run_cp "$bd/runtime.env" "$STATE_DIR/runtime.env" + run_verchown_mode 600 "$STATE_DIR/runtime.env" + run_rsync -a --delete --checksum "$bd/release/" "$RELEASE_DIR/" || die "restore release failed" + service_start || die "service start after restore failed" + sleep 2 + service_active || die "service not active after restore" + wait_health || die "health failed after restore" + verify_restored "$bd" + msg "restored $rid" +} + +# verify restored artifacts hash vs manifest +verify_restored() { + bd=$1 + w=$(grep -E '^previous_worker_image=' "$bd/release-manifest.txt" | sed 's/^previous_worker_image=//') || die "manifest corrupt" + [ "$(env_value WORKER_IMAGE)" = "$w" ] || die "restored env mismatches manifest" + h=$(grep -E '^previous_server_sha256=' "$bd/release-manifest.txt" | sed 's/^previous_server_sha256=//') + now=$(sha256sum "$RELEASE_DIR/strategy-lab-server" | awk '{print $1}') + [ "$h" = "$now" ] || die "restored server hash mismatch: $now != $h" + msg "restored artifacts verified (env + sha256)" +} + +# ---------------- main deploy flow ---------------- +main_deploy() { + exec 9>"$STATE_DIR/deploy.lock" + flock -n 9 || die "another deploy/rollback is running (deploy.lock held)" + + msg "phase 1/6 precheck (live service)" + precheck_live + + # Backup BEFORE stopping: a backup failure must never turn into downtime — + # the service keeps running on untouched artifacts and the deploy simply + # refuses. + msg "phase 2/6 backup (env + release artifacts, service still up)" + DEPLOY_PHASE=backup; export DEPLOY_PHASE + backup + + msg "phase 3/6 stop service (block NEW work)" + service_stop || die "service stop failed" + # R1: capture the REAL return code of the stopped-state checks — not an + # &&-chain / if ! — so a `2` (= new work discovered) is distinguishable + # from plain failure and the caller can restore the old service untouched. + rc=0 + precheck_stopped || rc=$? + # A DB/environment error (rc=2) after stop is treated the same as the race + # branch: we cannot prove the system is quiet, so restore the old service + # untouched rather than swapping files in an unverifiable state. + if [ "$rc" -ne 0 ]; then + if [ "$rc" = "2" ]; then + "$SYSTEMCTL_CMD" --user start "$SERVICE" || note "could NOT re-start old service — MANUAL CHECK NEEDED" + note "restored old service WITHOUT replacement (post-stop check errored: DB/env unreadable)" + note "backup dir kept for inspection only: $BACKUP_ROOT/$RELEASE_ID" + exit 1 + fi + # rc=1: new work arrived between checks; the backup above is discarded (it + # recorded pre-inflight state and nothing was replaced) + "$SYSTEMCTL_CMD" --user start "$SERVICE" || note "could NOT re-start old service — MANUAL CHECK NEEDED" + note "restored old service WITHOUT replacement (new tasks preserved)" + note "backup dir kept for inspection only: $BACKUP_ROOT/$RELEASE_ID (release kept; runtime.env already restored bytes)" + exit 1 + fi + + msg "phase 4/6 swap (env WORKER_IMAGE + binary + dist)" + # R1: each step rc-checked independently (also inside an && chain) + DEPLOY_PHASE=swap; export DEPLOY_PHASE + if ! set_worker_image_id; then + note "env update failed"; DEPLOY_LOCK_HELD=1 restore_release_id "$RELEASE_ID"; exit 1 + fi + if ! swap_release; then + note "release swap failed"; DEPLOY_LOCK_HELD=1 restore_release_id "$RELEASE_ID"; exit 1 + fi + + msg "phase 5/6 start service + health" + if ! "$SYSTEMCTL_CMD" --user start "$SERVICE"; then + note "service start failed"; DEPLOY_LOCK_HELD=1 restore_release_id "$RELEASE_ID"; exit 1 + fi + sleep 2 + if ! wait_health; then + note "health failed; rolling back to $RELEASE_ID" + DEPLOY_LOCK_HELD=1 restore_release_id "$RELEASE_ID" + exit 1 + fi + + msg "phase 6/6 deploy $RELEASE_ID complete (image $IMAGE_ID)" + msg "rollback id: $RELEASE_ID (rollback.sh $RELEASE_ID)" +} + +case "${1:-deploy}" in + deploy) main_deploy ;; + restore) shift; restore_release_id "$@" ;; + precheck) precheck_live ;; + *) die "usage: deploy.sh [deploy|restore |precheck]" ;; +esac diff --git a/artifacts/etf-recovery-candidate/ops/rollback.sh b/artifacts/etf-recovery-candidate/ops/rollback.sh new file mode 100755 index 0000000..095330e --- /dev/null +++ b/artifacts/etf-recovery-candidate/ops/rollback.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# Strategy Lab 回滚 v3（供 leader 审查/执行；开发会话不操作生产）。 +# 备份位于 /release-backups//，且只在写入了 complete +# 标记后才可用；没有 complete 标记的备份一律拒绝恢复。 +# R3：取同一把 deploy.lock；顺序：in-flight 检查 → 停服务 → 替换在线文件 +# （运行中绝不写二进制/前端）→ 启动 → is-active + /api/health JSON +# status=="ok" + sha256 与 runtime.env 较验一致。 +# 用法： +# rollback.sh # 精确恢复该 id +# rollback.sh list # 只读列表（不取锁、不动任何东西） +set -eu + +SYSTEMCTL_CMD=${SYSTEMCTL_CMD:-systemctl} +CURL_CMD=${CURL_CMD:-curl} +CP_CMD=${CP_CMD:-cp} +MV_CMD=${MV_CMD:-mv} +STATE_DIR=${STATE_DIR:-/home/somhairle/.local/share/strategy-lab-production} +RELEASE_DIR=${RELEASE_DIR:-$STATE_DIR/release} +BACKUP_ROOT=${BACKUP_ROOT:-$STATE_DIR/release-backups} +SERVICE=${SERVICE:-strategy-lab-production} + +msg() { printf '[rollback] %s\n' "$1"; } +die() { printf '[rollback] FATAL: %s\n' "$1" >&2; exit 1; } + +env_value() { grep -E "^$1=" "$STATE_DIR/runtime.env" | head -1 | cut -d= -f2-; } + +db_scalar() { + if command -v sqlite3 >/dev/null 2>&1; then + sqlite3 "$1" "$2" + else + python3 - "$1" "$2" <<'PY' || die "db_scalar failed" +import sqlite3, sys +c = sqlite3.connect(sys.argv[1]) +row = c.execute(sys.argv[2]).fetchone() +print(row[0] if row is not None else 0) +PY + fi +} + +if [ "${1:-}" = "list" ]; then + for d in "$BACKUP_ROOT"/*/; do + [ -d "$d" ] || continue + img=$(grep -E '^image_id_to_use=' "$d/release-manifest.txt" 2>/dev/null | sed 's/^image_id_to_use=//') + done_mark="" + [ -f "$d/complete" ] && done_mark=" COMPLETE" + printf '%s\timage=%s%s\n' "$(basename "$d")" "$img" "$done_mark" + done + exit 0 +fi + +rid=${1:?usage: rollback.sh |list} +bd=$BACKUP_ROOT/$rid +[ -d "$bd" ] || die "no backup for release id: $rid" +[ -f "$bd/complete" ] || die "backup incomplete (no complete marker): $bd" +[ -f "$bd/runtime.env" ] || die "backup incomplete: runtime.env" +[ -f "$bd/release/strategy-lab-server" ] || die "backup incomplete: release" + +# R3: same lock as deploy +exec 9>"$STATE_DIR/deploy.lock" +flock -n 9 || die "another deploy/rollback is running (deploy.lock held)" + +# Refuse while user work is in-flight (a restore must not strand mid-run state) +DB_PATH=$(grep -E '^DB_PATH=' "$STATE_DIR/runtime.env" | head -1 | cut -s -d= -f2-) +if [ -n "${DB_PATH:-}" ]; then + n=$(db_scalar "$DB_PATH" "SELECT COUNT(*) FROM runs WHERE status IN ('queued','running')") + [ "$n" = "0" ] || die "in-flight runs: $n — cancel/finish before rollback" +fi + +old_img=$(grep -E '^previous_worker_image=' "$bd/release-manifest.txt" 2>/dev/null | sed 's/^previous_worker_image=//') || true + +# NEVER write on-disk artifacts while a binary is executing it: stop first. +"$SYSTEMCTL_CMD" --user stop "$SERVICE" || die "stop before restore failed" + +cp -p "$bd/runtime.env" "$STATE_DIR/runtime.env" || die "restore env failed" +chmod 600 "$STATE_DIR/runtime.env" +# --checksum: dist contents must win even when size+mtime collide +rsync -a --delete --checksum "$bd/release/" "$RELEASE_DIR/" || { + # files may be half-restored — bring back what the backup declares as service + "$SYSTEMCTL_CMD" --user start "$SERVICE" 2>/dev/null || true + die "restore release failed" +} + +"$SYSTEMCTL_CMD" --user start "$SERVICE" || die "service start after restore failed" +sleep 2 +if ! "$SYSTEMCTL_CMD" --user is-active "$SERVICE" >/dev/null 2>&1; then + die "service not active after restore; check journalctl --user -u $SERVICE" +fi + +# health JSON must report status ok +BIND=$(grep -E '^BIND=' "$STATE_DIR/runtime.env" | head -1 | cut -d= -f2-) +body=$("$CURL_CMD" -fsS -m 5 "http://$BIND/api/health" 2>/dev/null) || die "health unreachable" +printf '%s' "$body" | grep -q '"status":"ok"' || die "health not ok: $body" + +# Release-artifact identity: hash must equal the manifest's recorded old value +want_hash=$(grep -E '^previous_server_sha256=' "$bd/release-manifest.txt" | sed 's/^previous_server_sha256=//') +now_hash=$(sha256sum "$RELEASE_DIR/strategy-lab-server" | awk '{print $1}') +[ "$want_hash" = "$now_hash" ] || die "restored server hash mismatch: $now_hash != $want_hash" + +# env identity +[ "$(env_value WORKER_IMAGE)" = "$old_img" ] || die "restored env WORKER_IMAGE mismatch: $(env_value WORKER_IMAGE) != $old_img" + +msg "restored $rid: env verified, artifact sha256 verified, health ok." -- cgit v1.2.3