summaryrefslogtreecommitdiff
path: root/artifacts/etf-recovery-candidate/ops/rollback.sh
diff options
context:
space:
mode:
Diffstat (limited to 'artifacts/etf-recovery-candidate/ops/rollback.sh')
-rwxr-xr-xartifacts/etf-recovery-candidate/ops/rollback.sh102
1 files changed, 102 insertions, 0 deletions
diff --git a/artifacts/etf-recovery-candidate/ops/rollback.sh b/artifacts/etf-recovery-candidate/ops/rollback.sh
new file mode 100755
index 0000000..095330e
--- /dev/null
+++ b/artifacts/etf-recovery-candidate/ops/rollback.sh
@@ -0,0 +1,102 @@
+#!/bin/sh
+# Strategy Lab 回滚 v3(供 leader 审查/执行;开发会话不操作生产)。
+# 备份位于 <STATE_DIR>/release-backups/<RELEASE_ID>/,且只在写入了 complete
+# 标记后才可用;没有 complete 标记的备份一律拒绝恢复。
+# R3:取同一把 deploy.lock;顺序:in-flight 检查 → 停服务 → 替换在线文件
+# (运行中绝不写二进制/前端)→ 启动 → is-active + /api/health JSON
+# status=="ok" + sha256 与 runtime.env 较验一致。
+# 用法:
+# rollback.sh <RELEASE_ID> # 精确恢复该 id
+# rollback.sh list # 只读列表(不取锁、不动任何东西)
+set -eu
+
+SYSTEMCTL_CMD=${SYSTEMCTL_CMD:-systemctl}
+CURL_CMD=${CURL_CMD:-curl}
+CP_CMD=${CP_CMD:-cp}
+MV_CMD=${MV_CMD:-mv}
+STATE_DIR=${STATE_DIR:-/home/somhairle/.local/share/strategy-lab-production}
+RELEASE_DIR=${RELEASE_DIR:-$STATE_DIR/release}
+BACKUP_ROOT=${BACKUP_ROOT:-$STATE_DIR/release-backups}
+SERVICE=${SERVICE:-strategy-lab-production}
+
+msg() { printf '[rollback] %s\n' "$1"; }
+die() { printf '[rollback] FATAL: %s\n' "$1" >&2; exit 1; }
+
+env_value() { grep -E "^$1=" "$STATE_DIR/runtime.env" | head -1 | cut -d= -f2-; }
+
+db_scalar() {
+ if command -v sqlite3 >/dev/null 2>&1; then
+ sqlite3 "$1" "$2"
+ else
+ python3 - "$1" "$2" <<'PY' || die "db_scalar failed"
+import sqlite3, sys
+c = sqlite3.connect(sys.argv[1])
+row = c.execute(sys.argv[2]).fetchone()
+print(row[0] if row is not None else 0)
+PY
+ fi
+}
+
+if [ "${1:-}" = "list" ]; then
+ for d in "$BACKUP_ROOT"/*/; do
+ [ -d "$d" ] || continue
+ img=$(grep -E '^image_id_to_use=' "$d/release-manifest.txt" 2>/dev/null | sed 's/^image_id_to_use=//')
+ done_mark=""
+ [ -f "$d/complete" ] && done_mark=" COMPLETE"
+ printf '%s\timage=%s%s\n' "$(basename "$d")" "$img" "$done_mark"
+ done
+ exit 0
+fi
+
+rid=${1:?usage: rollback.sh <RELEASE_ID>|list}
+bd=$BACKUP_ROOT/$rid
+[ -d "$bd" ] || die "no backup for release id: $rid"
+[ -f "$bd/complete" ] || die "backup incomplete (no complete marker): $bd"
+[ -f "$bd/runtime.env" ] || die "backup incomplete: runtime.env"
+[ -f "$bd/release/strategy-lab-server" ] || die "backup incomplete: release"
+
+# R3: same lock as deploy
+exec 9>"$STATE_DIR/deploy.lock"
+flock -n 9 || die "another deploy/rollback is running (deploy.lock held)"
+
+# Refuse while user work is in-flight (a restore must not strand mid-run state)
+DB_PATH=$(grep -E '^DB_PATH=' "$STATE_DIR/runtime.env" | head -1 | cut -s -d= -f2-)
+if [ -n "${DB_PATH:-}" ]; then
+ n=$(db_scalar "$DB_PATH" "SELECT COUNT(*) FROM runs WHERE status IN ('queued','running')")
+ [ "$n" = "0" ] || die "in-flight runs: $n — cancel/finish before rollback"
+fi
+
+old_img=$(grep -E '^previous_worker_image=' "$bd/release-manifest.txt" 2>/dev/null | sed 's/^previous_worker_image=//') || true
+
+# NEVER write on-disk artifacts while a binary is executing it: stop first.
+"$SYSTEMCTL_CMD" --user stop "$SERVICE" || die "stop before restore failed"
+
+cp -p "$bd/runtime.env" "$STATE_DIR/runtime.env" || die "restore env failed"
+chmod 600 "$STATE_DIR/runtime.env"
+# --checksum: dist contents must win even when size+mtime collide
+rsync -a --delete --checksum "$bd/release/" "$RELEASE_DIR/" || {
+ # files may be half-restored — bring back what the backup declares as service
+ "$SYSTEMCTL_CMD" --user start "$SERVICE" 2>/dev/null || true
+ die "restore release failed"
+}
+
+"$SYSTEMCTL_CMD" --user start "$SERVICE" || die "service start after restore failed"
+sleep 2
+if ! "$SYSTEMCTL_CMD" --user is-active "$SERVICE" >/dev/null 2>&1; then
+ die "service not active after restore; check journalctl --user -u $SERVICE"
+fi
+
+# health JSON must report status ok
+BIND=$(grep -E '^BIND=' "$STATE_DIR/runtime.env" | head -1 | cut -d= -f2-)
+body=$("$CURL_CMD" -fsS -m 5 "http://$BIND/api/health" 2>/dev/null) || die "health unreachable"
+printf '%s' "$body" | grep -q '"status":"ok"' || die "health not ok: $body"
+
+# Release-artifact identity: hash must equal the manifest's recorded old value
+want_hash=$(grep -E '^previous_server_sha256=' "$bd/release-manifest.txt" | sed 's/^previous_server_sha256=//')
+now_hash=$(sha256sum "$RELEASE_DIR/strategy-lab-server" | awk '{print $1}')
+[ "$want_hash" = "$now_hash" ] || die "restored server hash mismatch: $now_hash != $want_hash"
+
+# env identity
+[ "$(env_value WORKER_IMAGE)" = "$old_img" ] || die "restored env WORKER_IMAGE mismatch: $(env_value WORKER_IMAGE) != $old_img"
+
+msg "restored $rid: env verified, artifact sha256 verified, health ok."