blob: 095330e3b1d62f26d8b20950f6b1e2cfd1127e35 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
#!/bin/sh
# Strategy Lab 回滚 v3（供 leader 审查/执行；开发会话不操作生产）。
# 备份位于 <STATE_DIR>/release-backups/<RELEASE_ID>/，且只在写入了 complete
# 标记后才可用；没有 complete 标记的备份一律拒绝恢复。
# R3：取同一把 deploy.lock；顺序：in-flight 检查 → 停服务 → 替换在线文件
# （运行中绝不写二进制/前端）→ 启动 → is-active + /api/health JSON
# status=="ok" + sha256 与 runtime.env 较验一致。
# 用法：
# rollback.sh <RELEASE_ID> # 精确恢复该 id
# rollback.sh list # 只读列表（不取锁、不动任何东西）
set -eu
SYSTEMCTL_CMD=${SYSTEMCTL_CMD:-systemctl}
CURL_CMD=${CURL_CMD:-curl}
CP_CMD=${CP_CMD:-cp}
MV_CMD=${MV_CMD:-mv}
STATE_DIR=${STATE_DIR:-/home/somhairle/.local/share/strategy-lab-production}
RELEASE_DIR=${RELEASE_DIR:-$STATE_DIR/release}
BACKUP_ROOT=${BACKUP_ROOT:-$STATE_DIR/release-backups}
SERVICE=${SERVICE:-strategy-lab-production}
msg() { printf '[rollback] %s\n' "$1"; }
die() { printf '[rollback] FATAL: %s\n' "$1" >&2; exit 1; }
env_value() { grep -E "^$1=" "$STATE_DIR/runtime.env" | head -1 | cut -d= -f2-; }
db_scalar() {
if command -v sqlite3 >/dev/null 2>&1; then
sqlite3 "$1" "$2"
else
python3 - "$1" "$2" <<'PY' || die "db_scalar failed"
import sqlite3, sys
c = sqlite3.connect(sys.argv[1])
row = c.execute(sys.argv[2]).fetchone()
print(row[0] if row is not None else 0)
PY
fi
}
if [ "${1:-}" = "list" ]; then
for d in "$BACKUP_ROOT"/*/; do
[ -d "$d" ] || continue
img=$(grep -E '^image_id_to_use=' "$d/release-manifest.txt" 2>/dev/null | sed 's/^image_id_to_use=//')
done_mark=""
[ -f "$d/complete" ] && done_mark=" COMPLETE"
printf '%s\timage=%s%s\n' "$(basename "$d")" "$img" "$done_mark"
done
exit 0
fi
rid=${1:?usage: rollback.sh <RELEASE_ID>|list}
bd=$BACKUP_ROOT/$rid
[ -d "$bd" ] || die "no backup for release id: $rid"
[ -f "$bd/complete" ] || die "backup incomplete (no complete marker): $bd"
[ -f "$bd/runtime.env" ] || die "backup incomplete: runtime.env"
[ -f "$bd/release/strategy-lab-server" ] || die "backup incomplete: release"
# R3: same lock as deploy
exec 9>"$STATE_DIR/deploy.lock"
flock -n 9 || die "another deploy/rollback is running (deploy.lock held)"
# Refuse while user work is in-flight (a restore must not strand mid-run state)
DB_PATH=$(grep -E '^DB_PATH=' "$STATE_DIR/runtime.env" | head -1 | cut -s -d= -f2-)
if [ -n "${DB_PATH:-}" ]; then
n=$(db_scalar "$DB_PATH" "SELECT COUNT(*) FROM runs WHERE status IN ('queued','running')")
[ "$n" = "0" ] || die "in-flight runs: $n — cancel/finish before rollback"
fi
old_img=$(grep -E '^previous_worker_image=' "$bd/release-manifest.txt" 2>/dev/null | sed 's/^previous_worker_image=//') || true
# NEVER write on-disk artifacts while a binary is executing it: stop first.
"$SYSTEMCTL_CMD" --user stop "$SERVICE" || die "stop before restore failed"
cp -p "$bd/runtime.env" "$STATE_DIR/runtime.env" || die "restore env failed"
chmod 600 "$STATE_DIR/runtime.env"
# --checksum: dist contents must win even when size+mtime collide
rsync -a --delete --checksum "$bd/release/" "$RELEASE_DIR/" || {
# files may be half-restored — bring back what the backup declares as service
"$SYSTEMCTL_CMD" --user start "$SERVICE" 2>/dev/null || true
die "restore release failed"
}
"$SYSTEMCTL_CMD" --user start "$SERVICE" || die "service start after restore failed"
sleep 2
if ! "$SYSTEMCTL_CMD" --user is-active "$SERVICE" >/dev/null 2>&1; then
die "service not active after restore; check journalctl --user -u $SERVICE"
fi
# health JSON must report status ok
BIND=$(grep -E '^BIND=' "$STATE_DIR/runtime.env" | head -1 | cut -d= -f2-)
body=$("$CURL_CMD" -fsS -m 5 "http://$BIND/api/health" 2>/dev/null) || die "health unreachable"
printf '%s' "$body" | grep -q '"status":"ok"' || die "health not ok: $body"
# Release-artifact identity: hash must equal the manifest's recorded old value
want_hash=$(grep -E '^previous_server_sha256=' "$bd/release-manifest.txt" | sed 's/^previous_server_sha256=//')
now_hash=$(sha256sum "$RELEASE_DIR/strategy-lab-server" | awk '{print $1}')
[ "$want_hash" = "$now_hash" ] || die "restored server hash mismatch: $now_hash != $want_hash"
# env identity
[ "$(env_value WORKER_IMAGE)" = "$old_img" ] || die "restored env WORKER_IMAGE mismatch: $(env_value WORKER_IMAGE) != $old_img"
msg "restored $rid: env verified, artifact sha256 verified, health ok."
|
