From 80480b7ce29477c35bba5d74537b7f013969608b Mon Sep 17 00:00:00 2001 From: gao Date: Sat, 26 Sep 2026 07:50:00 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(gitea):=20=E5=A4=87=E4=BB=BD=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E5=90=8E=E6=8C=89=E5=B0=8F=E6=97=B6=E8=A1=A5=E8=B7=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gitea/reliability/HANDOVER.md | 12 ++++++++ gitea/reliability/README.md | 4 ++- gitea/reliability/backup.py | 38 +++++++++++++++++++++++++- gitea/reliability/install.py | 4 +-- gitea/reliability/test_backup.py | 47 +++++++++++++++++++++++++++++++- 5 files changed, 100 insertions(+), 5 deletions(-) diff --git a/gitea/reliability/HANDOVER.md b/gitea/reliability/HANDOVER.md index 6374e3e..1ed87f0 100644 --- a/gitea/reliability/HANDOVER.md +++ b/gitea/reliability/HANDOVER.md @@ -30,3 +30,15 @@ 正式项目逐个接管时仍须迁移 secrets、workflow、部署凭据与 status contexts，并验收一次真实发布和回退。 当前只验证私有试点与平台运维，不能将其描述为所有 GitHub 部署已经迁移。 + +## 2026-09-26 备份补跑 + +03:30 的 Gitea 冷拷贝与本机 restic 加密校验成功，外置设备检查超时，所以定时任务退出码 1； +07:44 使用同一份本机快照补复制到 T5，外置全量 read-data 校验通过，无需再次停止服务。 +新快照本机 ID `cc2940aca52a5c759c40742e0af3c138609512eb1d968887f6d7a4f19b2c04f3`， +T5 ID `52f09d35e3fd31ce2aa3d24925ef101533dcff1f0810a203a983b307686ffdb6`。 +后台 launchd 状态审计于 07:45 再次通过，之前的外盘查询超时具有间歇性。 + +本次修复把 03:30 单次定时改为 03:30 起按小时补跑：CI 占用则下一小时重试； +外盘短暂不可用则优先复制已有本机加密快照；同一备份窗口通过后不重复停服务。 +这只能缩短临时故障造成的备份缺口。连续失败仍没有主动告警；密钥独立保管仍待用户确认。 diff --git a/gitea/reliability/README.md b/gitea/reliability/README.md index 85679b4..77dfa91 100644 --- a/gitea/reliability/README.md +++ b/gitea/reliability/README.md @@ -16,7 +16,8 @@ python3 ~/ops/gitea-reliability/backup.py backup python3 ~/ops/gitea-reliability/backup.py restore-check --external ``` -安装器仅维护三个当前用户 LaunchAgent：每天 03:30 一次备份、60 秒异常停机恢复守卫、每小时备份健康检查。 +安装器仅维护三个当前用户 LaunchAgent：每天 03:30 开始备份窗口，其后每小时检查一次，直到当天外盘快照校验成功；另有 60 秒异常停机恢复守卫、每小时备份健康检查。 +若 CI 占用，下一小时重试冷拷贝；若本机加密快照已成功而外盘复制失败，先补复制同一份快照，不再次暂停 Gitea。当天已验证后，后续小时只检查状态，不重做冷拷贝。 不加载历史 Compose 中的其它 runner。后台 Python 首次访问 T5，需要 macOS 允许“可移除宗卷”。 必须实际从 launchd 运行验证，终端成功不能代替后台权限验收。 日志在运行目录 `logs/`；检查 `launchctl print gui/$(id -u)/com.ljwx.gitea.backup` 的退出码。 @@ -24,6 +25,7 @@ python3 ~/ops/gitea-reliability/backup.py restore-check --external `status` 检查 API 健康、外置设备 UUID/真实挂载点、最近 36 小时成功回执、数据及镜像快照实际存在。 失败日志不会输出完整环境、命令 stderr 或凭据。它不发送邮件或聊天通知；应将非零退出接到已有监控。 +自动重试仍依赖用户登录、OrbStack 和 T5 在线；连续失败只记录在本机日志，没有主动通知。 ## 备份与恢复承诺 diff --git a/gitea/reliability/backup.py b/gitea/reliability/backup.py index f3d4183..3f977fd 100644 --- a/gitea/reliability/backup.py +++ b/gitea/reliability/backup.py @@ -85,6 +85,16 @@ def parse_counts(output): return dict(zip(keys, values)) +def backup_window(timestamp): + """The local backup day starts at 03:30, independent of run duration.""" + return (timestamp.astimezone() - dt.timedelta(hours=3, minutes=30)).date() + + +def record_window(stamp): + created = dt.datetime.strptime(stamp, "%Y%m%dT%H%M%SZ").replace(tzinfo=dt.timezone.utc) + return backup_window(created) + + class Backup: def __init__(self, config): self.c = config @@ -322,6 +332,30 @@ done < /tmp/repositories raise RuntimeError("Local backup succeeded; external copy FAILED") return self.replicate(record_file) + def scheduled(self, now=None): + """Retry a verified local copy first; take at most one cold copy per backup day.""" + window = backup_window(now or dt.datetime.now(dt.timezone.utc)) + success_file = self.root / "last-success.json" + previous = json.loads(success_file.read_text()) if success_file.exists() else None + previous_stamp = previous["created_utc"] if previous else "" + for record_file in sorted(self.root.glob("backup-*.json"), reverse=True): + record = json.loads(record_file.read_text()) + if record["created_utc"] <= previous_stamp: + break + if record.get("snapshot_id") and record.get("image_snapshot_id") and not record.get("external_verified"): + self.replicate(record_file) + if record_window(record["created_utc"]) == window: + self.status() + self.log("PASS: recovered the current backup window without another service pause") + return + break + previous = json.loads(success_file.read_text()) if success_file.exists() else None + if previous and previous.get("external_verified") and record_window(previous["created_utc"]) == window: + self.status() + self.log("PASS: current backup window already verified") + return + self.backup() + def replicate(self, record_file): record = json.loads(Path(record_file).read_text()) external = self.check_external() @@ -429,7 +463,7 @@ done < /tmp/repositories def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", default=str(Path.home() / "ops/gitea-reliability/config.json")) - parser.add_argument("command", choices=["backup", "replicate", "restore-check", "status", "guard"]) + parser.add_argument("command", choices=["backup", "scheduled", "replicate", "restore-check", "status", "guard"]) parser.add_argument("--record") parser.add_argument("--external", action="store_true") args = parser.parse_args() @@ -449,6 +483,8 @@ def main(): obj.health() elif args.command == "backup": obj.backup() + elif args.command == "scheduled": + obj.scheduled() elif args.command == "replicate": if not args.record: raise RuntimeError("replicate requires --record") diff --git a/gitea/reliability/install.py b/gitea/reliability/install.py index 068ffbb..684974d 100644 --- a/gitea/reliability/install.py +++ b/gitea/reliability/install.py @@ -37,7 +37,7 @@ def main(): agents.mkdir(parents=True, exist_ok=True) domain = f"gui/{os.getuid()}" jobs = { - "com.ljwx.gitea.backup": ("backup", {"StartCalendarInterval":{"Hour":3,"Minute":30}}), + "com.ljwx.gitea.backup": ("scheduled", {"StartCalendarInterval":[{"Hour":hour,"Minute":30} for hour in range(24)]}), "com.ljwx.gitea.watchdog": ("guard", {"StartInterval":60,"RunAtLoad":True}), "com.ljwx.gitea.backup-audit": ("status", {"StartInterval":3600,"RunAtLoad":True}), } @@ -56,7 +56,7 @@ def main(): plist.chmod(0o644) subprocess.run(["plutil", "-lint", str(plist)], check=True) subprocess.run(["launchctl", "bootstrap", domain, str(plist)], check=True) - print("Installed nightly 03:30 backup, 60s recovery guard, hourly backup audit (login session required)") + print("Installed 03:30 backup with hourly retries, 60s recovery guard, hourly backup audit (login session required)") if __name__ == "__main__": diff --git a/gitea/reliability/test_backup.py b/gitea/reliability/test_backup.py index 9c41bd5..79ac539 100644 --- a/gitea/reliability/test_backup.py +++ b/gitea/reliability/test_backup.py @@ -3,10 +3,55 @@ import tempfile from pathlib import Path from unittest.mock import patch import json -from backup import Backup, data_volume, external_disk, external_repository, parse_counts +import datetime as dt +from backup import Backup, backup_window, data_volume, external_disk, external_repository, parse_counts class BackupSafetyTests(unittest.TestCase): + def test_scheduled_backup_uses_local_0330_window(self): + zone = dt.datetime.now().astimezone().tzinfo + start = dt.datetime(2026, 9, 26, 3, 30, tzinfo=zone) + self.assertNotEqual(backup_window(start - dt.timedelta(seconds=1)), backup_window(start)) + self.assertEqual(backup_window(start), backup_window(start + dt.timedelta(hours=20))) + + def test_scheduled_backup_does_not_repeat_a_verified_cold_copy(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")}) + now = dt.datetime.now(dt.timezone.utc) + (root / "last-success.json").write_text(json.dumps({"created_utc": now.strftime("%Y%m%dT%H%M%SZ"), "external_verified": True})) + with patch.object(obj, "status") as status, patch.object(obj, "backup") as backup: + obj.scheduled(now) + status.assert_called_once() + backup.assert_not_called() + + def test_scheduled_backup_retries_external_copy_without_service_pause(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")}) + now = dt.datetime.now(dt.timezone.utc) + prior = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ") + current = now.strftime("%Y%m%dT%H%M%SZ") + (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True})) + pending = root / f"backup-{current}.json" + pending.write_text(json.dumps({"created_utc": current, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": False})) + with patch.object(obj, "replicate") as replicate, patch.object(obj, "status") as status, patch.object(obj, "backup") as backup: + obj.scheduled(now) + replicate.assert_called_once_with(pending) + status.assert_called_once() + backup.assert_not_called() + + def test_scheduled_backup_retries_after_busy_ci(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")}) + now = dt.datetime.now(dt.timezone.utc) + with patch.object(obj, "backup", side_effect=[RuntimeError("CI running"), None]) as backup: + with self.assertRaisesRegex(RuntimeError, "CI running"): + obj.scheduled(now) + obj.scheduled(now + dt.timedelta(hours=1)) + self.assertEqual(backup.call_count, 2) + def test_actual_compose_volume_is_used(self): self.assertEqual(data_volume({"Mounts": [{"Destination": "/data", "Type": "volume", "Name": "gitea_gitea-data"}]}), "gitea_gitea-data") From d8bb18a0916fae5374125a7999125273f7bcdba6 Mon Sep 17 00:00:00 2001 From: gao Date: Sat, 26 Sep 2026 07:53:38 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix(gitea):=20=E4=BF=9D=E7=95=99=E8=B7=A8?= =?UTF-8?q?=E6=97=A5=E5=A4=87=E4=BB=BD=E5=B9=B6=E6=81=A2=E5=A4=8D=E4=B8=AD?= =?UTF-8?q?=E6=96=AD=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gitea/reliability/HANDOVER.md | 1 + gitea/reliability/README.md | 2 +- gitea/reliability/backup.py | 17 ++++++++---- gitea/reliability/test_backup.py | 47 ++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/gitea/reliability/HANDOVER.md b/gitea/reliability/HANDOVER.md index 1ed87f0..6ff8b0c 100644 --- a/gitea/reliability/HANDOVER.md +++ b/gitea/reliability/HANDOVER.md @@ -41,4 +41,5 @@ T5 ID `52f09d35e3fd31ce2aa3d24925ef101533dcff1f0810a203a983b307686ffdb6`。 本次修复把 03:30 单次定时改为 03:30 起按小时补跑：CI 占用则下一小时重试； 外盘短暂不可用则优先复制已有本机加密快照；同一备份窗口通过后不重复停服务。 +跨日未完成的旧外置复制不阻止下一日生成新的本机快照；当日已复制但尚未发布 `last-success` 的回执会幂等补齐。 这只能缩短临时故障造成的备份缺口。连续失败仍没有主动告警；密钥独立保管仍待用户确认。 diff --git a/gitea/reliability/README.md b/gitea/reliability/README.md index 77dfa91..cdf4f73 100644 --- a/gitea/reliability/README.md +++ b/gitea/reliability/README.md @@ -17,7 +17,7 @@ python3 ~/ops/gitea-reliability/backup.py restore-check --external ``` 安装器仅维护三个当前用户 LaunchAgent：每天 03:30 开始备份窗口，其后每小时检查一次，直到当天外盘快照校验成功；另有 60 秒异常停机恢复守卫、每小时备份健康检查。 -若 CI 占用，下一小时重试冷拷贝；若本机加密快照已成功而外盘复制失败，先补复制同一份快照，不再次暂停 Gitea。当天已验证后，后续小时只检查状态，不重做冷拷贝。 +若 CI 占用，下一小时重试冷拷贝；若当天本机加密快照已成功而外盘复制失败，先补复制同一份快照，不再次暂停 Gitea。跨日仍未补成的旧快照不会挡住今天的新本机冷拷贝。当天已验证后，后续小时只检查状态，不重做冷拷贝。 不加载历史 Compose 中的其它 runner。后台 Python 首次访问 T5，需要 macOS 允许“可移除宗卷”。 必须实际从 launchd 运行验证，终端成功不能代替后台权限验收。 日志在运行目录 `logs/`；检查 `launchctl print gui/$(id -u)/com.ljwx.gitea.backup` 的退出码。 diff --git a/gitea/reliability/backup.py b/gitea/reliability/backup.py index 3f977fd..9a126d3 100644 --- a/gitea/reliability/backup.py +++ b/gitea/reliability/backup.py @@ -334,6 +334,7 @@ done < /tmp/repositories def scheduled(self, now=None): """Retry a verified local copy first; take at most one cold copy per backup day.""" + self.recover_interrupted_snapshot() window = backup_window(now or dt.datetime.now(dt.timezone.utc)) success_file = self.root / "last-success.json" previous = json.loads(success_file.read_text()) if success_file.exists() else None @@ -342,13 +343,17 @@ done < /tmp/repositories record = json.loads(record_file.read_text()) if record["created_utc"] <= previous_stamp: break - if record.get("snapshot_id") and record.get("image_snapshot_id") and not record.get("external_verified"): - self.replicate(record_file) - if record_window(record["created_utc"]) == window: - self.status() - self.log("PASS: recovered the current backup window without another service pause") - return + # A failed copy from an older window must not prevent a fresh + # local snapshot today when the external disk remains unavailable. + if record_window(record["created_utc"]) != window: break + if record.get("snapshot_id") and record.get("image_snapshot_id"): + # Also repair a crash after marking the record verified but + # before publishing last-success and the USB receipt. + self.replicate(record_file) + self.status() + self.log("PASS: recovered the current backup window without another service pause") + return previous = json.loads(success_file.read_text()) if success_file.exists() else None if previous and previous.get("external_verified") and record_window(previous["created_utc"]) == window: self.status() diff --git a/gitea/reliability/test_backup.py b/gitea/reliability/test_backup.py index 79ac539..b6169ae 100644 --- a/gitea/reliability/test_backup.py +++ b/gitea/reliability/test_backup.py @@ -41,6 +41,36 @@ class BackupSafetyTests(unittest.TestCase): status.assert_called_once() backup.assert_not_called() + def test_older_failed_external_copy_does_not_block_todays_local_backup(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")}) + now = dt.datetime.now(dt.timezone.utc) + prior = (now - dt.timedelta(days=2)).strftime("%Y%m%dT%H%M%SZ") + stale = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ") + (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True})) + (root / f"backup-{stale}.json").write_text(json.dumps({"created_utc": stale, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": False})) + with patch.object(obj, "replicate", side_effect=RuntimeError("USB unavailable")) as replicate, patch.object(obj, "backup") as backup: + obj.scheduled(now) + replicate.assert_not_called() + backup.assert_called_once() + + def test_current_verified_record_without_published_receipt_avoids_new_outage(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")}) + now = dt.datetime.now(dt.timezone.utc) + prior = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ") + current = now.strftime("%Y%m%dT%H%M%SZ") + (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True})) + pending = root / f"backup-{current}.json" + pending.write_text(json.dumps({"created_utc": current, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": True})) + with patch.object(obj, "replicate") as replicate, patch.object(obj, "status") as status, patch.object(obj, "backup") as backup: + obj.scheduled(now) + replicate.assert_called_once_with(pending) + status.assert_called_once() + backup.assert_not_called() + def test_scheduled_backup_retries_after_busy_ci(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -52,6 +82,23 @@ class BackupSafetyTests(unittest.TestCase): obj.scheduled(now + dt.timedelta(hours=1)) self.assertEqual(backup.call_count, 2) + def test_scheduled_retry_recovers_interrupted_service_before_external_copy(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")}) + now = dt.datetime.now(dt.timezone.utc) + prior = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ") + current = now.strftime("%Y%m%dT%H%M%SZ") + (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True})) + pending = root / f"backup-{current}.json" + pending.write_text(json.dumps({"created_utc": current, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": False})) + (root / "maintenance.json").write_text(json.dumps({"container": "gitea"})) + with patch.object(obj, "start_and_wait") as start, patch.object(obj, "replicate") as replicate, patch.object(obj, "status"): + obj.scheduled(now) + start.assert_called_once() + replicate.assert_called_once_with(pending) + self.assertFalse((root / "maintenance.json").exists()) + def test_actual_compose_volume_is_used(self): self.assertEqual(data_volume({"Mounts": [{"Destination": "/data", "Type": "volume", "Name": "gitea_gitea-data"}]}), "gitea_gitea-data")