fix(gitea): 保留跨日备份并恢复中断服务

gitea/reliability/HANDOVER.md

@@ -41,4 +41,5 @@ T5 ID `52f09d35e3fd31ce2aa3d24925ef101533dcff1f0810a203a983b307686ffdb6`。
 
 本次修复把 03:30 单次定时改为 03:30 起按小时补跑：CI 占用则下一小时重试；
 外盘短暂不可用则优先复制已有本机加密快照；同一备份窗口通过后不重复停服务。
+跨日未完成的旧外置复制不阻止下一日生成新的本机快照；当日已复制但尚未发布 `last-success` 的回执会幂等补齐。
 这只能缩短临时故障造成的备份缺口。连续失败仍没有主动告警；密钥独立保管仍待用户确认。

gitea/reliability/README.md

@@ -17,7 +17,7 @@ python3 ~/ops/gitea-reliability/backup.py restore-check --external
 ```
 
 安装器仅维护三个当前用户 LaunchAgent：每天 03:30 开始备份窗口，其后每小时检查一次，直到当天外盘快照校验成功；另有 60 秒异常停机恢复守卫、每小时备份健康检查。
-若 CI 占用，下一小时重试冷拷贝；若本机加密快照已成功而外盘复制失败，先补复制同一份快照，不再次暂停 Gitea。当天已验证后，后续小时只检查状态，不重做冷拷贝。
+若 CI 占用，下一小时重试冷拷贝；若当天本机加密快照已成功而外盘复制失败，先补复制同一份快照，不再次暂停 Gitea。跨日仍未补成的旧快照不会挡住今天的新本机冷拷贝。当天已验证后，后续小时只检查状态，不重做冷拷贝。
 不加载历史 Compose 中的其它 runner。后台 Python 首次访问 T5，需要 macOS 允许“可移除宗卷”。
 必须实际从 launchd 运行验证，终端成功不能代替后台权限验收。
 日志在运行目录 `logs/`；检查 `launchctl print gui/$(id -u)/com.ljwx.gitea.backup` 的退出码。

gitea/reliability/backup.py

@@ -334,6 +334,7 @@ done < /tmp/repositories
 
     def scheduled(self, now=None):
         """Retry a verified local copy first; take at most one cold copy per backup day."""
+        self.recover_interrupted_snapshot()
         window = backup_window(now or dt.datetime.now(dt.timezone.utc))
         success_file = self.root / "last-success.json"
         previous = json.loads(success_file.read_text()) if success_file.exists() else None
@@ -342,13 +343,17 @@ done < /tmp/repositories
             record = json.loads(record_file.read_text())
             if record["created_utc"] <= previous_stamp:
                 break
-            if record.get("snapshot_id") and record.get("image_snapshot_id") and not record.get("external_verified"):
+            # A failed copy from an older window must not prevent a fresh
-                self.replicate(record_file)
+            # local snapshot today when the external disk remains unavailable.
-                if record_window(record["created_utc"]) == window:
+            if record_window(record["created_utc"]) != window:
-                    self.status()
-                    self.log("PASS: recovered the current backup window without another service pause")
-                    return
                 break
+            if record.get("snapshot_id") and record.get("image_snapshot_id"):
+                # Also repair a crash after marking the record verified but
+                # before publishing last-success and the USB receipt.
+                self.replicate(record_file)
+                self.status()
+                self.log("PASS: recovered the current backup window without another service pause")
+                return
         previous = json.loads(success_file.read_text()) if success_file.exists() else None
         if previous and previous.get("external_verified") and record_window(previous["created_utc"]) == window:
             self.status()

gitea/reliability/test_backup.py

@@ -41,6 +41,36 @@ class BackupSafetyTests(unittest.TestCase):
                 status.assert_called_once()
                 backup.assert_not_called()
 
+    def test_older_failed_external_copy_does_not_block_todays_local_backup(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            now = dt.datetime.now(dt.timezone.utc)
+            prior = (now - dt.timedelta(days=2)).strftime("%Y%m%dT%H%M%SZ")
+            stale = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ")
+            (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True}))
+            (root / f"backup-{stale}.json").write_text(json.dumps({"created_utc": stale, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": False}))
+            with patch.object(obj, "replicate", side_effect=RuntimeError("USB unavailable")) as replicate, patch.object(obj, "backup") as backup:
+                obj.scheduled(now)
+                replicate.assert_not_called()
+                backup.assert_called_once()
+
+    def test_current_verified_record_without_published_receipt_avoids_new_outage(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            now = dt.datetime.now(dt.timezone.utc)
+            prior = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ")
+            current = now.strftime("%Y%m%dT%H%M%SZ")
+            (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True}))
+            pending = root / f"backup-{current}.json"
+            pending.write_text(json.dumps({"created_utc": current, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": True}))
+            with patch.object(obj, "replicate") as replicate, patch.object(obj, "status") as status, patch.object(obj, "backup") as backup:
+                obj.scheduled(now)
+                replicate.assert_called_once_with(pending)
+                status.assert_called_once()
+                backup.assert_not_called()
+
     def test_scheduled_backup_retries_after_busy_ci(self):
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)
@@ -52,6 +82,23 @@ class BackupSafetyTests(unittest.TestCase):
                 obj.scheduled(now + dt.timedelta(hours=1))
                 self.assertEqual(backup.call_count, 2)
 
+    def test_scheduled_retry_recovers_interrupted_service_before_external_copy(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            obj = Backup({"state_dir": directory, "restic": "unused", "password_file": str(root / "key"), "local_repository": str(root / "repo")})
+            now = dt.datetime.now(dt.timezone.utc)
+            prior = (now - dt.timedelta(days=1)).strftime("%Y%m%dT%H%M%SZ")
+            current = now.strftime("%Y%m%dT%H%M%SZ")
+            (root / "last-success.json").write_text(json.dumps({"created_utc": prior, "external_verified": True}))
+            pending = root / f"backup-{current}.json"
+            pending.write_text(json.dumps({"created_utc": current, "snapshot_id": "data", "image_snapshot_id": "image", "external_verified": False}))
+            (root / "maintenance.json").write_text(json.dumps({"container": "gitea"}))
+            with patch.object(obj, "start_and_wait") as start, patch.object(obj, "replicate") as replicate, patch.object(obj, "status"):
+                obj.scheduled(now)
+                start.assert_called_once()
+                replicate.assert_called_once_with(pending)
+            self.assertFalse((root / "maintenance.json").exists())
+
     def test_actual_compose_volume_is_used(self):
         self.assertEqual(data_volume({"Mounts": [{"Destination": "/data", "Type": "volume", "Name": "gitea_gitea-data"}]}), "gitea_gitea-data")