gao/ljwx-deploy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(gitops): harden promotion pipeline and evidence flow

Browse Source
This commit is contained in:
deploy-promoter[bot]
2026-03-05 05:37:42 +01:00
parent cdcca5b9d8
commit 552a899a6d
37 changed files with 962 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-promoter-image.yml
+1 -2
View File
@@ -4,8 +4,7 @@ on:
push:
branches-ignore: [gh-pages]
paths:
- scripts/promoter/promote.py
- scripts/promoter/Dockerfile
- scripts/**
- .github/workflows/build-promoter-image.yml
workflow_dispatch:
.github/workflows/deploy-repo-gate.yml
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
import json, pathlib, sys
allowed = (
'envs/', 'release/'
'release/',
)
changed = [line.strip() for line in pathlib.Path('.changed-files.txt').read_text(encoding='utf-8').splitlines() if line.strip()]
event = json.loads(pathlib.Path('${{ github.event_path }}').read_text(encoding='utf-8'))
.github/workflows/monthly-rollback-drill.yml
+32
View File
@@ -0,0 +1,32 @@
name: monthly-rollback-drill
on:
schedule:
# Monthly on day 1, 18:00 UTC
- cron: "0 18 1 * *"
workflow_dispatch:
permissions:
contents: read
jobs:
drill:
if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Setup uv
uses: astral-sh/setup-uv@v4
- name: Run rollback drill (dry-run only)
run: bash scripts/ops/run_rollback_drill.sh
.github/workflows/nightly-evidence.yml
+4 -2
View File
@@ -39,14 +39,16 @@ jobs:
run: |
uvx --with pyyaml --with jsonschema python scripts/evidence/validate.py
uvx --with pyyaml --with jsonschema python scripts/evidence/collect.py --out evidence/index.json --summary evidence/summary/latest.md
uvx --with pyyaml python scripts/promoter/queue_metrics.py --queue release/queue.yaml --out evidence/metrics/queue-health.json
- name: Build publish payload (atomic staging)
run: |
rm -rf .publish-tmp
mkdir -p .publish-tmp/evidence/summary
mkdir -p .publish-tmp/evidence/summary .publish-tmp/evidence/metrics
cp pages/index.html pages/app.js .publish-tmp/
cp evidence/index.json .publish-tmp/evidence/index.json
cp evidence/summary/latest.md .publish-tmp/evidence/summary/latest.md
cp evidence/metrics/queue-health.json .publish-tmp/evidence/metrics/queue-health.json
python3 - <<'PY'
import json, pathlib
feed = pathlib.Path('.publish-tmp/evidence/index.json')
@@ -87,7 +89,7 @@ jobs:
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git add -A
extra_tracked="$(git ls-files | grep -Ev '^(index\.html|app\.js|evidence/index\.json|evidence/summary/latest\.md)$' || true)"
extra_tracked="$(git ls-files | grep -Ev '^(index\.html|app\.js|evidence/index\.json|evidence/summary/latest\.md|evidence/metrics/queue-health\.json)$' || true)"
if [ -n "${extra_tracked}" ]; then
echo "Unexpected tracked files in gh-pages branch:"
echo "${extra_tracked}"
README.md
+1
View File
@@ -7,6 +7,7 @@ GitOps 配置仓库，使用 ArgoCD 管理 Kubernetes 集群。
- 运行验证：`bash scripts/verify.sh`
- Harbor 复制：`docs/harbor-replication-ghcr.md`
- Promoter/Argo/Smoke 运维：`docs/ops-runbook.md`
- 回滚演练：`docs/rollback-drill.md`
- Grafana 可观测看板 + TLS（GitOps）：`apps/ljwx-platform-observability/`
- 双 k3s 同代码部署规范：`docs/dual-k3s-deployment.md`
- CronJob secret 示例：`cluster/deploy-promoter-secret.example.yaml`
apps/agent-platform/.gitignore
+3
View File
@@ -0,0 +1,3 @@
base/secret.yaml
base/secret.local.yaml
base/secret.*.local.yaml
apps/agent-platform/base/README.md
+9
View File
@@ -0,0 +1,9 @@
# agent-platform secrets
`secret.example.yaml` in this directory is a template manifest and must not contain real credentials.
Production practice:
1. Manage secrets via secret manager / encrypted secret workflow.
2. Inject runtime values at deploy time.
3. Do not commit concrete token/password/host values.
apps/agent-platform/base/secret.example.yaml
+18
View File
@@ -0,0 +1,18 @@
apiVersion: v1
kind: Secret
metadata:
name: agent-platform-db
type: Opaque
stringData:
# Replace via secret manager or encrypted secret pipeline in non-local environments.
DATABASE_URL: postgresql+asyncpg://agent:<db_password>@<db_host>:5432/agent_platform
---
apiVersion: v1
kind: Secret
metadata:
name: agent-platform-app
type: Opaque
stringData:
REDIS_URL: redis://<redis_host>:6379/0
ANTHROPIC_AUTH_TOKEN: <set_via_secret_manager>
JWT_SECRET_KEY: <generate_64_hex_chars>
cluster/deploy-promoter-cronjob.yaml
+44 -27
View File
@@ -19,15 +19,33 @@ spec:
restartPolicy: OnFailure
containers:
- name: promoter
image: ghcr.io/astral-sh/uv:python3.11-alpine
imagePullPolicy: IfNotPresent
image: ghcr.io/brunogaosz/ljwx-deploy-promoter:main
imagePullPolicy: Always
env:
- name: GIT_BRANCH
value: main
- name: DEPLOY_REPO_URL
value: https://github.com/BrunoGaoSZ/ljwx-deploy.git
- name: SERVICE_MAP_PATH
value: release/services.local-k3s.yaml
- name: SKIP_REGISTRY_CHECK
value: "1"
- name: SKIP_EVIDENCE_COLLECT
value: "0"
- name: RETRY_MAX
value: "10"
- name: DEPLOY_REPO_TOKEN
valueFrom:
secretKeyRef:
name: deploy-promoter-secret
key: deploy_repo_token
optional: true
- name: GITHUB_TOKEN
valueFrom:
secretKeyRef:
name: deploy-promoter-secret
key: github_token
optional: true
- name: HARBOR_URL
valueFrom:
secretKeyRef:
@@ -46,32 +64,31 @@ spec:
name: deploy-promoter-secret
key: harbor_password
optional: true
- name: GITHUB_TOKEN
valueFrom:
secretKeyRef:
name: deploy-promoter-secret
key: github_token
optional: true
command:
- /bin/sh
- -c
- /bin/bash
- -lc
- |
set -euo pipefail
apk add --no-cache bash git openssh-client curl py3-pip
pip3 install --no-cache-dir pyyaml jsonschema
rm -rf /work/repo
git clone --depth=1 "https://x-access-token:${GITHUB_TOKEN}@github.com/BrunoGaoSZ/ljwx-deploy.git" /work/repo
cd /work/repo
export LOCAL_REPO_DIR=/work/repo
bash scripts/promoter/promote.sh
uvx --with pyyaml --with jsonschema python scripts/evidence/collect.py --out evidence/index.json --summary evidence/summary/latest.md
if git diff --quiet; then
echo "No promoter changes"
exit 0
if [ -z "${DEPLOY_REPO_TOKEN:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then
export DEPLOY_REPO_TOKEN="${GITHUB_TOKEN}"
fi
git config user.name "deploy-promoter[bot]"
git config user.email "deploy-promoter[bot]@users.noreply.github.com"
git add release/queue.yaml envs/dev evidence/records evidence/index.json evidence/summary/latest.md
git commit -m "chore(promoter): auto promote pending services [skip ci]"
git pull --rebase origin "${GIT_BRANCH}"
git push origin "HEAD:${GIT_BRANCH}"
if [ -z "${DEPLOY_REPO_TOKEN:-}" ]; then
echo "缺少 DEPLOY_REPO_TOKEN（请在 deploy-promoter-secret 中配置 deploy_repo_token）"
exit 1
fi
cmd=(
python3 /app/scripts/promoter/promote.py
--deploy-repo-url "${DEPLOY_REPO_URL}"
--deploy-repo-token "${DEPLOY_REPO_TOKEN}"
--service-map "${SERVICE_MAP_PATH}"
--push-branch "${GIT_BRANCH}"
--retry-max "${RETRY_MAX}"
)
if [ "${SKIP_REGISTRY_CHECK:-0}" = "1" ] || [ "${SKIP_REGISTRY_CHECK:-}" = "true" ] || [ "${SKIP_REGISTRY_CHECK:-}" = "TRUE" ]; then
cmd+=(--skip-registry-check)
fi
if [ "${SKIP_EVIDENCE_COLLECT:-0}" = "1" ] || [ "${SKIP_EVIDENCE_COLLECT:-}" = "true" ] || [ "${SKIP_EVIDENCE_COLLECT:-}" = "TRUE" ]; then
cmd+=(--skip-evidence-collect)
fi
"${cmd[@]}"
cluster/deploy-promoter-secret.example.yaml
+5 -1
View File
@@ -5,7 +5,11 @@ metadata:
namespace: dev
type: Opaque
stringData:
github_token: "<github_token_with_repo_workflow>"
# Use least privilege token/GitHub App token for deploy repo only.
# Required scope: repository contents write (workflow scope not required here).
deploy_repo_token: "<github_app_or_fine_grained_token>"
# Backward compatible key (optional, can be omitted after migration).
github_token: "<legacy_token_optional>"
harbor_url: "https://harbor.example.com"
harbor_username: "<harbor_user>"
harbor_password: "<harbor_password>"
cluster/smoke-runner-cronjob.yaml
+23 -8
View File
@@ -20,13 +20,21 @@ spec:
restartPolicy: Never
containers:
- name: smoke-runner
image: ghcr.io/astral-sh/uv:python3.11-alpine
imagePullPolicy: IfNotPresent
image: ghcr.io/brunogaosz/ljwx-deploy-promoter:main
imagePullPolicy: Always
env:
- name: GIT_BRANCH
value: main
- name: DEPLOY_REPO_URL
value: https://github.com/BrunoGaoSZ/ljwx-deploy.git
- name: SMOKE_TARGETS
value: scripts/smoke/targets.local-k3s.json
- name: DEPLOY_REPO_TOKEN
valueFrom:
secretKeyRef:
name: deploy-promoter-secret
key: deploy_repo_token
optional: true
- name: ARGOCD_SERVER
valueFrom:
secretKeyRef:
@@ -50,23 +58,30 @@ spec:
- -c
- |
set -euo pipefail
apk add --no-cache git openssh-client curl py3-pip
pip3 install --no-cache-dir pyyaml jsonschema
if [ -z "${DEPLOY_REPO_TOKEN:-}" ] && [ -n "${GITHUB_TOKEN:-}" ]; then
export DEPLOY_REPO_TOKEN="${GITHUB_TOKEN}"
fi
if [ -z "${DEPLOY_REPO_TOKEN:-}" ]; then
echo "缺少 DEPLOY_REPO_TOKEN（请在 deploy-promoter-secret 中配置 deploy_repo_token）"
exit 1
fi
rm -rf /work/repo
git clone --depth=1 "https://x-access-token:${GITHUB_TOKEN}@github.com/BrunoGaoSZ/ljwx-deploy.git" /work/repo
git clone --depth=1 "https://x-access-token:${DEPLOY_REPO_TOKEN}@${DEPLOY_REPO_URL#https://}" /work/repo
cd /work/repo
set +e
uvx --with pyyaml --with jsonschema python scripts/smoke/run_smoke.py --targets "${SMOKE_TARGETS}"
python3 scripts/smoke/run_smoke.py --targets "${SMOKE_TARGETS}" --queue release/queue.yaml
SMOKE_EXIT=$?
set -e
uvx --with pyyaml --with jsonschema python scripts/evidence/collect.py --out evidence/index.json --summary evidence/summary/latest.md
python3 scripts/evidence/validate.py
python3 scripts/evidence/collect.py --out evidence/index.json --summary evidence/summary/latest.md
python3 scripts/promoter/queue_metrics.py --queue release/queue.yaml --out evidence/metrics/queue-health.json
if git diff --quiet; then
echo "No smoke updates"
exit "${SMOKE_EXIT}"
fi
git config user.name "smoke-runner[bot]"
git config user.email "smoke-runner[bot]@users.noreply.github.com"
git add evidence/records evidence/index.json evidence/summary/latest.md
git add evidence/records evidence/index.json evidence/summary/latest.md evidence/metrics/queue-health.json
git commit -m "chore(smoke): update smoke evidence [skip ci]"
git pull --rebase origin "${GIT_BRANCH}"
git push origin "HEAD:${GIT_BRANCH}"
docs/ops-runbook.md
+3
View File
@@ -14,6 +14,7 @@ This runbook covers day-2 operations for the async promotion path:
Required secret (not committed with real values):
- `cluster/deploy-promoter-secret.example.yaml`
- 推荐使用 `deploy_repo_token`（最小权限）替代通用 `github_token`
Cluster rule:
@@ -86,6 +87,7 @@ python3 scripts/evidence/collect.py --out evidence/index.json --summary evidence
```
Results are recorded in each evidence file under `tests.smoke`.
Queue health metrics are generated at `evidence/metrics/queue-health.json`.
## 4) Common failures
@@ -118,3 +120,4 @@ Results are recorded in each evidence file under `tests.smoke`.
6. Argo auto-sync applies new revision.
7. Smoke (profile-based targets) writes pass/fail to evidence record.
8. Pages feed updates with latest `evidence/index.json`.
9. Queue health feed updates at `evidence/metrics/queue-health.json`.
docs/pages.md
+3 -1
View File
@@ -16,11 +16,13 @@ After first successful publish, the URLs are typically:
- Dashboard: `https://<owner>.github.io/<repo>/`
- Feed JSON: `https://<owner>.github.io/<repo>/evidence/index.json`
- Queue health: `https://<owner>.github.io/<repo>/evidence/metrics/queue-health.json`
Example for `BrunoGaoSZ/ljwx-deploy`:
- `https://brunogaosz.github.io/ljwx-deploy/`
- `https://brunogaosz.github.io/ljwx-deploy/evidence/index.json`
- `https://brunogaosz.github.io/ljwx-deploy/evidence/metrics/queue-health.json`
## Access Control Notes
@@ -32,7 +34,7 @@ Example for `BrunoGaoSZ/ljwx-deploy`:
`nightly-evidence` now publishes with an explicit branch-content contract:
- only `index.html`, `app.js`, `evidence/index.json`, `evidence/summary/latest.md` are kept in `gh-pages`
- only `index.html`, `app.js`, `evidence/index.json`, `evidence/summary/latest.md`, `evidence/metrics/queue-health.json` are kept in `gh-pages`
- all other files in `gh-pages` root are deleted before each publish
This prevents deployment manifests or source files from being accidentally retained in the Pages branch.
docs/rollback-drill.md
+32
View File
@@ -0,0 +1,32 @@
# Rollback Drill (Monthly)
目标：将“回滚能力”作为固定演练项，而不是仅在事故时临时验证。
## 范围
演练走标准链路，但只做 dry-run，不写入生产分支：
`latest promoted -> synthetic pending rollback entry -> promoter dry-run`
## 本地执行
```bash
bash scripts/ops/run_rollback_drill.sh
```
脚本行为：
1. 复制当前仓库到临时目录。
2. 读取 `release/queue.yaml`，取最近一条 `promoted` 记录。
3. 生成一条同镜像 digest 的 `pending` 回滚演练记录。
4. 校验队列结构并执行 promoter dry-run。
5. 删除临时目录，不保留任何提交。
## CI 定时
工作流：`.github/workflows/monthly-rollback-drill.yml`
- 定时：每月一次
- 模式：dry-run only
- 目的：持续验证“可回滚”而非仅验证“可发布”
envs/README.md
+17
View File
@@ -0,0 +1,17 @@
# envs 目录说明（冻结）
截至 `2026-03-05`，`envs/` 仅作为历史快照/兼容参考，不再作为 GitOps 主发布入口。
当前主路径：
- 入列：`release/queue.yaml`
- 映射：`release/services*.yaml`
- 部署触发：`apps/*/overlays/*/kustomization.yaml` 由 promoter 更新
- 证据链：`evidence/records/*` + `evidence/index.json`
规则：
1. 常规发布流程不得依赖 `envs/` 内容变更。
2. 若必须修改 `envs/`（兼容或迁移任务），需走 `platform-admin` 特殊审批路径。
3. 新服务接入请使用 `factory/onboarding/services.catalog.yaml` 与 `scripts/factory/onboard_services.sh`。
evidence/README.md
+3 -1
View File
@@ -9,13 +9,14 @@
- `env`
- `source.repo`
- `source.commit`
- `image.harbor`
- `image.deployed`
- `deploy.deployRepoCommit`
## Optional fields
- `source.workflowRun`
- `image.ghcr`
- `image.harbor` (legacy compatibility field)
- `deploy.argocdApp`
- `deploy.syncedAt`
- `tests.smoke`
@@ -42,6 +43,7 @@ source:
image:
ghcr: ghcr.io/example/frontend@sha256:1111111111111111111111111111111111111111111111111111111111111111
harbor: harbor.omniverseai.net/app/frontend@sha256:2222222222222222222222222222222222222222222222222222222222222222
deployed: ghcr.io/example/frontend@sha256:1111111111111111111111111111111111111111111111111111111111111111
deploy:
deployRepoCommit: abcdef1234567890
argocdApp: frontend-dev
evidence/index.json
+4 -2
View File
@@ -852,7 +852,8 @@
},
"image": {
"ghcr": "ghcr.io/brunogaosz/ljwx-platform@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"harbor": "harbor.omniverseai.net/app/ljwx-platform@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
"harbor": "harbor.omniverseai.net/app/ljwx-platform@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"deployed": "ghcr.io/brunogaosz/ljwx-platform@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
},
"deploy": {
"deployRepoCommit": "1d679035ed8a78e7e0820228ae59d1e420da5695",
@@ -883,7 +884,8 @@
},
"image": {
"ghcr": "ghcr.io/brunogaosz/ljwx-website@sha256:b6bee6292395deda36062a25cdbae5a7eb81477823736bd3dacbcdc5e6be1641",
"harbor": "harbor.omniverseai.net/app/ljwx-website@sha256:b6bee6292395deda36062a25cdbae5a7eb81477823736bd3dacbcdc5e6be1641"
"harbor": "harbor.omniverseai.net/app/ljwx-website@sha256:b6bee6292395deda36062a25cdbae5a7eb81477823736bd3dacbcdc5e6be1641",
"deployed": "ghcr.io/brunogaosz/ljwx-website@sha256:b6bee6292395deda36062a25cdbae5a7eb81477823736bd3dacbcdc5e6be1641"
},
"deploy": {
"deployRepoCommit": "3e1ea9e61cef4876d0807b0e9ec5711ce298449a",
evidence/metrics/queue-health.json
+27
View File
@@ -0,0 +1,27 @@
{
"generatedAt": "2026-03-05T04:35:02Z",
"counts": {
"pending": 0,
"promoted": 27,
"failed": 0,
"superseded": 5
},
"pending": {
"oldestCreatedAt": "",
"maxAgeSeconds": 0,
"staleThresholdSeconds": 1800,
"staleIds": [],
"byServiceEnv": {}
},
"failed": {
"last24h": 0,
"byServiceEnv": {}
},
"retries": {
"pendingMaxAttempts": 0,
"failedMaxAttempts": 0
},
"ratios": {
"supersededVsPromoted": 0.1562
}
}
evidence/records/20260303-ljwx-platform-sha-e2e001.yaml
+1
View File
@@ -8,6 +8,7 @@ source:
image:
ghcr: ghcr.io/brunogaosz/ljwx-platform@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
harbor: harbor.omniverseai.net/app/ljwx-platform@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
deployed: ghcr.io/brunogaosz/ljwx-platform@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
deploy:
deployRepoCommit: 1d679035ed8a78e7e0820228ae59d1e420da5695
queueId: 2026-03-03T060500Z-ljwx-platform-sha-e2e001
evidence/records/20260303-ljwx-website-sha-931cfe96df25.yaml
+1
View File
@@ -8,6 +8,7 @@ source:
image:
ghcr: ghcr.io/brunogaosz/ljwx-website@sha256:b6bee6292395deda36062a25cdbae5a7eb81477823736bd3dacbcdc5e6be1641
harbor: harbor.omniverseai.net/app/ljwx-website@sha256:b6bee6292395deda36062a25cdbae5a7eb81477823736bd3dacbcdc5e6be1641
deployed: ghcr.io/brunogaosz/ljwx-website@sha256:b6bee6292395deda36062a25cdbae5a7eb81477823736bd3dacbcdc5e6be1641
deploy:
deployRepoCommit: 3e1ea9e61cef4876d0807b0e9ec5711ce298449a
queueId: 2026-03-02T151550Z-ljwx-website-sha-931cfe96df25
evidence/schema/evidence.schema.json
+7 -1
View File
@@ -52,7 +52,7 @@
"image": {
"type": "object",
"required": [
"harbor"
"deployed"
],
"additionalProperties": true,
"properties": {
@@ -63,6 +63,12 @@
]
},
"harbor": {
"type": [
"string",
"null"
]
},
"deployed": {
"type": "string",
"minLength": 3
}
evidence/summary/latest.md
+1 -1
View File
@@ -1,6 +1,6 @@
# Latest Evidence Summary
| service | env | harbor digest | syncedAt | smoke | links |
| service | env | deployed digest | syncedAt | smoke | links |
| --- | --- | --- | --- | --- | --- |
| ljwx-stock-agent | dev | `077c33f22a0d2950` | 2026-03-05T04:10:13Z | pass | - |
| ljwx-stock-qlib-predict | dev | `9b310ac3e5f1079a` | 2026-03-05T01:16:35Z | pending | - |
pages/app.js
+20 -7
View File
@@ -1,6 +1,8 @@
(function () {
const FEED_URL = "./evidence/index.json";
const METRICS_URL = "./evidence/metrics/queue-health.json";
let allRecords = [];
let queueMetrics = null;
const envFilter = document.getElementById("env-filter");
const testFilter = document.getElementById("test-filter");
@@ -11,11 +13,11 @@
return String(v ?? "").replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
}
function digestFromHarbor(harbor) {
if (!harbor) return "-";
const idx = harbor.indexOf("@sha256:");
if (idx === -1) return harbor;
return "sha256:" + harbor.slice(idx + 8, idx + 8 + 16);
function digestFromImage(imageRef) {
if (!imageRef) return "-";
const idx = imageRef.indexOf("@sha256:");
if (idx === -1) return imageRef;
return "sha256:" + imageRef.slice(idx + 8, idx + 8 + 16);
}
function smokeStatus(rec) {
@@ -57,14 +59,17 @@
<td><code>${text(r.evidenceId || "-")}</code></td>
<td>${text(r.service || "-")}</td>
<td>${text(r.env || "-")}</td>
<td><code>${text(digestFromHarbor(r?.image?.harbor || ""))}</code></td>
<td><code>${text(digestFromImage(r?.image?.deployed || r?.image?.harbor || ""))}</code></td>
<td>${text(r?.deploy?.syncedAt || r?.promotedAt || "-")}</td>
<td><span class="pill ${klass}">${text(smoke)}</span></td>
<td>${linksHtml(r)}</td>
</tr>`;
}).join("") || '<tr><td colspan="7">No records</td></tr>';
metaEl.textContent = `records: ${filtered.length} / ${allRecords.length}`;
const queueMeta = queueMetrics?.counts
? ` | queue pending=${queueMetrics.counts.pending}, failed=${queueMetrics.counts.failed}, superseded=${queueMetrics.counts.superseded}`
: "";
metaEl.textContent = `records: ${filtered.length} / ${allRecords.length}${queueMeta}`;
}
function fillEnvOptions() {
@@ -85,6 +90,14 @@
}
allRecords = await res.json();
if (!Array.isArray(allRecords)) allRecords = [];
const metricsRes = await fetch(METRICS_URL, { cache: "no-store" });
if (metricsRes.ok) {
const payload = await metricsRes.json();
if (payload && typeof payload === "object") {
queueMetrics = payload;
}
}
fillEnvOptions();
render();
}
pages/index.html
+1 -1
View File
@@ -29,7 +29,7 @@
<th>evidenceId</th>
<th>service</th>
<th>env</th>
<th>harbor digest</th>
<th>deployed digest</th>
<th>syncedAt</th>
<th>smoke</th>
<th>links</th>
scripts/ci/run_checks.sh
+2 -1
View File
@@ -4,5 +4,6 @@ set -euo pipefail
uvx --with pyyaml --with jsonschema python scripts/evidence/validate.py
uvx --with pyyaml --with jsonschema python scripts/evidence/collect.py --out evidence/index.json
uvx --with pyyaml python scripts/promoter/validate_queue.py
uvx --with pyyaml python scripts/promoter/queue_metrics.py --queue release/queue.yaml --out /tmp/queue-health.json
bash scripts/promoter/promote.sh --dry-run
uvx --with pyyaml --with jsonschema python scripts/smoke/run_smoke.py --dry-run
uvx --with pyyaml --with jsonschema python scripts/smoke/run_smoke.py --dry-run --allow-failures
scripts/evidence/collect.py
+5 -3
View File
@@ -83,7 +83,7 @@ def write_summary(records: list[dict[str, Any]], out_path: Path) -> None:
lines = [
"# Latest Evidence Summary",
"",
"| service | env | harbor digest | syncedAt | smoke | links |",
"| service | env | deployed digest | syncedAt | smoke | links |",
"| --- | --- | --- | --- | --- | --- |",
]
@@ -91,8 +91,10 @@ def write_summary(records: list[dict[str, Any]], out_path: Path) -> None:
service = str(record.get("service", "-"))
env = str(record.get("env", "-"))
image = record.get("image", {})
harbor = image.get("harbor", "") if isinstance(image, dict) else ""
digest = short_digest(str(harbor)) or "-"
deployed = ""
if isinstance(image, dict):
deployed = str(image.get("deployed", "") or image.get("harbor", ""))
digest = short_digest(deployed) or "-"
deploy = record.get("deploy", {})
synced_at = deploy.get("syncedAt", "-") if isinstance(deploy, dict) else "-"
tests = record.get("tests", {})
scripts/evidence/validate.py
+1 -1
View File
@@ -62,7 +62,7 @@ def main() -> int:
("env",),
("source", "repo"),
("source", "commit"),
("image", "harbor"),
("image", "deployed"),
("deploy", "deployRepoCommit"),
]
for key_path in required:
scripts/ops/run_rollback_drill.sh
Executable
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
cd "$ROOT_DIR"
TMP_DIR="$(mktemp -d)"
trap 'rm -rf "$TMP_DIR"' EXIT
echo "[rollback-drill] 准备临时仓库副本: $TMP_DIR/repo"
cp -R . "$TMP_DIR/repo"
cd "$TMP_DIR/repo"
echo "[rollback-drill] 选择最近 promoted 条目并构造回滚入列项"
python3 - <<'PY'
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
import copy
import yaml
queue_path = Path("release/queue.yaml")
payload = yaml.safe_load(queue_path.read_text(encoding="utf-8")) or {}
promoted = payload.get("promoted", [])
if not isinstance(promoted, list) or not promoted:
raise SystemExit("promoted 列表为空，无法执行回滚演练")
def parse_ts(value: str) -> datetime:
if not value:
return datetime.min.replace(tzinfo=timezone.utc)
try:
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
except ValueError:
return datetime.min.replace(tzinfo=timezone.utc)
candidates = [item for item in promoted if isinstance(item, dict)]
if not candidates:
raise SystemExit("promoted 无有效条目")
candidates.sort(
key=lambda item: parse_ts(str(item.get("promotedAt", ""))) or parse_ts(str(item.get("createdAt", ""))),
reverse=True,
)
base = candidates[0]
service = str(base.get("service", "")).strip()
env = str(base.get("env", "")).strip()
source = copy.deepcopy(base.get("source", {}))
if not service or not env or not isinstance(source, dict):
raise SystemExit("最近 promoted 条目字段不完整，无法演练")
now = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
entry = {
"id": f"{now}-{service}-{env}-rollback-drill",
"service": service,
"env": env,
"source": source,
"createdAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"status": "pending",
"attempts": 0,
"lastError": "",
"promotedAt": "",
"supersededAt": "",
"failedAt": "",
}
pending = payload.get("pending", [])
if not isinstance(pending, list):
pending = []
pending.append(entry)
payload["pending"] = pending
queue_path.write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8")
print(f"rollback drill entry created: {entry['id']}")
PY
echo "[rollback-drill] 校验队列结构"
uvx --with pyyaml python scripts/promoter/validate_queue.py --queue release/queue.yaml
echo "[rollback-drill] 执行 promoter dry-run（仅模拟，不提交）"
uvx --with pyyaml --with jsonschema python scripts/promoter/promote.py \
--dry-run \
--local-repo-dir .
echo "[rollback-drill] 演练完成"
scripts/promoter/Dockerfile
+5 -5
View File
@@ -1,11 +1,11 @@
FROM python:3.11-alpine
FROM ghcr.io/astral-sh/uv:python3.11-alpine
RUN apk add --no-cache git curl
RUN apk add --no-cache bash git curl openssh-client ca-certificates
WORKDIR /app
COPY scripts/promoter/promote.py /app/promote.py
COPY scripts /app/scripts
RUN pip install --no-cache-dir pyyaml jsonschema
RUN uv pip install --system pyyaml jsonschema
ENTRYPOINT ["python3", "/app/promote.py"]
ENTRYPOINT ["python3", "/app/scripts/promoter/promote.py"]
scripts/promoter/README.md
+4 -1
View File
@@ -7,10 +7,11 @@ Promoter reads `release/queue.yaml` and auto-promotes ready releases by mutating
- Per-service serialization: at most one pending entry per `service+env`.
- Older pending entries are moved to `superseded` (not failed).
- Promotion uses queue `source.ghcr` (standard image path) by default.
- Optional registry readiness check can be enabled with `HARBOR_URL`.
- Optional registry readiness check can be enabled with `HARBOR_URL` (probe Harbor-mapped image first).
- Idempotent rerun: already-promoted items are not promoted again.
- Promotion target resolution is centralized in `release/services.yaml`.
- One codebase can target multiple clusters by switching `SERVICE_MAP_PATH`.
- Queue/evidence/summary/metrics are committed in one promotion transaction.
## Main Command
@@ -73,9 +74,11 @@ kubectl patch serviceaccount promoter -n shared-platform \
- `HARBOR_URL` (optional; empty by default)
- `HARBOR_USER`, `HARBOR_PASS`
- `SKIP_REGISTRY_CHECK` (`1/true` to pass `--skip-registry-check`)
- `SKIP_EVIDENCE_COLLECT` (`1/true` to skip evidence index/summary/metrics generation)
- `RETRY_MAX` (default: `10`)
- `DRY_RUN` (`1` or `0`)
- `SERVICE_MAP_PATH` (default: `release/services.yaml`)
- `GIT_BRANCH` (default: `main`)
## Queue file
scripts/promoter/promote.py
+127 -17
View File
@@ -17,7 +17,9 @@ from typing import Any
try:
import yaml
except Exception as exc: # noqa: BLE001
raise SystemExit(f"PyYAML is required. Install with: pip3 install pyyaml\n{exc}")
raise SystemExit(
f"PyYAML is required. Install with: uvx --with pyyaml python <script>\n{exc}"
)
def now_rfc3339() -> str:
@@ -171,6 +173,25 @@ def harbor_manifest_ready(
return get.returncode == 0 and get.stdout.strip() == "200"
def image_belongs_to_harbor(image_ref: str, harbor_url: str) -> bool:
harbor_host = host_from_url(harbor_url)
image_host, _ = image_registry_and_repo(image_ref)
if not harbor_host:
return False
return bool(image_host) and image_host == harbor_host
def select_registry_probe_image(
harbor_image: str, deploy_image: str, harbor_url: str
) -> str:
# In strict Harbor mode, probe Harbor repository first.
if harbor_image and image_belongs_to_harbor(harbor_image, harbor_url):
return harbor_image
if deploy_image and image_belongs_to_harbor(deploy_image, harbor_url):
return deploy_image
return ""
def ghcr_repo(ghcr_ref: str) -> str:
return ghcr_ref.split("@", 1)[0] if "@" in ghcr_ref else ghcr_ref
@@ -398,10 +419,16 @@ def process_pending(
continue
should_check_registry = (not skip_registry_check) and bool(harbor_url.strip())
if should_check_registry and not harbor_manifest_ready(
deploy_image, digest, harbor_url, harbor_user, harbor_pass
):
continue
if should_check_registry:
probe_image = select_registry_probe_image(
harbor_image=harbor_image,
deploy_image=deploy_image,
harbor_url=harbor_url,
)
if probe_image and not harbor_manifest_ready(
probe_image, digest, harbor_url, harbor_user, harbor_pass
):
continue
overlay_path = repo_dir / overlay_rel
if not overlay_path.exists():
@@ -450,8 +477,8 @@ def process_pending(
},
"image": {
"ghcr": ghcr_ref,
# Keep legacy key for schema compatibility; value is the actual deployed image.
"harbor": f"{deploy_image}@{digest}",
# Keep legacy key for schema compatibility; prefer Harbor mirror when available.
"harbor": f"{(harbor_image or deploy_image)}@{digest}",
"deployed": f"{deploy_image}@{digest}",
},
"deploy": {
@@ -489,6 +516,7 @@ def process_pending(
promoted_meta.append(
{
"service": service,
"env": env,
"tag": tag,
"evidence_path": str(evidence_path),
"overlay_path": overlay_rel,
@@ -536,6 +564,8 @@ def validate_queue_shape(queue: dict[str, Any]) -> None:
def commit_and_push(
repo_dir: Path,
promoted_meta: list[dict[str, str]],
push_branch: str,
collect_evidence: bool,
dry_run: bool,
) -> None:
if dry_run:
@@ -543,6 +573,29 @@ def commit_and_push(
return
run(["python3", "scripts/evidence/validate.py"], cwd=repo_dir)
if collect_evidence:
run(
[
"python3",
"scripts/evidence/collect.py",
"--out",
"evidence/index.json",
"--summary",
"evidence/summary/latest.md",
],
cwd=repo_dir,
)
run(
[
"python3",
"scripts/promoter/queue_metrics.py",
"--queue",
"release/queue.yaml",
"--out",
"evidence/metrics/queue-health.json",
],
cwd=repo_dir,
)
run(["git", "config", "user.name", "deploy-promoter[bot]"], cwd=repo_dir)
run(
@@ -556,6 +609,14 @@ def commit_and_push(
)
stage_paths = ["release/queue.yaml", "evidence/records"]
if collect_evidence:
stage_paths.extend(
[
"evidence/index.json",
"evidence/summary/latest.md",
"evidence/metrics/queue-health.json",
]
)
stage_paths.extend(
item["overlay_path"] for item in promoted_meta if item.get("overlay_path")
)
@@ -568,11 +629,16 @@ def commit_and_push(
return
if promoted_meta:
svc = promoted_meta[0]["service"]
tag = promoted_meta[0]["tag"]
message = f"promote(dev): {svc} {tag} [skip ci]"
env_set = sorted({item.get("env", "dev") for item in promoted_meta})
scope = env_set[0] if len(env_set) == 1 else "multi"
if len(promoted_meta) == 1:
svc = promoted_meta[0]["service"]
tag = promoted_meta[0]["tag"]
message = f"promote({scope}): {svc} {tag} [skip ci]"
else:
message = f"promote({scope}): {len(promoted_meta)} services [skip ci]"
else:
message = "promote(dev): queue normalize [skip ci]"
message = "promote(ops): queue normalize [skip ci]"
run(["git", "commit", "-m", message], cwd=repo_dir)
sha = run(["git", "rev-parse", "HEAD"], cwd=repo_dir).stdout.strip()
@@ -594,13 +660,41 @@ def commit_and_push(
if touched:
run(["python3", "scripts/evidence/validate.py"], cwd=repo_dir)
run(
["git", "add", *[str(p.relative_to(repo_dir)) for p in touched]],
cwd=repo_dir,
)
add_paths = [str(p.relative_to(repo_dir)) for p in touched]
if collect_evidence:
run(
[
"python3",
"scripts/evidence/collect.py",
"--out",
"evidence/index.json",
"--summary",
"evidence/summary/latest.md",
],
cwd=repo_dir,
)
run(
[
"python3",
"scripts/promoter/queue_metrics.py",
"--queue",
"release/queue.yaml",
"--out",
"evidence/metrics/queue-health.json",
],
cwd=repo_dir,
)
add_paths.extend(
[
"evidence/index.json",
"evidence/summary/latest.md",
"evidence/metrics/queue-health.json",
]
)
run(["git", "add", *add_paths], cwd=repo_dir)
run(["git", "commit", "--amend", "--no-edit"], cwd=repo_dir)
run(["git", "push", "origin", "HEAD:main"], cwd=repo_dir)
run(["git", "push", "origin", f"HEAD:{push_branch}"], cwd=repo_dir)
def main() -> int:
@@ -620,6 +714,11 @@ def main() -> int:
parser.add_argument(
"--service-map", default=os.getenv("SERVICE_MAP_PATH", "release/services.yaml")
)
parser.add_argument(
"--push-branch",
default=os.getenv("GIT_BRANCH", "main"),
help="target branch in deploy repo",
)
parser.add_argument(
"--harbor-url",
default=os.getenv("HARBOR_URL", ""),
@@ -637,6 +736,11 @@ def main() -> int:
action="store_true",
help="skip registry manifest check (transparent-cache mode)",
)
parser.add_argument(
"--skip-evidence-collect",
action="store_true",
help="skip evidence index/summary/metrics collection before commit",
)
args = parser.parse_args()
dry_run = bool(args.dry_run) or os.getenv("DRY_RUN", "0") in {
@@ -692,7 +796,13 @@ def main() -> int:
for path, data in evidence_changes.items():
yaml_dump(path, data)
commit_and_push(repo_dir, promoted_meta, dry_run=False)
commit_and_push(
repo_dir=repo_dir,
promoted_meta=promoted_meta,
push_branch=args.push_branch,
collect_evidence=not args.skip_evidence_collect,
dry_run=False,
)
print("Promotion completed")
return 0
finally:
scripts/promoter/promote.sh
+3
View File
@@ -9,5 +9,8 @@ args=("$@")
if [[ "${SKIP_REGISTRY_CHECK:-0}" == "1" || "${SKIP_REGISTRY_CHECK:-}" == "true" || "${SKIP_REGISTRY_CHECK:-}" == "TRUE" ]]; then
args+=("--skip-registry-check")
fi
if [[ "${SKIP_EVIDENCE_COLLECT:-0}" == "1" || "${SKIP_EVIDENCE_COLLECT:-}" == "true" || "${SKIP_EVIDENCE_COLLECT:-}" == "TRUE" ]]; then
args+=("--skip-evidence-collect")
fi
python3 scripts/promoter/promote.py "${args[@]}"
scripts/promoter/queue_metrics.py
Executable
+184
View File
@@ -0,0 +1,184 @@
#!/usr/bin/env python3
"""Build queue health metrics for ops visibility."""
from __future__ import annotations
import argparse
import json
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
try:
import yaml
except Exception as exc: # noqa: BLE001
raise SystemExit(
f"PyYAML is required. Install with: uvx --with pyyaml python <script>\n{exc}"
)
def now_utc() -> datetime:
return datetime.now(timezone.utc)
def now_rfc3339() -> str:
return now_utc().strftime("%Y-%m-%dT%H:%M:%SZ")
def parse_ts(value: Any) -> datetime:
if not isinstance(value, str) or not value:
return datetime.min.replace(tzinfo=timezone.utc)
normalized = value.strip().replace("Z", "+00:00")
try:
parsed = datetime.fromisoformat(normalized)
except ValueError:
return datetime.min.replace(tzinfo=timezone.utc)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
def load_yaml(path: Path) -> dict[str, Any]:
payload = yaml.safe_load(path.read_text(encoding="utf-8"))
if payload is None:
return {}
if not isinstance(payload, dict):
raise ValueError(f"queue root must be mapping: {path}")
return payload
def ensure_queue_shape(payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
out: dict[str, list[dict[str, Any]]] = {}
for key in ("pending", "promoted", "failed", "superseded"):
raw = payload.get(key, [])
if not isinstance(raw, list):
raise ValueError(f"queue key must be list: {key}")
out[key] = [entry for entry in raw if isinstance(entry, dict)]
return out
def ts_from_entry(entry: dict[str, Any], fields: list[str]) -> datetime:
for field in fields:
parsed = parse_ts(entry.get(field))
if parsed != datetime.min.replace(tzinfo=timezone.utc):
return parsed
return datetime.min.replace(tzinfo=timezone.utc)
def service_env_key(entry: dict[str, Any]) -> str:
service = str(entry.get("service", "unknown")).strip() or "unknown"
env = str(entry.get("env", "unknown")).strip() or "unknown"
return f"{service}/{env}"
def grouped_counts(entries: list[dict[str, Any]]) -> dict[str, int]:
counts: dict[str, int] = {}
for entry in entries:
key = service_env_key(entry)
counts[key] = counts.get(key, 0) + 1
return dict(sorted(counts.items(), key=lambda item: item[0]))
def build_metrics(
queue: dict[str, list[dict[str, Any]]], stale_threshold_seconds: int
) -> dict[str, Any]:
current = now_utc()
pending = queue["pending"]
promoted = queue["promoted"]
failed = queue["failed"]
superseded = queue["superseded"]
pending_times: list[datetime] = [
ts
for ts in (ts_from_entry(item, ["createdAt"]) for item in pending)
if ts != datetime.min.replace(tzinfo=timezone.utc)
]
oldest_pending = min(pending_times) if pending_times else None
max_pending_age_seconds = (
int((current - oldest_pending).total_seconds()) if oldest_pending else 0
)
stale_ids = []
for item in pending:
created = ts_from_entry(item, ["createdAt"])
if created == datetime.min.replace(tzinfo=timezone.utc):
continue
age_seconds = int((current - created).total_seconds())
if age_seconds >= stale_threshold_seconds:
stale_ids.append(str(item.get("id", "")))
recent_failed_cutoff = current - timedelta(hours=24)
failed_24h = 0
for item in failed:
failed_at = ts_from_entry(item, ["failedAt", "createdAt"])
if failed_at >= recent_failed_cutoff:
failed_24h += 1
superseded_denominator = len(promoted) + len(superseded)
superseded_ratio = (
round(len(superseded) / superseded_denominator, 4)
if superseded_denominator > 0
else 0.0
)
pending_attempts = [int(item.get("attempts", 0)) for item in pending]
failed_attempts = [int(item.get("attempts", 0)) for item in failed]
return {
"generatedAt": now_rfc3339(),
"counts": {
"pending": len(pending),
"promoted": len(promoted),
"failed": len(failed),
"superseded": len(superseded),
},
"pending": {
"oldestCreatedAt": oldest_pending.strftime("%Y-%m-%dT%H:%M:%SZ")
if oldest_pending
else "",
"maxAgeSeconds": max_pending_age_seconds,
"staleThresholdSeconds": stale_threshold_seconds,
"staleIds": stale_ids,
"byServiceEnv": grouped_counts(pending),
},
"failed": {
"last24h": failed_24h,
"byServiceEnv": grouped_counts(failed),
},
"retries": {
"pendingMaxAttempts": max(pending_attempts) if pending_attempts else 0,
"failedMaxAttempts": max(failed_attempts) if failed_attempts else 0,
},
"ratios": {
"supersededVsPromoted": superseded_ratio,
},
}
def main() -> int:
parser = argparse.ArgumentParser(
description="Generate release queue health metrics"
)
parser.add_argument("--queue", type=Path, default=Path("release/queue.yaml"))
parser.add_argument(
"--out", type=Path, default=Path("evidence/metrics/queue-health.json")
)
parser.add_argument("--stale-threshold-seconds", type=int, default=1800)
args = parser.parse_args()
if args.stale_threshold_seconds < 1:
raise SystemExit("stale-threshold-seconds must be positive integer")
queue_payload = ensure_queue_shape(load_yaml(args.queue))
metrics = build_metrics(queue_payload, args.stale_threshold_seconds)
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(
json.dumps(metrics, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
print(f"已写入队列健康指标: {args.out}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/promoter/validate_queue.py
+173 -31
View File
@@ -1,10 +1,12 @@
#!/usr/bin/env python3
"""Validate release/queue.yaml structure for gate checks."""
"""Validate release/queue.yaml structure and state contracts."""
from __future__ import annotations
import argparse
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
@@ -16,25 +18,49 @@ except Exception as exc: # noqa: BLE001
)
QUEUE_STATES = ("pending", "promoted", "failed", "superseded")
DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
def load_yaml(path: Path) -> Any:
data = yaml.safe_load(path.read_text(encoding="utf-8"))
return {} if data is None else data
def main() -> int:
parser = argparse.ArgumentParser(description="Validate release queue shape")
parser.add_argument("--queue", default="release/queue.yaml", type=Path)
args = parser.parse_args()
def parse_rfc3339(value: Any) -> datetime | None:
if not isinstance(value, str):
return None
text = value.strip()
if not text:
return None
normalized = text.replace("Z", "+00:00")
try:
parsed = datetime.fromisoformat(normalized)
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
data = load_yaml(args.queue)
if not isinstance(data, dict):
print("queue root must be mapping")
return 1
for key in ["pending", "promoted", "failed", "superseded"]:
if key not in data or not isinstance(data[key], list):
print(f"missing list key: {key}")
return 1
def as_non_empty_str(value: Any) -> str | None:
if not isinstance(value, str):
return None
stripped = value.strip()
if not stripped:
return None
return stripped
def validate_entry(
state: str,
index: int,
entry: dict[str, Any],
seen_ids: set[str],
pending_service_env: set[tuple[str, str]],
) -> list[str]:
errors: list[str] = []
prefix = f"{state}[{index}]"
required = [
"id",
@@ -49,25 +75,141 @@ def main() -> int:
"supersededAt",
"failedAt",
]
for state in ["pending", "promoted", "failed", "superseded"]:
for idx, item in enumerate(data[state]):
if not isinstance(item, dict):
print(f"{state}[{idx}] must be mapping")
return 1
for field in required:
if field not in item:
print(f"{state}[{idx}] missing field: {field}")
return 1
source = item.get("source", {})
if not isinstance(source, dict):
print(f"{state}[{idx}].source must be mapping")
return 1
for sf in ["ghcr", "tag", "digest"]:
if sf not in source:
print(f"{state}[{idx}].source missing field: {sf}")
return 1
for field in required:
if field not in entry:
errors.append(f"{prefix} 缺少字段: {field}")
print("release queue validation passed")
entry_id = as_non_empty_str(entry.get("id"))
if not entry_id:
errors.append(f"{prefix}.id 必须是非空字符串")
elif entry_id in seen_ids:
errors.append(f"{prefix}.id 重复: {entry_id}")
else:
seen_ids.add(entry_id)
service = as_non_empty_str(entry.get("service"))
env = as_non_empty_str(entry.get("env"))
if not service:
errors.append(f"{prefix}.service 必须是非空字符串")
if not env:
errors.append(f"{prefix}.env 必须是非空字符串")
status = as_non_empty_str(entry.get("status"))
if not status:
errors.append(f"{prefix}.status 必须是非空字符串")
elif status not in QUEUE_STATES:
errors.append(f"{prefix}.status 非法: {status}")
elif status != state:
errors.append(f"{prefix}.status 与所在队列不一致: {status} != {state}")
attempts = entry.get("attempts")
if not isinstance(attempts, int) or attempts < 0:
errors.append(f"{prefix}.attempts 必须是 >= 0 的整数")
created_at = parse_rfc3339(entry.get("createdAt"))
if created_at is None:
errors.append(f"{prefix}.createdAt 必须是 RFC3339 时间戳")
for ts_field in ("promotedAt", "supersededAt", "failedAt"):
value = entry.get(ts_field)
if value in ("", None):
continue
if parse_rfc3339(value) is None:
errors.append(f"{prefix}.{ts_field} 必须是 RFC3339 时间戳或空字符串")
# State timestamp contract
if state == "pending":
for ts_field in ("promotedAt", "supersededAt", "failedAt"):
if as_non_empty_str(entry.get(ts_field)):
errors.append(f"{prefix}.{ts_field} 在 pending 状态必须为空")
if service and env:
key = (service, env)
if key in pending_service_env:
errors.append(
f"{prefix} 同一 service/env 存在多个 pending: {service}/{env}"
)
else:
pending_service_env.add(key)
elif state == "promoted":
if parse_rfc3339(entry.get("promotedAt")) is None:
errors.append(f"{prefix}.promotedAt 在 promoted 状态必须为 RFC3339 时间")
elif state == "failed":
if parse_rfc3339(entry.get("failedAt")) is None:
errors.append(f"{prefix}.failedAt 在 failed 状态必须为 RFC3339 时间")
elif state == "superseded":
if parse_rfc3339(entry.get("supersededAt")) is None:
errors.append(
f"{prefix}.supersededAt 在 superseded 状态必须为 RFC3339 时间"
)
source = entry.get("source", {})
if not isinstance(source, dict):
errors.append(f"{prefix}.source 必须是 mapping")
return errors
ghcr = as_non_empty_str(source.get("ghcr"))
tag = as_non_empty_str(source.get("tag"))
digest = as_non_empty_str(source.get("digest"))
if not ghcr:
errors.append(f"{prefix}.source.ghcr 必须是非空字符串")
if not tag:
errors.append(f"{prefix}.source.tag 必须是非空字符串")
if not digest:
errors.append(f"{prefix}.source.digest 必须是非空字符串")
elif not DIGEST_PATTERN.match(digest):
errors.append(f"{prefix}.source.digest 格式非法: {digest}")
if ghcr and "@" in ghcr:
_, ghcr_digest = ghcr.rsplit("@", 1)
if digest and ghcr_digest != digest:
errors.append(
f"{prefix}.source.ghcr 与 source.digest 不一致: {ghcr_digest} != {digest}"
)
return errors
def main() -> int:
parser = argparse.ArgumentParser(description="Validate release queue shape")
parser.add_argument("--queue", default="release/queue.yaml", type=Path)
args = parser.parse_args()
data = load_yaml(args.queue)
if not isinstance(data, dict):
print("queue 根节点必须是 mapping")
return 1
for key in QUEUE_STATES:
if key not in data or not isinstance(data[key], list):
print(f"缺少队列列表字段: {key}")
return 1
all_errors: list[str] = []
seen_ids: set[str] = set()
pending_service_env: set[tuple[str, str]] = set()
for state in QUEUE_STATES:
entries = data.get(state, [])
for idx, item in enumerate(entries):
if not isinstance(item, dict):
all_errors.append(f"{state}[{idx}] 必须是 mapping")
continue
all_errors.extend(
validate_entry(
state=state,
index=idx,
entry=item,
seen_ids=seen_ids,
pending_service_env=pending_service_env,
)
)
if all_errors:
print("release queue 校验失败:")
for message in all_errors:
print(f"- {message}")
return 1
print("release queue 校验通过")
return 0
scripts/smoke/README.md
+27 -7
View File
@@ -4,21 +4,41 @@ Runs post-deploy smoke checks and writes results to YAML evidence records.
## Logic
1. Find latest promoted record for each target service/environment from `evidence/records/*.yaml`.
2. Wait for Argo app `Synced + Healthy` (if Argo credentials/app name are configured).
3. Check service endpoint readiness (`HTTP 2xx/3xx`).
4. Update evidence record `tests.smoke.status`, `tests.smoke.checkedAt`, `tests.smoke.details`.
1. Resolve `queueId` for each target (prefer target `queue_id`, fallback to latest promoted entry in `release/queue.yaml`).
2. Find evidence record by `service/environment/queueId` from `evidence/records/*.yaml` (prefer `smoke=pending`).
3. Wait for Argo app `Synced + Healthy` (if Argo credentials/app name are configured).
4. Check service endpoint readiness (`HTTP 2xx/3xx`).
5. Update evidence record `tests.smoke.status`, `tests.smoke.checkedAt`, `tests.smoke.details`.
## Queue precision
To pin smoke to a specific promotion:
```json
{
"service": "frontend",
"environment": "dev",
"queue_id": "20260305T010000Z-frontend-sha-abc1234",
"argocd_app": "frontend-dev",
"endpoint": "http://frontend.dev.svc.cluster.local/"
}
```
Without `queue_id`, runner auto-resolves latest promoted queue entry for `service/environment`.
## Usage
```bash
# dry run (still performs checks, does not write files)
python3 scripts/smoke/run_smoke.py --dry-run
python3 scripts/smoke/run_smoke.py --dry-run --queue release/queue.yaml
# CI structural check mode (do not fail pipeline on endpoint/network reachability)
python3 scripts/smoke/run_smoke.py --dry-run --allow-failures --queue release/queue.yaml
# normal run
ARGOCD_SERVER=https://argocd.example.com \
ARGOCD_TOKEN=*** \
python3 scripts/smoke/run_smoke.py
python3 scripts/smoke/run_smoke.py --queue release/queue.yaml
```
## Cluster profiles
@@ -31,5 +51,5 @@ Use target files instead of forking smoke logic:
Example:
```bash
python3 scripts/smoke/run_smoke.py --targets scripts/smoke/targets.orbstack-k3s-cn.json
python3 scripts/smoke/run_smoke.py --targets scripts/smoke/targets.orbstack-k3s-cn.json --queue release/queue.yaml
```
scripts/smoke/run_smoke.py
+84 -7
View File
@@ -39,6 +39,15 @@ def read_yaml(path: Path) -> dict[str, Any]:
return raw
def read_queue(path: Path) -> dict[str, Any]:
raw = yaml.safe_load(path.read_text(encoding="utf-8"))
if raw is None:
return {}
if not isinstance(raw, dict):
raise ValueError(f"queue root must be mapping: {path}")
return raw
def write_yaml(path: Path, payload: dict[str, Any]) -> None:
path.write_text(
yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8"
@@ -73,7 +82,11 @@ def record_files(evidence_dir: Path) -> list[Path]:
def find_record_path(
evidence_dir: Path, service: str, environment: str, queue_id: str | None
evidence_dir: Path,
service: str,
environment: str,
queue_id: str | None,
prefer_pending: bool,
) -> Path | None:
candidates: list[tuple[datetime, Path]] = []
for path in record_files(evidence_dir):
@@ -100,6 +113,8 @@ def find_record_path(
)
if smoke_status not in {"pending", "pass", "fail", "unknown", None}:
continue
if prefer_pending and smoke_status != "pending":
continue
candidates.append((record_timestamp(record), path))
@@ -110,6 +125,39 @@ def find_record_path(
return candidates[0][1]
def latest_promoted_queue_id(
queue_payload: dict[str, Any], service: str, env: str
) -> str:
promoted = queue_payload.get("promoted", [])
if not isinstance(promoted, list):
return ""
candidates: list[tuple[datetime, str]] = []
for item in promoted:
if not isinstance(item, dict):
continue
if str(item.get("service", "")).strip() != service:
continue
if str(item.get("env", "")).strip() != env:
continue
queue_id = str(item.get("id", "")).strip()
if not queue_id:
continue
promoted_at = parse_ts(item.get("promotedAt"))
created_at = parse_ts(item.get("createdAt"))
ts = (
promoted_at
if promoted_at > datetime.min.replace(tzinfo=timezone.utc)
else created_at
)
candidates.append((ts, queue_id))
if not candidates:
return ""
candidates.sort(key=lambda item: item[0], reverse=True)
return candidates[0][1]
def http_get(
url: str, headers: dict[str, str] | None = None, timeout: float = 5.0
) -> tuple[int, str]:
@@ -193,21 +241,42 @@ def update_smoke_record(path: Path, ok: bool, details: str, dry_run: bool) -> No
write_yaml(path, record)
def run_target(target: dict[str, Any], args: argparse.Namespace) -> tuple[str, str]:
def run_target(
target: dict[str, Any], args: argparse.Namespace, queue_payload: dict[str, Any]
) -> tuple[str, str]:
service = str(target.get("service", "")).strip()
environment = str(target.get("environment", "dev")).strip()
queue_id = target.get("queue_id")
queue_id = str(target.get("queue_id", "")).strip()
app = str(target.get("argocd_app", "")).strip()
endpoint = str(target.get("endpoint", "")).strip()
if not service or not endpoint:
return "fail", f"invalid target: {target}"
record_path = find_record_path(args.evidence_dir, service, environment, queue_id)
resolved_queue_id = queue_id or latest_promoted_queue_id(
queue_payload=queue_payload,
service=service,
env=environment,
)
record_path = find_record_path(
evidence_dir=args.evidence_dir,
service=service,
environment=environment,
queue_id=resolved_queue_id or None,
prefer_pending=True,
)
if not record_path:
record_path = find_record_path(
evidence_dir=args.evidence_dir,
service=service,
environment=environment,
queue_id=resolved_queue_id or None,
prefer_pending=False,
)
if not record_path:
return (
"skip",
f"{service}: no promoted evidence record for {service}/{environment}",
f"{service}: no promoted evidence record for {service}/{environment} (queue={resolved_queue_id or '-'})",
)
ok_argocd, argocd_details = wait_for_argocd_health(
@@ -241,12 +310,18 @@ def parse_args() -> argparse.Namespace:
parser.add_argument(
"--targets", type=Path, default=Path("scripts/smoke/targets.json")
)
parser.add_argument("--queue", type=Path, default=Path("release/queue.yaml"))
parser.add_argument("--evidence-dir", type=Path, default=Path("evidence/records"))
parser.add_argument("--argocd-server", default=os.getenv("ARGOCD_SERVER", ""))
parser.add_argument("--argocd-token", default=os.getenv("ARGOCD_TOKEN", ""))
parser.add_argument("--timeout-seconds", type=int, default=180)
parser.add_argument("--interval-seconds", type=int, default=10)
parser.add_argument("--dry-run", action="store_true")
parser.add_argument(
"--allow-failures",
action="store_true",
help="return zero even when smoke failures exist (useful for CI structural checks)",
)
return parser.parse_args()
@@ -254,12 +329,13 @@ def main() -> int:
args = parse_args()
config = read_json(args.targets)
targets = list(config.get("targets", []))
queue_payload = read_queue(args.queue)
passed = 0
failed = 0
skipped = 0
for target in targets:
outcome, message = run_target(target, args)
outcome, message = run_target(target, args, queue_payload)
print(message)
if outcome == "pass":
passed += 1
@@ -275,10 +351,11 @@ def main() -> int:
"failed": failed,
"skipped": skipped,
"dry_run": args.dry_run,
"allow_failures": args.allow_failures,
}
)
)
return 0 if failed == 0 else 1
return 0 if failed == 0 or args.allow_failures else 1
if __name__ == "__main__":
scripts/verify.sh
+1
View File
@@ -10,6 +10,7 @@ uvx --with pyyaml python scripts/promoter/validate_queue.py
echo "[verify] validate and collect evidence feed"
uvx --with pyyaml --with jsonschema python scripts/evidence/validate.py
uvx --with pyyaml --with jsonschema python scripts/evidence/collect.py --out /tmp/evidence-index.json --summary /tmp/evidence-latest.md
uvx --with pyyaml python scripts/promoter/queue_metrics.py --queue release/queue.yaml --out /tmp/queue-health.json
echo "[verify] run promoter dry-run on local repo"
python3 scripts/promoter/promote.py --dry-run --local-repo-dir .