365 lines
12 KiB
Python
365 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Export markdown knowledge sources into a portable artifact."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import shutil
|
|
import tempfile
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
from .common import (
|
|
ARTIFACT_SCHEMA_RELATIVE,
|
|
JSONMapping,
|
|
KnowledgeContractError,
|
|
build_document_id,
|
|
control_files_changed,
|
|
extract_markdown_title,
|
|
load_review_entries,
|
|
load_source_rules,
|
|
load_validator,
|
|
match_globs,
|
|
parse_diff_entries,
|
|
repo_slug,
|
|
resolve_document_review,
|
|
require_non_empty_str,
|
|
utc_now,
|
|
validate_payload,
|
|
write_json,
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
"""Parse CLI arguments."""
|
|
|
|
parser = argparse.ArgumentParser(description="Export a knowledge artifact")
|
|
parser.add_argument(
|
|
"--repo-root", type=Path, required=True, help="Business repo root"
|
|
)
|
|
parser.add_argument(
|
|
"--deploy-root",
|
|
type=Path,
|
|
required=True,
|
|
help="ljwx-deploy root used for contracts",
|
|
)
|
|
parser.add_argument(
|
|
"--config",
|
|
type=Path,
|
|
default=Path("knowledge.sources.yaml"),
|
|
help="Path to knowledge.sources.yaml relative to repo root",
|
|
)
|
|
parser.add_argument(
|
|
"--review-ledger",
|
|
type=Path,
|
|
default=Path("knowledge.review.yaml"),
|
|
help="Path to knowledge.review.yaml relative to repo root",
|
|
)
|
|
parser.add_argument("--source-repo", required=True, help="GitHub owner/repo")
|
|
parser.add_argument("--source-ref", required=True, help="Git ref")
|
|
parser.add_argument("--source-sha", required=True, help="Git commit SHA")
|
|
parser.add_argument("--workflow-name", required=True, help="Workflow name")
|
|
parser.add_argument("--run-id", type=int, required=True, help="GitHub run id")
|
|
parser.add_argument(
|
|
"--run-attempt", type=int, required=True, help="GitHub run attempt"
|
|
)
|
|
parser.add_argument(
|
|
"--changed-files",
|
|
type=Path,
|
|
help="Path to git diff --name-status output. If omitted, export all matches.",
|
|
)
|
|
parser.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
required=True,
|
|
help="Artifact zip output path",
|
|
)
|
|
parser.add_argument(
|
|
"--summary-json",
|
|
type=Path,
|
|
required=True,
|
|
help="Where to write export summary JSON",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def resolve_rule_for_path(path: str, rules: list[object]) -> object | None:
|
|
"""Find a single matching rule for a source path."""
|
|
|
|
matched: list[object] = []
|
|
for rule in rules:
|
|
if not hasattr(rule, "include") or not hasattr(rule, "exclude"):
|
|
continue
|
|
include = getattr(rule, "include")
|
|
exclude = getattr(rule, "exclude")
|
|
if not isinstance(include, tuple) or not isinstance(exclude, tuple):
|
|
continue
|
|
if match_globs(path, include, exclude):
|
|
matched.append(rule)
|
|
if len(matched) > 1:
|
|
raise KnowledgeContractError(f"路径命中了多个 source rule: {path}")
|
|
return matched[0] if matched else None
|
|
|
|
|
|
def build_manifest_document(
|
|
*,
|
|
source_repo: str,
|
|
source_sha: str,
|
|
relative_path: str,
|
|
source_id: str,
|
|
visibility: str,
|
|
source_type: str,
|
|
doc_type: str,
|
|
product_area: str,
|
|
owner: str,
|
|
title_strategy: str,
|
|
title_prefix: str,
|
|
tags: tuple[str, ...],
|
|
content: str,
|
|
review_status: str,
|
|
approved_by: str | None,
|
|
effective_from: str | None,
|
|
expires_at: str | None,
|
|
checksum: str,
|
|
reason: str | None,
|
|
) -> tuple[JSONMapping, str]:
|
|
"""Build one upsert artifact document entry and its content payload."""
|
|
|
|
document_id = build_document_id(source_repo, relative_path)
|
|
title = extract_markdown_title(content, relative_path, title_strategy)
|
|
if title_prefix:
|
|
title = f"{title_prefix}{title}"
|
|
content_path = f"documents/{document_id}.md"
|
|
source_ref = f"{source_repo}@{source_sha}:{relative_path}"
|
|
payload: JSONMapping = {
|
|
"op": "upsert",
|
|
"document_id": document_id,
|
|
"source_id": source_id,
|
|
"source_path": relative_path,
|
|
"source_ref": source_ref,
|
|
"content_path": content_path,
|
|
"title": title,
|
|
"visibility": visibility,
|
|
"source_type": source_type,
|
|
"doc_type": doc_type,
|
|
"product_area": product_area,
|
|
"owner": owner,
|
|
"review_status": review_status,
|
|
"version": source_sha,
|
|
"checksum": checksum,
|
|
"tags": list(tags),
|
|
"approved_by": approved_by,
|
|
"effective_from": effective_from,
|
|
"expires_at": expires_at,
|
|
"reason": reason,
|
|
}
|
|
return payload, content_path
|
|
|
|
|
|
def build_invalidate_document(
|
|
*,
|
|
source_repo: str,
|
|
source_sha: str,
|
|
relative_path: str,
|
|
source_id: str,
|
|
) -> JSONMapping:
|
|
"""Build one invalidate artifact document entry."""
|
|
|
|
return {
|
|
"op": "invalidate",
|
|
"document_id": build_document_id(source_repo, relative_path),
|
|
"source_id": source_id,
|
|
"source_path": relative_path,
|
|
"source_ref": f"{source_repo}@{source_sha}:{relative_path}",
|
|
"content_path": None,
|
|
"title": None,
|
|
"visibility": None,
|
|
"source_type": None,
|
|
"doc_type": None,
|
|
"product_area": None,
|
|
"owner": None,
|
|
"review_status": None,
|
|
"version": None,
|
|
"checksum": None,
|
|
"tags": [],
|
|
"approved_by": None,
|
|
"effective_from": None,
|
|
"expires_at": None,
|
|
"reason": "source_deleted",
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
"""CLI entry point."""
|
|
|
|
args = parse_args()
|
|
repo_root = args.repo_root.resolve()
|
|
deploy_root = args.deploy_root.resolve()
|
|
config_path = (repo_root / args.config).resolve()
|
|
review_ledger_path = (repo_root / args.review_ledger).resolve()
|
|
output_path = args.output.resolve()
|
|
summary_path = args.summary_json.resolve()
|
|
|
|
service, rules = load_source_rules(config_path, deploy_root)
|
|
review_entries = load_review_entries(review_ledger_path, deploy_root)
|
|
artifact_validator = load_validator(deploy_root / ARTIFACT_SCHEMA_RELATIVE)
|
|
diff_entries = (
|
|
parse_diff_entries(args.changed_files.resolve()) if args.changed_files else []
|
|
)
|
|
|
|
changed_upserts = {
|
|
entry.path
|
|
for entry in diff_entries
|
|
if entry.change_type == "upsert" and entry.path.endswith(".md")
|
|
}
|
|
changed_deletes = {
|
|
entry.path for entry in diff_entries if entry.change_type == "delete"
|
|
}
|
|
export_all = not diff_entries or control_files_changed(
|
|
diff_entries,
|
|
{args.config.as_posix(), args.review_ledger.as_posix()},
|
|
)
|
|
|
|
live_documents: list[JSONMapping] = []
|
|
deleted_documents: list[JSONMapping] = []
|
|
staged_documents: list[tuple[str, str]] = []
|
|
|
|
if export_all:
|
|
candidate_paths = sorted(
|
|
path.relative_to(repo_root).as_posix()
|
|
for path in repo_root.rglob("*.md")
|
|
if path.is_file()
|
|
)
|
|
else:
|
|
candidate_paths = sorted(changed_upserts)
|
|
|
|
for relative_path in candidate_paths:
|
|
rule = resolve_rule_for_path(relative_path, rules)
|
|
if rule is None:
|
|
continue
|
|
content = (repo_root / relative_path).read_text(encoding="utf-8")
|
|
resolved_review = resolve_document_review(
|
|
rule=rule,
|
|
relative_path=relative_path,
|
|
content=content,
|
|
review_entries=review_entries,
|
|
)
|
|
rule_source_id = require_non_empty_str(
|
|
getattr(rule, "source_id"), "rule.source_id"
|
|
)
|
|
payload, content_path = build_manifest_document(
|
|
source_repo=args.source_repo,
|
|
source_sha=args.source_sha,
|
|
relative_path=relative_path,
|
|
source_id=rule_source_id,
|
|
visibility=require_non_empty_str(
|
|
getattr(rule, "visibility"), "rule.visibility"
|
|
),
|
|
source_type=require_non_empty_str(
|
|
getattr(rule, "source_type"), "rule.source_type"
|
|
),
|
|
doc_type=require_non_empty_str(getattr(rule, "doc_type"), "rule.doc_type"),
|
|
product_area=require_non_empty_str(
|
|
getattr(rule, "product_area"), "rule.product_area"
|
|
),
|
|
owner=require_non_empty_str(getattr(rule, "owner"), "rule.owner"),
|
|
title_strategy=require_non_empty_str(
|
|
getattr(rule, "title_strategy"), "rule.title_strategy"
|
|
),
|
|
title_prefix=require_non_empty_str(
|
|
getattr(rule, "title_prefix", ""), "rule.title_prefix"
|
|
)
|
|
if getattr(rule, "title_prefix", "") != ""
|
|
else "",
|
|
tags=getattr(rule, "tags"),
|
|
content=content,
|
|
review_status=resolved_review.review_status,
|
|
approved_by=resolved_review.approved_by,
|
|
effective_from=resolved_review.effective_from,
|
|
expires_at=resolved_review.expires_at,
|
|
checksum=resolved_review.checksum,
|
|
reason=resolved_review.reason,
|
|
)
|
|
live_documents.append(payload)
|
|
staged_documents.append((content_path, content))
|
|
|
|
for relative_path in sorted(changed_deletes):
|
|
rule = resolve_rule_for_path(relative_path, rules)
|
|
if rule is None:
|
|
continue
|
|
deleted_documents.append(
|
|
build_invalidate_document(
|
|
source_repo=args.source_repo,
|
|
source_sha=args.source_sha,
|
|
relative_path=relative_path,
|
|
source_id=require_non_empty_str(
|
|
getattr(rule, "source_id"), "rule.source_id"
|
|
),
|
|
)
|
|
)
|
|
|
|
all_documents = deleted_documents + live_documents
|
|
artifact_id = f"{repo_slug(args.source_repo)}-{args.source_sha[:12]}"
|
|
summary: JSONMapping = {
|
|
"artifact_id": artifact_id,
|
|
"artifact_file": str(output_path),
|
|
"deleted_documents": len(deleted_documents),
|
|
"exported": bool(all_documents),
|
|
"service": service,
|
|
"source_repo": args.source_repo,
|
|
"upsert_documents": len(live_documents),
|
|
}
|
|
|
|
if not all_documents:
|
|
write_json(summary_path, summary)
|
|
return 0
|
|
|
|
manifest: JSONMapping = {
|
|
"schema_version": "0.1.0",
|
|
"artifact_id": artifact_id,
|
|
"generated_at": utc_now(),
|
|
"export_mode": "full" if export_all else "changed",
|
|
"source": {
|
|
"repo": args.source_repo,
|
|
"service": service,
|
|
"commit": args.source_sha,
|
|
"ref": args.source_ref,
|
|
"workflow": args.workflow_name,
|
|
"run_id": args.run_id,
|
|
"run_attempt": args.run_attempt,
|
|
},
|
|
"documents": all_documents,
|
|
}
|
|
validate_payload(artifact_validator, manifest, "knowledge artifact manifest")
|
|
|
|
staging_root = Path(
|
|
tempfile.mkdtemp(prefix="knowledge-export-", dir=output_path.parent.as_posix())
|
|
)
|
|
try:
|
|
manifest_path = staging_root / "manifest.json"
|
|
write_json(manifest_path, manifest)
|
|
for content_path, content in staged_documents:
|
|
target_path = staging_root / content_path
|
|
target_path.parent.mkdir(parents=True, exist_ok=True)
|
|
target_path.write_text(content, encoding="utf-8")
|
|
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with zipfile.ZipFile(
|
|
output_path, "w", compression=zipfile.ZIP_DEFLATED
|
|
) as archive:
|
|
for path in sorted(staging_root.rglob("*")):
|
|
if path.is_dir():
|
|
continue
|
|
archive.write(path, path.relative_to(staging_root))
|
|
finally:
|
|
shutil.rmtree(staging_root, ignore_errors=True)
|
|
|
|
write_json(summary_path, summary)
|
|
print(json.dumps(summary, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|