Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e3917e05e0 |
@@ -66,6 +66,9 @@ jobs:
|
||||
- name: Validate release queue structure
|
||||
run: uvx --with pyyaml python scripts/promoter/validate_queue.py
|
||||
|
||||
- name: Validate knowledge publish records
|
||||
run: uvx --with pyyaml --with jsonschema python -m scripts.knowledge.validate_state --deploy-root . --env dev
|
||||
|
||||
- name: Evidence validate + collect (dry run)
|
||||
run: |
|
||||
uvx --with pyyaml --with jsonschema python scripts/evidence/validate.py
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
name: knowledge-ingest
|
||||
|
||||
on:
|
||||
repository_dispatch:
|
||||
types:
|
||||
- knowledge_artifact_ready
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_repo:
|
||||
description: "source repository (owner/repo)"
|
||||
required: true
|
||||
type: string
|
||||
source_run_id:
|
||||
description: "source workflow run id"
|
||||
required: true
|
||||
type: string
|
||||
artifact_name:
|
||||
description: "artifact name to ingest"
|
||||
required: true
|
||||
type: string
|
||||
target_env:
|
||||
description: "target environment"
|
||||
required: true
|
||||
default: dev
|
||||
type: choice
|
||||
options:
|
||||
- dev
|
||||
skip_dify:
|
||||
description: "skip live Dify publication"
|
||||
required: true
|
||||
default: "false"
|
||||
type: choice
|
||||
options:
|
||||
- "false"
|
||||
- "true"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
group: knowledge-ingest-${{ github.event_name }}-${{ github.event_name == 'repository_dispatch' && github.event.client_payload.source_repo || inputs.source_repo }}-${{ github.event_name == 'repository_dispatch' && github.event.client_payload.source_run_id || inputs.source_run_id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
ingest:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
KNOWLEDGE_DIFY_BASE_URL: ${{ secrets.KNOWLEDGE_DIFY_BASE_URL }}
|
||||
KNOWLEDGE_DIFY_API_KEY: ${{ secrets.KNOWLEDGE_DIFY_API_KEY }}
|
||||
KNOWLEDGE_GITHUB_TOKEN: ${{ secrets.DEPLOY_REPO_TOKEN || github.token }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Setup uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Resolve inputs
|
||||
id: prep
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${GITHUB_EVENT_NAME}" = "repository_dispatch" ]; then
|
||||
echo "source_repo=${{ github.event.client_payload.source_repo }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "source_run_id=${{ github.event.client_payload.source_run_id }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "artifact_name=${{ github.event.client_payload.artifact_name }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "target_env=${{ github.event.client_payload.target_env || 'dev' }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "skip_dify=${{ github.event.client_payload.skip_dify || 'false' }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "source_sha=${{ github.event.client_payload.source_sha || '' }}" >> "${GITHUB_OUTPUT}"
|
||||
else
|
||||
echo "source_repo=${{ inputs.source_repo }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "source_run_id=${{ inputs.source_run_id }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "artifact_name=${{ inputs.artifact_name }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "target_env=${{ inputs.target_env }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "skip_dify=${{ inputs.skip_dify }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "source_sha=" >> "${GITHUB_OUTPUT}"
|
||||
fi
|
||||
|
||||
- name: Ingest artifact
|
||||
run: |
|
||||
uvx --with pyyaml --with jsonschema python -m scripts.knowledge.ingest_artifact \
|
||||
--deploy-root . \
|
||||
--env "${{ steps.prep.outputs.target_env }}" \
|
||||
--source-repo "${{ steps.prep.outputs.source_repo }}" \
|
||||
--source-run-id "${{ steps.prep.outputs.source_run_id }}" \
|
||||
--artifact-name "${{ steps.prep.outputs.artifact_name }}" \
|
||||
--github-token "${KNOWLEDGE_GITHUB_TOKEN}" \
|
||||
--dify-base-url "${KNOWLEDGE_DIFY_BASE_URL}" \
|
||||
--dify-api-key "${KNOWLEDGE_DIFY_API_KEY}" \
|
||||
${{ steps.prep.outputs.skip_dify == 'true' && '--skip-dify' || '' }} \
|
||||
--summary-json /tmp/knowledge-ingest-summary.json
|
||||
cat /tmp/knowledge-ingest-summary.json
|
||||
|
||||
- name: Validate committed knowledge state
|
||||
run: |
|
||||
uvx --with pyyaml --with jsonschema python -m scripts.knowledge.validate_state \
|
||||
--deploy-root . \
|
||||
--env "${{ steps.prep.outputs.target_env }}"
|
||||
|
||||
- name: Commit knowledge state
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet -- knowledge; then
|
||||
echo "No knowledge changes to commit"
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "knowledge-publisher[bot]"
|
||||
git config user.email "knowledge-publisher[bot]@users.noreply.github.com"
|
||||
git add knowledge
|
||||
git commit -m "feat(knowledge): ingest artifact from ${{ steps.prep.outputs.source_repo }}@${{ steps.prep.outputs.source_sha || 'workflow' }}"
|
||||
git push
|
||||
@@ -0,0 +1,84 @@
|
||||
name: knowledge-reconcile
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target_env:
|
||||
description: "target environment"
|
||||
required: true
|
||||
default: dev
|
||||
type: choice
|
||||
options:
|
||||
- dev
|
||||
source_repo:
|
||||
description: "optional source repo filter"
|
||||
required: false
|
||||
type: string
|
||||
document_id:
|
||||
description: "optional document id filter"
|
||||
required: false
|
||||
type: string
|
||||
skip_dify:
|
||||
description: "skip live Dify publication"
|
||||
required: true
|
||||
default: "false"
|
||||
type: choice
|
||||
options:
|
||||
- "false"
|
||||
- "true"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
reconcile:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
KNOWLEDGE_DIFY_BASE_URL: ${{ secrets.KNOWLEDGE_DIFY_BASE_URL }}
|
||||
KNOWLEDGE_DIFY_API_KEY: ${{ secrets.KNOWLEDGE_DIFY_API_KEY }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Setup uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Reconcile publish records
|
||||
run: |
|
||||
uvx --with pyyaml --with jsonschema python -m scripts.knowledge.reconcile_records \
|
||||
--deploy-root . \
|
||||
--env "${{ inputs.target_env }}" \
|
||||
${{ inputs.source_repo != '' && format('--source-repo "{0}"', inputs.source_repo) || '' }} \
|
||||
${{ inputs.document_id != '' && format('--document-id "{0}"', inputs.document_id) || '' }} \
|
||||
--dify-base-url "${KNOWLEDGE_DIFY_BASE_URL}" \
|
||||
--dify-api-key "${KNOWLEDGE_DIFY_API_KEY}" \
|
||||
${{ inputs.skip_dify == 'true' && '--skip-dify' || '' }} \
|
||||
--summary-json /tmp/knowledge-reconcile-summary.json
|
||||
cat /tmp/knowledge-reconcile-summary.json
|
||||
|
||||
- name: Validate committed knowledge state
|
||||
run: |
|
||||
uvx --with pyyaml --with jsonschema python -m scripts.knowledge.validate_state \
|
||||
--deploy-root . \
|
||||
--env "${{ inputs.target_env }}"
|
||||
|
||||
- name: Commit reconciled state
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet -- knowledge; then
|
||||
echo "No knowledge changes to commit"
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "knowledge-publisher[bot]"
|
||||
git config user.email "knowledge-publisher[bot]@users.noreply.github.com"
|
||||
git add knowledge
|
||||
git commit -m "feat(knowledge): reconcile published records"
|
||||
git push
|
||||
@@ -0,0 +1,90 @@
|
||||
name: knowledge-unpublish
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target_env:
|
||||
description: "target environment"
|
||||
required: true
|
||||
default: dev
|
||||
type: choice
|
||||
options:
|
||||
- dev
|
||||
source_repo:
|
||||
description: "source repository (owner/repo)"
|
||||
required: true
|
||||
type: string
|
||||
document_id:
|
||||
description: "document id to unpublish"
|
||||
required: true
|
||||
type: string
|
||||
reason:
|
||||
description: "unpublish reason"
|
||||
required: true
|
||||
default: manual_unpublish
|
||||
type: string
|
||||
skip_dify:
|
||||
description: "skip live Dify publication"
|
||||
required: true
|
||||
default: "false"
|
||||
type: choice
|
||||
options:
|
||||
- "false"
|
||||
- "true"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
unpublish:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
KNOWLEDGE_DIFY_BASE_URL: ${{ secrets.KNOWLEDGE_DIFY_BASE_URL }}
|
||||
KNOWLEDGE_DIFY_API_KEY: ${{ secrets.KNOWLEDGE_DIFY_API_KEY }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Setup uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Unpublish record
|
||||
run: |
|
||||
uvx --with pyyaml --with jsonschema python -m scripts.knowledge.unpublish_records \
|
||||
--deploy-root . \
|
||||
--env "${{ inputs.target_env }}" \
|
||||
--source-repo "${{ inputs.source_repo }}" \
|
||||
--document-id "${{ inputs.document_id }}" \
|
||||
--reason "${{ inputs.reason }}" \
|
||||
--dify-base-url "${KNOWLEDGE_DIFY_BASE_URL}" \
|
||||
--dify-api-key "${KNOWLEDGE_DIFY_API_KEY}" \
|
||||
${{ inputs.skip_dify == 'true' && '--skip-dify' || '' }} \
|
||||
--summary-json /tmp/knowledge-unpublish-summary.json
|
||||
cat /tmp/knowledge-unpublish-summary.json
|
||||
|
||||
- name: Validate committed knowledge state
|
||||
run: |
|
||||
uvx --with pyyaml --with jsonschema python -m scripts.knowledge.validate_state \
|
||||
--deploy-root . \
|
||||
--env "${{ inputs.target_env }}"
|
||||
|
||||
- name: Commit unpublished state
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet -- knowledge; then
|
||||
echo "No knowledge changes to commit"
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "knowledge-publisher[bot]"
|
||||
git config user.email "knowledge-publisher[bot]@users.noreply.github.com"
|
||||
git add knowledge
|
||||
git commit -m "feat(knowledge): unpublish ${{ inputs.document_id }}"
|
||||
git push
|
||||
@@ -2,27 +2,37 @@
|
||||
|
||||
## Positioning
|
||||
|
||||
The raw source of truth stays in Git-managed repositories and approved source systems.
|
||||
Dify only consumes processed outputs.
|
||||
The raw source of truth stays in Git-managed business repositories and approved source systems.
|
||||
Dify only consumes centrally processed outputs.
|
||||
|
||||
The baseline Git-managed knowledge repository is `ljwx-knowledge` with:
|
||||
The current MVP is a two-layer pipeline:
|
||||
|
||||
- `content/public/**`
|
||||
- `content/internal/**`
|
||||
- `content/customers/{customer_id}/**`
|
||||
1. business repositories export standardized knowledge artifacts
|
||||
2. `ljwx-deploy` ingests those artifacts, applies centralized policy, and publishes to Dify dev
|
||||
|
||||
## Pipeline
|
||||
|
||||
1. collect
|
||||
### Layer 1: Business Repo Export
|
||||
|
||||
Each business repo only owns:
|
||||
|
||||
1. declare included markdown sources in `knowledge.sources.yaml`
|
||||
2. run light validation
|
||||
3. export a portable artifact (`manifest.json` + `documents/*.md`)
|
||||
4. dispatch the artifact to `ljwx-deploy`
|
||||
|
||||
### Layer 2: Central Ingest
|
||||
|
||||
`ljwx-deploy` owns:
|
||||
|
||||
1. artifact validation
|
||||
2. normalize
|
||||
3. classify
|
||||
4. deduplicate
|
||||
5. chunk
|
||||
6. tag
|
||||
7. apply ACL
|
||||
8. review
|
||||
9. publish
|
||||
10. invalidate or unpublish
|
||||
6. apply ACL
|
||||
7. publish / invalidate / republish
|
||||
8. write publish records and event history in Git
|
||||
|
||||
## Publish Targets
|
||||
|
||||
@@ -30,6 +40,13 @@ The baseline Git-managed knowledge repository is `ljwx-knowledge` with:
|
||||
- internal retrieval collections for platform and support use cases
|
||||
- customer-scoped retrieval collections when tenant isolation is required
|
||||
|
||||
MVP scope:
|
||||
|
||||
1. markdown only
|
||||
2. `public` / `internal` visibility only
|
||||
3. dev Dify dataset publication first
|
||||
4. retrieval targets stay declared in config but are not yet executed by the MVP workflow
|
||||
|
||||
## Invalidate Triggers
|
||||
|
||||
Publishing is not append-only. A document or chunk must be invalidated when:
|
||||
@@ -44,26 +61,29 @@ The target configuration lives in `platform/knowledge/publish-targets.yaml`.
|
||||
|
||||
## Directory and ACL Contract
|
||||
|
||||
- `platform/knowledge/source-registry.yaml` defines which repositories and paths are collectible.
|
||||
- `platform/knowledge/source-registry.yaml` defines which repositories can export artifacts and which source IDs are centrally allowed.
|
||||
- `platform/knowledge/taxonomy.yaml` defines the path-to-classification boundary.
|
||||
- `platform/knowledge/acl-policies.yaml` defines visibility-specific `acl_expression_template` and allowed publish targets.
|
||||
- `platform/contracts/knowledge-document.schema.yaml` and `platform/contracts/knowledge-chunk.schema.yaml` define the minimum manifest shape for normalize/classify/chunk outputs.
|
||||
- `platform/contracts/knowledge-source-config.schema.yaml` defines business repo export config.
|
||||
- `platform/contracts/knowledge-artifact.schema.yaml` defines the exported artifact manifest.
|
||||
- `platform/contracts/knowledge-document.schema.yaml` and `platform/contracts/knowledge-chunk.schema.yaml` define the minimum normalize/classify/chunk outputs.
|
||||
- `platform/contracts/knowledge-publish-record.schema.yaml` defines the committed central publication record.
|
||||
|
||||
## Minimum Execution Flow
|
||||
|
||||
The current MVP path is:
|
||||
|
||||
1. ingest markdown sources from `ljwx-knowledge`
|
||||
2. emit normalized document manifests
|
||||
3. enrich `doc_type`, `product_area`, and `acl_expression`
|
||||
4. chunk approved documents
|
||||
5. publish to local dev datasets
|
||||
6. invalidate or unpublish by `document_id` and republish when a rollback is needed
|
||||
1. business repo exports a markdown artifact from `knowledge.sources.yaml`
|
||||
2. `knowledge-ingest` validates and snapshots the artifact in `ljwx-deploy/knowledge/artifacts/dev/`
|
||||
3. central scripts emit normalized/classified/chunk manifests
|
||||
4. central publish resolves dataset mapping from `platform/knowledge/publish-targets.yaml`
|
||||
5. central workflow writes publish records to `knowledge/publish-records/dev/`
|
||||
6. `knowledge-unpublish` and `knowledge-reconcile` provide the minimal invalidate / republish loop
|
||||
|
||||
## Audit And Metrics
|
||||
|
||||
The current MVP also emits:
|
||||
The current MVP emits:
|
||||
|
||||
1. `state/<env>/events.jsonl` for lifecycle events
|
||||
2. `state/<env>/audit-events.jsonl` for cross-component audit evidence
|
||||
3. `state/<env>/metrics.json` for the latest publish or rollback counters
|
||||
1. `knowledge/events/<env>/events.jsonl` for append-only lifecycle events
|
||||
2. `knowledge/publish-records/<env>/**/*.json` for the latest central publication state
|
||||
3. `knowledge/artifacts/<env>/**` for replayable raw artifacts and derived manifests
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
$schema: https://json-schema.org/draft/2020-12/schema
|
||||
$id: https://lingjingwanxiang.cn/schemas/knowledge-artifact.schema.yaml
|
||||
title: KnowledgeArtifact
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required:
|
||||
- schema_version
|
||||
- artifact_id
|
||||
- generated_at
|
||||
- export_mode
|
||||
- source
|
||||
- documents
|
||||
properties:
|
||||
schema_version:
|
||||
type: string
|
||||
minLength: 1
|
||||
artifact_id:
|
||||
type: string
|
||||
minLength: 1
|
||||
generated_at:
|
||||
type: string
|
||||
format: date-time
|
||||
export_mode:
|
||||
type: string
|
||||
enum:
|
||||
- changed
|
||||
- full
|
||||
source:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required:
|
||||
- repo
|
||||
- service
|
||||
- commit
|
||||
- ref
|
||||
- workflow
|
||||
- run_id
|
||||
- run_attempt
|
||||
properties:
|
||||
repo:
|
||||
type: string
|
||||
minLength: 1
|
||||
service:
|
||||
type: string
|
||||
minLength: 1
|
||||
commit:
|
||||
type: string
|
||||
minLength: 1
|
||||
ref:
|
||||
type: string
|
||||
minLength: 1
|
||||
workflow:
|
||||
type: string
|
||||
minLength: 1
|
||||
run_id:
|
||||
type: integer
|
||||
minimum: 1
|
||||
run_attempt:
|
||||
type: integer
|
||||
minimum: 1
|
||||
documents:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required:
|
||||
- op
|
||||
- document_id
|
||||
- source_id
|
||||
- source_path
|
||||
- source_ref
|
||||
properties:
|
||||
op:
|
||||
type: string
|
||||
enum:
|
||||
- upsert
|
||||
- invalidate
|
||||
document_id:
|
||||
type: string
|
||||
minLength: 1
|
||||
source_id:
|
||||
type: string
|
||||
minLength: 1
|
||||
source_path:
|
||||
type: string
|
||||
minLength: 1
|
||||
source_ref:
|
||||
type: string
|
||||
minLength: 1
|
||||
content_path:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
title:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
visibility:
|
||||
anyOf:
|
||||
- type: string
|
||||
enum:
|
||||
- public
|
||||
- internal
|
||||
- type: "null"
|
||||
source_type:
|
||||
anyOf:
|
||||
- type: string
|
||||
enum:
|
||||
- repo_doc
|
||||
- faq
|
||||
- delivery_doc
|
||||
- deployment_doc
|
||||
- support_record
|
||||
- partner_reference
|
||||
- type: "null"
|
||||
doc_type:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
product_area:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
owner:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
review_status:
|
||||
anyOf:
|
||||
- type: string
|
||||
enum:
|
||||
- draft
|
||||
- reviewing
|
||||
- approved
|
||||
- rejected
|
||||
- type: "null"
|
||||
version:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
checksum:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
tags:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
approved_by:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
effective_from:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
format: date-time
|
||||
expires_at:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
format: date-time
|
||||
reason:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
@@ -0,0 +1,189 @@
|
||||
$schema: https://json-schema.org/draft/2020-12/schema
|
||||
$id: https://lingjingwanxiang.cn/schemas/knowledge-publish-record.schema.yaml
|
||||
title: KnowledgePublishRecord
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required:
|
||||
- schema_version
|
||||
- document_id
|
||||
- source_repo
|
||||
- source_id
|
||||
- source_path
|
||||
- visibility
|
||||
- checksum
|
||||
- status
|
||||
- artifact_ref
|
||||
- content_ref
|
||||
- normalized_ref
|
||||
- classified_ref
|
||||
- chunk_ref
|
||||
- latest_action
|
||||
- updated_at
|
||||
- targets
|
||||
properties:
|
||||
schema_version:
|
||||
type: string
|
||||
minLength: 1
|
||||
document_id:
|
||||
type: string
|
||||
minLength: 1
|
||||
source_repo:
|
||||
type: string
|
||||
minLength: 1
|
||||
source_id:
|
||||
type: string
|
||||
minLength: 1
|
||||
source_path:
|
||||
type: string
|
||||
minLength: 1
|
||||
source_ref:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
visibility:
|
||||
type: string
|
||||
enum:
|
||||
- public
|
||||
- internal
|
||||
checksum:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
title:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
source_type:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
doc_type:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
product_area:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
owner:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
review_status:
|
||||
anyOf:
|
||||
- type: string
|
||||
enum:
|
||||
- draft
|
||||
- reviewing
|
||||
- approved
|
||||
- rejected
|
||||
- type: "null"
|
||||
acl_expression:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
version:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
tags:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
status:
|
||||
type: string
|
||||
enum:
|
||||
- published
|
||||
- invalidated
|
||||
- unpublished
|
||||
- deduplicated
|
||||
duplicate_of:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
artifact_ref:
|
||||
type: string
|
||||
minLength: 1
|
||||
content_ref:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
normalized_ref:
|
||||
type: string
|
||||
minLength: 1
|
||||
classified_ref:
|
||||
type: string
|
||||
minLength: 1
|
||||
chunk_ref:
|
||||
type: string
|
||||
minLength: 1
|
||||
chunk_count:
|
||||
type: integer
|
||||
minimum: 0
|
||||
latest_action:
|
||||
type: string
|
||||
enum:
|
||||
- publish
|
||||
- invalidate
|
||||
- unpublish
|
||||
- republish
|
||||
latest_reason:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
published_at:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
format: date-time
|
||||
updated_at:
|
||||
type: string
|
||||
format: date-time
|
||||
targets:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required:
|
||||
- target_id
|
||||
- target_type
|
||||
- dataset_name
|
||||
- dataset_id
|
||||
- document_name
|
||||
- remote_document_id
|
||||
- status
|
||||
properties:
|
||||
target_id:
|
||||
type: string
|
||||
minLength: 1
|
||||
target_type:
|
||||
type: string
|
||||
minLength: 1
|
||||
dataset_name:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
dataset_id:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
document_name:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
remote_document_id:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
status:
|
||||
type: string
|
||||
enum:
|
||||
- published
|
||||
- invalidated
|
||||
- unpublished
|
||||
- skipped
|
||||
updated_at:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
format: date-time
|
||||
@@ -0,0 +1,101 @@
|
||||
$schema: https://json-schema.org/draft/2020-12/schema
|
||||
$id: https://lingjingwanxiang.cn/schemas/knowledge-source-config.schema.yaml
|
||||
title: KnowledgeSourceConfig
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required:
|
||||
- version
|
||||
- service
|
||||
- sources
|
||||
properties:
|
||||
version:
|
||||
type: string
|
||||
minLength: 1
|
||||
service:
|
||||
type: string
|
||||
minLength: 1
|
||||
sources:
|
||||
type: array
|
||||
minItems: 1
|
||||
items:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required:
|
||||
- id
|
||||
- include
|
||||
- visibility
|
||||
- source_type
|
||||
- doc_type
|
||||
- product_area
|
||||
- owner
|
||||
- review_status
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
minLength: 1
|
||||
include:
|
||||
type: array
|
||||
minItems: 1
|
||||
items:
|
||||
type: string
|
||||
minLength: 1
|
||||
exclude:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
minLength: 1
|
||||
visibility:
|
||||
type: string
|
||||
enum:
|
||||
- public
|
||||
- internal
|
||||
source_type:
|
||||
type: string
|
||||
enum:
|
||||
- repo_doc
|
||||
- faq
|
||||
- delivery_doc
|
||||
- deployment_doc
|
||||
- support_record
|
||||
- partner_reference
|
||||
doc_type:
|
||||
type: string
|
||||
minLength: 1
|
||||
product_area:
|
||||
type: string
|
||||
minLength: 1
|
||||
owner:
|
||||
type: string
|
||||
minLength: 1
|
||||
review_status:
|
||||
type: string
|
||||
enum:
|
||||
- draft
|
||||
- reviewing
|
||||
- approved
|
||||
- rejected
|
||||
tags:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
title_strategy:
|
||||
type: string
|
||||
enum:
|
||||
- first_heading
|
||||
- filename
|
||||
title_prefix:
|
||||
type: string
|
||||
approved_by:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
effective_from:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
format: date-time
|
||||
expires_at:
|
||||
type:
|
||||
- string
|
||||
- "null"
|
||||
format: date-time
|
||||
@@ -1,4 +1,4 @@
|
||||
version: 0.1.0
|
||||
version: 0.2.0
|
||||
|
||||
targets:
|
||||
dify-public:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: 0.1.0
|
||||
version: 0.2.0
|
||||
|
||||
sources:
|
||||
- id: ljwx-knowledge-public
|
||||
@@ -46,3 +46,13 @@ sources:
|
||||
- content/faq/**
|
||||
visibility: public
|
||||
owner: team-product
|
||||
|
||||
artifact_sources:
|
||||
- repo: BrunoGaoSZ/ljwx-bookstore
|
||||
service: ljwx-bookstore
|
||||
source_ids:
|
||||
- bookstore-public-readme
|
||||
- bookstore-internal-ai-plan
|
||||
allowed_visibilities:
|
||||
- public
|
||||
- internal
|
||||
|
||||
@@ -52,11 +52,15 @@ config_versions:
|
||||
service_capability_map: platform/contracts/service-capability-map.yaml@0.2.1
|
||||
capability_gateway_request: platform/contracts/capability-gateway-request.schema.json@0.1.0
|
||||
audit_event_schema: platform/contracts/audit-event.schema.json@0.1.0
|
||||
knowledge_source_registry: platform/knowledge/source-registry.yaml@0.1.0
|
||||
knowledge_source_registry: platform/knowledge/source-registry.yaml@0.2.0
|
||||
knowledge_publish_targets: platform/knowledge/publish-targets.yaml@0.2.0
|
||||
knowledge_taxonomy: platform/knowledge/taxonomy.yaml@0.1.0
|
||||
knowledge_acl_policies: platform/knowledge/acl-policies.yaml@0.1.0
|
||||
knowledge_document_schema: platform/contracts/knowledge-document.schema.yaml@0.1.1
|
||||
knowledge_chunk_schema: platform/contracts/knowledge-chunk.schema.yaml@0.1.1
|
||||
knowledge_source_config_schema: platform/contracts/knowledge-source-config.schema.yaml@0.1.0
|
||||
knowledge_artifact_schema: platform/contracts/knowledge-artifact.schema.yaml@0.1.0
|
||||
knowledge_publish_record_schema: platform/contracts/knowledge-publish-record.schema.yaml@0.1.0
|
||||
workflow_templates_repo: BrunoGaoSZ/ljwx-workflow-templates@406980fc99f7
|
||||
workflow_build_template: BrunoGaoSZ/ljwx-workflow-templates/.github/workflows/build-ghcr.yml@406980fc99f7
|
||||
workflow_enqueue_template: BrunoGaoSZ/ljwx-workflow-templates/.github/workflows/enqueue-release.yml@406980fc99f7
|
||||
|
||||
@@ -5,6 +5,7 @@ uvx --with pyyaml --with jsonschema python scripts/evidence/validate.py
|
||||
uvx --with pyyaml --with jsonschema python scripts/evidence/collect.py --out evidence/index.json
|
||||
uvx --with pyyaml --with jsonschema python scripts/platform/validate_router_contracts.py
|
||||
uvx --with pyyaml --with jsonschema python scripts/platform/validate_capability_contracts.py
|
||||
uvx --with pyyaml --with jsonschema python -m scripts.knowledge.validate_state --deploy-root . --env dev
|
||||
uvx --with pyyaml python scripts/promoter/validate_queue.py
|
||||
uvx --with pyyaml python scripts/promoter/queue_metrics.py --queue release/queue.yaml --out /tmp/queue-health.json
|
||||
bash scripts/promoter/promote.sh --dry-run
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Knowledge supply chain scripts."""
|
||||
@@ -0,0 +1,681 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for the knowledge supply chain MVP."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
try:
|
||||
import jsonschema
|
||||
except ImportError as exc:
|
||||
raise SystemExit(
|
||||
f"缺少依赖 jsonschema,请使用 uvx --with jsonschema --with pyyaml python <script>\n{exc}"
|
||||
) from exc
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError as exc:
|
||||
raise SystemExit(
|
||||
f"缺少依赖 PyYAML,请使用 uvx --with pyyaml --with jsonschema python <script>\n{exc}"
|
||||
) from exc
|
||||
|
||||
|
||||
JSONMapping = dict[str, object]
|
||||
|
||||
SOURCE_CONFIG_SCHEMA_RELATIVE: Final[Path] = Path(
|
||||
"platform/contracts/knowledge-source-config.schema.yaml"
|
||||
)
|
||||
ARTIFACT_SCHEMA_RELATIVE: Final[Path] = Path(
|
||||
"platform/contracts/knowledge-artifact.schema.yaml"
|
||||
)
|
||||
PUBLISH_RECORD_SCHEMA_RELATIVE: Final[Path] = Path(
|
||||
"platform/contracts/knowledge-publish-record.schema.yaml"
|
||||
)
|
||||
DOCUMENT_SCHEMA_RELATIVE: Final[Path] = Path(
|
||||
"platform/contracts/knowledge-document.schema.yaml"
|
||||
)
|
||||
CHUNK_SCHEMA_RELATIVE: Final[Path] = Path(
|
||||
"platform/contracts/knowledge-chunk.schema.yaml"
|
||||
)
|
||||
SOURCE_REGISTRY_RELATIVE: Final[Path] = Path("platform/knowledge/source-registry.yaml")
|
||||
ACL_POLICIES_RELATIVE: Final[Path] = Path("platform/knowledge/acl-policies.yaml")
|
||||
PUBLISH_TARGETS_RELATIVE: Final[Path] = Path("platform/knowledge/publish-targets.yaml")
|
||||
DEFAULT_MAX_CHUNK_CHARACTERS: Final[int] = 1200
|
||||
|
||||
|
||||
class KnowledgeContractError(Exception):
|
||||
"""Raised when a knowledge contract is invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChangeEntry:
|
||||
"""Represents one git diff path change."""
|
||||
|
||||
change_type: str
|
||||
path: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SourceRule:
|
||||
"""Represents one source rule declared by a business repository."""
|
||||
|
||||
source_id: str
|
||||
include: tuple[str, ...]
|
||||
exclude: tuple[str, ...]
|
||||
visibility: str
|
||||
source_type: str
|
||||
doc_type: str
|
||||
product_area: str
|
||||
owner: str
|
||||
review_status: str
|
||||
tags: tuple[str, ...]
|
||||
title_strategy: str
|
||||
title_prefix: str
|
||||
approved_by: str | None
|
||||
effective_from: str | None
|
||||
expires_at: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgePaths:
|
||||
"""Filesystem layout inside ljwx-deploy for knowledge state."""
|
||||
|
||||
artifacts_root: Path
|
||||
events_path: Path
|
||||
publish_records_root: Path
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
"""Return an ISO-8601 timestamp in UTC."""
|
||||
|
||||
return datetime.now(tz=UTC).isoformat()
|
||||
|
||||
|
||||
def ensure_directory(path: Path) -> None:
|
||||
"""Create a directory tree when it does not exist."""
|
||||
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def ensure_parent(path: Path) -> None:
|
||||
"""Create the parent directory for a file path."""
|
||||
|
||||
ensure_directory(path.parent)
|
||||
|
||||
|
||||
def write_json(path: Path, payload: JSONMapping | list[JSONMapping]) -> None:
|
||||
"""Write a JSON file with deterministic formatting."""
|
||||
|
||||
ensure_parent(path)
|
||||
path.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def load_json_mapping(path: Path) -> JSONMapping:
|
||||
"""Load a JSON object from disk."""
|
||||
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, dict):
|
||||
raise KnowledgeContractError(f"{path} 根节点必须是 object")
|
||||
return {str(key): value for key, value in raw.items()}
|
||||
|
||||
|
||||
def load_yaml_mapping(path: Path) -> JSONMapping:
|
||||
"""Load a YAML mapping from disk."""
|
||||
|
||||
raw = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
if raw is None:
|
||||
return {}
|
||||
if not isinstance(raw, dict):
|
||||
raise KnowledgeContractError(f"{path} 根节点必须是 mapping")
|
||||
return {str(key): value for key, value in raw.items()}
|
||||
|
||||
|
||||
def write_jsonl(path: Path, rows: list[JSONMapping]) -> None:
|
||||
"""Write JSONL rows to disk."""
|
||||
|
||||
ensure_parent(path)
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for row in rows:
|
||||
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True))
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def append_jsonl(path: Path, row: JSONMapping) -> None:
|
||||
"""Append one JSONL row to disk."""
|
||||
|
||||
ensure_parent(path)
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True))
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def load_jsonl(path: Path) -> list[JSONMapping]:
|
||||
"""Load JSONL records from disk when present."""
|
||||
|
||||
if not path.exists():
|
||||
return []
|
||||
rows: list[JSONMapping] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
raw = json.loads(stripped)
|
||||
if not isinstance(raw, dict):
|
||||
raise KnowledgeContractError(f"{path} 存在非 mapping JSONL 记录")
|
||||
rows.append({str(key): value for key, value in raw.items()})
|
||||
return rows
|
||||
|
||||
|
||||
def load_validator(path: Path) -> jsonschema.Draft202012Validator:
|
||||
"""Load a JSON schema validator from YAML."""
|
||||
|
||||
schema = load_yaml_mapping(path)
|
||||
jsonschema.Draft202012Validator.check_schema(schema)
|
||||
return jsonschema.Draft202012Validator(schema)
|
||||
|
||||
|
||||
def validate_payload(
|
||||
validator: jsonschema.Draft202012Validator,
|
||||
payload: JSONMapping,
|
||||
payload_name: str,
|
||||
) -> None:
|
||||
"""Validate one payload against a schema."""
|
||||
|
||||
try:
|
||||
validator.validate(payload)
|
||||
except jsonschema.ValidationError as exc:
|
||||
raise KnowledgeContractError(
|
||||
f"{payload_name} 不符合 schema: {exc.message}"
|
||||
) from exc
|
||||
|
||||
|
||||
def require_mapping(value: object, field_path: str) -> JSONMapping:
|
||||
"""Require a JSON mapping."""
|
||||
|
||||
if not isinstance(value, dict):
|
||||
raise KnowledgeContractError(f"{field_path} 必须是 mapping")
|
||||
return {str(key): item for key, item in value.items()}
|
||||
|
||||
|
||||
def require_list(value: object, field_path: str) -> list[object]:
|
||||
"""Require a list."""
|
||||
|
||||
if not isinstance(value, list):
|
||||
raise KnowledgeContractError(f"{field_path} 必须是列表")
|
||||
return value
|
||||
|
||||
|
||||
def require_non_empty_str(value: object, field_path: str) -> str:
|
||||
"""Require a non-empty string."""
|
||||
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise KnowledgeContractError(f"{field_path} 必须是非空字符串")
|
||||
return value.strip()
|
||||
|
||||
|
||||
def optional_non_empty_str(value: object, field_path: str) -> str | None:
|
||||
"""Read an optional non-empty string."""
|
||||
|
||||
if value in ("", None):
|
||||
return None
|
||||
return require_non_empty_str(value, field_path)
|
||||
|
||||
|
||||
def require_str_list(value: object, field_path: str) -> list[str]:
|
||||
"""Require a list of strings."""
|
||||
|
||||
items = require_list(value, field_path)
|
||||
results: list[str] = []
|
||||
for index, item in enumerate(items):
|
||||
results.append(require_non_empty_str(item, f"{field_path}[{index}]"))
|
||||
return results
|
||||
|
||||
|
||||
def sha256_text(text: str) -> str:
|
||||
"""Return the SHA-256 digest of a text payload."""
|
||||
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def slugify_fragment(value: str) -> str:
|
||||
"""Convert a string into a filesystem-safe slug."""
|
||||
|
||||
normalized = re.sub(r"[^a-zA-Z0-9]+", "-", value).strip("-").lower()
|
||||
return normalized or "unknown"
|
||||
|
||||
|
||||
def repo_slug(repo: str) -> str:
|
||||
"""Return a stable directory slug for an owner/repo name."""
|
||||
|
||||
return repo.replace("/", "__")
|
||||
|
||||
|
||||
def build_document_id(source_repo: str, source_path: str) -> str:
|
||||
"""Build a deterministic document identifier from repo and path."""
|
||||
|
||||
repo_name = source_repo.split("/")[-1]
|
||||
base_slug = slugify_fragment(f"{repo_name}-{source_path}")
|
||||
if len(base_slug) <= 96:
|
||||
return base_slug
|
||||
digest = hashlib.sha1(f"{source_repo}:{source_path}".encode("utf-8")).hexdigest()[
|
||||
:12
|
||||
]
|
||||
return f"{base_slug[:80]}-{digest}"
|
||||
|
||||
|
||||
def match_globs(
|
||||
path: str, includes: tuple[str, ...], excludes: tuple[str, ...]
|
||||
) -> bool:
|
||||
"""Return True when a path matches include/exclude rules."""
|
||||
|
||||
included = any(fnmatch.fnmatch(path, pattern) for pattern in includes)
|
||||
if not included:
|
||||
return False
|
||||
return not any(fnmatch.fnmatch(path, pattern) for pattern in excludes)
|
||||
|
||||
|
||||
def parse_diff_entries(path: Path) -> list[ChangeEntry]:
|
||||
"""Parse git diff --name-status output into flattened path changes."""
|
||||
|
||||
if not path.exists():
|
||||
return []
|
||||
|
||||
results: list[ChangeEntry] = []
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split("\t")
|
||||
status = require_non_empty_str(parts[0], "diff.status")
|
||||
base_status = status[0]
|
||||
if base_status in {"A", "C", "M", "T"}:
|
||||
if len(parts) < 2:
|
||||
raise KnowledgeContractError(f"无效 diff 行: {line}")
|
||||
results.append(ChangeEntry(change_type="upsert", path=parts[-1]))
|
||||
continue
|
||||
if base_status == "D":
|
||||
if len(parts) != 2:
|
||||
raise KnowledgeContractError(f"无效删除 diff 行: {line}")
|
||||
results.append(ChangeEntry(change_type="delete", path=parts[1]))
|
||||
continue
|
||||
if base_status == "R":
|
||||
if len(parts) != 3:
|
||||
raise KnowledgeContractError(f"无效重命名 diff 行: {line}")
|
||||
results.append(ChangeEntry(change_type="delete", path=parts[1]))
|
||||
results.append(ChangeEntry(change_type="upsert", path=parts[2]))
|
||||
continue
|
||||
raise KnowledgeContractError(f"暂不支持的 diff 状态: {status}")
|
||||
return results
|
||||
|
||||
|
||||
def extract_markdown_title(content: str, source_path: str, strategy: str) -> str:
|
||||
"""Resolve a document title from markdown content."""
|
||||
|
||||
if strategy == "filename":
|
||||
return Path(source_path).stem.replace("-", " ").replace("_", " ").strip()
|
||||
|
||||
for line in content.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("# "):
|
||||
return stripped.removeprefix("# ").strip()
|
||||
|
||||
return Path(source_path).stem.replace("-", " ").replace("_", " ").strip()
|
||||
|
||||
|
||||
def chunk_markdown(
|
||||
content: str, max_characters: int = DEFAULT_MAX_CHUNK_CHARACTERS
|
||||
) -> list[str]:
|
||||
"""Chunk markdown into paragraph groups."""
|
||||
|
||||
paragraphs = [item.strip() for item in content.split("\n\n") if item.strip()]
|
||||
if not paragraphs:
|
||||
return []
|
||||
|
||||
chunks: list[str] = []
|
||||
current_parts: list[str] = []
|
||||
current_length = 0
|
||||
for paragraph in paragraphs:
|
||||
paragraph_length = len(paragraph)
|
||||
if current_parts and current_length + paragraph_length > max_characters:
|
||||
chunks.append("\n\n".join(current_parts))
|
||||
current_parts = [paragraph]
|
||||
current_length = paragraph_length
|
||||
continue
|
||||
current_parts.append(paragraph)
|
||||
current_length += paragraph_length
|
||||
|
||||
if current_parts:
|
||||
chunks.append("\n\n".join(current_parts))
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def knowledge_paths(deploy_root: Path, environment: str) -> KnowledgePaths:
|
||||
"""Resolve the filesystem layout used by central knowledge state."""
|
||||
|
||||
base_root = deploy_root / "knowledge"
|
||||
return KnowledgePaths(
|
||||
artifacts_root=base_root / "artifacts" / environment,
|
||||
events_path=base_root / "events" / environment / "events.jsonl",
|
||||
publish_records_root=base_root / "publish-records" / environment,
|
||||
)
|
||||
|
||||
|
||||
def load_source_rules(
|
||||
config_path: Path, deploy_root: Path
|
||||
) -> tuple[str, list[SourceRule]]:
|
||||
"""Load and validate source config rules."""
|
||||
|
||||
validator = load_validator(deploy_root / SOURCE_CONFIG_SCHEMA_RELATIVE)
|
||||
payload = load_yaml_mapping(config_path)
|
||||
validate_payload(validator, payload, config_path.as_posix())
|
||||
|
||||
service = require_non_empty_str(
|
||||
payload.get("service"), "knowledge.sources.yaml.service"
|
||||
)
|
||||
raw_sources = require_list(payload.get("sources"), "knowledge.sources.yaml.sources")
|
||||
|
||||
rules: list[SourceRule] = []
|
||||
for index, raw_source in enumerate(raw_sources):
|
||||
source = require_mapping(raw_source, f"knowledge.sources.yaml.sources[{index}]")
|
||||
rules.append(
|
||||
SourceRule(
|
||||
source_id=require_non_empty_str(
|
||||
source.get("id"), f"knowledge.sources.yaml.sources[{index}].id"
|
||||
),
|
||||
include=tuple(
|
||||
require_str_list(
|
||||
source.get("include"),
|
||||
f"knowledge.sources.yaml.sources[{index}].include",
|
||||
)
|
||||
),
|
||||
exclude=tuple(
|
||||
require_str_list(
|
||||
source.get("exclude", []),
|
||||
f"knowledge.sources.yaml.sources[{index}].exclude",
|
||||
)
|
||||
),
|
||||
visibility=require_non_empty_str(
|
||||
source.get("visibility"),
|
||||
f"knowledge.sources.yaml.sources[{index}].visibility",
|
||||
),
|
||||
source_type=require_non_empty_str(
|
||||
source.get("source_type"),
|
||||
f"knowledge.sources.yaml.sources[{index}].source_type",
|
||||
),
|
||||
doc_type=require_non_empty_str(
|
||||
source.get("doc_type"),
|
||||
f"knowledge.sources.yaml.sources[{index}].doc_type",
|
||||
),
|
||||
product_area=require_non_empty_str(
|
||||
source.get("product_area"),
|
||||
f"knowledge.sources.yaml.sources[{index}].product_area",
|
||||
),
|
||||
owner=require_non_empty_str(
|
||||
source.get("owner"),
|
||||
f"knowledge.sources.yaml.sources[{index}].owner",
|
||||
),
|
||||
review_status=require_non_empty_str(
|
||||
source.get("review_status"),
|
||||
f"knowledge.sources.yaml.sources[{index}].review_status",
|
||||
),
|
||||
tags=tuple(
|
||||
require_str_list(
|
||||
source.get("tags", []),
|
||||
f"knowledge.sources.yaml.sources[{index}].tags",
|
||||
)
|
||||
),
|
||||
title_strategy=optional_non_empty_str(
|
||||
source.get("title_strategy"),
|
||||
f"knowledge.sources.yaml.sources[{index}].title_strategy",
|
||||
)
|
||||
or "first_heading",
|
||||
title_prefix=optional_non_empty_str(
|
||||
source.get("title_prefix"),
|
||||
f"knowledge.sources.yaml.sources[{index}].title_prefix",
|
||||
)
|
||||
or "",
|
||||
approved_by=optional_non_empty_str(
|
||||
source.get("approved_by"),
|
||||
f"knowledge.sources.yaml.sources[{index}].approved_by",
|
||||
),
|
||||
effective_from=optional_non_empty_str(
|
||||
source.get("effective_from"),
|
||||
f"knowledge.sources.yaml.sources[{index}].effective_from",
|
||||
),
|
||||
expires_at=optional_non_empty_str(
|
||||
source.get("expires_at"),
|
||||
f"knowledge.sources.yaml.sources[{index}].expires_at",
|
||||
),
|
||||
)
|
||||
)
|
||||
return service, rules
|
||||
|
||||
|
||||
def load_source_registry(deploy_root: Path) -> dict[str, dict[str, object]]:
|
||||
"""Load artifact source registry entries keyed by repository."""
|
||||
|
||||
payload = load_yaml_mapping(deploy_root / SOURCE_REGISTRY_RELATIVE)
|
||||
raw_registry = require_list(
|
||||
payload.get("artifact_sources", []),
|
||||
"platform/knowledge/source-registry.yaml.artifact_sources",
|
||||
)
|
||||
registry: dict[str, dict[str, object]] = {}
|
||||
for index, raw_entry in enumerate(raw_registry):
|
||||
entry = require_mapping(
|
||||
raw_entry,
|
||||
f"platform/knowledge/source-registry.yaml.artifact_sources[{index}]",
|
||||
)
|
||||
repo_name = require_non_empty_str(
|
||||
entry.get("repo"),
|
||||
f"platform/knowledge/source-registry.yaml.artifact_sources[{index}].repo",
|
||||
)
|
||||
registry[repo_name] = {
|
||||
"service": require_non_empty_str(
|
||||
entry.get("service"),
|
||||
"platform/knowledge/source-registry.yaml.artifact_sources."
|
||||
f"[{index}].service",
|
||||
),
|
||||
"source_ids": require_str_list(
|
||||
entry.get("source_ids", []),
|
||||
"platform/knowledge/source-registry.yaml.artifact_sources."
|
||||
f"[{index}].source_ids",
|
||||
),
|
||||
"allowed_visibilities": require_str_list(
|
||||
entry.get("allowed_visibilities", []),
|
||||
"platform/knowledge/source-registry.yaml.artifact_sources."
|
||||
f"[{index}].allowed_visibilities",
|
||||
),
|
||||
}
|
||||
return registry
|
||||
|
||||
|
||||
def load_acl_policies(deploy_root: Path) -> dict[str, dict[str, object]]:
|
||||
"""Load ACL policies keyed by visibility."""
|
||||
|
||||
payload = load_yaml_mapping(deploy_root / ACL_POLICIES_RELATIVE)
|
||||
raw_policies = require_mapping(
|
||||
payload.get("policies"),
|
||||
"platform/knowledge/acl-policies.yaml.policies",
|
||||
)
|
||||
policies: dict[str, dict[str, object]] = {}
|
||||
for visibility, raw_policy in raw_policies.items():
|
||||
policy = require_mapping(
|
||||
raw_policy,
|
||||
f"platform/knowledge/acl-policies.yaml.policies.{visibility}",
|
||||
)
|
||||
policies[visibility] = {
|
||||
"acl_expression_template": require_non_empty_str(
|
||||
policy.get("acl_expression_template"),
|
||||
"platform/knowledge/acl-policies.yaml.policies."
|
||||
f"{visibility}.acl_expression_template",
|
||||
),
|
||||
"allowed_targets": require_str_list(
|
||||
policy.get("allowed_targets", []),
|
||||
"platform/knowledge/acl-policies.yaml.policies."
|
||||
f"{visibility}.allowed_targets",
|
||||
),
|
||||
"review_required": bool(policy.get("review_required", False)),
|
||||
}
|
||||
return policies
|
||||
|
||||
|
||||
def load_publish_targets(deploy_root: Path) -> dict[str, dict[str, object]]:
|
||||
"""Load publish targets keyed by target id."""
|
||||
|
||||
payload = load_yaml_mapping(deploy_root / PUBLISH_TARGETS_RELATIVE)
|
||||
raw_targets = require_mapping(
|
||||
payload.get("targets"),
|
||||
"platform/knowledge/publish-targets.yaml.targets",
|
||||
)
|
||||
targets: dict[str, dict[str, object]] = {}
|
||||
for target_id, raw_target in raw_targets.items():
|
||||
target = require_mapping(
|
||||
raw_target, f"platform/knowledge/publish-targets.yaml.targets.{target_id}"
|
||||
)
|
||||
targets[target_id] = {
|
||||
"type": require_non_empty_str(
|
||||
target.get("type"),
|
||||
f"platform/knowledge/publish-targets.yaml.targets.{target_id}.type",
|
||||
),
|
||||
"visibility": require_non_empty_str(
|
||||
target.get("visibility"),
|
||||
"platform/knowledge/publish-targets.yaml.targets."
|
||||
f"{target_id}.visibility",
|
||||
),
|
||||
"dataset_name": optional_non_empty_str(
|
||||
target.get("dataset_name"),
|
||||
"platform/knowledge/publish-targets.yaml.targets."
|
||||
f"{target_id}.dataset_name",
|
||||
),
|
||||
}
|
||||
return targets
|
||||
|
||||
|
||||
def load_publish_record(path: Path) -> JSONMapping:
|
||||
"""Load one publish record file."""
|
||||
|
||||
return load_json_mapping(path)
|
||||
|
||||
|
||||
def load_publish_records(root: Path) -> dict[str, JSONMapping]:
|
||||
"""Load all publish records under a directory."""
|
||||
|
||||
if not root.exists():
|
||||
return {}
|
||||
records: dict[str, JSONMapping] = {}
|
||||
for path in sorted(root.rglob("*.json")):
|
||||
record = load_publish_record(path)
|
||||
document_id = require_non_empty_str(
|
||||
record.get("document_id"), f"{path}.document_id"
|
||||
)
|
||||
records[document_id] = record
|
||||
return records
|
||||
|
||||
|
||||
def build_acl_expression(
|
||||
visibility: str, policies: dict[str, dict[str, object]]
|
||||
) -> str:
|
||||
"""Resolve the ACL expression for a visibility."""
|
||||
|
||||
policy = policies.get(visibility)
|
||||
if policy is None:
|
||||
raise KnowledgeContractError(f"未注册 visibility ACL policy: {visibility}")
|
||||
acl_template = policy.get("acl_expression_template")
|
||||
return require_non_empty_str(acl_template, f"acl policy for {visibility}")
|
||||
|
||||
|
||||
def build_event(
|
||||
*,
|
||||
action: str,
|
||||
artifact_id: str,
|
||||
document_id: str,
|
||||
source_repo: str,
|
||||
status: str,
|
||||
reason: str,
|
||||
target_ids: list[str],
|
||||
) -> JSONMapping:
|
||||
"""Build one append-only event record."""
|
||||
|
||||
compact_time = utc_now().replace(":", "").replace("-", "")
|
||||
return {
|
||||
"event_id": f"knowledge-{action}-{document_id}-{compact_time}",
|
||||
"timestamp": utc_now(),
|
||||
"artifact_id": artifact_id,
|
||||
"document_id": document_id,
|
||||
"source_repo": source_repo,
|
||||
"action": action,
|
||||
"status": status,
|
||||
"reason": reason,
|
||||
"target_ids": target_ids,
|
||||
}
|
||||
|
||||
|
||||
def artifact_document_name(document_id: str) -> str:
|
||||
"""Return the stable remote document name used in Dify."""
|
||||
|
||||
return document_id
|
||||
|
||||
|
||||
def artifact_snapshot_root(
|
||||
deploy_root: Path,
|
||||
environment: str,
|
||||
source_repo: str,
|
||||
artifact_id: str,
|
||||
) -> Path:
|
||||
"""Return the snapshot root for one artifact."""
|
||||
|
||||
return (
|
||||
knowledge_paths(deploy_root, environment).artifacts_root
|
||||
/ repo_slug(source_repo)
|
||||
/ artifact_id
|
||||
)
|
||||
|
||||
|
||||
def record_path_for_document(
|
||||
deploy_root: Path,
|
||||
environment: str,
|
||||
source_repo: str,
|
||||
document_id: str,
|
||||
) -> Path:
|
||||
"""Return the publish-record path for a document."""
|
||||
|
||||
return (
|
||||
knowledge_paths(deploy_root, environment).publish_records_root
|
||||
/ repo_slug(source_repo)
|
||||
/ f"{document_id}.json"
|
||||
)
|
||||
|
||||
|
||||
def cast_mapping_list(value: object, field_path: str) -> list[JSONMapping]:
|
||||
"""Cast a list payload into a list of mappings."""
|
||||
|
||||
items = require_list(value, field_path)
|
||||
mappings: list[JSONMapping] = []
|
||||
for index, item in enumerate(items):
|
||||
mappings.append(require_mapping(item, f"{field_path}[{index}]"))
|
||||
return mappings
|
||||
|
||||
|
||||
def optional_mapping(value: object, field_path: str) -> JSONMapping | None:
|
||||
"""Return an optional mapping."""
|
||||
|
||||
if value is None:
|
||||
return None
|
||||
return require_mapping(value, field_path)
|
||||
|
||||
|
||||
def cast_str(value: object) -> str:
|
||||
"""Cast a value known to be a string."""
|
||||
|
||||
return cast(str, value)
|
||||
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Minimal Dify service API client for knowledge publication."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from .common import (
|
||||
JSONMapping,
|
||||
optional_non_empty_str,
|
||||
require_mapping,
|
||||
require_non_empty_str,
|
||||
)
|
||||
|
||||
|
||||
REQUEST_TIMEOUT_SECONDS: Final[int] = 15
|
||||
|
||||
|
||||
class DifyClientError(Exception):
|
||||
"""Raised when a Dify API request fails."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DifyDataset:
|
||||
"""Dify dataset summary."""
|
||||
|
||||
dataset_id: str
|
||||
name: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DifyDocument:
|
||||
"""Dify document summary."""
|
||||
|
||||
document_id: str
|
||||
name: str
|
||||
|
||||
|
||||
class DifyClient:
|
||||
"""Minimal service API client for Dify dataset operations."""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str) -> None:
|
||||
if not base_url.strip():
|
||||
raise DifyClientError("缺少 Dify base URL")
|
||||
if not api_key.strip():
|
||||
raise DifyClientError("缺少 Dify API key")
|
||||
self._base_url = base_url.rstrip("/")
|
||||
self._api_key = api_key
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
payload: JSONMapping | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
expect_empty: bool = False,
|
||||
) -> JSONMapping:
|
||||
url = f"{self._base_url}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urlencode(query)}"
|
||||
|
||||
data: bytes | None = None
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
if payload is not None:
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
|
||||
request = Request(url=url, data=data, method=method, headers=headers)
|
||||
try:
|
||||
with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
|
||||
raw_body = response.read().decode("utf-8").strip()
|
||||
except HTTPError as exc:
|
||||
error_body = exc.read().decode("utf-8", errors="replace")
|
||||
raise DifyClientError(
|
||||
f"Dify 请求失败: {method} {path} -> HTTP {exc.code}: {error_body}"
|
||||
) from exc
|
||||
except URLError as exc:
|
||||
raise DifyClientError(
|
||||
f"Dify 请求失败: {method} {path} -> {exc.reason}"
|
||||
) from exc
|
||||
|
||||
if expect_empty and not raw_body:
|
||||
return {}
|
||||
if not raw_body:
|
||||
return {}
|
||||
try:
|
||||
payload_object = json.loads(raw_body)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise DifyClientError(f"Dify 返回了无效 JSON: {raw_body}") from exc
|
||||
if not isinstance(payload_object, dict):
|
||||
raise DifyClientError("Dify 返回根节点必须是 object")
|
||||
return {str(key): value for key, value in payload_object.items()}
|
||||
|
||||
def list_datasets(self) -> list[DifyDataset]:
|
||||
"""List Dify datasets visible to the service API token."""
|
||||
|
||||
payload = self._request("GET", "/datasets", query={"page": "1", "limit": "100"})
|
||||
raw_items = payload.get("data", [])
|
||||
if not isinstance(raw_items, list):
|
||||
raise DifyClientError("/datasets.data 必须是列表")
|
||||
|
||||
results: list[DifyDataset] = []
|
||||
for index, raw_item in enumerate(raw_items):
|
||||
item = require_mapping(raw_item, f"/datasets.data[{index}]")
|
||||
results.append(
|
||||
DifyDataset(
|
||||
dataset_id=require_non_empty_str(
|
||||
item.get("id"), f"/datasets.data[{index}].id"
|
||||
),
|
||||
name=require_non_empty_str(
|
||||
item.get("name"), f"/datasets.data[{index}].name"
|
||||
),
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
def ensure_dataset(self, dataset_name: str) -> DifyDataset:
|
||||
"""Return an existing dataset or create it on demand."""
|
||||
|
||||
for dataset in self.list_datasets():
|
||||
if dataset.name == dataset_name:
|
||||
return dataset
|
||||
|
||||
payload = self._request(
|
||||
"POST",
|
||||
"/datasets",
|
||||
payload={
|
||||
"name": dataset_name,
|
||||
"description": f"中央知识流水线管理的 {dataset_name} dataset",
|
||||
"indexing_technique": "high_quality",
|
||||
"permission": "only_me",
|
||||
},
|
||||
)
|
||||
dataset_id = require_non_empty_str(payload.get("id"), "POST /datasets.id")
|
||||
created_name = require_non_empty_str(payload.get("name"), "POST /datasets.name")
|
||||
return DifyDataset(dataset_id=dataset_id, name=created_name)
|
||||
|
||||
def list_documents(self, dataset_id: str) -> list[DifyDocument]:
|
||||
"""List documents in one dataset."""
|
||||
|
||||
payload = self._request(
|
||||
"GET",
|
||||
f"/datasets/{dataset_id}/documents",
|
||||
query={"page": "1", "limit": "100"},
|
||||
)
|
||||
raw_items = payload.get("data", [])
|
||||
if not isinstance(raw_items, list):
|
||||
raise DifyClientError("/documents.data 必须是列表")
|
||||
|
||||
results: list[DifyDocument] = []
|
||||
for index, raw_item in enumerate(raw_items):
|
||||
item = require_mapping(
|
||||
raw_item, f"/datasets/{dataset_id}/documents.data[{index}]"
|
||||
)
|
||||
results.append(
|
||||
DifyDocument(
|
||||
document_id=require_non_empty_str(
|
||||
item.get("id"),
|
||||
f"/datasets/{dataset_id}/documents.data[{index}].id",
|
||||
),
|
||||
name=require_non_empty_str(
|
||||
item.get("name"),
|
||||
f"/datasets/{dataset_id}/documents.data[{index}].name",
|
||||
),
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
def find_document_id(self, dataset_id: str, document_name: str) -> str | None:
|
||||
"""Find a document by its stable name."""
|
||||
|
||||
for document in self.list_documents(dataset_id):
|
||||
if document.name == document_name:
|
||||
return document.document_id
|
||||
return None
|
||||
|
||||
def upsert_document(
|
||||
self,
|
||||
*,
|
||||
dataset_id: str,
|
||||
document_name: str,
|
||||
content: str,
|
||||
document_id: str | None,
|
||||
) -> str:
|
||||
"""Create or update one Dify text document."""
|
||||
|
||||
if document_id:
|
||||
payload = self._request(
|
||||
"POST",
|
||||
f"/datasets/{dataset_id}/documents/{document_id}/update_by_text",
|
||||
payload={
|
||||
"name": document_name,
|
||||
"text": content,
|
||||
"process_rule": {"mode": "automatic"},
|
||||
},
|
||||
)
|
||||
updated_id = optional_non_empty_str(
|
||||
payload.get("document_id"),
|
||||
f"/datasets/{dataset_id}/documents/{document_id}/update_by_text.document_id",
|
||||
)
|
||||
return updated_id or document_id
|
||||
|
||||
payload = self._request(
|
||||
"POST",
|
||||
f"/datasets/{dataset_id}/document/create_by_text",
|
||||
payload={
|
||||
"name": document_name,
|
||||
"text": content,
|
||||
"indexing_technique": "high_quality",
|
||||
"process_rule": {"mode": "automatic"},
|
||||
},
|
||||
)
|
||||
raw_document = require_mapping(
|
||||
payload.get("document"),
|
||||
f"/datasets/{dataset_id}/document/create_by_text.document",
|
||||
)
|
||||
return require_non_empty_str(
|
||||
raw_document.get("id"),
|
||||
f"/datasets/{dataset_id}/document/create_by_text.document.id",
|
||||
)
|
||||
|
||||
def delete_document(self, dataset_id: str, document_id: str) -> None:
|
||||
"""Delete one Dify document."""
|
||||
|
||||
self._request(
|
||||
"DELETE",
|
||||
f"/datasets/{dataset_id}/documents/{document_id}",
|
||||
expect_empty=True,
|
||||
)
|
||||
@@ -0,0 +1,343 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export markdown knowledge sources into a portable artifact."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from .common import (
|
||||
ARTIFACT_SCHEMA_RELATIVE,
|
||||
JSONMapping,
|
||||
KnowledgeContractError,
|
||||
build_document_id,
|
||||
extract_markdown_title,
|
||||
load_source_rules,
|
||||
load_validator,
|
||||
match_globs,
|
||||
parse_diff_entries,
|
||||
repo_slug,
|
||||
require_non_empty_str,
|
||||
sha256_text,
|
||||
utc_now,
|
||||
validate_payload,
|
||||
write_json,
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse CLI arguments."""
|
||||
|
||||
parser = argparse.ArgumentParser(description="Export a knowledge artifact")
|
||||
parser.add_argument(
|
||||
"--repo-root", type=Path, required=True, help="Business repo root"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-root",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="ljwx-deploy root used for contracts",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
type=Path,
|
||||
default=Path("knowledge.sources.yaml"),
|
||||
help="Path to knowledge.sources.yaml relative to repo root",
|
||||
)
|
||||
parser.add_argument("--source-repo", required=True, help="GitHub owner/repo")
|
||||
parser.add_argument("--source-ref", required=True, help="Git ref")
|
||||
parser.add_argument("--source-sha", required=True, help="Git commit SHA")
|
||||
parser.add_argument("--workflow-name", required=True, help="Workflow name")
|
||||
parser.add_argument("--run-id", type=int, required=True, help="GitHub run id")
|
||||
parser.add_argument(
|
||||
"--run-attempt", type=int, required=True, help="GitHub run attempt"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--changed-files",
|
||||
type=Path,
|
||||
help="Path to git diff --name-status output. If omitted, export all matches.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Artifact zip output path",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--summary-json",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Where to write export summary JSON",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def resolve_rule_for_path(path: str, rules: list[object]) -> object | None:
|
||||
"""Find a single matching rule for a source path."""
|
||||
|
||||
matched: list[object] = []
|
||||
for rule in rules:
|
||||
if not hasattr(rule, "include") or not hasattr(rule, "exclude"):
|
||||
continue
|
||||
include = getattr(rule, "include")
|
||||
exclude = getattr(rule, "exclude")
|
||||
if not isinstance(include, tuple) or not isinstance(exclude, tuple):
|
||||
continue
|
||||
if match_globs(path, include, exclude):
|
||||
matched.append(rule)
|
||||
if len(matched) > 1:
|
||||
raise KnowledgeContractError(f"路径命中了多个 source rule: {path}")
|
||||
return matched[0] if matched else None
|
||||
|
||||
|
||||
def build_manifest_document(
|
||||
*,
|
||||
repo_root: Path,
|
||||
source_repo: str,
|
||||
source_sha: str,
|
||||
relative_path: str,
|
||||
source_id: str,
|
||||
visibility: str,
|
||||
source_type: str,
|
||||
doc_type: str,
|
||||
product_area: str,
|
||||
owner: str,
|
||||
review_status: str,
|
||||
title_strategy: str,
|
||||
title_prefix: str,
|
||||
tags: tuple[str, ...],
|
||||
approved_by: str | None,
|
||||
effective_from: str | None,
|
||||
expires_at: str | None,
|
||||
) -> tuple[JSONMapping, str, str]:
|
||||
"""Build one upsert artifact document entry and its content payload."""
|
||||
|
||||
file_path = repo_root / relative_path
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
document_id = build_document_id(source_repo, relative_path)
|
||||
title = extract_markdown_title(content, relative_path, title_strategy)
|
||||
if title_prefix:
|
||||
title = f"{title_prefix}{title}"
|
||||
checksum = sha256_text(content)
|
||||
content_path = f"documents/{document_id}.md"
|
||||
source_ref = f"{source_repo}@{source_sha}:{relative_path}"
|
||||
payload: JSONMapping = {
|
||||
"op": "upsert",
|
||||
"document_id": document_id,
|
||||
"source_id": source_id,
|
||||
"source_path": relative_path,
|
||||
"source_ref": source_ref,
|
||||
"content_path": content_path,
|
||||
"title": title,
|
||||
"visibility": visibility,
|
||||
"source_type": source_type,
|
||||
"doc_type": doc_type,
|
||||
"product_area": product_area,
|
||||
"owner": owner,
|
||||
"review_status": review_status,
|
||||
"version": source_sha,
|
||||
"checksum": checksum,
|
||||
"tags": list(tags),
|
||||
"approved_by": approved_by,
|
||||
"effective_from": effective_from,
|
||||
"expires_at": expires_at,
|
||||
"reason": None,
|
||||
}
|
||||
return payload, content_path, content
|
||||
|
||||
|
||||
def build_invalidate_document(
|
||||
*,
|
||||
source_repo: str,
|
||||
source_sha: str,
|
||||
relative_path: str,
|
||||
source_id: str,
|
||||
) -> JSONMapping:
|
||||
"""Build one invalidate artifact document entry."""
|
||||
|
||||
return {
|
||||
"op": "invalidate",
|
||||
"document_id": build_document_id(source_repo, relative_path),
|
||||
"source_id": source_id,
|
||||
"source_path": relative_path,
|
||||
"source_ref": f"{source_repo}@{source_sha}:{relative_path}",
|
||||
"content_path": None,
|
||||
"title": None,
|
||||
"visibility": None,
|
||||
"source_type": None,
|
||||
"doc_type": None,
|
||||
"product_area": None,
|
||||
"owner": None,
|
||||
"review_status": None,
|
||||
"version": None,
|
||||
"checksum": None,
|
||||
"tags": [],
|
||||
"approved_by": None,
|
||||
"effective_from": None,
|
||||
"expires_at": None,
|
||||
"reason": "source_deleted",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entry point."""
|
||||
|
||||
args = parse_args()
|
||||
repo_root = args.repo_root.resolve()
|
||||
deploy_root = args.deploy_root.resolve()
|
||||
config_path = (repo_root / args.config).resolve()
|
||||
output_path = args.output.resolve()
|
||||
summary_path = args.summary_json.resolve()
|
||||
|
||||
service, rules = load_source_rules(config_path, deploy_root)
|
||||
artifact_validator = load_validator(deploy_root / ARTIFACT_SCHEMA_RELATIVE)
|
||||
diff_entries = (
|
||||
parse_diff_entries(args.changed_files.resolve()) if args.changed_files else []
|
||||
)
|
||||
|
||||
changed_upserts = {
|
||||
entry.path for entry in diff_entries if entry.change_type == "upsert"
|
||||
}
|
||||
changed_deletes = {
|
||||
entry.path for entry in diff_entries if entry.change_type == "delete"
|
||||
}
|
||||
export_all = not diff_entries
|
||||
|
||||
live_documents: list[JSONMapping] = []
|
||||
deleted_documents: list[JSONMapping] = []
|
||||
staged_documents: list[tuple[str, str]] = []
|
||||
|
||||
if export_all:
|
||||
candidate_paths = sorted(
|
||||
path.relative_to(repo_root).as_posix()
|
||||
for path in repo_root.rglob("*.md")
|
||||
if path.is_file()
|
||||
)
|
||||
else:
|
||||
candidate_paths = sorted(changed_upserts)
|
||||
|
||||
for relative_path in candidate_paths:
|
||||
rule = resolve_rule_for_path(relative_path, rules)
|
||||
if rule is None:
|
||||
continue
|
||||
rule_source_id = require_non_empty_str(
|
||||
getattr(rule, "source_id"), "rule.source_id"
|
||||
)
|
||||
payload, content_path, content = build_manifest_document(
|
||||
repo_root=repo_root,
|
||||
source_repo=args.source_repo,
|
||||
source_sha=args.source_sha,
|
||||
relative_path=relative_path,
|
||||
source_id=rule_source_id,
|
||||
visibility=require_non_empty_str(
|
||||
getattr(rule, "visibility"), "rule.visibility"
|
||||
),
|
||||
source_type=require_non_empty_str(
|
||||
getattr(rule, "source_type"), "rule.source_type"
|
||||
),
|
||||
doc_type=require_non_empty_str(getattr(rule, "doc_type"), "rule.doc_type"),
|
||||
product_area=require_non_empty_str(
|
||||
getattr(rule, "product_area"), "rule.product_area"
|
||||
),
|
||||
owner=require_non_empty_str(getattr(rule, "owner"), "rule.owner"),
|
||||
review_status=require_non_empty_str(
|
||||
getattr(rule, "review_status"), "rule.review_status"
|
||||
),
|
||||
title_strategy=require_non_empty_str(
|
||||
getattr(rule, "title_strategy"), "rule.title_strategy"
|
||||
),
|
||||
title_prefix=require_non_empty_str(
|
||||
getattr(rule, "title_prefix", ""), "rule.title_prefix"
|
||||
)
|
||||
if getattr(rule, "title_prefix", "") != ""
|
||||
else "",
|
||||
tags=getattr(rule, "tags"),
|
||||
approved_by=getattr(rule, "approved_by"),
|
||||
effective_from=getattr(rule, "effective_from"),
|
||||
expires_at=getattr(rule, "expires_at"),
|
||||
)
|
||||
live_documents.append(payload)
|
||||
staged_documents.append((content_path, content))
|
||||
|
||||
for relative_path in sorted(changed_deletes):
|
||||
rule = resolve_rule_for_path(relative_path, rules)
|
||||
if rule is None:
|
||||
continue
|
||||
deleted_documents.append(
|
||||
build_invalidate_document(
|
||||
source_repo=args.source_repo,
|
||||
source_sha=args.source_sha,
|
||||
relative_path=relative_path,
|
||||
source_id=require_non_empty_str(
|
||||
getattr(rule, "source_id"), "rule.source_id"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
all_documents = deleted_documents + live_documents
|
||||
artifact_id = f"{repo_slug(args.source_repo)}-{args.source_sha[:12]}"
|
||||
summary: JSONMapping = {
|
||||
"artifact_id": artifact_id,
|
||||
"artifact_file": str(output_path),
|
||||
"deleted_documents": len(deleted_documents),
|
||||
"exported": bool(all_documents),
|
||||
"service": service,
|
||||
"source_repo": args.source_repo,
|
||||
"upsert_documents": len(live_documents),
|
||||
}
|
||||
|
||||
if not all_documents:
|
||||
write_json(summary_path, summary)
|
||||
return 0
|
||||
|
||||
manifest: JSONMapping = {
|
||||
"schema_version": "0.1.0",
|
||||
"artifact_id": artifact_id,
|
||||
"generated_at": utc_now(),
|
||||
"export_mode": "full" if export_all else "changed",
|
||||
"source": {
|
||||
"repo": args.source_repo,
|
||||
"service": service,
|
||||
"commit": args.source_sha,
|
||||
"ref": args.source_ref,
|
||||
"workflow": args.workflow_name,
|
||||
"run_id": args.run_id,
|
||||
"run_attempt": args.run_attempt,
|
||||
},
|
||||
"documents": all_documents,
|
||||
}
|
||||
validate_payload(artifact_validator, manifest, "knowledge artifact manifest")
|
||||
|
||||
staging_root = Path(
|
||||
tempfile.mkdtemp(prefix="knowledge-export-", dir=output_path.parent.as_posix())
|
||||
)
|
||||
try:
|
||||
manifest_path = staging_root / "manifest.json"
|
||||
write_json(manifest_path, manifest)
|
||||
for content_path, content in staged_documents:
|
||||
target_path = staging_root / content_path
|
||||
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
target_path.write_text(content, encoding="utf-8")
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(
|
||||
output_path, "w", compression=zipfile.ZIP_DEFLATED
|
||||
) as archive:
|
||||
for path in sorted(staging_root.rglob("*")):
|
||||
if path.is_dir():
|
||||
continue
|
||||
archive.write(path, path.relative_to(staging_root))
|
||||
finally:
|
||||
shutil.rmtree(staging_root, ignore_errors=True)
|
||||
|
||||
write_json(summary_path, summary)
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""GitHub Actions artifact helpers for the knowledge supply chain."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from .common import JSONMapping, ensure_parent, require_mapping, require_non_empty_str
|
||||
|
||||
|
||||
REQUEST_TIMEOUT_SECONDS = 20
|
||||
|
||||
|
||||
class GitHubArtifactError(Exception):
|
||||
"""Raised when a GitHub artifact request fails."""
|
||||
|
||||
|
||||
def _request_json(url: str, token: str) -> JSONMapping:
|
||||
request = Request(
|
||||
url=url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
|
||||
body = response.read().decode("utf-8")
|
||||
except HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", errors="replace")
|
||||
raise GitHubArtifactError(
|
||||
f"GitHub API 请求失败: HTTP {exc.code}: {detail}"
|
||||
) from exc
|
||||
except URLError as exc:
|
||||
raise GitHubArtifactError(f"GitHub API 请求失败: {exc.reason}") from exc
|
||||
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise GitHubArtifactError(f"GitHub API 返回无效 JSON: {body}") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise GitHubArtifactError("GitHub API 返回根节点必须是 object")
|
||||
return {str(key): value for key, value in payload.items()}
|
||||
|
||||
|
||||
def download_artifact_zip(
|
||||
*,
|
||||
repository: str,
|
||||
run_id: int,
|
||||
artifact_name: str,
|
||||
token: str,
|
||||
output_path: Path,
|
||||
) -> Path:
|
||||
"""Download one named artifact from a workflow run."""
|
||||
|
||||
artifacts_url = (
|
||||
f"https://api.github.com/repos/{repository}/actions/runs/{run_id}/artifacts"
|
||||
)
|
||||
listing = _request_json(artifacts_url, token)
|
||||
raw_items = listing.get("artifacts", [])
|
||||
if not isinstance(raw_items, list):
|
||||
raise GitHubArtifactError("/artifacts 必须是列表")
|
||||
|
||||
artifact_id: int | None = None
|
||||
for index, raw_item in enumerate(raw_items):
|
||||
item = require_mapping(raw_item, f"artifacts[{index}]")
|
||||
candidate_name = require_non_empty_str(
|
||||
item.get("name"), f"artifacts[{index}].name"
|
||||
)
|
||||
if candidate_name != artifact_name:
|
||||
continue
|
||||
raw_id = item.get("id")
|
||||
if not isinstance(raw_id, int) or raw_id <= 0:
|
||||
raise GitHubArtifactError(f"artifacts[{index}].id 必须是正整数")
|
||||
artifact_id = raw_id
|
||||
break
|
||||
|
||||
if artifact_id is None:
|
||||
raise GitHubArtifactError(
|
||||
f"未找到 artifact: repo={repository} run_id={run_id} name={artifact_name}"
|
||||
)
|
||||
|
||||
download_url = (
|
||||
f"https://api.github.com/repos/{repository}/actions/artifacts/{artifact_id}/zip"
|
||||
)
|
||||
request = Request(
|
||||
url=download_url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
|
||||
payload = response.read()
|
||||
except HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", errors="replace")
|
||||
raise GitHubArtifactError(
|
||||
f"下载 artifact 失败: HTTP {exc.code}: {detail}"
|
||||
) from exc
|
||||
except URLError as exc:
|
||||
raise GitHubArtifactError(f"下载 artifact 失败: {exc.reason}") from exc
|
||||
|
||||
ensure_parent(output_path)
|
||||
output_path.write_bytes(payload)
|
||||
return output_path
|
||||
@@ -0,0 +1,858 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Consume one business-repo knowledge artifact in ljwx-deploy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
from .common import (
|
||||
ARTIFACT_SCHEMA_RELATIVE,
|
||||
CHUNK_SCHEMA_RELATIVE,
|
||||
DOCUMENT_SCHEMA_RELATIVE,
|
||||
JSONMapping,
|
||||
KnowledgeContractError,
|
||||
PUBLISH_RECORD_SCHEMA_RELATIVE,
|
||||
append_jsonl,
|
||||
artifact_document_name,
|
||||
artifact_snapshot_root,
|
||||
build_acl_expression,
|
||||
build_event,
|
||||
cast_mapping_list,
|
||||
chunk_markdown,
|
||||
knowledge_paths,
|
||||
load_acl_policies,
|
||||
load_json_mapping,
|
||||
load_publish_records,
|
||||
load_publish_targets,
|
||||
load_source_registry,
|
||||
load_validator,
|
||||
optional_non_empty_str,
|
||||
record_path_for_document,
|
||||
require_mapping,
|
||||
require_non_empty_str,
|
||||
sha256_text,
|
||||
utc_now,
|
||||
validate_payload,
|
||||
write_json,
|
||||
write_jsonl,
|
||||
)
|
||||
from .dify_client import DifyClient, DifyClientError
|
||||
from .github_artifacts import GitHubArtifactError, download_artifact_zip
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse CLI arguments."""
|
||||
|
||||
parser = argparse.ArgumentParser(description="Ingest a knowledge artifact")
|
||||
parser.add_argument(
|
||||
"--deploy-root", type=Path, required=True, help="ljwx-deploy root"
|
||||
)
|
||||
parser.add_argument("--env", default="dev", help="Target environment")
|
||||
parser.add_argument("--artifact-file", type=Path, help="Local artifact zip")
|
||||
parser.add_argument("--source-repo", help="GitHub owner/repo")
|
||||
parser.add_argument("--source-run-id", type=int, help="Source workflow run id")
|
||||
parser.add_argument("--artifact-name", help="Artifact name in source run")
|
||||
parser.add_argument("--github-token", help="GitHub token for artifact download")
|
||||
parser.add_argument(
|
||||
"--dify-base-url",
|
||||
default="",
|
||||
help="Dify service API base URL (or use KNOWLEDGE_DIFY_BASE_URL)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dify-api-key",
|
||||
default="",
|
||||
help="Dify service API key (or use KNOWLEDGE_DIFY_API_KEY)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-dify",
|
||||
action="store_true",
|
||||
help="Skip live Dify publication and only write Git state",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--summary-json",
|
||||
type=Path,
|
||||
help="Optional summary output path",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def resolve_artifact_zip(args: argparse.Namespace) -> Path:
|
||||
"""Resolve the artifact zip either locally or from GitHub Actions."""
|
||||
|
||||
if args.artifact_file:
|
||||
return args.artifact_file.resolve()
|
||||
|
||||
source_repo = require_non_empty_str(args.source_repo, "--source-repo")
|
||||
artifact_name = require_non_empty_str(args.artifact_name, "--artifact-name")
|
||||
github_token = require_non_empty_str(args.github_token, "--github-token")
|
||||
if not isinstance(args.source_run_id, int) or args.source_run_id <= 0:
|
||||
raise KnowledgeContractError("--source-run-id 必须是正整数")
|
||||
|
||||
temp_root = Path(tempfile.mkdtemp(prefix="knowledge-ingest-"))
|
||||
artifact_path = temp_root / f"{artifact_name}.zip"
|
||||
try:
|
||||
return download_artifact_zip(
|
||||
repository=source_repo,
|
||||
run_id=args.source_run_id,
|
||||
artifact_name=artifact_name,
|
||||
token=github_token,
|
||||
output_path=artifact_path,
|
||||
)
|
||||
except GitHubArtifactError as exc:
|
||||
shutil.rmtree(temp_root, ignore_errors=True)
|
||||
raise KnowledgeContractError(str(exc)) from exc
|
||||
|
||||
|
||||
def extract_artifact(artifact_zip: Path) -> Path:
|
||||
"""Extract an artifact zip into a temporary directory."""
|
||||
|
||||
if not artifact_zip.exists():
|
||||
raise KnowledgeContractError(f"artifact zip 不存在: {artifact_zip}")
|
||||
extract_root = Path(tempfile.mkdtemp(prefix="knowledge-artifact-"))
|
||||
with zipfile.ZipFile(artifact_zip) as archive:
|
||||
archive.extractall(extract_root)
|
||||
return extract_root
|
||||
|
||||
|
||||
def validate_source_registration(
|
||||
*,
|
||||
deploy_root: Path,
|
||||
source_repo: str,
|
||||
service_name: str,
|
||||
documents: list[JSONMapping],
|
||||
) -> None:
|
||||
"""Ensure the source repo and source ids are registered centrally."""
|
||||
|
||||
registry = load_source_registry(deploy_root)
|
||||
entry = registry.get(source_repo)
|
||||
if entry is None:
|
||||
raise KnowledgeContractError(
|
||||
f"source repo 未在中央 registry 注册: {source_repo}"
|
||||
)
|
||||
registered_service = require_non_empty_str(entry.get("service"), "registry.service")
|
||||
if registered_service != service_name:
|
||||
raise KnowledgeContractError(
|
||||
f"source repo service 不匹配: {source_repo} -> {registered_service} != {service_name}"
|
||||
)
|
||||
registered_source_ids = {
|
||||
require_non_empty_str(item, "registry.source_ids[]")
|
||||
for item in cast(list[str], entry.get("source_ids", []))
|
||||
}
|
||||
allowed_visibilities = {
|
||||
require_non_empty_str(item, "registry.allowed_visibilities[]")
|
||||
for item in cast(list[str], entry.get("allowed_visibilities", []))
|
||||
}
|
||||
for index, document in enumerate(documents):
|
||||
source_id = require_non_empty_str(
|
||||
document.get("source_id"), f"artifact.documents[{index}].source_id"
|
||||
)
|
||||
if source_id not in registered_source_ids:
|
||||
raise KnowledgeContractError(
|
||||
f"source_id 未在中央 registry 注册: {source_repo}/{source_id}"
|
||||
)
|
||||
visibility = optional_non_empty_str(
|
||||
document.get("visibility"), f"artifact.documents[{index}].visibility"
|
||||
)
|
||||
if visibility and visibility not in allowed_visibilities:
|
||||
raise KnowledgeContractError(
|
||||
f"visibility 不在 source repo 白名单中: {source_repo}/{visibility}"
|
||||
)
|
||||
|
||||
|
||||
def load_artifact_manifest(extract_root: Path, deploy_root: Path) -> JSONMapping:
|
||||
"""Load and validate manifest.json from an extracted artifact."""
|
||||
|
||||
manifest_path = extract_root / "manifest.json"
|
||||
if not manifest_path.exists():
|
||||
raise KnowledgeContractError(f"artifact 缺少 manifest.json: {extract_root}")
|
||||
manifest = load_json_mapping(manifest_path)
|
||||
validator = load_validator(deploy_root / ARTIFACT_SCHEMA_RELATIVE)
|
||||
validate_payload(validator, manifest, "knowledge artifact manifest")
|
||||
return manifest
|
||||
|
||||
|
||||
def copy_artifact_snapshot(
|
||||
*,
|
||||
extract_root: Path,
|
||||
deploy_root: Path,
|
||||
environment: str,
|
||||
source_repo: str,
|
||||
artifact_id: str,
|
||||
) -> Path:
|
||||
"""Persist the raw artifact payload into ljwx-deploy."""
|
||||
|
||||
snapshot_root = artifact_snapshot_root(
|
||||
deploy_root, environment, source_repo, artifact_id
|
||||
)
|
||||
if snapshot_root.exists():
|
||||
shutil.rmtree(snapshot_root)
|
||||
snapshot_root.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(extract_root, snapshot_root)
|
||||
return snapshot_root
|
||||
|
||||
|
||||
def load_artifact_content(snapshot_root: Path, document_entry: JSONMapping) -> str:
|
||||
"""Load one markdown content file from the snapshot."""
|
||||
|
||||
content_path = optional_non_empty_str(
|
||||
document_entry.get("content_path"), "content_path"
|
||||
)
|
||||
if content_path is None:
|
||||
raise KnowledgeContractError("upsert 文档缺少 content_path")
|
||||
absolute_path = snapshot_root / content_path
|
||||
if not absolute_path.exists():
|
||||
raise KnowledgeContractError(f"artifact content 不存在: {absolute_path}")
|
||||
return absolute_path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def build_normalized_document(
|
||||
document_entry: JSONMapping,
|
||||
acl_policies: dict[str, dict[str, object]],
|
||||
) -> JSONMapping:
|
||||
"""Build a normalized document contract row."""
|
||||
|
||||
visibility = require_non_empty_str(
|
||||
document_entry.get("visibility"), "document.visibility"
|
||||
)
|
||||
normalized: JSONMapping = {
|
||||
"document_id": require_non_empty_str(
|
||||
document_entry.get("document_id"), "document.document_id"
|
||||
),
|
||||
"title": require_non_empty_str(document_entry.get("title"), "document.title"),
|
||||
"source_type": require_non_empty_str(
|
||||
document_entry.get("source_type"), "document.source_type"
|
||||
),
|
||||
"source_ref": require_non_empty_str(
|
||||
document_entry.get("source_ref"), "document.source_ref"
|
||||
),
|
||||
"product_area": require_non_empty_str(
|
||||
document_entry.get("product_area"), "document.product_area"
|
||||
),
|
||||
"visibility": visibility,
|
||||
"acl_expression": build_acl_expression(visibility, acl_policies),
|
||||
"customer_id": None,
|
||||
"doc_type": require_non_empty_str(
|
||||
document_entry.get("doc_type"), "document.doc_type"
|
||||
),
|
||||
"owner": require_non_empty_str(document_entry.get("owner"), "document.owner"),
|
||||
"version": require_non_empty_str(
|
||||
document_entry.get("version"), "document.version"
|
||||
),
|
||||
"checksum": require_non_empty_str(
|
||||
document_entry.get("checksum"), "document.checksum"
|
||||
),
|
||||
"review_status": require_non_empty_str(
|
||||
document_entry.get("review_status"), "document.review_status"
|
||||
),
|
||||
"approved_by": optional_non_empty_str(
|
||||
document_entry.get("approved_by"), "document.approved_by"
|
||||
),
|
||||
"effective_from": optional_non_empty_str(
|
||||
document_entry.get("effective_from"), "document.effective_from"
|
||||
),
|
||||
"expires_at": optional_non_empty_str(
|
||||
document_entry.get("expires_at"), "document.expires_at"
|
||||
),
|
||||
"tags": list(cast(list[str], document_entry.get("tags", []))),
|
||||
}
|
||||
return normalized
|
||||
|
||||
|
||||
def build_chunk_rows(
|
||||
normalized_document: JSONMapping, content: str
|
||||
) -> list[JSONMapping]:
|
||||
"""Build chunk contract rows for one normalized document."""
|
||||
|
||||
document_id = require_non_empty_str(
|
||||
normalized_document.get("document_id"), "normalized.document_id"
|
||||
)
|
||||
visibility = require_non_empty_str(
|
||||
normalized_document.get("visibility"), "normalized.visibility"
|
||||
)
|
||||
acl_expression = require_non_empty_str(
|
||||
normalized_document.get("acl_expression"), "normalized.acl_expression"
|
||||
)
|
||||
chunks: list[JSONMapping] = []
|
||||
for index, chunk_text in enumerate(chunk_markdown(content)):
|
||||
chunks.append(
|
||||
{
|
||||
"chunk_id": f"{document_id}:chunk:{index:04d}",
|
||||
"document_id": document_id,
|
||||
"chunk_index": index,
|
||||
"heading_path": [],
|
||||
"visibility": visibility,
|
||||
"acl_expression": acl_expression,
|
||||
"customer_id": None,
|
||||
"token_count": max(1, len(chunk_text.strip()) // 4),
|
||||
"embedding_model": None,
|
||||
"text_hash": sha256_text(chunk_text),
|
||||
"published_targets": [],
|
||||
}
|
||||
)
|
||||
return chunks
|
||||
|
||||
|
||||
def validate_contract_rows(
|
||||
*,
|
||||
deploy_root: Path,
|
||||
normalized_rows: list[JSONMapping],
|
||||
chunk_rows: list[JSONMapping],
|
||||
) -> None:
|
||||
"""Validate normalized document and chunk rows."""
|
||||
|
||||
document_validator = load_validator(deploy_root / DOCUMENT_SCHEMA_RELATIVE)
|
||||
chunk_validator = load_validator(deploy_root / CHUNK_SCHEMA_RELATIVE)
|
||||
for row in normalized_rows:
|
||||
document_id = require_non_empty_str(
|
||||
row.get("document_id"), "normalized.document_id"
|
||||
)
|
||||
validate_payload(document_validator, row, f"normalized<{document_id}>")
|
||||
for row in chunk_rows:
|
||||
chunk_id = require_non_empty_str(row.get("chunk_id"), "chunk.chunk_id")
|
||||
validate_payload(chunk_validator, row, f"chunk<{chunk_id}>")
|
||||
|
||||
|
||||
def build_publishable_target_ids(
|
||||
*,
|
||||
normalized_document: JSONMapping,
|
||||
acl_policies: dict[str, dict[str, object]],
|
||||
publish_targets: dict[str, dict[str, object]],
|
||||
) -> list[str]:
|
||||
"""Resolve the Dify targets allowed for a normalized document."""
|
||||
|
||||
visibility = require_non_empty_str(
|
||||
normalized_document.get("visibility"), "visibility"
|
||||
)
|
||||
policy = acl_policies.get(visibility)
|
||||
if policy is None:
|
||||
raise KnowledgeContractError(f"未找到 ACL policy: {visibility}")
|
||||
allowed_targets = cast(list[str], policy.get("allowed_targets", []))
|
||||
|
||||
resolved: list[str] = []
|
||||
for target_id in allowed_targets:
|
||||
target = publish_targets.get(target_id)
|
||||
if target is None:
|
||||
continue
|
||||
target_type = require_non_empty_str(
|
||||
target.get("type"), f"target.{target_id}.type"
|
||||
)
|
||||
target_visibility = require_non_empty_str(
|
||||
target.get("visibility"), f"target.{target_id}.visibility"
|
||||
)
|
||||
if target_type != "dify_dataset":
|
||||
continue
|
||||
if target_visibility != visibility:
|
||||
continue
|
||||
resolved.append(target_id)
|
||||
return resolved
|
||||
|
||||
|
||||
def remove_remote_targets(
|
||||
*,
|
||||
record: JSONMapping,
|
||||
client: DifyClient | None,
|
||||
skip_dify: bool,
|
||||
) -> list[JSONMapping]:
|
||||
"""Remove remote documents recorded on a publish record."""
|
||||
|
||||
updated_targets: list[JSONMapping] = []
|
||||
raw_targets = cast_mapping_list(record.get("targets", []), "record.targets")
|
||||
now = utc_now()
|
||||
for target in raw_targets:
|
||||
dataset_id = optional_non_empty_str(
|
||||
target.get("dataset_id"), "target.dataset_id"
|
||||
)
|
||||
remote_document_id = optional_non_empty_str(
|
||||
target.get("remote_document_id"), "target.remote_document_id"
|
||||
)
|
||||
if not skip_dify and client and dataset_id and remote_document_id:
|
||||
client.delete_document(dataset_id, remote_document_id)
|
||||
target["status"] = "invalidated"
|
||||
target["updated_at"] = now
|
||||
updated_targets.append(target)
|
||||
return updated_targets
|
||||
|
||||
|
||||
def choose_duplicate_of(
|
||||
*,
|
||||
records: dict[str, JSONMapping],
|
||||
document_id: str,
|
||||
visibility: str,
|
||||
checksum: str,
|
||||
) -> str | None:
|
||||
"""Find an already-published record with the same checksum."""
|
||||
|
||||
for existing_document_id, record in records.items():
|
||||
if existing_document_id == document_id:
|
||||
continue
|
||||
if optional_non_empty_str(record.get("status"), "record.status") != "published":
|
||||
continue
|
||||
if (
|
||||
optional_non_empty_str(record.get("visibility"), "record.visibility")
|
||||
!= visibility
|
||||
):
|
||||
continue
|
||||
if (
|
||||
optional_non_empty_str(record.get("checksum"), "record.checksum")
|
||||
!= checksum
|
||||
):
|
||||
continue
|
||||
return existing_document_id
|
||||
return None
|
||||
|
||||
|
||||
def publish_targets_to_dify(
|
||||
*,
|
||||
client: DifyClient | None,
|
||||
skip_dify: bool,
|
||||
target_ids: list[str],
|
||||
publish_targets: dict[str, dict[str, object]],
|
||||
document_name: str,
|
||||
content: str,
|
||||
existing_record: JSONMapping | None,
|
||||
) -> list[JSONMapping]:
|
||||
"""Publish one document to all resolved Dify targets."""
|
||||
|
||||
existing_target_by_id: dict[str, JSONMapping] = {}
|
||||
if existing_record is not None:
|
||||
for item in cast_mapping_list(
|
||||
existing_record.get("targets", []), "record.targets"
|
||||
):
|
||||
target_id = require_non_empty_str(
|
||||
item.get("target_id"), "record.targets[].target_id"
|
||||
)
|
||||
existing_target_by_id[target_id] = item
|
||||
|
||||
results: list[JSONMapping] = []
|
||||
now = utc_now()
|
||||
for target_id in target_ids:
|
||||
target = publish_targets[target_id]
|
||||
dataset_name = require_non_empty_str(
|
||||
target.get("dataset_name"), f"publish-targets.{target_id}.dataset_name"
|
||||
)
|
||||
dataset_id: str | None = None
|
||||
remote_document_id: str | None = None
|
||||
existing_target = existing_target_by_id.get(target_id)
|
||||
if existing_target is not None:
|
||||
dataset_id = optional_non_empty_str(
|
||||
existing_target.get("dataset_id"), "existing_target.dataset_id"
|
||||
)
|
||||
remote_document_id = optional_non_empty_str(
|
||||
existing_target.get("remote_document_id"),
|
||||
"existing_target.remote_document_id",
|
||||
)
|
||||
|
||||
if not skip_dify:
|
||||
if client is None:
|
||||
raise KnowledgeContractError("Dify client 未初始化")
|
||||
dataset = client.ensure_dataset(dataset_name)
|
||||
dataset_id = dataset.dataset_id
|
||||
if remote_document_id is None:
|
||||
remote_document_id = client.find_document_id(dataset_id, document_name)
|
||||
remote_document_id = client.upsert_document(
|
||||
dataset_id=dataset_id,
|
||||
document_name=document_name,
|
||||
content=content,
|
||||
document_id=remote_document_id,
|
||||
)
|
||||
|
||||
results.append(
|
||||
{
|
||||
"target_id": target_id,
|
||||
"target_type": require_non_empty_str(
|
||||
target.get("type"), f"target.{target_id}.type"
|
||||
),
|
||||
"dataset_name": dataset_name,
|
||||
"dataset_id": dataset_id,
|
||||
"document_name": document_name,
|
||||
"remote_document_id": remote_document_id,
|
||||
"status": "published" if not skip_dify else "skipped",
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def maybe_build_dify_client(args: argparse.Namespace) -> DifyClient | None:
|
||||
"""Create a Dify client when live publication is enabled."""
|
||||
|
||||
if args.skip_dify:
|
||||
return None
|
||||
base_url = require_non_empty_str(args.dify_base_url, "--dify-base-url")
|
||||
api_key = require_non_empty_str(args.dify_api_key, "--dify-api-key")
|
||||
try:
|
||||
return DifyClient(base_url=base_url, api_key=api_key)
|
||||
except DifyClientError as exc:
|
||||
raise KnowledgeContractError(str(exc)) from exc
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entry point."""
|
||||
|
||||
args = parse_args()
|
||||
deploy_root = args.deploy_root.resolve()
|
||||
client = maybe_build_dify_client(args)
|
||||
artifact_zip = resolve_artifact_zip(args)
|
||||
extract_root = extract_artifact(artifact_zip)
|
||||
|
||||
try:
|
||||
manifest = load_artifact_manifest(extract_root, deploy_root)
|
||||
artifact_id = require_non_empty_str(
|
||||
manifest.get("artifact_id"), "manifest.artifact_id"
|
||||
)
|
||||
source = require_mapping(manifest.get("source"), "manifest.source")
|
||||
source_repo = require_non_empty_str(source.get("repo"), "manifest.source.repo")
|
||||
service_name = require_non_empty_str(
|
||||
source.get("service"), "manifest.source.service"
|
||||
)
|
||||
raw_documents = cast_mapping_list(
|
||||
manifest.get("documents"), "manifest.documents"
|
||||
)
|
||||
validate_source_registration(
|
||||
deploy_root=deploy_root,
|
||||
source_repo=source_repo,
|
||||
service_name=service_name,
|
||||
documents=raw_documents,
|
||||
)
|
||||
|
||||
snapshot_root = copy_artifact_snapshot(
|
||||
extract_root=extract_root,
|
||||
deploy_root=deploy_root,
|
||||
environment=args.env,
|
||||
source_repo=source_repo,
|
||||
artifact_id=artifact_id,
|
||||
)
|
||||
artifact_ref = (
|
||||
(snapshot_root / "manifest.json").relative_to(deploy_root).as_posix()
|
||||
)
|
||||
normalized_path = snapshot_root / "normalized-documents.jsonl"
|
||||
classified_path = snapshot_root / "classified-documents.jsonl"
|
||||
chunks_path = snapshot_root / "chunks.jsonl"
|
||||
|
||||
acl_policies = load_acl_policies(deploy_root)
|
||||
publish_targets = load_publish_targets(deploy_root)
|
||||
publish_record_validator = load_validator(
|
||||
deploy_root / PUBLISH_RECORD_SCHEMA_RELATIVE
|
||||
)
|
||||
records_root = knowledge_paths(deploy_root, args.env).publish_records_root
|
||||
records = load_publish_records(records_root)
|
||||
|
||||
normalized_rows: list[JSONMapping] = []
|
||||
classified_rows: list[JSONMapping] = []
|
||||
all_chunk_rows: list[JSONMapping] = []
|
||||
events: list[JSONMapping] = []
|
||||
published_documents = 0
|
||||
invalidated_documents = 0
|
||||
|
||||
invalidate_entries = [
|
||||
item
|
||||
for item in raw_documents
|
||||
if require_non_empty_str(item.get("op"), "document.op") == "invalidate"
|
||||
]
|
||||
upsert_entries = [
|
||||
item
|
||||
for item in raw_documents
|
||||
if require_non_empty_str(item.get("op"), "document.op") == "upsert"
|
||||
]
|
||||
|
||||
for document_entry in invalidate_entries:
|
||||
document_id = require_non_empty_str(
|
||||
document_entry.get("document_id"), "document.document_id"
|
||||
)
|
||||
existing_record = records.get(document_id)
|
||||
if existing_record is None:
|
||||
events.append(
|
||||
build_event(
|
||||
action="invalidate",
|
||||
artifact_id=artifact_id,
|
||||
document_id=document_id,
|
||||
source_repo=source_repo,
|
||||
status="ignored",
|
||||
reason="source_deleted_without_existing_record",
|
||||
target_ids=[],
|
||||
)
|
||||
)
|
||||
continue
|
||||
updated_targets = remove_remote_targets(
|
||||
record=existing_record,
|
||||
client=client,
|
||||
skip_dify=args.skip_dify,
|
||||
)
|
||||
existing_record["status"] = "invalidated"
|
||||
existing_record["latest_action"] = "invalidate"
|
||||
existing_record["latest_reason"] = require_non_empty_str(
|
||||
document_entry.get("reason"), "document.reason"
|
||||
)
|
||||
existing_record["updated_at"] = utc_now()
|
||||
existing_record["targets"] = updated_targets
|
||||
validate_payload(
|
||||
publish_record_validator,
|
||||
existing_record,
|
||||
f"publish-record<{document_id}>",
|
||||
)
|
||||
record_path = record_path_for_document(
|
||||
deploy_root, args.env, source_repo, document_id
|
||||
)
|
||||
write_json(record_path, existing_record)
|
||||
records[document_id] = existing_record
|
||||
invalidated_documents += 1
|
||||
events.append(
|
||||
build_event(
|
||||
action="invalidate",
|
||||
artifact_id=artifact_id,
|
||||
document_id=document_id,
|
||||
source_repo=source_repo,
|
||||
status="invalidated",
|
||||
reason=require_non_empty_str(
|
||||
document_entry.get("reason"), "document.reason"
|
||||
),
|
||||
target_ids=[
|
||||
require_non_empty_str(item.get("target_id"), "target.target_id")
|
||||
for item in updated_targets
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
for document_entry in upsert_entries:
|
||||
document_id = require_non_empty_str(
|
||||
document_entry.get("document_id"), "document.document_id"
|
||||
)
|
||||
existing_record = records.get(document_id)
|
||||
content = load_artifact_content(snapshot_root, document_entry)
|
||||
normalized_document = build_normalized_document(
|
||||
document_entry, acl_policies
|
||||
)
|
||||
normalized_rows.append(normalized_document)
|
||||
classified_rows.append(dict(normalized_document))
|
||||
chunk_rows = build_chunk_rows(normalized_document, content)
|
||||
all_chunk_rows.extend(chunk_rows)
|
||||
|
||||
visibility = require_non_empty_str(
|
||||
normalized_document.get("visibility"), "normalized.visibility"
|
||||
)
|
||||
checksum = require_non_empty_str(
|
||||
normalized_document.get("checksum"), "normalized.checksum"
|
||||
)
|
||||
review_status = require_non_empty_str(
|
||||
normalized_document.get("review_status"), "normalized.review_status"
|
||||
)
|
||||
publish_target_ids = build_publishable_target_ids(
|
||||
normalized_document=normalized_document,
|
||||
acl_policies=acl_policies,
|
||||
publish_targets=publish_targets,
|
||||
)
|
||||
duplicate_of = choose_duplicate_of(
|
||||
records=records,
|
||||
document_id=document_id,
|
||||
visibility=visibility,
|
||||
checksum=checksum,
|
||||
)
|
||||
content_ref = (
|
||||
(
|
||||
snapshot_root
|
||||
/ require_non_empty_str(
|
||||
document_entry.get("content_path"), "document.content_path"
|
||||
)
|
||||
)
|
||||
.relative_to(deploy_root)
|
||||
.as_posix()
|
||||
)
|
||||
base_record: JSONMapping = {
|
||||
"schema_version": "0.1.0",
|
||||
"document_id": document_id,
|
||||
"source_repo": source_repo,
|
||||
"source_id": require_non_empty_str(
|
||||
document_entry.get("source_id"), "document.source_id"
|
||||
),
|
||||
"source_path": require_non_empty_str(
|
||||
document_entry.get("source_path"), "document.source_path"
|
||||
),
|
||||
"source_ref": require_non_empty_str(
|
||||
document_entry.get("source_ref"), "document.source_ref"
|
||||
),
|
||||
"visibility": visibility,
|
||||
"checksum": checksum,
|
||||
"title": require_non_empty_str(
|
||||
document_entry.get("title"), "document.title"
|
||||
),
|
||||
"source_type": require_non_empty_str(
|
||||
document_entry.get("source_type"), "document.source_type"
|
||||
),
|
||||
"doc_type": require_non_empty_str(
|
||||
document_entry.get("doc_type"), "document.doc_type"
|
||||
),
|
||||
"product_area": require_non_empty_str(
|
||||
document_entry.get("product_area"), "document.product_area"
|
||||
),
|
||||
"owner": require_non_empty_str(
|
||||
document_entry.get("owner"), "document.owner"
|
||||
),
|
||||
"review_status": review_status,
|
||||
"acl_expression": require_non_empty_str(
|
||||
normalized_document.get("acl_expression"),
|
||||
"normalized.acl_expression",
|
||||
),
|
||||
"version": require_non_empty_str(
|
||||
document_entry.get("version"), "document.version"
|
||||
),
|
||||
"tags": list(cast(list[str], document_entry.get("tags", []))),
|
||||
"status": "published",
|
||||
"duplicate_of": None,
|
||||
"artifact_ref": artifact_ref,
|
||||
"content_ref": content_ref,
|
||||
"normalized_ref": normalized_path.relative_to(deploy_root).as_posix(),
|
||||
"classified_ref": classified_path.relative_to(deploy_root).as_posix(),
|
||||
"chunk_ref": chunks_path.relative_to(deploy_root).as_posix(),
|
||||
"chunk_count": len(chunk_rows),
|
||||
"latest_action": "publish",
|
||||
"latest_reason": "artifact_ingest",
|
||||
"published_at": utc_now(),
|
||||
"updated_at": utc_now(),
|
||||
"targets": [],
|
||||
}
|
||||
|
||||
if review_status != "approved":
|
||||
if existing_record is not None:
|
||||
base_record["targets"] = remove_remote_targets(
|
||||
record=existing_record,
|
||||
client=client,
|
||||
skip_dify=args.skip_dify,
|
||||
)
|
||||
base_record["status"] = "invalidated"
|
||||
base_record["latest_action"] = "invalidate"
|
||||
base_record["latest_reason"] = f"review_status={review_status}"
|
||||
base_record["published_at"] = None
|
||||
invalidated_documents += 1
|
||||
events.append(
|
||||
build_event(
|
||||
action="invalidate",
|
||||
artifact_id=artifact_id,
|
||||
document_id=document_id,
|
||||
source_repo=source_repo,
|
||||
status="invalidated",
|
||||
reason=f"review_status={review_status}",
|
||||
target_ids=[],
|
||||
)
|
||||
)
|
||||
elif duplicate_of is not None:
|
||||
if existing_record is not None:
|
||||
base_record["targets"] = remove_remote_targets(
|
||||
record=existing_record,
|
||||
client=client,
|
||||
skip_dify=args.skip_dify,
|
||||
)
|
||||
base_record["status"] = "deduplicated"
|
||||
base_record["duplicate_of"] = duplicate_of
|
||||
base_record["latest_action"] = "invalidate"
|
||||
base_record["latest_reason"] = f"duplicate_of={duplicate_of}"
|
||||
base_record["published_at"] = None
|
||||
invalidated_documents += 1
|
||||
events.append(
|
||||
build_event(
|
||||
action="invalidate",
|
||||
artifact_id=artifact_id,
|
||||
document_id=document_id,
|
||||
source_repo=source_repo,
|
||||
status="deduplicated",
|
||||
reason=f"duplicate_of={duplicate_of}",
|
||||
target_ids=[],
|
||||
)
|
||||
)
|
||||
else:
|
||||
if existing_record is not None:
|
||||
previous_targets = {
|
||||
require_non_empty_str(item.get("target_id"), "record.target_id")
|
||||
for item in cast_mapping_list(
|
||||
existing_record.get("targets", []), "record.targets"
|
||||
)
|
||||
}
|
||||
if set(publish_target_ids) != previous_targets:
|
||||
remove_remote_targets(
|
||||
record=existing_record,
|
||||
client=client,
|
||||
skip_dify=args.skip_dify,
|
||||
)
|
||||
try:
|
||||
base_record["targets"] = publish_targets_to_dify(
|
||||
client=client,
|
||||
skip_dify=args.skip_dify,
|
||||
target_ids=publish_target_ids,
|
||||
publish_targets=publish_targets,
|
||||
document_name=artifact_document_name(document_id),
|
||||
content=content,
|
||||
existing_record=existing_record,
|
||||
)
|
||||
except DifyClientError as exc:
|
||||
raise KnowledgeContractError(str(exc)) from exc
|
||||
published_documents += 1
|
||||
events.append(
|
||||
build_event(
|
||||
action="publish",
|
||||
artifact_id=artifact_id,
|
||||
document_id=document_id,
|
||||
source_repo=source_repo,
|
||||
status="published" if not args.skip_dify else "staged",
|
||||
reason="artifact_ingest",
|
||||
target_ids=[
|
||||
require_non_empty_str(
|
||||
item.get("target_id"), "target.target_id"
|
||||
)
|
||||
for item in cast_mapping_list(
|
||||
base_record.get("targets", []), "record.targets"
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
validate_payload(
|
||||
publish_record_validator,
|
||||
base_record,
|
||||
f"publish-record<{document_id}>",
|
||||
)
|
||||
record_path = record_path_for_document(
|
||||
deploy_root, args.env, source_repo, document_id
|
||||
)
|
||||
write_json(record_path, base_record)
|
||||
records[document_id] = base_record
|
||||
|
||||
validate_contract_rows(
|
||||
deploy_root=deploy_root,
|
||||
normalized_rows=normalized_rows,
|
||||
chunk_rows=all_chunk_rows,
|
||||
)
|
||||
write_jsonl(normalized_path, normalized_rows)
|
||||
write_jsonl(classified_path, classified_rows)
|
||||
write_jsonl(chunks_path, all_chunk_rows)
|
||||
for event in events:
|
||||
append_jsonl(knowledge_paths(deploy_root, args.env).events_path, event)
|
||||
|
||||
summary: JSONMapping = {
|
||||
"artifact_id": artifact_id,
|
||||
"environment": args.env,
|
||||
"published_documents": published_documents,
|
||||
"invalidated_documents": invalidated_documents,
|
||||
"normalized_documents": len(normalized_rows),
|
||||
"classified_documents": len(classified_rows),
|
||||
"chunk_rows": len(all_chunk_rows),
|
||||
"snapshot_root": snapshot_root.relative_to(deploy_root).as_posix(),
|
||||
"source_repo": source_repo,
|
||||
}
|
||||
if args.summary_json:
|
||||
write_json(args.summary_json.resolve(), summary)
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
return 0
|
||||
finally:
|
||||
artifact_temp_root = artifact_zip.parent
|
||||
if artifact_temp_root.name.startswith("knowledge-ingest-"):
|
||||
shutil.rmtree(artifact_temp_root, ignore_errors=True)
|
||||
shutil.rmtree(extract_root, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Replay published knowledge records into Dify dev targets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .common import (
|
||||
JSONMapping,
|
||||
KnowledgeContractError,
|
||||
PUBLISH_RECORD_SCHEMA_RELATIVE,
|
||||
append_jsonl,
|
||||
artifact_document_name,
|
||||
build_event,
|
||||
cast_mapping_list,
|
||||
knowledge_paths,
|
||||
load_publish_records,
|
||||
load_validator,
|
||||
optional_non_empty_str,
|
||||
require_non_empty_str,
|
||||
utc_now,
|
||||
validate_payload,
|
||||
write_json,
|
||||
)
|
||||
from .dify_client import DifyClient, DifyClientError
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse CLI arguments."""
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Reconcile published knowledge records"
|
||||
)
|
||||
parser.add_argument("--deploy-root", type=Path, required=True)
|
||||
parser.add_argument("--env", default="dev")
|
||||
parser.add_argument("--source-repo")
|
||||
parser.add_argument("--document-id")
|
||||
parser.add_argument("--dify-base-url", default="")
|
||||
parser.add_argument("--dify-api-key", default="")
|
||||
parser.add_argument("--skip-dify", action="store_true")
|
||||
parser.add_argument("--summary-json", type=Path)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def maybe_build_dify_client(args: argparse.Namespace) -> DifyClient | None:
|
||||
"""Create a Dify client when live reconciliation is enabled."""
|
||||
|
||||
if args.skip_dify:
|
||||
return None
|
||||
try:
|
||||
return DifyClient(
|
||||
base_url=require_non_empty_str(args.dify_base_url, "--dify-base-url"),
|
||||
api_key=require_non_empty_str(args.dify_api_key, "--dify-api-key"),
|
||||
)
|
||||
except DifyClientError as exc:
|
||||
raise KnowledgeContractError(str(exc)) from exc
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entry point."""
|
||||
|
||||
args = parse_args()
|
||||
deploy_root = args.deploy_root.resolve()
|
||||
client = maybe_build_dify_client(args)
|
||||
records = load_publish_records(
|
||||
knowledge_paths(deploy_root, args.env).publish_records_root
|
||||
)
|
||||
validator = load_validator(deploy_root / PUBLISH_RECORD_SCHEMA_RELATIVE)
|
||||
|
||||
reconciled = 0
|
||||
for document_id, record in records.items():
|
||||
source_repo = require_non_empty_str(
|
||||
record.get("source_repo"), "record.source_repo"
|
||||
)
|
||||
if args.source_repo and source_repo != args.source_repo:
|
||||
continue
|
||||
if args.document_id and document_id != args.document_id:
|
||||
continue
|
||||
status = require_non_empty_str(record.get("status"), "record.status")
|
||||
if status != "published":
|
||||
continue
|
||||
content_ref = optional_non_empty_str(
|
||||
record.get("content_ref"), "record.content_ref"
|
||||
)
|
||||
if content_ref is None:
|
||||
continue
|
||||
content_path = deploy_root / content_ref
|
||||
if not content_path.exists():
|
||||
raise KnowledgeContractError(f"content_ref 不存在: {content_path}")
|
||||
content = content_path.read_text(encoding="utf-8")
|
||||
|
||||
updated_targets: list[JSONMapping] = []
|
||||
for target in cast_mapping_list(record.get("targets", []), "record.targets"):
|
||||
target_id = require_non_empty_str(
|
||||
target.get("target_id"), "target.target_id"
|
||||
)
|
||||
dataset_name = require_non_empty_str(
|
||||
target.get("dataset_name"), "target.dataset_name"
|
||||
)
|
||||
dataset_id = optional_non_empty_str(
|
||||
target.get("dataset_id"), "target.dataset_id"
|
||||
)
|
||||
remote_document_id = optional_non_empty_str(
|
||||
target.get("remote_document_id"), "target.remote_document_id"
|
||||
)
|
||||
if not args.skip_dify:
|
||||
if client is None:
|
||||
raise KnowledgeContractError("Dify client 未初始化")
|
||||
dataset = client.ensure_dataset(dataset_name)
|
||||
dataset_id = dataset.dataset_id
|
||||
if remote_document_id is None:
|
||||
remote_document_id = client.find_document_id(
|
||||
dataset.dataset_id, artifact_document_name(document_id)
|
||||
)
|
||||
remote_document_id = client.upsert_document(
|
||||
dataset_id=dataset.dataset_id,
|
||||
document_name=artifact_document_name(document_id),
|
||||
content=content,
|
||||
document_id=remote_document_id,
|
||||
)
|
||||
updated_targets.append(
|
||||
{
|
||||
"target_id": target_id,
|
||||
"target_type": require_non_empty_str(
|
||||
target.get("target_type"), "target.target_type"
|
||||
),
|
||||
"dataset_name": dataset_name,
|
||||
"dataset_id": dataset_id,
|
||||
"document_name": artifact_document_name(document_id),
|
||||
"remote_document_id": remote_document_id,
|
||||
"status": "published" if not args.skip_dify else "skipped",
|
||||
"updated_at": utc_now(),
|
||||
}
|
||||
)
|
||||
|
||||
record["targets"] = updated_targets
|
||||
record["latest_action"] = "republish"
|
||||
record["latest_reason"] = "manual_reconcile"
|
||||
record["updated_at"] = utc_now()
|
||||
validate_payload(validator, record, f"publish-record<{document_id}>")
|
||||
source_repo_slug = source_repo.replace("/", "__")
|
||||
record_path = (
|
||||
knowledge_paths(deploy_root, args.env).publish_records_root
|
||||
/ source_repo_slug
|
||||
/ f"{document_id}.json"
|
||||
)
|
||||
write_json(record_path, record)
|
||||
append_jsonl(
|
||||
knowledge_paths(deploy_root, args.env).events_path,
|
||||
build_event(
|
||||
action="republish",
|
||||
artifact_id=require_non_empty_str(
|
||||
record.get("artifact_ref"), "record.artifact_ref"
|
||||
),
|
||||
document_id=document_id,
|
||||
source_repo=source_repo,
|
||||
status="published" if not args.skip_dify else "staged",
|
||||
reason="manual_reconcile",
|
||||
target_ids=[
|
||||
require_non_empty_str(item.get("target_id"), "target.target_id")
|
||||
for item in updated_targets
|
||||
],
|
||||
),
|
||||
)
|
||||
reconciled += 1
|
||||
|
||||
summary: JSONMapping = {
|
||||
"environment": args.env,
|
||||
"document_id": args.document_id or "",
|
||||
"source_repo": args.source_repo or "",
|
||||
"reconciled_records": reconciled,
|
||||
}
|
||||
if args.summary_json:
|
||||
write_json(args.summary_json.resolve(), summary)
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Manually unpublish one knowledge document from central targets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .common import (
|
||||
JSONMapping,
|
||||
KnowledgeContractError,
|
||||
PUBLISH_RECORD_SCHEMA_RELATIVE,
|
||||
append_jsonl,
|
||||
build_event,
|
||||
cast_mapping_list,
|
||||
knowledge_paths,
|
||||
load_validator,
|
||||
record_path_for_document,
|
||||
require_non_empty_str,
|
||||
utc_now,
|
||||
validate_payload,
|
||||
write_json,
|
||||
)
|
||||
from .dify_client import DifyClient, DifyClientError
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse CLI arguments."""
|
||||
|
||||
parser = argparse.ArgumentParser(description="Unpublish one knowledge document")
|
||||
parser.add_argument("--deploy-root", type=Path, required=True)
|
||||
parser.add_argument("--env", default="dev")
|
||||
parser.add_argument("--source-repo", required=True)
|
||||
parser.add_argument("--document-id", required=True)
|
||||
parser.add_argument("--reason", default="manual_unpublish")
|
||||
parser.add_argument("--dify-base-url", default="")
|
||||
parser.add_argument("--dify-api-key", default="")
|
||||
parser.add_argument("--skip-dify", action="store_true")
|
||||
parser.add_argument("--summary-json", type=Path)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def maybe_build_dify_client(args: argparse.Namespace) -> DifyClient | None:
|
||||
"""Create a Dify client when live publication is enabled."""
|
||||
|
||||
if args.skip_dify:
|
||||
return None
|
||||
try:
|
||||
return DifyClient(
|
||||
base_url=require_non_empty_str(args.dify_base_url, "--dify-base-url"),
|
||||
api_key=require_non_empty_str(args.dify_api_key, "--dify-api-key"),
|
||||
)
|
||||
except DifyClientError as exc:
|
||||
raise KnowledgeContractError(str(exc)) from exc
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entry point."""
|
||||
|
||||
args = parse_args()
|
||||
deploy_root = args.deploy_root.resolve()
|
||||
client = maybe_build_dify_client(args)
|
||||
record_path = record_path_for_document(
|
||||
deploy_root, args.env, args.source_repo, args.document_id
|
||||
)
|
||||
if not record_path.exists():
|
||||
raise KnowledgeContractError(f"未找到 publish record: {record_path}")
|
||||
|
||||
record = json.loads(record_path.read_text(encoding="utf-8"))
|
||||
if not isinstance(record, dict):
|
||||
raise KnowledgeContractError(
|
||||
f"publish record 根节点必须是 object: {record_path}"
|
||||
)
|
||||
payload = {str(key): value for key, value in record.items()}
|
||||
|
||||
updated_targets: list[JSONMapping] = []
|
||||
for target in cast_mapping_list(payload.get("targets", []), "record.targets"):
|
||||
dataset_id = target.get("dataset_id")
|
||||
remote_document_id = target.get("remote_document_id")
|
||||
if (
|
||||
not args.skip_dify
|
||||
and client is not None
|
||||
and isinstance(dataset_id, str)
|
||||
and dataset_id
|
||||
and isinstance(remote_document_id, str)
|
||||
and remote_document_id
|
||||
):
|
||||
client.delete_document(dataset_id, remote_document_id)
|
||||
target["status"] = "unpublished"
|
||||
target["updated_at"] = utc_now()
|
||||
updated_targets.append(target)
|
||||
|
||||
payload["status"] = "unpublished"
|
||||
payload["latest_action"] = "unpublish"
|
||||
payload["latest_reason"] = args.reason
|
||||
payload["updated_at"] = utc_now()
|
||||
payload["targets"] = updated_targets
|
||||
|
||||
validator = load_validator(deploy_root / PUBLISH_RECORD_SCHEMA_RELATIVE)
|
||||
validate_payload(validator, payload, f"publish-record<{args.document_id}>")
|
||||
write_json(record_path, payload)
|
||||
append_jsonl(
|
||||
knowledge_paths(deploy_root, args.env).events_path,
|
||||
build_event(
|
||||
action="unpublish",
|
||||
artifact_id=require_non_empty_str(
|
||||
payload.get("artifact_ref"), "record.artifact_ref"
|
||||
),
|
||||
document_id=args.document_id,
|
||||
source_repo=args.source_repo,
|
||||
status="unpublished",
|
||||
reason=args.reason,
|
||||
target_ids=[
|
||||
require_non_empty_str(item.get("target_id"), "target.target_id")
|
||||
for item in updated_targets
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
summary: JSONMapping = {
|
||||
"document_id": args.document_id,
|
||||
"environment": args.env,
|
||||
"source_repo": args.source_repo,
|
||||
"targets": len(updated_targets),
|
||||
"status": "unpublished",
|
||||
}
|
||||
if args.summary_json:
|
||||
write_json(args.summary_json.resolve(), summary)
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate committed knowledge publish records."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError as exc:
|
||||
raise SystemExit(
|
||||
f"缺少依赖 PyYAML,请使用 uvx --with pyyaml --with jsonschema python <script>\n{exc}"
|
||||
) from exc
|
||||
|
||||
from .common import (
|
||||
JSONMapping,
|
||||
KnowledgeContractError,
|
||||
PUBLISH_RECORD_SCHEMA_RELATIVE,
|
||||
knowledge_paths,
|
||||
load_publish_records,
|
||||
load_validator,
|
||||
optional_non_empty_str,
|
||||
require_non_empty_str,
|
||||
validate_payload,
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse CLI arguments."""
|
||||
|
||||
parser = argparse.ArgumentParser(description="Validate knowledge publish records")
|
||||
parser.add_argument(
|
||||
"--deploy-root", type=Path, default=Path("."), help="ljwx-deploy root"
|
||||
)
|
||||
parser.add_argument("--env", default="dev")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_platform_version(path: Path) -> JSONMapping:
|
||||
"""Load release/platform-version.yaml."""
|
||||
|
||||
raw = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, dict):
|
||||
raise KnowledgeContractError(f"{path} 根节点必须是 mapping")
|
||||
return {str(key): value for key, value in raw.items()}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""CLI entry point."""
|
||||
|
||||
args = parse_args()
|
||||
deploy_root = args.deploy_root.resolve()
|
||||
validator = load_validator(deploy_root / PUBLISH_RECORD_SCHEMA_RELATIVE)
|
||||
records = load_publish_records(
|
||||
knowledge_paths(deploy_root, args.env).publish_records_root
|
||||
)
|
||||
release_version = load_platform_version(
|
||||
deploy_root / "release/platform-version.yaml"
|
||||
)
|
||||
config_versions = release_version.get("config_versions")
|
||||
if not isinstance(config_versions, dict):
|
||||
raise KnowledgeContractError(
|
||||
"release/platform-version.yaml.config_versions 缺失"
|
||||
)
|
||||
publish_record_ref = optional_non_empty_str(
|
||||
config_versions.get("knowledge_publish_record_schema"),
|
||||
"release/platform-version.yaml.config_versions.knowledge_publish_record_schema",
|
||||
)
|
||||
expected_ref = f"{PUBLISH_RECORD_SCHEMA_RELATIVE.as_posix()}@0.1.0"
|
||||
if publish_record_ref != expected_ref:
|
||||
raise KnowledgeContractError(
|
||||
"release/platform-version.yaml 中 knowledge_publish_record_schema "
|
||||
f"必须等于 {expected_ref}"
|
||||
)
|
||||
|
||||
for document_id, record in records.items():
|
||||
validate_payload(validator, record, f"publish-record<{document_id}>")
|
||||
artifact_ref = require_non_empty_str(
|
||||
record.get("artifact_ref"), "record.artifact_ref"
|
||||
)
|
||||
normalized_ref = require_non_empty_str(
|
||||
record.get("normalized_ref"), "record.normalized_ref"
|
||||
)
|
||||
classified_ref = require_non_empty_str(
|
||||
record.get("classified_ref"), "record.classified_ref"
|
||||
)
|
||||
chunk_ref = require_non_empty_str(record.get("chunk_ref"), "record.chunk_ref")
|
||||
for relative_path in (artifact_ref, normalized_ref, classified_ref, chunk_ref):
|
||||
absolute_path = deploy_root / relative_path
|
||||
if not absolute_path.exists():
|
||||
raise KnowledgeContractError(
|
||||
f"publish record 引用了不存在的文件: {absolute_path}"
|
||||
)
|
||||
|
||||
print(f"knowledge publish record 校验通过 ({len(records)} 条)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KnowledgeContractError as exc:
|
||||
print(str(exc))
|
||||
raise SystemExit(1) from exc
|
||||
@@ -31,6 +31,9 @@ uvx --with pyyaml --with jsonschema python scripts/evidence/validate.py
|
||||
uvx --with pyyaml --with jsonschema python scripts/evidence/collect.py --out /tmp/evidence-index.json --summary /tmp/evidence-latest.md
|
||||
uvx --with pyyaml python scripts/promoter/queue_metrics.py --queue release/queue.yaml --out /tmp/queue-health.json
|
||||
|
||||
echo "[verify] validate knowledge publish records"
|
||||
uvx --with pyyaml --with jsonschema python -m scripts.knowledge.validate_state --deploy-root . --env dev
|
||||
|
||||
echo "[verify] run promoter dry-run on local repo"
|
||||
python3 scripts/promoter/promote.py --dry-run --local-repo-dir .
|
||||
echo "[verify] promoter supports local simulation without Harbor via --skip-registry-check"
|
||||
|
||||
Reference in New Issue
Block a user