From fd6316432843cb67ad373e519970f7ce15a29d97 Mon Sep 17 00:00:00 2001 From: csh Date: Wed, 15 Jul 2026 15:59:37 +0800 Subject: [PATCH] :wrench: chore(ci): inline skill sync and standardize git identity --- .gitea/ci/sync_thirdparty_skills.sh | 238 --------- .gitea/ci/update_thirdparty_skills.sh | 227 --------- .gitea/workflows/sync-tsl-playbook.yml | 8 +- .gitea/workflows/update-thirdparty-skills.yml | 471 +++++++++++++++++- test/test_thirdparty_skills_pipeline.py | 94 +++- 5 files changed, 544 insertions(+), 494 deletions(-) delete mode 100644 .gitea/ci/sync_thirdparty_skills.sh delete mode 100644 .gitea/ci/update_thirdparty_skills.sh diff --git a/.gitea/ci/sync_thirdparty_skills.sh b/.gitea/ci/sync_thirdparty_skills.sh deleted file mode 100644 index a2ceda25..00000000 --- a/.gitea/ci/sync_thirdparty_skills.sh +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -REPO_DIR="${REPO_DIR:-$(pwd)}" -THIRDPARTY_BRANCH="${THIRDPARTY_BRANCH:-thirdparty/skill}" -TARGET_BRANCH="${TARGET_BRANCH:-main}" -MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}" -COMMIT_AUTHOR_NAME="${COMMIT_AUTHOR_NAME:-ci[bot]}" -COMMIT_AUTHOR_EMAIL="${COMMIT_AUTHOR_EMAIL:-ci-bot@local}" - -emit_sources_tsv() { - python3 - "$MANIFEST_PATH" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as fh: - data = json.load(fh) - -for entry in data["sources"]: - print( - "\x1f".join( - [ - entry["id"], - entry["snapshot_dir"], - entry["sync_mode"], - entry["source_list"], - entry.get("skills_subdir", ""), - entry.get("output_name", entry["id"]), - entry.get("platform_config", ""), - entry.get("template_root", ""), - entry.get("data_dir", ""), - entry.get("scripts_dir", ""), - "\x1e".join(entry.get("include_skill_dirs", [])), - ] - ) - ) -PY -} - -render_skill() { - local snapshot_root="$1" - local output_dir="$2" - local platform_config_rel="$3" - local template_root_rel="$4" - local data_dir_rel="$5" - local scripts_dir_rel="$6" - - python3 - "$snapshot_root" "$output_dir" "$platform_config_rel" "$template_root_rel" "$data_dir_rel" "$scripts_dir_rel" <<'PY' -import json -import pathlib -import shutil -import sys - -snapshot_root = pathlib.Path(sys.argv[1]) -output_dir = pathlib.Path(sys.argv[2]) -platform_config_path = snapshot_root / sys.argv[3] -template_root = snapshot_root / sys.argv[4] -data_dir = snapshot_root / sys.argv[5] -scripts_dir = snapshot_root / sys.argv[6] - -config = json.loads(platform_config_path.read_text(encoding="utf-8")) -skill_template = (template_root / "base" / "skill-content.md").read_text(encoding="utf-8") -quick_reference = "" -if config.get("sections", {}).get("quickReference"): - quick_reference = "\n" + (template_root / "base" / "quick-reference.md").read_text(encoding="utf-8") - -def render_frontmatter(frontmatter): - if not frontmatter: - return "" - - lines = ["---"] - for key, value in frontmatter.items(): - if any(ch in value for ch in ':"\n'): - value = value.replace('"', '\\"') - lines.append(f'{key}: "{value}"') - else: - lines.append(f"{key}: {value}") - lines.extend(["---", ""]) - return "\n".join(lines) - -content = skill_template -content = content.replace("{{TITLE}}", config["title"]) -content = content.replace("{{DESCRIPTION}}", config["description"]) -content = content.replace("{{SCRIPT_PATH}}", config["scriptPath"]) -content = content.replace("{{SKILL_OR_WORKFLOW}}", config["skillOrWorkflow"]) -content = content.replace("{{QUICK_REFERENCE}}", quick_reference) - -if output_dir.exists(): - shutil.rmtree(output_dir) -output_dir.mkdir(parents=True, exist_ok=True) - -(output_dir / "SKILL.md").write_text( - render_frontmatter(config.get("frontmatter")) + content, - encoding="utf-8", -) -if data_dir.exists(): - shutil.copytree(data_dir, output_dir / "data") -if scripts_dir.exists(): - shutil.copytree(scripts_dir, output_dir / "scripts") -PY -} - -tracked_skill_exists() { - local name="$1" - # conflict only if a first-party SKILL.md exists directly under skills// - [ -f "skills/$name/SKILL.md" ] -} - -skill_dir_included() { - local name="$1" - local include_skill_dirs="$2" - - [ -n "$include_skill_dirs" ] || return 0 - - local IFS=$'\x1e' - local expected - read -r -a included <<< "$include_skill_dirs" - for expected in "${included[@]}"; do - if [ "$name" = "$expected" ]; then - return 0 - fi - done - - return 1 -} - - -cd "$REPO_DIR" - -git config user.name "$COMMIT_AUTHOR_NAME" -git config user.email "$COMMIT_AUTHOR_EMAIL" - -git fetch origin "$THIRDPARTY_BRANCH" -git fetch origin "$TARGET_BRANCH" - -tmp_dir="$(mktemp -d)" -cleanup() { - rm -rf "$tmp_dir" -} -trap cleanup EXIT - -git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH" - -mkdir -p "skills/thirdparty/.sources" - -sources_file="$tmp_dir/sources.tsv" -if ! emit_sources_tsv > "$sources_file"; then - echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2 - exit 1 -fi - -while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do - [ -n "$source_id" ] || continue - - if [ -f "$source_list" ]; then - while IFS= read -r name; do - [ -n "$name" ] || continue - rm -rf "skills/$name" - done < "$source_list" - fi -done < "$sources_file" - -declare -A owners=() -while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do - [ -n "$source_id" ] || continue - - git archive --format=tar "origin/${THIRDPARTY_BRANCH}" "$snapshot_dir" | tar -xf - -C "$tmp_dir" - snapshot_root="$tmp_dir/$snapshot_dir" - - names=() - case "$sync_mode" in - copy_skill_dirs) - source_skills_dir="$snapshot_root/$skills_subdir" - if [ ! -d "$source_skills_dir" ]; then - echo "ERROR: $skills_subdir not found in snapshot $snapshot_dir" >&2 - exit 1 - fi - - for dir in "$source_skills_dir"/*; do - [ -d "$dir" ] || continue - name="$(basename "$dir")" - if ! skill_dir_included "$name" "$include_skill_dirs"; then - continue - fi - if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then - echo "ERROR: duplicate third-party skill name: $name" >&2 - exit 1 - fi - if tracked_skill_exists "$name"; then - echo "ERROR: skill name conflict with tracked skill: $name" >&2 - exit 1 - fi - - rm -rf "skills/thirdparty/$name" - cp -R "$dir" "skills/thirdparty/$name" - names+=("$name") - owners["$name"]="$source_id" - done - ;; - render_skill) - name="$output_name" - if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then - echo "ERROR: duplicate third-party skill name: $name" >&2 - exit 1 - fi - if tracked_skill_exists "$name"; then - echo "ERROR: skill name conflict with tracked skill: $name" >&2 - exit 1 - fi - - render_skill "$snapshot_root" "skills/thirdparty/$name" "$platform_config" "$template_root" "$data_dir" "$scripts_dir" - names+=("$name") - owners["$name"]="$source_id" - ;; - *) - echo "ERROR: unsupported sync mode: $sync_mode" >&2 - exit 1 - ;; - esac - - printf "%s\n" "${names[@]}" | sort > "$source_list" -done < "$sources_file" - -git add skills - -if git diff --cached --quiet; then - echo "No third-party skills to sync." - exit 0 -fi - -git commit -m ":package: deps(skills): sync thirdparty skills" - -TOKEN="${WORKFLOW:-}" -if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then - git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git" -fi - -git push origin "$TARGET_BRANCH" diff --git a/.gitea/ci/update_thirdparty_skills.sh b/.gitea/ci/update_thirdparty_skills.sh deleted file mode 100644 index 29547dc1..00000000 --- a/.gitea/ci/update_thirdparty_skills.sh +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -REPO_DIR="${REPO_DIR:-$(pwd)}" -TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}" -MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}" -COMMIT_AUTHOR_NAME="${COMMIT_AUTHOR_NAME:-ci[bot]}" -COMMIT_AUTHOR_EMAIL="${COMMIT_AUTHOR_EMAIL:-ci-bot@local}" - -retry_cmd() { - local retries="$1" - shift - local delay="$1" - shift - - local attempt=1 - while true; do - if "$@"; then - return 0 - fi - if [ "$attempt" -ge "$retries" ]; then - return 1 - fi - echo "Retry ($attempt/$retries): $*" >&2 - sleep "$delay" - attempt=$((attempt + 1)) - done -} - -github_owner_repo() { - case "$1" in - https://github.com/*) - echo "$1" | sed -E 's#^https://github.com/([^/]+/[^/.]+)(\.git)?$#\1#' - ;; - http://github.com/*) - echo "$1" | sed -E 's#^http://github.com/([^/]+/[^/.]+)(\.git)?$#\1#' - ;; - git@github.com:*) - echo "$1" | sed -E 's#^git@github.com:([^/]+/[^/.]+)(\.git)?$#\1#' - ;; - *) - return 1 - ;; - esac -} - -resolve_latest_sha() { - local repo="$1" - local ref="$2" - local tmp_json="$3" - local gh_repo="$4" - local sha="" - - if [ -n "$gh_repo" ]; then - local api_url="https://api.github.com/repos/${gh_repo}/commits/${ref}" - if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$api_url" -o "$tmp_json"; then - sha="$(sed -n 's/^[[:space:]]*"sha":[[:space:]]*"\([0-9a-f]\{40\}\)".*/\1/p' "$tmp_json" | head -n 1)" - if [ -n "$sha" ]; then - echo "$sha" - return 0 - fi - fi - fi - - sha="$(retry_cmd 3 2 git -c http.version=HTTP/1.1 ls-remote "$repo" "refs/heads/$ref" | awk 'NR==1 {print $1}')" - if [ -n "$sha" ]; then - echo "$sha" - return 0 - fi - - return 1 -} - -emit_sources_tsv() { - python3 - "$MANIFEST_PATH" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as fh: - data = json.load(fh) - -for entry in data["sources"]: - print( - "\x1f".join( - [ - entry["id"], - entry["upstream_repo"], - entry.get("upstream_ref", "main"), - entry["snapshot_dir"], - entry["sync_mode"], - "\x1e".join(entry.get("remove_paths", [])), - ] - ) - ) -PY -} - -read_source_metadata_value() { - local key="$1" - local source_file="$2" - - if [ ! -f "$source_file" ]; then - return 0 - fi - - sed -n "s/^- ${key}:[[:space:]]*//p" "$source_file" | head -n 1 -} - -remove_snapshot_paths() { - local snapshot_dir="$1" - local remove_paths="$2" - - [ -n "$remove_paths" ] || return 0 - - local IFS=$'\x1e' - read -r -a paths <<< "$remove_paths" - for path in "${paths[@]}"; do - [ -n "$path" ] || continue - rm -rf "$snapshot_dir/$path" - done -} - -cd "$REPO_DIR" - -git config user.name "$COMMIT_AUTHOR_NAME" -git config user.email "$COMMIT_AUTHOR_EMAIL" - -tmp_dir="$(mktemp -d)" -cleanup() { - rm -rf "$tmp_dir" -} -trap cleanup EXIT - -if [ ! -f "$MANIFEST_PATH" ]; then - echo "ERROR: third-party manifest not found: $MANIFEST_PATH" >&2 - exit 1 -fi - -manifest_copy="$tmp_dir/thirdparty_skills.json" -cp "$MANIFEST_PATH" "$manifest_copy" -MANIFEST_PATH="$manifest_copy" - -git fetch origin "$TARGET_BRANCH" -git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH" - -sources_file="$tmp_dir/sources.tsv" -if ! emit_sources_tsv > "$sources_file"; then - echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2 - exit 1 -fi - -changed=0 -while IFS=$'\x1f' read -r source_id upstream_repo upstream_ref snapshot_dir sync_mode remove_paths; do - [ -n "$source_id" ] || continue - remove_paths_md="${remove_paths//$'\x1e'/,}" - - gh_repo="" - if gh_repo="$(github_owner_repo "$upstream_repo" 2>/dev/null)"; then - : - fi - - latest_sha="$(resolve_latest_sha "$upstream_repo" "$upstream_ref" "$tmp_dir/${source_id}-latest.json" "$gh_repo" || true)" - if [ -z "$latest_sha" ]; then - echo "ERROR: failed to resolve upstream ref for ${source_id}: $upstream_repo $upstream_ref" >&2 - exit 1 - fi - - current_sha="$(read_source_metadata_value "Ref" "$snapshot_dir/SOURCE.md")" - current_remove_paths="$(read_source_metadata_value "Remove-Paths" "$snapshot_dir/SOURCE.md")" - - if [ "$latest_sha" = "$current_sha" ] && [ "$remove_paths_md" = "$current_remove_paths" ]; then - echo "Third-party snapshot is up to date for ${source_id}: $latest_sha" - continue - fi - - rm -rf "$snapshot_dir" - mkdir -p "$snapshot_dir" - - snapshot_loaded=0 - if [ -n "$gh_repo" ]; then - tar_url="https://codeload.github.com/${gh_repo}/tar.gz/${latest_sha}" - if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$tar_url" -o "$tmp_dir/${source_id}.tar.gz"; then - tar -xzf "$tmp_dir/${source_id}.tar.gz" -C "$snapshot_dir" --strip-components=1 - snapshot_loaded=1 - fi - fi - - if [ "$snapshot_loaded" -eq 0 ]; then - upstream_dir="$tmp_dir/${source_id}-upstream" - git init "$upstream_dir" >/dev/null - git -C "$upstream_dir" remote add origin "$upstream_repo" - retry_cmd 3 2 git -C "$upstream_dir" fetch --depth 1 origin "$latest_sha" - git -C "$upstream_dir" checkout --detach FETCH_HEAD - git -C "$upstream_dir" archive --format=tar HEAD | tar -xf - -C "$snapshot_dir" - fi - - remove_snapshot_paths "$snapshot_dir" "$remove_paths" - - snapshot_date="$(date -u +%Y-%m-%d)" - cat > "$snapshot_dir/SOURCE.md" <&2 + sleep "$delay" + attempt=$((attempt + 1)) + done + } + + github_owner_repo() { + case "$1" in + https://github.com/*) + echo "$1" | sed -E 's#^https://github.com/([^/]+/[^/.]+)(\.git)?$#\1#' + ;; + http://github.com/*) + echo "$1" | sed -E 's#^http://github.com/([^/]+/[^/.]+)(\.git)?$#\1#' + ;; + git@github.com:*) + echo "$1" | sed -E 's#^git@github.com:([^/]+/[^/.]+)(\.git)?$#\1#' + ;; + *) + return 1 + ;; + esac + } + + resolve_latest_sha() { + local repo="$1" + local ref="$2" + local tmp_json="$3" + local gh_repo="$4" + local sha="" + + if [ -n "$gh_repo" ]; then + local api_url="https://api.github.com/repos/${gh_repo}/commits/${ref}" + if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$api_url" -o "$tmp_json"; then + sha="$(sed -n 's/^[[:space:]]*"sha":[[:space:]]*"\([0-9a-f]\{40\}\)".*/\1/p' "$tmp_json" | head -n 1)" + if [ -n "$sha" ]; then + echo "$sha" + return 0 + fi + fi + fi + + sha="$(retry_cmd 3 2 git -c http.version=HTTP/1.1 ls-remote "$repo" "refs/heads/$ref" | awk 'NR==1 {print $1}')" + if [ -n "$sha" ]; then + echo "$sha" + return 0 + fi + + return 1 + } + + emit_sources_tsv() { + python3 - "$MANIFEST_PATH" <<'PY' + import json + import sys + + with open(sys.argv[1], encoding="utf-8") as fh: + data = json.load(fh) + + for entry in data["sources"]: + print( + "\x1f".join( + [ + entry["id"], + entry["upstream_repo"], + entry.get("upstream_ref", "main"), + entry["snapshot_dir"], + entry["sync_mode"], + "\x1e".join(entry.get("remove_paths", [])), + ] + ) + ) + PY + } + + read_source_metadata_value() { + local key="$1" + local source_file="$2" + + if [ ! -f "$source_file" ]; then + return 0 + fi + + sed -n "s/^- ${key}:[[:space:]]*//p" "$source_file" | head -n 1 + } + + remove_snapshot_paths() { + local snapshot_dir="$1" + local remove_paths="$2" + + [ -n "$remove_paths" ] || return 0 + + local IFS=$'\x1e' + read -r -a paths <<< "$remove_paths" + for path in "${paths[@]}"; do + [ -n "$path" ] || continue + rm -rf "$snapshot_dir/$path" + done + } + + cd "$REPO_DIR" + + git config user.name "$GIT_USER_NAME" + git config user.email "$GIT_USER_EMAIL" + + tmp_dir="$(mktemp -d)" + cleanup() { + rm -rf "$tmp_dir" + } + trap cleanup EXIT + + if [ ! -f "$MANIFEST_PATH" ]; then + echo "ERROR: third-party manifest not found: $MANIFEST_PATH" >&2 + exit 1 + fi + + manifest_copy="$tmp_dir/thirdparty_skills.json" + cp "$MANIFEST_PATH" "$manifest_copy" + MANIFEST_PATH="$manifest_copy" + + git fetch origin "$TARGET_BRANCH" + git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH" + + sources_file="$tmp_dir/sources.tsv" + if ! emit_sources_tsv > "$sources_file"; then + echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2 + exit 1 + fi + + changed=0 + while IFS=$'\x1f' read -r source_id upstream_repo upstream_ref snapshot_dir sync_mode remove_paths; do + [ -n "$source_id" ] || continue + remove_paths_md="${remove_paths//$'\x1e'/,}" + + gh_repo="" + if gh_repo="$(github_owner_repo "$upstream_repo" 2>/dev/null)"; then + : + fi + + latest_sha="$(resolve_latest_sha "$upstream_repo" "$upstream_ref" "$tmp_dir/${source_id}-latest.json" "$gh_repo" || true)" + if [ -z "$latest_sha" ]; then + echo "ERROR: failed to resolve upstream ref for ${source_id}: $upstream_repo $upstream_ref" >&2 + exit 1 + fi + + current_sha="$(read_source_metadata_value "Ref" "$snapshot_dir/SOURCE.md")" + current_remove_paths="$(read_source_metadata_value "Remove-Paths" "$snapshot_dir/SOURCE.md")" + + if [ "$latest_sha" = "$current_sha" ] && [ "$remove_paths_md" = "$current_remove_paths" ]; then + echo "Third-party snapshot is up to date for ${source_id}: $latest_sha" + continue + fi + + rm -rf "$snapshot_dir" + mkdir -p "$snapshot_dir" + + snapshot_loaded=0 + if [ -n "$gh_repo" ]; then + tar_url="https://codeload.github.com/${gh_repo}/tar.gz/${latest_sha}" + if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$tar_url" -o "$tmp_dir/${source_id}.tar.gz"; then + tar -xzf "$tmp_dir/${source_id}.tar.gz" -C "$snapshot_dir" --strip-components=1 + snapshot_loaded=1 + fi + fi + + if [ "$snapshot_loaded" -eq 0 ]; then + upstream_dir="$tmp_dir/${source_id}-upstream" + git init "$upstream_dir" >/dev/null + git -C "$upstream_dir" remote add origin "$upstream_repo" + retry_cmd 3 2 git -C "$upstream_dir" fetch --depth 1 origin "$latest_sha" + git -C "$upstream_dir" checkout --detach FETCH_HEAD + git -C "$upstream_dir" archive --format=tar HEAD | tar -xf - -C "$snapshot_dir" + fi + + remove_snapshot_paths "$snapshot_dir" "$remove_paths" + + snapshot_date="$(date -u +%Y-%m-%d)" + cat > "$snapshot_dir/SOURCE.md" </ + [ -f "skills/$name/SKILL.md" ] + } + + skill_dir_included() { + local name="$1" + local include_skill_dirs="$2" + + [ -n "$include_skill_dirs" ] || return 0 + + local IFS=$'\x1e' + local expected + read -r -a included <<< "$include_skill_dirs" + for expected in "${included[@]}"; do + if [ "$name" = "$expected" ]; then + return 0 + fi + done + + return 1 + } + + + cd "$REPO_DIR" + + git config user.name "$GIT_USER_NAME" + git config user.email "$GIT_USER_EMAIL" + + git fetch origin "$THIRDPARTY_BRANCH" + git fetch origin "$TARGET_BRANCH" + + tmp_dir="$(mktemp -d)" + cleanup() { + rm -rf "$tmp_dir" + } + trap cleanup EXIT + + git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH" + + mkdir -p "skills/thirdparty/.sources" + + sources_file="$tmp_dir/sources.tsv" + if ! emit_sources_tsv > "$sources_file"; then + echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2 + exit 1 + fi + + while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do + [ -n "$source_id" ] || continue + + if [ -f "$source_list" ]; then + while IFS= read -r name; do + [ -n "$name" ] || continue + rm -rf "skills/$name" + done < "$source_list" + fi + done < "$sources_file" + + declare -A owners=() + while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do + [ -n "$source_id" ] || continue + + git archive --format=tar "origin/${THIRDPARTY_BRANCH}" "$snapshot_dir" | tar -xf - -C "$tmp_dir" + snapshot_root="$tmp_dir/$snapshot_dir" + + names=() + case "$sync_mode" in + copy_skill_dirs) + source_skills_dir="$snapshot_root/$skills_subdir" + if [ ! -d "$source_skills_dir" ]; then + echo "ERROR: $skills_subdir not found in snapshot $snapshot_dir" >&2 + exit 1 + fi + + for dir in "$source_skills_dir"/*; do + [ -d "$dir" ] || continue + name="$(basename "$dir")" + if ! skill_dir_included "$name" "$include_skill_dirs"; then + continue + fi + if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then + echo "ERROR: duplicate third-party skill name: $name" >&2 + exit 1 + fi + if tracked_skill_exists "$name"; then + echo "ERROR: skill name conflict with tracked skill: $name" >&2 + exit 1 + fi + + rm -rf "skills/thirdparty/$name" + cp -R "$dir" "skills/thirdparty/$name" + names+=("$name") + owners["$name"]="$source_id" + done + ;; + render_skill) + name="$output_name" + if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then + echo "ERROR: duplicate third-party skill name: $name" >&2 + exit 1 + fi + if tracked_skill_exists "$name"; then + echo "ERROR: skill name conflict with tracked skill: $name" >&2 + exit 1 + fi + + render_skill "$snapshot_root" "skills/thirdparty/$name" "$platform_config" "$template_root" "$data_dir" "$scripts_dir" + names+=("$name") + owners["$name"]="$source_id" + ;; + *) + echo "ERROR: unsupported sync mode: $sync_mode" >&2 + exit 1 + ;; + esac + + printf "%s\n" "${names[@]}" | sort > "$source_list" + done < "$sources_file" + + git add skills + + if git diff --cached --quiet; then + echo "No third-party skills to sync." + exit 0 + fi + + git commit -m ":package: deps(skills): sync thirdparty skills" + + TOKEN="${WORKFLOW:-}" + if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then + git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git" + fi + + git push origin "$TARGET_BRANCH" + SYNC_THIRDPARTY + # END sync_thirdparty_skills echo "✅ Update and sync finished." diff --git a/test/test_thirdparty_skills_pipeline.py b/test/test_thirdparty_skills_pipeline.py index 18276406..1da9a7cf 100644 --- a/test/test_thirdparty_skills_pipeline.py +++ b/test/test_thirdparty_skills_pipeline.py @@ -3,6 +3,7 @@ import os import shutil import subprocess import tempfile +import textwrap import unittest from pathlib import Path @@ -10,6 +11,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] MANIFEST = ROOT / ".gitea" / "ci" / "thirdparty_skills.json" WORKFLOW = ROOT / ".gitea" / "workflows" / "update-thirdparty-skills.yml" +TSL_SYNC_WORKFLOW = ROOT / ".gitea" / "workflows" / "sync-tsl-playbook.yml" LEGACY_WORKFLOW = ROOT / ".gitea" / "workflows" / "update-thirdparty-superpowers.yml" UPDATE_SCRIPT = ROOT / ".gitea" / "ci" / "update_thirdparty_skills.sh" SYNC_SCRIPT = ROOT / ".gitea" / "ci" / "sync_thirdparty_skills.sh" @@ -44,6 +46,16 @@ def run_command(*args: str, cwd: Path | None = None) -> subprocess.CompletedProc ) +def extract_workflow_region(name: str) -> str: + text = WORKFLOW.read_text(encoding="utf-8") + begin = f"# BEGIN {name}" + end = f"# END {name}" + if begin not in text or end not in text: + raise AssertionError(f"workflow region markers not found: {name}") + body = text.split(begin, 1)[1].split(end, 1)[0] + return textwrap.dedent(body).strip() + "\n" + + class ThirdpartySkillsPipelineTests(unittest.TestCase): def test_manifest_declares_all_thirdparty_sources(self): data = load_manifest() @@ -137,14 +149,18 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase): superpowers = next(item for item in data["sources"] if item["id"] == "superpowers") self.assertEqual(superpowers["remove_paths"], ["skills/ui-ux-pro-max"]) - def test_workflow_uses_generic_scripts_and_single_serial_job(self): + def test_workflow_inlines_update_and_sync_in_single_serial_job(self): text = WORKFLOW.read_text(encoding="utf-8") self.assertFalse(LEGACY_WORKFLOW.exists()) + self.assertFalse(UPDATE_SCRIPT.exists()) + self.assertFalse(SYNC_SCRIPT.exists()) self.assertIn("update_and_sync:", text) self.assertNotIn("\n update:\n", text) self.assertNotIn("\n sync:\n", text) - self.assertIn("bash .gitea/ci/update_thirdparty_skills.sh", text) - self.assertIn("bash .gitea/ci/sync_thirdparty_skills.sh", text) + self.assertIn("# BEGIN update_thirdparty_snapshots", text) + self.assertIn("# BEGIN sync_thirdparty_skills", text) + self.assertNotIn("update_thirdparty_skills.sh", text) + self.assertNotIn("sync_thirdparty_skills.sh", text) self.assertNotIn("git merge", text) self.assertNotIn("git pull", text) @@ -153,19 +169,30 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase): self.assertIn("concurrency:", text) self.assertIn("update-thirdparty-${{ github.repository }}", text) self.assertIn('MANIFEST_PATH: ".gitea/ci/thirdparty_skills.json"', text) - self.assertIn('TARGET_BRANCH="$THIRDPARTY_BRANCH" bash .gitea/ci/update_thirdparty_skills.sh', text) - self.assertIn('TARGET_BRANCH="main" \\', text) - self.assertIn('MANIFEST_PATH="$MANIFEST_PATH" \\', text) + self.assertIn("update_thirdparty_snapshots", text) + self.assertIn("sync_thirdparty_skills", text) - def test_generic_scripts_exist_and_use_manifest(self): - update_text = UPDATE_SCRIPT.read_text(encoding="utf-8") - sync_text = SYNC_SCRIPT.read_text(encoding="utf-8") - self.assertIn('MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"', update_text) - self.assertIn('MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"', sync_text) - self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"', update_text) - self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-main}"', sync_text) - self.assertIn(':package: deps(thirdparty): update snapshots', update_text) - self.assertIn(':package: deps(skills): sync thirdparty skills', sync_text) + def test_inline_workflow_exposes_manifest_and_publish_contract(self): + text = WORKFLOW.read_text(encoding="utf-8") + self.assertIn('MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"', text) + self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"', text) + self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-main}"', text) + self.assertIn(':package: deps(thirdparty): update snapshots', text) + self.assertIn(':package: deps(skills): sync thirdparty skills', text) + self.assertIn('git push origin "$TARGET_BRANCH"', text) + + def test_ci_committers_share_explicit_git_identity(self): + workflow_text = TSL_SYNC_WORKFLOW.read_text(encoding="utf-8") + thirdparty_text = WORKFLOW.read_text(encoding="utf-8") + + for text in (workflow_text, thirdparty_text): + self.assertIn('GIT_USER_NAME: "ci[bot]"', text) + self.assertIn('GIT_USER_EMAIL: "ci[bot]@tinysoft.com.cn"', text) + self.assertIn('git config user.name "$GIT_USER_NAME"', text) + self.assertIn('git config user.email "$GIT_USER_EMAIL"', text) + self.assertNotIn("COMMIT_AUTHOR_NAME", text) + self.assertNotIn("COMMIT_AUTHOR_EMAIL", text) + self.assertNotIn("@local", text) def test_skills_doc_points_to_generic_thirdparty_sources(self): text = SKILLS_MD.read_text(encoding="utf-8") @@ -193,8 +220,8 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase): self.assertTrue((UI_UX_PRO_MAX_DIR / "data").is_dir()) self.assertTrue((UI_UX_PRO_MAX_DIR / "scripts").is_dir()) - def test_update_script_materializes_manifest_before_target_checkout(self): - text = UPDATE_SCRIPT.read_text(encoding="utf-8") + def test_inline_update_materializes_manifest_before_target_checkout(self): + text = WORKFLOW.read_text(encoding="utf-8") self.assertIn('manifest_copy="$tmp_dir/thirdparty_skills.json"', text) self.assertIn('cp "$MANIFEST_PATH" "$manifest_copy"', text) self.assertIn('MANIFEST_PATH="$manifest_copy"', text) @@ -208,8 +235,8 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase): text.index('git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"'), ) - def test_sync_script_assumes_thirdparty_snapshot_is_already_clean(self): - text = SYNC_SCRIPT.read_text(encoding="utf-8") + def test_inline_sync_assumes_thirdparty_snapshot_is_already_clean(self): + text = WORKFLOW.read_text(encoding="utf-8") self.assertIn('"\\x1f".join(', text) self.assertIn("while IFS=$'\\x1f' read -r", text) self.assertIn("include_skill_dirs", text) @@ -218,7 +245,7 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase): self.assertNotIn("exclude_skill_dirs", text) self.assertNotIn("is_excluded_skill_dir", text) - def test_sync_script_generates_karpathy_outputs_in_temp_repo(self): + def test_inline_sync_generates_karpathy_outputs_in_temp_repo(self): with tempfile.TemporaryDirectory() as tmp_dir: tmp_root = Path(tmp_dir) mirror = tmp_root / "origin.git" @@ -265,9 +292,30 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase): json.dumps(manifest_data, indent=2) + "\n", encoding="utf-8", ) - shutil.copy2(SYNC_SCRIPT, work / ".gitea" / "ci" / "sync_thirdparty_skills.sh") - - sync_result = run_command("bash", ".gitea/ci/sync_thirdparty_skills.sh", cwd=work) + sync_script = extract_workflow_region("sync_thirdparty_skills") + script_path = work / ".sync-thirdparty-test.sh" + script_path.write_text( + 'export GIT_USER_NAME="test"\n' + 'export GIT_USER_EMAIL="test@example.invalid"\n' + + sync_script, + encoding="utf-8", + newline="\n", + ) + env = os.environ.copy() + env.update( + { + "REPO_DIR": str(work), + "THIRDPARTY_BRANCH": "thirdparty/skill", + "MANIFEST_PATH": ".gitea/ci/thirdparty_skills.json", + } + ) + sync_result = subprocess.run( + ["bash", script_path.name], + cwd=work, + env=env, + capture_output=True, + text=True, + ) self.assertEqual( sync_result.returncode, 0,