569 lines
20 KiB
YAML
569 lines
20 KiB
YAML
name: 🪄 Update Third-Party Skills
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: "@daily"
|
|
|
|
concurrency:
|
|
group: update-thirdparty-${{ github.repository }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
WORKSPACE_DIR: "/home/workspace"
|
|
THIRDPARTY_BRANCH: "thirdparty/skill"
|
|
MANIFEST_PATH: ".gitea/ci/thirdparty_skills.json"
|
|
GIT_USER_NAME: "ci[bot]"
|
|
GIT_USER_EMAIL: "ci[bot]@tinysoft.com.cn"
|
|
|
|
jobs:
|
|
update_and_sync:
|
|
name: ♻️ Update thirdparty and sync main
|
|
runs-on: ubuntu-22.04
|
|
|
|
steps:
|
|
- name: 🧰 Prepare repo
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
echo "========================================"
|
|
echo "📦 Prepare repo in WORKSPACE_DIR"
|
|
echo "========================================"
|
|
|
|
REPO_NAME="${{ github.event.repository.name }}"
|
|
TOKEN="${{ secrets.WORKFLOW }}"
|
|
mkdir -p "${{ env.WORKSPACE_DIR }}"
|
|
REPO_DIR="$(mktemp -d "${{ env.WORKSPACE_DIR }}/${REPO_NAME}.XXXXXX")"
|
|
if [ -n "$TOKEN" ]; then
|
|
REPO_URL="https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${{ github.repository }}.git"
|
|
else
|
|
REPO_URL="${GITHUB_SERVER_URL}/${{ github.repository }}.git"
|
|
fi
|
|
|
|
git clone "$REPO_URL" "$REPO_DIR"
|
|
|
|
git -C "$REPO_DIR" fetch origin main
|
|
git -C "$REPO_DIR" checkout -B main origin/main
|
|
|
|
git config --global --add safe.directory "$REPO_DIR"
|
|
echo "REPO_DIR=$REPO_DIR" >> "$GITHUB_ENV"
|
|
echo "✅ Repo prepared"
|
|
|
|
- name: ♻️ Update thirdparty and sync main
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
cd "$REPO_DIR"
|
|
|
|
echo "========================================"
|
|
echo "📦 Refresh thirdparty/skill snapshots"
|
|
echo "========================================"
|
|
|
|
before_ref=""
|
|
if git show-ref --verify --quiet "refs/remotes/origin/$THIRDPARTY_BRANCH"; then
|
|
before_ref="$(git rev-parse "origin/$THIRDPARTY_BRANCH")"
|
|
echo "📌 Previous $THIRDPARTY_BRANCH: $before_ref"
|
|
else
|
|
echo "📌 Previous $THIRDPARTY_BRANCH: <missing>"
|
|
fi
|
|
|
|
export MANIFEST_PATH="$MANIFEST_PATH"
|
|
# BEGIN update_thirdparty_snapshots
|
|
TARGET_BRANCH="$THIRDPARTY_BRANCH" bash <<'UPDATE_THIRDPARTY'
|
|
set -euo pipefail
|
|
|
|
REPO_DIR="${REPO_DIR:-$(pwd)}"
|
|
TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"
|
|
MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"
|
|
|
|
retry_cmd() {
|
|
local retries="$1"
|
|
shift
|
|
local delay="$1"
|
|
shift
|
|
|
|
local attempt=1
|
|
while true; do
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
if [ "$attempt" -ge "$retries" ]; then
|
|
return 1
|
|
fi
|
|
echo "Retry ($attempt/$retries): $*" >&2
|
|
sleep "$delay"
|
|
attempt=$((attempt + 1))
|
|
done
|
|
}
|
|
|
|
github_owner_repo() {
|
|
case "$1" in
|
|
https://github.com/*)
|
|
echo "$1" | sed -E 's#^https://github.com/([^/]+/[^/.]+)(\.git)?$#\1#'
|
|
;;
|
|
http://github.com/*)
|
|
echo "$1" | sed -E 's#^http://github.com/([^/]+/[^/.]+)(\.git)?$#\1#'
|
|
;;
|
|
git@github.com:*)
|
|
echo "$1" | sed -E 's#^git@github.com:([^/]+/[^/.]+)(\.git)?$#\1#'
|
|
;;
|
|
*)
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
resolve_latest_sha() {
|
|
local repo="$1"
|
|
local ref="$2"
|
|
local tmp_json="$3"
|
|
local gh_repo="$4"
|
|
local sha=""
|
|
|
|
if [ -n "$gh_repo" ]; then
|
|
local api_url="https://api.github.com/repos/${gh_repo}/commits/${ref}"
|
|
if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$api_url" -o "$tmp_json"; then
|
|
sha="$(sed -n 's/^[[:space:]]*"sha":[[:space:]]*"\([0-9a-f]\{40\}\)".*/\1/p' "$tmp_json" | head -n 1)"
|
|
if [ -n "$sha" ]; then
|
|
echo "$sha"
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
sha="$(retry_cmd 3 2 git -c http.version=HTTP/1.1 ls-remote "$repo" "refs/heads/$ref" | awk 'NR==1 {print $1}')"
|
|
if [ -n "$sha" ]; then
|
|
echo "$sha"
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
emit_sources_tsv() {
|
|
python3 - "$MANIFEST_PATH" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as fh:
|
|
data = json.load(fh)
|
|
|
|
for entry in data["sources"]:
|
|
print(
|
|
"\x1f".join(
|
|
[
|
|
entry["id"],
|
|
entry["upstream_repo"],
|
|
entry.get("upstream_ref", "main"),
|
|
entry["snapshot_dir"],
|
|
entry["sync_mode"],
|
|
"\x1e".join(entry.get("remove_paths", [])),
|
|
]
|
|
)
|
|
)
|
|
PY
|
|
}
|
|
|
|
read_source_metadata_value() {
|
|
local key="$1"
|
|
local source_file="$2"
|
|
|
|
if [ ! -f "$source_file" ]; then
|
|
return 0
|
|
fi
|
|
|
|
sed -n "s/^- ${key}:[[:space:]]*//p" "$source_file" | head -n 1
|
|
}
|
|
|
|
remove_snapshot_paths() {
|
|
local snapshot_dir="$1"
|
|
local remove_paths="$2"
|
|
|
|
[ -n "$remove_paths" ] || return 0
|
|
|
|
local IFS=$'\x1e'
|
|
read -r -a paths <<< "$remove_paths"
|
|
for path in "${paths[@]}"; do
|
|
[ -n "$path" ] || continue
|
|
rm -rf "$snapshot_dir/$path"
|
|
done
|
|
}
|
|
|
|
cd "$REPO_DIR"
|
|
|
|
git config user.name "$GIT_USER_NAME"
|
|
git config user.email "$GIT_USER_EMAIL"
|
|
|
|
tmp_dir="$(mktemp -d)"
|
|
cleanup() {
|
|
rm -rf "$tmp_dir"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
if [ ! -f "$MANIFEST_PATH" ]; then
|
|
echo "ERROR: third-party manifest not found: $MANIFEST_PATH" >&2
|
|
exit 1
|
|
fi
|
|
|
|
manifest_copy="$tmp_dir/thirdparty_skills.json"
|
|
cp "$MANIFEST_PATH" "$manifest_copy"
|
|
MANIFEST_PATH="$manifest_copy"
|
|
|
|
git fetch origin "$TARGET_BRANCH"
|
|
git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"
|
|
|
|
sources_file="$tmp_dir/sources.tsv"
|
|
if ! emit_sources_tsv > "$sources_file"; then
|
|
echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2
|
|
exit 1
|
|
fi
|
|
|
|
changed=0
|
|
while IFS=$'\x1f' read -r source_id upstream_repo upstream_ref snapshot_dir sync_mode remove_paths; do
|
|
[ -n "$source_id" ] || continue
|
|
remove_paths_md="${remove_paths//$'\x1e'/,}"
|
|
|
|
gh_repo=""
|
|
if gh_repo="$(github_owner_repo "$upstream_repo" 2>/dev/null)"; then
|
|
:
|
|
fi
|
|
|
|
latest_sha="$(resolve_latest_sha "$upstream_repo" "$upstream_ref" "$tmp_dir/${source_id}-latest.json" "$gh_repo" || true)"
|
|
if [ -z "$latest_sha" ]; then
|
|
echo "ERROR: failed to resolve upstream ref for ${source_id}: $upstream_repo $upstream_ref" >&2
|
|
exit 1
|
|
fi
|
|
|
|
current_sha="$(read_source_metadata_value "Ref" "$snapshot_dir/SOURCE.md")"
|
|
current_remove_paths="$(read_source_metadata_value "Remove-Paths" "$snapshot_dir/SOURCE.md")"
|
|
|
|
if [ "$latest_sha" = "$current_sha" ] && [ "$remove_paths_md" = "$current_remove_paths" ]; then
|
|
echo "Third-party snapshot is up to date for ${source_id}: $latest_sha"
|
|
continue
|
|
fi
|
|
|
|
rm -rf "$snapshot_dir"
|
|
mkdir -p "$snapshot_dir"
|
|
|
|
snapshot_loaded=0
|
|
if [ -n "$gh_repo" ]; then
|
|
tar_url="https://codeload.github.com/${gh_repo}/tar.gz/${latest_sha}"
|
|
if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$tar_url" -o "$tmp_dir/${source_id}.tar.gz"; then
|
|
tar -xzf "$tmp_dir/${source_id}.tar.gz" -C "$snapshot_dir" --strip-components=1
|
|
snapshot_loaded=1
|
|
fi
|
|
fi
|
|
|
|
if [ "$snapshot_loaded" -eq 0 ]; then
|
|
upstream_dir="$tmp_dir/${source_id}-upstream"
|
|
git init "$upstream_dir" >/dev/null
|
|
git -C "$upstream_dir" remote add origin "$upstream_repo"
|
|
retry_cmd 3 2 git -C "$upstream_dir" fetch --depth 1 origin "$latest_sha"
|
|
git -C "$upstream_dir" checkout --detach FETCH_HEAD
|
|
git -C "$upstream_dir" archive --format=tar HEAD | tar -xf - -C "$snapshot_dir"
|
|
fi
|
|
|
|
remove_snapshot_paths "$snapshot_dir" "$remove_paths"
|
|
|
|
snapshot_date="$(date -u +%Y-%m-%d)"
|
|
cat > "$snapshot_dir/SOURCE.md" <<EOF
|
|
# Source
|
|
|
|
- Repo: ${upstream_repo%".git"}
|
|
- Ref: $latest_sha
|
|
- Remove-Paths: $remove_paths_md
|
|
- Snapshot: $snapshot_date
|
|
- Sync-Mode: $sync_mode
|
|
- Notes: vendored into playbook branch $TARGET_BRANCH
|
|
EOF
|
|
|
|
git add -A "$snapshot_dir"
|
|
changed=1
|
|
done < "$sources_file"
|
|
|
|
if [ "$changed" -eq 0 ] || git diff --cached --quiet; then
|
|
echo "All third-party snapshots are up to date."
|
|
exit 0
|
|
fi
|
|
|
|
git commit -m ":package: deps(thirdparty): update snapshots"
|
|
|
|
TOKEN="${WORKFLOW:-}"
|
|
if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
|
|
git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
|
|
fi
|
|
|
|
git push origin "$TARGET_BRANCH"
|
|
UPDATE_THIRDPARTY
|
|
# END update_thirdparty_snapshots
|
|
|
|
git fetch origin "$THIRDPARTY_BRANCH"
|
|
after_ref="$(git rev-parse "origin/$THIRDPARTY_BRANCH")"
|
|
echo "📌 Current $THIRDPARTY_BRANCH: $after_ref"
|
|
|
|
if [ "$after_ref" = "$before_ref" ]; then
|
|
echo "✅ No thirdparty snapshot change; skip main sync."
|
|
exit 0
|
|
fi
|
|
|
|
echo "========================================"
|
|
echo "🔄 Sync skills into main"
|
|
echo "========================================"
|
|
|
|
git fetch origin main
|
|
git checkout -B main origin/main
|
|
|
|
# BEGIN sync_thirdparty_skills
|
|
TARGET_BRANCH="main" \
|
|
THIRDPARTY_BRANCH="$THIRDPARTY_BRANCH" \
|
|
MANIFEST_PATH="$MANIFEST_PATH" \
|
|
bash <<'SYNC_THIRDPARTY'
|
|
set -euo pipefail
|
|
|
|
REPO_DIR="${REPO_DIR:-$(pwd)}"
|
|
THIRDPARTY_BRANCH="${THIRDPARTY_BRANCH:-thirdparty/skill}"
|
|
TARGET_BRANCH="${TARGET_BRANCH:-main}"
|
|
MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"
|
|
|
|
emit_sources_tsv() {
|
|
python3 - "$MANIFEST_PATH" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as fh:
|
|
data = json.load(fh)
|
|
|
|
for entry in data["sources"]:
|
|
print(
|
|
"\x1f".join(
|
|
[
|
|
entry["id"],
|
|
entry["snapshot_dir"],
|
|
entry["sync_mode"],
|
|
entry["source_list"],
|
|
entry.get("skills_subdir", ""),
|
|
entry.get("output_name", entry["id"]),
|
|
entry.get("platform_config", ""),
|
|
entry.get("template_root", ""),
|
|
entry.get("data_dir", ""),
|
|
entry.get("scripts_dir", ""),
|
|
"\x1e".join(entry.get("include_skill_dirs", [])),
|
|
]
|
|
)
|
|
)
|
|
PY
|
|
}
|
|
|
|
render_skill() {
|
|
local snapshot_root="$1"
|
|
local output_dir="$2"
|
|
local platform_config_rel="$3"
|
|
local template_root_rel="$4"
|
|
local data_dir_rel="$5"
|
|
local scripts_dir_rel="$6"
|
|
|
|
python3 - "$snapshot_root" "$output_dir" "$platform_config_rel" "$template_root_rel" "$data_dir_rel" "$scripts_dir_rel" <<'PY'
|
|
import json
|
|
import pathlib
|
|
import shutil
|
|
import sys
|
|
|
|
snapshot_root = pathlib.Path(sys.argv[1])
|
|
output_dir = pathlib.Path(sys.argv[2])
|
|
platform_config_path = snapshot_root / sys.argv[3]
|
|
template_root = snapshot_root / sys.argv[4]
|
|
data_dir = snapshot_root / sys.argv[5]
|
|
scripts_dir = snapshot_root / sys.argv[6]
|
|
|
|
config = json.loads(platform_config_path.read_text(encoding="utf-8"))
|
|
skill_template = (template_root / "base" / "skill-content.md").read_text(encoding="utf-8")
|
|
quick_reference = ""
|
|
if config.get("sections", {}).get("quickReference"):
|
|
quick_reference = "\n" + (template_root / "base" / "quick-reference.md").read_text(encoding="utf-8")
|
|
|
|
def render_frontmatter(frontmatter):
|
|
if not frontmatter:
|
|
return ""
|
|
|
|
lines = ["---"]
|
|
for key, value in frontmatter.items():
|
|
if any(ch in value for ch in ':"\n'):
|
|
value = value.replace('"', '\\"')
|
|
lines.append(f'{key}: "{value}"')
|
|
else:
|
|
lines.append(f"{key}: {value}")
|
|
lines.extend(["---", ""])
|
|
return "\n".join(lines)
|
|
|
|
content = skill_template
|
|
content = content.replace("{{TITLE}}", config["title"])
|
|
content = content.replace("{{DESCRIPTION}}", config["description"])
|
|
content = content.replace("{{SCRIPT_PATH}}", config["scriptPath"])
|
|
content = content.replace("{{SKILL_OR_WORKFLOW}}", config["skillOrWorkflow"])
|
|
content = content.replace("{{QUICK_REFERENCE}}", quick_reference)
|
|
|
|
if output_dir.exists():
|
|
shutil.rmtree(output_dir)
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
(output_dir / "SKILL.md").write_text(
|
|
render_frontmatter(config.get("frontmatter")) + content,
|
|
encoding="utf-8",
|
|
)
|
|
if data_dir.exists():
|
|
shutil.copytree(data_dir, output_dir / "data")
|
|
if scripts_dir.exists():
|
|
shutil.copytree(scripts_dir, output_dir / "scripts")
|
|
PY
|
|
}
|
|
|
|
tracked_skill_exists() {
|
|
local name="$1"
|
|
# conflict only if a first-party SKILL.md exists directly under skills/<name>/
|
|
[ -f "skills/$name/SKILL.md" ]
|
|
}
|
|
|
|
skill_dir_included() {
|
|
local name="$1"
|
|
local include_skill_dirs="$2"
|
|
|
|
[ -n "$include_skill_dirs" ] || return 0
|
|
|
|
local IFS=$'\x1e'
|
|
local expected
|
|
read -r -a included <<< "$include_skill_dirs"
|
|
for expected in "${included[@]}"; do
|
|
if [ "$name" = "$expected" ]; then
|
|
return 0
|
|
fi
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
|
|
cd "$REPO_DIR"
|
|
|
|
git config user.name "$GIT_USER_NAME"
|
|
git config user.email "$GIT_USER_EMAIL"
|
|
|
|
git fetch origin "$THIRDPARTY_BRANCH"
|
|
git fetch origin "$TARGET_BRANCH"
|
|
|
|
tmp_dir="$(mktemp -d)"
|
|
cleanup() {
|
|
rm -rf "$tmp_dir"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"
|
|
|
|
mkdir -p "skills/thirdparty/.sources"
|
|
|
|
sources_file="$tmp_dir/sources.tsv"
|
|
if ! emit_sources_tsv > "$sources_file"; then
|
|
echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2
|
|
exit 1
|
|
fi
|
|
|
|
while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do
|
|
[ -n "$source_id" ] || continue
|
|
|
|
if [ -f "$source_list" ]; then
|
|
while IFS= read -r name; do
|
|
[ -n "$name" ] || continue
|
|
rm -rf "skills/$name"
|
|
done < "$source_list"
|
|
fi
|
|
done < "$sources_file"
|
|
|
|
declare -A owners=()
|
|
while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do
|
|
[ -n "$source_id" ] || continue
|
|
|
|
git archive --format=tar "origin/${THIRDPARTY_BRANCH}" "$snapshot_dir" | tar -xf - -C "$tmp_dir"
|
|
snapshot_root="$tmp_dir/$snapshot_dir"
|
|
|
|
names=()
|
|
case "$sync_mode" in
|
|
copy_skill_dirs)
|
|
source_skills_dir="$snapshot_root/$skills_subdir"
|
|
if [ ! -d "$source_skills_dir" ]; then
|
|
echo "ERROR: $skills_subdir not found in snapshot $snapshot_dir" >&2
|
|
exit 1
|
|
fi
|
|
|
|
for dir in "$source_skills_dir"/*; do
|
|
[ -d "$dir" ] || continue
|
|
name="$(basename "$dir")"
|
|
if ! skill_dir_included "$name" "$include_skill_dirs"; then
|
|
continue
|
|
fi
|
|
if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then
|
|
echo "ERROR: duplicate third-party skill name: $name" >&2
|
|
exit 1
|
|
fi
|
|
if tracked_skill_exists "$name"; then
|
|
echo "ERROR: skill name conflict with tracked skill: $name" >&2
|
|
exit 1
|
|
fi
|
|
|
|
rm -rf "skills/thirdparty/$name"
|
|
cp -R "$dir" "skills/thirdparty/$name"
|
|
names+=("$name")
|
|
owners["$name"]="$source_id"
|
|
done
|
|
;;
|
|
render_skill)
|
|
name="$output_name"
|
|
if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then
|
|
echo "ERROR: duplicate third-party skill name: $name" >&2
|
|
exit 1
|
|
fi
|
|
if tracked_skill_exists "$name"; then
|
|
echo "ERROR: skill name conflict with tracked skill: $name" >&2
|
|
exit 1
|
|
fi
|
|
|
|
render_skill "$snapshot_root" "skills/thirdparty/$name" "$platform_config" "$template_root" "$data_dir" "$scripts_dir"
|
|
names+=("$name")
|
|
owners["$name"]="$source_id"
|
|
;;
|
|
*)
|
|
echo "ERROR: unsupported sync mode: $sync_mode" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
printf "%s\n" "${names[@]}" | sort > "$source_list"
|
|
done < "$sources_file"
|
|
|
|
git add skills
|
|
|
|
if git diff --cached --quiet; then
|
|
echo "No third-party skills to sync."
|
|
exit 0
|
|
fi
|
|
|
|
git commit -m ":package: deps(skills): sync thirdparty skills"
|
|
|
|
TOKEN="${WORKFLOW:-}"
|
|
if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
|
|
git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
|
|
fi
|
|
|
|
git push origin "$TARGET_BRANCH"
|
|
SYNC_THIRDPARTY
|
|
# END sync_thirdparty_skills
|
|
|
|
echo "✅ Update and sync finished."
|
|
|
|
- name: 🧹 Clean temporary repo
|
|
if: always()
|
|
run: |
|
|
rm -rf "$REPO_DIR"
|