csh/playbook
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🔧 chore(ci): inline skill sync and standardize git identity

Browse Source
This commit is contained in:
csh
2026-07-20 09:08:36 +08:00
parent ff07c79818
commit fd63164328
5 changed files with 544 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/ci/sync_thirdparty_skills.sh
-238
View File
@@ -1,238 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
REPO_DIR="${REPO_DIR:-$(pwd)}"
THIRDPARTY_BRANCH="${THIRDPARTY_BRANCH:-thirdparty/skill}"
TARGET_BRANCH="${TARGET_BRANCH:-main}"
MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"
COMMIT_AUTHOR_NAME="${COMMIT_AUTHOR_NAME:-ci[bot]}"
COMMIT_AUTHOR_EMAIL="${COMMIT_AUTHOR_EMAIL:-ci-bot@local}"
emit_sources_tsv() {
python3 - "$MANIFEST_PATH" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as fh:
data = json.load(fh)
for entry in data["sources"]:
print(
"\x1f".join(
[
entry["id"],
entry["snapshot_dir"],
entry["sync_mode"],
entry["source_list"],
entry.get("skills_subdir", ""),
entry.get("output_name", entry["id"]),
entry.get("platform_config", ""),
entry.get("template_root", ""),
entry.get("data_dir", ""),
entry.get("scripts_dir", ""),
"\x1e".join(entry.get("include_skill_dirs", [])),
]
)
)
PY
}
render_skill() {
local snapshot_root="$1"
local output_dir="$2"
local platform_config_rel="$3"
local template_root_rel="$4"
local data_dir_rel="$5"
local scripts_dir_rel="$6"
python3 - "$snapshot_root" "$output_dir" "$platform_config_rel" "$template_root_rel" "$data_dir_rel" "$scripts_dir_rel" <<'PY'
import json
import pathlib
import shutil
import sys
snapshot_root = pathlib.Path(sys.argv[1])
output_dir = pathlib.Path(sys.argv[2])
platform_config_path = snapshot_root / sys.argv[3]
template_root = snapshot_root / sys.argv[4]
data_dir = snapshot_root / sys.argv[5]
scripts_dir = snapshot_root / sys.argv[6]
config = json.loads(platform_config_path.read_text(encoding="utf-8"))
skill_template = (template_root / "base" / "skill-content.md").read_text(encoding="utf-8")
quick_reference = ""
if config.get("sections", {}).get("quickReference"):
quick_reference = "\n" + (template_root / "base" / "quick-reference.md").read_text(encoding="utf-8")
def render_frontmatter(frontmatter):
if not frontmatter:
return ""
lines = ["---"]
for key, value in frontmatter.items():
if any(ch in value for ch in ':"\n'):
value = value.replace('"', '\\"')
lines.append(f'{key}: "{value}"')
else:
lines.append(f"{key}: {value}")
lines.extend(["---", ""])
return "\n".join(lines)
content = skill_template
content = content.replace("{{TITLE}}", config["title"])
content = content.replace("{{DESCRIPTION}}", config["description"])
content = content.replace("{{SCRIPT_PATH}}", config["scriptPath"])
content = content.replace("{{SKILL_OR_WORKFLOW}}", config["skillOrWorkflow"])
content = content.replace("{{QUICK_REFERENCE}}", quick_reference)
if output_dir.exists():
shutil.rmtree(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
(output_dir / "SKILL.md").write_text(
render_frontmatter(config.get("frontmatter")) + content,
encoding="utf-8",
)
if data_dir.exists():
shutil.copytree(data_dir, output_dir / "data")
if scripts_dir.exists():
shutil.copytree(scripts_dir, output_dir / "scripts")
PY
}
tracked_skill_exists() {
local name="$1"
# conflict only if a first-party SKILL.md exists directly under skills/<name>/
[ -f "skills/$name/SKILL.md" ]
}
skill_dir_included() {
local name="$1"
local include_skill_dirs="$2"
[ -n "$include_skill_dirs" ] || return 0
local IFS=$'\x1e'
local expected
read -r -a included <<< "$include_skill_dirs"
for expected in "${included[@]}"; do
if [ "$name" = "$expected" ]; then
return 0
fi
done
return 1
}
cd "$REPO_DIR"
git config user.name "$COMMIT_AUTHOR_NAME"
git config user.email "$COMMIT_AUTHOR_EMAIL"
git fetch origin "$THIRDPARTY_BRANCH"
git fetch origin "$TARGET_BRANCH"
tmp_dir="$(mktemp -d)"
cleanup() {
rm -rf "$tmp_dir"
}
trap cleanup EXIT
git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"
mkdir -p "skills/thirdparty/.sources"
sources_file="$tmp_dir/sources.tsv"
if ! emit_sources_tsv > "$sources_file"; then
echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2
exit 1
fi
while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do
[ -n "$source_id" ] || continue
if [ -f "$source_list" ]; then
while IFS= read -r name; do
[ -n "$name" ] || continue
rm -rf "skills/$name"
done < "$source_list"
fi
done < "$sources_file"
declare -A owners=()
while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do
[ -n "$source_id" ] || continue
git archive --format=tar "origin/${THIRDPARTY_BRANCH}" "$snapshot_dir" | tar -xf - -C "$tmp_dir"
snapshot_root="$tmp_dir/$snapshot_dir"
names=()
case "$sync_mode" in
copy_skill_dirs)
source_skills_dir="$snapshot_root/$skills_subdir"
if [ ! -d "$source_skills_dir" ]; then
echo "ERROR: $skills_subdir not found in snapshot $snapshot_dir" >&2
exit 1
fi
for dir in "$source_skills_dir"/*; do
[ -d "$dir" ] || continue
name="$(basename "$dir")"
if ! skill_dir_included "$name" "$include_skill_dirs"; then
continue
fi
if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then
echo "ERROR: duplicate third-party skill name: $name" >&2
exit 1
fi
if tracked_skill_exists "$name"; then
echo "ERROR: skill name conflict with tracked skill: $name" >&2
exit 1
fi
rm -rf "skills/thirdparty/$name"
cp -R "$dir" "skills/thirdparty/$name"
names+=("$name")
owners["$name"]="$source_id"
done
;;
render_skill)
name="$output_name"
if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then
echo "ERROR: duplicate third-party skill name: $name" >&2
exit 1
fi
if tracked_skill_exists "$name"; then
echo "ERROR: skill name conflict with tracked skill: $name" >&2
exit 1
fi
render_skill "$snapshot_root" "skills/thirdparty/$name" "$platform_config" "$template_root" "$data_dir" "$scripts_dir"
names+=("$name")
owners["$name"]="$source_id"
;;
*)
echo "ERROR: unsupported sync mode: $sync_mode" >&2
exit 1
;;
esac
printf "%s\n" "${names[@]}" | sort > "$source_list"
done < "$sources_file"
git add skills
if git diff --cached --quiet; then
echo "No third-party skills to sync."
exit 0
fi
git commit -m ":package: deps(skills): sync thirdparty skills"
TOKEN="${WORKFLOW:-}"
if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
fi
git push origin "$TARGET_BRANCH"
.gitea/ci/update_thirdparty_skills.sh
-227
View File
@@ -1,227 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
REPO_DIR="${REPO_DIR:-$(pwd)}"
TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"
MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"
COMMIT_AUTHOR_NAME="${COMMIT_AUTHOR_NAME:-ci[bot]}"
COMMIT_AUTHOR_EMAIL="${COMMIT_AUTHOR_EMAIL:-ci-bot@local}"
retry_cmd() {
local retries="$1"
shift
local delay="$1"
shift
local attempt=1
while true; do
if "$@"; then
return 0
fi
if [ "$attempt" -ge "$retries" ]; then
return 1
fi
echo "Retry ($attempt/$retries): $*" >&2
sleep "$delay"
attempt=$((attempt + 1))
done
}
github_owner_repo() {
case "$1" in
https://github.com/*)
echo "$1" | sed -E 's#^https://github.com/([^/]+/[^/.]+)(\.git)?$#\1#'
;;
http://github.com/*)
echo "$1" | sed -E 's#^http://github.com/([^/]+/[^/.]+)(\.git)?$#\1#'
;;
git@github.com:*)
echo "$1" | sed -E 's#^git@github.com:([^/]+/[^/.]+)(\.git)?$#\1#'
;;
*)
return 1
;;
esac
}
resolve_latest_sha() {
local repo="$1"
local ref="$2"
local tmp_json="$3"
local gh_repo="$4"
local sha=""
if [ -n "$gh_repo" ]; then
local api_url="https://api.github.com/repos/${gh_repo}/commits/${ref}"
if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$api_url" -o "$tmp_json"; then
sha="$(sed -n 's/^[[:space:]]*"sha":[[:space:]]*"\([0-9a-f]\{40\}\)".*/\1/p' "$tmp_json" | head -n 1)"
if [ -n "$sha" ]; then
echo "$sha"
return 0
fi
fi
fi
sha="$(retry_cmd 3 2 git -c http.version=HTTP/1.1 ls-remote "$repo" "refs/heads/$ref" | awk 'NR==1 {print $1}')"
if [ -n "$sha" ]; then
echo "$sha"
return 0
fi
return 1
}
emit_sources_tsv() {
python3 - "$MANIFEST_PATH" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as fh:
data = json.load(fh)
for entry in data["sources"]:
print(
"\x1f".join(
[
entry["id"],
entry["upstream_repo"],
entry.get("upstream_ref", "main"),
entry["snapshot_dir"],
entry["sync_mode"],
"\x1e".join(entry.get("remove_paths", [])),
]
)
)
PY
}
read_source_metadata_value() {
local key="$1"
local source_file="$2"
if [ ! -f "$source_file" ]; then
return 0
fi
sed -n "s/^- ${key}:[[:space:]]*//p" "$source_file" | head -n 1
}
remove_snapshot_paths() {
local snapshot_dir="$1"
local remove_paths="$2"
[ -n "$remove_paths" ] || return 0
local IFS=$'\x1e'
read -r -a paths <<< "$remove_paths"
for path in "${paths[@]}"; do
[ -n "$path" ] || continue
rm -rf "$snapshot_dir/$path"
done
}
cd "$REPO_DIR"
git config user.name "$COMMIT_AUTHOR_NAME"
git config user.email "$COMMIT_AUTHOR_EMAIL"
tmp_dir="$(mktemp -d)"
cleanup() {
rm -rf "$tmp_dir"
}
trap cleanup EXIT
if [ ! -f "$MANIFEST_PATH" ]; then
echo "ERROR: third-party manifest not found: $MANIFEST_PATH" >&2
exit 1
fi
manifest_copy="$tmp_dir/thirdparty_skills.json"
cp "$MANIFEST_PATH" "$manifest_copy"
MANIFEST_PATH="$manifest_copy"
git fetch origin "$TARGET_BRANCH"
git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"
sources_file="$tmp_dir/sources.tsv"
if ! emit_sources_tsv > "$sources_file"; then
echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2
exit 1
fi
changed=0
while IFS=$'\x1f' read -r source_id upstream_repo upstream_ref snapshot_dir sync_mode remove_paths; do
[ -n "$source_id" ] || continue
remove_paths_md="${remove_paths//$'\x1e'/,}"
gh_repo=""
if gh_repo="$(github_owner_repo "$upstream_repo" 2>/dev/null)"; then
:
fi
latest_sha="$(resolve_latest_sha "$upstream_repo" "$upstream_ref" "$tmp_dir/${source_id}-latest.json" "$gh_repo" || true)"
if [ -z "$latest_sha" ]; then
echo "ERROR: failed to resolve upstream ref for ${source_id}: $upstream_repo $upstream_ref" >&2
exit 1
fi
current_sha="$(read_source_metadata_value "Ref" "$snapshot_dir/SOURCE.md")"
current_remove_paths="$(read_source_metadata_value "Remove-Paths" "$snapshot_dir/SOURCE.md")"
if [ "$latest_sha" = "$current_sha" ] && [ "$remove_paths_md" = "$current_remove_paths" ]; then
echo "Third-party snapshot is up to date for ${source_id}: $latest_sha"
continue
fi
rm -rf "$snapshot_dir"
mkdir -p "$snapshot_dir"
snapshot_loaded=0
if [ -n "$gh_repo" ]; then
tar_url="https://codeload.github.com/${gh_repo}/tar.gz/${latest_sha}"
if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$tar_url" -o "$tmp_dir/${source_id}.tar.gz"; then
tar -xzf "$tmp_dir/${source_id}.tar.gz" -C "$snapshot_dir" --strip-components=1
snapshot_loaded=1
fi
fi
if [ "$snapshot_loaded" -eq 0 ]; then
upstream_dir="$tmp_dir/${source_id}-upstream"
git init "$upstream_dir" >/dev/null
git -C "$upstream_dir" remote add origin "$upstream_repo"
retry_cmd 3 2 git -C "$upstream_dir" fetch --depth 1 origin "$latest_sha"
git -C "$upstream_dir" checkout --detach FETCH_HEAD
git -C "$upstream_dir" archive --format=tar HEAD | tar -xf - -C "$snapshot_dir"
fi
remove_snapshot_paths "$snapshot_dir" "$remove_paths"
snapshot_date="$(date -u +%Y-%m-%d)"
cat > "$snapshot_dir/SOURCE.md" <<EOF
# Source
- Repo: ${upstream_repo%".git"}
- Ref: $latest_sha
- Remove-Paths: $remove_paths_md
- Snapshot: $snapshot_date
- Sync-Mode: $sync_mode
- Notes: vendored into playbook branch $TARGET_BRANCH
EOF
git add -A "$snapshot_dir"
changed=1
done < "$sources_file"
if [ "$changed" -eq 0 ] || git diff --cached --quiet; then
echo "All third-party snapshots are up to date."
exit 0
fi
git commit -m ":package: deps(thirdparty): update snapshots"
TOKEN="${WORKFLOW:-}"
if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
fi
git push origin "$TARGET_BRANCH"
.gitea/workflows/sync-tsl-playbook.yml
+4 -4
View File
@@ -13,6 +13,8 @@ concurrency:
env:
WORKSPACE_DIR: "/home/workspace"
TARGET_BRANCH: "tsl-playbook"
GIT_USER_NAME: "ci[bot]"
GIT_USER_EMAIL: "ci[bot]@tinysoft.com.cn"
jobs:
sync:
@@ -73,8 +75,6 @@ jobs:
REPO_DIR="${REPO_DIR:-$(pwd)}"
TARGET_BRANCH="${TARGET_BRANCH:-tsl-playbook}"
BUILD_SCRIPT="${BUILD_SCRIPT:-scripts/build_tsl_playbook.py}"
COMMIT_AUTHOR_NAME="${COMMIT_AUTHOR_NAME:-ci[bot]}"
COMMIT_AUTHOR_EMAIL="${COMMIT_AUTHOR_EMAIL:-ci-bot@local}"
cd "$REPO_DIR"
@@ -82,8 +82,8 @@ jobs:
echo "🔨 Build bundle and sync $TARGET_BRANCH"
echo "========================================"
git config user.name "$COMMIT_AUTHOR_NAME"
git config user.email "$COMMIT_AUTHOR_EMAIL"
git config user.name "$GIT_USER_NAME"
git config user.email "$GIT_USER_EMAIL"
source_sha="$(git rev-parse HEAD)"
source_short="$(git rev-parse --short HEAD)"
.gitea/workflows/update-thirdparty-skills.yml
+469 -2
View File
@@ -16,6 +16,8 @@ env:
WORKSPACE_DIR: "/home/workspace"
THIRDPARTY_BRANCH: "thirdparty/skill"
MANIFEST_PATH: ".gitea/ci/thirdparty_skills.json"
GIT_USER_NAME: "ci[bot]"
GIT_USER_EMAIL: "ci[bot]@tinysoft.com.cn"
jobs:
update_and_sync:
@@ -70,7 +72,234 @@ jobs:
fi
export MANIFEST_PATH="$MANIFEST_PATH"
TARGET_BRANCH="$THIRDPARTY_BRANCH" bash .gitea/ci/update_thirdparty_skills.sh
# BEGIN update_thirdparty_snapshots
TARGET_BRANCH="$THIRDPARTY_BRANCH" bash <<'UPDATE_THIRDPARTY'
set -euo pipefail
REPO_DIR="${REPO_DIR:-$(pwd)}"
TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"
MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"
retry_cmd() {
local retries="$1"
shift
local delay="$1"
shift
local attempt=1
while true; do
if "$@"; then
return 0
fi
if [ "$attempt" -ge "$retries" ]; then
return 1
fi
echo "Retry ($attempt/$retries): $*" >&2
sleep "$delay"
attempt=$((attempt + 1))
done
}
github_owner_repo() {
case "$1" in
https://github.com/*)
echo "$1" | sed -E 's#^https://github.com/([^/]+/[^/.]+)(\.git)?$#\1#'
;;
http://github.com/*)
echo "$1" | sed -E 's#^http://github.com/([^/]+/[^/.]+)(\.git)?$#\1#'
;;
git@github.com:*)
echo "$1" | sed -E 's#^git@github.com:([^/]+/[^/.]+)(\.git)?$#\1#'
;;
*)
return 1
;;
esac
}
resolve_latest_sha() {
local repo="$1"
local ref="$2"
local tmp_json="$3"
local gh_repo="$4"
local sha=""
if [ -n "$gh_repo" ]; then
local api_url="https://api.github.com/repos/${gh_repo}/commits/${ref}"
if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$api_url" -o "$tmp_json"; then
sha="$(sed -n 's/^[[:space:]]*"sha":[[:space:]]*"\([0-9a-f]\{40\}\)".*/\1/p' "$tmp_json" | head -n 1)"
if [ -n "$sha" ]; then
echo "$sha"
return 0
fi
fi
fi
sha="$(retry_cmd 3 2 git -c http.version=HTTP/1.1 ls-remote "$repo" "refs/heads/$ref" | awk 'NR==1 {print $1}')"
if [ -n "$sha" ]; then
echo "$sha"
return 0
fi
return 1
}
emit_sources_tsv() {
python3 - "$MANIFEST_PATH" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as fh:
data = json.load(fh)
for entry in data["sources"]:
print(
"\x1f".join(
[
entry["id"],
entry["upstream_repo"],
entry.get("upstream_ref", "main"),
entry["snapshot_dir"],
entry["sync_mode"],
"\x1e".join(entry.get("remove_paths", [])),
]
)
)
PY
}
read_source_metadata_value() {
local key="$1"
local source_file="$2"
if [ ! -f "$source_file" ]; then
return 0
fi
sed -n "s/^- ${key}:[[:space:]]*//p" "$source_file" | head -n 1
}
remove_snapshot_paths() {
local snapshot_dir="$1"
local remove_paths="$2"
[ -n "$remove_paths" ] || return 0
local IFS=$'\x1e'
read -r -a paths <<< "$remove_paths"
for path in "${paths[@]}"; do
[ -n "$path" ] || continue
rm -rf "$snapshot_dir/$path"
done
}
cd "$REPO_DIR"
git config user.name "$GIT_USER_NAME"
git config user.email "$GIT_USER_EMAIL"
tmp_dir="$(mktemp -d)"
cleanup() {
rm -rf "$tmp_dir"
}
trap cleanup EXIT
if [ ! -f "$MANIFEST_PATH" ]; then
echo "ERROR: third-party manifest not found: $MANIFEST_PATH" >&2
exit 1
fi
manifest_copy="$tmp_dir/thirdparty_skills.json"
cp "$MANIFEST_PATH" "$manifest_copy"
MANIFEST_PATH="$manifest_copy"
git fetch origin "$TARGET_BRANCH"
git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"
sources_file="$tmp_dir/sources.tsv"
if ! emit_sources_tsv > "$sources_file"; then
echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2
exit 1
fi
changed=0
while IFS=$'\x1f' read -r source_id upstream_repo upstream_ref snapshot_dir sync_mode remove_paths; do
[ -n "$source_id" ] || continue
remove_paths_md="${remove_paths//$'\x1e'/,}"
gh_repo=""
if gh_repo="$(github_owner_repo "$upstream_repo" 2>/dev/null)"; then
:
fi
latest_sha="$(resolve_latest_sha "$upstream_repo" "$upstream_ref" "$tmp_dir/${source_id}-latest.json" "$gh_repo" || true)"
if [ -z "$latest_sha" ]; then
echo "ERROR: failed to resolve upstream ref for ${source_id}: $upstream_repo $upstream_ref" >&2
exit 1
fi
current_sha="$(read_source_metadata_value "Ref" "$snapshot_dir/SOURCE.md")"
current_remove_paths="$(read_source_metadata_value "Remove-Paths" "$snapshot_dir/SOURCE.md")"
if [ "$latest_sha" = "$current_sha" ] && [ "$remove_paths_md" = "$current_remove_paths" ]; then
echo "Third-party snapshot is up to date for ${source_id}: $latest_sha"
continue
fi
rm -rf "$snapshot_dir"
mkdir -p "$snapshot_dir"
snapshot_loaded=0
if [ -n "$gh_repo" ]; then
tar_url="https://codeload.github.com/${gh_repo}/tar.gz/${latest_sha}"
if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$tar_url" -o "$tmp_dir/${source_id}.tar.gz"; then
tar -xzf "$tmp_dir/${source_id}.tar.gz" -C "$snapshot_dir" --strip-components=1
snapshot_loaded=1
fi
fi
if [ "$snapshot_loaded" -eq 0 ]; then
upstream_dir="$tmp_dir/${source_id}-upstream"
git init "$upstream_dir" >/dev/null
git -C "$upstream_dir" remote add origin "$upstream_repo"
retry_cmd 3 2 git -C "$upstream_dir" fetch --depth 1 origin "$latest_sha"
git -C "$upstream_dir" checkout --detach FETCH_HEAD
git -C "$upstream_dir" archive --format=tar HEAD | tar -xf - -C "$snapshot_dir"
fi
remove_snapshot_paths "$snapshot_dir" "$remove_paths"
snapshot_date="$(date -u +%Y-%m-%d)"
cat > "$snapshot_dir/SOURCE.md" <<EOF
# Source
- Repo: ${upstream_repo%".git"}
- Ref: $latest_sha
- Remove-Paths: $remove_paths_md
- Snapshot: $snapshot_date
- Sync-Mode: $sync_mode
- Notes: vendored into playbook branch $TARGET_BRANCH
EOF
git add -A "$snapshot_dir"
changed=1
done < "$sources_file"
if [ "$changed" -eq 0 ] || git diff --cached --quiet; then
echo "All third-party snapshots are up to date."
exit 0
fi
git commit -m ":package: deps(thirdparty): update snapshots"
TOKEN="${WORKFLOW:-}"
if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
fi
git push origin "$TARGET_BRANCH"
UPDATE_THIRDPARTY
# END update_thirdparty_snapshots
git fetch origin "$THIRDPARTY_BRANCH"
after_ref="$(git rev-parse "origin/$THIRDPARTY_BRANCH")"
@@ -88,10 +317,248 @@ jobs:
git fetch origin main
git checkout -B main origin/main
# BEGIN sync_thirdparty_skills
TARGET_BRANCH="main" \
THIRDPARTY_BRANCH="$THIRDPARTY_BRANCH" \
MANIFEST_PATH="$MANIFEST_PATH" \
bash .gitea/ci/sync_thirdparty_skills.sh
bash <<'SYNC_THIRDPARTY'
set -euo pipefail
REPO_DIR="${REPO_DIR:-$(pwd)}"
THIRDPARTY_BRANCH="${THIRDPARTY_BRANCH:-thirdparty/skill}"
TARGET_BRANCH="${TARGET_BRANCH:-main}"
MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"
emit_sources_tsv() {
python3 - "$MANIFEST_PATH" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as fh:
data = json.load(fh)
for entry in data["sources"]:
print(
"\x1f".join(
[
entry["id"],
entry["snapshot_dir"],
entry["sync_mode"],
entry["source_list"],
entry.get("skills_subdir", ""),
entry.get("output_name", entry["id"]),
entry.get("platform_config", ""),
entry.get("template_root", ""),
entry.get("data_dir", ""),
entry.get("scripts_dir", ""),
"\x1e".join(entry.get("include_skill_dirs", [])),
]
)
)
PY
}
render_skill() {
local snapshot_root="$1"
local output_dir="$2"
local platform_config_rel="$3"
local template_root_rel="$4"
local data_dir_rel="$5"
local scripts_dir_rel="$6"
python3 - "$snapshot_root" "$output_dir" "$platform_config_rel" "$template_root_rel" "$data_dir_rel" "$scripts_dir_rel" <<'PY'
import json
import pathlib
import shutil
import sys
snapshot_root = pathlib.Path(sys.argv[1])
output_dir = pathlib.Path(sys.argv[2])
platform_config_path = snapshot_root / sys.argv[3]
template_root = snapshot_root / sys.argv[4]
data_dir = snapshot_root / sys.argv[5]
scripts_dir = snapshot_root / sys.argv[6]
config = json.loads(platform_config_path.read_text(encoding="utf-8"))
skill_template = (template_root / "base" / "skill-content.md").read_text(encoding="utf-8")
quick_reference = ""
if config.get("sections", {}).get("quickReference"):
quick_reference = "\n" + (template_root / "base" / "quick-reference.md").read_text(encoding="utf-8")
def render_frontmatter(frontmatter):
if not frontmatter:
return ""
lines = ["---"]
for key, value in frontmatter.items():
if any(ch in value for ch in ':"\n'):
value = value.replace('"', '\\"')
lines.append(f'{key}: "{value}"')
else:
lines.append(f"{key}: {value}")
lines.extend(["---", ""])
return "\n".join(lines)
content = skill_template
content = content.replace("{{TITLE}}", config["title"])
content = content.replace("{{DESCRIPTION}}", config["description"])
content = content.replace("{{SCRIPT_PATH}}", config["scriptPath"])
content = content.replace("{{SKILL_OR_WORKFLOW}}", config["skillOrWorkflow"])
content = content.replace("{{QUICK_REFERENCE}}", quick_reference)
if output_dir.exists():
shutil.rmtree(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
(output_dir / "SKILL.md").write_text(
render_frontmatter(config.get("frontmatter")) + content,
encoding="utf-8",
)
if data_dir.exists():
shutil.copytree(data_dir, output_dir / "data")
if scripts_dir.exists():
shutil.copytree(scripts_dir, output_dir / "scripts")
PY
}
tracked_skill_exists() {
local name="$1"
# conflict only if a first-party SKILL.md exists directly under skills/<name>/
[ -f "skills/$name/SKILL.md" ]
}
skill_dir_included() {
local name="$1"
local include_skill_dirs="$2"
[ -n "$include_skill_dirs" ] || return 0
local IFS=$'\x1e'
local expected
read -r -a included <<< "$include_skill_dirs"
for expected in "${included[@]}"; do
if [ "$name" = "$expected" ]; then
return 0
fi
done
return 1
}
cd "$REPO_DIR"
git config user.name "$GIT_USER_NAME"
git config user.email "$GIT_USER_EMAIL"
git fetch origin "$THIRDPARTY_BRANCH"
git fetch origin "$TARGET_BRANCH"
tmp_dir="$(mktemp -d)"
cleanup() {
rm -rf "$tmp_dir"
}
trap cleanup EXIT
git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"
mkdir -p "skills/thirdparty/.sources"
sources_file="$tmp_dir/sources.tsv"
if ! emit_sources_tsv > "$sources_file"; then
echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2
exit 1
fi
while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do
[ -n "$source_id" ] || continue
if [ -f "$source_list" ]; then
while IFS= read -r name; do
[ -n "$name" ] || continue
rm -rf "skills/$name"
done < "$source_list"
fi
done < "$sources_file"
declare -A owners=()
while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do
[ -n "$source_id" ] || continue
git archive --format=tar "origin/${THIRDPARTY_BRANCH}" "$snapshot_dir" | tar -xf - -C "$tmp_dir"
snapshot_root="$tmp_dir/$snapshot_dir"
names=()
case "$sync_mode" in
copy_skill_dirs)
source_skills_dir="$snapshot_root/$skills_subdir"
if [ ! -d "$source_skills_dir" ]; then
echo "ERROR: $skills_subdir not found in snapshot $snapshot_dir" >&2
exit 1
fi
for dir in "$source_skills_dir"/*; do
[ -d "$dir" ] || continue
name="$(basename "$dir")"
if ! skill_dir_included "$name" "$include_skill_dirs"; then
continue
fi
if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then
echo "ERROR: duplicate third-party skill name: $name" >&2
exit 1
fi
if tracked_skill_exists "$name"; then
echo "ERROR: skill name conflict with tracked skill: $name" >&2
exit 1
fi
rm -rf "skills/thirdparty/$name"
cp -R "$dir" "skills/thirdparty/$name"
names+=("$name")
owners["$name"]="$source_id"
done
;;
render_skill)
name="$output_name"
if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then
echo "ERROR: duplicate third-party skill name: $name" >&2
exit 1
fi
if tracked_skill_exists "$name"; then
echo "ERROR: skill name conflict with tracked skill: $name" >&2
exit 1
fi
render_skill "$snapshot_root" "skills/thirdparty/$name" "$platform_config" "$template_root" "$data_dir" "$scripts_dir"
names+=("$name")
owners["$name"]="$source_id"
;;
*)
echo "ERROR: unsupported sync mode: $sync_mode" >&2
exit 1
;;
esac
printf "%s\n" "${names[@]}" | sort > "$source_list"
done < "$sources_file"
git add skills
if git diff --cached --quiet; then
echo "No third-party skills to sync."
exit 0
fi
git commit -m ":package: deps(skills): sync thirdparty skills"
TOKEN="${WORKFLOW:-}"
if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
fi
git push origin "$TARGET_BRANCH"
SYNC_THIRDPARTY
# END sync_thirdparty_skills
echo "✅ Update and sync finished."
test/test_thirdparty_skills_pipeline.py
+71 -23
View File
@@ -3,6 +3,7 @@ import os
import shutil
import subprocess
import tempfile
import textwrap
import unittest
from pathlib import Path
@@ -10,6 +11,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MANIFEST = ROOT / ".gitea" / "ci" / "thirdparty_skills.json"
WORKFLOW = ROOT / ".gitea" / "workflows" / "update-thirdparty-skills.yml"
TSL_SYNC_WORKFLOW = ROOT / ".gitea" / "workflows" / "sync-tsl-playbook.yml"
LEGACY_WORKFLOW = ROOT / ".gitea" / "workflows" / "update-thirdparty-superpowers.yml"
UPDATE_SCRIPT = ROOT / ".gitea" / "ci" / "update_thirdparty_skills.sh"
SYNC_SCRIPT = ROOT / ".gitea" / "ci" / "sync_thirdparty_skills.sh"
@@ -44,6 +46,16 @@ def run_command(*args: str, cwd: Path | None = None) -> subprocess.CompletedProc
)
def extract_workflow_region(name: str) -> str:
text = WORKFLOW.read_text(encoding="utf-8")
begin = f"# BEGIN {name}"
end = f"# END {name}"
if begin not in text or end not in text:
raise AssertionError(f"workflow region markers not found: {name}")
body = text.split(begin, 1)[1].split(end, 1)[0]
return textwrap.dedent(body).strip() + "\n"
class ThirdpartySkillsPipelineTests(unittest.TestCase):
def test_manifest_declares_all_thirdparty_sources(self):
data = load_manifest()
@@ -137,14 +149,18 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
superpowers = next(item for item in data["sources"] if item["id"] == "superpowers")
self.assertEqual(superpowers["remove_paths"], ["skills/ui-ux-pro-max"])
def test_workflow_uses_generic_scripts_and_single_serial_job(self):
def test_workflow_inlines_update_and_sync_in_single_serial_job(self):
text = WORKFLOW.read_text(encoding="utf-8")
self.assertFalse(LEGACY_WORKFLOW.exists())
self.assertFalse(UPDATE_SCRIPT.exists())
self.assertFalse(SYNC_SCRIPT.exists())
self.assertIn("update_and_sync:", text)
self.assertNotIn("\n update:\n", text)
self.assertNotIn("\n sync:\n", text)
self.assertIn("bash .gitea/ci/update_thirdparty_skills.sh", text)
self.assertIn("bash .gitea/ci/sync_thirdparty_skills.sh", text)
self.assertIn("# BEGIN update_thirdparty_snapshots", text)
self.assertIn("# BEGIN sync_thirdparty_skills", text)
self.assertNotIn("update_thirdparty_skills.sh", text)
self.assertNotIn("sync_thirdparty_skills.sh", text)
self.assertNotIn("git merge", text)
self.assertNotIn("git pull", text)
@@ -153,19 +169,30 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
self.assertIn("concurrency:", text)
self.assertIn("update-thirdparty-${{ github.repository }}", text)
self.assertIn('MANIFEST_PATH: ".gitea/ci/thirdparty_skills.json"', text)
self.assertIn('TARGET_BRANCH="$THIRDPARTY_BRANCH" bash .gitea/ci/update_thirdparty_skills.sh', text)
self.assertIn('TARGET_BRANCH="main" \\', text)
self.assertIn('MANIFEST_PATH="$MANIFEST_PATH" \\', text)
self.assertIn("update_thirdparty_snapshots", text)
self.assertIn("sync_thirdparty_skills", text)
def test_generic_scripts_exist_and_use_manifest(self):
update_text = UPDATE_SCRIPT.read_text(encoding="utf-8")
sync_text = SYNC_SCRIPT.read_text(encoding="utf-8")
self.assertIn('MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"', update_text)
self.assertIn('MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"', sync_text)
self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"', update_text)
self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-main}"', sync_text)
self.assertIn(':package: deps(thirdparty): update snapshots', update_text)
self.assertIn(':package: deps(skills): sync thirdparty skills', sync_text)
def test_inline_workflow_exposes_manifest_and_publish_contract(self):
text = WORKFLOW.read_text(encoding="utf-8")
self.assertIn('MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"', text)
self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"', text)
self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-main}"', text)
self.assertIn(':package: deps(thirdparty): update snapshots', text)
self.assertIn(':package: deps(skills): sync thirdparty skills', text)
self.assertIn('git push origin "$TARGET_BRANCH"', text)
def test_ci_committers_share_explicit_git_identity(self):
workflow_text = TSL_SYNC_WORKFLOW.read_text(encoding="utf-8")
thirdparty_text = WORKFLOW.read_text(encoding="utf-8")
for text in (workflow_text, thirdparty_text):
self.assertIn('GIT_USER_NAME: "ci[bot]"', text)
self.assertIn('GIT_USER_EMAIL: "ci[bot]@tinysoft.com.cn"', text)
self.assertIn('git config user.name "$GIT_USER_NAME"', text)
self.assertIn('git config user.email "$GIT_USER_EMAIL"', text)
self.assertNotIn("COMMIT_AUTHOR_NAME", text)
self.assertNotIn("COMMIT_AUTHOR_EMAIL", text)
self.assertNotIn("@local", text)
def test_skills_doc_points_to_generic_thirdparty_sources(self):
text = SKILLS_MD.read_text(encoding="utf-8")
@@ -193,8 +220,8 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
self.assertTrue((UI_UX_PRO_MAX_DIR / "data").is_dir())
self.assertTrue((UI_UX_PRO_MAX_DIR / "scripts").is_dir())
def test_update_script_materializes_manifest_before_target_checkout(self):
text = UPDATE_SCRIPT.read_text(encoding="utf-8")
def test_inline_update_materializes_manifest_before_target_checkout(self):
text = WORKFLOW.read_text(encoding="utf-8")
self.assertIn('manifest_copy="$tmp_dir/thirdparty_skills.json"', text)
self.assertIn('cp "$MANIFEST_PATH" "$manifest_copy"', text)
self.assertIn('MANIFEST_PATH="$manifest_copy"', text)
@@ -208,8 +235,8 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
text.index('git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"'),
)
def test_sync_script_assumes_thirdparty_snapshot_is_already_clean(self):
text = SYNC_SCRIPT.read_text(encoding="utf-8")
def test_inline_sync_assumes_thirdparty_snapshot_is_already_clean(self):
text = WORKFLOW.read_text(encoding="utf-8")
self.assertIn('"\\x1f".join(', text)
self.assertIn("while IFS=$'\\x1f' read -r", text)
self.assertIn("include_skill_dirs", text)
@@ -218,7 +245,7 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
self.assertNotIn("exclude_skill_dirs", text)
self.assertNotIn("is_excluded_skill_dir", text)
def test_sync_script_generates_karpathy_outputs_in_temp_repo(self):
def test_inline_sync_generates_karpathy_outputs_in_temp_repo(self):
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_root = Path(tmp_dir)
mirror = tmp_root / "origin.git"
@@ -265,9 +292,30 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
json.dumps(manifest_data, indent=2) + "\n",
encoding="utf-8",
)
shutil.copy2(SYNC_SCRIPT, work / ".gitea" / "ci" / "sync_thirdparty_skills.sh")
sync_result = run_command("bash", ".gitea/ci/sync_thirdparty_skills.sh", cwd=work)
sync_script = extract_workflow_region("sync_thirdparty_skills")
script_path = work / ".sync-thirdparty-test.sh"
script_path.write_text(
'export GIT_USER_NAME="test"\n'
'export GIT_USER_EMAIL="test@example.invalid"\n'
+ sync_script,
encoding="utf-8",
newline="\n",
)
env = os.environ.copy()
env.update(
{
"REPO_DIR": str(work),
"THIRDPARTY_BRANCH": "thirdparty/skill",
"MANIFEST_PATH": ".gitea/ci/thirdparty_skills.json",
}
)
sync_result = subprocess.run(
["bash", script_path.name],
cwd=work,
env=env,
capture_output=True,
text=True,
)
self.assertEqual(
sync_result.returncode,
0,