🔧 chore(ci): inline skill sync and standardize git identity
This commit is contained in:
@@ -1,238 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
REPO_DIR="${REPO_DIR:-$(pwd)}"
|
|
||||||
THIRDPARTY_BRANCH="${THIRDPARTY_BRANCH:-thirdparty/skill}"
|
|
||||||
TARGET_BRANCH="${TARGET_BRANCH:-main}"
|
|
||||||
MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"
|
|
||||||
COMMIT_AUTHOR_NAME="${COMMIT_AUTHOR_NAME:-ci[bot]}"
|
|
||||||
COMMIT_AUTHOR_EMAIL="${COMMIT_AUTHOR_EMAIL:-ci-bot@local}"
|
|
||||||
|
|
||||||
emit_sources_tsv() {
|
|
||||||
python3 - "$MANIFEST_PATH" <<'PY'
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
|
|
||||||
with open(sys.argv[1], encoding="utf-8") as fh:
|
|
||||||
data = json.load(fh)
|
|
||||||
|
|
||||||
for entry in data["sources"]:
|
|
||||||
print(
|
|
||||||
"\x1f".join(
|
|
||||||
[
|
|
||||||
entry["id"],
|
|
||||||
entry["snapshot_dir"],
|
|
||||||
entry["sync_mode"],
|
|
||||||
entry["source_list"],
|
|
||||||
entry.get("skills_subdir", ""),
|
|
||||||
entry.get("output_name", entry["id"]),
|
|
||||||
entry.get("platform_config", ""),
|
|
||||||
entry.get("template_root", ""),
|
|
||||||
entry.get("data_dir", ""),
|
|
||||||
entry.get("scripts_dir", ""),
|
|
||||||
"\x1e".join(entry.get("include_skill_dirs", [])),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
)
|
|
||||||
PY
|
|
||||||
}
|
|
||||||
|
|
||||||
render_skill() {
|
|
||||||
local snapshot_root="$1"
|
|
||||||
local output_dir="$2"
|
|
||||||
local platform_config_rel="$3"
|
|
||||||
local template_root_rel="$4"
|
|
||||||
local data_dir_rel="$5"
|
|
||||||
local scripts_dir_rel="$6"
|
|
||||||
|
|
||||||
python3 - "$snapshot_root" "$output_dir" "$platform_config_rel" "$template_root_rel" "$data_dir_rel" "$scripts_dir_rel" <<'PY'
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import shutil
|
|
||||||
import sys
|
|
||||||
|
|
||||||
snapshot_root = pathlib.Path(sys.argv[1])
|
|
||||||
output_dir = pathlib.Path(sys.argv[2])
|
|
||||||
platform_config_path = snapshot_root / sys.argv[3]
|
|
||||||
template_root = snapshot_root / sys.argv[4]
|
|
||||||
data_dir = snapshot_root / sys.argv[5]
|
|
||||||
scripts_dir = snapshot_root / sys.argv[6]
|
|
||||||
|
|
||||||
config = json.loads(platform_config_path.read_text(encoding="utf-8"))
|
|
||||||
skill_template = (template_root / "base" / "skill-content.md").read_text(encoding="utf-8")
|
|
||||||
quick_reference = ""
|
|
||||||
if config.get("sections", {}).get("quickReference"):
|
|
||||||
quick_reference = "\n" + (template_root / "base" / "quick-reference.md").read_text(encoding="utf-8")
|
|
||||||
|
|
||||||
def render_frontmatter(frontmatter):
|
|
||||||
if not frontmatter:
|
|
||||||
return ""
|
|
||||||
|
|
||||||
lines = ["---"]
|
|
||||||
for key, value in frontmatter.items():
|
|
||||||
if any(ch in value for ch in ':"\n'):
|
|
||||||
value = value.replace('"', '\\"')
|
|
||||||
lines.append(f'{key}: "{value}"')
|
|
||||||
else:
|
|
||||||
lines.append(f"{key}: {value}")
|
|
||||||
lines.extend(["---", ""])
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
content = skill_template
|
|
||||||
content = content.replace("{{TITLE}}", config["title"])
|
|
||||||
content = content.replace("{{DESCRIPTION}}", config["description"])
|
|
||||||
content = content.replace("{{SCRIPT_PATH}}", config["scriptPath"])
|
|
||||||
content = content.replace("{{SKILL_OR_WORKFLOW}}", config["skillOrWorkflow"])
|
|
||||||
content = content.replace("{{QUICK_REFERENCE}}", quick_reference)
|
|
||||||
|
|
||||||
if output_dir.exists():
|
|
||||||
shutil.rmtree(output_dir)
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
(output_dir / "SKILL.md").write_text(
|
|
||||||
render_frontmatter(config.get("frontmatter")) + content,
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
if data_dir.exists():
|
|
||||||
shutil.copytree(data_dir, output_dir / "data")
|
|
||||||
if scripts_dir.exists():
|
|
||||||
shutil.copytree(scripts_dir, output_dir / "scripts")
|
|
||||||
PY
|
|
||||||
}
|
|
||||||
|
|
||||||
tracked_skill_exists() {
|
|
||||||
local name="$1"
|
|
||||||
# conflict only if a first-party SKILL.md exists directly under skills/<name>/
|
|
||||||
[ -f "skills/$name/SKILL.md" ]
|
|
||||||
}
|
|
||||||
|
|
||||||
skill_dir_included() {
|
|
||||||
local name="$1"
|
|
||||||
local include_skill_dirs="$2"
|
|
||||||
|
|
||||||
[ -n "$include_skill_dirs" ] || return 0
|
|
||||||
|
|
||||||
local IFS=$'\x1e'
|
|
||||||
local expected
|
|
||||||
read -r -a included <<< "$include_skill_dirs"
|
|
||||||
for expected in "${included[@]}"; do
|
|
||||||
if [ "$name" = "$expected" ]; then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
cd "$REPO_DIR"
|
|
||||||
|
|
||||||
git config user.name "$COMMIT_AUTHOR_NAME"
|
|
||||||
git config user.email "$COMMIT_AUTHOR_EMAIL"
|
|
||||||
|
|
||||||
git fetch origin "$THIRDPARTY_BRANCH"
|
|
||||||
git fetch origin "$TARGET_BRANCH"
|
|
||||||
|
|
||||||
tmp_dir="$(mktemp -d)"
|
|
||||||
cleanup() {
|
|
||||||
rm -rf "$tmp_dir"
|
|
||||||
}
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"
|
|
||||||
|
|
||||||
mkdir -p "skills/thirdparty/.sources"
|
|
||||||
|
|
||||||
sources_file="$tmp_dir/sources.tsv"
|
|
||||||
if ! emit_sources_tsv > "$sources_file"; then
|
|
||||||
echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do
|
|
||||||
[ -n "$source_id" ] || continue
|
|
||||||
|
|
||||||
if [ -f "$source_list" ]; then
|
|
||||||
while IFS= read -r name; do
|
|
||||||
[ -n "$name" ] || continue
|
|
||||||
rm -rf "skills/$name"
|
|
||||||
done < "$source_list"
|
|
||||||
fi
|
|
||||||
done < "$sources_file"
|
|
||||||
|
|
||||||
declare -A owners=()
|
|
||||||
while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do
|
|
||||||
[ -n "$source_id" ] || continue
|
|
||||||
|
|
||||||
git archive --format=tar "origin/${THIRDPARTY_BRANCH}" "$snapshot_dir" | tar -xf - -C "$tmp_dir"
|
|
||||||
snapshot_root="$tmp_dir/$snapshot_dir"
|
|
||||||
|
|
||||||
names=()
|
|
||||||
case "$sync_mode" in
|
|
||||||
copy_skill_dirs)
|
|
||||||
source_skills_dir="$snapshot_root/$skills_subdir"
|
|
||||||
if [ ! -d "$source_skills_dir" ]; then
|
|
||||||
echo "ERROR: $skills_subdir not found in snapshot $snapshot_dir" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
for dir in "$source_skills_dir"/*; do
|
|
||||||
[ -d "$dir" ] || continue
|
|
||||||
name="$(basename "$dir")"
|
|
||||||
if ! skill_dir_included "$name" "$include_skill_dirs"; then
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then
|
|
||||||
echo "ERROR: duplicate third-party skill name: $name" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if tracked_skill_exists "$name"; then
|
|
||||||
echo "ERROR: skill name conflict with tracked skill: $name" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
rm -rf "skills/thirdparty/$name"
|
|
||||||
cp -R "$dir" "skills/thirdparty/$name"
|
|
||||||
names+=("$name")
|
|
||||||
owners["$name"]="$source_id"
|
|
||||||
done
|
|
||||||
;;
|
|
||||||
render_skill)
|
|
||||||
name="$output_name"
|
|
||||||
if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then
|
|
||||||
echo "ERROR: duplicate third-party skill name: $name" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if tracked_skill_exists "$name"; then
|
|
||||||
echo "ERROR: skill name conflict with tracked skill: $name" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
render_skill "$snapshot_root" "skills/thirdparty/$name" "$platform_config" "$template_root" "$data_dir" "$scripts_dir"
|
|
||||||
names+=("$name")
|
|
||||||
owners["$name"]="$source_id"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "ERROR: unsupported sync mode: $sync_mode" >&2
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
printf "%s\n" "${names[@]}" | sort > "$source_list"
|
|
||||||
done < "$sources_file"
|
|
||||||
|
|
||||||
git add skills
|
|
||||||
|
|
||||||
if git diff --cached --quiet; then
|
|
||||||
echo "No third-party skills to sync."
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
git commit -m ":package: deps(skills): sync thirdparty skills"
|
|
||||||
|
|
||||||
TOKEN="${WORKFLOW:-}"
|
|
||||||
if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
|
|
||||||
git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
|
|
||||||
fi
|
|
||||||
|
|
||||||
git push origin "$TARGET_BRANCH"
|
|
||||||
@@ -1,227 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
REPO_DIR="${REPO_DIR:-$(pwd)}"
|
|
||||||
TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"
|
|
||||||
MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"
|
|
||||||
COMMIT_AUTHOR_NAME="${COMMIT_AUTHOR_NAME:-ci[bot]}"
|
|
||||||
COMMIT_AUTHOR_EMAIL="${COMMIT_AUTHOR_EMAIL:-ci-bot@local}"
|
|
||||||
|
|
||||||
retry_cmd() {
|
|
||||||
local retries="$1"
|
|
||||||
shift
|
|
||||||
local delay="$1"
|
|
||||||
shift
|
|
||||||
|
|
||||||
local attempt=1
|
|
||||||
while true; do
|
|
||||||
if "$@"; then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
if [ "$attempt" -ge "$retries" ]; then
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
echo "Retry ($attempt/$retries): $*" >&2
|
|
||||||
sleep "$delay"
|
|
||||||
attempt=$((attempt + 1))
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
github_owner_repo() {
|
|
||||||
case "$1" in
|
|
||||||
https://github.com/*)
|
|
||||||
echo "$1" | sed -E 's#^https://github.com/([^/]+/[^/.]+)(\.git)?$#\1#'
|
|
||||||
;;
|
|
||||||
http://github.com/*)
|
|
||||||
echo "$1" | sed -E 's#^http://github.com/([^/]+/[^/.]+)(\.git)?$#\1#'
|
|
||||||
;;
|
|
||||||
git@github.com:*)
|
|
||||||
echo "$1" | sed -E 's#^git@github.com:([^/]+/[^/.]+)(\.git)?$#\1#'
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
return 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
resolve_latest_sha() {
|
|
||||||
local repo="$1"
|
|
||||||
local ref="$2"
|
|
||||||
local tmp_json="$3"
|
|
||||||
local gh_repo="$4"
|
|
||||||
local sha=""
|
|
||||||
|
|
||||||
if [ -n "$gh_repo" ]; then
|
|
||||||
local api_url="https://api.github.com/repos/${gh_repo}/commits/${ref}"
|
|
||||||
if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$api_url" -o "$tmp_json"; then
|
|
||||||
sha="$(sed -n 's/^[[:space:]]*"sha":[[:space:]]*"\([0-9a-f]\{40\}\)".*/\1/p' "$tmp_json" | head -n 1)"
|
|
||||||
if [ -n "$sha" ]; then
|
|
||||||
echo "$sha"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
sha="$(retry_cmd 3 2 git -c http.version=HTTP/1.1 ls-remote "$repo" "refs/heads/$ref" | awk 'NR==1 {print $1}')"
|
|
||||||
if [ -n "$sha" ]; then
|
|
||||||
echo "$sha"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
emit_sources_tsv() {
|
|
||||||
python3 - "$MANIFEST_PATH" <<'PY'
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
|
|
||||||
with open(sys.argv[1], encoding="utf-8") as fh:
|
|
||||||
data = json.load(fh)
|
|
||||||
|
|
||||||
for entry in data["sources"]:
|
|
||||||
print(
|
|
||||||
"\x1f".join(
|
|
||||||
[
|
|
||||||
entry["id"],
|
|
||||||
entry["upstream_repo"],
|
|
||||||
entry.get("upstream_ref", "main"),
|
|
||||||
entry["snapshot_dir"],
|
|
||||||
entry["sync_mode"],
|
|
||||||
"\x1e".join(entry.get("remove_paths", [])),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
)
|
|
||||||
PY
|
|
||||||
}
|
|
||||||
|
|
||||||
read_source_metadata_value() {
|
|
||||||
local key="$1"
|
|
||||||
local source_file="$2"
|
|
||||||
|
|
||||||
if [ ! -f "$source_file" ]; then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
sed -n "s/^- ${key}:[[:space:]]*//p" "$source_file" | head -n 1
|
|
||||||
}
|
|
||||||
|
|
||||||
remove_snapshot_paths() {
|
|
||||||
local snapshot_dir="$1"
|
|
||||||
local remove_paths="$2"
|
|
||||||
|
|
||||||
[ -n "$remove_paths" ] || return 0
|
|
||||||
|
|
||||||
local IFS=$'\x1e'
|
|
||||||
read -r -a paths <<< "$remove_paths"
|
|
||||||
for path in "${paths[@]}"; do
|
|
||||||
[ -n "$path" ] || continue
|
|
||||||
rm -rf "$snapshot_dir/$path"
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
cd "$REPO_DIR"
|
|
||||||
|
|
||||||
git config user.name "$COMMIT_AUTHOR_NAME"
|
|
||||||
git config user.email "$COMMIT_AUTHOR_EMAIL"
|
|
||||||
|
|
||||||
tmp_dir="$(mktemp -d)"
|
|
||||||
cleanup() {
|
|
||||||
rm -rf "$tmp_dir"
|
|
||||||
}
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
if [ ! -f "$MANIFEST_PATH" ]; then
|
|
||||||
echo "ERROR: third-party manifest not found: $MANIFEST_PATH" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
manifest_copy="$tmp_dir/thirdparty_skills.json"
|
|
||||||
cp "$MANIFEST_PATH" "$manifest_copy"
|
|
||||||
MANIFEST_PATH="$manifest_copy"
|
|
||||||
|
|
||||||
git fetch origin "$TARGET_BRANCH"
|
|
||||||
git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"
|
|
||||||
|
|
||||||
sources_file="$tmp_dir/sources.tsv"
|
|
||||||
if ! emit_sources_tsv > "$sources_file"; then
|
|
||||||
echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
changed=0
|
|
||||||
while IFS=$'\x1f' read -r source_id upstream_repo upstream_ref snapshot_dir sync_mode remove_paths; do
|
|
||||||
[ -n "$source_id" ] || continue
|
|
||||||
remove_paths_md="${remove_paths//$'\x1e'/,}"
|
|
||||||
|
|
||||||
gh_repo=""
|
|
||||||
if gh_repo="$(github_owner_repo "$upstream_repo" 2>/dev/null)"; then
|
|
||||||
:
|
|
||||||
fi
|
|
||||||
|
|
||||||
latest_sha="$(resolve_latest_sha "$upstream_repo" "$upstream_ref" "$tmp_dir/${source_id}-latest.json" "$gh_repo" || true)"
|
|
||||||
if [ -z "$latest_sha" ]; then
|
|
||||||
echo "ERROR: failed to resolve upstream ref for ${source_id}: $upstream_repo $upstream_ref" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
current_sha="$(read_source_metadata_value "Ref" "$snapshot_dir/SOURCE.md")"
|
|
||||||
current_remove_paths="$(read_source_metadata_value "Remove-Paths" "$snapshot_dir/SOURCE.md")"
|
|
||||||
|
|
||||||
if [ "$latest_sha" = "$current_sha" ] && [ "$remove_paths_md" = "$current_remove_paths" ]; then
|
|
||||||
echo "Third-party snapshot is up to date for ${source_id}: $latest_sha"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
rm -rf "$snapshot_dir"
|
|
||||||
mkdir -p "$snapshot_dir"
|
|
||||||
|
|
||||||
snapshot_loaded=0
|
|
||||||
if [ -n "$gh_repo" ]; then
|
|
||||||
tar_url="https://codeload.github.com/${gh_repo}/tar.gz/${latest_sha}"
|
|
||||||
if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$tar_url" -o "$tmp_dir/${source_id}.tar.gz"; then
|
|
||||||
tar -xzf "$tmp_dir/${source_id}.tar.gz" -C "$snapshot_dir" --strip-components=1
|
|
||||||
snapshot_loaded=1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$snapshot_loaded" -eq 0 ]; then
|
|
||||||
upstream_dir="$tmp_dir/${source_id}-upstream"
|
|
||||||
git init "$upstream_dir" >/dev/null
|
|
||||||
git -C "$upstream_dir" remote add origin "$upstream_repo"
|
|
||||||
retry_cmd 3 2 git -C "$upstream_dir" fetch --depth 1 origin "$latest_sha"
|
|
||||||
git -C "$upstream_dir" checkout --detach FETCH_HEAD
|
|
||||||
git -C "$upstream_dir" archive --format=tar HEAD | tar -xf - -C "$snapshot_dir"
|
|
||||||
fi
|
|
||||||
|
|
||||||
remove_snapshot_paths "$snapshot_dir" "$remove_paths"
|
|
||||||
|
|
||||||
snapshot_date="$(date -u +%Y-%m-%d)"
|
|
||||||
cat > "$snapshot_dir/SOURCE.md" <<EOF
|
|
||||||
# Source
|
|
||||||
|
|
||||||
- Repo: ${upstream_repo%".git"}
|
|
||||||
- Ref: $latest_sha
|
|
||||||
- Remove-Paths: $remove_paths_md
|
|
||||||
- Snapshot: $snapshot_date
|
|
||||||
- Sync-Mode: $sync_mode
|
|
||||||
- Notes: vendored into playbook branch $TARGET_BRANCH
|
|
||||||
EOF
|
|
||||||
|
|
||||||
git add -A "$snapshot_dir"
|
|
||||||
changed=1
|
|
||||||
done < "$sources_file"
|
|
||||||
|
|
||||||
if [ "$changed" -eq 0 ] || git diff --cached --quiet; then
|
|
||||||
echo "All third-party snapshots are up to date."
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
git commit -m ":package: deps(thirdparty): update snapshots"
|
|
||||||
|
|
||||||
TOKEN="${WORKFLOW:-}"
|
|
||||||
if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
|
|
||||||
git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
|
|
||||||
fi
|
|
||||||
|
|
||||||
git push origin "$TARGET_BRANCH"
|
|
||||||
@@ -13,6 +13,8 @@ concurrency:
|
|||||||
env:
|
env:
|
||||||
WORKSPACE_DIR: "/home/workspace"
|
WORKSPACE_DIR: "/home/workspace"
|
||||||
TARGET_BRANCH: "tsl-playbook"
|
TARGET_BRANCH: "tsl-playbook"
|
||||||
|
GIT_USER_NAME: "ci[bot]"
|
||||||
|
GIT_USER_EMAIL: "ci[bot]@tinysoft.com.cn"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
sync:
|
sync:
|
||||||
@@ -73,8 +75,6 @@ jobs:
|
|||||||
REPO_DIR="${REPO_DIR:-$(pwd)}"
|
REPO_DIR="${REPO_DIR:-$(pwd)}"
|
||||||
TARGET_BRANCH="${TARGET_BRANCH:-tsl-playbook}"
|
TARGET_BRANCH="${TARGET_BRANCH:-tsl-playbook}"
|
||||||
BUILD_SCRIPT="${BUILD_SCRIPT:-scripts/build_tsl_playbook.py}"
|
BUILD_SCRIPT="${BUILD_SCRIPT:-scripts/build_tsl_playbook.py}"
|
||||||
COMMIT_AUTHOR_NAME="${COMMIT_AUTHOR_NAME:-ci[bot]}"
|
|
||||||
COMMIT_AUTHOR_EMAIL="${COMMIT_AUTHOR_EMAIL:-ci-bot@local}"
|
|
||||||
|
|
||||||
cd "$REPO_DIR"
|
cd "$REPO_DIR"
|
||||||
|
|
||||||
@@ -82,8 +82,8 @@ jobs:
|
|||||||
echo "🔨 Build bundle and sync $TARGET_BRANCH"
|
echo "🔨 Build bundle and sync $TARGET_BRANCH"
|
||||||
echo "========================================"
|
echo "========================================"
|
||||||
|
|
||||||
git config user.name "$COMMIT_AUTHOR_NAME"
|
git config user.name "$GIT_USER_NAME"
|
||||||
git config user.email "$COMMIT_AUTHOR_EMAIL"
|
git config user.email "$GIT_USER_EMAIL"
|
||||||
|
|
||||||
source_sha="$(git rev-parse HEAD)"
|
source_sha="$(git rev-parse HEAD)"
|
||||||
source_short="$(git rev-parse --short HEAD)"
|
source_short="$(git rev-parse --short HEAD)"
|
||||||
|
|||||||
@@ -16,6 +16,8 @@ env:
|
|||||||
WORKSPACE_DIR: "/home/workspace"
|
WORKSPACE_DIR: "/home/workspace"
|
||||||
THIRDPARTY_BRANCH: "thirdparty/skill"
|
THIRDPARTY_BRANCH: "thirdparty/skill"
|
||||||
MANIFEST_PATH: ".gitea/ci/thirdparty_skills.json"
|
MANIFEST_PATH: ".gitea/ci/thirdparty_skills.json"
|
||||||
|
GIT_USER_NAME: "ci[bot]"
|
||||||
|
GIT_USER_EMAIL: "ci[bot]@tinysoft.com.cn"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
update_and_sync:
|
update_and_sync:
|
||||||
@@ -70,7 +72,234 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
export MANIFEST_PATH="$MANIFEST_PATH"
|
export MANIFEST_PATH="$MANIFEST_PATH"
|
||||||
TARGET_BRANCH="$THIRDPARTY_BRANCH" bash .gitea/ci/update_thirdparty_skills.sh
|
# BEGIN update_thirdparty_snapshots
|
||||||
|
TARGET_BRANCH="$THIRDPARTY_BRANCH" bash <<'UPDATE_THIRDPARTY'
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
REPO_DIR="${REPO_DIR:-$(pwd)}"
|
||||||
|
TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"
|
||||||
|
MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"
|
||||||
|
|
||||||
|
retry_cmd() {
|
||||||
|
local retries="$1"
|
||||||
|
shift
|
||||||
|
local delay="$1"
|
||||||
|
shift
|
||||||
|
|
||||||
|
local attempt=1
|
||||||
|
while true; do
|
||||||
|
if "$@"; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [ "$attempt" -ge "$retries" ]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "Retry ($attempt/$retries): $*" >&2
|
||||||
|
sleep "$delay"
|
||||||
|
attempt=$((attempt + 1))
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
github_owner_repo() {
|
||||||
|
case "$1" in
|
||||||
|
https://github.com/*)
|
||||||
|
echo "$1" | sed -E 's#^https://github.com/([^/]+/[^/.]+)(\.git)?$#\1#'
|
||||||
|
;;
|
||||||
|
http://github.com/*)
|
||||||
|
echo "$1" | sed -E 's#^http://github.com/([^/]+/[^/.]+)(\.git)?$#\1#'
|
||||||
|
;;
|
||||||
|
git@github.com:*)
|
||||||
|
echo "$1" | sed -E 's#^git@github.com:([^/]+/[^/.]+)(\.git)?$#\1#'
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_latest_sha() {
|
||||||
|
local repo="$1"
|
||||||
|
local ref="$2"
|
||||||
|
local tmp_json="$3"
|
||||||
|
local gh_repo="$4"
|
||||||
|
local sha=""
|
||||||
|
|
||||||
|
if [ -n "$gh_repo" ]; then
|
||||||
|
local api_url="https://api.github.com/repos/${gh_repo}/commits/${ref}"
|
||||||
|
if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$api_url" -o "$tmp_json"; then
|
||||||
|
sha="$(sed -n 's/^[[:space:]]*"sha":[[:space:]]*"\([0-9a-f]\{40\}\)".*/\1/p' "$tmp_json" | head -n 1)"
|
||||||
|
if [ -n "$sha" ]; then
|
||||||
|
echo "$sha"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
sha="$(retry_cmd 3 2 git -c http.version=HTTP/1.1 ls-remote "$repo" "refs/heads/$ref" | awk 'NR==1 {print $1}')"
|
||||||
|
if [ -n "$sha" ]; then
|
||||||
|
echo "$sha"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
emit_sources_tsv() {
|
||||||
|
python3 - "$MANIFEST_PATH" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
with open(sys.argv[1], encoding="utf-8") as fh:
|
||||||
|
data = json.load(fh)
|
||||||
|
|
||||||
|
for entry in data["sources"]:
|
||||||
|
print(
|
||||||
|
"\x1f".join(
|
||||||
|
[
|
||||||
|
entry["id"],
|
||||||
|
entry["upstream_repo"],
|
||||||
|
entry.get("upstream_ref", "main"),
|
||||||
|
entry["snapshot_dir"],
|
||||||
|
entry["sync_mode"],
|
||||||
|
"\x1e".join(entry.get("remove_paths", [])),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
read_source_metadata_value() {
|
||||||
|
local key="$1"
|
||||||
|
local source_file="$2"
|
||||||
|
|
||||||
|
if [ ! -f "$source_file" ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
sed -n "s/^- ${key}:[[:space:]]*//p" "$source_file" | head -n 1
|
||||||
|
}
|
||||||
|
|
||||||
|
remove_snapshot_paths() {
|
||||||
|
local snapshot_dir="$1"
|
||||||
|
local remove_paths="$2"
|
||||||
|
|
||||||
|
[ -n "$remove_paths" ] || return 0
|
||||||
|
|
||||||
|
local IFS=$'\x1e'
|
||||||
|
read -r -a paths <<< "$remove_paths"
|
||||||
|
for path in "${paths[@]}"; do
|
||||||
|
[ -n "$path" ] || continue
|
||||||
|
rm -rf "$snapshot_dir/$path"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
cd "$REPO_DIR"
|
||||||
|
|
||||||
|
git config user.name "$GIT_USER_NAME"
|
||||||
|
git config user.email "$GIT_USER_EMAIL"
|
||||||
|
|
||||||
|
tmp_dir="$(mktemp -d)"
|
||||||
|
cleanup() {
|
||||||
|
rm -rf "$tmp_dir"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
if [ ! -f "$MANIFEST_PATH" ]; then
|
||||||
|
echo "ERROR: third-party manifest not found: $MANIFEST_PATH" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
manifest_copy="$tmp_dir/thirdparty_skills.json"
|
||||||
|
cp "$MANIFEST_PATH" "$manifest_copy"
|
||||||
|
MANIFEST_PATH="$manifest_copy"
|
||||||
|
|
||||||
|
git fetch origin "$TARGET_BRANCH"
|
||||||
|
git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"
|
||||||
|
|
||||||
|
sources_file="$tmp_dir/sources.tsv"
|
||||||
|
if ! emit_sources_tsv > "$sources_file"; then
|
||||||
|
echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
changed=0
|
||||||
|
while IFS=$'\x1f' read -r source_id upstream_repo upstream_ref snapshot_dir sync_mode remove_paths; do
|
||||||
|
[ -n "$source_id" ] || continue
|
||||||
|
remove_paths_md="${remove_paths//$'\x1e'/,}"
|
||||||
|
|
||||||
|
gh_repo=""
|
||||||
|
if gh_repo="$(github_owner_repo "$upstream_repo" 2>/dev/null)"; then
|
||||||
|
:
|
||||||
|
fi
|
||||||
|
|
||||||
|
latest_sha="$(resolve_latest_sha "$upstream_repo" "$upstream_ref" "$tmp_dir/${source_id}-latest.json" "$gh_repo" || true)"
|
||||||
|
if [ -z "$latest_sha" ]; then
|
||||||
|
echo "ERROR: failed to resolve upstream ref for ${source_id}: $upstream_repo $upstream_ref" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
current_sha="$(read_source_metadata_value "Ref" "$snapshot_dir/SOURCE.md")"
|
||||||
|
current_remove_paths="$(read_source_metadata_value "Remove-Paths" "$snapshot_dir/SOURCE.md")"
|
||||||
|
|
||||||
|
if [ "$latest_sha" = "$current_sha" ] && [ "$remove_paths_md" = "$current_remove_paths" ]; then
|
||||||
|
echo "Third-party snapshot is up to date for ${source_id}: $latest_sha"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -rf "$snapshot_dir"
|
||||||
|
mkdir -p "$snapshot_dir"
|
||||||
|
|
||||||
|
snapshot_loaded=0
|
||||||
|
if [ -n "$gh_repo" ]; then
|
||||||
|
tar_url="https://codeload.github.com/${gh_repo}/tar.gz/${latest_sha}"
|
||||||
|
if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$tar_url" -o "$tmp_dir/${source_id}.tar.gz"; then
|
||||||
|
tar -xzf "$tmp_dir/${source_id}.tar.gz" -C "$snapshot_dir" --strip-components=1
|
||||||
|
snapshot_loaded=1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$snapshot_loaded" -eq 0 ]; then
|
||||||
|
upstream_dir="$tmp_dir/${source_id}-upstream"
|
||||||
|
git init "$upstream_dir" >/dev/null
|
||||||
|
git -C "$upstream_dir" remote add origin "$upstream_repo"
|
||||||
|
retry_cmd 3 2 git -C "$upstream_dir" fetch --depth 1 origin "$latest_sha"
|
||||||
|
git -C "$upstream_dir" checkout --detach FETCH_HEAD
|
||||||
|
git -C "$upstream_dir" archive --format=tar HEAD | tar -xf - -C "$snapshot_dir"
|
||||||
|
fi
|
||||||
|
|
||||||
|
remove_snapshot_paths "$snapshot_dir" "$remove_paths"
|
||||||
|
|
||||||
|
snapshot_date="$(date -u +%Y-%m-%d)"
|
||||||
|
cat > "$snapshot_dir/SOURCE.md" <<EOF
|
||||||
|
# Source
|
||||||
|
|
||||||
|
- Repo: ${upstream_repo%".git"}
|
||||||
|
- Ref: $latest_sha
|
||||||
|
- Remove-Paths: $remove_paths_md
|
||||||
|
- Snapshot: $snapshot_date
|
||||||
|
- Sync-Mode: $sync_mode
|
||||||
|
- Notes: vendored into playbook branch $TARGET_BRANCH
|
||||||
|
EOF
|
||||||
|
|
||||||
|
git add -A "$snapshot_dir"
|
||||||
|
changed=1
|
||||||
|
done < "$sources_file"
|
||||||
|
|
||||||
|
if [ "$changed" -eq 0 ] || git diff --cached --quiet; then
|
||||||
|
echo "All third-party snapshots are up to date."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
git commit -m ":package: deps(thirdparty): update snapshots"
|
||||||
|
|
||||||
|
TOKEN="${WORKFLOW:-}"
|
||||||
|
if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
|
||||||
|
git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
|
||||||
|
fi
|
||||||
|
|
||||||
|
git push origin "$TARGET_BRANCH"
|
||||||
|
UPDATE_THIRDPARTY
|
||||||
|
# END update_thirdparty_snapshots
|
||||||
|
|
||||||
git fetch origin "$THIRDPARTY_BRANCH"
|
git fetch origin "$THIRDPARTY_BRANCH"
|
||||||
after_ref="$(git rev-parse "origin/$THIRDPARTY_BRANCH")"
|
after_ref="$(git rev-parse "origin/$THIRDPARTY_BRANCH")"
|
||||||
@@ -88,10 +317,248 @@ jobs:
|
|||||||
git fetch origin main
|
git fetch origin main
|
||||||
git checkout -B main origin/main
|
git checkout -B main origin/main
|
||||||
|
|
||||||
|
# BEGIN sync_thirdparty_skills
|
||||||
TARGET_BRANCH="main" \
|
TARGET_BRANCH="main" \
|
||||||
THIRDPARTY_BRANCH="$THIRDPARTY_BRANCH" \
|
THIRDPARTY_BRANCH="$THIRDPARTY_BRANCH" \
|
||||||
MANIFEST_PATH="$MANIFEST_PATH" \
|
MANIFEST_PATH="$MANIFEST_PATH" \
|
||||||
bash .gitea/ci/sync_thirdparty_skills.sh
|
bash <<'SYNC_THIRDPARTY'
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
REPO_DIR="${REPO_DIR:-$(pwd)}"
|
||||||
|
THIRDPARTY_BRANCH="${THIRDPARTY_BRANCH:-thirdparty/skill}"
|
||||||
|
TARGET_BRANCH="${TARGET_BRANCH:-main}"
|
||||||
|
MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"
|
||||||
|
|
||||||
|
emit_sources_tsv() {
|
||||||
|
python3 - "$MANIFEST_PATH" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
with open(sys.argv[1], encoding="utf-8") as fh:
|
||||||
|
data = json.load(fh)
|
||||||
|
|
||||||
|
for entry in data["sources"]:
|
||||||
|
print(
|
||||||
|
"\x1f".join(
|
||||||
|
[
|
||||||
|
entry["id"],
|
||||||
|
entry["snapshot_dir"],
|
||||||
|
entry["sync_mode"],
|
||||||
|
entry["source_list"],
|
||||||
|
entry.get("skills_subdir", ""),
|
||||||
|
entry.get("output_name", entry["id"]),
|
||||||
|
entry.get("platform_config", ""),
|
||||||
|
entry.get("template_root", ""),
|
||||||
|
entry.get("data_dir", ""),
|
||||||
|
entry.get("scripts_dir", ""),
|
||||||
|
"\x1e".join(entry.get("include_skill_dirs", [])),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
render_skill() {
|
||||||
|
local snapshot_root="$1"
|
||||||
|
local output_dir="$2"
|
||||||
|
local platform_config_rel="$3"
|
||||||
|
local template_root_rel="$4"
|
||||||
|
local data_dir_rel="$5"
|
||||||
|
local scripts_dir_rel="$6"
|
||||||
|
|
||||||
|
python3 - "$snapshot_root" "$output_dir" "$platform_config_rel" "$template_root_rel" "$data_dir_rel" "$scripts_dir_rel" <<'PY'
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
|
||||||
|
snapshot_root = pathlib.Path(sys.argv[1])
|
||||||
|
output_dir = pathlib.Path(sys.argv[2])
|
||||||
|
platform_config_path = snapshot_root / sys.argv[3]
|
||||||
|
template_root = snapshot_root / sys.argv[4]
|
||||||
|
data_dir = snapshot_root / sys.argv[5]
|
||||||
|
scripts_dir = snapshot_root / sys.argv[6]
|
||||||
|
|
||||||
|
config = json.loads(platform_config_path.read_text(encoding="utf-8"))
|
||||||
|
skill_template = (template_root / "base" / "skill-content.md").read_text(encoding="utf-8")
|
||||||
|
quick_reference = ""
|
||||||
|
if config.get("sections", {}).get("quickReference"):
|
||||||
|
quick_reference = "\n" + (template_root / "base" / "quick-reference.md").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
def render_frontmatter(frontmatter):
|
||||||
|
if not frontmatter:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
lines = ["---"]
|
||||||
|
for key, value in frontmatter.items():
|
||||||
|
if any(ch in value for ch in ':"\n'):
|
||||||
|
value = value.replace('"', '\\"')
|
||||||
|
lines.append(f'{key}: "{value}"')
|
||||||
|
else:
|
||||||
|
lines.append(f"{key}: {value}")
|
||||||
|
lines.extend(["---", ""])
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
content = skill_template
|
||||||
|
content = content.replace("{{TITLE}}", config["title"])
|
||||||
|
content = content.replace("{{DESCRIPTION}}", config["description"])
|
||||||
|
content = content.replace("{{SCRIPT_PATH}}", config["scriptPath"])
|
||||||
|
content = content.replace("{{SKILL_OR_WORKFLOW}}", config["skillOrWorkflow"])
|
||||||
|
content = content.replace("{{QUICK_REFERENCE}}", quick_reference)
|
||||||
|
|
||||||
|
if output_dir.exists():
|
||||||
|
shutil.rmtree(output_dir)
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
(output_dir / "SKILL.md").write_text(
|
||||||
|
render_frontmatter(config.get("frontmatter")) + content,
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
if data_dir.exists():
|
||||||
|
shutil.copytree(data_dir, output_dir / "data")
|
||||||
|
if scripts_dir.exists():
|
||||||
|
shutil.copytree(scripts_dir, output_dir / "scripts")
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
tracked_skill_exists() {
|
||||||
|
local name="$1"
|
||||||
|
# conflict only if a first-party SKILL.md exists directly under skills/<name>/
|
||||||
|
[ -f "skills/$name/SKILL.md" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
skill_dir_included() {
|
||||||
|
local name="$1"
|
||||||
|
local include_skill_dirs="$2"
|
||||||
|
|
||||||
|
[ -n "$include_skill_dirs" ] || return 0
|
||||||
|
|
||||||
|
local IFS=$'\x1e'
|
||||||
|
local expected
|
||||||
|
read -r -a included <<< "$include_skill_dirs"
|
||||||
|
for expected in "${included[@]}"; do
|
||||||
|
if [ "$name" = "$expected" ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
cd "$REPO_DIR"
|
||||||
|
|
||||||
|
git config user.name "$GIT_USER_NAME"
|
||||||
|
git config user.email "$GIT_USER_EMAIL"
|
||||||
|
|
||||||
|
git fetch origin "$THIRDPARTY_BRANCH"
|
||||||
|
git fetch origin "$TARGET_BRANCH"
|
||||||
|
|
||||||
|
tmp_dir="$(mktemp -d)"
|
||||||
|
cleanup() {
|
||||||
|
rm -rf "$tmp_dir"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"
|
||||||
|
|
||||||
|
mkdir -p "skills/thirdparty/.sources"
|
||||||
|
|
||||||
|
sources_file="$tmp_dir/sources.tsv"
|
||||||
|
if ! emit_sources_tsv > "$sources_file"; then
|
||||||
|
echo "ERROR: failed to load third-party manifest: $MANIFEST_PATH" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do
|
||||||
|
[ -n "$source_id" ] || continue
|
||||||
|
|
||||||
|
if [ -f "$source_list" ]; then
|
||||||
|
while IFS= read -r name; do
|
||||||
|
[ -n "$name" ] || continue
|
||||||
|
rm -rf "skills/$name"
|
||||||
|
done < "$source_list"
|
||||||
|
fi
|
||||||
|
done < "$sources_file"
|
||||||
|
|
||||||
|
declare -A owners=()
|
||||||
|
while IFS=$'\x1f' read -r source_id snapshot_dir sync_mode source_list skills_subdir output_name platform_config template_root data_dir scripts_dir include_skill_dirs; do
|
||||||
|
[ -n "$source_id" ] || continue
|
||||||
|
|
||||||
|
git archive --format=tar "origin/${THIRDPARTY_BRANCH}" "$snapshot_dir" | tar -xf - -C "$tmp_dir"
|
||||||
|
snapshot_root="$tmp_dir/$snapshot_dir"
|
||||||
|
|
||||||
|
names=()
|
||||||
|
case "$sync_mode" in
|
||||||
|
copy_skill_dirs)
|
||||||
|
source_skills_dir="$snapshot_root/$skills_subdir"
|
||||||
|
if [ ! -d "$source_skills_dir" ]; then
|
||||||
|
echo "ERROR: $skills_subdir not found in snapshot $snapshot_dir" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for dir in "$source_skills_dir"/*; do
|
||||||
|
[ -d "$dir" ] || continue
|
||||||
|
name="$(basename "$dir")"
|
||||||
|
if ! skill_dir_included "$name" "$include_skill_dirs"; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then
|
||||||
|
echo "ERROR: duplicate third-party skill name: $name" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if tracked_skill_exists "$name"; then
|
||||||
|
echo "ERROR: skill name conflict with tracked skill: $name" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -rf "skills/thirdparty/$name"
|
||||||
|
cp -R "$dir" "skills/thirdparty/$name"
|
||||||
|
names+=("$name")
|
||||||
|
owners["$name"]="$source_id"
|
||||||
|
done
|
||||||
|
;;
|
||||||
|
render_skill)
|
||||||
|
name="$output_name"
|
||||||
|
if [ -n "${owners[$name]:-}" ] && [ "${owners[$name]}" != "$source_id" ]; then
|
||||||
|
echo "ERROR: duplicate third-party skill name: $name" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if tracked_skill_exists "$name"; then
|
||||||
|
echo "ERROR: skill name conflict with tracked skill: $name" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
render_skill "$snapshot_root" "skills/thirdparty/$name" "$platform_config" "$template_root" "$data_dir" "$scripts_dir"
|
||||||
|
names+=("$name")
|
||||||
|
owners["$name"]="$source_id"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "ERROR: unsupported sync mode: $sync_mode" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
printf "%s\n" "${names[@]}" | sort > "$source_list"
|
||||||
|
done < "$sources_file"
|
||||||
|
|
||||||
|
git add skills
|
||||||
|
|
||||||
|
if git diff --cached --quiet; then
|
||||||
|
echo "No third-party skills to sync."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
git commit -m ":package: deps(skills): sync thirdparty skills"
|
||||||
|
|
||||||
|
TOKEN="${WORKFLOW:-}"
|
||||||
|
if [ -n "$TOKEN" ] && [ -n "${GITHUB_SERVER_URL:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ]; then
|
||||||
|
git remote set-url origin "https://oauth2:${TOKEN}@${GITHUB_SERVER_URL#https://}/${GITHUB_REPOSITORY}.git"
|
||||||
|
fi
|
||||||
|
|
||||||
|
git push origin "$TARGET_BRANCH"
|
||||||
|
SYNC_THIRDPARTY
|
||||||
|
# END sync_thirdparty_skills
|
||||||
|
|
||||||
echo "✅ Update and sync finished."
|
echo "✅ Update and sync finished."
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import os
|
|||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import textwrap
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -10,6 +11,7 @@ from pathlib import Path
|
|||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
MANIFEST = ROOT / ".gitea" / "ci" / "thirdparty_skills.json"
|
MANIFEST = ROOT / ".gitea" / "ci" / "thirdparty_skills.json"
|
||||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "update-thirdparty-skills.yml"
|
WORKFLOW = ROOT / ".gitea" / "workflows" / "update-thirdparty-skills.yml"
|
||||||
|
TSL_SYNC_WORKFLOW = ROOT / ".gitea" / "workflows" / "sync-tsl-playbook.yml"
|
||||||
LEGACY_WORKFLOW = ROOT / ".gitea" / "workflows" / "update-thirdparty-superpowers.yml"
|
LEGACY_WORKFLOW = ROOT / ".gitea" / "workflows" / "update-thirdparty-superpowers.yml"
|
||||||
UPDATE_SCRIPT = ROOT / ".gitea" / "ci" / "update_thirdparty_skills.sh"
|
UPDATE_SCRIPT = ROOT / ".gitea" / "ci" / "update_thirdparty_skills.sh"
|
||||||
SYNC_SCRIPT = ROOT / ".gitea" / "ci" / "sync_thirdparty_skills.sh"
|
SYNC_SCRIPT = ROOT / ".gitea" / "ci" / "sync_thirdparty_skills.sh"
|
||||||
@@ -44,6 +46,16 @@ def run_command(*args: str, cwd: Path | None = None) -> subprocess.CompletedProc
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_workflow_region(name: str) -> str:
|
||||||
|
text = WORKFLOW.read_text(encoding="utf-8")
|
||||||
|
begin = f"# BEGIN {name}"
|
||||||
|
end = f"# END {name}"
|
||||||
|
if begin not in text or end not in text:
|
||||||
|
raise AssertionError(f"workflow region markers not found: {name}")
|
||||||
|
body = text.split(begin, 1)[1].split(end, 1)[0]
|
||||||
|
return textwrap.dedent(body).strip() + "\n"
|
||||||
|
|
||||||
|
|
||||||
class ThirdpartySkillsPipelineTests(unittest.TestCase):
|
class ThirdpartySkillsPipelineTests(unittest.TestCase):
|
||||||
def test_manifest_declares_all_thirdparty_sources(self):
|
def test_manifest_declares_all_thirdparty_sources(self):
|
||||||
data = load_manifest()
|
data = load_manifest()
|
||||||
@@ -137,14 +149,18 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
|
|||||||
superpowers = next(item for item in data["sources"] if item["id"] == "superpowers")
|
superpowers = next(item for item in data["sources"] if item["id"] == "superpowers")
|
||||||
self.assertEqual(superpowers["remove_paths"], ["skills/ui-ux-pro-max"])
|
self.assertEqual(superpowers["remove_paths"], ["skills/ui-ux-pro-max"])
|
||||||
|
|
||||||
def test_workflow_uses_generic_scripts_and_single_serial_job(self):
|
def test_workflow_inlines_update_and_sync_in_single_serial_job(self):
|
||||||
text = WORKFLOW.read_text(encoding="utf-8")
|
text = WORKFLOW.read_text(encoding="utf-8")
|
||||||
self.assertFalse(LEGACY_WORKFLOW.exists())
|
self.assertFalse(LEGACY_WORKFLOW.exists())
|
||||||
|
self.assertFalse(UPDATE_SCRIPT.exists())
|
||||||
|
self.assertFalse(SYNC_SCRIPT.exists())
|
||||||
self.assertIn("update_and_sync:", text)
|
self.assertIn("update_and_sync:", text)
|
||||||
self.assertNotIn("\n update:\n", text)
|
self.assertNotIn("\n update:\n", text)
|
||||||
self.assertNotIn("\n sync:\n", text)
|
self.assertNotIn("\n sync:\n", text)
|
||||||
self.assertIn("bash .gitea/ci/update_thirdparty_skills.sh", text)
|
self.assertIn("# BEGIN update_thirdparty_snapshots", text)
|
||||||
self.assertIn("bash .gitea/ci/sync_thirdparty_skills.sh", text)
|
self.assertIn("# BEGIN sync_thirdparty_skills", text)
|
||||||
|
self.assertNotIn("update_thirdparty_skills.sh", text)
|
||||||
|
self.assertNotIn("sync_thirdparty_skills.sh", text)
|
||||||
self.assertNotIn("git merge", text)
|
self.assertNotIn("git merge", text)
|
||||||
self.assertNotIn("git pull", text)
|
self.assertNotIn("git pull", text)
|
||||||
|
|
||||||
@@ -153,19 +169,30 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
|
|||||||
self.assertIn("concurrency:", text)
|
self.assertIn("concurrency:", text)
|
||||||
self.assertIn("update-thirdparty-${{ github.repository }}", text)
|
self.assertIn("update-thirdparty-${{ github.repository }}", text)
|
||||||
self.assertIn('MANIFEST_PATH: ".gitea/ci/thirdparty_skills.json"', text)
|
self.assertIn('MANIFEST_PATH: ".gitea/ci/thirdparty_skills.json"', text)
|
||||||
self.assertIn('TARGET_BRANCH="$THIRDPARTY_BRANCH" bash .gitea/ci/update_thirdparty_skills.sh', text)
|
self.assertIn("update_thirdparty_snapshots", text)
|
||||||
self.assertIn('TARGET_BRANCH="main" \\', text)
|
self.assertIn("sync_thirdparty_skills", text)
|
||||||
self.assertIn('MANIFEST_PATH="$MANIFEST_PATH" \\', text)
|
|
||||||
|
|
||||||
def test_generic_scripts_exist_and_use_manifest(self):
|
def test_inline_workflow_exposes_manifest_and_publish_contract(self):
|
||||||
update_text = UPDATE_SCRIPT.read_text(encoding="utf-8")
|
text = WORKFLOW.read_text(encoding="utf-8")
|
||||||
sync_text = SYNC_SCRIPT.read_text(encoding="utf-8")
|
self.assertIn('MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"', text)
|
||||||
self.assertIn('MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"', update_text)
|
self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"', text)
|
||||||
self.assertIn('MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}"', sync_text)
|
self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-main}"', text)
|
||||||
self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}"', update_text)
|
self.assertIn(':package: deps(thirdparty): update snapshots', text)
|
||||||
self.assertIn('TARGET_BRANCH="${TARGET_BRANCH:-main}"', sync_text)
|
self.assertIn(':package: deps(skills): sync thirdparty skills', text)
|
||||||
self.assertIn(':package: deps(thirdparty): update snapshots', update_text)
|
self.assertIn('git push origin "$TARGET_BRANCH"', text)
|
||||||
self.assertIn(':package: deps(skills): sync thirdparty skills', sync_text)
|
|
||||||
|
def test_ci_committers_share_explicit_git_identity(self):
|
||||||
|
workflow_text = TSL_SYNC_WORKFLOW.read_text(encoding="utf-8")
|
||||||
|
thirdparty_text = WORKFLOW.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
for text in (workflow_text, thirdparty_text):
|
||||||
|
self.assertIn('GIT_USER_NAME: "ci[bot]"', text)
|
||||||
|
self.assertIn('GIT_USER_EMAIL: "ci[bot]@tinysoft.com.cn"', text)
|
||||||
|
self.assertIn('git config user.name "$GIT_USER_NAME"', text)
|
||||||
|
self.assertIn('git config user.email "$GIT_USER_EMAIL"', text)
|
||||||
|
self.assertNotIn("COMMIT_AUTHOR_NAME", text)
|
||||||
|
self.assertNotIn("COMMIT_AUTHOR_EMAIL", text)
|
||||||
|
self.assertNotIn("@local", text)
|
||||||
|
|
||||||
def test_skills_doc_points_to_generic_thirdparty_sources(self):
|
def test_skills_doc_points_to_generic_thirdparty_sources(self):
|
||||||
text = SKILLS_MD.read_text(encoding="utf-8")
|
text = SKILLS_MD.read_text(encoding="utf-8")
|
||||||
@@ -193,8 +220,8 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
|
|||||||
self.assertTrue((UI_UX_PRO_MAX_DIR / "data").is_dir())
|
self.assertTrue((UI_UX_PRO_MAX_DIR / "data").is_dir())
|
||||||
self.assertTrue((UI_UX_PRO_MAX_DIR / "scripts").is_dir())
|
self.assertTrue((UI_UX_PRO_MAX_DIR / "scripts").is_dir())
|
||||||
|
|
||||||
def test_update_script_materializes_manifest_before_target_checkout(self):
|
def test_inline_update_materializes_manifest_before_target_checkout(self):
|
||||||
text = UPDATE_SCRIPT.read_text(encoding="utf-8")
|
text = WORKFLOW.read_text(encoding="utf-8")
|
||||||
self.assertIn('manifest_copy="$tmp_dir/thirdparty_skills.json"', text)
|
self.assertIn('manifest_copy="$tmp_dir/thirdparty_skills.json"', text)
|
||||||
self.assertIn('cp "$MANIFEST_PATH" "$manifest_copy"', text)
|
self.assertIn('cp "$MANIFEST_PATH" "$manifest_copy"', text)
|
||||||
self.assertIn('MANIFEST_PATH="$manifest_copy"', text)
|
self.assertIn('MANIFEST_PATH="$manifest_copy"', text)
|
||||||
@@ -208,8 +235,8 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
|
|||||||
text.index('git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"'),
|
text.index('git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"'),
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_sync_script_assumes_thirdparty_snapshot_is_already_clean(self):
|
def test_inline_sync_assumes_thirdparty_snapshot_is_already_clean(self):
|
||||||
text = SYNC_SCRIPT.read_text(encoding="utf-8")
|
text = WORKFLOW.read_text(encoding="utf-8")
|
||||||
self.assertIn('"\\x1f".join(', text)
|
self.assertIn('"\\x1f".join(', text)
|
||||||
self.assertIn("while IFS=$'\\x1f' read -r", text)
|
self.assertIn("while IFS=$'\\x1f' read -r", text)
|
||||||
self.assertIn("include_skill_dirs", text)
|
self.assertIn("include_skill_dirs", text)
|
||||||
@@ -218,7 +245,7 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
|
|||||||
self.assertNotIn("exclude_skill_dirs", text)
|
self.assertNotIn("exclude_skill_dirs", text)
|
||||||
self.assertNotIn("is_excluded_skill_dir", text)
|
self.assertNotIn("is_excluded_skill_dir", text)
|
||||||
|
|
||||||
def test_sync_script_generates_karpathy_outputs_in_temp_repo(self):
|
def test_inline_sync_generates_karpathy_outputs_in_temp_repo(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||||
tmp_root = Path(tmp_dir)
|
tmp_root = Path(tmp_dir)
|
||||||
mirror = tmp_root / "origin.git"
|
mirror = tmp_root / "origin.git"
|
||||||
@@ -265,9 +292,30 @@ class ThirdpartySkillsPipelineTests(unittest.TestCase):
|
|||||||
json.dumps(manifest_data, indent=2) + "\n",
|
json.dumps(manifest_data, indent=2) + "\n",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
shutil.copy2(SYNC_SCRIPT, work / ".gitea" / "ci" / "sync_thirdparty_skills.sh")
|
sync_script = extract_workflow_region("sync_thirdparty_skills")
|
||||||
|
script_path = work / ".sync-thirdparty-test.sh"
|
||||||
sync_result = run_command("bash", ".gitea/ci/sync_thirdparty_skills.sh", cwd=work)
|
script_path.write_text(
|
||||||
|
'export GIT_USER_NAME="test"\n'
|
||||||
|
'export GIT_USER_EMAIL="test@example.invalid"\n'
|
||||||
|
+ sync_script,
|
||||||
|
encoding="utf-8",
|
||||||
|
newline="\n",
|
||||||
|
)
|
||||||
|
env = os.environ.copy()
|
||||||
|
env.update(
|
||||||
|
{
|
||||||
|
"REPO_DIR": str(work),
|
||||||
|
"THIRDPARTY_BRANCH": "thirdparty/skill",
|
||||||
|
"MANIFEST_PATH": ".gitea/ci/thirdparty_skills.json",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
sync_result = subprocess.run(
|
||||||
|
["bash", script_path.name],
|
||||||
|
cwd=work,
|
||||||
|
env=env,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
sync_result.returncode,
|
sync_result.returncode,
|
||||||
0,
|
0,
|
||||||
|
|||||||
Reference in New Issue
Block a user