#!/usr/bin/env bash set -euo pipefail REPO_DIR="${REPO_DIR:-$(pwd)}" TARGET_BRANCH="${TARGET_BRANCH:-thirdparty/skill}" MANIFEST_PATH="${MANIFEST_PATH:-.gitea/ci/thirdparty_skills.json}" COMMIT_AUTHOR_NAME="${COMMIT_AUTHOR_NAME:-ci[bot]}" COMMIT_AUTHOR_EMAIL="${COMMIT_AUTHOR_EMAIL:-ci-bot@local}" retry_cmd() { local retries="$1" shift local delay="$1" shift local attempt=1 while true; do if "$@"; then return 0 fi if [ "$attempt" -ge "$retries" ]; then return 1 fi echo "Retry ($attempt/$retries): $*" >&2 sleep "$delay" attempt=$((attempt + 1)) done } github_owner_repo() { case "$1" in https://github.com/*) echo "$1" | sed -E 's#^https://github.com/([^/]+/[^/.]+)(\.git)?$#\1#' ;; http://github.com/*) echo "$1" | sed -E 's#^http://github.com/([^/]+/[^/.]+)(\.git)?$#\1#' ;; git@github.com:*) echo "$1" | sed -E 's#^git@github.com:([^/]+/[^/.]+)(\.git)?$#\1#' ;; *) return 1 ;; esac } resolve_latest_sha() { local repo="$1" local ref="$2" local tmp_json="$3" local gh_repo="$4" local sha="" if [ -n "$gh_repo" ]; then local api_url="https://api.github.com/repos/${gh_repo}/commits/${ref}" if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$api_url" -o "$tmp_json"; then sha="$(sed -n 's/^[[:space:]]*"sha":[[:space:]]*"\([0-9a-f]\{40\}\)".*/\1/p' "$tmp_json" | head -n 1)" if [ -n "$sha" ]; then echo "$sha" return 0 fi fi fi sha="$(retry_cmd 3 2 git -c http.version=HTTP/1.1 ls-remote "$repo" "refs/heads/$ref" | awk 'NR==1 {print $1}')" if [ -n "$sha" ]; then echo "$sha" return 0 fi return 1 } emit_sources_tsv() { python3 - "$MANIFEST_PATH" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as fh: data = json.load(fh) for entry in data["sources"]: print( "\t".join( [ entry["id"], entry["upstream_repo"], entry.get("upstream_ref", "main"), entry["snapshot_dir"], entry["sync_mode"], ] ) ) PY } cd "$REPO_DIR" git config user.name "$COMMIT_AUTHOR_NAME" git config user.email "$COMMIT_AUTHOR_EMAIL" git fetch origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH" tmp_dir="$(mktemp -d)" cleanup() { rm -rf "$tmp_dir" } trap cleanup EXIT changed=0 while IFS=$'\t' read -r source_id upstream_repo upstream_ref snapshot_dir sync_mode; do [ -n "$source_id" ] || continue gh_repo="" if gh_repo="$(github_owner_repo "$upstream_repo" 2>/dev/null)"; then : fi latest_sha="$(resolve_latest_sha "$upstream_repo" "$upstream_ref" "$tmp_dir/${source_id}-latest.json" "$gh_repo" || true)" if [ -z "$latest_sha" ]; then echo "ERROR: failed to resolve upstream ref for ${source_id}: $upstream_repo $upstream_ref" >&2 exit 1 fi current_sha="" if [ -f "$snapshot_dir/SOURCE.md" ]; then current_sha="$(sed -n 's/^- Ref:[[:space:]]*//p' "$snapshot_dir/SOURCE.md" | head -n 1)" fi if [ "$latest_sha" = "$current_sha" ]; then echo "Third-party snapshot is up to date for ${source_id}: $latest_sha" continue fi rm -rf "$snapshot_dir" mkdir -p "$snapshot_dir" snapshot_loaded=0 if [ -n "$gh_repo" ]; then tar_url="https://codeload.github.com/${gh_repo}/tar.gz/${latest_sha}" if retry_cmd 3 2 curl -fsSL --retry 3 --retry-delay 2 "$tar_url" -o "$tmp_dir/${source_id}.tar.gz"; then tar -xzf "$tmp_dir/${source_id}.tar.gz" -C "$snapshot_dir" --strip-components=1 snapshot_loaded=1 fi fi if [ "$snapshot_loaded" -eq 0 ]; then upstream_dir="$tmp_dir/${source_id}-upstream" git init "$upstream_dir" >/dev/null git -C "$upstream_dir" remote add origin "$upstream_repo" retry_cmd 3 2 git -C "$upstream_dir" fetch --depth 1 origin "$latest_sha" git -C "$upstream_dir" checkout --detach FETCH_HEAD git -C "$upstream_dir" archive --format=tar HEAD | tar -xf - -C "$snapshot_dir" fi snapshot_date="$(date -u +%Y-%m-%d)" cat > "$snapshot_dir/SOURCE.md" <