97 lines
2.2 KiB
Bash
97 lines
2.2 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
show_help() {
|
|
cat << 'USAGE'
|
|
Stream model IDs on stdin, emit one JSON object per line (NDJSON).
|
|
|
|
Usage:
|
|
hf_enrich_models.sh [MODEL_ID ...]
|
|
cat ids.txt | hf_enrich_models.sh
|
|
baseline_hf_api.sh 50 | jq -r '.[].id' | hf_enrich_models.sh
|
|
|
|
Description:
|
|
Reads newline-separated model IDs and fetches basic metadata for each.
|
|
Outputs NDJSON with id, downloads, likes, pipeline_tag, tags.
|
|
Uses HF_TOKEN for auth if the environment variable is set.
|
|
|
|
Examples:
|
|
hf_enrich_models.sh gpt2 distilbert-base-uncased
|
|
baseline_hf_api.sh 50 | jq -r '.[].id' | hf_enrich_models.sh | jq -s 'sort_by(.downloads)'
|
|
HF_TOKEN=your_token hf_enrich_models.sh microsoft/DialoGPT-medium
|
|
USAGE
|
|
}
|
|
|
|
if [[ "${1:-}" == "--help" ]]; then
|
|
show_help
|
|
exit 0
|
|
fi
|
|
|
|
if ! command -v jq >/dev/null 2>&1; then
|
|
echo "Error: jq is required but not installed" >&2
|
|
exit 1
|
|
fi
|
|
|
|
headers=()
|
|
if [[ -n "${HF_TOKEN:-}" ]]; then
|
|
headers=(-H "Authorization: Bearer ${HF_TOKEN}")
|
|
fi
|
|
|
|
emit_error() {
|
|
local model_id="$1"
|
|
local message="$2"
|
|
jq -cn --arg id "$model_id" --arg error "$message" '{id: $id, error: $error}'
|
|
}
|
|
|
|
process_id() {
|
|
local model_id="$1"
|
|
|
|
if [[ -z "$model_id" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local url="https://huggingface.co/api/models/${model_id}"
|
|
local response
|
|
response=$(curl -s "${headers[@]}" "$url" 2>/dev/null || true)
|
|
|
|
if [[ -z "$response" ]]; then
|
|
emit_error "$model_id" "request_failed"
|
|
return 0
|
|
fi
|
|
|
|
if ! jq -e . >/dev/null 2>&1 <<<"$response"; then
|
|
emit_error "$model_id" "invalid_json"
|
|
return 0
|
|
fi
|
|
|
|
if jq -e '.error' >/dev/null 2>&1 <<<"$response"; then
|
|
emit_error "$model_id" "not_found"
|
|
return 0
|
|
fi
|
|
|
|
jq -c --arg id "$model_id" '{
|
|
id: (.id // $id),
|
|
downloads: (.downloads // 0),
|
|
likes: (.likes // 0),
|
|
pipeline_tag: (.pipeline_tag // "unknown"),
|
|
tags: (.tags // [])
|
|
}' <<<"$response" 2>/dev/null || emit_error "$model_id" "parse_failed"
|
|
}
|
|
|
|
if [[ $# -gt 0 ]]; then
|
|
for model_id in "$@"; do
|
|
process_id "$model_id"
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -t 0 ]]; then
|
|
show_help
|
|
exit 1
|
|
fi
|
|
|
|
while IFS= read -r model_id; do
|
|
process_id "$model_id"
|
|
done
|