#!/bin/bash # Find models associated with papers on Hugging Face # Usage: ./find_models_by_paper.sh [arXiv_id|search_term] # Optional: Set HF_TOKEN environment variable for private/gated models set -e # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Help function show_help() { echo -e "${BLUE}Find models associated with papers on Hugging Face${NC}" echo "" echo -e "${YELLOW}Usage:${NC}" echo " $0 [OPTIONS] [search_term|arXiv_id]" echo "" echo -e "${YELLOW}Options:${NC}" echo " --help Show this help message" echo " --token Use HF_TOKEN environment variable (if set)" echo "" echo -e "${YELLOW}Environment:${NC}" echo " HF_TOKEN Optional: Hugging Face token for private/gated models" echo "" echo -e "${YELLOW}Examples:${NC}" echo " $0 1910.01108 # Search by arXiv ID" echo " $0 distilbert # Search by model name" echo " $0 transformer # Search by keyword" echo " HF_TOKEN=your_token $0 1910.01108 # Use authentication" echo "" echo -e "${YELLOW}Description:${NC}" echo "This script finds Hugging Face models that are associated with research papers." echo "It searches for models that have arXiv IDs in their tags or mentions papers in their metadata." echo "" echo -e "${YELLOW}Notes:${NC}" echo "• HF_TOKEN is optional for public models" echo "• Use HF_TOKEN for private repositories or gated models" echo "• HF_TOKEN enables higher rate limits for heavy usage" } # Parse arguments USE_TOKEN=false POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do case $1 in --help) show_help exit 0 ;; --token) USE_TOKEN=true shift ;; -*) echo -e "${RED}Unknown option: $1${NC}" show_help exit 1 ;; *) POSITIONAL_ARGS+=("$1") shift ;; esac done set -- "${POSITIONAL_ARGS[@]}" if [[ $# -eq 0 ]]; then echo -e "${RED}Error: Please provide a search term or arXiv ID${NC}" echo -e "Use ${YELLOW}$0 --help${NC} for usage information" exit 1 fi SEARCH_TERM="$1" # Set up authentication header if HF_TOKEN is available if [[ -n "$HF_TOKEN" ]] && [[ "$USE_TOKEN" == true || -n "$HF_TOKEN" ]]; then AUTH_HEADER="-H \"Authorization: Bearer $HF_TOKEN\"" echo -e "${BLUE}Using HF_TOKEN for authentication${NC}" else AUTH_HEADER="" if [[ -n "$HF_TOKEN" ]]; then echo -e "${YELLOW}HF_TOKEN found but not using it (add --token flag to use)${NC}" fi fi # Check if the input looks like an arXiv ID (format: YYYY.NNNNN or YYYY.NNNNNNN) if [[ "$SEARCH_TERM" =~ ^[0-9]{4}\.[0-9]{4,7}$ ]]; then echo -e "${BLUE}Searching for models associated with arXiv paper: $SEARCH_TERM${NC}" SEARCH_QUERY="arxiv%3A$SEARCH_TERM" IS_ARXIV_SEARCH=true else echo -e "${BLUE}Searching for models related to: $SEARCH_TERM${NC}" SEARCH_QUERY="$SEARCH_TERM" IS_ARXIV_SEARCH=false fi # Function to extract arXiv IDs from tags extract_arxiv_ids() { local tags="$1" echo "$tags" | jq -r '.[] | select(. | startswith("arxiv:")) | split(":")[1]' 2>/dev/null || true } # Function to get paper title from arXiv ID get_paper_title() { local arxiv_id="$1" # Try to get paper title from Hugging Face tags if available # This is a simplified approach - in practice, you might want to call arXiv API echo "Paper Title (arXiv:$arxiv_id)" } # Search for models API_URL="https://huggingface.co/api/models" echo -e "${YELLOW}Searching Hugging Face API...${NC}" # Build curl command with authentication if available CURL_CMD="curl -s $AUTH_HEADER \"$API_URL?search=$SEARCH_QUERY&limit=50\"" echo -e "${BLUE}API Query: $API_URL?search=$SEARCH_QUERY&limit=50${NC}" # Execute the API call if [[ -n "$HF_TOKEN" ]]; then RESPONSE=$(curl -s -H "Authorization: Bearer $HF_TOKEN" "$API_URL?search=$SEARCH_QUERY&limit=50" || true) else RESPONSE=$(curl -s "$API_URL?search=$SEARCH_QUERY&limit=50" || true) fi # Check if we got a valid response if [[ -z "$RESPONSE" ]] || [[ "$RESPONSE" == "[]" ]]; then echo -e "${RED}No models found for search term: $SEARCH_TERM${NC}" # If arXiv search failed, try without arxiv: prefix if [[ "$IS_ARXIV_SEARCH" == true ]]; then echo -e "${YELLOW}Trying broader search without arxiv: prefix...${NC}" SEARCH_QUERY="$SEARCH_TERM" IS_ARXIV_SEARCH=false if [[ -n "$HF_TOKEN" ]]; then RESPONSE=$(curl -s -H "Authorization: Bearer $HF_TOKEN" "$API_URL?search=$SEARCH_QUERY&limit=50" || true) else RESPONSE=$(curl -s "$API_URL?search=$SEARCH_QUERY&limit=50" || true) fi if [[ -z "$RESPONSE" ]] || [[ "$RESPONSE" == "[]" ]]; then echo -e "${RED}Still no results found. Try a different search term.${NC}" exit 1 fi else exit 1 fi fi # Process the results echo -e "${GREEN}Found models! Processing results...${NC}" # Use jq to process the JSON response and find models with paper associations MODELS_WITH_PAPERS=$(echo "$RESPONSE" | jq -r ' .[] | select(.id != null) | { id: .id, arxiv_tags: [.tags[] | select(. | startswith("arxiv:"))] | join("; "), downloads: (.downloads // 0), likes: (.likes // 0), task: (.pipeline_tag // "unknown"), library: (.library_name // "unknown") } | @base64' 2>/dev/null || true) # Count total results TOTAL_MODELS=$(echo "$RESPONSE" | jq 'length' 2>/dev/null || echo "0") MODELS_WITH_PAPERS_COUNT=$(echo "$MODELS_WITH_PAPERS" | wc -l) echo -e "${BLUE}Results Summary:${NC}" echo -e " Total models found: $TOTAL_MODELS" echo -e " Models with paper associations: $MODELS_WITH_PAPERS_COUNT" echo "" if [[ -z "$MODELS_WITH_PAPERS" ]]; then # Show all models even if no paper associations found echo -e "${YELLOW}No explicit paper associations found. Showing all matching models:${NC}" echo "$RESPONSE" | jq -r ' .[] | select(.id != null) | "📦 \(.id) Task: \(.pipeline_tag // "unknown") Downloads: \(.downloads // 0) Likes: \(.likes // 0) Library: \(.library_name // "unknown") ---" ' 2>/dev/null || echo "Failed to parse response" else # Show models with paper associations echo -e "${GREEN}Models with paper associations:${NC}" echo "$MODELS_WITH_PAPERS" | while read -r model_data; do if [[ -n "$model_data" ]]; then # Decode base64 and show formatted echo "$model_data" | base64 -d | jq -r ' "📄 \(.id) arXiv: \(.arxiv_tags) Task: \(.task) Downloads: \(.downloads) Likes: \(.likes) Library: \(.library) ---" ' 2>/dev/null || echo "Failed to parse model data" fi done fi # Additional search tips echo "" echo -e "${BLUE}Search Tips:${NC}" echo "• Try searching with the full arXiv ID (e.g., 1910.01108)" echo "• Try searching with the paper title keywords" echo "• Try searching with the model name" echo "• Use HF_TOKEN for private models or higher rate limits" echo "" echo -e "${BLUE}Examples to try:${NC}" echo " $0 1910.01108 # DistilBERT paper" echo " $0 1810.04805 # BERT paper" echo " $0 1706.03762 # Attention is All You Need paper" echo " $0 roberta # RoBERTa models" echo " $0 transformer # Transformer models" echo " HF_TOKEN=your_token $0 1910.01108 # Use authentication"