📦 deps(thirdparty): update snapshots
This commit is contained in:
ci[bot]
+230
@@ -0,0 +1,230 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Find models associated with papers on Hugging Face
|
||||
# Usage: ./find_models_by_paper.sh [arXiv_id|search_term]
|
||||
# Optional: Set HF_TOKEN environment variable for private/gated models
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Help function
|
||||
show_help() {
|
||||
echo -e "${BLUE}Find models associated with papers on Hugging Face${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Usage:${NC}"
|
||||
echo " $0 [OPTIONS] [search_term|arXiv_id]"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Options:${NC}"
|
||||
echo " --help Show this help message"
|
||||
echo " --token Use HF_TOKEN environment variable (if set)"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Environment:${NC}"
|
||||
echo " HF_TOKEN Optional: Hugging Face token for private/gated models"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Examples:${NC}"
|
||||
echo " $0 1910.01108 # Search by arXiv ID"
|
||||
echo " $0 distilbert # Search by model name"
|
||||
echo " $0 transformer # Search by keyword"
|
||||
echo " HF_TOKEN=your_token $0 1910.01108 # Use authentication"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Description:${NC}"
|
||||
echo "This script finds Hugging Face models that are associated with research papers."
|
||||
echo "It searches for models that have arXiv IDs in their tags or mentions papers in their metadata."
|
||||
echo ""
|
||||
echo -e "${YELLOW}Notes:${NC}"
|
||||
echo "• HF_TOKEN is optional for public models"
|
||||
echo "• Use HF_TOKEN for private repositories or gated models"
|
||||
echo "• HF_TOKEN enables higher rate limits for heavy usage"
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
USE_TOKEN=false
|
||||
POSITIONAL_ARGS=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--help)
|
||||
show_help
|
||||
exit 0
|
||||
;;
|
||||
--token)
|
||||
USE_TOKEN=true
|
||||
shift
|
||||
;;
|
||||
-*)
|
||||
echo -e "${RED}Unknown option: $1${NC}"
|
||||
show_help
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
POSITIONAL_ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
set -- "${POSITIONAL_ARGS[@]}"
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
echo -e "${RED}Error: Please provide a search term or arXiv ID${NC}"
|
||||
echo -e "Use ${YELLOW}$0 --help${NC} for usage information"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SEARCH_TERM="$1"
|
||||
|
||||
# Set up authentication header if HF_TOKEN is available
|
||||
if [[ -n "$HF_TOKEN" ]] && [[ "$USE_TOKEN" == true || -n "$HF_TOKEN" ]]; then
|
||||
AUTH_HEADER="-H \"Authorization: Bearer $HF_TOKEN\""
|
||||
echo -e "${BLUE}Using HF_TOKEN for authentication${NC}"
|
||||
else
|
||||
AUTH_HEADER=""
|
||||
if [[ -n "$HF_TOKEN" ]]; then
|
||||
echo -e "${YELLOW}HF_TOKEN found but not using it (add --token flag to use)${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check if the input looks like an arXiv ID (format: YYYY.NNNNN or YYYY.NNNNNNN)
|
||||
if [[ "$SEARCH_TERM" =~ ^[0-9]{4}\.[0-9]{4,7}$ ]]; then
|
||||
echo -e "${BLUE}Searching for models associated with arXiv paper: $SEARCH_TERM${NC}"
|
||||
SEARCH_QUERY="arxiv%3A$SEARCH_TERM"
|
||||
IS_ARXIV_SEARCH=true
|
||||
else
|
||||
echo -e "${BLUE}Searching for models related to: $SEARCH_TERM${NC}"
|
||||
SEARCH_QUERY="$SEARCH_TERM"
|
||||
IS_ARXIV_SEARCH=false
|
||||
fi
|
||||
|
||||
# Function to extract arXiv IDs from tags
|
||||
extract_arxiv_ids() {
|
||||
local tags="$1"
|
||||
echo "$tags" | jq -r '.[] | select(. | startswith("arxiv:")) | split(":")[1]' 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Function to get paper title from arXiv ID
|
||||
get_paper_title() {
|
||||
local arxiv_id="$1"
|
||||
# Try to get paper title from Hugging Face tags if available
|
||||
# This is a simplified approach - in practice, you might want to call arXiv API
|
||||
echo "Paper Title (arXiv:$arxiv_id)"
|
||||
}
|
||||
|
||||
# Search for models
|
||||
API_URL="https://huggingface.co/api/models"
|
||||
echo -e "${YELLOW}Searching Hugging Face API...${NC}"
|
||||
|
||||
# Build curl command with authentication if available
|
||||
CURL_CMD="curl -s $AUTH_HEADER \"$API_URL?search=$SEARCH_QUERY&limit=50\""
|
||||
echo -e "${BLUE}API Query: $API_URL?search=$SEARCH_QUERY&limit=50${NC}"
|
||||
|
||||
# Execute the API call
|
||||
if [[ -n "$HF_TOKEN" ]]; then
|
||||
RESPONSE=$(curl -s -H "Authorization: Bearer $HF_TOKEN" "$API_URL?search=$SEARCH_QUERY&limit=50" || true)
|
||||
else
|
||||
RESPONSE=$(curl -s "$API_URL?search=$SEARCH_QUERY&limit=50" || true)
|
||||
fi
|
||||
|
||||
# Check if we got a valid response
|
||||
if [[ -z "$RESPONSE" ]] || [[ "$RESPONSE" == "[]" ]]; then
|
||||
echo -e "${RED}No models found for search term: $SEARCH_TERM${NC}"
|
||||
|
||||
# If arXiv search failed, try without arxiv: prefix
|
||||
if [[ "$IS_ARXIV_SEARCH" == true ]]; then
|
||||
echo -e "${YELLOW}Trying broader search without arxiv: prefix...${NC}"
|
||||
SEARCH_QUERY="$SEARCH_TERM"
|
||||
IS_ARXIV_SEARCH=false
|
||||
|
||||
if [[ -n "$HF_TOKEN" ]]; then
|
||||
RESPONSE=$(curl -s -H "Authorization: Bearer $HF_TOKEN" "$API_URL?search=$SEARCH_QUERY&limit=50" || true)
|
||||
else
|
||||
RESPONSE=$(curl -s "$API_URL?search=$SEARCH_QUERY&limit=50" || true)
|
||||
fi
|
||||
|
||||
if [[ -z "$RESPONSE" ]] || [[ "$RESPONSE" == "[]" ]]; then
|
||||
echo -e "${RED}Still no results found. Try a different search term.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Process the results
|
||||
echo -e "${GREEN}Found models! Processing results...${NC}"
|
||||
|
||||
# Use jq to process the JSON response and find models with paper associations
|
||||
MODELS_WITH_PAPERS=$(echo "$RESPONSE" | jq -r '
|
||||
.[] |
|
||||
select(.id != null) |
|
||||
{
|
||||
id: .id,
|
||||
arxiv_tags: [.tags[] | select(. | startswith("arxiv:"))] | join("; "),
|
||||
downloads: (.downloads // 0),
|
||||
likes: (.likes // 0),
|
||||
task: (.pipeline_tag // "unknown"),
|
||||
library: (.library_name // "unknown")
|
||||
}
|
||||
| @base64' 2>/dev/null || true)
|
||||
|
||||
# Count total results
|
||||
TOTAL_MODELS=$(echo "$RESPONSE" | jq 'length' 2>/dev/null || echo "0")
|
||||
MODELS_WITH_PAPERS_COUNT=$(echo "$MODELS_WITH_PAPERS" | wc -l)
|
||||
|
||||
echo -e "${BLUE}Results Summary:${NC}"
|
||||
echo -e " Total models found: $TOTAL_MODELS"
|
||||
echo -e " Models with paper associations: $MODELS_WITH_PAPERS_COUNT"
|
||||
echo ""
|
||||
|
||||
if [[ -z "$MODELS_WITH_PAPERS" ]]; then
|
||||
# Show all models even if no paper associations found
|
||||
echo -e "${YELLOW}No explicit paper associations found. Showing all matching models:${NC}"
|
||||
echo "$RESPONSE" | jq -r '
|
||||
.[] |
|
||||
select(.id != null) |
|
||||
"📦 \(.id)
|
||||
Task: \(.pipeline_tag // "unknown")
|
||||
Downloads: \(.downloads // 0)
|
||||
Likes: \(.likes // 0)
|
||||
Library: \(.library_name // "unknown")
|
||||
---"
|
||||
' 2>/dev/null || echo "Failed to parse response"
|
||||
else
|
||||
# Show models with paper associations
|
||||
echo -e "${GREEN}Models with paper associations:${NC}"
|
||||
echo "$MODELS_WITH_PAPERS" | while read -r model_data; do
|
||||
if [[ -n "$model_data" ]]; then
|
||||
# Decode base64 and show formatted
|
||||
echo "$model_data" | base64 -d | jq -r '
|
||||
"📄 \(.id)
|
||||
arXiv: \(.arxiv_tags)
|
||||
Task: \(.task)
|
||||
Downloads: \(.downloads)
|
||||
Likes: \(.likes)
|
||||
Library: \(.library)
|
||||
---"
|
||||
' 2>/dev/null || echo "Failed to parse model data"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Additional search tips
|
||||
echo ""
|
||||
echo -e "${BLUE}Search Tips:${NC}"
|
||||
echo "• Try searching with the full arXiv ID (e.g., 1910.01108)"
|
||||
echo "• Try searching with the paper title keywords"
|
||||
echo "• Try searching with the model name"
|
||||
echo "• Use HF_TOKEN for private models or higher rate limits"
|
||||
echo ""
|
||||
echo -e "${BLUE}Examples to try:${NC}"
|
||||
echo " $0 1910.01108 # DistilBERT paper"
|
||||
echo " $0 1810.04805 # BERT paper"
|
||||
echo " $0 1706.03762 # Attention is All You Need paper"
|
||||
echo " $0 roberta # RoBERTa models"
|
||||
echo " $0 transformer # Transformer models"
|
||||
echo " HF_TOKEN=your_token $0 1910.01108 # Use authentication"
|
||||
Reference in New Issue
Block a user