# Source registry — every domain AI Atlas crawls, with trust tier (1 official primary … 4 unverified), crawl policy and owner org.
# `key` is stable; `organization` refers to registry/organizations.yaml keys. Rate limits are requests/minute per domain.
sources:
  # ---------------------------------------------------------------- major AI labs (tier 1, official)
  - key: openai.com
    name: OpenAI — news & developer platform
    domain: openai.com
    organization: openai
    tier: 1
    kind: website
    category: lab
    base_url: https://openai.com
    rate_limit_per_min: 20
    crawl_interval_s: 1800
    priority: 0
  - key: platform.openai.com
    name: OpenAI Platform docs
    domain: platform.openai.com
    organization: openai
    tier: 1
    kind: docs
    category: lab
    base_url: https://platform.openai.com
    rate_limit_per_min: 12
    crawl_interval_s: 3600
    priority: 0
  - key: anthropic.com
    name: Anthropic — news
    domain: anthropic.com
    organization: anthropic
    tier: 1
    kind: website
    category: lab
    base_url: https://www.anthropic.com
    rate_limit_per_min: 20
    crawl_interval_s: 1800
    priority: 0
  - key: docs.claude.com
    name: Claude Developer Platform docs
    domain: docs.claude.com
    organization: anthropic
    tier: 1
    kind: docs
    category: lab
    base_url: https://docs.claude.com
    rate_limit_per_min: 20
    crawl_interval_s: 3600
    priority: 0
  - key: deepmind.google
    name: Google DeepMind — blog
    domain: deepmind.google
    organization: google-deepmind
    tier: 1
    kind: website
    category: lab
    base_url: https://deepmind.google
    rate_limit_per_min: 20
    crawl_interval_s: 3600
    priority: 0
  - key: ai.google.dev
    name: Google AI for Developers (Gemini API docs)
    domain: ai.google.dev
    organization: google
    tier: 1
    kind: docs
    category: lab
    base_url: https://ai.google.dev
    rate_limit_per_min: 20
    crawl_interval_s: 3600
    priority: 0
  - key: ai.meta.com
    name: Meta AI — blog & research
    domain: ai.meta.com
    organization: meta-ai
    tier: 1
    kind: website
    category: lab
    base_url: https://ai.meta.com
    rate_limit_per_min: 15
    crawl_interval_s: 3600
    priority: 0
  - key: llama.com
    name: Llama — official site
    domain: llama.com
    organization: meta-ai
    tier: 1
    kind: website
    category: lab
    base_url: https://www.llama.com
    rate_limit_per_min: 15
    crawl_interval_s: 7200
    priority: 0
  - key: microsoft.com/research
    name: Microsoft Research — publications & blog
    domain: microsoft.com
    organization: microsoft
    tier: 1
    kind: feed
    category: lab
    base_url: https://www.microsoft.com/en-us/research
    rate_limit_per_min: 15
    crawl_interval_s: 7200
    priority: 1
  - key: blogs.nvidia.com
    name: NVIDIA — blog & developer blog
    domain: nvidia.com
    organization: nvidia
    tier: 1
    kind: feed
    category: hardware
    base_url: https://blogs.nvidia.com
    rate_limit_per_min: 15
    crawl_interval_s: 7200
    priority: 1
  - key: machinelearning.apple.com
    name: Apple Machine Learning Research
    domain: machinelearning.apple.com
    organization: apple
    tier: 1
    kind: feed
    category: lab
    base_url: https://machinelearning.apple.com
    rate_limit_per_min: 15
    crawl_interval_s: 21600
    priority: 1
  - key: mistral.ai
    name: Mistral AI — news, models & pricing
    domain: mistral.ai
    organization: mistral
    tier: 1
    kind: website
    category: lab
    base_url: https://mistral.ai
    rate_limit_per_min: 15
    crawl_interval_s: 3600
    priority: 0
  - key: docs.mistral.ai
    name: Mistral AI docs
    domain: docs.mistral.ai
    organization: mistral
    tier: 1
    kind: docs
    category: lab
    base_url: https://docs.mistral.ai
    rate_limit_per_min: 15
    crawl_interval_s: 3600
    priority: 0
  - key: qwenlm.github.io
    name: Qwen — official blog
    domain: qwenlm.github.io
    organization: qwen
    tier: 1
    kind: feed
    category: lab
    base_url: https://qwenlm.github.io
    rate_limit_per_min: 15
    crawl_interval_s: 3600
    priority: 0
  - key: deepseek.com
    name: DeepSeek — site & API docs
    domain: deepseek.com
    organization: deepseek
    tier: 1
    kind: docs
    category: lab
    base_url: https://api-docs.deepseek.com
    rate_limit_per_min: 15
    crawl_interval_s: 3600
    priority: 0
  - key: cohere.com
    name: Cohere — docs & blog
    domain: cohere.com
    organization: cohere
    tier: 1
    kind: docs
    category: lab
    base_url: https://docs.cohere.com
    rate_limit_per_min: 15
    crawl_interval_s: 7200
    priority: 1
  - key: x.ai
    name: xAI — docs
    domain: x.ai
    organization: xai
    tier: 1
    kind: docs
    category: lab
    base_url: https://docs.x.ai
    rate_limit_per_min: 12
    crawl_interval_s: 7200
    priority: 1
  # ---------------------------------------------------------------- model & code ecosystems
  - key: huggingface.co
    name: Hugging Face Hub (public pages, model cards, papers)
    domain: huggingface.co
    organization: huggingface
    tier: 2
    kind: repository
    category: hub
    base_url: https://huggingface.co
    rate_limit_per_min: 30
    crawl_interval_s: 3600
    priority: 0
    notes: Model cards are tier-1 statements by their authors; hub metadata (downloads, likes) is community signal.
  - key: github.com
    name: GitHub public repositories (HTML, releases.atom, raw files)
    domain: github.com
    organization: github
    tier: 2
    kind: repository
    category: code
    base_url: https://github.com
    rate_limit_per_min: 20
    crawl_interval_s: 3600
    priority: 0
  - key: pypi.org
    name: PyPI public package metadata
    domain: pypi.org
    organization: python-software-foundation
    tier: 2
    kind: registry
    category: code
    base_url: https://pypi.org
    rate_limit_per_min: 30
    crawl_interval_s: 21600
    priority: 1
  # ---------------------------------------------------------------- research
  - key: arxiv.org
    name: arXiv (Atom API + RSS)
    domain: arxiv.org
    organization: arxiv
    tier: 1
    kind: feed
    category: research
    base_url: https://export.arxiv.org
    rate_limit_per_min: 4
    crawl_interval_s: 21600
    priority: 0
    robots_policy: documented-exception
    notes: export.arxiv.org/robots.txt disallows generic crawlers but the arXiv API Terms of Use designate the host for programmatic access (≤ 1 request / 3 s); we use 4/min.
  - key: openreview.net
    name: OpenReview
    domain: openreview.net
    organization: openreview
    tier: 1
    kind: registry
    category: research
    base_url: https://api2.openreview.net
    rate_limit_per_min: 10
    crawl_interval_s: 86400
    priority: 1
  # ---------------------------------------------------------------- providers (pricing pages, model lists)
  - key: openrouter.ai
    name: OpenRouter public model & pricing listing
    domain: openrouter.ai
    organization: openrouter
    tier: 2
    kind: registry
    category: provider
    base_url: https://openrouter.ai
    rate_limit_per_min: 10
    crawl_interval_s: 3600
    priority: 0
    notes: Aggregated secondary pricing across providers; tier 2 by design — official provider pages win conflicts.
  - key: groq.com
    name: Groq — docs & pricing
    domain: groq.com
    organization: groq
    tier: 1
    kind: docs
    category: provider
    base_url: https://console.groq.com
    rate_limit_per_min: 12
    crawl_interval_s: 7200
    priority: 1
  - key: together.ai
    name: Together AI — pricing
    domain: together.ai
    organization: together-ai
    tier: 1
    kind: website
    category: provider
    base_url: https://www.together.ai
    rate_limit_per_min: 12
    crawl_interval_s: 7200
    priority: 1
  - key: fireworks.ai
    name: Fireworks AI — pricing
    domain: fireworks.ai
    organization: fireworks-ai
    tier: 1
    kind: website
    category: provider
    base_url: https://fireworks.ai
    rate_limit_per_min: 12
    crawl_interval_s: 7200
    priority: 1
  # ---------------------------------------------------------------- benchmarks / leaderboards
  - key: aider.chat
    name: Aider polyglot leaderboard (data file on GitHub)
    domain: aider.chat
    organization: aider
    tier: 2
    kind: leaderboard
    category: benchmark
    base_url: https://aider.chat/docs/leaderboards/
    rate_limit_per_min: 10
    crawl_interval_s: 21600
    priority: 1
  - key: swebench.com
    name: SWE-bench leaderboards
    domain: swebench.com
    organization: swe-bench
    tier: 2
    kind: leaderboard
    category: benchmark
    base_url: https://www.swebench.com
    rate_limit_per_min: 10
    crawl_interval_s: 21600
    priority: 1
  - key: livebench.ai
    name: LiveBench
    domain: livebench.ai
    organization: livebench
    tier: 2
    kind: leaderboard
    category: benchmark
    base_url: https://livebench.ai
    rate_limit_per_min: 10
    crawl_interval_s: 43200
    priority: 2
  - key: artificialanalysis.ai
    name: Artificial Analysis
    domain: artificialanalysis.ai
    organization: artificial-analysis
    tier: 2
    kind: leaderboard
    category: benchmark
    base_url: https://artificialanalysis.ai
    rate_limit_per_min: 6
    crawl_interval_s: 43200
    priority: 2
  # ---------------------------------------------------------------- hardware manufacturers
  - key: apple.com
    name: Apple — Mac tech specs
    domain: apple.com
    organization: apple
    tier: 1
    kind: website
    category: hardware
    base_url: https://www.apple.com
    rate_limit_per_min: 10
    crawl_interval_s: 86400
    priority: 1
  - key: nvidia.com
    name: NVIDIA — data center product pages
    domain: nvidia.com
    organization: nvidia
    tier: 1
    kind: website
    category: hardware
    base_url: https://www.nvidia.com
    rate_limit_per_min: 10
    crawl_interval_s: 86400
    priority: 1
  - key: amd.com
    name: AMD — Instinct accelerators
    domain: amd.com
    organization: amd
    tier: 1
    kind: website
    category: hardware
    base_url: https://www.amd.com
    rate_limit_per_min: 6
    crawl_interval_s: 86400
    priority: 2
  # ---------------------------------------------------------------- curated registries (AI Atlas itself)
  - key: ai-atlas.registry
    name: AI Atlas curated registry (YAML, versioned in git, every entry carries its source URL)
    domain: ai-atlas.co
    organization: ai-atlas
    tier: 2
    kind: registry
    category: registry
    base_url: https://www.ai-atlas.co/sources
    rate_limit_per_min: 1000
    crawl_interval_s: 86400
    priority: 1