{
 "source": "https://agentswelcome.dev/api/crawlers",
 "registry_updated": "2026-07-06",
 "copied": "2026-09-07",
 "records": [
  {
   "token": "ClaudeBot",
   "name": "ClaudeBot",
   "operator": "Anthropic",
   "purpose": "training",
   "ua_substring": "ClaudeBot"
  },
  {
   "token": "Claude-User",
   "name": "Claude-User",
   "operator": "Anthropic",
   "purpose": "inference",
   "ua_substring": "Claude-User"
  },
  {
   "token": "Claude-SearchBot",
   "name": "Claude-SearchBot",
   "operator": "Anthropic",
   "purpose": "search",
   "ua_substring": "Claude-SearchBot"
  },
  {
   "token": "GPTBot",
   "name": "GPTBot",
   "operator": "OpenAI",
   "purpose": "training",
   "ua_substring": "GPTBot"
  },
  {
   "token": "OAI-SearchBot",
   "name": "OAI-SearchBot",
   "operator": "OpenAI",
   "purpose": "search",
   "ua_substring": "OAI-SearchBot"
  },
  {
   "token": "ChatGPT-User",
   "name": "ChatGPT-User",
   "operator": "OpenAI",
   "purpose": "inference",
   "ua_substring": "ChatGPT-User"
  },
  {
   "token": "PerplexityBot",
   "name": "PerplexityBot",
   "operator": "Perplexity",
   "purpose": "search",
   "ua_substring": "PerplexityBot"
  },
  {
   "token": "Perplexity-User",
   "name": "Perplexity-User",
   "operator": "Perplexity",
   "purpose": "inference",
   "ua_substring": "Perplexity-User"
  },
  {
   "token": "Google-Extended",
   "name": "Google-Extended",
   "operator": "Google",
   "purpose": "training",
   "ua_substring": "(none — token only)"
  },
  {
   "token": "GoogleOther",
   "name": "GoogleOther",
   "operator": "Google",
   "purpose": "search",
   "ua_substring": "GoogleOther"
  },
  {
   "token": "Google-CloudVertexBot",
   "name": "Google-CloudVertexBot / Gemini agents",
   "operator": "Google",
   "purpose": "inference",
   "ua_substring": "Google-CloudVertexBot"
  },
  {
   "token": "Bingbot",
   "name": "Bingbot",
   "operator": "Microsoft",
   "purpose": "search",
   "ua_substring": "bingbot"
  },
  {
   "token": "Amazonbot",
   "name": "Amazonbot",
   "operator": "Amazon",
   "purpose": "search",
   "ua_substring": "Amazonbot"
  },
  {
   "token": "Applebot-Extended",
   "name": "Applebot-Extended",
   "operator": "Apple",
   "purpose": "training",
   "ua_substring": "(none — token only)"
  },
  {
   "token": "meta-externalagent",
   "name": "Meta-ExternalAgent",
   "operator": "Meta",
   "purpose": "training",
   "ua_substring": "meta-externalagent"
  },
  {
   "token": "CCBot",
   "name": "CCBot",
   "operator": "Common Crawl",
   "purpose": "training",
   "ua_substring": "CCBot"
  },
  {
   "token": "Bytespider",
   "name": "Bytespider",
   "operator": "ByteDance",
   "purpose": "training",
   "ua_substring": "Bytespider"
  },
  {
   "token": "DuckAssistBot",
   "name": "DuckAssistBot",
   "operator": "DuckDuckGo",
   "purpose": "inference",
   "ua_substring": "DuckAssistBot"
  },
  {
   "token": "OAI-AdsBot",
   "name": "OAI-AdsBot",
   "operator": "OpenAI",
   "purpose": "ad-verification",
   "ua_substring": "OAI-AdsBot"
  },
  {
   "token": "Google-Agent",
   "name": "Google-Agent",
   "operator": "Google",
   "purpose": "inference",
   "ua_substring": "Google-Agent"
  },
  {
   "token": "MistralAI-User",
   "name": "MistralAI-User",
   "operator": "Mistral AI",
   "purpose": "inference",
   "ua_substring": "MistralAI-User"
  },
  {
   "token": "Diffbot",
   "name": "Diffbot",
   "operator": "Diffbot",
   "purpose": "data-aggregation",
   "ua_substring": "Diffbot"
  },
  {
   "token": "Diffbot-User",
   "name": "Diffbot-User",
   "operator": "Diffbot",
   "purpose": "inference",
   "ua_substring": "Diffbot-User"
  },
  {
   "token": "ImagesiftBot",
   "name": "ImagesiftBot",
   "operator": "ImageSift (Hive)",
   "purpose": "data-aggregation",
   "ua_substring": "ImagesiftBot"
  },
  {
   "token": "ICC-Crawler",
   "name": "ICC-Crawler",
   "operator": "NICT (National Institute of Information and Communications Technology)",
   "purpose": "training",
   "ua_substring": "ICC-Crawler"
  },
  {
   "token": "cohere-ai",
   "name": "cohere-ai",
   "operator": "Cohere",
   "purpose": "inference",
   "ua_substring": "cohere-ai"
  },
  {
   "token": "Meta-WebIndexer",
   "name": "Meta-WebIndexer",
   "operator": "Meta",
   "purpose": "search",
   "ua_substring": "Meta-WebIndexer"
  },
  {
   "token": "(none — agentic browser; no published robots.txt token)",
   "name": "ChatGPT Atlas (agent mode)",
   "operator": "OpenAI",
   "purpose": "agentic-browsing",
   "ua_substring": ""
  },
  {
   "token": "(none — agentic browser; no published robots.txt token)",
   "name": "Perplexity Comet (assistant/agent)",
   "operator": "Perplexity",
   "purpose": "agentic-browsing",
   "ua_substring": ""
  },
  {
   "token": "(none — agentic browser/agent; no published robots.txt token)",
   "name": "OpenAI Operator (Computer-Using Agent)",
   "operator": "OpenAI",
   "purpose": "agentic-browsing",
   "ua_substring": ""
  },
  {
   "token": "(none — agentic browser; no published robots.txt token; successor Google-Agent carries a token)",
   "name": "Project Mariner",
   "operator": "Google",
   "purpose": "agentic-browsing",
   "ua_substring": ""
  },
  {
   "token": "Applebot",
   "name": "Applebot",
   "operator": "Apple",
   "purpose": "search",
   "ua_substring": "Applebot"
  },
  {
   "token": "meta-externalfetcher",
   "name": "meta-externalfetcher",
   "operator": "Meta",
   "purpose": "inference",
   "ua_substring": "meta-externalfetcher"
  },
  {
   "token": "meta-externalads",
   "name": "meta-externalads",
   "operator": "Meta",
   "purpose": "ad-verification",
   "ua_substring": "meta-externalads"
  },
  {
   "token": "AI2Bot",
   "name": "AI2Bot",
   "operator": "Allen Institute for AI (Ai2)",
   "purpose": "training",
   "ua_substring": "AI2Bot"
  },
  {
   "token": "anthropic-ai",
   "name": "anthropic-ai",
   "operator": "Anthropic",
   "purpose": "training",
   "ua_substring": "anthropic-ai"
  },
  {
   "token": "Bravebot",
   "name": "Bravebot",
   "operator": "Brave",
   "purpose": "search",
   "ua_substring": "Bravebot"
  },
  {
   "token": "kagi-fetcher",
   "name": "kagi-fetcher",
   "operator": "Kagi",
   "purpose": "inference",
   "ua_substring": "kagi-fetcher"
  },
  {
   "token": "bedrockbot",
   "name": "bedrockbot",
   "operator": "Amazon",
   "purpose": "inference",
   "ua_substring": "bedrockbot"
  },
  {
   "token": "cohere-training-data-crawler",
   "name": "cohere-training-data-crawler",
   "operator": "Cohere",
   "purpose": "training",
   "ua_substring": "cohere-training-data-crawler"
  },
  {
   "token": "DuckDuckBot",
   "name": "DuckDuckBot",
   "operator": "DuckDuckGo",
   "purpose": "search",
   "ua_substring": "DuckDuckBot"
  }
 ]
}