{
  "name": "AI crawlers list by Depra AI",
  "url": "https://depra.ai/ai-crawlers",
  "description": "46 AI and search bot user-agent tokens, each looked up on its vendor's own crawler page where one exists. \"Vendor docs not found\" means we found no page from the vendor that covers that bot, so no claim is made about it.",
  "license": "Free to use and share. Please credit Depra AI and link to https://depra.ai/ai-crawlers.",
  "checked": "2026-10-07",
  "count": 46,
  "crawlers": [
    {
      "token": "GPTBot",
      "vendor": "OpenAI",
      "purpose": "Training: collects content to train OpenAI's foundation models",
      "followsRobotsTxt": "Yes",
      "source": "https://developers.openai.com/api/docs/bots"
    },
    {
      "token": "OAI-SearchBot",
      "vendor": "OpenAI",
      "purpose": "Search: finds pages to show in ChatGPT search answers",
      "followsRobotsTxt": "Yes",
      "source": "https://developers.openai.com/api/docs/bots"
    },
    {
      "token": "ChatGPT-User",
      "vendor": "OpenAI",
      "purpose": "User-triggered: visits a page when someone asks ChatGPT or a custom GPT about it",
      "followsRobotsTxt": "May not apply, says OpenAI",
      "source": "https://developers.openai.com/api/docs/bots"
    },
    {
      "token": "OAI-AdsBot",
      "vendor": "OpenAI",
      "purpose": "Checks the landing pages of ads submitted to ChatGPT; not used for training",
      "followsRobotsTxt": "Not stated",
      "source": "https://developers.openai.com/api/docs/bots"
    },
    {
      "token": "Googlebot",
      "vendor": "Google",
      "purpose": "Search: crawls for Google Search, including AI Overviews and AI Mode",
      "followsRobotsTxt": "Yes",
      "source": "https://developers.google.com/search/docs/appearance/ai-features"
    },
    {
      "token": "Google-Extended",
      "vendor": "Google",
      "purpose": "Control token, no crawler: covers Gemini model training and grounding in Gemini Apps and Vertex AI",
      "followsRobotsTxt": "Token only",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers"
    },
    {
      "token": "GoogleOther",
      "vendor": "Google",
      "purpose": "Generic crawler for Google product teams; its rules affect no specific product",
      "followsRobotsTxt": "Yes",
      "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers"
    },
    {
      "token": "ClaudeBot",
      "vendor": "Anthropic",
      "purpose": "Training: collects web content for Anthropic's AI models",
      "followsRobotsTxt": "Yes",
      "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "token": "Claude-SearchBot",
      "vendor": "Anthropic",
      "purpose": "Search: crawls to improve search results in Claude",
      "followsRobotsTxt": "Yes",
      "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "token": "Claude-User",
      "vendor": "Anthropic",
      "purpose": "User-triggered: fetches a page when someone asks Claude a question",
      "followsRobotsTxt": "Yes",
      "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "token": "anthropic-ai",
      "vendor": "Anthropic",
      "purpose": "Older token, not on Anthropic's current crawler page",
      "followsRobotsTxt": "Vendor docs not found",
      "source": null
    },
    {
      "token": "PerplexityBot",
      "vendor": "Perplexity",
      "purpose": "Search: surfaces and links sites in Perplexity; not used for model training",
      "followsRobotsTxt": "Yes",
      "source": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
    },
    {
      "token": "Perplexity-User",
      "vendor": "Perplexity",
      "purpose": "User-triggered: visits a page to answer a user's question",
      "followsRobotsTxt": "Generally ignores it, says Perplexity",
      "source": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
    },
    {
      "token": "Meta-ExternalAgent",
      "vendor": "Meta",
      "purpose": "Training and products: training foundation AI models or improving products",
      "followsRobotsTxt": "Yes",
      "source": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers"
    },
    {
      "token": "Meta-WebIndexer",
      "vendor": "Meta",
      "purpose": "Search: improves Meta AI search results",
      "followsRobotsTxt": "Yes",
      "source": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers"
    },
    {
      "token": "Meta-ExternalFetcher",
      "vendor": "Meta",
      "purpose": "User-triggered: fetches single links at a user's request",
      "followsRobotsTxt": "May bypass it, says Meta",
      "source": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers"
    },
    {
      "token": "FacebookBot",
      "vendor": "Meta",
      "purpose": "Older token, not on Meta's current crawler page",
      "followsRobotsTxt": "Vendor docs not found",
      "source": null
    },
    {
      "token": "Applebot",
      "vendor": "Apple",
      "purpose": "Search: powers search in Spotlight, Siri and Safari",
      "followsRobotsTxt": "Yes, and follows Googlebot rules when none name Applebot",
      "source": "https://support.apple.com/en-us/119829"
    },
    {
      "token": "Applebot-Extended",
      "vendor": "Apple",
      "purpose": "Control token, no crawler: opts content out of training Apple's foundation models",
      "followsRobotsTxt": "Token only",
      "source": "https://support.apple.com/en-us/119829"
    },
    {
      "token": "Amazonbot",
      "vendor": "Amazon",
      "purpose": "Improves Amazon products and services; may be used to train Amazon AI models",
      "followsRobotsTxt": "Yes",
      "source": "https://developer.amazon.com/amazonbot"
    },
    {
      "token": "Amzn-SearchBot",
      "vendor": "Amazon",
      "purpose": "Search: search in Amazon products; not used for model training",
      "followsRobotsTxt": "Yes",
      "source": "https://developer.amazon.com/amazonbot"
    },
    {
      "token": "Amzn-User",
      "vendor": "Amazon",
      "purpose": "User-triggered: for example Alexa questions that need current information",
      "followsRobotsTxt": "May not follow every rule, says Amazon",
      "source": "https://developer.amazon.com/amazonbot"
    },
    {
      "token": "bingbot",
      "vendor": "Microsoft",
      "purpose": "Search: crawls for Bing",
      "followsRobotsTxt": "Yes",
      "source": "https://blogs.bing.com/webmaster/2012/5/To-crawl-or-not-to-crawl,-that-is-BingBot-s-questi/"
    },
    {
      "token": "msnbot",
      "vendor": "Microsoft",
      "purpose": "Older Bing token; bingbot reads a msnbot group when it has no group of its own",
      "followsRobotsTxt": "Read by bingbot as a fallback",
      "source": "https://blogs.bing.com/webmaster/2012/5/To-crawl-or-not-to-crawl,-that-is-BingBot-s-questi/"
    },
    {
      "token": "CCBot",
      "vendor": "Common Crawl",
      "purpose": "Builds Common Crawl's free, open archive of web crawl data",
      "followsRobotsTxt": "Yes",
      "source": "https://commoncrawl.org/ccbot"
    },
    {
      "token": "DuckAssistBot",
      "vendor": "DuckDuckGo",
      "purpose": "Search: fetches pages in real time for AI-assisted answers; not used for training",
      "followsRobotsTxt": "Yes",
      "source": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot"
    },
    {
      "token": "MistralAI-User",
      "vendor": "Mistral",
      "purpose": "User-triggered: visits a page when a user asks Mistral's assistant a question",
      "followsRobotsTxt": "Yes",
      "source": "https://docs.mistral.ai/robots"
    },
    {
      "token": "MistralAI-Index",
      "vendor": "Mistral",
      "purpose": "Search: crawls for indexing only; not used for training",
      "followsRobotsTxt": "Yes",
      "source": "https://docs.mistral.ai/robots"
    },
    {
      "token": "MistralAI-Training",
      "vendor": "Mistral",
      "purpose": "Training: builds datasets for Mistral AI models",
      "followsRobotsTxt": "Yes",
      "source": "https://docs.mistral.ai/robots"
    },
    {
      "token": "AI2Bot",
      "vendor": "Allen Institute for AI",
      "purpose": "Training: content used to train open language models",
      "followsRobotsTxt": "Not stated",
      "source": "https://allenai.org/crawler"
    },
    {
      "token": "AhrefsBot",
      "vendor": "Ahrefs",
      "purpose": "Search and SEO index: powers the Yep search engine and the Ahrefs platform",
      "followsRobotsTxt": "Yes",
      "source": "https://ahrefs.com/robot"
    },
    {
      "token": "cohere-ai",
      "vendor": "Cohere",
      "purpose": "Cohere says it runs no crawlers for model training today and lists no bot names",
      "followsRobotsTxt": "Vendor lists no bots",
      "source": "https://docs.cohere.com/docs/cohere-web-crawlers"
    },
    {
      "token": "cohere-training-data-crawler",
      "vendor": "Cohere",
      "purpose": "Not listed by Cohere, as above",
      "followsRobotsTxt": "Vendor lists no bots",
      "source": "https://docs.cohere.com/docs/cohere-web-crawlers"
    },
    {
      "token": "Diffbot",
      "vendor": "Diffbot",
      "purpose": "Crawls the web to build Diffbot's general search engine",
      "followsRobotsTxt": "Yes by default; can be overridden by agreement",
      "source": "https://www.diffbot.com/docs/crawl/faq/robots-txt"
    },
    {
      "token": "PetalBot",
      "vendor": "Huawei",
      "purpose": "Search: indexes sites for Petal Search",
      "followsRobotsTxt": "Yes",
      "source": "https://webmaster.petalsearch.com/site/petalbot"
    },
    {
      "token": "ImagesiftBot",
      "vendor": "Hive (ImageSift)",
      "purpose": "Collects public images, alt text and page text so ImageSift products can find similar images",
      "followsRobotsTxt": "Yes, and follows Googlebot rules when none name ImagesiftBot",
      "source": "https://imagesift.com/about"
    },
    {
      "token": "SemrushBot-OCOB",
      "vendor": "Semrush",
      "purpose": "Crawls pages for Semrush's Content Toolkit",
      "followsRobotsTxt": "Yes",
      "source": "https://www.semrush.com/bot/"
    },
    {
      "token": "Omgilibot",
      "vendor": "Webz.io",
      "purpose": "Older Webz.io crawler, replaced by Webzio and Webzio-extended",
      "followsRobotsTxt": "Not stated for Omgilibot",
      "source": "https://webz.io/blog/company/from-omgilibot-to-the-webzbot-duo-a-powerful-leap-for-ethical-and-comprehensive-data-collection/"
    },
    {
      "token": "Bytespider",
      "vendor": "ByteDance",
      "purpose": "Search: the crawler of Toutiao Search, per its webmaster help page (in Chinese)",
      "followsRobotsTxt": "Not stated on that page",
      "source": "https://zhanzhang.toutiao.com/docs/intro/26899"
    },
    {
      "token": "TikTokSpider",
      "vendor": "ByteDance",
      "purpose": "Vendor docs not found",
      "followsRobotsTxt": "Vendor docs not found",
      "source": null
    },
    {
      "token": "DeepSeekBot",
      "vendor": "DeepSeek",
      "purpose": "Vendor docs not found",
      "followsRobotsTxt": "Vendor docs not found",
      "source": null
    },
    {
      "token": "Kangaroo Bot",
      "vendor": "Kangaroo LLM",
      "purpose": "Vendor docs not found",
      "followsRobotsTxt": "Vendor docs not found",
      "source": null
    },
    {
      "token": "Timpibot",
      "vendor": "Timpi",
      "purpose": "Vendor docs not found",
      "followsRobotsTxt": "Vendor docs not found",
      "source": null
    },
    {
      "token": "xAI-Bot",
      "vendor": "xAI",
      "purpose": "Vendor docs not found",
      "followsRobotsTxt": "Vendor docs not found",
      "source": null
    },
    {
      "token": "Grok-User",
      "vendor": "xAI",
      "purpose": "Vendor docs not found",
      "followsRobotsTxt": "Vendor docs not found",
      "source": null
    },
    {
      "token": "YouBot",
      "vendor": "You.com",
      "purpose": "Search: discovers and indexes pages for the You.com search engine",
      "followsRobotsTxt": "Yes",
      "source": "https://you.com/docs/youbot"
    }
  ]
}