More bots get added all the time, you can get the most recent version of these lists at https://perishablepress.com/ultimate-ai-block-list/
# Allow all other bots, such as search engines
User-agent: *
Disallow:
# Block AI bots
User-agent: .ai
User-agent: -ai
User-agent: _ai
User-agent: ai.
User-agent: ai-
User-agent: ai_
User-agent: ai=
User-agent: AddSearchBot
User-agent: Agentic
User-agent: AgentQL
User-agent: Agent 3
User-agent: Agent API
User-agent: Agent-
User-agent: AI Agent
User-agent: AI Article Writer
User-agent: AI Assistant
User-agent: AI Chat
User-agent: AI Content Detector
User-agent: AI Detection
User-agent: AI Dungeon
User-agent: AI Journalist
User-agent: AI Legion
User-agent: AI RAG
User-agent: AI Search
User-agent: AI SEO Crawler
User-agent: AI Training
User-agent: AI Web
User-agent: AI Writer
User-agent: AI2
User-agent: AIBot
User-agent: aiHitBot
User-agent: AIMatrix
User-agent: Airesearch
User-agent: AISearch
User-agent: AITraining
User-agent: Alexa
User-agent: Alice Yandex
User-agent: AliGenie
User-agent: AliyunSec
User-agent: Alpha AI
User-agent: AlphaAI
User-agent: Amazon
User-agent: Amelia
User-agent: AndersPinkBot
User-agent: AndiBot
User-agent: Anomura
User-agent: Anonymous AI
User-agent: Anthropic
User-agent: AnyPicker
User-agent: Anyword
User-agent: Applebot
User-agent: Aria AI
User-agent: Aria Browse
User-agent: Articoolo
User-agent: Ask AI
User-agent: Atlassian
User-agent: AutoGen
User-agent: Automated Writer
User-agent: AutoML
User-agent: Autonomous RAG
User-agent: AutoRAG
User-agent: Awario
User-agent: AWS Trainium
User-agent: Azure
User-agent: BabyAGI
User-agent: BabyCatAGI
User-agent: BardBot
User-agent: Basic RAG
User-agent: Bedrock
User-agent: Big Sur
User-agent: Bigsur
User-agent: Botsonic
User-agent: Brightbot
User-agent: Browser MCP Agent
User-agent: Browser Use
User-agent: BuddyBot
User-agent: Bytebot
User-agent: ByteDance
User-agent: Bytespider
User-agent: CarynAI
User-agent: CatBoost
User-agent: CC-Crawler
User-agent: CCBot
User-agent: Chai
User-agent: Channel3Bot
User-agent: Character
User-agent: Charstar AI
User-agent: Chatbot
User-agent: Chatsonic
User-agent: ChatUser
User-agent: Chinchilla
User-agent: Claude
User-agent: ClearScope
User-agent: Clearview
User-agent: Cognitive AI
User-agent: Cohere
User-agent: Common Crawl
User-agent: CommonCrawl
User-agent: Compass
User-agent: Content Harmony
User-agent: Content King
User-agent: Content Optimizer
User-agent: Content Samurai
User-agent: ContentAtScale
User-agent: ContentBot
User-agent: Contentedge
User-agent: ContentShake
User-agent: Conversion AI
User-agent: Copilot
User-agent: CopyAI
User-agent: Copymatic
User-agent: Copyscape
User-agent: CoreWeave
User-agent: Corrective RAG
User-agent: Cotoyogi
User-agent: CRAB
User-agent: Crawl4AI
User-agent: CrawlQ AI
User-agent: Crawlspace
User-agent: Crew AI
User-agent: CrewAI
User-agent: Crushon AI
User-agent: Cypex
User-agent: DALL-E
User-agent: DarkBard
User-agent: DarkBERT
User-agent: DataFor
User-agent: DataProvider
User-agent: Datenbank Crawler
User-agent: DeepAI
User-agent: Deep AI
User-agent: DeepL
User-agent: DeepMind
User-agent: Deep Research
User-agent: DeepResearch
User-agent: DeepSeek
User-agent: Devin
User-agent: Diffbot
User-agent: Dolma
User-agent: Doubao AI
User-agent: DuckAssistBot
User-agent: DuckDuckGo Chat
User-agent: DuckDuckGo-Enhanced
User-agent: Echobot
User-agent: Echobox
User-agent: Elixir
User-agent: FacebookBot
User-agent: FacebookExternalHit
User-agent: Factset
User-agent: Falcon
User-agent: FIRE-1
User-agent: Firebase
User-agent: Firecrawl
User-agent: Flyriver
User-agent: Frase AI
User-agent: FriendlyCrawler
User-agent: Gato
User-agent: Gemini
User-agent: Gemma
User-agent: Gen AI
User-agent: GenAI
User-agent: Generative
User-agent: Genspark
User-agent: Gentoo-chat
User-agent: Ghostwriter
User-agent: GigaChat
User-agent: GLM
User-agent: GodMode
User-agent: Goose
User-agent: GPT
User-agent: Grammarly
User-agent: Grendizer
User-agent: Grok
User-agent: Groq
User-agent: GT Bot
User-agent: GTBot
User-agent: GTP
User-agent: Hemingway Editor
User-agent: Hetzner
User-agent: Hugging
User-agent: Hunyuan
User-agent: Hybrid Search RAG
User-agent: Hypotenuse AI
User-agent: iAsk
User-agent: IbouBot
User-agent: ICC-Crawler
User-agent: Ideogram
User-agent: ImageGen
User-agent: Imagen
User-agent: ImagesiftBot
User-agent: ImageSpider
User-agent: img2dataset
User-agent: imgproxy
User-agent: INK Editor
User-agent: INKforall
User-agent: Instructor
User-agent: IntelliSeek
User-agent: Inferkit
User-agent: ISSCyberRiskCrawler
User-agent: Janitor AI
User-agent: Jasper
User-agent: Jenni AI
User-agent: Julius AI
User-agent: Kafkai
User-agent: Kaggle
User-agent: Kangaroo
User-agent: Keyword Density AI
User-agent: Kimi
User-agent: Klaviyo
User-agent: Knowledge
User-agent: KomoBot
User-agent: Kruti
User-agent: KunatoCrawler
User-agent: LAIONDownloader
User-agent: LangChain
User-agent: Langfuse
User-agent: Language AI
User-agent: Le Chat
User-agent: Lensa
User-agent: Leonardo
User-agent: Lightpanda
User-agent: LightRAG
User-agent: LinerBot
User-agent: LinkupBot
User-agent: LLaMA
User-agent: LCC
User-agent: LLM
User-agent: Local RAG Agent
User-agent: Lovable
User-agent: Magistral
User-agent: magpie-crawler
User-agent: MAI-Image
User-agent: Manus
User-agent: MarketMuse
User-agent: MBZUAI
User-agent: Meltwater
User-agent: Meta-AI
User-agent: Meta-External
User-agent: Meta-Webindexer
User-agent: Meta AI
User-agent: MetaAI
User-agent: MetaTagBot
User-agent: Middleware
User-agent: Midjourney
User-agent: Mini AGI
User-agent: MiniMax
User-agent: Mintlify
User-agent: Mistral
User-agent: Mixtral
User-agent: model-training
User-agent: Molmo
User-agent: Monica
User-agent: Nano
User-agent: Narrative
User-agent: NeevaBot
User-agent: netEstate
User-agent: Neural Text
User-agent: NeuralSEO
User-agent: NinjaAI
User-agent: NodeZero
User-agent: NotebookLM
User-agent: Nova Act
User-agent: NovaAct
User-agent: Nytheon
User-agent: OAI-SearchBot
User-agent: OAI SearchBot
User-agent: OASIS
User-agent: Olivia
User-agent: OLM
User-agent: Omgili
User-agent: Onyx
User-agent: Open AI
User-agent: Open Interpreter
User-agent: OpenAGI
User-agent: OpenAI
User-agent: OpenBot
User-agent: OpenInterpreter
User-agent: OpenPi
User-agent: OpenRouter
User-agent: OpenText AI
User-agent: Operator
User-agent: Outwrite
User-agent: OVHcloud
User-agent: Owlin
User-agent: Page Analyzer AI
User-agent: PaLM 2
User-agent: PanguBot
User-agent: Panscient
User-agent: Paperlibot
User-agent: Paraphraser.io
User-agent: peer39_crawler
User-agent: Perflexity
User-agent: Perplexity
User-agent: Petal
User-agent: Phind
User-agent: PiplBot
User-agent: Pixmo
User-agent: Playwright
User-agent: PoeBot
User-agent: PoeSearchBot
User-agent: Poggio
User-agent: Poseidon
User-agent: ProWritingAid
User-agent: Proximic
User-agent: PulsarRPA
User-agent: Puppeteer
User-agent: Python AI
User-agent: Python lxml
User-agent: PyTorch
User-agent: Qualified
User-agent: Quark
User-agent: QuillBot
User-agent: Qopywriter
User-agent: Qwen
User-agent: RAG Agent
User-agent: RAG Azure AI
User-agent: RAG Chatbot
User-agent: RAG Database
User-agent: RAG IS
User-agent: RAG Pipeline
User-agent: RAG Search
User-agent: RAG with
User-agent: RAG-
User-agent: RAG_
User-agent: Raptor
User-agent: React Agent
User-agent: Reasoning
User-agent: Redis AI RAG
User-agent: Reka Flash
User-agent: Replicate
User-agent: RobotSpider
User-agent: RunPod
User-agent: Rytr
User-agent: SaplingAI
User-agent: SBIntuitionsBot
User-agent: Scala
User-agent: Scalenut
User-agent: Scrap
User-agent: ScriptBook
User-agent: Seekr
User-agent: SEObot
User-agent: SEO Content Machine
User-agent: SEO Robot
User-agent: SemrushBot
User-agent: Sentibot
User-agent: Serper
User-agent: ShapBot
User-agent: Sidetrade
User-agent: Simplified AI
User-agent: Sitefinity
User-agent: Skydancer
User-agent: Skyvern
User-agent: SlickWrite
User-agent: SmartBot
User-agent: Sonic
User-agent: Sora
User-agent: Spider/2
User-agent: SpiderCreator
User-agent: Spin Rewrite
User-agent: Spinbot
User-agent: Stability
User-agent: Stable Diffusion
User-agent: StableDiffusionBot
User-agent: Stagehand
User-agent: Stealth
User-agent: Sudowrite
User-agent: SummalyBot
User-agent: Super Agent
User-agent: Superagent
User-agent: SuperAGI
User-agent: Surfer AI
User-agent: Taiko
User-agent: TavilyBot
User-agent: TerraCotta
User-agent: Text Blaze
User-agent: TextCortex
User-agent: Thinkbot
User-agent: Thordata
User-agent: TikTokSpider
User-agent: Timpibot
User-agent: Tinybird
User-agent: Together
User-agent: Traefik
User-agent: Tulu
User-agent: Tülu
User-agent: TurnitinBot
User-agent: TwinAgent
User-agent: uAgents
User-agent: UI-TARS
User-agent: VelenPublicWebCrawler
User-agent: Venus Chub AI
User-agent: Vidnami AI
User-agent: Vision RAG
User-agent: WARDBot
User-agent: WebSurfer
User-agent: WebText
User-agent: WebVoyager
User-agent: Webzio
User-agent: WeChat
User-agent: Whisper
User-agent: WordAI
User-agent: Wordtune
User-agent: WPBot
User-agent: Writecream
User-agent: WriterZen
User-agent: Writescope
User-agent: Writesonic
User-agent: WRTNBot
User-agent: xAI
User-agent: Xanthorox
User-agent: xBot
User-agent: XLNet
User-agent: YaK
User-agent: YaML
User-agent: YandexAdditional
User-agent: YouBot
User-agent: Zanista
User-agent: Zendesk
User-agent: Zero
User-agent: Zhipu
User-agent: Zhuque AI
User-agent: Zimm
Diallow: /
<IfModule mod_rewrite.c>
RewriteEngine On
RewriteCond %{HTTP_USER_AGENT} (\.ai|-ai|_ai|ai\.|ai-|ai_|ai=|AddSearchBot|Agentic|AgentQL|Agent\ 3|Agent\ API|Agent-|AI\ Agent|AI\ Article\ Writer|AI\ Assistant|AI\ Chat|AI\ Content\ Detector|AI\ Detection|AI\ Dungeon) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (AI\ Journalist|AI\ Legion|AI\ RAG|AI\ Search|AI\ SEO\ Crawler|AI\ Training|AI\ Web|AI\ Writer|AI2|AIBot|aiHitBot|AIMatrix|Airesearch|AISearch|AITraining|Alexa|Alice\ Yandex|AliGenie|AliyunSec|Alpha\ AI) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (AlphaAI|Amazon|Amelia|AndersPinkBot|AndiBot|Anomura|Anonymous\ AI|Anthropic|AnyPicker|Anyword|Applebot|Aria\ AI|Aria\ Browse|Articoolo|Ask\ AI|Atlassian|AutoGen|Automated\ Writer|AutoML|Autonomous\ RAG) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (AutoRAG|Awario|AWS\ Trainium|Azure|BabyAGI|BabyCatAGI|BardBot|Basic\ RAG|Bedrock|Big\ Sur|Bigsur|Botsonic|Brightbot|Browser\ MCP\ Agent|Browser\ Use|BuddyBot|Bytebot|ByteDance|Bytespider|CarynAI) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (CatBoost|CC-Crawler|CCBot|Chai|Channel3Bot|Character|Charstar\ AI|Chatbot|Chatsonic|ChatUser|Chinchilla|Claude|ClearScope|Clearview|Cognitive\ AI|Cohere|Common\ Crawl|CommonCrawl|Compass|Content\ Harmony) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Content\ King|Content\ Optimizer|Content\ Samurai|ContentAtScale|ContentBot|Contentedge|ContentShake|Conversion\ AI|Copilot|CopyAI|Copymatic|Copyscape|CoreWeave|Corrective\ RAG|Cotoyogi|CRAB|Crawl4AI|CrawlQ\ AI|Crawlspace|Crew\ AI) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (CrewAI|Crushon\ AI|Cypex|DALL-E|DarkBard|DarkBERT|DataFor|DataProvider|Datenbank\ Crawler|DeepAI|Deep\ AI|DeepL|DeepMind|Deep\ Research|DeepResearch|DeepSeek|Devin|Diffbot|Dolma|Doubao\ AI) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (DuckAssistBot|DuckDuckGo\ Chat|DuckDuckGo-Enhanced|Echobot|Echobox|Elixir|FacebookBot|FacebookExternalHit|Factset|Falcon|FIRE-1|Firebase|Firecrawl|Flyriver|Frase\ AI|FriendlyCrawler|Gato|Gemini|Gemma|Gen\ AI) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (GenAI|Generative|Genspark|Gentoo-chat|Ghostwriter|GigaChat|GLM|GodMode|Goose|GPT|Grammarly|Grendizer|Grok|Groq|GT\ Bot|GTBot|GTP|Hemingway\ Editor|Hetzner|Hugging) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Hunyuan|Hybrid\ Search\ RAG|Hypotenuse\ AI|iAsk|IbouBot|ICC-Crawler|Ideogram|ImageGen|Imagen|ImagesiftBot|ImageSpider|img2dataset|imgproxy|INK\ Editor|INKforall|Instructor|IntelliSeek|Inferkit|ISSCyberRiskCrawler|Janitor\ AI) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Jasper|Jenni\ AI|Julius\ AI|Kafkai|Kaggle|Kangaroo|Keyword\ Density\ AI|Kimi|Klaviyo|Knowledge|KomoBot|Kruti|KunatoCrawler|LAIONDownloader|LangChain|Langfuse|Language\ AI|Le\ Chat|Lensa|Leonardo) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Lightpanda|LightRAG|LinerBot|LinkupBot|LLaMA|LCC|LLM|Local\ RAG\ Agent|Lovable|Magistral|magpie-crawler|MAI-Image|Manus|MarketMuse|MBZUAI|Meltwater|Meta-AI|Meta-External|Meta-Webindexer|Meta\ AI) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (MetaAI|MetaTagBot|Middleware|Midjourney|Mini\ AGI|MiniMax|Mintlify|Mistral|Mixtral|model-training|Molmo|Monica|Nano|Narrative|NeevaBot|netEstate|Neural\ Text|NeuralSEO|NinjaAI|NodeZero) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (NotebookLM|Nova\ Act|NovaAct|Nytheon|OAI-SearchBot|OAI\ SearchBot|OASIS|Olivia|OLM|Omgili|Onyx|Open\ AI|Open\ Interpreter|OpenAGI|OpenAI|OpenBot|OpenInterpreter|OpenPi|OpenRouter|OpenText\ AI) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Operator|Outwrite|OVHcloud|Owlin|Page\ Analyzer\ AI|PaLM\ 2|PanguBot|Panscient|Paperlibot|Paraphraser\.io|peer39_crawler|Perflexity|Perplexity|Petal|Phind|PiplBot|Pixmo|Playwright|PoeBot|PoeSearchBot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Poggio|Poseidon|ProWritingAid|Proximic|PulsarRPA|Puppeteer|Python\ AI|Python\ lxml|PyTorch|Qualified|Quark|QuillBot|Qopywriter|Qwen|RAG\ Agent|RAG\ Azure\ AI|RAG\ Chatbot|RAG\ Database|RAG\ IS|RAG\ Pipeline) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (RAG\ Search|RAG\ with|RAG-|RAG_|Raptor|React\ Agent|Reasoning|Redis\ AI\ RAG|Reka\ Flash|Replicate|RobotSpider|RunPod|Rytr|SaplingAI|SBIntuitionsBot|Scala|Scalenut|Scrap|ScriptBook|Seekr) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (SEObot|SEO\ Content\ Machine|SEO\ Robot|SemrushBot|Sentibot|Serper|ShapBot|Sidetrade|Simplified\ AI|Sitefinity|Skydancer|Skyvern|SlickWrite|SmartBot|Sonic|Sora|Spider/2|SpiderCreator|Spin\ Rewrite|Spinbot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Stability|Stable\ Diffusion|StableDiffusionBot|Stagehand|Stealth|Sudowrite|SummalyBot|Super\ Agent|Superagent|SuperAGI|Surfer\ AI|Taiko|TavilyBot|TerraCotta|Text\ Blaze|TextCortex|Thinkbot|Thordata|TikTokSpider|Timpibot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Tinybird|Together|Traefik|Tulu|Tülu|TurnitinBot|TwinAgent|uAgents|UI-TARS|VelenPublicWebCrawler|Venus\ Chub\ AI|Vidnami\ AI|Vision\ RAG|WARDBot|WebSurfer|WebText|WebVoyager|Webzio|WeChat|Whisper) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (WordAI|Wordtune|WPBot|Writecream|WriterZen|Writescope|Writesonic|WRTNBot|xAI|Xanthorox|xBot|XLNet|YaK|YaML|YandexAdditional|YouBot|Zanista|Zendesk|Zero|Zhipu) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Zhuque\ AI|Zimm) [NC]
RewriteRule (.*) - [F,L]
</IfModule>