50 lines
1.2 KiB
Plaintext
50 lines
1.2 KiB
Plaintext
User-agent: *
|
|
Crawl-delay: 5
|
|
Disallow: /assets/
|
|
Disallow: /dogma-jimfinium/assets/
|
|
Disallow: /*/assets/
|
|
Disallow: /shepich_resume.pdf
|
|
|
|
|
|
# Block AI bots.
|
|
User-agent: GPTBot
|
|
User-agent: OAI-SearchBot
|
|
User-agent: ChatGPT-User
|
|
User-agent: anthropic-ai
|
|
User-agent: ClaudeBot
|
|
User-agent: claude-web
|
|
User-agent: PerplexityBot
|
|
User-agent: Perplexity-User
|
|
User-agent: Google-Extended
|
|
User-agent: Googlebot-Image
|
|
User-agent: Bingbot
|
|
User-agent: Amazonbot
|
|
User-agent: Applebot
|
|
User-agent: Applebot-Extended
|
|
User-agent: FacebookBot
|
|
User-agent: meta-externalagent
|
|
User-agent: LinkedInBot
|
|
User-agent: Bytespider
|
|
User-agent: DuckAssistBot
|
|
User-agent: cohere-ai
|
|
User-agent: AI2Bot
|
|
User-agent: CCBot
|
|
User-agent: Diffbot
|
|
User-agent: omgili
|
|
User-agent: Timpibot
|
|
User-agent: YouBot
|
|
User-agent: MistralAI-User
|
|
User-agent: GoogleAgent-Mariner
|
|
User-agent: Standard Chrome UA
|
|
User-agent: No known UA
|
|
User-agent: Knowatoa AI Search Console
|
|
User-agent: Merkle robots.txt Tester
|
|
User-agent: Block GPT model training
|
|
User-agent: Allow ChatGPT user traffic
|
|
User-agent: ChatGPT Atlas
|
|
User-agent: OpenAI Operator
|
|
User-agent: Google Project Mariner
|
|
User-agent: Anthropic Computer Use
|
|
User-agent: xAI Grok
|
|
Disallow: /
|