38 lines
628 B
Plaintext
38 lines
628 B
Plaintext
# KNUTH Ceramics — robots.txt
|
|
# https://knuthceramics.com
|
|
|
|
User-agent: *
|
|
Allow: /
|
|
Disallow: /checkout
|
|
Disallow: /mock-payment
|
|
Disallow: /success
|
|
Disallow: /admin
|
|
|
|
# AI Search Crawlers — explicitly allowed for citation visibility
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: ChatGPT-User
|
|
Allow: /
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
|
|
# Training-only crawlers — blocked (prevents scraping without citation benefit)
|
|
User-agent: CCBot
|
|
Disallow: /
|
|
|
|
Sitemap: https://knuthceramics.com/sitemap.xml
|