# robots.txt for promptprogen.com
User-agent: *
Allow: /
Disallow: /api/

# AI search crawlers - these decide whether you get cited in AI answers
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Bingbot
Allow: /

# User-triggered fetches (someone asks an assistant to read your page)
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

# Training crawlers - blocked. Does not affect Google rankings or AI-search citations.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

Sitemap: https://promptprogen.com/sitemap.xml