User-agent: *
Allow: /
Disallow: /admin/
Disallow: /login
Disallow: /register
Disallow: /profile
Disallow: /verify
Disallow: /test-
Disallow: /api/auth/
Disallow: /api/admin/
Disallow: /api/stripe/
Crawl-delay: 1

# Search Engine Crawlers
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Welcome AI Crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: CCBot
Allow: /

Sitemap: https://allratestoday.com/sitemap.xml

# AI Documentation
# LLM Guide: https://allratestoday.com/llms.txt
# API Docs: https://allratestoday.com/api/openapi.json
# OpenAPI Spec: https://allratestoday.com/openapi.json