# FitBudd robots.txt — AI & SEO Optimized (Referrals + Crawl Budget)

# Major AI and Search Crawlers (Explicit Allow)
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: claude-web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: YouBot
User-agent: DuckAssistBot
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Bingbot
User-agent: BingPreview
User-agent: DuckDuckBot
Allow: /

# Block low-value pages and unnecessary parameters
Disallow: /search/
Disallow: /404
Disallow: /404.html
Disallow: /401
Disallow: /401
Disallow: /*?edit=*
Disallow: /*?cms_editor=*
Disallow: /*?preview=*
Disallow: /*?nocache=*

# Explicitly allow useful dynamic and static assets
Allow: /*?
Allow: /*=
Allow: /assets/
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /videos/
Allow: /favicon.ico
Allow: /robots.txt
Allow: /sitemap.xml

# Default rule for all other crawlers
User-agent: *
Allow: /
Disallow: /search/
Disallow: /404$
Disallow: /404.html$
Disallow: /401$
Disallow: /401.html$
Disallow: /*?edit=*
Disallow: /*?cms_editor=*
Disallow: /*?preview=*
Disallow: /*?nocache=*
Allow: /*?
Allow: /*=
Allow: /assets/
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /videos/
Allow: /favicon.ico
Allow: /robots.txt
Allow: /sitemap.xml

Sitemap: https://www.fitbudd.com/sitemap.xml