# ===========================================================
# robots.txt for joshivaibhav.com
# Last updated: March 2026
# Managed via: Yoast SEO (WordPress)
# Purpose: Allow all search engines + AI crawlers
# ===========================================================

# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow:

User-agent: AdsBot-Google
Disallow: /

User-agent: AdsBot-Google-Mobile
Disallow: /

Sitemap: https://joshivaibhav.com/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK

# ===========================================================
# AI CRAWLER ACCESS — Added manually below Yoast block
# Allows content to be cited by ChatGPT, Gemini, Perplexity,
# Grok, Claude, Bing Copilot, and other AI engines
# ===========================================================

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

# OpenAI — ChatGPT browsing plugin
User-agent: ChatGPT-User
Allow: /

# Google — Gemini & AI Overviews
User-agent: Google-Extended
Allow: /

# Anthropic — Claude AI
User-agent: ClaudeBot
Allow: /

# Common Crawl — used by many AI training datasets
User-agent: CCBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Apple — used by Apple Intelligence
User-agent: Applebot
Allow: /

# Applebot Extended — Apple AI training
User-agent: Applebot-Extended
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# You.com AI search
User-agent: YouBot
Allow: /