# robots.txt for anshulrana.in
# Anshul Rana - SEO, AEO & GEO Expert

# One group, many agents. Consecutive User-agent lines form a single record, so
# every crawler named here gets the identical rule set below. This matters:
# a crawler only obeys the MOST SPECIFIC group that names it, so a bot with its
# own group ignores "User-agent: *" entirely. Splitting these out into separate
# per-agent groups meant the Disallow lines under * never applied to Googlebot.

# Standard search engine crawlers
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
# AI crawlers - explicitly allowed for AEO and GEO visibility
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
Allow: /
# SSI fragments: served only as internal subrequests, never as pages
Disallow: /includes/
# POST-only form and tool endpoints. They answer a GET with a 4xx, which
# Search Console files under "Blocked due to other 4xx issue". Nothing here is
# a page, so keeping crawlers out of them is the fix.
Disallow: /audit-submit.php
Disallow: /tools/tool-report.php
Disallow: /tools/proxy.php
# Non-public paths
Disallow: /private/
Disallow: /admin/
Disallow: /tmp/

# Block poor-reputation crawlers with high crawl-budget cost
User-agent: Bytespider
Disallow: /

# General fallback - same rules as the named crawlers above
User-agent: *
Allow: /
Disallow: /includes/
Disallow: /audit-submit.php
Disallow: /tools/tool-report.php
Disallow: /tools/proxy.php
Disallow: /private/
Disallow: /admin/
Disallow: /tmp/

# Sitemap
# Only real XML sitemaps belong in a Sitemap: directive. llms.txt was listed
# here previously; it is markdown, so every fetch failed XML parsing. It lives
# at https://anshulrana.in/llms.txt and is linked from the site instead.
Sitemap: https://anshulrana.in/sitemap.xml