User-agent: *
Allow: /

# Per AGENTS.md §9 + the LLM-crawler discoverability rule:
# the public marketing surface is open. The /api, /dashboard, /admin
# and /login routes are intentionally absent from this site
# (R220 marketing-only cut, R221 unchanged) so no Disallow lines
# are needed for them.

Sitemap: https://www.webhost24.in/sitemap-index.xml

# AI training crawlers: explicit allowlist for the well-known
# LLM discovery surfaces. These crawlers respect robots.txt
# for opt-in/opt-out semantics.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Bytespider
Allow: /

# Host: webhost24.in