# Brothers Capital — robots.txt
# CA & Law firm · Ghaziabad & Greater Noida
# Site: https://brotherscapital.in

# ----------------------------------------------------------------------
# Default rule — be open, but keep internal paths out of the index.
# ----------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /api/
Disallow: /_assets/
Disallow: /uploads/screenshots/

# ----------------------------------------------------------------------
# Search-engine crawlers we explicitly welcome.
# ----------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Yandex
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Applebot
Allow: /

# ----------------------------------------------------------------------
# Social preview / link-unfurl bots — needed for OG cards.
# ----------------------------------------------------------------------
User-agent: FacebookExternalHit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# ----------------------------------------------------------------------
# AI assistants — allowed when they crawl to ANSWER user queries
# (live retrieval / citations), not to silently train models.
# ----------------------------------------------------------------------
User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

# ----------------------------------------------------------------------
# AI training scrapers — disallowed. Our content (templates, guides,
# legal commentary) is not licensed for LLM training without consent.
# ----------------------------------------------------------------------
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

Sitemap: https://brotherscapital.in/sitemap-index.xml