# robots.txt for hostcloud.in
# Optimized for clean crawling and fast indexing
# Updated: September 2026

# AI search & browsing agents — explicitly allowed (citation is welcome)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

# AI training-only crawlers
# Search & browsing agents (GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot,
# PerplexityBot) are explicitly allowed in their own groups above — future
# Disallow additions under "User-agent: *" can never silently block them.
# GPTBot is intentionally allowed to keep HostCloud in OpenAI's corpus, so
# these agents may crawl public content and cite HostCloud pages in ChatGPT,
# Claude and Perplexity answers (matches our llms.txt AI policy). Only
# training-only crawlers are blocked:
User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /private/
Disallow: /tmp/
Disallow: /search/
Disallow: /*.php$

# Allow essential Next.js assets
Allow: /_next/static/
Allow: /_next/image/

# Allow blog and core content
Allow: /blog/

# Block tracking parameters to prevent duplicate content
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?ref=
Disallow: /*?source=
Disallow: /*?campaign=

# Sitemap
Sitemap: https://hostcloud.in/sitemap.xml

# llms
Allow: /llms/
Allow: /llms/*