# Hentai AI Chat — public crawl policy
# Updated: 2026-10-07
# All public content is accessible without a login or entry wall.
# These rules permit crawling by agents that honour robots.txt.
# User-initiated retrieval and extended control tokens follow provider semantics.
# Permission does not guarantee indexing, citation or model inclusion.

User-agent: *
Allow: /

# Google: Search indexing, including Google Search features
User-agent: Googlebot
Allow: /

# Google: Image indexing and image-dependent search features
User-agent: Googlebot-Image
Allow: /

# Google: Video indexing and video search features
User-agent: Googlebot-Video
Allow: /

# Google: Gemini training and Gemini/Vertex grounding data-use control; not a separate HTTP crawler
User-agent: Google-Extended
Allow: /

# Microsoft: Bing search indexing; AI usage also governed by documented page-level controls
User-agent: bingbot
Allow: /

# Apple: Search indexing and answer context; collected data may also be used for training
User-agent: Applebot
Allow: /

# Apple: Foundation-model training data-use control; does not crawl pages
User-agent: Applebot-Extended
Allow: /

# OpenAI: ChatGPT search indexing
User-agent: OAI-SearchBot
Allow: /

# OpenAI: User-requested retrieval; robots.txt may not apply
User-agent: ChatGPT-User
Allow: /

# OpenAI: Collection potentially used for foundation-model training
User-agent: GPTBot
Allow: /

# Anthropic: Search indexing and relevance
User-agent: Claude-SearchBot
Allow: /

# Anthropic: User-requested retrieval; honors robots.txt per operator documentation
User-agent: Claude-User
Allow: /

# Anthropic: Collection potentially contributing to model training
User-agent: ClaudeBot
Allow: /

# Perplexity: Search indexing; not foundation-model training
User-agent: PerplexityBot
Allow: /

# Perplexity: User-requested retrieval; generally ignores robots.txt; not foundation-model training
User-agent: Perplexity-User
Allow: /

# Amazon: Product/service improvement; data may train Amazon AI models
User-agent: Amazonbot
Allow: /

# Amazon: Amazon search experiences; not generative-model training
User-agent: Amzn-SearchBot
Allow: /

# Amazon: User-requested live retrieval; may not follow all robots.txt directives; not generative-model training
User-agent: Amzn-User
Allow: /

# Common Crawl: Open web corpus for research/analysis and potential downstream model training
User-agent: CCBot
Allow: /

Sitemap: https://hentai-ai-chat.org/sitemap.xml