# robots.txt - RuDao.org
#
# Every search engine and AI crawler is welcome on the book itself.
# The only closed doors are the author's private chat archive and the
# API endpoints behind it.
#
# NOTE on structure: all crawlers below share ONE rule set, written as
# consecutive User-agent lines. That matters - a crawler that finds its
# own named group ignores every other group, so a Disallow placed under
# "User-agent: *" would never reach Googlebot. Keep them together.

User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Baiduspider
User-agent: Bingbot
User-agent: YandexBot
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: CCBot
User-agent: AI2Bot
User-agent: Amazonbot
User-agent: *
Allow: /

# /sitemap and the three public-domain Chinese classics at /ch/lunyu,
# /ch/daodejing and /ch/artofwar are public and covered by "Allow: /".

# Private: the author's chat archive and every function endpoint
Disallow: /admin-chat-log
Disallow: /admin-chat-log.html
Disallow: /api/

# Sitemap
# Only rudao.org - declaring the pages.dev / github.io mirrors here would
# point crawlers at duplicate content and dilute the canonical domain.
Sitemap: https://rudao.org/sitemap.xml