# No Disallow rules by design.
#
# robots.txt controls CRAWLING, not INDEXING. A disallowed URL that Google
# already knows about stays in the index as a URL-only entry with no title or
# description, which is what Search Console reports as "Indexed, though blocked
# by robots.txt". Blocking also stops Google from seeing a page's noindex tag,
# canonical, or redirect target.
#
# To keep a page OUT of search results, set `noindex: true` in its front matter
# (layouts/base.njk emits the meta tag) and `sitemap: false`. Do not add it here.
User-agent: *
Allow: /

# AI crawlers: explicitly permitted, content is intended for AI citation.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

Sitemap: https://www.borong.com/sitemap.xml