# =====================================================
# robots.txt — thecabinchiangmai.com
# Purpose: Index Cleanup + Crawl Budget Optimization
# Last updated: 2026-08-11
# =====================================================
# Edit via cPanel > File Manager > public_html/robots.txt
# Rank Math cannot write this file (DISALLOW_FILE_EDIT)
# =====================================================
# Change log (2026-08-11):
#   - /llms/*.md = AI answer-engine content only.
#     -> Search crawlers (Googlebot/Bingbot etc. via "User-agent: *")
#        are DISALLOWED from /llms/. Saves crawl budget; keeps .md
#        out of search.
#     -> Named AI crawlers get their OWN group that ALLOWS /llms/.
#   - No noindex/X-Robots-Tag is used on .md (it cannot work once
#     crawling is blocked). Index-safety therefore relies ENTIRELY on
#     Google never DISCOVERING a .md URL. Hard rules:
#         * never list .md in any XML sitemap
#         * never link .md via <link rel="alternate"> or any
#           Google-crawlable link
#         * .md discovery is via llms.txt ONLY
#     If Google ever finds a .md URL, it may show as a URL-only result.
#   - NOTE: robots groups do NOT inherit. The AI-bot group only obeys
#     its own rules, so any new "*" Disallow that must also bind AI
#     bots has to be duplicated into that group.
# Change log (2026-04-22):
#   - Removed 4 Disallow: /feed/ rules (.htaccess 301s feeds to parent)
#   - Removed Disallow: /video/ (.htaccess now returns 410 Gone)
#   - Temporarily removed Disallow: /*?* (specific ?s=, ?p= added)
#   - TODO (~2026-06-01): restore Disallow: /*?* after deindex
# =====================================================

User-agent: *

# --- Core WordPress Admin ---
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# --- Specific query-string blocks (temporary, while /*?* is off) ---
Disallow: /?s=
Disallow: /?p=
Disallow: /?replytocom=
Disallow: /search/

# --- PDF Files ---
Disallow: /*.pdf$

# --- LLM markdown files (/llms/) — search crawlers blocked ---
# For AI answer engines only (see AI-crawler group below).
# Keeps .md out of search + saves crawl budget.
Disallow: /llms/

# --- WP-JSON / REST API ---
Disallow: /wp-json/

# --- WordPress Internals ---
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/wpcf7_uploads/
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-cron.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-register.php
Disallow: /wp-trackback.php

# --- Trackbacks ---
Disallow: /trackback/
Disallow: /*/trackback/

# --- Author Archives (prevent user enumeration) ---
Disallow: /author/

# --- Embed Endpoints ---
Disallow: /embed/
Disallow: /*/embed/

# --- Allow CSS/JS/Images for page rendering ---
Allow: /wp-content/uploads/
Allow: /wp-content/themes/*.css
Allow: /wp-content/themes/*.js
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js

# =====================================================
# AI ANSWER-ENGINE CRAWLERS
# Self-contained group (does NOT inherit "*"). Allowed to read
# /llms/*.md. Security paths re-declared here because no inheritance.
# Add more bots as needed.
# =====================================================
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User

Allow: /llms/
Disallow: /wp-admin/
Disallow: /wp-json/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/

# =====================================================
# Sitemap (global; applies to all groups)
# =====================================================
Sitemap: https://www.thecabinchiangmai.com/sitemap_index.xml