User-agent: *

# BLOCK: Admin & platform internals
Disallow: /conceptxx/
Disallow: /admin/
Disallow: /gumbo/admin/

# BLOCK: User account / private pages
Disallow: /gumbo/user/
Disallow: /gumbo/checkout/
Disallow: /gumbo/login.cfm
Disallow: /gumbo/forgot_password.cfm
Disallow: /login/
Disallow: /user-dashboard/

# BLOCK: Partial/fragment includes only — NEVER block /gumbo/home/ itself.
# The homepage canonical is "/" and "/" 301s into /gumbo/home/, so disallowing
# that folder told Google not to crawl the homepage. Duplicate consolidation is
# already handled by the 301 + canonical, not by robots.
Disallow: /gumbo/home/incs/

# BLOCK: Functional/utility pages
Disallow: /index.cfm
Disallow: /gumbo/index.cfm
Disallow: /gumbo/follow/
Disallow: /calendar.cfm
Disallow: /catering/startreg.cfm
# NOTE: /gumbo/contactus.cfm was blocked here until 2026-07-17. Search Console
# showed it earning 679 impressions at avg position 7.8 while disallowed. A
# contact page carries NAP (name/address/phone) and is a strong local-SEO and
# AI-citation asset — blocking it was self-defeating. Now crawlable.

# BLOCK: Review gate - functional page, not indexable content
Disallow: /review/

# BLOCK: Dynamic pages with infinite URL variation
Disallow: /gumbo/menu/index.cfm?
Disallow: /gumbo/events/?

# ALLOW: Core public pages
Allow: /
Allow: /gumbo/home/
Allow: /gumbo/menu/
Allow: /menu/
Allow: /gumbo/reservations/
Allow: /gumbo/experiences/
Allow: /gumbo/events/
Allow: /gumbo/rewards/
Allow: /gumbo/updates/
Allow: /radio/
Allow: /sitemap.xml


# ============================================================
# AI SEARCH / ANSWER ENGINES
# ============================================================
# We WANT to be cited by AI assistants — a growing share of "best BBQ in Baton
# Rouge" / "where should I eat" questions get answered there now.
#
# CRITICAL robots.txt RULE: a bot obeys ONLY the most specific User-agent group
# that matches it. Naming a bot here means it IGNORES the "User-agent: *" block
# above ENTIRELY — including every Disallow in it. So each group below MUST
# repeat the protected paths, or naming a bot would silently expose /conceptxx/,
# customer accounts and checkout to it. Keep these Disallow lines in sync with
# the "*" block above.
#
# Honest note on what this does and does NOT control:
#  - Google AI Overviews / AI Mode run on normal Googlebot. There is NO way to
#    opt in or out of AI Overviews separately. Google-Extended below governs
#    Gemini/Vertex grounding ONLY — not AI Overviews.
#  - These bots would already be allowed via "*". Listing them is an explicit
#    opt-in so a future blanket block upstream doesn't silently cut us out.

# --- OpenAI: ChatGPT browsing + ChatGPT Search ---
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# --- Google: Gemini / Vertex grounding (NOT AI Overviews) ---
User-agent: Google-Extended
# --- Perplexity ---
User-agent: PerplexityBot
User-agent: Perplexity-User
# --- Anthropic: Claude ---
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: anthropic-ai
# --- Apple Intelligence / Siri ---
User-agent: Applebot-Extended
# --- Common Crawl (feeds many LLM datasets) ---
User-agent: CCBot
# --- Meta AI ---
User-agent: meta-externalagent
Disallow: /conceptxx/
Disallow: /admin/
Disallow: /gumbo/admin/
Disallow: /gumbo/user/
Disallow: /gumbo/checkout/
Disallow: /gumbo/login.cfm
Disallow: /gumbo/forgot_password.cfm
Disallow: /login/
Disallow: /user-dashboard/
Disallow: /gumbo/home/incs/
Disallow: /index.cfm
Disallow: /gumbo/index.cfm
Disallow: /gumbo/follow/
Disallow: /calendar.cfm
Disallow: /catering/startreg.cfm
Disallow: /review/
Allow: /

Sitemap: https://www.thesmokeypit.com/sitemap.xml