# ==============================================================================
# robots.txt | Diamond Power Washers | diamondpowerwashers.com
# Optimized for: SEO + AEO (Answer Engine Optimization) + GEO (Generative
# Engine Optimization) | Last Updated: July 2026
# ==============================================================================
#
# HOW THIS FILE WORKS (critical robots.txt rule):
#   A crawler obeys ONLY the single most-specific User-agent group that
#   matches it, and IGNORES the * group entirely. That means a bot with its
#   own "Allow: /" group would skip every Disallow in the * group and crawl
#   /wp-admin/, search results, etc.
#
#   So this file has exactly two kinds of bots:
#   ALLOWED bots  -> get NO group at all. They fall back to * and inherit
#                    the WordPress/thin-content protections below.
#   BLOCKED bots  -> get their own group with "Disallow: /".
#
#   Never add an "Allow: /" group for a bot you want to allow.
#
# INTENTIONALLY ALLOWED (no group needed — they follow *):
#   Search:      Googlebot, Googlebot-Image, Bingbot, BingPreview,
#                DuckDuckBot, Applebot, YouBot, Yandex, Baiduspider
#   AI answers/citations (AEO/GEO): OAI-SearchBot, ChatGPT-User, ClaudeBot,
#                Claude-SearchBot, Claude-User, PerplexityBot,
#                Perplexity-User, DuckAssistBot, Meta-ExternalFetcher,
#                Google-Extended (Gemini grounding — NOT AI Overviews;
#                AI Overviews are part of Search and use Googlebot)
#
# CONTENT SIGNALS:
#   ai-input=yes  -> AI may READ content for real-time answers (AEO/GEO).
#   ai-train=no   -> AI may NOT use content to train foundation models.
#   Content-Signal is a Cloudflare-backed proposal (contentsignals.org) —
#   only honored by participating crawlers; the Disallow groups below are
#   the enforcement layer.
# ==============================================================================


# ==============================================================================
# GLOBAL RULES — every allowed bot inherits these
# ==============================================================================

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-json/
Disallow: /wp-content/uploads/wpo/
Disallow: /?s=
Disallow: /search/
Disallow: /page/
Disallow: /tag/
Disallow: /author/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /comments/


# ==============================================================================
# GOOGLE — special cases only
# Googlebot / Googlebot-Image / Google-Extended: allowed via * (no group).
# Storebot-Google -> Google Shopping; no value for a service business.
# ==============================================================================

User-agent: Storebot-Google
Disallow: /


# ==============================================================================
# APPLE
# Applebot (Siri/Spotlight/Safari + Apple Intelligence search citations):
#   allowed via * — no group here, on purpose.
# Applebot-Extended -> Apple foundation-model TRAINING opt-out only.
#   Blocking it does NOT remove us from Siri or Apple Intelligence answers
#   (those use Applebot). Blocked to enforce ai-train=no.
# ==============================================================================

User-agent: Applebot-Extended
Disallow: /


# ==============================================================================
# AI TRAINING CRAWLERS — BLOCK
# These scrape content to train foundation models. Zero ranking or citation
# value in return. Blocking these is what makes ai-train=no real.
# The user-triggered fetchers that DO produce citations (ChatGPT-User,
# Claude-User, Perplexity-User, Meta-ExternalFetcher, OAI-SearchBot,
# ClaudeBot, PerplexityBot) are allowed via * above.
# ==============================================================================

# OpenAI model training (OAI-SearchBot / ChatGPT-User stay allowed)
User-agent: GPTBot
Disallow: /

# Common Crawl — bulk corpus used for open AI training datasets
User-agent: CCBot
Disallow: /

# ByteDance / TikTok training scraper
User-agent: Bytespider
Disallow: /

# Meta AI model training (Meta-ExternalFetcher, the user-initiated
# fetcher that powers Meta AI citations, stays allowed via *)
User-agent: Meta-ExternalAgent
Disallow: /

# Amazon — Alexa/model training; negligible local-service search value
User-agent: Amazonbot
Disallow: /

# Allen Institute for AI training crawler
User-agent: AI2Bot
Disallow: /

# Cohere AI training
User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

# Diffbot — content scraper feeding AI training pipelines
User-agent: Diffbot
Disallow: /

# Timpi / Webz.io / ImageSift — dataset scrapers
User-agent: Timpibot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# Huawei PanGu training
User-agent: PanguBot
Disallow: /

# Cloudflare internal browser renderer (not a search/answer bot)
User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

# NOTE: ia_archiver (Internet Archive / Wayback Machine) is deliberately
# NOT blocked — it is an archive, not an AI trainer, and archive presence
# is a useful provenance/entity signal.


# ==============================================================================
# SEO TOOL CRAWLERS — BLOCK (COMPETITOR INTELLIGENCE PROTECTION)
# Prevents competitors from auditing our structure, keyword targets, and
# internal linking via paid tools. Our own SEMrush position tracking and
# GSC-based reporting still work (API + Google data, not these crawlers).
# ==============================================================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: MauiBot
Disallow: /


# ==============================================================================
# KNOWN BAD / SPAM CRAWLERS — BLOCK
# ==============================================================================

User-agent: SiteAuditBot
Disallow: /

User-agent: CheckMarkNetwork
Disallow: /

User-agent: GrapeshotCrawler
Disallow: /

User-agent: FullStoryBot
Disallow: /


# ==============================================================================
# SITEMAP DECLARATION
# ==============================================================================

Sitemap: https://diamondpowerwashers.com/sitemap_index.xml
User-agent: *
Disallow: /wp-content/uploads/wpo/wpo-plugins-tables-list.json