# =================================================== # robots.txt # =================================================== # Standard WordPress directives User-agent: * Disallow: /wp-admin/ Disallow: /imunify-bot-check Allow: /wp-admin/admin-ajax.php # Content Signals: machine-readable AI-use preferences for any crawler # not explicitly named below (and reinforces intent for the named ones). # search = allow indexing/citations | ai-input = allow real-time AI answers # ai-train = do not use for model training | use=reference = index/excerpt/link back only Content-Signal: search=yes, ai-input=yes, ai-train=no, use=reference # XML Sitemap Sitemap: https://www.senatorinn.com/sitemap_index.xml # --------------------------------------------------- # AI Crawlers # --------------------------------------------------- # --- OpenAI --- # Training crawler - blocked (opt out of model training) User-agent: GPTBot Disallow: / # ChatGPT Search / citations - allowed User-agent: OAI-SearchBot Allow: / # User-initiated fetch (fires when a ChatGPT user asks about a specific page) - allowed User-agent: ChatGPT-User Allow: / # --- Anthropic --- # Training crawler - blocked (opt out of model training) User-agent: ClaudeBot Disallow: / # In-product search indexing (Claude's web search tool) - allowed User-agent: Claude-SearchBot Allow: / # User-initiated fetch (fires when a Claude user asks about a specific page) - allowed User-agent: Claude-User Allow: / # --- Perplexity --- # Search index / citations (not a training crawler) - allowed User-agent: PerplexityBot Allow: / # User-initiated fetch (fires when a Perplexity user asks about a specific page) - allowed User-agent: Perplexity-User Allow: / # --- Google AI --- # Gemini/Vertex AI training + grounding - blocked (does not affect normal Google Search or AI Overviews) User-agent: Google-Extended Disallow: / # --- Apple --- # Siri, Spotlight, Safari suggestions - allowed User-agent: Applebot Allow: / # Apple Intelligence / foundation model training - blocked User-agent: Applebot-Extended Disallow: / # --- Microsoft --- # Bing Search + Copilot share one index; blocking would remove both, so left allowed User-agent: Bingbot Allow: / # --- Meta --- # General AI/Llama training crawler - blocked (opt out of model training) User-agent: Meta-ExternalAgent Disallow: / # User-initiated fetch (fires when a Meta AI user asks about a specific page) - allowed User-agent: Meta-ExternalFetcher Allow: / # --- Amazon --- # General crawler, may be used to train Amazon AI models - blocked User-agent: Amazonbot Disallow: / # Alexa/Rufus search eligibility - explicitly does not train models - allowed User-agent: Amzn-SearchBot Allow: / # User-initiated fetch (fires when an Alexa/Rufus user asks about a specific page) - allowed User-agent: Amzn-User Allow: / # --------------------------------------------------- # Low-value scraper bots (blocked) # --------------------------------------------------- # Common Crawl - bulk dataset crawler used by many third-party AI training sets User-agent: CCBot Disallow: / # ByteDance's crawler - history of non-compliance with robots.txt norms User-agent: Bytespider Disallow: /