# robots.txt for dedicatted.com
# Physical file at /opt/bitnami/wordpress/robots.txt (takes precedence over the
# virtual one WordPress/AIOSEO would serve). Versioned here so it survives an
# instance rebuild - see ops/README.md.
#
# IMPORTANT robots.txt semantics: a crawler obeys ONLY its most specific matching
# User-agent group and does NOT inherit "User-agent: *". The previous version gave
# each AI crawler a group containing just "Allow: /", so they were the only agents
# permitted into /wp-admin/ and the thin taxonomy archives that are deliberately
# excluded from everyone else. Each group below therefore repeats the Disallow set.
#
# Crawl-delay was also dropped entirely. The site actively courts AI citation
# (articles carry "Read summarized version with ChatGPT / Claude / Perplexity"
# buttons), so throttling those exact agents to one request per 5s worked against
# the goal. The origin sits behind CloudFront at ~0.2 load and does not need the
# protection; crawlers we do not want are blocked outright below rather than slowed.

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /industry/automotive
Disallow: /industry/financial-services
Disallow: /industry/healthcare
Disallow: /industry/manufacturing
Disallow: /industry/retail
Disallow: /industry/high-tech-software
Disallow: /industry/insurance
Disallow: /format/blog
Disallow: /format/case-study
Disallow: /format/whitepaper
Allow: /services/
Allow: /industries/

# --- AI assistants and AI search. Full access to real content, no throttle, but
# --- kept out of wp-admin and the thin archive pages.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Bingbot
User-agent: CCBot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: Bytespider
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /industry/automotive
Disallow: /industry/financial-services
Disallow: /industry/healthcare
Disallow: /industry/manufacturing
Disallow: /industry/retail
Disallow: /industry/high-tech-software
Disallow: /industry/insurance
Disallow: /format/blog
Disallow: /format/case-study
Disallow: /format/whitepaper
Allow: /services/
Allow: /industries/
Allow: /

# --- SEO scrapers and crawlers with no upside for this site.
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Baiduspider
Disallow: /

Sitemap: https://dedicatted.com/sitemap.xml