# Keep public pages available for search and AI discovery.
# Security is enforced by WAF/Fail2Ban, not robots.txt.

# High-volume cooperative crawlers may crawl every public page and image, but
# should pace requests. Crawler-specific groups do not inherit the wildcard
# exclusions, so repeat the non-public and exploit-path rules here.
User-agent: ClaudeBot
User-agent: meta-externalagent
User-agent: DotBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: SemrushBot
User-agent: serpstatbot
Crawl-delay: 1
Allow: /
Disallow: /admin/
Disallow: /logout
Disallow: /api/
Disallow: /.env
Disallow: /vendor/
Disallow: /var/
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /*.php$

User-agent: *
Allow: /

# Non-public or non-content endpoints.
Disallow: /admin/
Disallow: /logout
Disallow: /api/
Disallow: /.env
Disallow: /vendor/
Disallow: /var/

# Exploit/probe paths with no SEO value.
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /*.php$