# robots.txt for rattlesnakesolutions.com
# Updated: December 2, 2024

# Sitemap location
Sitemap: https://rattlesnakesolutions.com/sitemap.xml

# Allow all search engines by default
User-agent: *

# Block sensitive/admin directories
Disallow: /crotalus/
Disallow: /_scripts/
Disallow: /_downloads/
Disallow: /_ex/
Disallow: /_common/

# Block WordPress admin areas
Disallow: /fieldherper/wp-admin/
Disallow: /fieldherper/wp-includes/
Disallow: /fieldherper/wp-content/plugins/
Disallow: /fieldherper/wp-content/themes/
Disallow: /fieldherper/wp-content/cache/
Disallow: /fieldherper/wp-content/upgrade/

Disallow: /snakeblog/wp-admin/
Disallow: /snakeblog/wp-includes/
Disallow: /snakeblog/wp-content/plugins/
Disallow: /snakeblog/wp-content/themes/
Disallow: /snakeblog/wp-content/cache/
Disallow: /snakeblog/wp-content/upgrade/

# Allow WordPress uploads (images, etc.)
Allow: /fieldherper/wp-content/uploads/
Allow: /snakeblog/wp-content/uploads/

# Block WordPress login and registration pages
Disallow: /fieldherper/wp-login.php
Disallow: /fieldherper/wp-signup.php
Disallow: /snakeblog/wp-login.php
Disallow: /snakeblog/wp-signup.php

# Block query strings and common WordPress parameters
Disallow: /*?*replytocom=
Disallow: /*?s=
Disallow: /*/feed/
Disallow: /*/trackback/
Disallow: /*/embed/

# Block file types
Disallow: /*.log$
Disallow: /*.json$
Disallow: /*.md$
Disallow: /*.sql$
Disallow: /*.zip$
Disallow: /*.gz$

# Block specific files
Disallow: /wp-config.php
Disallow: /.htaccess
Disallow: /readme.html
Disallow: /license.txt

# Block development/promo subdirectories
Disallow: /promo/

# Crawl-delay for aggressive bots (helps reduce server load)
Crawl-delay: 1

# Specific rules for common bots

# Google - allow everything except blocked above
User-agent: Googlebot
Disallow: /crotalus/
Disallow: /_scripts/
Disallow: /_downloads/
Disallow: /_ex/
Disallow: /_common/
Disallow: /fieldherper/wp-admin/
Disallow: /snakeblog/wp-admin/
Allow: /fieldherper/wp-content/uploads/
Allow: /snakeblog/wp-content/uploads/

# Bing - same as Google
User-agent: Bingbot
Disallow: /crotalus/
Disallow: /_scripts/
Disallow: /_downloads/
Disallow: /_ex/
Disallow: /_common/
Disallow: /fieldherper/wp-admin/
Disallow: /snakeblog/wp-admin/
Allow: /fieldherper/wp-content/uploads/
Allow: /snakeblog/wp-content/uploads/

# Block bad bots and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# Block AI scrapers (optional - comment out if you want AI training)
User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Allow Google Images (for image search)
User-agent: Googlebot-Image
Allow: /_images/
Allow: /fieldherper/wp-content/uploads/
Allow: /snakeblog/wp-content/uploads/
Disallow: /crotalus/
Disallow: /_scripts/

# Allow Googlebot Video (for video content)
User-agent: Googlebot-Video
Allow: /

# End of robots.txt