User-agent: *

# Core WordPress folders (block backend access)
Disallow: /wp-admin/
Disallow: /wp-includes/
Allow: /wp-admin/admin-ajax.php

# Plugin directory (hide structure but allow needed assets)
Disallow: /wp-content/plugins/
Allow: /wp-content/plugins/*.js
Allow: /wp-content/plugins/*.css

# Allow frontend JS/CSS site-wide (for proper rendering)
Allow: *.css
Allow: *.js

# Crawl waste - reduce crawl budget issues
Disallow: /?s=
Disallow: /search
Disallow: /comments
Disallow: /trackback/
Disallow: */trackback/
Disallow: */feed/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /wp-json/
Disallow: /404.php

# Sensitive public files
Disallow: /readme.html
Disallow: /license.txt

# Block log, dump, and backup files
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.zip$
Disallow: /*.tar$

# Aggressive or non-essential bots (these are not search engines)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Yandex
Disallow: /

# Honeypot trap (optional - must have /honeypot.html file on site)
User-agent: *
Disallow: /honeypot.html