User-agent: *
Crawl-delay: 600  # Adjust the number of seconds as needed

# Disallow certain directories to reduce server load
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-json/
Disallow: /cgi-bin/

# Allow crawling of WordPress content that should be indexed
Allow: /wp-content/uploads/

# Specific rules for certain known bots (example: Facebook's crawler)
User-agent: facebookexternalhit/1.1
Crawl-delay: 20  # Adjust this if necessary
Disallow:

User-agent: meta-externalagent/1.1
Crawl-delay: 20  # Adjust this as needed
Disallow: