# Global rules for all crawlers
User-agent: *
Allow: /
Disallow: /staff/

# Specific rules for OpenAI's GPTBot
User-agent: GPTBot
Allow: /*
Disallow: /staff/

# Specific rules for Anthropic's Claude
User-agent: anthropic-ai
Allow: /*
Disallow: /staff/

# Specific rules for Google's AI bot
User-agent: Google-Extended
Allow: /*
Disallow: /staff/

# Common Crawl bot
User-agent: CCBot
Allow: /*
Disallow: /staff/

# Add crawl delay to prevent overwhelming the server
Crawl-delay: 10

# Sitemap location
Sitemap: https://www.gdindex.net/sitemap.xml