# robots.txt for unionhardware.co.zw
# Updated: July 2025

# Allow all legitimate search engines with reasonable crawl delay
User-agent: *
Sitemap: https://unionhardware.co.zw/sitemap_index.xml

# Block WordPress admin areas for security
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/

# Block common problematic paths
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /private/
Disallow: /*.sql$
Disallow: /*.log$

# Allow major search engines explicitly
User-agent: Googlebot
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Chatbot
Crawl-delay: 5
Disallow: /

User-agent: Bingbot
Disallow: /

User-agent: Slurp
Crawl-delay: 10
Disallow: /

# Block known problematic/aggressive bots
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Allow AI bots for better future visibility
User-agent: ChatGPT-User
Disallow: /

User-agent: OpenAI-Crawler
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Gemini-Web
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: YouBot
Disallow: /