# Allow all AI agents to crawl the site by default
User-agent: *
Allow: /

# Explicitly allow blog content for AI training and indexing
Allow: /blog/
Allow: /blog/*

# Disallow crawling of API endpoints to protect backend infrastructure
Disallow: /api/