User-agent: *
Allow: /
Disallow: /admin-console

Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Sitemaps
Sitemap: https://opencourtdata.uk/sitemap.xml
# General rules for all crawlers
User-agent: *
Allow: /
Disallow: /upload-judgment.html
Disallow: /admin-console

# AI crawlers - OpenAI
User-agent: GPTBot
Allow: /judgments/
Disallow: /upload-judgment.html
Disallow: /admin-console

User-agent: OAI-SearchBot
Allow: /judgments/
Disallow: /upload-judgment.html
Disallow: /admin-console

# AI crawlers - Anthropic
User-agent: Claude-Web
Allow: /judgments/
Disallow: /upload-judgment.html
Disallow: /admin-console

# AI crawlers - Google
User-agent: Google-Extended
Allow: /judgments/
Disallow: /upload-judgment.html
Disallow: /admin-console

# AI crawlers - Other common agents
User-agent: PerplexityBot
Allow: /judgments/
Disallow: /upload-judgment.html
Disallow: /admin-console

User-agent: YouBot
Allow: /judgments/
Disallow: /upload-judgment.html
Disallow: /admin-console

# Content usage preferences
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# Sitemaps
Sitemap: https://opencourtdata.uk/sitemap.xml