# EU AI Act Ready™ crawler policy

Sitemap: https://eu-ai-act-ready.eu/sitemap.xml


User-agent: *
Allow: /

Disallow: /private/
Disallow: /admin/
Disallow: /workspace/
Disallow: /api/
Disallow: /_worker/
Disallow: /functions/
Disallow: /tmp/
Disallow: /test/
Disallow: /draft/


# AI-readable public knowledge layer

Allow: /llms.txt
Allow: /ai.json
Allow: /authority.json
Allow: /entities.json
Allow: /intents.json
Allow: /adn.json
Allow: /ai-proof.json
Allow: /trust-state.json
Allow: /governance.json
Allow: /policy.json
Allow: /entity-graph.public.jsonld
Allow: /ai-signals.json
Allow: /ai-routes.json
Allow: /ai-validation.json
Allow: /signal-manifest.json
Allow: /signals/


# OpenAI

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /


# Google AI

User-agent: Google-Extended
Allow: /


# Anthropic Claude

User-agent: ClaudeBot
Allow: /


# Perplexity

User-agent: PerplexityBot
Allow: /


# Microsoft AI / Bing

User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /


# Meta AI

User-agent: Meta-ExternalAgent
Allow: /


# Amazon AI

User-agent: Amazonbot
Allow: /


# Apple AI

User-agent: Applebot
Allow: /


# Cohere

User-agent: cohere-ai
Allow: /


# You.com

User-agent: YouBot
Allow: /


# DeepSeek

User-agent: DeepSeekBot
Allow: /


# Research crawlers

User-agent: Diffbot
Allow: /

User-agent: Bytespider
Allow: /