User-agent: *
Disallow: /search
Allow: /

Sitemap: https://www.wagdoll.co.uk/sitemap.xml


###### BLOCK AI CRAWLERS ######

# OpenAI ChatGPT / GPTBot
User-agent: GPTBot
Disallow: /

# OpenAI Training Crawler
User-agent: ChatGPT-User
Disallow: /

# OpenAI All-in-one Block
User-agent: OAI-SearchBot
Disallow: /

# Anthropic (Claude)
User-agent: Claude-Web
Disallow: /

# Google AI (Bard / Gemini expansion)
User-agent: Google-Extended
Disallow: /

# Perplexity AI
User-agent: PerplexityBot
Disallow: /

# Common AI / LLM scraping bots
User-agent: CCBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: aiGPTBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /