# Robots.txt for Business Directory
# Updated for SEO and AI Optimization (AIO)

# Default rules for all crawlers
User-agent: *
Disallow: /backoffice
Disallow: /backoffice/*
Disallow: /profile
Disallow: /profile/*
Disallow: /validate-g-auth
Disallow: /report-review
Disallow: /checkout/*
Disallow: /signup
Disallow: /signin
Disallow: /update-password
Disallow: /ajax/*
Disallow: /pages/get-a-quote
Disallow: /*?page=*
Disallow: /*?q=*

# Allow important paths for SEO
Allow: /offices/*
Allow: /business/*
Allow: /articles/*
Allow: /services/*

# Crawl delay to prevent server overload
Crawl-delay: 1

# Google specific rules
User-agent: Googlebot
Allow: /
Disallow: /backoffice
Disallow: /profile
Disallow: /ajax/*

# Bing specific rules
User-agent: Bingbot
Allow: /
Disallow: /backoffice
Disallow: /profile
Disallow: /ajax/*
Crawl-delay: 2

# AI Crawlers - Control access for AI training
# GPTBot (OpenAI)
User-agent: GPTBot
Allow: /offices/
Allow: /business/
Allow: /articles/
Allow: /services/
Disallow: /backoffice
Disallow: /profile
Disallow: /ajax/*
Disallow: /checkout/*

# ChatGPT User Agent
User-agent: ChatGPT-User
Allow: /offices/
Allow: /business/
Allow: /articles/
Allow: /services/
Disallow: /backoffice
Disallow: /profile
Disallow: /ajax/*

# Claude Web (Anthropic)
User-agent: Claude-Web
Allow: /offices/
Allow: /business/
Allow: /articles/
Allow: /services/
Disallow: /backoffice
Disallow: /profile
Disallow: /ajax/*

# Google Bard / Gemini
User-agent: Google-Extended
Allow: /offices/
Allow: /business/
Allow: /articles/
Allow: /services/
Disallow: /backoffice
Disallow: /profile
Disallow: /ajax/*

# CCBot (Common Crawl - used by many AI models)
User-agent: CCBot
Allow: /offices/
Allow: /business/
Allow: /articles/
Disallow: /backoffice
Disallow: /profile
Disallow: /ajax/*
Crawl-delay: 5

# Perplexity AI
User-agent: PerplexityBot
Allow: /offices/
Allow: /business/
Allow: /articles/
Disallow: /backoffice
Disallow: /profile
Disallow: /ajax/*

# Block known bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Sitemap location (per-domain, absolute)
Sitemap: https://www.floristsbook.co.uk/sitemap_floristsbook.xml