# VisitDundee
# Official destination guide for Dundee, Scotland.
# Public visitor content may be indexed for search and discovery.

# General search crawlers
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /portal/
Disallow: /business/onboarding
Disallow: /business/submissions
Disallow: /business-claims/
Disallow: /login
Disallow: /register
Disallow: /auth/
Disallow: /password/
Disallow: /email/
Disallow: /sanctum/
Disallow: /webhooks/

# Avoid indexing internal search result combinations
Disallow: /search?

# Avoid crawling temporary or signed URLs
Disallow: /*?token=
Disallow: /*&token=
Disallow: /*?signature=
Disallow: /*&signature=
Disallow: /*?code=
Disallow: /*&code=

# OpenAI search/discovery crawler
User-agent: OAI-SearchBot
Allow: /

# Anthropic search/discovery crawler
User-agent: Claude-SearchBot
Allow: /

# Perplexity search/discovery crawler
User-agent: PerplexityBot
Allow: /

# AI model training / bulk data collection
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Canonical public content discovery
Sitemap: https://visitdundee.com/sitemap.xml