# General Crawlers - Allow access to public content
User-agent: *
Allow: /

# Block access to source and development files
Disallow: /src/
Disallow: /js/
Disallow: /*.json$
Disallow: /node_modules/
Disallow: /.*

# Allow important directories and files
Allow: /images/
Allow: /videos/
Allow: /css/
Allow: /sitemap.xml
Allow: /robots.txt

# Allow specific important files
Allow: /index.html
Allow: /contact.html
Allow: /index.css

# Sitemap locations for both domains
Sitemap: https://ascendedent.com/sitemap.xml
Sitemap: https://ascendedentertainment.com/sitemap.xml

# Crawl delay for respectful crawling
Crawl-delay: 1

# Search Engine Bots
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Social Media Bots
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# AI Training & Crawling Bots

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic / Claude
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

User-agent: Applebot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /

# Other AI Services
User-agent: Bytespider
Allow: /

User-agent: Diffbot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: AI2Bot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: Kangaroo Bot
Allow: /

User-agent: img2dataset
Allow: /

# Block common spam bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /