# Robots.txt for Make My Book
# Allow all well-behaved crawlers

User-agent: *
Allow: /

# Disallow legacy URLs with query parameters (these pages no longer exist)
Disallow: /pricing?lang=*
Disallow: /help?lang=*
Disallow: /?lang=*
Disallow: /blog?lang=*
Disallow: /story-themes?lang=*
Disallow: /unique-gifts?lang=*
Disallow: /features?lang=*
Disallow: /example-book/*?lang=*
Disallow: /how-it-works?lang=*
Disallow: /illustration-styles?lang=*
Disallow: /privacy?lang=*
Disallow: /terms?lang=*

# Disallow private areas - Login pages
Disallow: /login
Disallow: /he/login
Disallow: /en/login
Disallow: /es/login
Disallow: /fr/login
Disallow: /de/login
Disallow: /it/login

# Disallow private areas - Profile pages
Disallow: /profile
Disallow: /he/profile
Disallow: /en/profile
Disallow: /es/profile
Disallow: /fr/profile
Disallow: /de/profile
Disallow: /it/profile

# Disallow private areas - Console pages
Disallow: /console/
Disallow: /he/console/
Disallow: /en/console/
Disallow: /es/console/
Disallow: /fr/console/
Disallow: /de/console/
Disallow: /it/console/

# Disallow private areas - Create pages
Disallow: /console/create
Disallow: /he/console/create
Disallow: /en/console/create
Disallow: /es/console/create
Disallow: /fr/console/create
Disallow: /de/console/create
Disallow: /it/console/create

# Disallow private areas - Subscription/Payment pages
Disallow: /console/subscription
Disallow: /he/console/subscription
Disallow: /en/console/subscription
Disallow: /es/console/subscription
Disallow: /fr/console/subscription
Disallow: /de/console/subscription
Disallow: /it/console/subscription

# Disallow private areas - Book pages (user-specific content)
Disallow: /book
Disallow: /he/book
Disallow: /en/book
Disallow: /es/book
Disallow: /fr/book
Disallow: /de/book
Disallow: /it/book

# Disallow private areas - Pricing pages (internal pricing)
Disallow: /pricing_page
Disallow: /he/pricing_page
Disallow: /en/pricing_page
Disallow: /es/pricing_page
Disallow: /fr/pricing_page
Disallow: /de/pricing_page
Disallow: /it/pricing_page

# Disallow API and admin areas
Disallow: /api/
Disallow: /admin/
Disallow: /_/
Disallow: /firebase-debug.log
Disallow: /firestore-debug.log

# Allow important files
Allow: /sitemap.xml
Allow: /sitemapindex.xml
Allow: /humans.txt
Allow: /ads.txt
Allow: /.well-known/
Allow: /manifest.json
Allow: /sw.js
Allow: /browserconfig.xml
Allow: /google-site-verification.html

# Special instructions for different bots
User-agent: Googlebot
Crawl-delay: 1
Allow: /

User-agent: Bingbot
Crawl-delay: 5
Allow: /

User-agent: facebookexternalhit
Allow: /

# AI Bot Access - Allow AI bots to access content for training and responses
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

User-agent: CCBot
Allow: /
Crawl-delay: 1

User-agent: ClaudeBot
Allow: /
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: Google-Extended
Allow: /
Crawl-delay: 1

User-agent: Omgilibot
Allow: /
Crawl-delay: 1

# Sitemap locations
Sitemap: https://makemybook.app/sitemap.xml
Sitemap: https://makemybook.app/sitemapindex.xml
Sitemap: https://blog.makemybook.app/sitemap.xml
Sitemap: https://blog.makemybook.app/en/sitemap.xml
Sitemap: https://blog.makemybook.app/he/sitemap.xml
Sitemap: https://blog.makemybook.app/es/sitemap.xml
Sitemap: https://blog.makemybook.app/fr/sitemap.xml
Sitemap: https://blog.makemybook.app/de/sitemap.xml
Sitemap: https://blog.makemybook.app/it/sitemap.xml

# Allow crawling of anchor links for better SEO
Allow: /he/features
Allow: /en/features
Allow: /es/features
Allow: /fr/features
Allow: /de/features
Allow: /it/features

Allow: /he/how-it-works
Allow: /en/how-it-works
Allow: /es/how-it-works
Allow: /fr/how-it-works
Allow: /de/how-it-works
Allow: /it/how-it-works

Allow: /he/story-themes
Allow: /en/story-themes
Allow: /es/story-themes
Allow: /fr/story-themes
Allow: /de/story-themes
Allow: /it/story-themes

Allow: /he/illustration-styles
Allow: /en/illustration-styles
Allow: /es/illustration-styles
Allow: /fr/illustration-styles
Allow: /de/illustration-styles
Allow: /it/illustration-styles

Allow: /he/pricing
Allow: /en/pricing
Allow: /es/pricing
Allow: /fr/pricing
Allow: /de/pricing
Allow: /it/pricing

Allow: /he/faq
Allow: /en/faq
Allow: /es/faq
Allow: /fr/faq
Allow: /de/faq
Allow: /it/faq

Allow: /he/unique-gifts
Allow: /en/unique-gifts
Allow: /es/unique-gifts
Allow: /fr/unique-gifts
Allow: /de/unique-gifts
Allow: /it/unique-gifts

# Contact for crawler issues
# Contact: info@makemybook.app