# My Tool Studio - robots.txt
# Allow the public site and all tools. Hold research content (/blog, /insights)
# back from crawling for now. Block AJAX API and Next internals.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /blog/
Disallow: /insights/
Crawl-delay: 2

User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /blog/
Disallow: /insights/
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /blog/
Disallow: /insights/
Crawl-delay: 1

User-agent: GPTBot
Allow: /tools/
Disallow: /api/
Disallow: /_next/
Disallow: /blog/
Disallow: /insights/
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /tools/
Disallow: /api/
Disallow: /_next/
Disallow: /blog/
Disallow: /insights/
Crawl-delay: 2

User-agent: OAI-SearchBot
Allow: /tools/
Disallow: /api/
Disallow: /_next/
Disallow: /blog/
Disallow: /insights/
Crawl-delay: 2

User-agent: ClaudeBot
Allow: /tools/
Disallow: /api/
Disallow: /_next/
Disallow: /blog/
Disallow: /insights/
Crawl-delay: 2

User-agent: PerplexityBot
Allow: /tools/
Disallow: /api/
Disallow: /_next/
Disallow: /blog/
Disallow: /insights/
Crawl-delay: 2

User-agent: Google-Extended
Allow: /tools/
Disallow: /api/
Disallow: /_next/
Disallow: /blog/
Disallow: /insights/

Host: https://mytoolstudio.com
Sitemap: https://mytoolstudio.com/sitemap.xml
Llms: https://mytoolstudio.com/llms.txt