User-agent: * Disallow: # ========================================================= # ThaverTech - Robots.txt # Purpose: SEO + AEO + GEO + AI Search Visibility # Website: https://www.thavertech.com/ # ========================================================= # --------------------------------------------------------- # Allow OpenAI / ChatGPT crawlers # --------------------------------------------------------- User-agent: GPTBot Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / # --------------------------------------------------------- # Allow Anthropic / Claude crawlers # --------------------------------------------------------- User-agent: ClaudeBot Allow: / User-agent: Claude-SearchBot Allow: / User-agent: Claude-User Allow: / User-agent: anthropic-ai Allow: / # --------------------------------------------------------- # Allow Google crawlers # --------------------------------------------------------- User-agent: Googlebot Allow: / User-agent: Google-Extended Allow: / # --------------------------------------------------------- # Allow Microsoft / Bing # --------------------------------------------------------- User-agent: Bingbot Allow: / # --------------------------------------------------------- # Allow Perplexity # --------------------------------------------------------- User-agent: PerplexityBot Allow: / # --------------------------------------------------------- # Allow Meta AI crawler # --------------------------------------------------------- User-agent: meta-externalagent Allow: / User-agent: Meta-ExternalFetcher Allow: / # --------------------------------------------------------- # Allow Apple AI crawler/control token # --------------------------------------------------------- User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / # --------------------------------------------------------- # Allow Common Crawl # --------------------------------------------------------- User-agent: CCBot Allow: / # --------------------------------------------------------- # Allow ByteDance AI crawler # --------------------------------------------------------- User-agent: Bytespider Allow: / # --------------------------------------------------------- # Allow Amazon crawler # --------------------------------------------------------- User-agent: Amazonbot Allow: / # --------------------------------------------------------- # General web crawlers # --------------------------------------------------------- User-agent: * Allow: / # --------------------------------------------------------- # Sitemap # --------------------------------------------------------- Sitemap: https://www.thavertech.com/sitemap.xml