{"generatedAt":"2026-07-09T00:25:03.414Z","siteUrl":"https://hurtadvice.com","robotsUrl":"https://hurtadvice.com/robots.txt","note":"Public crawler readiness definitions only. This endpoint does not expose private request logs.","readiness":{"responseHeaders":["X-Discoverability-Crawler","X-Discoverability-Crawler-Type"],"optionalLoggingEnvironmentVariable":"CRAWLER_OBSERVABILITY_LOG","optionalLoggingEnabled":false,"categories":{"search":3,"ai-search":5,"ai-training":6,"ai-agent":4},"probeUserAgents":["Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)","Mozilla/5.0 (compatible; OAI-SearchBot/1.0)","Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0","Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)","Claude-SearchBot/1.0"]},"crawlers":[{"name":"Googlebot","category":"search","purpose":"Google Search crawling and rendering"},{"name":"Bingbot","category":"search","purpose":"Bing Search crawling and IndexNow recrawl validation"},{"name":"Applebot","category":"search","purpose":"Apple search and Siri web results"},{"name":"OAI-SearchBot","category":"ai-search","purpose":"OpenAI search retrieval and citation surfaces"},{"name":"ChatGPT-User","category":"ai-agent","purpose":"User-triggered ChatGPT browsing and agent requests"},{"name":"GPTBot","category":"ai-training","purpose":"OpenAI model training crawler governed by robots.txt"},{"name":"OAI-AdsBot","category":"ai-agent","purpose":"OpenAI ad landing-page validation crawler for submitted ChatGPT ads"},{"name":"Claude-Web","category":"ai-search","purpose":"Legacy Anthropic Claude web retrieval crawler token retained for compatibility"},{"name":"ClaudeBot","category":"ai-training","purpose":"Anthropic model training crawler governed by robots.txt"},{"name":"Claude-User","category":"ai-agent","purpose":"User-triggered Claude browsing and agent requests"},{"name":"Claude-SearchBot","category":"ai-search","purpose":"Anthropic Claude search retrieval and search-result quality crawler"},{"name":"PerplexityBot","category":"ai-search","purpose":"Perplexity answer-engine retrieval crawler"},{"name":"Perplexity-User","category":"ai-agent","purpose":"User-triggered Perplexity browsing and answer requests"},{"name":"YouBot","category":"ai-search","purpose":"You.com answer-engine retrieval crawler"},{"name":"Google-Extended","category":"ai-training","purpose":"Google AI training crawler governed by robots.txt"},{"name":"Applebot-Extended","category":"ai-training","purpose":"Apple AI training crawler governed by robots.txt"},{"name":"CCBot","category":"ai-training","purpose":"Common Crawl crawler used by search and AI datasets"},{"name":"Meta-ExternalAgent","category":"ai-training","purpose":"Meta external AI crawler governed by robots.txt"}]}