{"slug":"x402-agentharbor-agency-ai-crawlers-6cd48d","title":"Given a website domain, reports which AI crawlers may fetch its home page under ","host":"x402.agentharbor.agency","method":"GET","resource":"https://x402.agentharbor.agency/ai-crawlers","category":"other","description":"Given a website domain, reports which AI crawlers may fetch its home page under its robots.txt (RFC 9309): GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, PerplexityBot, Google-Extended, Applebot-Extended, CCBot and more, each allowed or blocked, plus Googlebot, listed sitemaps, wh","price_listed":0.01,"price_asked":0.01,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":208,"reported_calls_30d":3,"reported_payers_30d":3,"networks":["eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","queryParams":{"domain":"agentharbor.agency"},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET","HEAD","DELETE"],"type":"string"},"queryParams":{"additionalProperties":false,"properties":{"domain":{"description":"Bare public hostname, e.g. example.com","maxLength":253,"pattern":"^[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?(\\.[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+$","type":"string"}},"required":["domain"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-18","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.01,"price_match":true,"latency_ms":208,"error":null}],"description_full":"Given a website domain, reports which AI crawlers may fetch its home page under its robots.txt (RFC 9309): GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, PerplexityBot, Google-Extended, Applebot-Extended, CCBot and more, each allowed or blocked, plus Googlebot, listed sitemaps, whether /llms.txt is published and page-level noindex/nosnippet, with fixes. Static check, no browser. Not charged if robots.txt cannot be read (no response, or a bot block).","last_updated":"2026-10-15T10:18:39.378Z","schemes":["exact"]}