{"slug":"bess-x402-onrender-com-v1-web-robots-7fc7e0","title":"A site's robots.txt rules per user agent, whether AI crawlers (GPTBot, ClaudeBot","host":"bess-x402.onrender.com","method":"GET","resource":"https://bess-x402.onrender.com/v1/web/robots","category":"other","description":"A site's robots.txt rules per user agent, whether AI crawlers (GPTBot, ClaudeBot, Google-Extended, CCBot, PerplexityBot) are blocked, and the URLs listed in its sitemap. Use before crawling a site or for SEO audits.","price_listed":0.002,"price_asked":0.002,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":199,"reported_calls_30d":2,"reported_payers_30d":2,"networks":["eip155:137","eip155:42161","eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","queryParams":{"domain":"stripe.com","limit":20},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET","HEAD","DELETE"],"type":"string"},"queryParams":{"properties":{"domain":{"description":"Domain name","type":"string"},"limit":{"description":"Sitemap URLs to return (1-1000)","type":"integer"}},"required":["domain"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.002,"price_match":true,"latency_ms":199,"error":null}],"description_full":"A site's robots.txt rules per user agent, whether AI crawlers (GPTBot, ClaudeBot, Google-Extended, CCBot, PerplexityBot) are blocked, and the URLs listed in its sitemap. Use before crawling a site or for SEO audits.","last_updated":"2026-10-23T13:46:29.209Z","schemes":["exact"]}