{"slug":"pennyregwatch-com-v1-robots-476136","title":"Decide whether a specific crawler may fetch an exact public URL under robots.txt","host":"pennyregwatch.com","method":"GET","resource":"https://pennyregwatch.com/v1/robots","category":"search","description":"Decide whether a specific crawler may fetch an exact public URL under robots.txt rules. Returns ALLOW/DENY, the winning rule, crawl-delay, sitemap URLs, and live provenance. Use before an AI agent or crawler fetches a page.","price_listed":0.001,"price_asked":0.001,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":1438,"reported_calls_30d":3,"reported_payers_30d":3,"networks":["eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","queryParams":{"target":"https://example.com/private/page","user_agent":"GPTBot"},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET","HEAD","DELETE"],"type":"string"},"queryParams":{"properties":{"target":{"description":"Exact public page URL or hostname to test","type":"string"},"user_agent":{"default":"*","description":"Crawler product token, e.g. GPTBot, ClaudeBot, Googlebot","type":"string"}},"required":["target"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-09-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.001,"price_match":true,"latency_ms":1438,"error":null}],"description_full":"Decide whether a specific crawler may fetch an exact public URL under robots.txt rules. Returns ALLOW/DENY, the winning rule, crawl-delay, sitemap URLs, and live provenance. Use before an AI agent or crawler fetches a page.","last_updated":"2026-09-17T16:35:25.142Z","schemes":["exact"]}