{"slug":"agent-web-tools-agent-web-tools-workers-dev-v1-robots-c4676d","title":"May this agent crawl this URL?","host":"agent-web-tools.agent-web-tools.workers.dev","method":"GET","resource":"https://agent-web-tools.agent-web-tools.workers.dev/v1/robots","category":"search","description":"May this agent crawl this URL? robots.txt verdict per RFC 9309.","price_listed":0.001,"price_asked":0.001,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":381,"reported_calls_30d":0,"reported_payers_30d":0,"networks":["eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","queryParams":{"agent":"GPTBot","url":"https://example.com"},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"queryParams":{"properties":{"agent":{"description":"Your crawler's user-agent token (default *)","type":"string"},"url":{"description":"Public http(s) URL (ports 80/443)","type":"string"}},"required":["url"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-26","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.001,"price_match":true,"latency_ms":381,"error":null}],"description_full":"May this agent crawl this URL? robots.txt verdict per RFC 9309.","last_updated":"2026-10-26T15:44:56.573Z","schemes":["exact"]}