{"slug":"web-openverbs-com-v1-robots-2476de","title":"Fetch a site's robots.txt and evaluate crawl permissions","host":"web.openverbs.com","method":"POST","resource":"https://web.openverbs.com/v1/robots","category":"search","description":"Fetch a site's robots.txt and evaluate crawl permissions: given a URL (plus optional extra paths) and a user-agent, return whether each path is allowed or disallowed, the rule that matched, the user-agent group, the crawl-delay and any declared sitemaps. Implements the Robots Exclusion Protocol (RFC","price_listed":0.006,"price_asked":0.006,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":449,"reported_calls_30d":0,"reported_payers_30d":0,"networks":["eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"paths":["example"],"url":"https://example.com","userAgent":"example"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"additionalProperties":false,"properties":{"paths":{"description":"Optional extra paths (or same-origin URLs) to check in the same call, e.g. [\"/admin\", \"/search?q=x\"]. Up to 50.","items":{"maxLength":2048,"minLength":1,"type":"string"},"maxItems":50,"type":"array"},"url":{"description":"Public http(s) URL to check. Its origin's /robots.txt is fetched and its path is the first path evaluated.","format":"uri","maxLength":2048,"type":"string"},"userAgent":{"description":"User-agent token to evaluate rules for, e.g. \"Googlebot\". Defaults to \"*\".","maxLength":200,"minLength":1,"type":"string"}},"required":["url"],"type":"object"},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-04","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.006,"price_match":true,"latency_ms":449,"error":null}],"description_full":"Fetch a site's robots.txt and evaluate crawl permissions: given a URL (plus optional extra paths) and a user-agent, return whether each path is allowed or disallowed, the rule that matched, the user-agent group, the crawl-delay and any declared sitemaps. Implements the Robots Exclusion Protocol (RFC 9309) with longest-match-wins, Allow-over-Disallow tie-breaking and * / $ wildcards.","last_updated":"2026-10-04T13:06:08.403Z","schemes":["exact"]}