{"slug":"api-avedaris-com-v1-paid-robots-inspect-a7fcc8","title":"Fetch and parse a public site's robots.txt into crawl directives and sitemap hin","host":"api.avedaris.com","method":"POST","resource":"https://api.avedaris.com/v1/paid/robots-inspect","category":"search","description":"Fetch and parse a public site's robots.txt into crawl directives and sitemap hints. $0.003 USDC per call on Base.","price_listed":0.003,"price_asked":0.003,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":27,"reported_calls_30d":1,"reported_payers_30d":1,"networks":["eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"url":"https://example.com/"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"additionalProperties":false,"properties":{"url":{"description":"Public http(s) URL on the site to check; the site's /robots.txt is fetched and parsed into crawler groups and sitemaps. 8 to 8,192 characters.","maxLength":8192,"minLength":8,"type":"string"}},"required":["url"],"type":"object"},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"properties":{"result":{"description":"Output of robots.inspect. The fields below are the documented core; some responses add further detail fields.","properties":{"groups":{"description":"Parsed rule groups, each with userAgents, allow and disallow path lists. Empty when no robots.txt.","type":"array"},"robotsUrl":{"description":"The robots.txt URL that was fetched (site origin + /robots.txt).","type":"string"},"sitemaps":{"description":"Sitemap URLs declared in robots.txt.","type":"array"},"status":{"description":"HTTP status of the robots.txt fetch; 404 means the site has none.","type":"integer"}},"required":["robotsUrl","status","groups","sitemaps"],"type":"object"}},"required":["result"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.003,"price_match":true,"latency_ms":27,"error":null}],"description_full":"Fetch and parse a public site's robots.txt into crawl directives and sitemap hints. $0.003 USDC per call on Base.","last_updated":"2026-10-22T23:48:49.852Z","schemes":["exact"]}