{"slug":"tanod-dev-v1-robots-e90b64","title":"sitepeek (Tanod) robots.txt check","host":"tanod.dev","method":"POST","resource":"https://tanod.dev/v1/robots","category":"search","description":"sitepeek (Tanod) robots.txt check: whether a crawler may fetch a URL under RFC 9309, with the matching rule, crawl delay and sitemaps. Read-only.","price_listed":0.001,"price_asked":0.001,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":217,"reported_calls_30d":2,"reported_payers_30d":2,"networks":["eip155:137","eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"url":"https://www.google.com/search","user_agent":"Googlebot"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"additionalProperties":false,"properties":{"url":{"description":"Public http(s) URL to test; <origin>/robots.txt is fetched and the URL's path is matched against it.","maxLength":2048,"minLength":1,"type":"string"},"user_agent":{"default":"*","description":"Product token (e.g. Googlebot) or a full User-Agent string; '*' (default) = any crawler.","maxLength":200,"minLength":1,"type":"string"}},"required":["url"],"type":"object"},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"properties":{"allowed":{"type":"boolean"},"crawl_delay":{"type":["number","string","null"]},"fetch":{"enum":["ok","unavailable","unreachable"],"type":"string"},"group":{"type":["string","null"]},"http_status":{"type":["integer","null"]},"matched_rule":{"properties":{"line":{"type":"integer"},"pattern":{"type":"string"},"type":{"enum":["allow","disallow"],"type":"string"}},"type":["object","null"]},"path":{"type":"string"},"product_token":{"type":"string"},"reason":{"type":"string"},"robots_url":{"type":["string","null"]},"sitemaps":{"items":{"type":"string"},"type":"array"},"sitemaps_total":{"type":"integer"},"stats":{"type":["object","null"]},"truncated":{"type":"boolean"},"untrusted_content":{"type":"boolean"},"url":{"type":"string"},"user_agent":{"type":"string"}},"required":["url","robots_url","user_agent","product_token","path","http_status","fetch","allowed","matched_rule","group","crawl_delay","sitemaps","sitemaps_total","truncated","stats","reason","untrusted_content"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-10","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.001,"price_match":true,"latency_ms":217,"error":null}],"description_full":"sitepeek (Tanod) robots.txt check: whether a crawler may fetch a URL under RFC 9309, with the matching rule, crawl delay and sitemaps. Read-only.","last_updated":"2026-10-08T22:46:00.651Z","schemes":["exact"]}