{"slug":"toolvend-dev-parse-robots-561ac8","title":"Fetch and parse robots.txt (+ llms.txt if present) into structured JSON, includi","host":"toolvend.dev","method":"GET","resource":"https://toolvend.dev/parse/robots","category":"other","description":"Fetch and parse robots.txt (+ llms.txt if present) into structured JSON, including which AI crawlers are blocked.","price_listed":0.005,"price_asked":0.005,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":20,"reported_calls_30d":1,"reported_payers_30d":1,"networks":["eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","queryParams":{"url":"https://example.com"},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"queryParams":{"properties":{"url":{"description":"Public HTTP(S) site origin or URL","type":"string"}},"required":["url"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-09-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.005,"price_match":true,"latency_ms":20,"error":null}],"description_full":"Fetch and parse robots.txt (+ llms.txt if present) into structured JSON, including which AI crawlers are blocked.","last_updated":"2026-09-07T16:07:04.556Z","schemes":["exact"]}