{"slug":"aayatai-com-crawl-5c5f15","title":"Turn a small website, or one section of it, into clean Markdown","host":"aayatai.com","method":"GET","resource":"https://aayatai.com/crawl","category":"company","description":"Turn a small website, or one section of it, into clean Markdown: starts from your URL, finds pages from the sitemap or by following links on the same site, obeys robots.txt, and returns up to 10 pages (title, Markdown, address) in one call. Great for docs, blogs and company sites. ?url=https://examp","price_listed":0.03,"price_asked":0.03,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":20,"reported_calls_30d":1,"reported_payers_30d":1,"networks":["eip155:137","eip155:42161","eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","queryParams":{"maxPages":2,"url":"https://example.org"},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"queryParams":{"properties":{"maxPages":{"default":5,"description":"Most pages to return (1-10).","maximum":10,"minimum":1,"type":"integer"},"path":{"description":"Only crawl pages under this path (default: the start page's folder).","maxLength":300,"type":"string"},"url":{"description":"Start page, e.g. https://example.com/docs/.","format":"uri","maxLength":2048,"type":"string"}},"required":["url"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"properties":{"complete":{"description":"False if more matching pages were found than returned.","type":"boolean"},"count":{"type":"integer"},"crawledAt":{"type":"string"},"discoveredFrom":{"enum":["sitemap","links"],"type":"string"},"pages":{"items":{"properties":{"chars":{"type":"integer"},"markdown":{"type":"string"},"title":{"type":"string"},"truncated":{"type":"boolean"},"url":{"type":"string"}},"type":"object"},"type":"array"},"robots":{"description":"robots.txt status: ok, none or unreachable.","type":"string"},"skipped":{"description":"Pages not read and why (robots.txt, error, not HTML).","items":{"type":"object"},"type":"array"},"start":{"type":"string"}},"required":["start","pages","count","discoveredFrom","robots","complete"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-09-29","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.03,"price_match":true,"latency_ms":20,"error":null}],"description_full":"Turn a small website, or one section of it, into clean Markdown: starts from your URL, finds pages from the sitemap or by following links on the same site, obeys robots.txt, and returns up to 10 pages (title, Markdown, address) in one call. Great for docs, blogs and company sites. ?url=https://example.com/docs/&maxPages=10&path=/docs/","last_updated":"2026-09-29T00:02:34.181Z","schemes":["exact"]}