{"slug":"agentsvc-io-api-v1-proxy-site-crawl-894bd2","title":"Website crawler","host":"agentsvc.io","method":"POST","resource":"https://agentsvc.io/api/v1/proxy/site-crawl","category":"company","description":"Website crawler: read a whole website or one section of it (for example every page under /docs) as clean Markdown: starts at a URL, follows same-site links breadth-first and returns up to 20 pages with title and description. Use for documentation, product catalogs or company sites. For one page use ","price_listed":0.005,"price_asked":0.005,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":885,"reported_calls_30d":10,"reported_payers_30d":3,"networks":["eip155:137","eip155:42161","eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"max_chars":2000,"max_pages":3,"url":"https://agentsvc.io/docs"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"properties":{"max_chars":{"default":4000,"maximum":20000,"minimum":500,"type":"integer"},"max_pages":{"default":5,"maximum":20,"minimum":1,"type":"integer"},"path_prefix":{"description":"Only follow links whose path starts with this, e.g. /docs","type":"string"},"url":{"type":"string"}},"required":["url"]},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST","PUT","PATCH"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"properties":{"data":{"properties":{"crawled_at":{"type":"string"},"discovered_urls":{"type":"integer"},"ok_count":{"type":"integer"},"pages":{"items":{"properties":{"description":{"type":"string"},"markdown":{"type":"string"},"ok":{"type":"boolean"},"status":{"type":"integer"},"title":{"type":"string"},"url":{"type":"string"},"word_count":{"type":"integer"}},"type":"object"},"type":"array"},"pages_crawled":{"type":"integer"},"site":{"type":"string"},"skipped_non_html":{"type":"integer"},"took_ms":{"type":"integer"}},"required":["site","pages_crawled","ok_count","pages"],"type":"object"},"success":{"type":"boolean"}},"required":["success","data"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-10","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.005,"price_match":true,"latency_ms":885,"error":null}],"description_full":"Website crawler: read a whole website or one section of it (for example every page under /docs) as clean Markdown: starts at a URL, follows same-site links breadth-first and returns up to 20 pages with title and description. Use for documentation, product catalogs or company sites. For one page use web-scrape, for a known list of URLs web-read-batch. Charged only if at least one page was read.","last_updated":"2026-10-10T09:02:28.067Z","schemes":["exact"]}