{"slug":"agent402-tools-api-site-crawl-24ce10","title":"Crawl a website from a starting URL and return each page as clean markdown","host":"agent402.tools","method":"POST","resource":"https://agent402.tools/api/site-crawl","category":"search","description":"Crawl a website from a starting URL and return each page as clean markdown: breadth-first over internal links, bounded by page count and depth, honouring robots.txt, with per-page title, status, depth and outbound links. Use it when an agent needs a whole section of a site rather than one known page","price_listed":0.02,"price_asked":0.02,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":948,"reported_calls_30d":3,"reported_payers_30d":1,"networks":["algorand:wGHE2Pwdvd7S12BL5FaOP20EGYesN73ktiC1qzkkit8=","eip155:10","eip155:1329","eip155:137","eip155:143","eip155:42161","eip155:42220","eip155:43114","eip155:4663","eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","stellar:pubnet"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"limit":3,"maxDepth":1,"url":"https://example.com"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"properties":{"excludePatterns":{"description":"Never follow links whose URL contains any of these substrings (max 20)","items":{"type":"string"},"type":"array"},"format":{"description":"Page content format (default markdown)","enum":["markdown","text"],"type":"string"},"includePatterns":{"description":"Only follow links whose URL contains at least one of these substrings (max 20)","items":{"type":"string"},"type":"array"},"limit":{"description":"Max pages to fetch, 1-20 (default 10); failed fetches count toward it","type":"integer"},"maxCharsPerPage":{"description":"Cap on content characters per page, 200-20000 (default 8000)","type":"integer"},"maxDepth":{"description":"Link depth from the start URL, 0-2 (default 1)","type":"integer"},"sameHost":{"description":"true (default): stay on the start host (www and bare host count as one); false: also follow subdomains of the start site","type":"boolean"},"url":{"description":"Start URL","type":"string"}},"required":["url"]},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-09-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.02,"price_match":true,"latency_ms":948,"error":null}],"description_full":"Crawl a website from a starting URL and return each page as clean markdown: breadth-first over internal links, bounded by page count and depth, honouring robots.txt, with per-page title, status, depth and outbound links. Use it when an agent needs a whole section of a site rather than one known page.","last_updated":"2026-09-22T11:53:40.01Z","schemes":["exact","upto"]}