{"slug":"api-weblens-dev-crawl-696cae","title":"Crawl a site and get clean markdown for every page, in one call","host":"api.weblens.dev","method":"POST","resource":"https://api.weblens.dev/crawl","category":"search","description":"Crawl a site and get clean markdown for every page, in one call. Same-host BFS with depth and page-budget limits, robots.txt honoured by default. $0.0015 per requested page (1-25).","price_listed":0.015,"price_asked":0.015,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":511,"reported_calls_30d":1,"reported_payers_30d":1,"networks":["eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"limit":10,"maxDepth":2,"url":"https://example.com"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"properties":{"exclude":{"default":[],"description":"Skip URLs whose path+query contains one of these","items":{"maxLength":200,"minLength":1,"type":"string"},"maxItems":20,"type":"array"},"include":{"default":[],"description":"Only crawl URLs whose path+query contains one of these","items":{"maxLength":200,"minLength":1,"type":"string"},"maxItems":20,"type":"array"},"limit":{"default":10,"description":"Page budget — you are charged per requested page","example":10,"maximum":25,"minimum":1,"type":"number"},"maxChars":{"default":8000,"description":"Per-page content character cap","maximum":50000,"minimum":500,"type":"number"},"maxDepth":{"default":2,"description":"Link depth from the start URL (0 = start page only)","example":2,"maximum":3,"minimum":0,"type":"number"},"respectRobots":{"default":true,"description":"Honour robots.txt (default true; disable only for sites you control)","type":"boolean"},"timeout":{"default":10000,"description":"Request timeout in milliseconds","maximum":30000,"minimum":5000,"type":"number"},"url":{"description":"The URL to process","example":"https://example.com","format":"uri","type":"string"}},"required":["url"],"type":"object"},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"properties":{"crawledAt":{"type":"string"},"pages":{"description":"Per-page results with url, depth, status, title, content, truncated, error","items":{"properties":{"content":{"description":"Page as clean markdown","type":"string"},"depth":{"type":"integer"},"error":{"type":"string"},"status":{"enum":["success","failed"],"type":"string"},"title":{"type":"string"},"truncated":{"type":"boolean"},"url":{"type":"string"}},"type":"object"},"type":"array"},"requestId":{"type":"string"},"summary":{"description":"crawled, successful, failed, discovered, limit, maxDepth, robotsRespected","properties":{"crawled":{"type":"integer"},"discovered":{"type":"integer"},"failed":{"type":"integer"},"limit":{"type":"integer"},"maxDepth":{"type":"integer"},"robotsRespected":{"type":"boolean"},"successful":{"type":"integer"}},"type":"object"},"url":{"type":"string"}},"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-18","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.015,"price_match":true,"latency_ms":511,"error":null}],"description_full":"Crawl a site and get clean markdown for every page, in one call. Same-host BFS with depth and page-budget limits, robots.txt honoured by default. $0.0015 per requested page (1-25).","last_updated":"2026-10-13T18:08:28.117Z","schemes":["exact"]}