{"slug":"agent402-tools-api-site-map-7b3787","title":"Discover a website's URLs in one call","host":"agent402.tools","method":"POST","resource":"https://agent402.tools/api/site-map","category":"search","description":"Discover a website's URLs in one call: reads robots.txt, its declared sitemap(s) (sitemap indexes and gzipped sitemaps included, /sitemap.xml as the fallback) and the start page's internal links, then returns a same-host, normalized, deduplicated list (up to 500) with an optional substring filter. H","price_listed":0.005,"price_asked":0.005,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":1062,"reported_calls_30d":4,"reported_payers_30d":1,"networks":["algorand:wGHE2Pwdvd7S12BL5FaOP20EGYesN73ktiC1qzkkit8=","eip155:10","eip155:1329","eip155:137","eip155:143","eip155:42161","eip155:42220","eip155:43114","eip155:4663","eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","stellar:pubnet"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"limit":50,"url":"https://www.iana.org"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"properties":{"includeSubdomains":{"description":"Also keep URLs on subdomains of the start site (default false; www and bare host always count as one site)","type":"boolean"},"limit":{"description":"Max URLs to return, 1-500 (default 100)","type":"integer"},"search":{"description":"Optional case-insensitive substring filter applied to the discovered URLs","type":"string"},"url":{"description":"Start URL (the site's homepage or any page on it)","type":"string"}},"required":["url"]},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"properties":{"fetchedAt":{"type":"string"},"fetches":{"type":"integer"},"host":{"type":"string"},"search":{},"sitemapsRead":{"type":"integer"},"source":{"type":"string"},"sources":{"type":"object"},"total":{"type":"integer"},"truncated":{"type":"boolean"},"url":{"type":"string"},"urls":{"type":"array"},"warnings":{"type":"array"}},"required":["url","host","total","urls","sources","sitemapsRead","truncated","fetches","warnings","source","fetchedAt"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-09-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.005,"price_match":true,"latency_ms":1062,"error":null}],"description_full":"Discover a website's URLs in one call: reads robots.txt, its declared sitemap(s) (sitemap indexes and gzipped sitemaps included, /sitemap.xml as the fallback) and the start page's internal links, then returns a same-host, normalized, deduplicated list (up to 500) with an optional substring filter. Hard budgets: at most 6 fetches, 15 seconds, 5 MB. Use it to pick which pages to crawl or extract next.","last_updated":"2026-09-22T11:52:55.847Z","schemes":["exact","upto"]}