{"slug":"crawl-preflight-postnov01-workers-dev-crawl-check-domain-fadcaa","title":"Crawl pre-flight for a domain","host":"crawl-preflight.postnov01.workers.dev","method":"GET","resource":"https://crawl-preflight.postnov01.workers.dev/crawl-check/:domain","category":"search","description":"Crawl pre-flight for a domain: will you be blocked, is content behind a paywall, and how much is actually there. Returns per-bot robots.txt verdicts for 16 AI crawlers, crawlable page count from sitemaps, detected CDN and paid-access signals (x402, TollBit), plus a fetch/skip recommendation. Call be","price_listed":0.02,"price_asked":0.02,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":345,"reported_calls_30d":2,"reported_payers_30d":1,"networks":["eip155:137","eip155:42161","eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","pathParams":{"domain":"sitepoint.com"},"queryParams":{},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"pathParams":{"properties":{"domain":{"description":"Bare domain to check, without scheme or path","type":"string"}},"required":["domain"],"type":"object"},"queryParams":{"properties":{},"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-09-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.02,"price_match":true,"latency_ms":345,"error":null}],"description_full":"Crawl pre-flight for a domain: will you be blocked, is content behind a paywall, and how much is actually there. Returns per-bot robots.txt verdicts for 16 AI crawlers, crawlable page count from sitemaps, detected CDN and paid-access signals (x402, TollBit), plus a fetch/skip recommendation. Call before spending requests on an unknown domain.","last_updated":"2026-08-31T09:40:26.676Z","schemes":["exact"]}