{"slug":"minifetch-com-api-v1-x402-preflight-url-check-3ce43f","title":"Cheap robots.txt preflight before you call the Minifetch /extract and /run/seo-p","host":"minifetch.com","method":"GET","resource":"https://minifetch.com/api/v1/x402/preflight/url-check","category":"code","description":"Cheap robots.txt preflight before you call the Minifetch /extract and /run/seo-page-audit endpoints: returns whether the Minifetch user agent may fetch a target URL (allow/deny) plus its crawl-delay. If blocked or setting a crawl delay read https://minifetch.com/tutorials/unblock-minifetch . Checks ","price_listed":0.001,"price_asked":0.001,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":920,"reported_calls_30d":25,"reported_payers_30d":13,"networks":["eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","queryParams":{"url":"https://github.com"},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"queryParams":{"properties":{"fresh":{"description":"Optional. If true, bypasses the 24h robots.txt cache and re-fetches robots.txt for this check. Use after editing your robots.txt or crawl-delay (or adding/removing a Minifetch block) to confirm the change took effect.","type":"boolean"},"url":{"description":"The target URL to check against robots.txt","type":"string"}},"required":["url"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"properties":{"results":{"items":{"properties":{"data":{"properties":{"allowed":{"description":"Whether the minifetch user agent may fetch this URL per robots.txt","type":"boolean"},"crawlDelay":{"description":"Crawl-delay (seconds) for the minifetch UA; omitted when blocked or the domain is invalid","type":"number"},"message":{"description":"e.g. 'Access allowed by robots.txt', 'URL is blocked by robots.txt', or 'robots.txt not found, defaulting to allowed'","type":"string"},"minifetchCache":{"description":"robots.txt cache state (~24h TTL; pass fresh=true to bypass)","properties":{"cachedAt":{"type":"string"},"expiresAt":{"type":"string"},"hit":{"type":"boolean"}},"type":"object"},"proxy":{"description":"Fetch provenance — always minifetch UA here","properties":{"proxyType":{"enum":["minifetch"],"type":"string"},"userAgent":{"type":"string"}},"type":"object"},"url":{"description":"The URL checked, exactly as given — redirects are not followed","type":"string"}},"required":["url","allowed","message","proxy"],"type":"object"}},"type":"object"},"type":"array"},"success":{"type":"boolean"},"tip":{"description":"Present only when allowed=false: how site owners can unblock the minifetch user agent","properties":{"message":{"type":"string"},"url":{"type":"string"}},"type":"object"}},"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-09-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.001,"price_match":true,"latency_ms":920,"error":null}],"description_full":"Cheap robots.txt preflight before you call the Minifetch /extract and /run/seo-page-audit endpoints: returns whether the Minifetch user agent may fetch a target URL (allow/deny) plus its crawl-delay. If blocked or setting a crawl delay read https://minifetch.com/tutorials/unblock-minifetch . Checks the URL as given — does not follow redirects. Supports GET & POST","last_updated":"2026-09-22T19:22:31.677Z","schemes":["exact"]}