{"slug":"shop-lordevo-com-api-quick-can-fetch-d3c6c1","title":"Can I fetch this URL?","host":"shop.lordevo.com","method":"GET","resource":"https://shop.lordevo.com/api/quick/can-fetch","category":"search","description":"Can I fetch this URL? robots.txt + AI-crawler permission gate: Pre-fetch gate for agents and scrapers: is this exact URL allowed by robots.txt (RFC 9309: missing = allow, 5xx/unreachable = disallow) for generic agents and 16 named AI agents/crawlers (ChatGPT-User, Claude-User, PerplexityBot, GPTBot,","price_listed":0.01,"price_asked":0.01,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":313,"reported_calls_30d":0,"reported_payers_30d":0,"networks":["eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","queryParams":{"url":"https://example.com"},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"queryParams":{"properties":{"url":{"description":"public http(s) URL or bare domain","type":"string"}},"required":["url"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"properties":{"agents":{"additionalProperties":{"type":"boolean"},"type":"object"},"allowed":{"type":"boolean"},"content_signal":{"type":["object","null"]},"crawl_delay_seconds":{"type":["number","null"]},"noai":{"type":"boolean"},"origin":{"type":"string"},"page":{"type":["object","null"]},"path":{"type":"string"},"robots_http_status":{"type":["integer","null"]},"robots_txt":{"enum":["present","missing","unreachable"],"type":"string"},"sitemaps":{"items":{"type":"string"},"type":"array"},"summary":{"type":"string"},"tool":{"type":"string"},"url":{"type":"string"},"verdict":{"enum":["allowed","disallowed","allowed_but_no_ai_use","allowed_with_crawl_delay"],"type":"string"}},"required":["tool","url","verdict","allowed","agents","robots_txt"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-26","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.01,"price_match":true,"latency_ms":313,"error":null}],"description_full":"Can I fetch this URL? robots.txt + AI-crawler permission gate: Pre-fetch gate for agents and scrapers: is this exact URL allowed by robots.txt (RFC 9309: missing = allow, 5xx/unreachable = disallow) for generic agents and 16 named AI agents/crawlers (ChatGPT-User, Claude-User, PerplexityBot, GPTBot,","last_updated":"2026-10-26T17:40:08.756Z","schemes":["exact"]}