{"slug":"intel-rallylive-ca-dev-ai-crawler-policy-2aa5b2","title":"AI-crawler policy of a website","host":"intel.rallylive.ca","method":"POST","resource":"https://intel.rallylive.ca/dev/ai-crawler-policy","category":"search","description":"AI-crawler policy of a website: reads robots.txt and reports, for 20 known AI/LLM crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, anthropic-ai, CCBot, Google-Extended, Applebot-Extended, PerplexityBot, Bytespider, Amazonbot, cohere-ai, meta-externalagent and more), whether the root is allo","price_listed":0.01,"price_asked":0.001,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":22,"reported_calls_30d":2,"reported_payers_30d":2,"networks":["eip155:8453","eip155:84532"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"domain":"nytimes.com"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"properties":{}},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-09-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.001,"price_match":false,"latency_ms":22,"error":null}],"description_full":"AI-crawler policy of a website: reads robots.txt and reports, for 20 known AI/LLM crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, anthropic-ai, CCBot, Google-Extended, Applebot-Extended, PerplexityBot, Bytespider, Amazonbot, cohere-ai, meta-externalagent and more), whether the root is allowed, blocked or unmentioned, plus an overall stance (open / selective / blocks-all-ai) and TDM/ai.txt hints. $0.01 per domain.","last_updated":"2026-09-13T10:27:04.811Z","schemes":["exact"]}