{"slug":"api-vextorium-com-v1-robots-txt-6cb6b8","title":"Parse a site's robots.txt","host":"api.vextorium.com","method":"POST","resource":"https://api.vextorium.com/v1/robots-txt","category":"search","description":"Parse a site's robots.txt: whether a user-agent may crawl a path, which AI crawlers (GPTBot, ClaudeBot, Google-Extended, PerplexityBot, CCBot...) are blocked, and declared sitemaps. Check before your agent scrapes.","price_listed":0.003,"price_asked":0.003,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":316,"reported_calls_30d":1,"reported_payers_30d":1,"networks":["eip155:10","eip155:5042","eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"agent":"GPTBot","route":"/section/world","url":"https://www.nytimes.com"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"properties":{"agent":{"description":"User-agent to test (optional)","type":"string"},"route":{"description":"Path to test (default /)","type":"string"},"url":{"type":"string"}},"required":["url"]},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"properties":{"ai_bots":{"properties":{"Amazonbot":{"type":["string","null"]},"Applebot-Extended":{"type":["string","null"]},"Bytespider":{"type":["string","null"]},"CCBot":{"type":["string","null"]},"ChatGPT-User":{"type":["string","null"]},"Claude-SearchBot":{"type":["string","null"]},"Claude-User":{"type":["string","null"]},"ClaudeBot":{"type":["string","null"]},"Diffbot":{"type":["string","null"]},"DuckAssistBot":{"type":["string","null"]},"GPTBot":{"type":["string","null"]},"Google-Extended":{"type":["string","null"]},"MistralAI-User":{"type":["string","null"]},"OAI-SearchBot":{"type":["string","null"]},"Perplexity-User":{"type":["string","null"]},"PerplexityBot":{"type":["string","null"]},"YouBot":{"type":["string","null"]},"anthropic-ai":{"type":["string","null"]},"cohere-ai":{"type":["string","null"]},"meta-externalagent":{"type":["string","null"]}},"type":["object","null"]},"blocked_ai_bots":{"type":["number","null"]},"groups":{"type":["number","null"]},"query":{"properties":{"agent":{"type":["string","null"]},"allowed":{"type":["boolean","null"]},"applied_group":{"items":{"type":["null","string"]},"type":["array","null"]},"route":{"type":["string","null"]}},"type":["object","null"]},"robots_txt":{"type":["boolean","null"]},"site":{"type":["string","null"]},"sitemaps":{"items":{"type":["null","string"]},"type":["array","null"]}},"type":["object","null"]},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-04","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.003,"price_match":true,"latency_ms":316,"error":null}],"description_full":"Parse a site's robots.txt: whether a user-agent may crawl a path, which AI crawlers (GPTBot, ClaudeBot, Google-Extended, PerplexityBot, CCBot...) are blocked, and declared sitemaps. Check before your agent scrapes.","last_updated":"2026-10-03T20:44:09.56Z","schemes":["exact"]}