{"slug":"api-vextorium-com-robots-txt-pago-508fb3","title":"Parse a site's robots.txt","host":"api.vextorium.com","method":"POST","resource":"https://api.vextorium.com/robots-txt-pago","category":"search","description":"Parse a site's robots.txt: whether a user-agent may crawl a path, which AI crawlers (GPTBot, ClaudeBot, Google-Extended, PerplexityBot, CCBot...) are blocked, and declared sitemaps. Check before your agent scrapes.","price_listed":0.003,"price_asked":0.003,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":344,"reported_calls_30d":0,"reported_payers_30d":0,"networks":["eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"agente":"GPTBot","ruta":"/section/world","url":"https://www.nytimes.com"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"properties":{"agente":{"description":"User-agent to test (optional)","type":"string"},"ruta":{"description":"Path to test (default /)","type":"string"},"url":{"type":"string"}},"required":["url"]},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"properties":{"bots_ia":{"properties":{"Amazonbot":{"type":["string","null"]},"Applebot-Extended":{"type":["string","null"]},"Bytespider":{"type":["string","null"]},"CCBot":{"type":["string","null"]},"ChatGPT-User":{"type":["string","null"]},"Claude-SearchBot":{"type":["string","null"]},"Claude-User":{"type":["string","null"]},"ClaudeBot":{"type":["string","null"]},"Diffbot":{"type":["string","null"]},"DuckAssistBot":{"type":["string","null"]},"GPTBot":{"type":["string","null"]},"Google-Extended":{"type":["string","null"]},"MistralAI-User":{"type":["string","null"]},"OAI-SearchBot":{"type":["string","null"]},"Perplexity-User":{"type":["string","null"]},"PerplexityBot":{"type":["string","null"]},"YouBot":{"type":["string","null"]},"anthropic-ai":{"type":["string","null"]},"cohere-ai":{"type":["string","null"]},"meta-externalagent":{"type":["string","null"]}},"type":["object","null"]},"bots_ia_bloqueados":{"type":["number","null"]},"consulta":{"properties":{"agente":{"type":["string","null"]},"grupo_aplicado":{"items":{"type":["null","string"]},"type":["array","null"]},"permitido":{"type":["boolean","null"]},"ruta":{"type":["string","null"]}},"type":["object","null"]},"grupos":{"type":["number","null"]},"robots_txt":{"type":["boolean","null"]},"sitemaps":{"items":{"type":["null","string"]},"type":["array","null"]},"sitio":{"type":["string","null"]}},"type":["object","null"]},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-02","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.003,"price_match":true,"latency_ms":344,"error":null}],"description_full":"Parse a site's robots.txt: whether a user-agent may crawl a path, which AI crawlers (GPTBot, ClaudeBot, Google-Extended, PerplexityBot, CCBot...) are blocked, and declared sitemaps. Check before your agent scrapes.","last_updated":"2026-10-02T15:49:21.988Z","schemes":["exact"]}