{"slug":"netintel-dev-robots-txt-analyze-7ded6a","title":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-age","host":"netintel.dev","method":"GET","resource":"https://netintel.dev/robots-txt/analyze","category":"search","description":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate sitemaps before scraping.","price_listed":0.01,"price_asked":0.01,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":941,"reported_calls_30d":1,"reported_payers_30d":1,"networks":["eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","queryParams":{"domain":"example.com"},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"queryParams":{"properties":{"domain":{"description":"Domain name to fetch robots.txt from (e.g. example.com)","type":"string"},"path":{"description":"URL path to check permission for (e.g. /api/data)","type":"string"},"user_agent":{"description":"Bot name to check rules for (default: *)","type":"string"}},"required":["domain"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"properties":{"domain":{"type":"string"},"findings":{"items":{"type":"object"},"type":"array"},"found":{"type":"boolean"},"grade":{"type":"string"},"path_check":{"type":"object"},"raw_content_preview":{"type":"string"},"robots_url":{"type":"string"},"rules":{"items":{"type":"object"},"type":"array"},"score":{"type":"number"},"sitemaps":{"items":{"type":"string"},"type":"array"},"status_code":{"type":"number"}},"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-09-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.01,"price_match":true,"latency_ms":941,"error":null}],"description_full":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate sitemaps before scraping.","last_updated":"2026-09-19T20:43:05.992Z","schemes":["exact"]}