{"slug":"saylorinnovations-com-api-web-robots-check-34eb38","title":"Check whether a crawler may fetch a path under a robots.txt per RFC 9309","host":"saylorinnovations.com","method":"GET","resource":"https://saylorinnovations.com/api/web/robots-check","category":"search","description":"Check whether a crawler may fetch a path under a robots.txt per RFC 9309 - matched group and rule, crawl delay and sitemaps, for Googlebot, GPTBot or any agent","price_listed":0.01,"price_asked":0.01,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":537,"reported_calls_30d":0,"reported_payers_30d":0,"networks":["eip155:137","eip155:42161","eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","queryParams":{"path":"/private/public/a.html","robots":"User-agent: *\nDisallow: /private/\nAllow: /private/public\n\nUser-agent: GPTBot\nDisallow: /\nSitemap: https://example.com/sitemap.xml","user_agent":"Googlebot"},"type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"method":{"enum":["GET"],"type":"string"},"queryParams":{"properties":{"path":{"description":"optional path to test, default /","type":"string"},"robots":{"description":"robots.txt contents, up to 12000 characters","type":"string"},"user_agent":{"description":"optional crawler token such as Googlebot or GPTBot, default *","type":"string"}},"required":["robots"],"type":"object"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"properties":{"allowed":{"description":"True when the crawler may fetch the path","type":"boolean"},"crawl_delay_seconds":{"description":"Crawl delay, in seconds","type":["number","null"]},"decision":{"description":"allowed or blocked","type":"string"},"group_selection":{"description":"Group selection","type":"string"},"groups_found":{"description":"Groups found","type":"number"},"matched_group":{"description":"Matched group","type":["array","null"]},"matched_rule":{"description":"The rule that decided the outcome, or null when no rule matched","properties":{"directive":{"description":"Allow or Disallow","type":["string","null"]},"pattern":{"description":"The path pattern of the deciding rule","type":["string","null"]}},"type":["object","null"]},"note":{"description":"Caveat or context for reading the result","type":"string"},"path":{"description":"Path that was tested","type":"string"},"rules_in_group":{"description":"Rules in group","type":"number"},"sitemaps":{"description":"Sitemap URLs declared in the file","type":"array"},"source":{"description":"Data source the values come from","type":"string"},"user_agent":{"description":"User-agent the rules were evaluated for","type":"string"}},"required":["allowed","decision"],"type":"object"},"type":{"const":"json","type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-20","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.01,"price_match":true,"latency_ms":537,"error":null}],"description_full":"Check whether a crawler may fetch a path under a robots.txt per RFC 9309 - matched group and rule, crawl delay and sitemaps, for Googlebot, GPTBot or any agent","last_updated":"2026-10-20T14:52:39.367Z","schemes":["exact"]}