Check whether a crawler may fetch a path under a robots.txt per RFC 9309
Check whether a crawler may fetch a path under a robots.txt per RFC 9309 - matched group and rule, crawl delay and sitemaps, for Googlebot, GPTBot or any agent
Answeringour last check, 2026-10-20
1 of 1checks answered this week
537 msmedian answer time
$0.01listed price per call
$0.01price it asked us
Paid test badge: not yet. The checks above are free: we call the tool without paying and read the payment request it sends back. The Verified badge needs paid calls whose answers match the promised output, and nobody can buy a badge.
Endpoint
GET https://saylorinnovations.com/api/web/robots-check
| Category | Search and research |
|---|---|
| Provider host | saylorinnovations.com |
| Networks | eip155:137, eip155:42161, eip155:8453, solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp |
| Payment schemes | exact |
| Self-reported calls, 30 days | 0 from 0 payers (the provider's figure, not ours) |
Our checks, last 30 days
| Day | Result | HTTP | Asked | Time |
|---|---|---|---|---|
| 2026-10-20 | valid payment request | 402 | $0.01 | 537 ms |
Example input (from the provider)
{
"method": "GET",
"queryParams": {
"path": "/private/public/a.html",
"robots": "User-agent: *\nDisallow: /private/\nAllow: /private/public\n\nUser-agent: GPTBot\nDisallow: /\nSitemap: https://example.com/sitemap.xml",
"user_agent": "Googlebot"
},
"type": "http"
}
Promised output schema (from the provider)
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"input": {
"additionalProperties": false,
"properties": {
"method": {
"enum": [
"GET"
],
"type": "string"
},
"queryParams": {
"properties": {
"path": {
"description": "optional path to test, default /",
"type": "string"
},
"robots": {
"description": "robots.txt contents, up to 12000 characters",
"type": "string"
},
"user_agent": {
"description": "optional crawler token such as Googlebot or GPTBot, default *",
"type": "string"
}
},
"required": [
"robots"
],
"type": "object"
},
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type",
"method"
],
"type": "object"
},
"output": {
"properties": {
"example": {
"properties": {
"allowed": {
"description": "True when the crawler may fetch the path",
"type": "boolean"
},
"crawl_delay_seconds": {
"description": "Crawl delay, in seconds",
"type": [
"number",
"null"
]
},
"decision": {
"description": "allowed or blocked",
"type": "string"
},
"group_selection": {
"description": "Group selection",
"type": "string"
},
"groups_found": {
"description": "Groups found",
"type": "number"
},
"matched_group": {
"description": "Matched group",
"type": [
"array",
"null"
]
},
"matched_rule": {
"description": "The rule that decided the outcome, or null when no rule matched",
"properties": {
"directive": {
"description": "Allow or Disallow",
"type": [
"string",
"null"
]
},
"pattern": {
"description": "The path pattern of the deciding rule",
"type": [
"string",
"null"
]
}
},
"type": [
"object",
"null"
]
},
"note": {
"description": "Caveat or context for reading the result",
"type": "string"
},
"path": {
"description": "Path that was tested",
"type": "string"
},
"rules_in_group": {
"description": "Rules in group",
"type": "number"
},
"sitemaps": {
"description": "Sitemap URLs declared in the file",
"type": "array"
},
"source": {
"description": "Data source the values come from",
"type": "string"
},
"user_agent": {
"description": "User-agent the rules were evaluated for",
"type": "string"
}
},
"required": [
"allowed",
"decision"
],
"type": "object"
},
"type": {
"const": "json",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
},
"required": [
"input"
],
"type": "object"
}