Interpret the robots.txt crawling rules that apply to a specific public URL and

Interpret the robots.txt crawling rules that apply to a specific public URL and user-agent before an automated fetch or crawl. Returns allow/disallow decision evidence, matched rules, crawl delay and sitemap hints from one bounded robots.txt read. This is protocol guidance, not authorization or an anti-bot bypass. Base mainnet USDC

Paid test badge: not yet. The checks above are free: we call the tool without paying and read the payment request it sends back. The Verified badge needs paid calls whose answers match the promised output, and nobody can buy a badge.

Endpoint

POST https://http--forest-gas-station-mainnet--lcjl27p8lmjs.code.run/robots/context

Our checks, last 30 days

Example input (from the provider)

{
  "body": {
    "url": "https://http--forest-gas-station-mainnet--lcjl27p8lmjs.code.run/fixtures/page",
    "user_agent": "ForestBot/1.0"
  },
  "bodyType": "json",
  "method": "POST",
  "type": "http"
}

Promised output schema (from the provider)

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "properties": {
    "input": {
      "additionalProperties": false,
      "properties": {
        "body": {
          "additionalProperties": false,
          "properties": {
            "url": {
              "description": "Public HTTP(S) URL to inspect, resolve, or compare.",
              "maxLength": 2048,
              "minLength": 1,
              "pattern": "^https?://",
              "type": "string"
            },
            "user_agent": {
              "description": "Public user agent value used to scope this lookup.",
              "maxLength": 128,
              "minLength": 1,
              "type": "string"
            }
          },
          "required": [
            "url"
          ],
          "type": "object"
        },
        "bodyType": {
          "enum": [
            "json",
            "form-data",
            "text"
          ],
          "type": "string"
        },
        "method": {
          "enum": [
            "POST"
          ],
          "type": "string"
        },
        "type": {
          "const": "http",
          "type": "string"
        }
      },
      "required": [
        "type",
        "method",
        "bodyType",
        "body"
      ],
      "type": "object"
    },
    "output": {
      "properties": {
        "example": {
          "additionalProperties": false,
          "properties": {
            "capability": {
              "const": "robots.context",
              "description": "Public capability value used to scope this lookup."
            },
            "data": {
              "additionalProperties": false,
              "description": "Public data value used to scope this lookup.",
              "properties": {
                "crawl_delay": {
                  "description": "Public crawl delay value used to scope this lookup.",
                  "minimum": 0,
                  "type": [
                    "number",
                    "null"
                  ]
                },
                "decision": {
                  "description": "Public decision value used to scope this lookup.",
                  "enum": [
                    "allowed",
                    "disallowed",
                    "unknown"
                  ]
                },
                "matched_rule": {
                  "anyOf": [
                    {
                      "additionalProperties": false,
                      "properties": {
                        "directive": {
                          "description": "Public directive value used to scope this lookup.",
                          "enum": [
                            "allow",
                            "disallow"
                          ]
                        },
                        "pattern": {
                          "description": "Public pattern value used to scope this lookup.",
                          "type": "string"
                        },
                        "specificity": {
                          "description": "Public specificity value used to scope this lookup.",
                          "minimum": 0,
                          "type": "integer"
                        }
                      },
                      "required": [
                        "directive",
                        "pattern",
                        "specificity"
                      ],
                      "type": "object"
                    },
                    {
                      "type": "null"
                    }
                  ],
                  "description": "Public matched rule value used to scope this lookup."
                },
                "matching_agents": {
                  "description": "Public matching agents value used to scope this lookup.",
                  "items": {
                    "type": "string"
                  },
                  "type": "array"
                },
                "rendered_javascript": {
                  "const": false,
                  "description": "Public rendered javascript value used to scope this lookup."
                },
                "robots_state": {
                  "description": "Public robots state value used to scope this lookup.",
                  "enum": [
                    "available",
                    "unavailable",
                    "temporarily_unavailable"
                  ]
                },
                "robots_url": {
                  "description": "Public robots url value used to scope this lookup.",
                  "type": "string"
                },
                "rules": {
                  "description": "Public rules value used to scope this lookup.",
                  "items": {
                    "additionalProperties": false,
                    "properties": {
                      "directive": {
                        "description": "Public directive value used to scope this lookup.",
                        "enum": [
                          "allow",
                          "disallow"
                        ]
                      },
                      "pattern": {
                        "description": "Public pattern value used to scope this lookup.",
                        "type": "string"
                      }
                    },
                    "required": [
                      "directive",
                      "pattern"
                    ],
                    "type": "object"
                  },
                  "maxItems": 100,
                  "type": "array"
                },
                "sitemaps": {
                  "description": "Public sitemaps value used to scope this lookup.",
                  "items": {
                    "type": "string"
                  },
                  "maxItems": 50,
                  "type": "array"
                },
                "status": {
                  "description": "Public s

This page as JSON