{"slug":"http-forest-gas-station-mainnet-lcjl27p8lmjs-code-run-robots-8b22c2","title":"Interpret the robots.txt crawling rules that apply to a specific public URL and ","host":"http--forest-gas-station-mainnet--lcjl27p8lmjs.code.run","method":"POST","resource":"https://http--forest-gas-station-mainnet--lcjl27p8lmjs.code.run/robots/context","category":"code","description":"Interpret the robots.txt crawling rules that apply to a specific public URL and user-agent before an automated fetch or crawl. Returns allow/disallow decision evidence, matched rules, crawl delay and sitemap hints from one bounded robots.txt read. This is protocol guidance, not authorization or an a","price_listed":0.001,"price_asked":0.001,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":929,"reported_calls_30d":2,"reported_payers_30d":2,"networks":["eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"url":"https://http--forest-gas-station-mainnet--lcjl27p8lmjs.code.run/fixtures/page","user_agent":"ForestBot/1.0"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"additionalProperties":false,"properties":{"url":{"description":"Public HTTP(S) URL to inspect, resolve, or compare.","maxLength":2048,"minLength":1,"pattern":"^https?://","type":"string"},"user_agent":{"description":"Public user agent value used to scope this lookup.","maxLength":128,"minLength":1,"type":"string"}},"required":["url"],"type":"object"},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"additionalProperties":false,"properties":{"capability":{"const":"robots.context","description":"Public capability value used to scope this lookup."},"data":{"additionalProperties":false,"description":"Public data value used to scope this lookup.","properties":{"crawl_delay":{"description":"Public crawl delay value used to scope this lookup.","minimum":0,"type":["number","null"]},"decision":{"description":"Public decision value used to scope this lookup.","enum":["allowed","disallowed","unknown"]},"matched_rule":{"anyOf":[{"additionalProperties":false,"properties":{"directive":{"description":"Public directive value used to scope this lookup.","enum":["allow","disallow"]},"pattern":{"description":"Public pattern value used to scope this lookup.","type":"string"},"specificity":{"description":"Public specificity value used to scope this lookup.","minimum":0,"type":"integer"}},"required":["directive","pattern","specificity"],"type":"object"},{"type":"null"}],"description":"Public matched rule value used to scope this lookup."},"matching_agents":{"description":"Public matching agents value used to scope this lookup.","items":{"type":"string"},"type":"array"},"rendered_javascript":{"const":false,"description":"Public rendered javascript value used to scope this lookup."},"robots_state":{"description":"Public robots state value used to scope this lookup.","enum":["available","unavailable","temporarily_unavailable"]},"robots_url":{"description":"Public robots url value used to scope this lookup.","type":"string"},"rules":{"description":"Public rules value used to scope this lookup.","items":{"additionalProperties":false,"properties":{"directive":{"description":"Public directive value used to scope this lookup.","enum":["allow","disallow"]},"pattern":{"description":"Public pattern value used to scope this lookup.","type":"string"}},"required":["directive","pattern"],"type":"object"},"maxItems":100,"type":"array"},"sitemaps":{"description":"Public sitemaps value used to scope this lookup.","items":{"type":"string"},"maxItems":50,"type":"array"},"status":{"description":"Public status value used to scope this lookup.","type":["integer","null"]},"target_url":{"description":"Public target url value used to scope this lookup.","type":"string"},"truncated":{"description":"Public truncated value used to scope this lookup.","type":"boolean"},"user_agent":{"description":"Public user agent value used to scope this lookup.","type":"string"}},"required":["target_url","robots_url","status","robots_state","user_agent","matching_agents","decision","matched_rule","rules","crawl_delay","sitemaps","truncated","rendered_javascript"],"type":"object"},"meta":{"additionalProperties":false,"description":"Public meta value used to scope this lookup.","properties":{"cache":{"description":"Public cache value used to scope this lookup.","enum":["hit","miss","bypass"]},"checked_at":{"description":"Public checked at value used to scope this lookup.","type":"string"},"duration_ms":{"description":"Public duration ms value used to scope this lookup.","minimum":0,"type":"integer"},"provider":{"description":"Public provider value used to scope this lookup.","type":["string","null"]}},"required":["provider","checked_at","cache","duration_ms"],"type":"object"},"ok":{"const":true,"description":"Public ok value used to scope this lookup."},"version":{"const":"1","description":"Package version whose public metadata should be checked."}},"required":["ok","capability","version","data","meta"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-10","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.001,"price_match":true,"latency_ms":929,"error":null}],"description_full":"Interpret the robots.txt crawling rules that apply to a specific public URL and user-agent before an automated fetch or crawl. Returns allow/disallow decision evidence, matched rules, crawl delay and sitemap hints from one bounded robots.txt read. This is protocol guidance, not authorization or an anti-bot bypass. Base mainnet USDC","last_updated":"2026-10-09T00:52:12.461Z","schemes":["exact"]}