{"slug":"signalharness-ai-api-agent-services-robots-policy-parse-invo-52eb93","title":"Parses robots.txt groups and applies deterministic longest-match allow and disal","host":"signalharness.ai","method":"POST","resource":"https://signalharness.ai/api/agent/services/robots_policy_parse/invoke","category":"other","description":"Parses robots.txt groups and applies deterministic longest-match allow and disallow semantics.","price_listed":0.005,"price_asked":0.005,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":2202,"reported_calls_30d":2,"reported_payers_30d":1,"networks":["eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"text":"User-agent: *\nAllow: /"},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"maxProperties":3,"minProperties":1,"properties":{"targetPath":{"maxLength":2048,"minLength":1,"type":"string"},"text":{"maxLength":65536,"minLength":0,"type":"string"},"url":{"maxLength":2048,"minLength":9,"pattern":"^https://","type":"string"},"userAgent":{"maxLength":256,"minLength":1,"type":"string"}},"required":[],"type":"object"},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"additionalProperties":false,"properties":{"artifacts":{"maxItems":0,"type":"array"},"executionId":{"type":"string"},"receipt":{"additionalProperties":true,"properties":{"receiptId":{"type":"string"},"resultSha256":{"pattern":"^[0-9a-f]{64}$","type":"string"},"serviceId":{"type":"string"}},"required":["receiptId","serviceId","resultSha256"],"type":"object"},"replay":{"type":"boolean"},"requestId":{"type":"string"},"result":{"$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"maxProperties":7,"minProperties":7,"properties":{"crawlAllowed":{"type":["boolean","null"]},"crawlDelay":{"type":["number","null"]},"groups":{"items":{"additionalProperties":false,"maxProperties":3,"minProperties":3,"properties":{"crawlDelay":{"type":["number","null"]},"rules":{"items":{"additionalProperties":false,"maxProperties":2,"minProperties":2,"properties":{"directive":{"enum":["allow","disallow"],"maxLength":8,"minLength":5,"type":"string"},"path":{"maxLength":2048,"minLength":0,"type":"string"}},"required":["directive","path"],"type":"object"},"maxItems":256,"minItems":0,"type":"array"},"userAgents":{"items":{"maxLength":256,"minLength":1,"type":"string"},"maxItems":32,"minItems":1,"type":"array"}},"required":["userAgents","rules","crawlDelay"],"type":"object"},"maxItems":256,"minItems":0,"type":"array"},"matchedGroup":{"type":["array","null"]},"matchedRule":{"type":["object","null"]},"sitemaps":{"items":{"maxLength":2048,"minLength":1,"type":"string"},"maxItems":256,"minItems":0,"type":"array"},"warnings":{"items":{"maxLength":1024,"minLength":1,"type":"string"},"maxItems":32,"minItems":0,"type":"array"}},"required":["groups","matchedGroup","matchedRule","crawlAllowed","sitemaps","crawlDelay","warnings"],"type":"object"},"status":{"const":"succeeded","type":"string"}},"required":["status","replay","requestId","executionId","result","receipt","artifacts"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-09-24","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.005,"price_match":true,"latency_ms":2202,"error":null}],"description_full":"Parses robots.txt groups and applies deterministic longest-match allow and disallow semantics.","last_updated":"2026-09-20T20:01:21.175Z","schemes":["exact"]}