{"slug":"api-aiskills402-com-v1-skills-robots-txt-policy-file-2c5df0","title":"Robots.txt Policy Writer — Turns a crawler policy written in words into a correc","host":"api.aiskills402.com","method":"GET","resource":"https://api.aiskills402.com/v1/skills/robots-txt-policy/file","category":"search","description":"Robots.txt Policy Writer — Turns a crawler policy written in words into a correct robots.txt, answered as the file only. Knows the mistakes that make a file do the opposite of what was meant - a crawler obeys only the one group that names it, so a GPTBot group silently drops every rule written for t","price_listed":0.03,"price_asked":0.03,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":373,"reported_calls_30d":2,"reported_payers_30d":2,"networks":["eip155:8453"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"method":"GET","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"headers":{"additionalProperties":{"type":"string"},"type":"object"},"method":{"enum":["GET","HEAD"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method"],"type":"object"},"output":{"properties":{"example":{"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-18","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.03,"price_match":true,"latency_ms":373,"error":null}],"description_full":"Robots.txt Policy Writer — Turns a crawler policy written in words into a correct robots.txt, answered as the file only. Knows the mistakes that make a file do the opposite of what was meant - a crawler obeys only the one group that names it, so a GPTBot group silently drops every rule written for the star group and the private paths must be repeated; longest match wins and Allow wins a tie; Google ignores Crawl-delay; Host is a Yandex-only line; Sitemap must be an absolute address; Google-Exte…","last_updated":"2026-10-16T20:53:53.454Z","schemes":["exact"]}