{"slug":"tariffmonkee-com-paid-crawler-diligence-logs-fa0ac5","title":"AI crawler traffic analysis from real server logs (up to 20,000 lines)","host":"tariffmonkee.com","method":"POST","resource":"https://tariffmonkee.com/paid/crawler-diligence-logs","category":"search","description":"AI crawler traffic analysis from real server logs (up to 20,000 lines): hits by crawler and operator, with reverse-DNS checks that flag IPs claiming a crawler User-Agent that do not belong to that operator. Optional revenue band from a rate the caller supplies.","price_listed":0.3,"price_asked":0.3,"state":"answering","state_label":"Answering","checks_7d":1,"answered_7d":1,"latency_ms_median":1218,"reported_calls_30d":1,"reported_payers_30d":1,"networks":["eip155:8453","solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp"],"badge":"unverified","paid_checks_7d":0,"paid_ok_7d":0,"example_input":{"body":{"log":"20.171.206.70 - - [18/Oct/2026:01:03:10 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n20.171.206.71 - - [18/Oct/2026:01:04:11 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n107.23.91.1 - - [18/Oct/2026:11:30:02 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"\n160.79.106.10 - - [18/Oct/2026:04:20:05 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)\"\n160.79.106.11 - - [18/Oct/2026:04:21:06 +0000] \"GET /faq HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)\"\n198.51.100.20 - - [18/Oct/2026:07:11:40 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)\"\n198.51.100.21 - - [18/Oct/2026:07:12:41 +0000] \"GET /docs/auth HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)\"\n52.70.240.171 - - [18/Oct/2026:09:02:15 +0000] \"GET /blog/pricing HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)\"\n52.70.240.171 - - [18/Oct/2026:09:03:16 +0000] \"GET /docs/api HTTP/1.1\" 200 4200 \"-\" \"Mozilla/5.0 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)\"\n203.0.113.10 - - [18/Oct/2026:12:00:33 +0000] \"GET / HTTP/1.1\" 200 5100 \"https://www.google.com/\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36\"\n203.0.113.11 - - [18/Oct/2026:12:01:33 +0000] \"GET /about HTTP/1.1\" 200 5100 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/141.0.0.0 Safari/537.36\"","rate_per_crawl":0.01},"bodyType":"json","method":"POST","type":"http"},"output_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"input":{"additionalProperties":false,"properties":{"body":{"properties":{"log":{"description":"Raw server log lines as one string. Up to 20,000 lines are read.","type":"string"},"rate_per_crawl":{"description":"Your assumed USD per crawl. Omit for counts only.","type":"number"}},"required":["log"],"type":"object"},"bodyType":{"enum":["json","form-data","text"],"type":"string"},"method":{"enum":["POST"],"type":"string"},"type":{"const":"http","type":"string"}},"required":["type","method","bodyType","body"],"type":"object"},"output":{"properties":{"example":{"properties":{"ai_crawler_hits":{"type":"number"},"by_crawler":{"items":{"properties":{"crawler":{"type":"string"},"distinct_ips":{"type":"number"},"hits":{"type":"number"},"kind":{"type":"string"},"operator":{"type":"string"},"spoofed_ips":{"type":"number"},"unverified_ips":{"type":"number"},"verified_ips":{"type":"number"}},"required":["crawler","operator","kind","hits","distinct_ips","verified_ips","spoofed_ips","unverified_ips"],"type":"object"},"type":"array"},"caveats":{"items":{"type":"string"},"type":"array"},"lines_parsed":{"type":"number"},"lines_unrecognized":{"type":"number"},"revenue_estimate":{},"spoofing_summary":{"type":"string"}},"required":["lines_parsed","lines_unrecognized","ai_crawler_hits","by_crawler","spoofing_summary","caveats"],"type":"object"},"type":{"type":"string"}},"required":["type"],"type":"object"}},"required":["input"],"type":"object"},"history":[{"day":"2026-10-18","reachable":true,"status":402,"valid_402":true,"asked_usdc":0.3,"price_match":true,"latency_ms":1218,"error":null}],"description_full":"AI crawler traffic analysis from real server logs (up to 20,000 lines): hits by crawler and operator, with reverse-DNS checks that flag IPs claiming a crawler User-Agent that do not belong to that operator. Optional revenue band from a rate the caller supplies.","last_updated":"2026-10-18T14:27:09Z","schemes":["exact"]}