#!/usr/bin/env python3
"""Fetches the Google results page for each fixture keyword through four SERP API routes, N passes,
and stores one raw response per call (paid results are read by score-paid-serp-check.py).
Resumable: a call whose file exists is skipped. One worker per route. Deterministic; no LLM.
Usage: scripts/run-paid-serp-check.py evidence/fixtures/paid-serp-keywords-v1.json <out-dir> [passes=2]
Cost on 2026-11-08 settled prices: about $0.85 for 200 calls (SerpApi is most of it)."""
import json, sys, os, subprocess, time, concurrent.futures as cf
fx = json.load(open(sys.argv[1])); out = sys.argv[2]; passes = int(sys.argv[3]) if len(sys.argv) > 3 else 2
raw = os.path.join(out, 'raw'); os.makedirs(raw, exist_ok=True)
LOC = fx['location']
def routes(q):
    return {
        'litescrape': ['litescrape.google.serp.ads', '--query', f'q={q}', '--query', f'location={LOC}', '--query', 'device=desktop', '--query', 'hl=en'],
        'serpapi': ['serpapi.google.serp.organic', '--query', 'engine=google', '--query', f'q={q}', '--query', f'location={LOC}', '--query', 'gl=us', '--query', 'hl=en', '--query', 'device=desktop', '--query', 'no_cache=true'],
        'dataforseo': ['dataforseo.x.serp-google-organic-live-advanced', '--method', 'POST', '--data', json.dumps([{'language_code': 'en', 'location_name': 'New York,New York,United States', 'keyword': q, 'device': 'desktop', 'depth': 10}])],
        'cloro': ['cloro.google.serp.organic', '--method', 'POST', '--data', json.dumps({'query': q, 'gl': 'US', 'pages': 1})],
    }
def call(label, argv, meta):
    f = os.path.join(raw, label + '.json')
    if os.path.exists(f): return
    t0 = time.time()
    p = subprocess.run(['perl', '-e', 'alarm 120; exec @ARGV', 'treg', 'call', *argv], capture_output=True, text=True)
    ms = int((time.time() - t0) * 1000)
    txt = p.stdout if p.stdout.strip() else p.stderr
    try: body = json.loads(txt)
    except Exception: body = {'_nonjson': txt[:2000]}
    json.dump({'_run': {**meta, 'label': label, 'latency_ms': ms, 'exit': p.returncode, 'ts': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}, 'response': body}, open(f, 'w'))
    print(label, ms, 'ms exit', p.returncode, flush=True)
for n in range(1, passes + 1):
    by = {}
    for i, q in enumerate(fx['keywords']):
        for r, argv in routes(q).items():
            by.setdefault(r, []).append((f'p{n}-{r}-k{i+1:02d}', argv, {'pass': n, 'route': r, 'q': q}))
    with cf.ThreadPoolExecutor(max_workers=len(by)) as ex:
        list(ex.map(lambda lst: [call(*j) for j in lst], by.values()))
    print('pass', n, 'done', flush=True)
