#!/usr/bin/env python3
"""Scores a paid-SERP run (scripts/run-paid-serp-check.py) with no provider call.
Per route: fetches answered, fetches with a text ad, ads top/bottom, sponsored Shopping and unpaid
product grids, advertiser domains found vs the union over all routes and passes, pass-to-pass churn.
Usage: scripts/score-paid-serp-check.py <fixture> <run-dir>   (reads <run-dir>/raw, writes summary.json, keywords.csv)"""
import json, sys, os, glob, csv, statistics
from urllib.parse import urlparse
fx = json.load(open(sys.argv[1])); run = sys.argv[2]
ROUTES = ['litescrape', 'serpapi', 'dataforseo', 'cloro']
def dom(u):
    if not u: return None
    u = u.split(' ')[0].strip()
    if '://' not in u: u = 'https://' + u
    h = (urlparse(u).hostname or '').lower()
    for p in ('www.',): h = h[len(p):] if h.startswith(p) else h
    return h or None
def parse(route, r):
    """returns dict(ok, ads=[(block, domain)], shopping_ads=int, product_grid=int, err)"""
    o = {'ok': False, 'ads': [], 'shopping_ads': 0, 'product_grid': 0, 'err': None}
    if not isinstance(r, dict) or '_nonjson' in r: o['err'] = 'nonjson'; return o
    if route == 'dataforseo':
        try: t = r['tasks'][0]
        except Exception: o['err'] = 'no task'; return o
        if t.get('status_code') != 20000: o['err'] = f"{t.get('status_code')} {t.get('status_message')}"; return o
        items = (t['result'][0].get('items') or [])
        o['ok'] = True
        first_org = min([i['rank_absolute'] for i in items if i['type'] == 'organic'] or [999])
        for i in items:
            if i['type'] == 'paid':
                o['ads'].append(('top' if i['rank_absolute'] < first_org else 'bottom', dom(i.get('breadcrumb')) or (i.get('website_name') or '').lower()))
            elif i['type'] == 'shopping': o['shopping_ads'] += len(i.get('items') or []) or 1
            elif i['type'] in ('popular_products', 'product_considerations'): o['product_grid'] += 1
        return o
    if route == 'cloro':
        res = r.get('result', r)
        if not isinstance(res, dict) or 'organicResults' not in res: o['err'] = str(r)[:120]; return o
        o['ok'] = True
        for k in ('ads', 'paidResults', 'sponsoredResults', 'topAds', 'bottomAds'):
            for a in res.get(k) or []: o['ads'].append((a.get('blockPosition') or a.get('position_block') or ('bottom' if k == 'bottomAds' else 'top'), (a.get('domain') or dom(a.get('url') or a.get('displayedUrl'))).removeprefix('www.')))
        o['product_grid'] = 1 if res.get('shoppingCards') else 0
        return o
    # litescrape and serpapi share SerpApi's schema
    if 'organic_results' not in r and 'search_metadata' not in r: o['err'] = str(r)[:120]; return o
    if (r.get('search_metadata') or {}).get('status') not in (None, 'Success'): o['err'] = r['search_metadata'].get('status'); return o
    o['ok'] = True
    for a in r.get('ads') or []:
        l = dom(a.get('link'))
        if not l or l.endswith('google.com'): l = dom(a.get('displayed_link')) or l  # aclk click URLs hide the advertiser
        o['ads'].append((a.get('block_position') or 'top', l))
    sh = r.get('shopping_results') or []
    if isinstance(sh, dict): sh = sum((v for v in sh.values() if isinstance(v, list)), [])
    o['shopping_ads'] = len(sh) + len(r.get('inline_shopping') or [])
    o['product_grid'] = 1 if r.get('immersive_products') else 0
    return o
recs = {}
for f in glob.glob(os.path.join(run, 'raw', '*.json')):
    d = json.load(open(f)); m = d['_run']
    p = parse(m['route'], d['response']); p['ms'] = m['latency_ms']
    recs[(m['pass'], m['route'], m['q'])] = p
passes = sorted({k[0] for k in recs}); K = fx['keywords']
union = {q: {a[1] for k, p in recs.items() if k[2] == q for a in p['ads'] if a[1]} for q in K}
summ = {'fixture': fx['name'], 'location': fx['location'], 'keywords': len(K), 'passes': passes, 'routes': {}}
for rt in ROUTES:
    rs = [p for k, p in recs.items() if k[1] == rt]
    ok = [p for p in rs if p['ok']]
    found = sum(len({a[1] for k, p in recs.items() if k[1] == rt and k[2] == q for a in p['ads'] if a[1]}) for q in K)
    churn = []; flips = 0
    for q in K:
        a, b = recs.get((1, rt, q)), recs.get((2, rt, q))
        if not (a and b and a['ok'] and b['ok']): continue
        A = {x[1] for x in a['ads']}; B = {x[1] for x in b['ads']}
        if bool(A) != bool(B): flips += 1
        if A or B: churn.append(len(A & B) / len(A | B))
    summ['routes'][rt] = {
        'calls': len(rs), 'answered': len(ok), 'errors': sorted({p['err'] for p in rs if p['err']}),
        'fetches_with_text_ad': sum(1 for p in ok if p['ads']),
        'text_ads': sum(len(p['ads']) for p in ok),
        'top': sum(1 for p in ok for a in p['ads'] if a[0] == 'top'), 'bottom': sum(1 for p in ok for a in p['ads'] if a[0] == 'bottom'),
        'middle': sum(1 for p in ok for a in p['ads'] if a[0] not in ('top', 'bottom')),
        'fetches_with_sponsored_shopping': sum(1 for p in ok if p['shopping_ads']),
        'fetches_with_product_grid': sum(1 for p in ok if p['product_grid']),
        'keywords_with_ad_any_pass': sum(1 for q in K if any(recs.get((n, rt, q), {}).get('ads') for n in passes)),
        'advertiser_domains_found': found, 'advertiser_domains_union': sum(len(v) for v in union.values()),
        'pass_presence_flips': flips, 'pass_domain_jaccard_mean': round(statistics.mean(churn), 2) if churn else None,
        'median_latency_ms': int(statistics.median([p['ms'] for p in ok])) if ok else None,
    }
summ['keywords_with_ad_on_any_route'] = sum(1 for q in K if union[q])
summ['advertiser_domains_union'] = sum(len(v) for v in union.values())
json.dump(summ, open(os.path.join(run, 'summary.json'), 'w'), indent=1)
with open(os.path.join(run, 'keywords.csv'), 'w', newline='') as fh:
    w = csv.writer(fh); w.writerow(['keyword', 'pass', 'route', 'answered', 'text_ads_top', 'text_ads_bottom', 'advertiser_domains', 'sponsored_shopping', 'product_grid'])
    for q in K:
        for n in passes:
            for rt in ROUTES:
                p = recs.get((n, rt, q))
                if not p: continue
                w.writerow([q, n, rt, int(p['ok']), sum(1 for a in p['ads'] if a[0] == 'top'), sum(1 for a in p['ads'] if a[0] == 'bottom'), ';'.join(sorted({a[1] for a in p['ads'] if a[1]})), p['shopping_ads'], p['product_grid']])
print(json.dumps(summ, indent=1))
