#!/usr/bin/env python3
"""Two questions a stored ads cohort run can answer with no provider call.
Landing pages: on each fixture advertiser's own Meta pages (Apify page scrape plus ScrapeCreators company
endpoint, deduplicated on ad id), how many ads carry a landing URL, how many distinct pages they point at
(query string and fragment removed) and how many ads share the most used page. No route filters by destination,
so "every ad pointing at one page" is this post-filter.
Countries: per Meta route, how many ads carry a populated country field; LinkedIn's per-ad country breakdown.
Usage: check-ad-landing-country.py <run-date> <out-dir>  (reads public/evidence/ads-runs/<run-date>/)"""
import glob, json, os, sys, csv
from collections import Counter, defaultdict
from urllib.parse import urlsplit
run, out = sys.argv[1], sys.argv[2]; os.makedirs(out, exist_ok=True)
def norm(u):
    p = urlsplit(u); return (p.netloc.lower().removeprefix('www.') + p.path.rstrip('/')) if p.netloc else None
def ads_of(r):
    x = r.get('response', r)
    if isinstance(x, list): return x
    return x.get('results') or x.get('searchResults') or x.get('ads') or []
def own_page(adv, a):
    # the Apify route is a keyword search, so it also holds other advertisers' ads that mention the name
    name = (a.get('pageName') or a.get('page_name') or (a.get('snapshot') or {}).get('pageName') or (a.get('snapshot') or {}).get('page_name') or '').lower()
    return adv.lower().removesuffix('.io').removesuffix('.com') in name
per = defaultdict(dict); countries = {}; total = Counter()
for f in sorted(glob.glob(f'public/evidence/ads-runs/{run}/*-a*.json')):
    r = json.load(open(f)); m = r['_run']; ads = ads_of(r); key = f"{m['platform']}-{m['provider']}"
    if m['platform'] == 'meta':
        c = countries.setdefault(key, {'ads': 0, 'with_country_field': 0}); c['ads'] += len(ads)
        for a in ads:
            s = a.get('snapshot') or {}
            if a.get('targetedOrReachedCountries') or a.get('targeted_or_reached_countries') or s.get('countryIsoCode') or s.get('country_iso_code'): c['with_country_field'] += 1
            if m['provider'] in ('apify', 'sc_company') and own_page(m['adv'], a):
                aid = str(a.get('adArchiveID') or a.get('ad_archive_id'))
                link = s.get('linkUrl') or s.get('link_url')
                if not link:
                    for card in s.get('cards') or []:
                        link = card.get('linkUrl') or card.get('link_url')
                        if link: break
                per[m['adv']].setdefault(aid, norm(link) if link else None)
    if m['platform'] == 'linkedin':
        c = countries.setdefault(key, {'ads': 0, 'with_country_field': 0}); c['ads'] += len(ads)
        c['with_country_field'] += sum(bool(a.get('impressionsByCountry')) for a in ads)
rows = []
for adv, ads in sorted(per.items()):
    links = [l for l in ads.values() if l]; cnt = Counter(links)
    rows.append({'advertiser': adv, 'ads': len(ads), 'with_landing': len(links), 'distinct_pages': len(cnt), 'ads_on_top_page': cnt.most_common(1)[0][1] if cnt else 0, 'top_page': cnt.most_common(1)[0][0] if cnt else ''})
tot = {k: sum(r[k] for r in rows) for k in ('ads', 'with_landing', 'distinct_pages', 'ads_on_top_page')}
json.dump({'run': run, 'landing_totals': tot, 'landing_per_advertiser': rows, 'country_fields': countries}, open(os.path.join(out, 'summary.json'), 'w'), indent=1)
w = csv.DictWriter(open(os.path.join(out, 'landing-pages.csv'), 'w', newline=''), fieldnames=list(rows[0])); w.writeheader(); w.writerows(rows)
print(json.dumps(tot)); print(json.dumps(countries)); [print(r) for r in rows]
