"""Per-outlet stats from counts_week.csv and items.csv.

Usage: python3 overlap.py counts_week.csv items.csv
Prints one row per outlet (mean/day, max, zero days, unique share) and the
cross-outlet title pairs that matched at Jaccard >= 0.6.
"""
import csv, re, sys, collections

def tokens(title):
    title = title.rsplit(" - ", 1)[0].lower()
    return set(re.findall(r"[a-z0-9]+", title))

def similar(a, b, threshold=0.6):
    return len(a & b) / max(1, len(a | b)) >= threshold

counts = collections.defaultdict(list)
for r in csv.DictReader(open(sys.argv[1])):
    counts[r["domain"]].append(int(r["items_24h"]))

items = list(csv.DictReader(open(sys.argv[2])))
seen, uniq = set(), []
for it in items:  # same story can be fresh on two runs
    if it["link"] not in seen:
        seen.add(it["link"])
        uniq.append(it)
toks = [tokens(i["title"]) for i in uniq]

shared = collections.defaultdict(int)
total = collections.defaultdict(int)
pairs = []
for i, a in enumerate(uniq):
    total[a["domain"]] += 1
    hit = None
    for j, b in enumerate(uniq):
        if i != j and a["domain"] != b["domain"] and similar(toks[i], toks[j]):
            hit = j
            break
    if hit is not None:
        shared[a["domain"]] += 1
        if i < hit:
            pairs.append((a["domain"], a["title"], uniq[hit]["domain"], uniq[hit]["title"]))

print("outlet\tmean\tmax\tzero_days\titems\tunique_share")
for d, v in sorted(counts.items(), key=lambda kv: -sum(kv[1])):
    n = total[d]
    share = f"{(n - shared[d]) / n:.0%}" if n else "-"
    print(f"{d}\t{sum(v)/len(v):.2f}\t{max(v)}\t{v.count(0)}\t{n}\t{share}")
print(f"\n{len(uniq)} unique items, {len(pairs)} matched pairs")
for p in pairs:
    print(" |", p[0], "|", p[1], "||", p[2], "|", p[3])