"""Count Google News items per outlet in the last 24 hours.
Usage: python3 count_sources.py domains.txt counts.csv
Appends one row per domain to the CSV: run_utc, domain, items_returned, items_24h, capped.
Also appends every fresh item (title, pubDate, link) to items.csv in the same folder
so the overlap check can run later.
"""
import csv, os, sys, time, urllib.parse, urllib.request
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
import xml.etree.ElementTree as ET

TERM = '("vertical farm" OR "vertical farming" OR "indoor farming")'   # fixed niche term, same for every outlet
CAP = 100                         # searches returned 100 to 103 items in our earlier test
DELAY = 3                         # seconds between requests

domains = [d.strip() for d in open(sys.argv[1]) if d.strip()]
out = sys.argv[2]
now = datetime.now(timezone.utc)
new = not os.path.exists(out)
items_path = os.path.join(os.path.dirname(os.path.abspath(out)), "items.csv")
new_items = not os.path.exists(items_path)
with open(out, "a", newline="") as f, open(items_path, "a", newline="") as g:
    w = csv.writer(f)
    wi = csv.writer(g)
    if new_items:
        wi.writerow(["run_utc", "domain", "title", "pubDate", "link"])
    if new:
        w.writerow(["run_utc", "domain", "items_returned", "items_24h", "capped"])
    for d in domains:
        q = f"site:{d} {TERM} when:1d"
        url = "https://news.google.com/rss/search?q=" + urllib.parse.quote_plus(q) + "&hl=en-US&gl=US&ceid=US:en"
        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
        try:
            root = ET.fromstring(urllib.request.urlopen(req, timeout=20).read())
        except Exception as e:            # one failed request should not lose the whole day
            print(f"{d}: {e}", file=sys.stderr)
            time.sleep(DELAY)
            continue
        items = list(root.iter("item"))
        fresh = [i for i in items if now - parsedate_to_datetime(i.findtext("pubDate")) <= timedelta(hours=24)]
        for i in fresh:
            wi.writerow([now.strftime("%Y-%m-%dT%H:%M:%SZ"), d, i.findtext("title"), i.findtext("pubDate"), i.findtext("link")])
        w.writerow([now.strftime("%Y-%m-%dT%H:%M:%SZ"), d, len(items), len(fresh), len(items) >= CAP])
        time.sleep(DELAY)