#!/usr/bin/env python3
"""GONE - removal tracker CLI (v0.2).

Checks a list of URLs against the state you intended for each one,
follows redirect chains hop by hop, reads meta robots and X-Robots-Tag,
checks robots.txt, and flags any URL that is still listed in a sitemap.

Input CSV columns (header optional):
    url,intent,target
    https://site.test/old-page,410,
    https://site.test/moved,301,https://site.test/new-page
    https://site.test/thin,noindex,
    https://site.test/match-preview,unavailable_after,2026-11-01T20:00:00Z

intent values: 410 | 404 | 301 | 308 | noindex | unavailable_after | 200
Rows with no intent use --default-intent (410 unless you change it).

Modes:
    --mode standard    judge each URL against its own intent (default)
    --mode facet       every URL is a filter combination with no results:
                       it should return 404 (or 410) at its own URL, not
                       redirect to a shared not-found page and not return 200
    --mode post-event  every URL is a page for an event that has ended:
                       expects unavailable_after (date in the target column)
                       unless the row says otherwise

Usage:
    python3 gone.py urls.csv --sitemap https://site.test/sitemap.xml \
        --out report            # writes report.csv and report.html

Needs Python 3.8+ and 'requests' (pip install requests).
"""
import argparse, csv, gzip, html, io, re, sys, time, json, threading
import urllib.robotparser
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from urllib.parse import urljoin, urlsplit, urlunsplit

try:
    import requests
except ImportError:
    sys.exit("gone.py needs 'requests': pip install requests")

UA = "GONE/0.2 removal-check (+https://nikkos.io/tools/gone)"
MAX_HOPS = 10
BODY_LIMIT = 512 * 1024
META_RE = re.compile(r'<meta\s[^>]*name\s*=\s*["\']?(robots|googlebot)["\']?[^>]*>', re.I)
CONTENT_RE = re.compile(r'content\s*=\s*["\']([^"\']*)["\']', re.I)
CANON_RE = re.compile(r'<link\s[^>]*rel\s*=\s*["\']?canonical["\']?[^>]*>', re.I)
HREF_RE = re.compile(r'href\s*=\s*["\']([^"\']*)["\']', re.I)


def norm(u):
    """Normalise a URL for comparison: lower-case scheme and host, drop fragment and default ports."""
    if not u:
        return ""
    s = urlsplit(u.strip())
    host = (s.hostname or "").lower()
    if s.port and not ((s.scheme == "http" and s.port == 80) or (s.scheme == "https" and s.port == 443)):
        host += f":{s.port}"
    return urlunsplit((s.scheme.lower(), host, s.path or "/", s.query, ""))


GONE_INTENTS = ("410", "404")
SEV_ORDER = {"High": 0, "Medium": 1, "Low": 2, "": 3}


def norm_intent(v):
    v = (v or "").strip().lower().replace("_", "-")
    v = v.replace("200 noindex", "noindex").replace("200-noindex", "noindex")
    if v in ("unavailable-after", "unavailable after", "expire", "expires"):
        return "unavailable_after"
    if v == "redirect":
        return "301"
    return v


def read_input(path, default_intent="410"):
    rows = []
    with open(path, newline="", encoding="utf-8-sig") as f:
        for i, r in enumerate(csv.reader(f)):
            if not r or not r[0].strip() or r[0].strip().startswith("#"):
                continue
            if i == 0 and r[0].strip().lower() == "url":
                continue
            url = r[0].strip()
            intent = norm_intent(r[1] if len(r) > 1 else "") or norm_intent(default_intent)
            target = r[2].strip() if len(r) > 2 else ""
            rows.append({"url": url, "intent": intent, "target": target})
    return rows


def fetch_sitemap(url, session, seen=None, depth=0, notes=None):
    """Return the set of <loc> URLs in a sitemap or sitemap index (follows child sitemaps, gzip ok)."""
    seen = seen if seen is not None else set()
    notes = notes if notes is not None else []
    locs = set()
    if url in seen or depth > 3:
        return locs
    seen.add(url)
    try:
        if url.startswith("file://") or not re.match(r"https?://", url):
            data = open(url.replace("file://", ""), "rb").read()
        else:
            r = session.get(url, timeout=30)
            if r.status_code != 200:
                notes.append(f"sitemap {url} returned {r.status_code}")
                return locs
            data = r.content
        if data[:2] == b"\x1f\x8b":
            data = gzip.decompress(data)
        root = ET.fromstring(data)
    except Exception as e:
        notes.append(f"sitemap {url} could not be read: {e}")
        return locs
    tag = root.tag.split("}")[-1]
    for loc in root.iter():
        if loc.tag.split("}")[-1] == "loc" and loc.text:
            if tag == "sitemapindex":
                locs |= fetch_sitemap(loc.text.strip(), session, seen, depth + 1, notes)
            else:
                locs.add(norm(loc.text))
    return locs


_robots_cache, _robots_lock = {}, threading.Lock()


def robots_blocked(url, session):
    """True if robots.txt disallows Googlebot from this URL, None if robots.txt could not be read."""
    s = urlsplit(url)
    key = f"{s.scheme}://{s.netloc}"
    with _robots_lock:
        rp = _robots_cache.get(key, "missing")
    if rp == "missing":
        rp = None
        try:
            r = session.get(key + "/robots.txt", timeout=15)
            if r.status_code == 200:
                rp = urllib.robotparser.RobotFileParser()
                rp.parse(r.text.splitlines())
            elif 400 <= r.status_code < 500:
                rp = "allow-all"
        except requests.RequestException:
            rp = None
        with _robots_lock:
            _robots_cache[key] = rp
    if rp is None:
        return None
    if rp == "allow-all":
        return False
    return not rp.can_fetch("Googlebot", url)


def check_url(url, session, delay):
    """Follow the chain one hop at a time. Return a dict with the chain and the final response details."""
    chain, cur, out = [], url, {}
    try:
        for _ in range(MAX_HOPS):
            time.sleep(delay)
            r = session.get(cur, allow_redirects=False, timeout=20, stream=True)
            loc = r.headers.get("Location")
            chain.append({"url": cur, "status": r.status_code})
            if r.status_code in (301, 302, 303, 307, 308) and loc:
                r.close()
                nxt = urljoin(cur, loc)
                if any(norm(c["url"]) == norm(nxt) for c in chain):
                    chain.append({"url": nxt, "status": "LOOP"})
                    out["error"] = "redirect loop"
                    break
                cur = nxt
                continue
            body = b""
            ctype = r.headers.get("Content-Type", "")
            if "html" in ctype.lower():
                for chunk in r.iter_content(16384):
                    body += chunk
                    if len(body) >= BODY_LIMIT:
                        break
            r.close()
            text = body.decode(r.encoding or "utf-8", errors="replace")
            metas = []
            for m in META_RE.finditer(text):
                c = CONTENT_RE.search(m.group(0))
                if c:
                    metas.append(f"{m.group(1).lower()}: {c.group(1)}")
            canon = ""
            cm = CANON_RE.search(text)
            if cm:
                h = HREF_RE.search(cm.group(0))
                canon = urljoin(cur, h.group(1)) if h else ""
            out.update({
                "final_url": cur, "final_status": r.status_code,
                "x_robots_tag": r.headers.get("X-Robots-Tag", ""),
                "meta_robots": " | ".join(metas), "canonical": canon,
            })
            break
        else:
            out["error"] = f"more than {MAX_HOPS} hops"
    except requests.RequestException as e:
        out["error"] = type(e).__name__ + ": " + str(e)[:160]
    out["chain"] = chain
    return out


def is_home(u):
    p = urlsplit(u or "")
    return (p.path or "/") in ("/", "/index.html", "/index.php", "") and not p.query


def verdict(row, res, in_sitemap, mode="standard", blocked=None, migration=False):
    """Compare intent with what the server did.

    Returns (verdict, severity, list of reasons). Severity is the worst issue found:
    High, Medium or Low. 404 vs 410 is Low: Google treats them almost the same.
    """
    intent, chain = row["intent"], res.get("chain", [])
    issues, warns = [], []  # each item: (severity, text)
    if res.get("error"):
        return "ERROR", "High", [res["error"]]
    first = chain[0]["status"] if chain else None
    hops = len(chain) - 1
    final = res.get("final_status")
    final_url = res.get("final_url")
    robots = (res.get("x_robots_tag", "") + " " + res.get("meta_robots", "")).lower()
    noindex = "noindex" in robots or "none" in robots.split()
    has_ua = "unavailable_after" in robots

    if hops > 3:
        warns.append(("High" if hops >= 5 else "Medium", f"{hops} redirect hops - Google advises no more than 3, and fewer than 5"))

    if mode == "facet":
        if first in (404, 410):
            pass
        elif hops:
            issues.append(("High", f"empty filter page redirects ({first}) to {final_url} - return 404 at its own URL instead"))
        elif first == 200:
            issues.append(("High", "empty filter page returns 200 - soft 404 risk, return 404"))
        else:
            issues.append(("Medium", f"empty filter page returns {first}, expected 404"))
    elif intent in GONE_INTENTS:
        want = int(intent)
        if first == want:
            pass
        elif first in (404, 410):
            warns.append(("Low", f"returns {first}, intended {want} - both are fine, Google treats them almost the same"))
        elif hops:
            sev = "High"
            txt = f"redirects ({first}) to {final_url} - final {final}, intended {want}"
            if is_home(final_url):
                txt += " - redirecting removed pages to the home page may be treated as a soft 404"
            issues.append((sev, txt))
        elif first == 200:
            issues.append(("High", f"returns 200, intended {want} - soft 404 risk" + (" (noindex does not save crawl)" if noindex else "")))
        else:
            issues.append(("Medium", f"returns {first}, intended {want}"))
        if blocked:
            issues.append(("High", "blocked in robots.txt - Google can't crawl it, so it never sees the 404/410"))
    elif intent in ("301", "308"):
        tgt = norm(row["target"])
        if not hops:
            issues.append(("High", f"no redirect - returns {first}"))
        else:
            if first not in (301, 308):
                issues.append(("Medium", f"first hop is {first}, not permanent (302/307 are a weak signal)"))
            elif str(first) != intent:
                warns.append(("Low", f"first hop is {first}, intended {intent}"))
            if tgt and norm(final_url) != tgt:
                issues.append(("High", f"lands on {final_url}, intended {row['target']}"))
            if final != 200:
                issues.append(("High", f"final URL returns {final}"))
            elif noindex:
                issues.append(("High", "final URL is noindex"))
    elif intent == "noindex":
        if hops:
            issues.append(("Medium", f"redirects ({first}) instead of serving 200 noindex"))
        elif final != 200:
            issues.append(("Medium", f"returns {final}, intended 200 noindex"))
        elif not noindex:
            issues.append(("High", "200 but no noindex in meta robots or X-Robots-Tag"))
        if blocked:
            issues.append(("High", "blocked in robots.txt - Google can't see the noindex"))
    elif intent == "unavailable_after":
        if hops or final != 200:
            if final in (404, 410) and not hops:
                warns.append(("Low", f"returns {final} - page is gone, not expiring; fine if that is the decision"))
            else:
                issues.append(("Medium", f"expected a live page with unavailable_after, got {first}" + (f" then {final}" if hops else "")))
        elif not has_ua:
            sev = "Medium" if mode == "post-event" else "High"
            when = row["target"] or "<event end time>"
            issues.append((sev, f"no unavailable_after - add <meta name=\"robots\" content=\"unavailable_after: {when}\"> or the X-Robots-Tag header"))
        if noindex and not has_ua:
            warns.append(("Low", "noindex used on a past-event page - Google still requests noindexed pages, so it does not save crawl"))
    elif intent == "200":
        if hops or final != 200:
            issues.append(("High", f"expected 200, got {first}" + (f" then {final}" if hops else "")))
        elif noindex:
            issues.append(("High", "200 but noindex"))
    else:
        warns.append(("Low", f"unknown intent '{intent}'"))

    if in_sitemap and intent != "200":
        if migration:
            warns.append(("Low", "still listed in the migration-tracking sitemap (expected during a move)"))
        else:
            issues.append(("High", "still listed in sitemap - dead or non-indexable URLs should not be in a sitemap"))

    allitems = issues + warns
    sev = min((i[0] for i in allitems), key=lambda x: SEV_ORDER[x], default="")
    texts = [f"[{i[0]}] {i[1]}" for i in allitems]
    if issues:
        return "MISMATCH", sev, texts
    if warns:
        return "WARN", sev, texts
    return "MATCH", "", []


def chain_str(chain):
    return " > ".join(f"{c['status']} {c['url']}" for c in chain)


def write_html(path, results, meta):
    counts = {}
    for r in results:
        counts[r["verdict"]] = counts.get(r["verdict"], 0) + 1
    col = {"MATCH": "#3ecf8e", "WARN": "#f5b84a", "MISMATCH": "#ff6b6b", "ERROR": "#ff6b6b"}
    order = {"MISMATCH": 0, "ERROR": 1, "WARN": 2, "MATCH": 3}
    rows = []
    for r in sorted(results, key=lambda x: (order.get(x["verdict"], 9), SEV_ORDER.get(x["severity"], 3))):
        rows.append(
            "<tr><td><span class='pill' style='color:{c}'>{v}</span></td><td class='m'>{u}</td><td>{i}</td>"
            "<td class='m'>{ch}</td><td class='m'>{rb}</td><td>{sm}</td><td>{sev}</td><td>{why}</td></tr>".format(
                c=col.get(r["verdict"], "#8b94a1"), v=r["verdict"], u=html.escape(r["url"]), sev=html.escape(r["severity"] or "-"),
                i=html.escape(r["intent"] + (" to " + r["target"] if r["target"] else "")),
                ch=html.escape(r["chain"]), rb=html.escape((r["x_robots_tag"] + " " + r["meta_robots"]).strip() or "-"),
                sm="yes" if r["in_sitemap"] == "yes" else ("-" if r["in_sitemap"] == "" else "no"),
                why=html.escape("; ".join(r["reasons"].split(" ; ")) if r["reasons"] else "-")))
    summary = " ".join(f"<div class='s'><b style='color:{col.get(k,'#8b94a1')}'>{v}</b><span>{k}</span></div>" for k, v in sorted(counts.items(), key=lambda kv: order.get(kv[0], 9)))
    doc = f"""<!doctype html><html lang="en"><head><meta charset="utf-8"><title>GONE report</title>
<style>body{{background:#0b0d10;color:#e7eaee;font-family:Satoshi,system-ui,sans-serif;margin:0;padding:32px}}
h1{{letter-spacing:.06em;margin:0 0 4px}}.mut{{color:#8b94a1;font-size:13px}}
.sum{{display:flex;gap:12px;margin:20px 0}}.s{{background:#12151a;border:1px solid #252b33;border-radius:10px;padding:12px 18px}}
.s b{{font-family:'IBM Plex Mono',monospace;font-size:24px;display:block}}.s span{{color:#8b94a1;font-size:12px}}
table{{border-collapse:collapse;width:100%;font-size:13px;background:#12151a}}th,td{{border-bottom:1px solid #252b33;padding:8px;text-align:left;vertical-align:top}}
th{{color:#8b94a1;font-weight:500;font-size:12px;text-transform:uppercase}}.m{{font-family:'IBM Plex Mono',monospace;font-size:12px;word-break:break-all}}
.pill{{font-family:'IBM Plex Mono',monospace;font-size:11px;border:1px solid currentColor;border-radius:99px;padding:2px 8px}}</style></head>
<body><h1>GONE</h1><div class="mut">Removal check run {html.escape(meta['run_at'])} - mode {html.escape(meta['mode'])} - {len(results)} URLs - sitemaps: {html.escape(', '.join(meta['sitemaps']) or 'none')}</div>
{''.join(f'<div class="mut">Note: {html.escape(n)}</div>' for n in meta['notes'])}
<div class="sum">{summary}</div>
<table><tr><th>Verdict</th><th>URL</th><th>Intent</th><th>Chain</th><th>Robots</th><th>In sitemap</th><th>Severity</th><th>Why</th></tr>{''.join(rows)}</table>
</body></html>"""
    open(path, "w", encoding="utf-8").write(doc)


def main():
    ap = argparse.ArgumentParser(description="GONE - check removed URLs against intent")
    ap.add_argument("input", help="CSV of url,intent,target")
    ap.add_argument("--sitemap", action="append", default=[], help="sitemap URL or local file (repeatable)")
    ap.add_argument("--out", default="gone-report", help="output prefix (writes .csv and .html)")
    ap.add_argument("--workers", type=int, default=4)
    ap.add_argument("--delay", type=float, default=0.2, help="seconds between requests per worker")
    ap.add_argument("--max", type=int, default=2000)
    ap.add_argument("--mode", choices=["standard", "facet", "post-event"], default="standard")
    ap.add_argument("--default-intent", default=None,
                    help="intent for rows with none (default 410; post-event mode defaults to unavailable_after)")
    ap.add_argument("--migration-sitemap", action="store_true",
                    help="the sitemap is an old-URL sitemap kept on purpose during a site move: listing is Low, not High")
    ap.add_argument("--no-robots", action="store_true", help="skip the robots.txt check")
    a = ap.parse_args()

    default_intent = a.default_intent or ("unavailable_after" if a.mode == "post-event" else "404" if a.mode == "facet" else "410")
    rows = read_input(a.input, default_intent)[: a.max]
    if not rows:
        sys.exit("no URLs in input")
    s = requests.Session()
    s.headers["User-Agent"] = UA
    notes, smap = [], set()
    for sm in a.sitemap:
        smap |= fetch_sitemap(sm, s, notes=notes)
    if a.sitemap:
        print(f"sitemap URLs loaded: {len(smap)}", file=sys.stderr)

    with ThreadPoolExecutor(max_workers=a.workers) as ex:
        res = list(ex.map(lambda r: check_url(r["url"], s, a.delay), rows))
        if a.no_robots:
            blocked = [None] * len(rows)
        else:
            blocked = list(ex.map(lambda r: robots_blocked(r["url"], s), rows))

    out = []
    for row, r, bl in zip(rows, res, blocked):
        insm = norm(row["url"]) in smap if a.sitemap else None
        v, sev, reasons = verdict(row, r, insm, a.mode, bl, a.migration_sitemap)
        out.append({
            "url": row["url"], "intent": row["intent"], "target": row["target"], "verdict": v,
            "severity": sev, "mode": a.mode,
            "robots_txt": "" if bl is None else ("blocked" if bl else "allowed"),
            "first_status": r["chain"][0]["status"] if r.get("chain") else "",
            "final_status": r.get("final_status", ""), "final_url": r.get("final_url", ""),
            "hops": max(len(r.get("chain", [])) - 1, 0), "chain": chain_str(r.get("chain", [])),
            "x_robots_tag": r.get("x_robots_tag", ""), "meta_robots": r.get("meta_robots", ""),
            "canonical": r.get("canonical", ""),
            "in_sitemap": "" if insm is None else ("yes" if insm else "no"),
            "reasons": " ; ".join(reasons),
        })
    fields = list(out[0].keys())
    with open(a.out + ".csv", "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        w.writerows(out)
    meta = {"run_at": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"), "sitemaps": a.sitemap, "notes": notes, "mode": a.mode}
    write_html(a.out + ".html", out, meta)
    tally = {}
    for o in out:
        tally[o["verdict"]] = tally.get(o["verdict"], 0) + 1
    print(json.dumps(tally), file=sys.stderr)
    print(f"wrote {a.out}.csv and {a.out}.html", file=sys.stderr)
    for n in notes:
        print("note:", n, file=sys.stderr)


if __name__ == "__main__":
    main()