#!/usr/bin/env python3
"""verify_js_url.py — render a URL WITH JavaScript via Apify and print the page text.

Why: some careers boards (Ashby, embedded Greenhouse, Workday, chime.com...)
return an empty shell without JS, so sweep sessions can't verify postings and
dump them into Manual Review blind. This renders the real page in a headless
browser (Apify's official apify/website-content-crawler actor) and extracts
the visible text. (Not apify/web-scraper or puppeteer-scraper: since they run
arbitrary page code, Apify now requires a one-time full-account-permission
approval in the console before the API will start them.)

Usage (on the Mac — Cowork sweep sandboxes block api.apify.com):
    python3 verify_js_url.py <url> [<url2> ...]   # print extracted text per URL
    python3 verify_js_url.py --json <url> ...     # print JSON: title/text/cost/run_id
    python3 verify_js_url.py --force <url>        # ignore the daily render cap

Budget: every render increments .verify_state.json's daily counter, shared
with the dashboard server's verifier loop. Over the cap (config
verifier.max_renders_per_day, default 10) this script refuses unless --force.
Each run's cost is appended to .apify_log.txt.

Sweep sessions that CANNOT reach api.apify.com should queue URLs instead by
adding entries to .verify_queue.json in this folder:

    [{"url": "https://...", "note": "why it needs a JS render", "added": "YYYY-MM-DD"}]

The dashboard server's verifier loop (see serve_dashboard.py) renders queued
URLs within ~30 min and writes each entry's "result" back into the same file
(status, page title, dead-signals found, first 4000 chars of text, cost).

Also imported by serve_dashboard.py for render() and the decision helpers
(board_url / dead_signals / role_on_page / decide). Stdlib only.
"""

import json
import re
import ssl
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from datetime import date, datetime
from pathlib import Path

def _jobdash_data_dir(_default):
    """Pi deployment: use the Nextcloud data-dir copy of the Job Search folder
    when jobdash_pi.json (written by deploy_to_pi.py) sits next to this script."""
    try:
        import json as _json
        return Path(_json.loads(
            (_default / "jobdash_pi.json").read_text())["data_dir"])
    except Exception:
        return _default
HERE = _jobdash_data_dir(Path(__file__).resolve().parent)
CONFIG_PATH = HERE / ".apify_config.json"
STATE_PATH = HERE / ".verify_state.json"
LOG_PATH = HERE / ".apify_log.txt"
API = "https://api.apify.com/v2"

SSL_REMEDY = ("no usable CA bundle for SSL verification — "
              "run: /Applications/Python*/Install Certificates.command "
              "or pip3 install certifi")

# Rows whose Notes match this need a JS-render verification…
NEEDS_VERIFY_RE = re.compile(
    r"unverifiab|cannot verify|can'?t verify|js[- ]?only|js[- ]?block|"
    r"javascript[- ]?(only|blocked|render|wall)|blocked by (js|javascript)|"
    r"requires? (js|javascript)", re.I)
# …unless they already carry a verifier verdict marker.
ALREADY_RE = re.compile(r"via js render|js render \d{4}-\d{2}-\d{2}", re.I)

# Signals on a DETAIL page that a posting is gone. Deliberately conservative.
DEAD_PATTERNS = [
    "no longer accepting applications", "no longer available", "no longer active",
    "no longer open", "job not found", "position not found", "posting not found",
    "page not found", "job you are looking for", "job you're looking for",
    "position you are looking for", "posting has closed", "job has closed",
    "position has been filled", "this job is closed", "this position is closed",
    "this posting is closed", "job posting has expired",
    "404 not found", "error 404", "404 error",
]

_STOP = {"the", "a", "an", "and", "or", "of", "for", "to", "in", "on", "at",
         "with", "i", "ii", "iii", "iv", "sr", "jr", "new", "us", "usa", "remote"}

class ApifyError(RuntimeError):
    pass


def load_config():
    if not CONFIG_PATH.exists():
        raise ApifyError(f"missing {CONFIG_PATH.name} next to this script")
    return json.loads(CONFIG_PATH.read_text())


_SSL_CTX = None


def get_ssl_context():
    """Verified SSL context, tried in order: (1) certifi bundle if importable,
    (2) macOS system bundle /etc/ssl/cert.pem, (3) stdlib default context.
    NEVER falls back to unverified SSL — these requests carry the API token.
    If nothing works, logs a one-line remedy to .apify_log.txt and raises."""
    global _SSL_CTX
    if _SSL_CTX is not None:
        return _SSL_CTX
    try:
        import certifi
        _SSL_CTX = ssl.create_default_context(cafile=certifi.where())
        return _SSL_CTX
    except Exception:
        pass
    try:
        if Path("/etc/ssl/cert.pem").exists():
            _SSL_CTX = ssl.create_default_context(cafile="/etc/ssl/cert.pem")
            return _SSL_CTX
    except Exception:
        pass
    try:
        _SSL_CTX = ssl.create_default_context()
        return _SSL_CTX
    except Exception:
        log_line("verify", SSL_REMEDY)
        raise ApifyError(SSL_REMEDY)


def _api(path, token, method="GET", body=None, timeout=90):
    url = f"{API}{path}{'&' if '?' in path else '?'}token={token}"
    data = json.dumps(body).encode() if body is not None else None
    req = urllib.request.Request(url, data=data, method=method,
                                 headers={"Content-Type": "application/json"})
    try:
        with urllib.request.urlopen(req, timeout=timeout,
                                    context=get_ssl_context()) as r:
            return json.loads(r.read().decode())
    except urllib.error.HTTPError as e:
        detail = e.read().decode(errors="replace")[:400]
        if e.code == 401:
            raise ApifyError("Apify token rejected (401) — rotate it at apify.com "
                             "> Settings > API & Integrations, update .apify_config.json")
        if e.code == 402 or "usage-limit" in detail or "not enough" in detail.lower():
            raise ApifyError("Apify credit exhausted (402) — wait for the monthly "
                             "$5 free credit reset or add billing at apify.com")
        raise ApifyError(f"Apify API HTTP {e.code} on {path}: {detail}")
    except urllib.error.URLError as e:
        if isinstance(getattr(e, "reason", None), ssl.SSLCertVerificationError):
            log_line("verify", SSL_REMEDY)
            raise ApifyError(f"SSL certificate verification failed reaching "
                             f"api.apify.com ({e.reason}). {SSL_REMEDY}")
        raise ApifyError(f"cannot reach api.apify.com ({e.reason}) — sandboxed "
                         "sessions are blocked; run on the Mac, or queue the URL "
                         "in .verify_queue.json for the dashboard server")


def render(urls, cfg=None):
    """Render URLs in a headless browser via apify/website-content-crawler.

    Returns (items, cost_usd, run_id, status); items are dicts
    {url, title, text} ordered to match the requested urls (missing pages
    yield a placeholder with empty text).

    Live-tested 2026-07-09 against careers.chime.com (a JS-only page):
    8,907 chars of rendered text, $0.0036, ~30 s.
    """
    cfg = cfg or load_config()
    v = cfg.get("verifier", {})
    token = cfg["token"]
    actor = v.get("actor_path", "apify~website-content-crawler")
    inp = {
        "startUrls": [{"url": u} for u in urls],
        "crawlerType": "playwright:firefox",
        "maxCrawlDepth": 0,
        "maxCrawlPages": len(urls),
        "maxResults": len(urls),
        "proxyConfiguration": {"useApifyProxy": True},
        "removeCookieWarnings": True,
        "saveHtml": False,
        "saveMarkdown": False,
    }
    mem = int(v.get("memory_mbytes", 2048))
    run_timeout = int(v.get("run_timeout_seconds", 300))
    run = _api(f"/acts/{actor}/runs?memory={mem}&timeout={run_timeout}",
               token, "POST", inp)["data"]
    run_id, ds_id = run["id"], run["defaultDatasetId"]
    status = run["status"]
    deadline = time.time() + run_timeout + 60
    while status in ("READY", "RUNNING") and time.time() < deadline:
        time.sleep(6)
        run = _api(f"/actor-runs/{run_id}", token)["data"]
        status = run["status"]
    cost = float(run.get("usageTotalUsd") or 0.0)
    raw = _api(f"/datasets/{ds_id}/items?clean=true&format=json", token) or []
    norm = []
    for i in raw:
        if isinstance(i, dict):
            norm.append({"url": str(i.get("url", "")),
                         "title": str((i.get("metadata") or {}).get("title", "")),
                         "text": str(i.get("text", ""))})
    by_url = {n["url"].rstrip("/"): n for n in norm}
    items = []
    for u in urls:
        it = by_url.get(u.rstrip("/"))
        if it is None and norm and len(urls) == 1:
            it = norm[0]  # single-URL run that redirected
        items.append(it or {"url": u, "title": "", "text": ""})
    return items, cost, run_id, status


# ---------- decision helpers (pure — used by serve_dashboard's verifier) ----------

def board_url(posting_url):
    """Best-effort URL of the company's own careers-board listing page."""
    p = urllib.parse.urlparse(posting_url)
    host = p.netloc.lower()
    parts = [s for s in p.path.split("/") if s]
    root = f"{p.scheme}://{p.netloc}/"
    if "myworkdayjobs.com" in host and "/job/" in p.path:
        return f"{p.scheme}://{p.netloc}{p.path.split('/job/')[0]}"
    for ats in ("greenhouse.io", "lever.co", "ashbyhq.com",
                "smartrecruiters.com", "workable.com", "recruitee.com"):
        if ats in host:
            return f"{p.scheme}://{p.netloc}/{parts[0]}" if parts else root
    return root


def title_tokens(role):
    role = re.sub(r"\(.*?\)", " ", str(role or ""))
    toks = [t for t in re.findall(r"[a-z0-9]+", role.lower())
            if len(t) > 2 and t not in _STOP]
    return toks


def role_on_page(role, text):
    """True if some line of the rendered text looks like this role title."""
    toks = title_tokens(role)
    if not toks or not text:
        return False
    for line in text.lower().splitlines():
        if not line.strip():
            continue
        hits = sum(1 for t in toks if t in line)
        if hits == len(toks) or (len(toks) >= 3 and hits / len(toks) >= 0.75):
            return True
    return False


def dead_signals(text):
    tl = (text or "").lower()
    return [pat for pat in DEAD_PATTERNS if pat in tl]


def decide(role, board_text, detail_text):
    """Verdict for a Manual Review row. Returns (verdict, evidence);
    verdict is 'verified' | 'dead' | 'inconclusive'. Conservative: anything
    mixed stays 'inconclusive' and is left for Sohaib's judgment."""
    dead = dead_signals(detail_text)
    on_detail = role_on_page(role, detail_text)
    on_board = role_on_page(role, board_text)
    d_len = len((detail_text or "").strip())
    b_len = len((board_text or "").strip())
    if d_len < 80 and b_len < 80:
        # both pages near-empty — a failed/blocked render, NOT evidence of death
        return "inconclusive", "render returned little/no text (failed render?)"
    if dead and not on_detail:
        return "dead", f"detail page says {dead[0]!r}"
    if on_board and not dead:
        where = "board and detail page" if on_detail else "board; detail page loads clean"
        return "verified", f"role title found on company {where}"
    if not on_board and not on_detail:
        if b_len >= 80:  # board rendered real content and the role isn't on it
            return "dead", ("role absent from company board and detail page"
                            + (f"; detail says {dead[0]!r}" if dead else ""))
        return "inconclusive", "role not found but board render was empty"
    return "inconclusive", "mixed signals (e.g. role on detail page but not on board)"


# ---------- shared budget counter ----------

def load_state():
    today = date.today().isoformat()
    try:
        st = json.loads(STATE_PATH.read_text())
    except Exception:
        st = {}
    if st.get("date") != today:
        st = {"date": today, "renders": 0}
    return st


def save_state(st):
    STATE_PATH.write_text(json.dumps(st))


def log_line(tag, msg):
    line = f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S')} [{tag}] {msg}\n"
    try:
        with open(LOG_PATH, "a") as f:
            f.write(line)
    except Exception:
        pass


# ---------- CLI ----------

def main(argv):
    args = [a for a in argv if not a.startswith("--")]
    as_json = "--json" in argv
    force = "--force" in argv
    if not args:
        print(__doc__)
        return 2
    cfg = load_config()
    cap = int(cfg.get("verifier", {}).get("max_renders_per_day", 10))
    st = load_state()
    if st["renders"] + len(args) > cap and not force:
        print(f"ERROR: daily render cap would be exceeded "
              f"({st['renders']}/{cap} used today; asking for {len(args)}). "
              f"Re-run with --force to override.", file=sys.stderr)
        return 3
    st["renders"] += len(args)
    save_state(st)
    items, cost, run_id, status = render(args, cfg)
    log_line("verify-cli", f"rendered {len(args)} url(s) run={run_id} "
                           f"status={status} cost=${cost:.4f} "
                           f"({st['renders']}/{cap} renders today)")
    if as_json:
        print(json.dumps({"run_id": run_id, "status": status,
                          "cost_usd": cost, "items": items}, indent=2))
    else:
        for it in items:
            print(f"===== {it['url']}")
            print(f"----- title: {it.get('title', '')}")
            sig = dead_signals(it.get("text", ""))
            if sig:
                print(f"----- dead-signals: {sig}")
            print(it.get("text", "") or "(no text extracted)")
            print()
        print(f"[run {run_id} {status}; cost ${cost:.4f}; "
              f"{st['renders']}/{cap} renders today]", file=sys.stderr)
    return 0 if status == "SUCCEEDED" else 1


if __name__ == "__main__":
    try:
        sys.exit(main(sys.argv[1:]))
    except ApifyError as e:
        print(f"ERROR: {e}", file=sys.stderr)
        sys.exit(1)