"""Step 3: Whatcom County assessor pages (sales history, improvements) for every tracked parcel.

Pages are cached forever in cache/assessor/<prop_id>.html (delete a file to refresh it). ~1 request/sec.
Usage: python3 pipeline/enrich_assessor.py [--refresh-days N]   (re-fetch cached pages older than N days)
Output: data/raw/assessor.json  (prop_id -> parsed record)
"""
import html, os, re, sys, time
from common import CACHE, get, save
import universe

DIR = os.path.join(CACHE, "assessor")
os.makedirs(DIR, exist_ok=True)
URL = "https://property.whatcomcounty.us/propertyaccess/Property.aspx?cid=0&year=2026&prop_id={}"


def flat(s):
    t = re.sub(r"<script.*?</script>|<style.*?</style>", "", s, flags=re.S)
    t = html.unescape(re.sub(r"<[^>]+>", "|", t))
    t = re.sub(r"\s+", " ", t)
    return re.sub(r"(\| ?)+", "|", t)


def money(s):
    return float(s.replace("$", "").replace(",", "")) if s else None


def parse(raw):
    t = flat(raw)
    out = {}
    m = re.search(r"Owner\|Name:\|(.*?)\|.*?Mailing Address:\|(.*?)\|(.*?)\|", t)
    if m:
        out.update(owner=m[1].strip(), mail=(m[2].strip() + " " + m[3].strip()).strip())
    m = re.search(r"\(=\) Market Value:\|=\|(\$[\d,]+)", t)
    out["market_value"] = money(m[1]) if m else None
    out["improvements"] = [dict(desc=d, code=c, sqft=float(s)) for d, c, s in
                           re.findall(r"Improvement #\d+:\|(.*?)\|State Code:\|(\d+)\|([\d.]+) sqft", t)]
    sales, seen = [], set()
    s = t.split("Sales History", 1)[-1].split("Payout Agreement", 1)[0]
    for r in re.finditer(r"\|(\d+)\|(\d\d)/(\d\d)/(\d{4})\|([^|]*)\|([^|]*)\|([^|]*)\|([^|]*)\|(?:[^|$]*\|){0,2}?(\$[\d,.]+)", s):
        rec = dict(date=f"{r[4]}-{r[2]}-{r[3]}", type=r[5].strip(), desc=r[6].strip(), grantor=r[7].strip(),
                   grantee=r[8].strip(), price=money(r[9]))
        key = (rec["date"], rec["grantor"], rec["grantee"], rec["price"])
        if key not in seen:  # pages repeat some rows
            seen.add(key)
            sales.append(rec)
    out["sales"] = sorted(sales, key=lambda x: x["date"])
    return out


if __name__ == "__main__":
    refresh_days = float(sys.argv[sys.argv.index("--refresh-days") + 1]) if "--refresh-days" in sys.argv else None
    permits, parcels, link, tracked, parcel_only = universe.build()
    finaled = {link[p["PERMIT_NO"]] for p in permits if p["STATUS"] == "FINALED" and p["PERMIT_NO"] in link}
    # priority: Bellingham finaled permit parcels, then other cities, then everything else tracked
    order = sorted(tracked, key=lambda pid: (pid not in finaled, universe.city(parcels[pid]) == "Bellingham", pid))
    todo = []
    for pid in order:
        fn = os.path.join(DIR, f"{pid}.html")
        if not os.path.exists(fn) or (refresh_days and time.time() - os.path.getmtime(fn) > refresh_days * 86400):
            todo.append(pid)
    # One request at a time (the server takes ~3 s/page). 2026-10-29: 3 parallel workers drew 503s after ~650 pages,
    # so stay sequential, and stop after 3 consecutive errors; just rerun later, since the cache makes it resumable.
    print(f"{len(tracked)} tracked parcels, {len(todo)} to fetch (~{len(todo) * 4 // 60 + 1} min)", flush=True)
    errors = 0
    for i, pid in enumerate(todo):
        try:
            page = get(URL.format(pid))
            if "Sales History" in page:
                open(os.path.join(DIR, f"{pid}.html"), "w").write(page)
            errors = 0
        except Exception as e:
            errors += 1
            print("  fail", pid, repr(e)[:120], flush=True)
            if errors >= 3:
                print("  3 errors in a row: assessor site is unhappy, stopping. Rerun later to resume.", flush=True)
                break
            time.sleep(30)
        if i % 100 == 0:
            print(f"  {i}/{len(todo)}", flush=True)
        time.sleep(1)
    out = {}
    for pid in tracked:
        fn = os.path.join(DIR, f"{pid}.html")
        if os.path.exists(fn):
            out[pid] = parse(open(fn).read())
    save("assessor.json", out)
    print(len(out), "assessor records parsed")