"""Merge Redfin search-page snapshots (data/redfin_search/*.json, from pipeline/redfin_search.js) into
data/redfin.json, keyed by county prop_id, for build.py.

Each snapshot is "every home built 2015+ that is for sale now or sold in the last year" per city. Snapshots
accumulate: a home seen in several runs keeps every observation (status/price by date), so list → pending →
sold and price cuts build up over time.

Matching: house number + street name + unit (+ city), falling back to the nearest tracked parcel within 30 m
with the same house number. Unmatched homes (mostly detached SFR, not middle housing) are kept in
data/redfin_unmatched.json for reference.
"""
import datetime, glob, json, math, os, re, collections
from common import ROOT, load

SUFFIX = {"STREET": "ST", "AVENUE": "AVE", "ROAD": "RD", "DRIVE": "DR", "LANE": "LN", "PLACE": "PL", "COURT": "CT",
          "CIRCLE": "CIR", "WAY": "WAY", "LOOP": "LOOP", "BOULEVARD": "BLVD", "TERRACE": "TER", "PARKWAY": "PKWY"}
MONTHS = {m: i + 1 for i, m in enumerate("JAN FEB MAR APR MAY JUN JUL AUG SEP OCT NOV DEC".split())}


def from_url(url):
    """https://www.redfin.com/WA/Bellingham/4691-Wade-St-98226/unit-4/home/15751513 -> ('4691 Wade St', 'Bellingham', '4')"""
    m = re.search(r"/WA/([^/]+)/([^/]+?)-(\d{5})(?:/unit-([^/]+))?/home/", url)
    if not m:
        return None, None, None
    return m[2].replace("-", " "), m[1].replace("-", " "), m[4]


def key(addr, unit=None):
    s = re.sub(r"\(.*?\)", " ", (addr or "").upper())                         # "(LOT 48)"
    m = re.search(r"\s(?:#|UNIT|APT|STE)\s*([\w-]+)\s*$", s)
    if m:
        unit, s = unit or m[1], s[:m.start()]
    w = [SUFFIX.get(t, t) for t in re.sub(r"[^\w\s]", " ", s).split()]
    if not w or not w[0][0].isdigit():
        return None
    name = [t for t in w[1:] if t not in ("N", "S", "E", "W", "NE", "NW", "SE", "SW")]
    return (w[0], name[0] if name else "", (unit or "").upper().lstrip("#").lstrip("0") or None)


def sold_iso(s):
    m = re.match(r"([A-Z]{3}) (\d{1,2}), (\d{4})", s or "")
    return datetime.date(int(m[3]), MONTHS[m[1]], int(m[2])).isoformat() if m else None


def dist_m(a, b, c, d):
    return math.hypot((a - c) * 111_000, (b - d) * 111_000 * math.cos(math.radians(a)))


if __name__ == "__main__":
    t = open(os.path.join(ROOT, "ui", "data", "data.js")).read()
    data = json.loads(t[t.index("{"):t.rstrip().rstrip(";").rindex("}") + 1])
    units = data["units"]
    by_key = collections.defaultdict(list)
    for u in units:
        k = key(u["addr"])
        if k:
            by_key[(k, u["city"].upper())].append(u)
            if k[2]:  # also index without unit, for Redfin rows that drop it
                by_key[((k[0], k[1], None), u["city"].upper())].append(u)

    obs = collections.defaultdict(list)   # redfin id -> observations
    for fn in sorted(glob.glob(os.path.join(ROOT, "data", "redfin_search", "*.json"))):
        for r in json.load(open(fn)):
            ua, uc, uu = from_url(r["url"])
            r["addr"] = r.get("addr") or ua
            r["city"] = r.get("city") or uc
            r["unit"] = uu
            r["sold_iso"] = sold_iso(r.get("sold_date"))
            # snapshots before 2026-10-30's fix could take a "$5000 credit" from the description as the price
            if (r.get("price_text") or 0) < 50000 and not r.get("price"):
                r["price_text"] = None
            obs[r["id"]].append(r)

    out, unmatched, how = {}, [], collections.Counter()
    for rid, rs in obs.items():
        r = rs[-1]
        k = key(r["addr"], r["unit"])
        cands = by_key.get((k, (r["city"] or "").upper())) if k else None
        if not cands and k and k[2]:
            cands = by_key.get(((k[0], k[1], None), (r["city"] or "").upper()))
        if cands and len(cands) > 1 and r.get("lat"):
            cands = sorted(cands, key=lambda u: dist_m(r["lat"], r["lon"], u["lat"] or 0, u["lon"] or 0))[:1]
        if cands:
            how["address"] += 1
        elif r.get("lat") and k:
            near = [u for u in units if u.get("lat") and key(u["addr"]) and key(u["addr"])[0] == k[0]
                    and dist_m(r["lat"], r["lon"], u["lat"], u["lon"]) < 30]
            cands = near[:1]
            if cands:
                how["nearby"] += 1
        if not cands:
            unmatched.append(r)
            continue
        u = cands[0]
        events = sorted({(x["seen"], "sold" if x["sold"] or x.get("sold_iso") else "listed", x.get("sold_iso"),
                          x.get("price") or x.get("price_text"), x.get("status"), x.get("days_listed")) for x in rs})
        out[str(u["pid"])] = dict(
            id=rid, url=r["url"], addr=r["addr"], status=r.get("status"), sold=bool(r["sold"] or r.get("sold_iso")),
            sold_date=r.get("sold_iso"), price=r.get("price") or r.get("price_text"), beds=r.get("beds"), baths=r.get("baths"),
            sqft=r.get("sqft") or r.get("sqft_text"), days_listed=r.get("days_listed"), blurb=r.get("blurb"),
            photos=[r["photo"]] if r.get("photo") else [],
            source=r.get("source"), seen=r["seen"],
            history=[dict(seen=e[0], kind=e[1], sold_date=e[2], price=e[3], status=e[4], days_listed=e[5]) for e in events],
        )
    json.dump(out, open(os.path.join(ROOT, "data", "redfin.json"), "w"), indent=0)
    json.dump(unmatched, open(os.path.join(ROOT, "data", "redfin_unmatched.json"), "w"))
    types = collections.Counter(next(u["type"] for u in units if str(u["pid"]) == pid) for pid in out)
    print(f"{len(obs)} Redfin homes -> {len(out)} matched to tracked parcels ({dict(how)}), {len(unmatched)} unmatched")
    print("matched by type:", types.most_common())
    print("matched sold/active:", collections.Counter("sold" if v["sold"] else "active" for v in out.values()))