"""Shared helpers for the pipeline. Run every script from the project root: python3 pipeline/<script>.py"""
import json, os, time, urllib.request, urllib.parse, datetime

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RAW = os.path.join(ROOT, "data", "raw")
CACHE = os.path.join(ROOT, "cache")
UA = {"User-Agent": "bellingham-middle-housing personal research script (python)"}
os.makedirs(RAW, exist_ok=True)


def get(url, timeout=60):
    return urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=timeout).read().decode("utf-8", "replace")


def arcgis_query(url, where, out_fields="*", geom=False, page=1000, order="OBJECTID", **extra):
    """Page through an ArcGIS REST layer query. Returns attribute dicts (plus lon/lat when geom)."""
    rows, off = [], 0
    while True:
        params = {"where": where, "outFields": out_fields, "returnGeometry": str(geom).lower(), "outSR": "4326",
                  "f": "json", "resultOffset": off, "resultRecordCount": page, "orderByFields": order, **extra}
        body = urllib.parse.urlencode(params).encode()
        d = json.loads(urllib.request.urlopen(urllib.request.Request(url + "/query", data=body, headers=UA), timeout=120).read())
        if "error" in d:
            raise RuntimeError(d["error"])
        feats = d.get("features", [])
        for f in feats:
            a = f["attributes"]
            if geom:
                g = f.get("geometry") or {}
                if "x" in g:
                    a["lon"], a["lat"] = g["x"], g["y"]
                elif "rings" in g and g["rings"]:  # polygon -> rough centroid of first ring
                    r = g["rings"][0]
                    a["lon"], a["lat"] = sum(p[0] for p in r) / len(r), sum(p[1] for p in r) / len(r)
            rows.append(a)
        if len(feats) < page and not d.get("exceededTransferLimit"):
            return rows
        off += len(feats)
        time.sleep(0.2)


def epoch_to_date(v):
    if isinstance(v, (int, float)):
        return datetime.datetime.fromtimestamp(v / 1000, datetime.timezone.utc).date().isoformat()
    return v


def save(name, obj):
    path = os.path.join(RAW, name)
    json.dump(obj, open(path, "w"))
    return path


def load(name):
    return json.load(open(os.path.join(RAW, name)))


def norm_addr(s):
    """'1447 PLATEAU PL  ' / '1447 Plateau Pl #B' -> '1447 PLATEAU PL' (unit stripped)."""
    if not s:
        return ""
    s = " ".join(str(s).upper().replace(".", "").replace(",", " ").split())
    for sep in (" #", " UNIT ", " APT ", " STE "):
        if sep in s:
            s = s.split(sep)[0]
    return s