#!/usr/bin/env python3
"""Scrape Bellingham's job-description library (cob.org) into structured data.

Steps:
  1. Parse the index table at /about/employment/job-descriptions
     -> bellingham/data/job_descriptions_index.csv
  2. Download every linked duty-statement PDF -> bellingham/raw/job-descriptions/
  3. Extract text from each PDF -> bellingham/data/job_description_texts.jsonl

Re-runnable: already-downloaded PDFs are skipped.
"""
import csv
import json
import re
import time
from pathlib import Path

import pdfplumber
import requests
from bs4 import BeautifulSoup

ROOT = Path(__file__).resolve().parent.parent
RAW = ROOT / "bellingham/raw/job-descriptions"
DATA = ROOT / "bellingham/data"
INDEX_URL = "https://cob.org/about/employment/job-descriptions"
UA = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
      "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"}


def scrape_index():
    html = requests.get(INDEX_URL, headers=UA, timeout=60).text
    soup = BeautifulSoup(html, "html.parser")
    table = soup.find("table")
    headers = [th.get_text(strip=True) for th in table.find_all("th")]
    rows = []
    for tr in table.find_all("tr"):
        tds = tr.find_all("td")
        if not tds:
            continue
        cells = [td.get_text(strip=True) for td in tds]
        link = tr.find("a", href=True)
        url = link["href"] if link else None
        if url and url.startswith("/"):
            url = "https://cob.org" + url
        rec = dict(zip(headers, cells)) if headers else {}
        rec["_cells"] = cells
        rec["pdf_url"] = url
        rows.append(rec)
    return headers, rows


def main():
    RAW.mkdir(parents=True, exist_ok=True)
    DATA.mkdir(parents=True, exist_ok=True)
    headers, rows = scrape_index()
    print("index columns:", headers)
    print("rows:", len(rows), " with pdf:", sum(1 for r in rows if r["pdf_url"]))

    fieldnames = (headers or []) + ["pdf_url", "pdf_file"]
    session = requests.Session()
    session.headers.update(UA)
    n_dl, n_skip, n_err = 0, 0, 0
    for r in rows:
        r.pop("_cells", None)
        r["pdf_file"] = ""
        if not r["pdf_url"]:
            continue
        name = re.sub(r"[^A-Za-z0-9._-]", "_", r["pdf_url"].split("/")[-1])
        dest = RAW / name
        r["pdf_file"] = name
        if dest.exists() and dest.stat().st_size > 500:
            n_skip += 1
            continue
        try:
            resp = session.get(r["pdf_url"], timeout=60)
            resp.raise_for_status()
            dest.write_bytes(resp.content)
            n_dl += 1
            time.sleep(0.25)
        except Exception as e:
            print("ERR", r["pdf_url"], e)
            n_err += 1
    print(f"downloaded {n_dl}, skipped {n_skip}, errors {n_err}")

    with open(DATA / "job_descriptions_index.csv", "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
        w.writeheader()
        w.writerows(rows)

    with open(DATA / "job_description_texts.jsonl", "w") as f:
        n_txt = 0
        for r in rows:
            if not r["pdf_file"]:
                continue
            dest = RAW / r["pdf_file"]
            if not dest.exists():
                continue
            try:
                with pdfplumber.open(dest) as pdf:
                    text = "\n".join((p.extract_text() or "") for p in pdf.pages)
            except Exception as e:
                text = ""
                print("TEXT ERR", r["pdf_file"], e)
            f.write(json.dumps({**r, "text": text}) + "\n")
            n_txt += 1
    print(f"extracted text for {n_txt} PDFs")


if __name__ == "__main__":
    main()