#!/usr/bin/env python3
# =============================================================================
# QWHPI — Quebec Weekly Housing Price Index
# Author  : Simon-Pierre Boucher
# Contact : contact@spboucher.ai
# File    : engine/scripts/02_geography.py
# Purpose : Step 2 — spatial join of all transactions to official boundaries,
#           validation against the raw city text field, persistence.
# =============================================================================
"""Geography build (Execution Order step 2).

Runs the spatial join (cached in ``data/processed/geo_join.parquet``),
validates the result against the raw ``city`` text field, and writes
validation tables + a report section.
"""

from __future__ import annotations

import sys
import time
import unicodedata
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

import polars as pl

from qwhpi.config import REPORTS_DIR, TABLES_DIR, ensure_dirs
from qwhpi.geography import SDA_VERSION, build_geo_join
from qwhpi.ingest import load_raw

LINES: list[str] = []


def note(line: str = "") -> None:
    LINES.append(line)
    print(line)


def norm_name(s: str) -> str:
    """Accent/case/punctuation-insensitive normalization for name comparison."""
    if s is None:
        return ""
    s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode()
    return (
        s.lower().replace("saint-", "st-").replace("sainte-", "ste-")
        .replace("'", "").replace("’", "").replace(".", "").replace(" ", "-")
    )


def main() -> None:
    ensure_dirs()
    tx = load_raw()

    t0 = time.time()
    geo = build_geo_join(tx)
    note("## Geography spatial join (SDA 1/20k, version " + SDA_VERSION + ")")
    note("")
    note(f"- Join computed/loaded in {time.time() - t0:.1f}s for {geo.height:,} transactions")

    method_counts = geo.group_by("join_method").len().sort("len", descending=True)
    for row in method_counts.iter_rows(named=True):
        note(f"- join_method={row['join_method']}: {row['len']:,} "
             f"({row['len'] / geo.height * 100:.3f}%)")

    n_regions = geo["region"].n_unique()
    n_munic = geo["geo_code"].n_unique()
    note(f"- Distinct regions matched: {n_regions} | municipalities: {n_munic:,}")

    region_dist = (
        geo.group_by(["region_code", "region"]).len()
        .sort("len", descending=True)
        .rename({"len": "n_transactions"})
    )
    region_dist.write_csv(TABLES_DIR / "geo_region_distribution.csv")
    note("")
    note("Transactions by administrative region:")
    for row in region_dist.iter_rows(named=True):
        note(f"- {row['region_code']} {row['region']}: {row['n_transactions']:,}")

    # ---------------------------------------------------------------- #
    # Validation vs raw city text field
    # ---------------------------------------------------------------- #
    joined = tx.select("id", "city").join(geo, on="id", how="left")
    comparable = joined.filter(pl.col("municipality").is_not_null())
    comp = comparable.with_columns(
        pl.col("city").map_elements(norm_name, return_dtype=pl.Utf8).alias("city_norm"),
        pl.col("municipality").map_elements(norm_name, return_dtype=pl.Utf8).alias("mun_norm"),
    ).with_columns(
        (
            (pl.col("city_norm") == pl.col("mun_norm"))
            | pl.col("mun_norm").str.contains(pl.col("city_norm"), literal=True)
            | pl.col("city_norm").str.contains(pl.col("mun_norm"), literal=True)
        ).alias("name_match")
    )
    match_rate = comp["name_match"].mean() * 100
    note("")
    note("### Validation: joined municipality vs raw `city` text")
    note(f"- Name agreement (normalized, containment-tolerant): {match_rate:.2f}%")

    mismatches = (
        comp.filter(~pl.col("name_match"))
        .group_by(["city", "municipality", "region"]).len()
        .sort("len", descending=True)
        .rename({"len": "n"})
    )
    mismatches.head(300).write_csv(TABLES_DIR / "geo_city_mismatch_top300.csv")
    note(f"- Mismatching pairs saved: outputs/tables/geo_city_mismatch_top300.csv "
         f"({mismatches.height:,} distinct pairs, {mismatches['n'].sum():,} transactions)")
    note("")
    note("Top 15 mismatch pairs (city text -> joined municipality):")
    for row in mismatches.head(15).iter_rows(named=True):
        note(f"- '{row['city']}' -> '{row['municipality']}' ({row['region']}): {row['n']:,}")

    report_path = REPORTS_DIR / "geography_report.md"
    header = (
        "<!--\n"
        "=============================================================================\n"
        "QWHPI — Quebec Weekly Housing Price Index\n"
        "Author  : Simon-Pierre Boucher\n"
        "Contact : contact@spboucher.ai\n"
        "File    : outputs/reports/geography_report.md\n"
        "Purpose : Step 2 geography join + validation report (from 02_geography.py)\n"
        "=============================================================================\n"
        "-->\n\n"
    )
    report_path.write_text(header + "\n".join(LINES) + "\n", encoding="utf-8")
    print(f"\nReport written to {report_path}")


if __name__ == "__main__":
    main()