#!/usr/bin/env python3
# =============================================================================
# QWHPI — Quebec Weekly Housing Price Index
# Author  : Simon-Pierre Boucher
# Contact : contact@spboucher.ai
# File    : engine/scripts/03_clean.py
# Purpose : Step 3 — build the flagged clean table, write the full exclusion
#           table and cleaning report. Nothing is silently deleted.
# =============================================================================
"""Cleaning build (Execution Order step 3)."""

from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

import polars as pl

from qwhpi.clean import (
    FLOORAREA_HI, FLOORAREA_LO, PRICE_CEILING, RATIO_HI, RATIO_LO,
    build_clean, research_sample,
)
from qwhpi.config import REPORTS_DIR, TABLES_DIR, ensure_dirs
from qwhpi.geography import build_geo_join
from qwhpi.ingest import load_raw

LINES: list[str] = []


def note(line: str = "") -> None:
    LINES.append(line)
    print(line)


def main() -> None:
    ensure_dirs()
    tx = load_raw()
    geo = build_geo_join(tx)
    df = build_clean(tx, geo, force=True)

    note("## Cleaning report")
    note("")
    note(f"- Input rows: {df.height:,} (all retained; exclusions are flags)")
    note(f"- Screens: ratio band [{RATIO_LO}, {RATIO_HI}] on amount/totalArValue; "
         f"price ceiling ${PRICE_CEILING:,}; duplicate flags; "
         f"floorArea sanitation [{FLOORAREA_LO}, {FLOORAREA_HI}] m².")
    note("- Provider pre-filter verified: no transaction below $50,000 exists in the raw file.")
    note("")

    # ------------------------------------------------------------------ #
    # Exclusion table
    # ------------------------------------------------------------------ #
    excl = (
        df.filter(pl.col("exclude_reason").is_not_null())
        .group_by("exclude_reason").len()
        .rename({"len": "n"})
        .with_columns(pl.col("n").cast(pl.Int64))
        .with_columns((pl.col("n") / df.height * 100).round(3).alias("pct_of_raw"))
        .sort("n", descending=True)
    )
    total_excl = excl["n"].sum() if excl.height else 0
    excl_full = pl.concat([
        excl,
        pl.DataFrame({
            "exclude_reason": ["TOTAL_EXCLUDED", "RETAINED_IN_RESEARCH_SAMPLE"],
            "n": [total_excl, df.height - total_excl],
            "pct_of_raw": [round(total_excl / df.height * 100, 3),
                           round((df.height - total_excl) / df.height * 100, 3)],
        }),
    ])
    excl_full.write_csv(TABLES_DIR / "exclusions.csv")
    note("### Exclusion table (outputs/tables/exclusions.csv)")
    for row in excl_full.iter_rows(named=True):
        note(f"- {row['exclude_reason']}: {row['n']:,} ({row['pct_of_raw']}%)")

    # Sanitation summary (rows kept, characteristic nulled).
    note("")
    note("### Characteristic sanitation (row retained, value nulled)")
    note(f"- floorArea outside [{FLOORAREA_LO}, {FLOORAREA_HI}] m²: "
         f"{df.filter(pl.col('fa_suspect')).height:,}")
    note(f"- yearBuilt > sale year + 1: {df.filter(pl.col('yb_suspect')).height:,}")

    # ------------------------------------------------------------------ #
    # Research sample summary
    # ------------------------------------------------------------------ #
    rs = research_sample(df)
    rs_all = research_sample(df, include_undetermined=True)
    note("")
    note("### Research sample")
    note(f"- Clean rows incl. indéterminé: {rs_all.height:,}")
    note(f"- Headline sample (3 determined types): {rs.height:,} "
         f"({rs.height / df.height * 100:.2f}% of raw)")
    by_type = rs.group_by("propertyType").len().sort("len", descending=True)
    for row in by_type.iter_rows(named=True):
        note(f"  - {row['propertyType']}: {row['len']:,}")

    # Weekly volume of the research sample (continuity check).
    weekly = rs.group_by("week").len().sort("week")
    note(f"- Weeks with >=1 research-sample transaction: {weekly.height} / 291")
    note(f"- Median weekly research-sample volume: {weekly['len'].median():,.0f}")

    # Missingness within the research sample (for the hedonic step).
    miss = pl.DataFrame({
        "column": ["floorArea", "yearBuilt", "buildingType", "ownerType"],
        "pct_missing": [
            round(rs["floorArea"].is_null().mean() * 100, 2),
            round(rs["yearBuilt"].is_null().mean() * 100, 2),
            round(rs["buildingType"].is_null().mean() * 100, 2),
            round(rs["ownerType"].is_null().mean() * 100, 2),
        ],
    })
    miss.write_csv(TABLES_DIR / "research_sample_missingness.csv")
    note("")
    note("### Missingness in headline research sample")
    for row in miss.iter_rows(named=True):
        note(f"- {row['column']}: {row['pct_missing']}%")

    report_path = REPORTS_DIR / "cleaning_report.md"
    header = (
        "<!--\n"
        "=============================================================================\n"
        "QWHPI — Quebec Weekly Housing Price Index\n"
        "Author  : Simon-Pierre Boucher\n"
        "Contact : contact@spboucher.ai\n"
        "File    : outputs/reports/cleaning_report.md\n"
        "Purpose : Step 3 cleaning report (generated by engine/scripts/03_clean.py)\n"
        "=============================================================================\n"
        "-->\n\n"
    )
    report_path.write_text(header + "\n".join(LINES) + "\n", encoding="utf-8")
    print(f"\nReport written to {report_path}")


if __name__ == "__main__":
    main()