# Author: Simon-Pierre Boucher — contact@spboucher.ai
"""
01_inspect_raw_data.py
----------------------
Load both raw datasets (data/raw/airbnb.csv, data/raw/rent.json), print
inspection summaries, and save a data-dictionary / inspection log to
results/logs/data_inspection.txt.

Note: the raw files are not distributed with the repository (see
data/raw/README.md). This step is only needed to rebuild the processed
parquet files from scratch; results/logs/data_inspection.txt already
contains its output from the original run.
"""

import json
import sys
from io import StringIO
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

import pandas as pd

from src.config import AIRBNB_RAW, RENT_RAW, LOG_DIR, require

RAW_HINT = ("Place the original raw files in data/raw/ to run steps 01-03. "
            "The processed parquet files in data/processed/ already allow "
            "running steps 04-10.")

LOG_PATH = LOG_DIR / "data_inspection.txt"

# ── helper to capture + print simultaneously ─────────────────────────────────
log_buf = StringIO()


def log(msg: str = "") -> None:
    """Print to stdout and buffer for the log file."""
    print(msg)
    log_buf.write(msg + "\n")


def inspect_airbnb() -> None:
    """Inspect the raw Airbnb CSV: dtypes, nulls, summaries, value counts."""
    log("=" * 80)
    log("AIRBNB DATASET  —  airbnb.csv")
    log("=" * 80)

    ab = pd.read_csv(AIRBNB_RAW)

    log(f"\nShape: {ab.shape[0]} rows x {ab.shape[1]} columns\n")

    # dtypes
    log("--- Dtypes ---")
    for col in ab.columns:
        log(f"  {col:30s}  {str(ab[col].dtype)}")

    # nulls
    log("\n--- Null counts ---")
    nulls = ab.isnull().sum()
    for col in ab.columns:
        n = nulls[col]
        pct = 100 * n / len(ab)
        log(f"  {col:30s}  {n:6d}  ({pct:5.1f}%)")

    # numeric summary
    log("\n--- Numeric summary ---")
    log(ab.describe().to_string())

    # key categorical value counts
    for col in ["city", "property_type", "price_category", "rating_category"]:
        log(f"\n--- Value counts: {col} ---")
        vc = ab[col].value_counts()
        for val, cnt in vc.items():
            log(f"  {str(val):50s}  {cnt}")

    # boolean columns
    for col in ["is_superhost", "is_guest_favorite", "pets_allowed"]:
        log(f"\n--- Value counts: {col} ---")
        vc = ab[col].value_counts()
        for val, cnt in vc.items():
            log(f"  {str(val):10s}  {cnt}")

    # sample rows
    log("\n--- First 5 rows ---")
    log(ab.head().to_string())


def inspect_rent() -> None:
    """Inspect the raw Realtor.ca JSON: structure, missingness, patterns."""
    log("\n" + "=" * 80)
    log("RENT DATASET  —  rent.json")
    log("=" * 80)

    with open(RENT_RAW, "r", encoding="utf-8") as f:
        rent_raw = json.load(f)

    log(f"\nTotal records: {len(rent_raw)}")

    # Flatten key fields into a DataFrame for inspection
    rows = []
    for rec in rent_raw:
        prop = rec.get("Property", {})
        addr = prop.get("Address", {})
        bld = rec.get("Building", {})
        rows.append({
            "address_text": addr.get("AddressText", ""),
            "latitude": addr.get("Latitude", ""),
            "longitude": addr.get("Longitude", ""),
            "property_type": prop.get("Type", ""),
            "lease_rent": prop.get("LeaseRent", ""),
            "lease_rent_unformatted": prop.get("LeaseRentUnformattedValue", ""),
            "building_type": bld.get("Type", ""),
            "bedrooms": bld.get("Bedrooms", ""),
            "bathrooms_total": bld.get("BathroomTotal", ""),
            "size_interior": bld.get("SizeInterior", ""),
            "stories_total": bld.get("StoriesTotal", ""),
            "postal_code": rec.get("PostalCode", ""),
            "province": rec.get("ProvinceName", ""),
            "scraped_at": rec.get("scrapedAt", ""),
        })

    rt = pd.DataFrame(rows)

    log(f"Shape (flattened): {rt.shape[0]} rows x {rt.shape[1]} columns\n")

    # dtypes
    log("--- Dtypes ---")
    for col in rt.columns:
        log(f"  {col:30s}  {str(rt[col].dtype)}")

    # blanks / nulls
    log("\n--- Empty-string + null counts ---")
    for col in rt.columns:
        n_null = rt[col].isnull().sum()
        n_blank = (rt[col] == "").sum()
        total_missing = n_null + n_blank
        pct = 100 * total_missing / len(rt)
        log(f"  {col:30s}  null={n_null:5d}  blank={n_blank:5d}  "
            f"total={total_missing:5d}  ({pct:5.1f}%)")

    # value counts for key categoricals
    for col in ["property_type", "building_type"]:
        log(f"\n--- Value counts: {col} ---")
        vc = rt[col].value_counts()
        for val, cnt in vc.items():
            log(f"  {str(val):40s}  {cnt}")

    # lease_rent patterns
    log("\n--- LeaseRent frequency patterns (top 10) ---")
    patterns = rt["lease_rent"].str.extract(r"(\$[\d,]+/\w+)")[0].value_counts().head(10)
    for val, cnt in patterns.items():
        log(f"  {val:30s}  {cnt}")

    # check rent period distribution
    log("\n--- LeaseRent period distribution ---")
    for period in ["Monthly", "Yearly", "Weekly"]:
        cnt = rt["lease_rent"].str.contains(period, na=False).sum()
        log(f"  {period:15s}  {cnt}")

    # sample rows
    log("\n--- First 5 rows (flattened) ---")
    log(rt.head().to_string())


def main() -> None:
    require(AIRBNB_RAW, RAW_HINT)
    require(RENT_RAW, RAW_HINT)

    inspect_airbnb()
    inspect_rent()

    LOG_PATH.write_text(log_buf.getvalue(), encoding="utf-8")
    log(f"\n>>> Inspection log saved to {LOG_PATH}")


if __name__ == "__main__":
    main()