spb/wp5_uqo Public

1# Author: Simon-Pierre Boucher — contact@spboucher.ai2"""301_inspect_raw_data.py4----------------------5Load both raw datasets (data/raw/airbnb.csv, data/raw/rent.json), print6inspection summaries, and save a data-dictionary / inspection log to7results/logs/data_inspection.txt.89Note: the raw files are not distributed with the repository (see10data/raw/README.md). This step is only needed to rebuild the processed11parquet files from scratch; results/logs/data_inspection.txt already12contains its output from the original run.13"""1415import json16import sys17from io import StringIO18from pathlib import Path1920sys.path.insert(0, str(Path(__file__).resolve().parent.parent))2122import pandas as pd2324from src.config import AIRBNB_RAW, RENT_RAW, LOG_DIR, require2526RAW_HINT = ("Place the original raw files in data/raw/ to run steps 01-03. "27            "The processed parquet files in data/processed/ already allow "28            "running steps 04-10.")2930LOG_PATH = LOG_DIR / "data_inspection.txt"3132# ── helper to capture + print simultaneously ─────────────────────────────────33log_buf = StringIO()343536def log(msg: str = "") -> None:37    """Print to stdout and buffer for the log file."""38    print(msg)39    log_buf.write(msg + "\n")404142def inspect_airbnb() -> None:43    """Inspect the raw Airbnb CSV: dtypes, nulls, summaries, value counts."""44    log("=" * 80)45    log("AIRBNB DATASET  —  airbnb.csv")46    log("=" * 80)4748    ab = pd.read_csv(AIRBNB_RAW)4950    log(f"\nShape: {ab.shape[0]} rows x {ab.shape[1]} columns\n")5152    # dtypes53    log("--- Dtypes ---")54    for col in ab.columns:55        log(f"  {col:30s}  {str(ab[col].dtype)}")5657    # nulls58    log("\n--- Null counts ---")59    nulls = ab.isnull().sum()60    for col in ab.columns:61        n = nulls[col]62        pct = 100 * n / len(ab)63        log(f"  {col:30s}  {n:6d}  ({pct:5.1f}%)")6465    # numeric summary66    log("\n--- Numeric summary ---")67    log(ab.describe().to_string())6869    # key categorical value counts70    for col in ["city", "property_type", "price_category", "rating_category"]:71        log(f"\n--- Value counts: {col} ---")72        vc = ab[col].value_counts()73        for val, cnt in vc.items():74            log(f"  {str(val):50s}  {cnt}")7576    # boolean columns77    for col in ["is_superhost", "is_guest_favorite", "pets_allowed"]:78        log(f"\n--- Value counts: {col} ---")79        vc = ab[col].value_counts()80        for val, cnt in vc.items():81            log(f"  {str(val):10s}  {cnt}")8283    # sample rows84    log("\n--- First 5 rows ---")85    log(ab.head().to_string())868788def inspect_rent() -> None:89    """Inspect the raw Realtor.ca JSON: structure, missingness, patterns."""90    log("\n" + "=" * 80)91    log("RENT DATASET  —  rent.json")92    log("=" * 80)9394    with open(RENT_RAW, "r", encoding="utf-8") as f:95        rent_raw = json.load(f)9697    log(f"\nTotal records: {len(rent_raw)}")9899    # Flatten key fields into a DataFrame for inspection100    rows = []101    for rec in rent_raw:102        prop = rec.get("Property", {})103        addr = prop.get("Address", {})104        bld = rec.get("Building", {})105        rows.append({106            "address_text": addr.get("AddressText", ""),107            "latitude": addr.get("Latitude", ""),108            "longitude": addr.get("Longitude", ""),109            "property_type": prop.get("Type", ""),110            "lease_rent": prop.get("LeaseRent", ""),111            "lease_rent_unformatted": prop.get("LeaseRentUnformattedValue", ""),112            "building_type": bld.get("Type", ""),113            "bedrooms": bld.get("Bedrooms", ""),114            "bathrooms_total": bld.get("BathroomTotal", ""),115            "size_interior": bld.get("SizeInterior", ""),116            "stories_total": bld.get("StoriesTotal", ""),117            "postal_code": rec.get("PostalCode", ""),118            "province": rec.get("ProvinceName", ""),119            "scraped_at": rec.get("scrapedAt", ""),120        })121122    rt = pd.DataFrame(rows)123124    log(f"Shape (flattened): {rt.shape[0]} rows x {rt.shape[1]} columns\n")125126    # dtypes127    log("--- Dtypes ---")128    for col in rt.columns:129        log(f"  {col:30s}  {str(rt[col].dtype)}")130131    # blanks / nulls132    log("\n--- Empty-string + null counts ---")133    for col in rt.columns:134        n_null = rt[col].isnull().sum()135        n_blank = (rt[col] == "").sum()136        total_missing = n_null + n_blank137        pct = 100 * total_missing / len(rt)138        log(f"  {col:30s}  null={n_null:5d}  blank={n_blank:5d}  "139            f"total={total_missing:5d}  ({pct:5.1f}%)")140141    # value counts for key categoricals142    for col in ["property_type", "building_type"]:143        log(f"\n--- Value counts: {col} ---")144        vc = rt[col].value_counts()145        for val, cnt in vc.items():146            log(f"  {str(val):40s}  {cnt}")147148    # lease_rent patterns149    log("\n--- LeaseRent frequency patterns (top 10) ---")150    patterns = rt["lease_rent"].str.extract(r"(\$[\d,]+/\w+)")[0].value_counts().head(10)151    for val, cnt in patterns.items():152        log(f"  {val:30s}  {cnt}")153154    # check rent period distribution155    log("\n--- LeaseRent period distribution ---")156    for period in ["Monthly", "Yearly", "Weekly"]:157        cnt = rt["lease_rent"].str.contains(period, na=False).sum()158        log(f"  {period:15s}  {cnt}")159160    # sample rows161    log("\n--- First 5 rows (flattened) ---")162    log(rt.head().to_string())163164165def main() -> None:166    require(AIRBNB_RAW, RAW_HINT)167    require(RENT_RAW, RAW_HINT)168169    inspect_airbnb()170    inspect_rent()171172    LOG_PATH.write_text(log_buf.getvalue(), encoding="utf-8")173    log(f"\n>>> Inspection log saved to {LOG_PATH}")174175176if __name__ == "__main__":177    main()178