#!/usr/bin/env python3
# Author: Simon-Pierre Boucher
# Contact: contact@spboucher.ai
#
# profile_csv.py — deterministic CSV profiler. Stdlib only.
# Usage: python3 profile_csv.py <file.csv>
# Prints a JSON profile to stdout. Exit codes: 0 = OK, 2 = unreadable input.

import csv
import json
import statistics
import sys
from collections import Counter
from pathlib import Path

# Cap keeps runtime and memory bounded on very large files; 100k rows is
# enough for stable summary statistics on any practical column.
MAX_ROWS = 100_000

# A column is typed integer/float/date/boolean only if at least 95% of its
# non-empty values parse as that type; below that, mixed content is safer
# reported as "string" than as a false-precision numeric column.
TYPE_THRESHOLD = 0.95

# Top-N most frequent values reported for string columns; 5 shows the
# dominant categories without bloating the report.
TOP_VALUES = 5

# Values beyond 3 standard deviations from the mean are counted as outliers
# (classic three-sigma rule).
OUTLIER_SIGMAS = 3


def is_int(s):
    try:
        int(s)
        return True
    except ValueError:
        return False


def is_float(s):
    try:
        float(s)
        return True
    except ValueError:
        return False


def is_bool(s):
    return s.strip().lower() in ("true", "false", "yes", "no", "0", "1")


def is_date(s):
    s = s.strip()
    for sep in ("-", "/"):
        parts = s.split(sep)
        if len(parts) == 3 and all(p.isdigit() for p in parts):
            return True
    return False


def infer_type(values):
    """Return the dominant type of non-empty values per TYPE_THRESHOLD."""
    if not values:
        return "empty"
    n = len(values)
    for name, pred in (("integer", is_int), ("float", is_float),
                       ("boolean", is_bool), ("date", is_date)):
        if sum(1 for v in values if pred(v)) / n >= TYPE_THRESHOLD:
            return name
    return "string"


def read_rows(path):
    """Read the CSV, falling back to latin-1 if UTF-8 fails."""
    fallback = False
    try:
        with open(path, newline="", encoding="utf-8") as f:
            rows = list(csv.reader(f))
    except UnicodeDecodeError:
        fallback = True
        with open(path, newline="", encoding="latin-1") as f:
            rows = list(csv.reader(f))
    return rows, fallback


def profile_column(name, values):
    non_empty = [v for v in values if v.strip() != ""]
    col = {
        "name": name,
        "type": infer_type(non_empty),
        "nulls": len(values) - len(non_empty),
        "unique": len(set(non_empty)),
    }
    if col["type"] in ("integer", "float"):
        nums = [float(v) for v in non_empty if is_float(v)]
        if nums:
            col["min"] = min(nums)
            col["max"] = max(nums)
            col["mean"] = round(statistics.fmean(nums), 4)
            col["median"] = statistics.median(nums)
            if len(nums) > 1:
                sd = statistics.stdev(nums)
                col["stdev"] = round(sd, 4)
                if sd > 0:
                    m = statistics.fmean(nums)
                    col["outliers"] = sum(
                        1 for x in nums if abs(x - m) > OUTLIER_SIGMAS * sd)
    elif col["type"] == "string" and non_empty:
        col["top_values"] = Counter(non_empty).most_common(TOP_VALUES)
    return col


def main():
    if len(sys.argv) != 2:
        print("usage: profile_csv.py <file.csv>", file=sys.stderr)
        sys.exit(2)
    path = Path(sys.argv[1])
    if not path.is_file():
        print(f"error: file not found or not a regular file: {path}",
              file=sys.stderr)
        sys.exit(2)

    try:
        rows, encoding_fallback = read_rows(path)
    except OSError as e:
        print(f"error: cannot read {path}: {e}", file=sys.stderr)
        sys.exit(2)

    profile = {
        "file": str(path),
        "encoding_fallback": encoding_fallback,
        "truncated": False,
        "rows": 0,
        "columns": [],
        "duplicate_rows": 0,
        "ragged_rows": 0,
    }

    if not rows:
        print(json.dumps(profile, indent=2))
        return

    header, data = rows[0], rows[1:]
    if len(data) > MAX_ROWS:
        data = data[:MAX_ROWS]
        profile["truncated"] = True

    width = len(header)
    profile["ragged_rows"] = sum(1 for r in data if len(r) != width)
    # Ragged rows are padded/clipped so every column still gets profiled.
    normalized = [(r + [""] * width)[:width] for r in data]

    profile["rows"] = len(normalized)
    profile["duplicate_rows"] = len(normalized) - len(
        {tuple(r) for r in normalized})
    profile["columns"] = [
        profile_column(name, [r[i] for r in normalized])
        for i, name in enumerate(header)
    ]

    print(json.dumps(profile, indent=2))


if __name__ == "__main__":
    main()