spb/ultra-sharp-agent-skills Public

1#!/usr/bin/env python32# Author: Simon-Pierre Boucher3# Contact: contact@spboucher.ai4#5# profile_csv.py — deterministic CSV profiler. Stdlib only.6# Usage: python3 profile_csv.py <file.csv>7# Prints a JSON profile to stdout. Exit codes: 0 = OK, 2 = unreadable input.89import csv10import json11import statistics12import sys13from collections import Counter14from pathlib import Path1516# Cap keeps runtime and memory bounded on very large files; 100k rows is17# enough for stable summary statistics on any practical column.18MAX_ROWS = 100_0001920# A column is typed integer/float/date/boolean only if at least 95% of its21# non-empty values parse as that type; below that, mixed content is safer22# reported as "string" than as a false-precision numeric column.23TYPE_THRESHOLD = 0.952425# Top-N most frequent values reported for string columns; 5 shows the26# dominant categories without bloating the report.27TOP_VALUES = 52829# Values beyond 3 standard deviations from the mean are counted as outliers30# (classic three-sigma rule).31OUTLIER_SIGMAS = 3323334def is_int(s):35    try:36        int(s)37        return True38    except ValueError:39        return False404142def is_float(s):43    try:44        float(s)45        return True46    except ValueError:47        return False484950def is_bool(s):51    return s.strip().lower() in ("true", "false", "yes", "no", "0", "1")525354def is_date(s):55    s = s.strip()56    for sep in ("-", "/"):57        parts = s.split(sep)58        if len(parts) == 3 and all(p.isdigit() for p in parts):59            return True60    return False616263def infer_type(values):64    """Return the dominant type of non-empty values per TYPE_THRESHOLD."""65    if not values:66        return "empty"67    n = len(values)68    for name, pred in (("integer", is_int), ("float", is_float),69                       ("boolean", is_bool), ("date", is_date)):70        if sum(1 for v in values if pred(v)) / n >= TYPE_THRESHOLD:71            return name72    return "string"737475def read_rows(path):76    """Read the CSV, falling back to latin-1 if UTF-8 fails."""77    fallback = False78    try:79        with open(path, newline="", encoding="utf-8") as f:80            rows = list(csv.reader(f))81    except UnicodeDecodeError:82        fallback = True83        with open(path, newline="", encoding="latin-1") as f:84            rows = list(csv.reader(f))85    return rows, fallback868788def profile_column(name, values):89    non_empty = [v for v in values if v.strip() != ""]90    col = {91        "name": name,92        "type": infer_type(non_empty),93        "nulls": len(values) - len(non_empty),94        "unique": len(set(non_empty)),95    }96    if col["type"] in ("integer", "float"):97        nums = [float(v) for v in non_empty if is_float(v)]98        if nums:99            col["min"] = min(nums)100            col["max"] = max(nums)101            col["mean"] = round(statistics.fmean(nums), 4)102            col["median"] = statistics.median(nums)103            if len(nums) > 1:104                sd = statistics.stdev(nums)105                col["stdev"] = round(sd, 4)106                if sd > 0:107                    m = statistics.fmean(nums)108                    col["outliers"] = sum(109                        1 for x in nums if abs(x - m) > OUTLIER_SIGMAS * sd)110    elif col["type"] == "string" and non_empty:111        col["top_values"] = Counter(non_empty).most_common(TOP_VALUES)112    return col113114115def main():116    if len(sys.argv) != 2:117        print("usage: profile_csv.py <file.csv>", file=sys.stderr)118        sys.exit(2)119    path = Path(sys.argv[1])120    if not path.is_file():121        print(f"error: file not found or not a regular file: {path}",122              file=sys.stderr)123        sys.exit(2)124125    try:126        rows, encoding_fallback = read_rows(path)127    except OSError as e:128        print(f"error: cannot read {path}: {e}", file=sys.stderr)129        sys.exit(2)130131    profile = {132        "file": str(path),133        "encoding_fallback": encoding_fallback,134        "truncated": False,135        "rows": 0,136        "columns": [],137        "duplicate_rows": 0,138        "ragged_rows": 0,139    }140141    if not rows:142        print(json.dumps(profile, indent=2))143        return144145    header, data = rows[0], rows[1:]146    if len(data) > MAX_ROWS:147        data = data[:MAX_ROWS]148        profile["truncated"] = True149150    width = len(header)151    profile["ragged_rows"] = sum(1 for r in data if len(r) != width)152    # Ragged rows are padded/clipped so every column still gets profiled.153    normalized = [(r + [""] * width)[:width] for r in data]154155    profile["rows"] = len(normalized)156    profile["duplicate_rows"] = len(normalized) - len(157        {tuple(r) for r in normalized})158    profile["columns"] = [159        profile_column(name, [r[i] for r in normalized])160        for i, name in enumerate(header)161    ]162163    print(json.dumps(profile, indent=2))164165166if __name__ == "__main__":167    main()168