1"""Generate reports/data_dictionary.md from the interim/processed parquet schemas plus curated field descriptions."""2from __future__ import annotations3import pandas as pd4from src.common import ROOT, P, get_logger56log = get_logger("reports.dictionary")7DESC = {8 "uid": "Fabri-Ka listing id (hash of store + platform product id); stable across syncs",9 "store_id": "canonical store domain (merchant identifier)", "external_id": "platform-local product id (not unique across stores)",10 "url": "canonical product URL on the merchant site", "title": "listing title as published", "title_norm": "ASCII-folded, lower-cased, punctuation-stripped title",11 "description_head": "first 600 characters of the merchant description", "description_len": "length of the full description (chars)",12 "price": "listed price = MINIMUM variant price at the snapshot (merchant currency)", "price_max": "maximum variant price", "compare_at_price": "merchant strike-through price (on-sale indicator)",13 "currency": "merchant currency code", "category": "Fabri-Ka canonical category (20 keys, rule-based inference; audited in reports/phase2_data_audit.md)",14 "product_type": "merchant's own product type", "tags_list": "merchant tags (JSON list)", "n_tags": "number of tags", "vendor": "merchant's brand/vendor field (often the store's own name)",15 "available": "merchant availability flag (1 in stock, 0 sold out, NULL unknown)", "content_hash": "hash of listing content used by the platform's upsert",16 "first_seen": "epoch seconds of first observation", "last_seen": "epoch seconds of last observation", "first_seen_utc": "first observation (UTC)", "last_seen_utc": "last observation (UTC)",17 "active": "1 if seen in the last two syncs", "miss_count": "consecutive misses (0–2)", "listing_status": "published | excluded:<reason> | quarantine:<reason> (platform rules)",18 "price_on_request": "1 if the merchant shows no price", "n_variants": "number of variants (Shopify/Woo/Wix details)", "has_sku": "a merchant SKU is present", "sku": "first merchant SKU (64 chars max)",19 "grams": "variant weight in grams (Shopify grams or Woo kg×1000) — first variant", "n_options": "number of option dimensions", "variant_prices_min": "min of variant prices in details",20 "variant_prices_max": "max of variant prices in details", "n_variant_prices": "number of variant prices", "on_sale_flag": "Shopify on_sale flag", "merchant_categories": "merchant's own category path (Woo/Wix)",21 "published_at": "Shopify published_at", "created_at": "Shopify created_at", "n_images": "number of images",22 "f_published": "active = 1 and listing_status = 'published'", "f_live_store": "store enabled with >= 1 published listing", "store_kind": "standard | revendeur | collectif | hors_mission",23 "f_hors_mission": "store flagged out of mission by the platform", "dup_group_size": "number of listings with the same normalised title in the store", "f_dup_in_store": "duplicated title within store",24 "f_dup_rep": "representative (first-seen) listing of its (store, title) group", "f_sibling_dup": "store is a non-representative mirror domain of another store",25 "f_analysis_assort": "ASSORTMENT SAMPLE membership", "store_cad_share": "share of the store's listings priced in CAD", "f_cad": "CAD listing in a CAD store (>= 95 %)",26 "f_price_ok": "0 < price <= 50 000, not on request", "f_analysis_price": "PRICE SAMPLE membership", "log_price": "natural log of price (price sample)", "on_sale": "compare_at_price > price",27 "has_variants": "price_max > price or n_variants > 1",28 "id": "store domain", "name": "store name", "platform": "e-commerce platform (connector)", "city": "city/sector label from the FSA centroid or directory", "region_db": "Fabri-Ka region (directory / site / FSA map)",29 "region_pip": "region containing the FSA centroid (official MERN layer, point-in-polygon)", "region_final": "region_db, else region_pip, else 'Inconnue'", "region_source": "db | pip | unknown",30 "region_conflict": "region_db ≠ region_pip (both present)", "has_geo": "FSA centroid available", "lat": "FSA centroid latitude", "lng": "FSA centroid longitude", "postal_prefix": "forward sortation area (first 3 characters)",31 "phone_shared": "phone shared by >= 3 stores (directory contamination; never used)", "origin_class": "store-level origin class A/C/E assigned by the platform from discovery-source priors + site signals (NOT product provenance)",32 "origin_confidence": "platform confidence in origin_class", "origin_evidence": "retained textual evidence for origin_class", "certified": "store carries >= 1 structured third-party certification",33 "cert_adq": "Aliments du Québec member", "cert_lpq": "Les Produits du Québec certified", "cert_bio": "CARTV organic", "cert_labels": "certification labels", "language": "site language",34 "n_sources": "number of discovery sources", "first_source": "first discovery source", "reg_status": "registry verification status (verified/probable)", "product_count": "platform's published count",35 "n_listings": "assortment-sample listings", "n_categories": "distinct categories in the assortment sample", "share_autre": "share of 'autre'", "median_price": "median listed price (CAD)",36 "share_available": "share available = 1", "share_on_sale": "share on sale", "main_category": "modal category", "first_listing_utc": "earliest first_seen", "last_listing_utc": "latest first_seen",37 "last_sync_utc": "last sync", "fail_streak": "consecutive failed syncs", "sibling_group": "mirror-domain group id", "sibling_rep": "representative store of the group", "is_rep": "is the representative", "n_titles": "distinct titles",38 "uid_a": "listing a", "uid_b": "listing b", "tier": "matching tier A/B/C/D", "score": "rapidfuzz score", "comparable": "variant-structure comparability flag", "log_gap": "|log price a − log price b|",39 "group_id": "match group (connected component over tiers A–C)", "n": "group size", "n_stores": "stores in group", "n_cat": "categories in group", "nv": "variant count", "vendor_norm": "normalised brand", "ts": "epoch", "ts_utc": "UTC time",40 "found": "listings found in the sync", "added": "added", "updated": "updated", "removed": "removed", "status": "ok | error | presumed_closed",41}424344def main():45 out = ["# Data dictionary — WP38", "", "Generated by `src/reports/data_dictionary.py` from the parquet schemas. Raw snapshot fields are described in `reports/00_source_audit.md`.", ""]46 for f in ("listings_clean", "stores_master", "sibling_stores", "match_pairs", "match_groups", "sync_log"):47 p = P["interim"] / f"{f}.parquet"48 if not p.exists(): continue49 df = pd.read_parquet(p)50 out += [f"## `data/interim/{f}.parquet` — {len(df):,} rows × {df.shape[1]} columns".replace(",", " "), "", "| column | dtype | non-null | description |", "|---|---|---:|---|"]51 for c in df.columns:52 out.append(f"| `{c}` | {df[c].dtype} | {df[c].notna().mean():.3f} | {DESC.get(c, '')} |")53 out.append("")54 (ROOT / "reports" / "data_dictionary.md").write_text("\n".join(out)); log.info("data dictionary written")555657if __name__ == "__main__":58 main()59