1"""Calibrate name-similarity rules on UPC ground truth (Metro <-> Super C, same brand key, same size bucket).23For every cross-banner candidate pair in a (brand, size) block we compute several scorers and the GTIN-equality label,4then report precision / recall by threshold. The chosen thresholds are frozen in the preregistration.5"""6import itertools, pandas as pd, numpy as np7from rapidfuzz import fuzz8from src.common import path, get_logger9from src.matching.match_products import prepare, num_tokens1011log = get_logger("calib")12MULTI_RE = r"\b(ou|or|assortis?|assorted|selectionnes?|varietes?|variétés|au choix|et plus)\b|,"1314def main():15 prod = pd.read_parquet(path("processed") / "products.parquet"); ret = pd.read_parquet(path("processed") / "retailers.parquet")16 p = prepare(prod, ret); g = p[p.gtin.notna() & (p.bkey != "") & (p.skey != "")]17 rows = []18 for (bk, sk), block in g.groupby(["bkey", "skey"]):19 m = block[block.source == "metro"]; s = block[block.source == "superc"]20 if len(m) == 0 or len(s) == 0: continue21 for a, b in itertools.product(m.itertuples(), s.itertuples()):22 rows.append(dict(uid_a=a.uid, uid_b=b.uid, same_gtin=(a.gtin == b.gtin),23 set_ratio=fuzz.token_set_ratio(a.nkey, b.nkey) / 100, sort_ratio=fuzz.token_sort_ratio(a.nkey, b.nkey) / 100,24 ratio=fuzz.ratio(a.nkey, b.nkey) / 100, tokens_equal=(set(a.nkey.split()) == set(b.nkey.split())),25 nums_equal=(num_tokens(a.nkey) == num_tokens(b.nkey)),26 jaccard=len(set(a.nkey.split()) & set(b.nkey.split())) / max(len(set(a.nkey.split()) | set(b.nkey.split())), 1)))27 d = pd.DataFrame(rows)28 log.info("candidate pairs in (brand,size) blocks: %d ; positives (same UPC): %d ; negatives: %d", len(d), d.same_gtin.sum(), (~d.same_gtin).sum())29 npos = d.same_gtin.sum()30 out = []31 for scorer in ["set_ratio", "sort_ratio", "ratio", "jaccard"]:32 for thr in [0.70, 0.75, 0.80, 0.85, 0.90, 0.95, 1.0]:33 sel = d[(d[scorer] >= thr) & d.nums_equal]34 out.append(dict(scorer=scorer, thr=thr, n=len(sel), precision=sel.same_gtin.mean() if len(sel) else np.nan, recall=sel.same_gtin.sum() / npos))35 sel = d[d.tokens_equal & d.nums_equal]; out.append(dict(scorer="tokens_equal", thr=1.0, n=len(sel), precision=sel.same_gtin.mean(), recall=sel.same_gtin.sum() / npos))36 res = pd.DataFrame(out); res.to_csv(path("processed") / "match_threshold_calibration.csv", index=False)37 log.info("\n%s", res.round(3).to_string(index=False))38 # what do the residual false positives at sort_ratio>=0.9 look like?39 fp = d[(d.sort_ratio >= 0.9) & d.nums_equal & ~d.same_gtin]40 txt = prod.set_index("uid")41 log.info("false positives at sort_ratio>=0.90: %d ; examples:", len(fp))42 for _, x in fp.sample(min(10, len(fp)), random_state=1).iterrows():43 a, b = txt.loc[x.uid_a], txt.loc[x.uid_b]44 log.info(" [%.2f] %s | %s | %s <-> %s | %s | %s", x.sort_ratio, a.brand, a["name"], a.size_label, b.brand, b["name"], b.size_label)45 # same name & size but different UPC: how often is the name literally identical?46 ident = d[(d.ratio == 1.0) & ~d.same_gtin]47 log.info("literally identical cleaned names with different UPC: %d of %d identical-name pairs (%.1f%%) — name-identical ≠ product-identical (variants hidden in UPC)",48 len(ident), (d.ratio == 1.0).sum(), 100 * len(ident) / max((d.ratio == 1.0).sum(), 1))4950if __name__ == "__main__":51 main()52