1"""Unit normalisation for grocery listings (CAD; g / ml / unit).23Re-implements and extends Food-Ka's parser (data/raw/foodka-src/foodka/normalize.py) so that the research4pipeline does not depend on production code. Differences are documented in reports/phase1_data_audit.md.5"""6from __future__ import annotations7import re, unicodedata8from dataclasses import dataclass910_UNITS = {"kg": (1000.0, "g"), "g": (1.0, "g"), "mg": (0.001, "g"), "lb": (453.592, "g"), "lbs": (453.592, "g"),11 "oz": (28.3495, "g"), "l": (1000.0, "ml"), "ml": (1.0, "ml"), "cl": (10.0, "ml"),12 "un": (1.0, "un"), "unite": (1.0, "un"), "unites": (1.0, "un"), "ea": (1.0, "un"), "each": (1.0, "un"),13 "pk": (1.0, "un"), "ch": (1.0, "un"), "pc": (1.0, "un"), "pcs": (1.0, "un"), "units": (1.0, "un"), "unit": (1.0, "un")}14_SIZE_RE = re.compile(r"(?:(\d+)\s*[x×*]\s*)?(\d+(?:[.,]\d+)?)\s*(kg|g|mg|lb|lbs|oz|ml|cl|l|un|unites?|ea|each|pk|ch|pcs?|units?)\b", re.I)15_RANGE_RE = re.compile(r"(\d+(?:[.,]\d+)?)\s*(?:-|–|a|à)\s*(\d+(?:[.,]\d+)?)\s*(kg|g|lb|lbs|oz|ml|l)\b", re.I)16_PERKG_RE = re.compile(r"/\s*(kg|lb|100\s*g|100\s*ml|l|ch|un)\b", re.I)17_MULTI_RE = re.compile(r"^\s*(\d+)\s*(?:/|pour|for)\s*(?:(\d{1,4}(?:[.,]\d{1,2})?)\s*\$)?")18_APPROX_RE = re.compile(r"environ|approx|moyenne|avg|about|~", re.I)19_DEPOSIT_RE = re.compile(r"(cons(igne)?\.?\s*[\d,\.]+\s*\$\s*/\s*\w+\.?|\d+[,\.]\d+\s*\$\s*/\s*100\s*(g|ml))", re.I)20_SOLD_INDIV_RE = re.compile(r"vendu individuellement|sold individually|a l'unite|à l'unité|l'unite|unite$", re.I)2122def strip_accents(s: str) -> str:23 return "".join(c for c in unicodedata.normalize("NFKD", s or "") if not unicodedata.combining(c))2425@dataclass26class Size:27 qty: float # in base unit (g, ml, un)28 base: str # 'g' | 'ml' | 'un'29 approx: bool = False30 source: str = "size_label" # where it was parsed from31 pack: int = 1 # multipack count ('12 x 355 ml' -> 12)3233def parse_size(label: str | None, name: str | None = None) -> Size | None:34 """Parse '450 g', '2 L', '12 x 355 ml', '6 un', 'Vendu individuellement', '1,36 kg'; falls back to the name."""35 for text, src in ((label, "size_label"), (name, "name")):36 if not text:37 continue38 t = strip_accents(text).lower().replace("\u00a0", " ")39 m = _RANGE_RE.search(t)40 if m: # "1,2 - 1,5 kg" -> midpoint, approximate (checked before the simple pattern)41 a, b = float(m.group(1).replace(",", ".")), float(m.group(2).replace(",", "."))42 factor, base = _UNITS[m.group(3).lower()]43 return Size((a + b) / 2 * factor, base, True, src)44 m = _SIZE_RE.search(t)45 if m:46 mult = int(m.group(1)) if m.group(1) else 147 qty = float(m.group(2).replace(",", "."))48 factor, base = _UNITS[m.group(3).lower()]49 total = mult * qty * factor50 if total > 0:51 return Size(total, base, bool(_APPROX_RE.search(t)), src, mult)52 if src == "size_label" and _SOLD_INDIV_RE.search(t):53 return Size(1.0, "un", False, src)54 return None5556@dataclass57class PriceLabel:58 per_weight_unit: str | None = None # 'kg','lb','100 g','100 ml','l' when the displayed price is per weight/volume59 multibuy_n: int | None = None # 'n / x $'60 multibuy_total: float | None = None6162def parse_price_label(label: str | None) -> PriceLabel:63 pl = PriceLabel()64 if not label:65 return pl66 t = strip_accents(label).lower().replace(" ", " ")67 m = _PERKG_RE.search(t)68 if m:69 pl.per_weight_unit = re.sub(r"\s+", " ", m.group(1))70 m = _MULTI_RE.match(t)71 if m:72 pl.multibuy_n = int(m.group(1))73 if m.group(2):74 pl.multibuy_total = float(m.group(2).replace(",", "."))75 return pl7677def unit_price(price: float | None, size: Size | None, label: PriceLabel | None = None) -> tuple[float, str] | None:78 """Comparable price: $/100 g, $/100 ml or $/unit. Returns None when not computable."""79 if price is None or price <= 0:80 return None81 if label and label.per_weight_unit:82 u = label.per_weight_unit83 if u == "kg": return (round(price / 10.0, 6), "100 g")84 if u == "lb": return (round(price / 4.53592, 6), "100 g")85 if u == "100 g": return (price, "100 g")86 if u == "100 ml": return (price, "100 ml")87 if u == "l": return (round(price / 10.0, 6), "100 ml")88 if size is None:89 return None90 if size.base == "un":91 return (round(price / size.qty, 6), "un")92 return (round(price / size.qty * 100.0, 6), f"100 {size.base}")939495def size_residual(label: str | None) -> str:96 """Text of the size label that is NOT the package size itself (e.g. 'calibre 21-25', 'gros', 'bio'), with deposit97 and unit-price fragments removed — used as extra discriminating tokens in matching."""98 if not label:99 return ""100 t = strip_accents(label).lower().replace("\u00a0", " ")101 t = _DEPOSIT_RE.sub(" ", t)102 t = _RANGE_RE.sub(" ", t); t = _SIZE_RE.sub(" ", t)103 t = re.sub(r"[^a-z0-9%]+", " ", t)104 return " ".join(sorted(w for w in t.split() if len(w) > 1))105