#!/usr/bin/env python3 # -*- coding: utf-8 -*- """Génère les tableaux LaTeX (summary stats + régression) avec vrais chiffres.""" import json, re, warnings import numpy as np, pandas as pd import statsmodels.formula.api as smf warnings.filterwarnings("ignore") CUR_YEAR = 2026 data = json.load(open("data/processed/maisons_avec_prix.json", encoding="utf-8")) def parse_area(s): if not s or not isinstance(s, str): return np.nan t=s.lower(); m=("m2" in t) or ("m²" in t) n=re.sub(r"[^0-9.,\s]","",t).strip().replace(" ","").replace(",","") try: v=float(n) except: return np.nan return (v*10.7639 if m else v) if v>0 else np.nan def clean_city(c): if not c or not isinstance(c,str): return None c=c.split("(")[0].strip(); return re.sub(r"\s+"," ",c).lower().strip() or None def to_int(x): try: return int(x) except: return np.nan AMEN={"pool":["swimming pool","piscine","pool"],"waterfront":["waterfront","bord de l'eau","navigable","lakefront","riverfront","au bord de"], "garage":["garage"],"fireplace":["fireplace","foyer","wood stove","poêle"], "central_air":["central air","air conditionné","climatisation","a/c","heat-pump","thermopompe","air exchanger"], "finished_bsmt":["totally finished","finished basement","sous-sol fini","basement"], "near_water":["near water","waterfront","navigable"],"cul_de_sac":["cul-de-sac","cul de sac"], "golf":["golf"],"ski":["ski"],"near_park":["park","parc"], "near_school":["school","école","cegep","university","université","elementary","high school"], "near_highway":["highway","autoroute"],"transit":["public transit","public transportation","transport"], "no_rear_neighbor":["no backyard neighbors","sans voisin arrière"]} def side(s): s=s.lower() if "ft" in s: f=re.search(r"([\d.]+)\s*ft",s); i=re.search(r"([\d.]+)\s*in",s) return (float(f.group(1)) if f else 0)+(float(i.group(1))/12 if i else 0) m=re.search(r"([\d.,]+)\s*m",s) if m: try: return float(m.group(1).replace(",","."))*3.28084 except: return None return None def rarea(d): if not d or "x" not in d: return None a,_,b=d.partition("x"); la,lb=side(a),side(b) return la*lb if (la and lb and 10 else np.nan; o["n_levels_r"]=len(lv); return o rows=[] for d in data: p=d.get("property") or {}; ad=d.get("address") or {}; ft=d.get("features") or [] blob=(" ".join(ft)+" "+(d.get("description") or "")+" "+(d.get("title") or "")).lower() yb=to_int(p.get("yearBuilt")); age=(CUR_YEAR-yb) if (not pd.isna(yb) and 1700<=yb<=CUR_YEAR) else np.nan pt=p.get("buildingType") or p.get("type"); st=p.get("storeys") or p.get("levels") try: st=float(str(st).replace(",",".")) except: st=np.nan rec={"price":d.get("priceNumber"),"source":d.get("source"),"sqft":parse_area(p.get("squareFeet")), "lot_sqft":parse_area(p.get("lotSize")),"beds":to_int(p.get("bedrooms")),"baths":to_int(p.get("bathrooms")), "parking":to_int(p.get("parkingSpaces")),"storeys":st,"age":age,"ptype":(pt or "Unknown").strip(), "city":clean_city(ad.get("city"))} for n,kw in AMEN.items(): rec[n]=int(any(k in blob for k in kw)) rec["n_images"]=len(d.get("images") or []); rec["virtual_tour"]=int(bool(d.get("virtualTourUrl"))) rec.update(prooms(d.get("rooms"))); rows.append(rec) df=pd.DataFrame(rows) df=df[df["price"].notna()]; df=df[(df["price"]>=50_000)&(df["price"]<=15_000_000)] df.loc[(df["sqft"]<300)|(df["sqft"]>20_000),"sqft"]=np.nan df.loc[(df["lot_sqft"]<200)|(df["lot_sqft"]>5_000_000),"lot_sqft"]=np.nan df.loc[df["beds"]>15,"beds"]=np.nan; df.loc[df["baths"]>12,"baths"]=np.nan df["ppsf"]=df["price"]/df["sqft"]; lo,hi=df["ppsf"].quantile([0.005,0.995]) df=df[~(df["ppsf"].notna()&((df["ppsf"]hi)))] ov=df[df["sqft"].notna()&df["room_area"].notna()&(df["room_area"]>200)] ratio=(ov["sqft"]/ov["room_area"]).median(); re_=df["room_area"]*ratio ok=df["room_area"].notna()&(re_.between(300,20000)) df["sqft_final"]=df["sqft"]; fill=df["sqft_final"].isna()&ok; df.loc[fill,"sqft_final"]=re_[fill] df["has_sqft"]=df["sqft_final"].notna().astype(int) df["sqft_imp"]=df.groupby("ptype")["sqft_final"].transform(lambda s:s.fillna(s.median())).fillna(df["sqft_final"].median()) df["has_lot"]=df["lot_sqft"].notna().astype(int); df["lot_sqft"]=df["lot_sqft"].fillna(df["lot_sqft"].median()) df["has_age"]=df["age"].notna().astype(int); df["age"]=df["age"].fillna(df["age"].median()).clip(lower=0) for c in ["beds","baths","storeys"]: df[c]=df[c].fillna(df[c].median()) df["parking"]=df["parking"].fillna(0) df["ln_price"]=np.log(df["price"]); df["ln_sqft"]=np.log(df["sqft_imp"]) df["ln_lot"]=np.log(df["lot_sqft"]); df["age2"]=df["age"]**2; df["n_images_c"]=df["n_images"].clip(upper=60) vc=df["city"].value_counts(); keep=set(vc[vc>=20].index) df["city_fe"]=df["city"].where(df["city"].isin(keep),"__other__").fillna("__other__") tc=df["ptype"].value_counts(); df["ptype_fe"]=df["ptype"].where(df["ptype"].isin(set(tc[tc>=30].index)),"Other") amen=list(AMEN.keys()); rv=["n_rooms","n_levels_r","has_primary_bed","has_basement","r_office","r_family","r_workshop","r_winecellar","r_sunroom","r_mezzanine","r_library","r_laundry"] extra=["n_images_c","virtual_tour"]+rv base="ln_lot + has_lot + beds + baths + parking + storeys + age + age2 + has_age + C(source) + C(ptype_fe) + "+" + ".join(amen)+" + "+" + ".join(extra)+" + C(city_fe)" fA="ln_price ~ ln_sqft + "+base; fB="ln_price ~ "+base dfA=df[df["has_sqft"]==1].copy() mA=smf.ols(fA,data=dfA).fit(cov_type="HC3"); mB=smf.ols(fB,data=df).fit(cov_type="HC3") sdA,sdB=np.std(mA.resid),np.std(mB.resid) # ---------- summary stats table ---------- def q(s,p): return s.quantile(p) def fmt(x,d=0): if d==0: return f"{x:,.0f}".replace(",","{,}") return f"{x:,.{d}f}".replace(",","{,}") svars=[("Price (CAD)","price",0),("Living area (ft$^2$)","sqft_final",0), ("Lot area (ft$^2$)","lot_sqft",0),("Bedrooms","beds",1),("Bathrooms","baths",1), ("Parking spaces","parking",1),("Storeys","storeys",1),("Age (years)","age",1), ("Rooms (Realtor)","n_rooms",1),("Photos","n_images",0)] lines=[] for lab,c,d in svars: s=df[c].dropna(); lines.append(f"{lab} & {len(s):,} & {fmt(s.mean(),d)} & {fmt(s.std(),d)} & {fmt(q(s,.25),d)} & {fmt(q(s,.5),d)} & {fmt(q(s,.75),d)} \\\\".replace(",","{,}")) share_r=(df.source=="realtor").mean()*100 tbl_sum=r"""% Auto-generated \begin{table}[htbp] \centering \begin{threeparttable} \caption{Summary Statistics} \label{tab:summary_stats} \begin{tabular}{@{}lcccccc@{}} \toprule Variable & $N$ & Mean & Std.\ Dev. & $p_{25}$ & Median & $p_{75}$ \\ \midrule """+"\n".join(" "+l for l in lines)+r""" \bottomrule \end{tabular} \begin{tablenotes}[flushleft] \footnotesize \item \textit{Notes:} Descriptive statistics for the cleaned sample of """+f"{len(df):,}".replace(",","{,}")+r""" single-family and townhouse listings (Realtor.ca and DuProprio), scraped on 3~July~2026. """+f"{share_r:.0f}"+r"""\% of listings originate from Realtor.ca. Living area is the reported area, or reconstructed from room dimensions where missing. Continuous variables are bounded by the outlier rules described in Section~\ref{sec:data}; price-per-square-foot is trimmed at the 0.5th and 99.5th percentiles. \end{tablenotes} \end{threeparttable} \end{table} """ open("paper/tables/summary_stats.tex","w").write(tbl_sum) # ---------- regression table ---------- def star(p): return "^{***}" if p<.01 else "^{**}" if p<.05 else "^{*}" if p<.1 else "" disp=[("ln_sqft",r"$\ln(\text{Living area})$"),("ln_lot",r"$\ln(\text{Lot area})$"), ("beds","Bedrooms"),("baths","Bathrooms"),("parking","Parking"),("age","Age"), ("central_air","Central A/C"),("transit","Public transit"),("waterfront","Waterfront"), ("pool","Pool"),("garage","Garage"),("fireplace","Fireplace"),("near_school","Near school"), ("has_primary_bed","Primary bedroom"),("r_winecellar","Wine cellar"), ("n_images_c","Photos (count)"),("virtual_tour","Virtual tour"), ("C(source)[T.realtor]","Realtor.ca (vs DuProprio)")] def cell(m,key): if key not in m.params.index: return "","" b=m.params[key]; se=m.bse[key]; p=m.pvalues[key] sign="$-$" if b<0 else "" return f"{sign}{abs(b):.3f}${star(p)}$", f"({se:.3f})" rowsL=[] for key,lab in disp: a1,a2=cell(mA,key); b1,b2=cell(mB,key) rowsL.append(f" {lab} & {a1} & {b1} \\\\") rowsL.append(f" & {a2} & {b2} \\\\[3pt]") tbl_reg=r"""% Auto-generated \begin{table}[htbp] \centering \begin{threeparttable} \caption{Hedonic Regression Results} \label{tab:regression} \begin{tabular}{@{}lcc@{}} \toprule & (1) Model~A & (2) Model~B \\ & \textit{rich} & \textit{broad} \\ \midrule """+"\n".join(rowsL)+r""" \midrule City FE & Yes & Yes \\ Property-type FE & Yes & Yes \\ Living area & Yes & No \\ Observations & """+f"{int(mA.nobs):,}".replace(",","{,}")+r""" & """+f"{int(mB.nobs):,}".replace(",","{,}")+r""" \\ $R^2$ & """+f"{mA.rsquared:.3f}"+r""" & """+f"{mB.rsquared:.3f}"+r""" \\ Adjusted $R^2$ & """+f"{mA.rsquared_adj:.3f}"+r""" & """+f"{mB.rsquared_adj:.3f}"+r""" \\ \bottomrule \end{tabular} \begin{tablenotes}[flushleft] \footnotesize \item \textit{Notes:} OLS estimates of Equation~\eqref{eq:hedonic}. The dependent variable is $\ln(\text{Price})$. Model~A includes $\ln(\text{Living area})$ and is estimated on listings with a measured (reported or room-reconstructed) area; Model~B omits living area and is estimated on the full cleaned sample. All specifications include city and property-type fixed effects and the full set of amenity, room-detail, and listing-signal controls (Section~\ref{sec:methodology}); only selected coefficients are reported. Heteroskedasticity-robust (HC3) standard errors in parentheses. $^{***}$, $^{**}$, $^{*}$ denote significance at the 1\%, 5\%, and 10\% levels. \end{tablenotes} \end{threeparttable} \end{table} """ open("paper/tables/regression.tex","w").write(tbl_reg) # key numbers for the text predA=np.exp(mA.predict(df))*np.exp(sdA**2/2); predB=np.exp(mB.predict(df))*np.exp(sdB**2/2) df["pred"]=np.where(df["has_sqft"]==1,predA,predB); df["rp"]=(df["price"]/df["pred"]-1)*100 print("N_clean",len(df),"N_A",int(mA.nobs),"N_B",int(mB.nobs)) print("R2_A %.4f adj %.4f errpct %.1f"%(mA.rsquared,mA.rsquared_adj,(np.exp(sdA)-1)*100)) print("R2_B %.4f adj %.4f errpct %.1f"%(mB.rsquared,mB.rsquared_adj,(np.exp(sdB)-1)*100)) print("median|err| %.1f within15 %.0f within25 %.0f"%(df.rp.abs().median(),(df.rp.abs()<=15).mean()*100,(df.rp.abs()<=25).mean()*100)) print("nb_cities_fe",df.city_fe.nunique(),"median_price",int(df.price.median())) cf={k.split("T.")[1].rstrip("]"):v for k,v in mA.params.items() if k.startswith("C(city_fe)")} top=pd.Series(cf).sort_values(ascending=False) print("TOP:",[(c,round((np.exp(b)-1)*100)) for c,b in top.head(5).items()]) print("BOT:",[(c,round((np.exp(b)-1)*100)) for c,b in top.tail(3).items()]) print("share_realtor %.1f"%((df.source=='realtor').mean()*100)) print("Tables written to paper/tables/")