Python 57%
TeX 42.9%
1#!/usr/bin/env python32# -*- coding: utf-8 -*-3"""Génère les tableaux LaTeX (summary stats + régression) avec vrais chiffres."""4import json, re, warnings5import numpy as np, pandas as pd6import statsmodels.formula.api as smf7warnings.filterwarnings("ignore")8CUR_YEAR = 20269data = json.load(open("data/processed/maisons_avec_prix.json", encoding="utf-8"))1011def parse_area(s):12 if not s or not isinstance(s, str): return np.nan13 t=s.lower(); m=("m2" in t) or ("m²" in t)14 n=re.sub(r"[^0-9.,\s]","",t).strip().replace(" ","").replace(",","")15 try: v=float(n)16 except: return np.nan17 return (v*10.7639 if m else v) if v>0 else np.nan18def clean_city(c):19 if not c or not isinstance(c,str): return None20 c=c.split("(")[0].strip(); return re.sub(r"\s+"," ",c).lower().strip() or None21def to_int(x):22 try: return int(x)23 except: return np.nan24AMEN={"pool":["swimming pool","piscine","pool"],"waterfront":["waterfront","bord de l'eau","navigable","lakefront","riverfront","au bord de"],25"garage":["garage"],"fireplace":["fireplace","foyer","wood stove","poêle"],26"central_air":["central air","air conditionné","climatisation","a/c","heat-pump","thermopompe","air exchanger"],27"finished_bsmt":["totally finished","finished basement","sous-sol fini","basement"],28"near_water":["near water","waterfront","navigable"],"cul_de_sac":["cul-de-sac","cul de sac"],29"golf":["golf"],"ski":["ski"],"near_park":["park","parc"],30"near_school":["school","école","cegep","university","université","elementary","high school"],31"near_highway":["highway","autoroute"],"transit":["public transit","public transportation","transport"],32"no_rear_neighbor":["no backyard neighbors","sans voisin arrière"]}33def side(s):34 s=s.lower()35 if "ft" in s:36 f=re.search(r"([\d.]+)\s*ft",s); i=re.search(r"([\d.]+)\s*in",s)37 return (float(f.group(1)) if f else 0)+(float(i.group(1))/12 if i else 0)38 m=re.search(r"([\d.,]+)\s*m",s)39 if m:40 try: return float(m.group(1).replace(",","."))*3.2808441 except: return None42 return None43def rarea(d):44 if not d or "x" not in d: return None45 a,_,b=d.partition("x"); la,lb=side(a),side(b)46 return la*lb if (la and lb and 1<la<200 and 1<lb<200) else None47RF={"r_office":["office"],"r_family":["family room"],"r_workshop":["workshop"],48"r_winecellar":["wine cellar"],"r_sunroom":["sunroom","solarium"],"r_mezzanine":["mezzanine"],49"r_library":["library"],"r_laundry":["laundry"]}50def prooms(rooms):51 o={"room_area":np.nan,"n_rooms":0,"has_primary_bed":0,"has_basement":0,"n_levels_r":0}52 for k in RF: o[k]=053 if not rooms: return o54 ar=0.0; lv=set()55 for r in rooms:56 nm=(r.get("name") or "").lower(); l=(r.get("level") or "").lower(); o["n_rooms"]+=157 if l: lv.add(l)58 if "basement" in l or "sous-sol" in l: o["has_basement"]=159 if "primary bedroom" in nm: o["has_primary_bed"]=160 for k,kw in RF.items():61 if any(w in nm for w in kw): o[k]=162 a=rarea(r.get("dimensions"))63 if a: ar+=a64 o["room_area"]=ar if ar>0 else np.nan; o["n_levels_r"]=len(lv); return o65rows=[]66for d in data:67 p=d.get("property") or {}; ad=d.get("address") or {}; ft=d.get("features") or []68 blob=(" ".join(ft)+" "+(d.get("description") or "")+" "+(d.get("title") or "")).lower()69 yb=to_int(p.get("yearBuilt")); age=(CUR_YEAR-yb) if (not pd.isna(yb) and 1700<=yb<=CUR_YEAR) else np.nan70 pt=p.get("buildingType") or p.get("type"); st=p.get("storeys") or p.get("levels")71 try: st=float(str(st).replace(",","."))72 except: st=np.nan73 rec={"price":d.get("priceNumber"),"source":d.get("source"),"sqft":parse_area(p.get("squareFeet")),74 "lot_sqft":parse_area(p.get("lotSize")),"beds":to_int(p.get("bedrooms")),"baths":to_int(p.get("bathrooms")),75 "parking":to_int(p.get("parkingSpaces")),"storeys":st,"age":age,"ptype":(pt or "Unknown").strip(),76 "city":clean_city(ad.get("city"))}77 for n,kw in AMEN.items(): rec[n]=int(any(k in blob for k in kw))78 rec["n_images"]=len(d.get("images") or []); rec["virtual_tour"]=int(bool(d.get("virtualTourUrl")))79 rec.update(prooms(d.get("rooms"))); rows.append(rec)80df=pd.DataFrame(rows)81df=df[df["price"].notna()]; df=df[(df["price"]>=50_000)&(df["price"]<=15_000_000)]82df.loc[(df["sqft"]<300)|(df["sqft"]>20_000),"sqft"]=np.nan83df.loc[(df["lot_sqft"]<200)|(df["lot_sqft"]>5_000_000),"lot_sqft"]=np.nan84df.loc[df["beds"]>15,"beds"]=np.nan; df.loc[df["baths"]>12,"baths"]=np.nan85df["ppsf"]=df["price"]/df["sqft"]; lo,hi=df["ppsf"].quantile([0.005,0.995])86df=df[~(df["ppsf"].notna()&((df["ppsf"]<lo)|(df["ppsf"]>hi)))]87ov=df[df["sqft"].notna()&df["room_area"].notna()&(df["room_area"]>200)]88ratio=(ov["sqft"]/ov["room_area"]).median(); re_=df["room_area"]*ratio89ok=df["room_area"].notna()&(re_.between(300,20000))90df["sqft_final"]=df["sqft"]; fill=df["sqft_final"].isna()&ok; df.loc[fill,"sqft_final"]=re_[fill]91df["has_sqft"]=df["sqft_final"].notna().astype(int)92df["sqft_imp"]=df.groupby("ptype")["sqft_final"].transform(lambda s:s.fillna(s.median())).fillna(df["sqft_final"].median())93df["has_lot"]=df["lot_sqft"].notna().astype(int); df["lot_sqft"]=df["lot_sqft"].fillna(df["lot_sqft"].median())94df["has_age"]=df["age"].notna().astype(int); df["age"]=df["age"].fillna(df["age"].median()).clip(lower=0)95for c in ["beds","baths","storeys"]: df[c]=df[c].fillna(df[c].median())96df["parking"]=df["parking"].fillna(0)97df["ln_price"]=np.log(df["price"]); df["ln_sqft"]=np.log(df["sqft_imp"])98df["ln_lot"]=np.log(df["lot_sqft"]); df["age2"]=df["age"]**2; df["n_images_c"]=df["n_images"].clip(upper=60)99vc=df["city"].value_counts(); keep=set(vc[vc>=20].index)100df["city_fe"]=df["city"].where(df["city"].isin(keep),"__other__").fillna("__other__")101tc=df["ptype"].value_counts(); df["ptype_fe"]=df["ptype"].where(df["ptype"].isin(set(tc[tc>=30].index)),"Other")102103amen=list(AMEN.keys()); rv=["n_rooms","n_levels_r","has_primary_bed","has_basement","r_office","r_family","r_workshop","r_winecellar","r_sunroom","r_mezzanine","r_library","r_laundry"]104extra=["n_images_c","virtual_tour"]+rv105base="ln_lot + has_lot + beds + baths + parking + storeys + age + age2 + has_age + C(source) + C(ptype_fe) + "+" + ".join(amen)+" + "+" + ".join(extra)+" + C(city_fe)"106fA="ln_price ~ ln_sqft + "+base; fB="ln_price ~ "+base107dfA=df[df["has_sqft"]==1].copy()108mA=smf.ols(fA,data=dfA).fit(cov_type="HC3"); mB=smf.ols(fB,data=df).fit(cov_type="HC3")109sdA,sdB=np.std(mA.resid),np.std(mB.resid)110111# ---------- summary stats table ----------112def q(s,p): return s.quantile(p)113def fmt(x,d=0):114 if d==0: return f"{x:,.0f}".replace(",","{,}")115 return f"{x:,.{d}f}".replace(",","{,}")116svars=[("Price (CAD)","price",0),("Living area (ft$^2$)","sqft_final",0),117("Lot area (ft$^2$)","lot_sqft",0),("Bedrooms","beds",1),("Bathrooms","baths",1),118("Parking spaces","parking",1),("Storeys","storeys",1),("Age (years)","age",1),119("Rooms (Realtor)","n_rooms",1),("Photos","n_images",0)]120lines=[]121for lab,c,d in svars:122 s=df[c].dropna();123 lines.append(f"{lab} & {len(s):,} & {fmt(s.mean(),d)} & {fmt(s.std(),d)} & {fmt(q(s,.25),d)} & {fmt(q(s,.5),d)} & {fmt(q(s,.75),d)} \\\\".replace(",","{,}"))124share_r=(df.source=="realtor").mean()*100125tbl_sum=r"""% Auto-generated126\begin{table}[htbp]127 \centering128 \begin{threeparttable}129 \caption{Summary Statistics}130 \label{tab:summary_stats}131 \begin{tabular}{@{}lcccccc@{}}132 \toprule133 Variable & $N$ & Mean & Std.\ Dev. & $p_{25}$ & Median & $p_{75}$ \\134 \midrule135"""+"\n".join(" "+l for l in lines)+r"""136 \bottomrule137 \end{tabular}138 \begin{tablenotes}[flushleft]139 \footnotesize140 \item \textit{Notes:} Descriptive statistics for the cleaned sample of """+f"{len(df):,}".replace(",","{,}")+r""" single-family and townhouse listings (Realtor.ca and DuProprio), scraped on 3~July~2026. """+f"{share_r:.0f}"+r"""\% of listings originate from Realtor.ca. Living area is the reported area, or reconstructed from room dimensions where missing. Continuous variables are bounded by the outlier rules described in Section~\ref{sec:data}; price-per-square-foot is trimmed at the 0.5th and 99.5th percentiles.141 \end{tablenotes}142 \end{threeparttable}143\end{table}144"""145open("paper/tables/summary_stats.tex","w").write(tbl_sum)146147# ---------- regression table ----------148def star(p): return "^{***}" if p<.01 else "^{**}" if p<.05 else "^{*}" if p<.1 else ""149disp=[("ln_sqft",r"$\ln(\text{Living area})$"),("ln_lot",r"$\ln(\text{Lot area})$"),150("beds","Bedrooms"),("baths","Bathrooms"),("parking","Parking"),("age","Age"),151("central_air","Central A/C"),("transit","Public transit"),("waterfront","Waterfront"),152("pool","Pool"),("garage","Garage"),("fireplace","Fireplace"),("near_school","Near school"),153("has_primary_bed","Primary bedroom"),("r_winecellar","Wine cellar"),154("n_images_c","Photos (count)"),("virtual_tour","Virtual tour"),155("C(source)[T.realtor]","Realtor.ca (vs DuProprio)")]156def cell(m,key):157 if key not in m.params.index: return "",""158 b=m.params[key]; se=m.bse[key]; p=m.pvalues[key]159 sign="$-$" if b<0 else ""160 return f"{sign}{abs(b):.3f}${star(p)}$", f"({se:.3f})"161rowsL=[]162for key,lab in disp:163 a1,a2=cell(mA,key); b1,b2=cell(mB,key)164 rowsL.append(f" {lab} & {a1} & {b1} \\\\")165 rowsL.append(f" & {a2} & {b2} \\\\[3pt]")166tbl_reg=r"""% Auto-generated167\begin{table}[htbp]168 \centering169 \begin{threeparttable}170 \caption{Hedonic Regression Results}171 \label{tab:regression}172 \begin{tabular}{@{}lcc@{}}173 \toprule174 & (1) Model~A & (2) Model~B \\175 & \textit{rich} & \textit{broad} \\176 \midrule177"""+"\n".join(rowsL)+r"""178 \midrule179 City FE & Yes & Yes \\180 Property-type FE & Yes & Yes \\181 Living area & Yes & No \\182 Observations & """+f"{int(mA.nobs):,}".replace(",","{,}")+r""" & """+f"{int(mB.nobs):,}".replace(",","{,}")+r""" \\183 $R^2$ & """+f"{mA.rsquared:.3f}"+r""" & """+f"{mB.rsquared:.3f}"+r""" \\184 Adjusted $R^2$ & """+f"{mA.rsquared_adj:.3f}"+r""" & """+f"{mB.rsquared_adj:.3f}"+r""" \\185 \bottomrule186 \end{tabular}187 \begin{tablenotes}[flushleft]188 \footnotesize189 \item \textit{Notes:} OLS estimates of Equation~\eqref{eq:hedonic}. The dependent variable is $\ln(\text{Price})$. Model~A includes $\ln(\text{Living area})$ and is estimated on listings with a measured (reported or room-reconstructed) area; Model~B omits living area and is estimated on the full cleaned sample. All specifications include city and property-type fixed effects and the full set of amenity, room-detail, and listing-signal controls (Section~\ref{sec:methodology}); only selected coefficients are reported. Heteroskedasticity-robust (HC3) standard errors in parentheses. $^{***}$, $^{**}$, $^{*}$ denote significance at the 1\%, 5\%, and 10\% levels.190 \end{tablenotes}191 \end{threeparttable}192\end{table}193"""194open("paper/tables/regression.tex","w").write(tbl_reg)195196# key numbers for the text197predA=np.exp(mA.predict(df))*np.exp(sdA**2/2); predB=np.exp(mB.predict(df))*np.exp(sdB**2/2)198df["pred"]=np.where(df["has_sqft"]==1,predA,predB); df["rp"]=(df["price"]/df["pred"]-1)*100199print("N_clean",len(df),"N_A",int(mA.nobs),"N_B",int(mB.nobs))200print("R2_A %.4f adj %.4f errpct %.1f"%(mA.rsquared,mA.rsquared_adj,(np.exp(sdA)-1)*100))201print("R2_B %.4f adj %.4f errpct %.1f"%(mB.rsquared,mB.rsquared_adj,(np.exp(sdB)-1)*100))202print("median|err| %.1f within15 %.0f within25 %.0f"%(df.rp.abs().median(),(df.rp.abs()<=15).mean()*100,(df.rp.abs()<=25).mean()*100))203print("nb_cities_fe",df.city_fe.nunique(),"median_price",int(df.price.median()))204cf={k.split("T.")[1].rstrip("]"):v for k,v in mA.params.items() if k.startswith("C(city_fe)")}205top=pd.Series(cf).sort_values(ascending=False)206print("TOP:",[(c,round((np.exp(b)-1)*100)) for c,b in top.head(5).items()])207print("BOT:",[(c,round((np.exp(b)-1)*100)) for c,b in top.tail(3).items()])208print("share_realtor %.1f"%((df.source=='realtor').mean()*100))209print("Tables written to paper/tables/")210