spb/uqo-imm1003

1"""rapport.py — intègre le « Rapport immobilier Québec » (article LaTeX, 26 sections, 67 figures PDF,228 tableaux générés, valeurs clés, données HPI ACI + FRED) sous forme de mini-livre web."""3from __future__ import annotations45import hashlib6import json7import os8import re9import shutil10import subprocess11from datetime import datetime1213from latex2html import Ctx, convert_chapter, strip_comments, read_group, plain_text14from bib import parse_bib1516REPORT_DIR = "Rapport_Immobilier_Quebec"171819def load_key_values(path: str) -> dict:20    vals = {}21    for m in re.finditer(r"\\newcommand\{\\(Vc[A-Za-z]+)\}\{([^}]*)\}", open(path, encoding="utf-8").read()):22        vals[m.group(1)] = m.group(2)23    return vals242526def preprocess(src_dir: str) -> tuple[str, list[dict], dict]:27    """Retourne (texte des sections, liste [{'key','num','title','tex','appendix'}], valeurs clés)."""28    main = open(os.path.join(src_dir, "rapport_immobilier_quebec.tex"), encoding="utf-8").read()29    vals = load_key_values(os.path.join(src_dir, "tables", "valeurs_cles.tex"))30    # corps : de la première \section à \bibliographystyle, puis annexes après \appendix31    start = main.index("\\section{Introduction}")32    bib_pos = main.index("\\bibliographystyle")33    app_pos = main.index("\\appendix")34    body = main[start:bib_pos] + "\n\\appendix\n" + main[app_pos + len("\\appendix"):main.index("\\end{document}")]35    # inputs de tableaux36    def inline_input(m):37        p = os.path.join(src_dir, m.group(1) + (".tex" if not m.group(1).endswith(".tex") else ""))38        return open(p, encoding="utf-8").read() if os.path.exists(p) else ""39    body = re.sub(r"\\input\{([^}]+)\}", inline_input, body)40    # valeurs clés41    for k, v in sorted(vals.items(), key=lambda kv: -len(kv[0])):42        body = re.sub(r"\\" + k + r"(\{\})?(?![A-Za-z])", lambda m, v=v: v, body)43    body = strip_comments(body)44    # découpage en sections → « chapitres »45    parts = re.split(r"(?=^\\section\{)", body, flags=re.M)46    sections = []47    appendix = False48    n = 049    for chunk in parts:50        if not chunk.strip():51            continue52        if not chunk.startswith("\\section{"):53            if "\\appendix" in chunk:54                appendix = True55            continue56        if "\\appendix" in chunk:57            # l'annexe commence après ce chunk58            chunk_main, _, rest = chunk.partition("\\appendix")59            chunk = chunk_main60            pending_app = True61        else:62            pending_app = False63        title, k = read_group(chunk, len("\\section"))64        if appendix:65            num = chr(ord("A") + sum(1 for s in sections if s["appendix"]))66        else:67            n += 168            num = str(n)69        tex = "\\chapter{" + title + "}" + chunk[k:]70        tex = tex.replace("\\subsubsection{", "\\subsection@@{").replace("\\subsection{", "\\section{").replace("\\subsection@@{", "\\subsection{")71        tex = tex.replace("\\clearpage", "")72        sections.append({"key": f"rapport-{num}", "num": num, "title": title, "tex": tex, "appendix": appendix})73        if pending_app:74            appendix = True75    return body, sections, vals767778def convert_figures(src_dir: str, images: list[str], cache_dir: str, out_dir: str) -> dict[int, str]:79    """Convertit les PDF de figures en SVG (cache) et les copie dans out_dir. Retourne {index: html}."""80    os.makedirs(cache_dir, exist_ok=True)81    os.makedirs(out_dir, exist_ok=True)82    out = {}83    for idx, rel in enumerate(images):84        src = os.path.join(src_dir, rel)85        if not os.path.exists(src):86            out[idx] = ""87            continue88        if rel.endswith(".png") or rel.endswith(".jpg"):89            name = os.path.basename(rel)90            shutil.copy2(src, os.path.join(out_dir, name))91            out[idx] = f'<span class="figimg"><img src="/assets/rapport/{name}" alt="" loading="lazy"></span>'92            continue93        h = hashlib.sha1((rel + str(os.path.getmtime(src))).encode()).hexdigest()[:12]94        name = os.path.splitext(os.path.basename(rel))[0] + "-" + h + ".svg"95        cached = os.path.join(cache_dir, name)96        if not os.path.exists(cached):97            subprocess.run(["pdftocairo", "-svg", src, cached], check=True, timeout=120)98        shutil.copy2(cached, os.path.join(out_dir, name))99        head = open(cached, encoding="utf-8", errors="replace").read(600)100        m = re.search(r'width="([\d.]+)pt" height="([\d.]+)pt"', head)101        attrs = f' width="{int(float(m.group(1)) * 4 / 3)}" height="{int(float(m.group(2)) * 4 / 3)}"' if m else ""102        out[idx] = f'<span class="figimg"><img src="/assets/rapport/{name}" alt="Figure du rapport" loading="lazy" decoding="async"{attrs}></span>'103    return out104105106def extract_hpi(src_dir: str) -> dict:107    """Extrait les séries HPI (mensuel non désaisonnalisé) des marchés d'intérêt."""108    import openpyxl109    path = os.path.join(src_dir, "data", "Not Seasonally Adjusted (M).xlsx")110    wb = openpyxl.load_workbook(path, read_only=True, data_only=True)111    wanted = {"AGGREGATE": "Canada", "QUEBEC": "Québec (province)", "MONTREAL_CMA": "RMR de Montréal", "QUEBEC_CMA": "RMR de Québec",112              "ESTRIE": "Estrie", "MAURICIE": "Mauricie", "CENTRE_DU_QUEBEC": "Centre-du-Québec", "ONTARIO": "Ontario",113              "GREATER_TORONTO": "Grand Toronto", "GREATER_VANCOUVER": "Grand Vancouver", "OTTAWA": "Ottawa", "CALGARY": "Calgary",114              "BRITISH_COLUMBIA": "Colombie-Britannique", "ALBERTA": "Alberta", "HALIFAX_DARTMOUTH": "Halifax", "WINNIPEG": "Winnipeg"}115    cols = ["Composite_HPI", "Single_Family_HPI", "One_Storey_HPI", "Two_Storey_HPI", "Townhouse_HPI", "Apartment_HPI", "Composite_Benchmark", "Single_Family_Benchmark", "Apartment_Benchmark"]116    dates = None117    series = {}118    for sheet, label in wanted.items():119        if sheet not in wb.sheetnames:120            continue121        ws = wb[sheet]122        rows = [r for r in ws.iter_rows(values_only=True) if r and r[0] is not None]123        header = [str(h) for h in rows[0]]124        idx = {c: header.index(c) for c in cols if c in header}125        data = [r for r in rows[1:] if isinstance(r[0], datetime)]126        d = [r[0].strftime("%Y-%m") for r in data]127        if dates is None or len(d) > len(dates):128            dates = d129        series[label] = {c: [(round(float(r[i]), 1) if isinstance(r[i], (int, float)) else None) for r in data] for c, i in idx.items()}130    return {"dates": dates, "series": series, "source": "ACI/CREA — Indice des prix des propriétés MLS®, mensuel non désaisonnalisé, janv. 2005 = 100"}131132133def extract_fred(src_dir: str) -> dict:134    meta = {"IR3TIB01CAM156N": ("Taux interbancaire 3 mois", "%"), "IRLTLT01CAM156N": ("Obligations 10 ans", "%"),135            "CPALTT01CAM659N": ("Inflation (IPC, variation annuelle)", "%"), "LRUNTTTTCAM156S": ("Taux de chômage", "%"),136            "DEXCAUS": ("Taux de change CAD/USD", "CAD par USD"), "NGDPRSAXDCCAQ": ("PIB réel (trimestriel)", "M$ 2017"),137            "POPTOTCAA647NWDB": ("Population", "habitants"), "CANCPIALLMINMEI": ("IPC (indice)", "2015 = 100")}138    out = {}139    for fid, (label, unit) in meta.items():140        p = os.path.join(src_dir, "data", "fred", fid + ".json")141        if not os.path.exists(p):142            continue143        d = json.load(open(p, encoding="utf-8"))144        obs = d.get("observations", []) if isinstance(d, dict) else d145        pts = []146        for o in obs:147            try:148                v = float(o["value"])149            except (ValueError, KeyError, TypeError):150                continue151            pts.append([o["date"][:7], round(v, 3)])152        if fid == "DEXCAUS":  # quotidien → mensuel (moyenne)153            agg = {}154            for dte, v in pts:155                agg.setdefault(dte, []).append(v)156            pts = [[k, round(sum(v) / len(v), 4)] for k, v in sorted(agg.items())]157        out[fid] = {"label": label, "unit": unit, "points": pts}158    return out159160161def build_report(ctx_code: str, root: str, out_dir: str, cache_dir: str, finalize_cb=None):162    """Convertit le rapport. Retourne dict {sections: [ChapterResult+meta], vals, bib, images_html, hpi, fred}."""163    src_dir = os.path.join(root, REPORT_DIR, "sources")164    body, sections, vals = preprocess(src_dir)165    bib = parse_bib(open(os.path.join(src_dir, "references.bib"), encoding="utf-8").read())166    ctx = Ctx(ctx_code + "-RAPPORT", bib)167    ctx.images = []168    results = []169    for s in sections:170        url = f"/rapport/{s['num']}/"171        r = convert_chapter(ctx, s["key"], s["num"], url, s["tex"])172        r.appendix = s["appendix"]173        results.append(r)174    img_html = convert_figures(src_dir, ctx.images, os.path.join(cache_dir, "rapport-fig"), os.path.join(out_dir, "assets", "rapport"))175    hpi = extract_hpi(src_dir)176    fred = extract_fred(src_dir)177    return {"sections": results, "vals": vals, "bib": bib, "ctx": ctx, "images": img_html, "hpi": hpi, "fred": fred, "src_dir": src_dir}178