spb/wp46_uqo

1"""Resumable, rate-limit-aware ingestion of 1-minute (and daily) bars into the private cache (outside git).23Layout: <cache>/bars/<dataset_id>/<symbol>/<start>_<end>.parquet + .json manifest (dataset version, sha256, rows).4Exports are chunked (≤ CHUNK_YEARS per request) because the server rejects very long ranges. Nothing is re-downloaded5when the manifest exists and the file hash matches."""6from __future__ import annotations78import json9import os10from datetime import date11from pathlib import Path1213from .client import FinDataClient, sha256_file1415CHUNK_YEARS = 516DATASET_FOR = {  # asset class -> 1-minute dataset used for realized measures17    "etf": "market_bars.etf.1m.split",18    "stock": "market_bars.stock.1m.split",19    "futures": "market_bars.futures.1m.continuous_ratio",20    "fx": "market_bars.fx.1m.raw",21    "crypto": "market_bars.crypto.1m.raw",22}23DAILY_FOR = {"etf": "market_bars.etf.1d.split_div", "stock": "market_bars.stock.1d.split_div",24             "futures": "market_bars.futures.1d.continuous_ratio", "crypto": "market_bars.crypto.1d.raw",25             "index": "market_bars.index.1d.raw"}262728def cache_root() -> Path:29    return Path(os.environ.get("WP46_CACHE", "~/wp46-cache")).expanduser()303132def chunks(start: str, end: str, years: int = CHUNK_YEARS) -> list[tuple[str, str]]:33    s, e = date.fromisoformat(start), date.fromisoformat(end)34    out = []35    y = s36    while y <= e:37        y2 = min(e, date(y.year + years - 1, 12, 31))38        out.append((y.isoformat(), y2.isoformat()))39        y = date(y2.year + 1, 1, 1)40    return out414243def ingest_symbol(fd: FinDataClient, dataset_id: str, symbol: str, start: str, end: str, cache: Path | None = None,44                  adjustment: str | None = None) -> list[dict]:45    cache = cache or cache_root()46    d = cache / "bars" / dataset_id / symbol47    d.mkdir(parents=True, exist_ok=True)48    out = []49    for a, b in chunks(start, end):50        f = d / f"{a}_{b}.parquet"51        m = f.with_suffix(".json")52        if m.exists() and f.exists():53            man = json.loads(m.read_text())54            if man.get("sha256") == sha256_file(f):55                out.append(man)56                continue57        man = fd.export(dataset_id, [symbol], a, b, f, adjustment=adjustment)58        man["symbol"] = symbol59        m.write_text(json.dumps(man, indent=1))60        out.append(man)61    return out62