"""Reference data: ticker listings, delisted log, splits and dividends → instruments / corporate_actions.""" from __future__ import annotations from typing import Any from findata_core.db import transaction from findata_market_firstrate import FirstRateMarketConnector from findata_ingest.lake import Lake from findata_ingest.tasks.common import record_raw_archive def run(job: dict[str, Any], lake: Lake, log) -> dict[str, Any]: params = job.get("params") or {} types = params.get("types") or ["stock", "etf"] conn_fr = FirstRateMarketConnector() stats: dict[str, Any] = {} for t in types: rows = conn_fr.ticker_listing(t) with transaction() as conn: for r in rows: iid = f"firstrate:{t}:{r['symbol']}" conn.execute( "INSERT INTO instruments(instrument_id, symbol, name, asset_class, venue, currency, first_date, last_date, status, source_id) " "VALUES (%s,%s,%s,%s,NULL,'USD',%s,%s,%s,'firstrate') ON CONFLICT (instrument_id) DO UPDATE SET name=EXCLUDED.name, " "first_date=EXCLUDED.first_date, last_date=EXCLUDED.last_date, status=EXCLUDED.status, updated_at=now()", (iid, r["symbol"], r["name"], t, r["first_date"], r["last_date"], r["status"])) if r["symbol"].endswith("-DELISTED"): base = r["symbol"][: -len("-DELISTED")] conn.execute("INSERT INTO instrument_aliases(instrument_id, symbol, valid_from, valid_to, source_id, note) VALUES (%s,%s,%s,%s,'firstrate','vendor -DELISTED suffix') " "ON CONFLICT DO NOTHING", (iid, base, r["first_date"], r["last_date"])) stats[f"instruments_{t}"] = len(rows) log.info("metadata.listing", type=t, rows=len(rows)) if "stock" in types: dl = conn_fr.delisted_log() with transaction() as conn: for r in dl: conn.execute("UPDATE instruments SET status='delisted', last_date=COALESCE(%s, last_date), updated_at=now() WHERE instrument_id=%s", (r["last_date"], f"firstrate:stock:{r['symbol']}")) stats["delisted_log"] = len(dl) for kind in ("splits", "dividends"): res, rows = conn_fr.meta_actions("stock", kind, lake.raw_dir("firstrate", "corporate_actions.stock")) with transaction() as conn: record_raw_archive(conn, "firstrate", "corporate_actions.stock", {"endpoint": "meta_file", "type": "stock", "metafile_type": kind}, str(res.path.relative_to(lake.root)), res.sha256, res.bytes, res.file_count, res.vendor_last_update, job.get("run_id")) n = 0 for r in rows: conn.execute("INSERT INTO corporate_actions(instrument_id, action_type, ex_date, value, source_id, ingest_run_id) VALUES (%s,%s,%s,%s,'firstrate',%s) ON CONFLICT DO NOTHING", (f"firstrate:stock:{r['symbol']}", r["action_type"], r["ex_date"], r["value"], job.get("run_id"))) n += 1 stats[kind] = n log.info("metadata.actions", kind=kind, rows=n) return stats