"""Single ingest worker: claims jobs from Postgres, holds the lake writer lock, runs the task registry. Also hosts the scheduler loop (idempotent enqueue with dedupe keys) so there is exactly one writer process.""" from __future__ import annotations import signal import time import traceback from datetime import datetime from zoneinfo import ZoneInfo from findata_core.config import get_settings from findata_core.logging import configure_logging, get_logger from findata_ingest import queue from findata_ingest.lake import Lake, WriterLock from findata_ingest.tasks import REGISTRY ET = ZoneInfo("America/New_York") _stop = False def _handle(sig, frame): global _stop _stop = True def schedule_tick(log) -> None: """Enqueue recurring jobs (dedupe keys make this idempotent). Times in America/New_York.""" now = datetime.now(ET) day = now.date().isoformat() # vendor daily update lands ~23:30–03:00 ET → poll from 02:00 every 30 min (incremental skips when vendor unchanged) if now.weekday() < 6 and (now.hour >= 2 or now.hour < 1): queue.enqueue("incremental", "market_bars.stock.1m.raw", {"type": "stock", "period": "day"}, priority=20, dedupe_key=f"incremental:stock:day:{day}:{now.hour}:{now.minute // 30}") if now.weekday() == 5 and now.hour == 6: # Saturday 06:00 ET — week archive repairs any missed day queue.enqueue("incremental", "market_bars.stock.1m.raw", {"type": "stock", "period": "week", "force": True}, priority=25, dedupe_key=f"incremental:stock:week:{day}") if now.weekday() == 6 and now.hour == 7: queue.enqueue("metadata", None, {"types": ["stock", "etf"]}, priority=30, dedupe_key=f"metadata:{day}") queue.enqueue("compaction", "market_bars.stock.1m.raw", {}, priority=60, dedupe_key=f"compaction:1m:{day}") queue.enqueue("compaction", "market_bars.stock.5m.raw", {}, priority=61, dedupe_key=f"compaction:5m:{day}") if now.hour == 5 and now.minute < 30: queue.enqueue("fundamentals", "fundamentals.financials", {"tracked": True}, priority=50, dedupe_key=f"fundamentals:{day}") if now.hour == 4 and now.minute < 30 and get_settings().backup_target: queue.enqueue("backup", None, {}, priority=70, dedupe_key=f"backup:{day}") if now.hour == 6 and now.minute < 30: queue.enqueue("quality", "market_bars.stock.1m.raw", {}, priority=80, dedupe_key=f"quality:{day}") queue.enqueue("gc", None, {}, priority=90, dedupe_key=f"gc:{day}") def main(once: bool = False, kinds: list[str] | None = None, with_scheduler: bool = True) -> None: configure_logging() log = get_logger("worker") signal.signal(signal.SIGTERM, _handle) signal.signal(signal.SIGINT, _handle) lake = Lake() requeued = queue.requeue_stale() if requeued: log.warning("worker.requeued_stale", n=requeued) last_tick = 0.0 with WriterLock(lake.root): log.info("worker.start", lake=str(lake.root), free_gb=round(lake.free_gb(), 1)) while not _stop: if with_scheduler and time.monotonic() - last_tick > 300: try: schedule_tick(log) except Exception as e: log.error("scheduler.error", error=str(e)) last_tick = time.monotonic() job = queue.claim_next(kinds) if job is None: if once: break time.sleep(5) continue jl = log.bind(job_id=str(job["id"]), kind=job["kind"], dataset=job.get("dataset_id"), attempt=job["attempts"]) jl.info("job.start") t0 = time.monotonic() try: fn = REGISTRY[job["kind"]] result = fn(job, lake, jl) queue.finish(job, True, {**(result or {}), "seconds": round(time.monotonic() - t0, 1)}) jl.info("job.done", seconds=round(time.monotonic() - t0, 1)) except Exception as e: err = f"{type(e).__name__}: {e}\n{traceback.format_exc()[-1500:]}" queue.finish(job, False, error=err) jl.error("job.failed", error=str(e)[:300], seconds=round(time.monotonic() - t0, 1)) if once: break log.info("worker.stop")