/** * earth-now.co * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * File: apps/ingest/src/runners.ts * Purpose: Per-source run functions (fetch → archive raw → parse → summary) shared by the CLI and the queue worker */ import { archiveRaw } from "./archive.js"; import { type FetchLike, defaultFetch } from "./fetch-like.js"; import { NOAA_CO2_URL, parseNoaaMonthlyCo2 } from "./sources/noaa-co2.js"; import { parseUsgsCount, usgsCountUrl } from "./sources/usgs.js"; /** USGS live counter parameters: quakes M ≥ 4.5 over the trailing 24 h. */ export const USGS_WINDOW_HOURS = 24; // Must match the registry definition of earthquakes_24h (catalog: M ≥ 2.5). export const USGS_MIN_MAGNITUDE = 2.5; export interface ObservationPoint { time: string; value: number; } export interface RunSummary { sourceId: string; observationCount: number; first: ObservationPoint | null; last: ObservationPoint | null; rawPath: string; sha256: string; } async function fetchText(url: string, fetchImpl: FetchLike): Promise { const res = await fetchImpl(url); if (!res.ok) throw new Error(`fetch failed: HTTP ${res.status} for ${url}`); return res.text(); } /** USGS FDSN: one observation — the M≥4.5 quake count over the trailing 24 h window. */ export async function runUsgsFdsn(fetchImpl: FetchLike = defaultFetch): Promise { const url = usgsCountUrl(USGS_WINDOW_HOURS, USGS_MIN_MAGNITUDE); const text = await fetchText(url, fetchImpl); const archived = await archiveRaw("usgs_fdsn", text); const count = parseUsgsCount(JSON.parse(text)); const point: ObservationPoint = { time: new Date().toISOString(), value: count }; return { sourceId: "usgs_fdsn", observationCount: 1, first: point, last: point, rawPath: archived.rawPath, sha256: archived.sha256, }; } /** NOAA GML Mauna Loa: full monthly CO₂ series (Keeling curve). */ export async function runNoaaGmlMlo(fetchImpl: FetchLike = defaultFetch): Promise { const text = await fetchText(NOAA_CO2_URL, fetchImpl); const archived = await archiveRaw("noaa_gml_mlo", text); const observations = parseNoaaMonthlyCo2(text); return { sourceId: "noaa_gml_mlo", observationCount: observations.length, first: observations[0] ?? null, last: observations[observations.length - 1] ?? null, rawPath: archived.rawPath, sha256: archived.sha256, }; } export interface SourceSpec { id: string; /** Scheduling cadence in ms (usgs 5 min, noaa daily). */ cadenceMs: number; run: (fetchImpl?: FetchLike) => Promise; } /** Every source the ingest service knows how to run, with its scheduling cadence. */ export const SOURCES: readonly SourceSpec[] = [ { id: "usgs_fdsn", cadenceMs: 5 * 60_000, run: runUsgsFdsn }, { id: "noaa_gml_mlo", cadenceMs: 24 * 3_600_000, run: runNoaaGmlMlo }, ]; export const KNOWN_SOURCE_IDS: readonly string[] = SOURCES.map((s) => s.id); /** Run one source by id. Throws on unknown id or on any fetch/parse failure. */ export async function runSource( sourceId: string, fetchImpl: FetchLike = defaultFetch, ): Promise { const spec = SOURCES.find((s) => s.id === sourceId); if (!spec) throw new Error(`Unknown source '${sourceId}'. Known: ${KNOWN_SOURCE_IDS.join(", ")}`); return spec.run(fetchImpl); }