/** * earth-now.co * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * File: apps/ingest/test/runners.test.ts * Purpose: End-to-end run functions (fetch → archive → parse → summary) with fixture-backed fakes, no network */ import { existsSync, mkdtempSync, readdirSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import type { FetchLike } from "../src/fetch-like"; import { KNOWN_SOURCE_IDS, runSource } from "../src/runners"; const fixture = (name: string): string => readFileSync(new URL(`./fixtures/${name}`, import.meta.url), "utf8"); let previousRawDir: string | undefined; beforeEach(() => { previousRawDir = process.env.RAW_ARCHIVE_DIR; process.env.RAW_ARCHIVE_DIR = mkdtempSync(join(tmpdir(), "earth-now-runners-")); }); afterEach(() => { if (previousRawDir === undefined) delete process.env.RAW_ARCHIVE_DIR; else process.env.RAW_ARCHIVE_DIR = previousRawDir; }); describe("runSource", () => { it("knows exactly the declared sources", () => { expect([...KNOWN_SOURCE_IDS]).toEqual(["usgs_fdsn", "noaa_gml_mlo"]); }); it("rejects unknown source ids listing the known ones", async () => { await expect(runSource("nope")).rejects.toThrow(/Unknown source 'nope'.*usgs_fdsn/); }); it("usgs_fdsn: fetches, archives the raw payload and summarizes one observation", async () => { const body = fixture("usgs_count_sample.json"); const fetchImpl: FetchLike = async () => ({ ok: true, status: 200, text: async () => body }); const summary = await runSource("usgs_fdsn", fetchImpl); expect(summary.sourceId).toBe("usgs_fdsn"); expect(summary.observationCount).toBe(1); expect(summary.first?.value).toBe(132); expect(existsSync(summary.rawPath)).toBe(true); expect(readFileSync(summary.rawPath, "utf8")).toBe(body); expect(existsSync(`${summary.rawPath}.sha256`)).toBe(true); }); it("noaa_gml_mlo: fetches, archives and summarizes the monthly series", async () => { const body = fixture("co2_mm_mlo_sample.txt"); const fetchImpl: FetchLike = async () => ({ ok: true, status: 200, text: async () => body }); const summary = await runSource("noaa_gml_mlo", fetchImpl); expect(summary.observationCount).toBe(15); expect(summary.first?.time).toBe("2024-01-15T00:00:00.000Z"); expect(summary.last?.value).toBe(429.64); expect(readFileSync(summary.rawPath, "utf8")).toBe(body); }); it("noaa_gml_mlo: still archives the raw payload even when parsing fails (truncated file)", async () => { const body = fixture("co2_mm_mlo_truncated.txt"); const fetchImpl: FetchLike = async () => ({ ok: true, status: 200, text: async () => body }); await expect(runSource("noaa_gml_mlo", fetchImpl)).rejects.toThrow(/truncated/); // The raw bytes were archived before parsing — nothing ingested is ever lost. const archivedDir = join(process.env.RAW_ARCHIVE_DIR ?? "", "noaa_gml_mlo"); expect(existsSync(archivedDir)).toBe(true); expect(readdirSync(archivedDir).some((f) => f.endsWith(".raw"))).toBe(true); }); });