spb/earth-now Public License

1/**2 * earth-now.co3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * File:    apps/ingest/src/archive.ts6 * Purpose: Raw payload archive — every ingested byte is kept (with sha256) for future re-fitting; S3-shaped interface, local FS for now7 */89import { createHash } from "node:crypto";10import { mkdir, writeFile } from "node:fs/promises";11import { join } from "node:path";1213export interface ArchiveOptions {14  /** Archive root. Defaults to $RAW_ARCHIVE_DIR, then "data/raw". */15  dir?: string;16  /** Timestamp override for deterministic tests. */17  now?: Date;18}1920export interface ArchiveResult {21  rawPath: string;22  checksumPath: string;23  sha256: string;24}2526/**27 * Write `${dir}/${sourceId}/${timestamp}.raw` plus a sibling `.sha256` checksum file.28 * We keep ALL raw data for re-fitting; when this moves to S3 only this module changes.29 */30export async function archiveRaw(31  sourceId: string,32  payload: string | Uint8Array,33  options: ArchiveOptions = {},34): Promise<ArchiveResult> {35  if (!/^[a-z0-9_-]+$/i.test(sourceId))36    throw new Error(`archiveRaw: invalid sourceId '${sourceId}' (path-safe [a-z0-9_-] only)`);37  const dir = options.dir ?? process.env.RAW_ARCHIVE_DIR ?? "data/raw";38  const now = options.now ?? new Date();39  // ISO timestamp made filename-safe: 2026-08-09T12:00:00.000Z → 2026-08-09T12-00-00-000Z40  const stamp = now.toISOString().replace(/[:.]/g, "-");41  const targetDir = join(dir, sourceId);42  await mkdir(targetDir, { recursive: true });4344  const data = typeof payload === "string" ? Buffer.from(payload, "utf8") : Buffer.from(payload);45  const sha256 = createHash("sha256").update(data).digest("hex");46  const fileName = `${stamp}.raw`;47  const rawPath = join(targetDir, fileName);48  const checksumPath = `${rawPath}.sha256`;4950  await writeFile(rawPath, data);51  await writeFile(checksumPath, `${sha256}  ${fileName}\n`, "utf8");52  return { rawPath, checksumPath, sha256 };53}54