spb/datacenterindex

1/** Read-only access to the raw document archive (MinIO/S3). Keys are `raw/<connector>/<doc>/<hash>.<ext>.zst|gz`. */2import { S3Client, GetObjectCommand, HeadBucketCommand } from "@aws-sdk/client-s3";3import * as zlib from "node:zlib";4import { getEnv } from "./env.js";56let _client: S3Client | null = null;7export function getS3(): S3Client {8  if (_client) return _client;9  const env = getEnv();10  _client = new S3Client({ endpoint: env.s3Endpoint, region: env.s3Region, forcePathStyle: true, credentials: { accessKeyId: env.s3AccessKey, secretAccessKey: env.s3SecretKey }, requestHandler: { requestTimeout: 20_000, connectionTimeout: 3_000 } });11  return _client;12}1314const zstd = zlib as unknown as { zstdDecompressSync?: (b: Buffer) => Buffer };1516/** Decompress by key suffix; when the suffix lies, try zstd then gzip then raw. */17export function decompress(key: string, body: Buffer): Buffer {18  const tryZstd = () => (zstd.zstdDecompressSync ? zstd.zstdDecompressSync(body) : null);19  const tryGz = () => zlib.gunzipSync(body);20  if (key.endsWith(".zst")) { try { const r = tryZstd(); if (r) return r; } catch { /* fall through */ } try { return tryGz(); } catch { return body; } }21  if (key.endsWith(".gz")) { try { return tryGz(); } catch { try { const r = tryZstd(); if (r) return r; } catch { /* */ } return body; } }22  try { const r = tryZstd(); if (r) return r; } catch { /* */ }23  try { return tryGz(); } catch { return body; }24}2526export async function getRawObject(key: string, maxBytes = 5 * 1024 * 1024): Promise<{ body: Buffer; contentType: string | null; truncated: boolean } | null> {27  try {28    const r = await getS3().send(new GetObjectCommand({ Bucket: getEnv().s3Bucket, Key: key }));29    const bytes = r.Body ? Buffer.from(await r.Body.transformToByteArray()) : Buffer.alloc(0);30    const full = decompress(key, bytes);31    const truncated = full.length > maxBytes;32    return { body: truncated ? full.subarray(0, maxBytes) : full, contentType: r.Metadata?.["content-type"] ?? null, truncated };33  } catch (e) {34    const err = e as { name?: string; $metadata?: { httpStatusCode?: number } };35    if (err.name === "NoSuchKey" || err.name === "NotFound" || err.$metadata?.httpStatusCode === 404) return null;36    throw e;37  }38}3940export async function storageOk(): Promise<boolean> {41  try {42    await Promise.race([getS3().send(new HeadBucketCommand({ Bucket: getEnv().s3Bucket })), new Promise((_, rej) => setTimeout(() => rej(new Error("timeout")), 2_500))]);43    return true;44  } catch {45    return false;46  }47}48