/** * ───────────────────────────────────────────── * SPB Drive — Personal Cloud Drive * ───────────────────────────────────────────── * Author : Simon-Pierre Boucher * Contact : contact@spboucher.ai * File : src/storage/blobs.mjs * Purpose : Content-addressed blob store — sha256 CAS, refcounts, GC * License : MIT © Simon-Pierre Boucher * ───────────────────────────────────────────── */ import { createHash, randomBytes } from 'node:crypto'; import { createReadStream, createWriteStream, existsSync } from 'node:fs'; import { mkdir, rename, rm, stat } from 'node:fs/promises'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; import { config } from '../config.mjs'; import { getDb } from '../db/db.mjs'; /** Absolute on-disk path for a blob sha. */ export function blobPath(sha) { return path.join(config.filesDir, sha.slice(0, 2), sha.slice(2, 4), sha); } /** * Store a readable stream in the CAS. Streams to a temp file while hashing, * then moves into place — identical content dedupes naturally. * @param {NodeJS.ReadableStream} stream * @returns {Promise<{sha: string, size: number, deduped: boolean}>} */ export async function putBlob(stream) { const tmp = path.join(config.cacheDir, 'tmp', `put-${randomBytes(8).toString('hex')}`); const hash = createHash('sha256'); let size = 0; await pipeline( stream, async function* (source) { for await (const chunk of source) { hash.update(chunk); size += chunk.length; yield chunk; } }, createWriteStream(tmp, { mode: 0o600 }), ); const sha = hash.digest('hex'); const dest = blobPath(sha); const deduped = existsSync(dest); if (deduped) { await rm(tmp, { force: true }); } else { await mkdir(path.dirname(dest), { recursive: true, mode: 0o700 }); await rename(tmp, dest); } getDb() .prepare( `INSERT INTO blobs (sha, size, refcount, created) VALUES (?, ?, 0, ?) ON CONFLICT(sha) DO NOTHING`, ) .run(sha, size, Date.now()); return { sha, size, deduped }; } /** Increment a blob's refcount (one node now points at it). */ export function refBlob(sha) { getDb().prepare('UPDATE blobs SET refcount = refcount + 1 WHERE sha = ?').run(sha); } /** Decrement a blob's refcount. Physical deletion happens in gcBlobs(). */ export function unrefBlob(sha) { getDb().prepare('UPDATE blobs SET refcount = MAX(refcount - 1, 0) WHERE sha = ?').run(sha); } /** Readable stream over a blob (optionally a byte range). */ export function blobStream(sha, { start, end } = {}) { return createReadStream(blobPath(sha), start !== undefined ? { start, end } : {}); } /** Blob size from disk (throws if missing). */ export async function blobSize(sha) { return (await stat(blobPath(sha))).size; } /** * Delete blobs with refcount 0 (and their cache artifacts). * @returns {Promise} number of blobs removed */ export async function gcBlobs() { const db = getDb(); const orphans = db.prepare('SELECT sha FROM blobs WHERE refcount <= 0').all(); for (const { sha } of orphans) { await rm(blobPath(sha), { force: true }); for (const sub of ['thumbs', 'transcode', 'office', 'peaks', 'text']) { const dir = path.join(config.cacheDir, sub); for (const suffix of ['', '.256.webp', '.512.webp', '.pdf', '.mp4', '.json', '.txt']) { await rm(path.join(dir, `${sha}${suffix}`), { force: true }); } } db.prepare('DELETE FROM blobs WHERE sha = ?').run(sha); } return orphans.length; } /** Aggregate storage stats for /healthz and the storage meter. */ export function storageStats() { const db = getDb(); const blobs = db.prepare('SELECT COUNT(*) AS n, COALESCE(SUM(size), 0) AS bytes FROM blobs WHERE refcount > 0').get(); const nodes = db.prepare("SELECT COUNT(*) AS n FROM nodes WHERE trashed_at IS NULL AND id != 1").get(); const byType = db .prepare( `SELECT CASE WHEN mime LIKE 'image/%' THEN 'images' WHEN mime LIKE 'video/%' THEN 'video' WHEN mime LIKE 'audio/%' THEN 'audio' WHEN mime LIKE 'application/pdf' OR mime LIKE '%word%' OR mime LIKE '%sheet%' OR mime LIKE '%presentation%' OR mime LIKE 'text/%' THEN 'docs' WHEN mime LIKE '%zip%' OR mime LIKE '%tar%' OR mime LIKE '%7z%' OR mime LIKE '%rar%' THEN 'archives' ELSE 'other' END AS bucket, COALESCE(SUM(size), 0) AS bytes, COUNT(*) AS n FROM nodes WHERE type = 'file' AND trashed_at IS NULL GROUP BY bucket`, ) .all(); return { blobCount: blobs.n, usedBytes: blobs.bytes, nodeCount: nodes.n, byType }; }