spb/spb-cloud

1/**2 * Détection des doublons exacts (SHA-256 du contenu déchiffré) dans le périmètre → data/dupes.json3 *   npx tsx --env-file=.env scripts/organize/dupes.ts4 */5import crypto from "crypto";6import fs from "fs";7import path from "path";8import { loadFileData } from "../../lib/storage";9import { prisma } from "../../lib/prisma";10import { DATA_DIR, filesInScope, pool } from "./common";1112export const DUPES_PATH = path.join(DATA_DIR, "dupes.json");1314async function main() {15  const files = await filesInScope();16  const hashes = new Map<string, string>(); // fileId → sha25617  let n = 0;18  await pool(files, 8, async (f) => {19    try {20      const buf = await loadFileData(f.storagePath, f.isEncrypted);21      hashes.set(f.id, crypto.createHash("sha256").update(buf).digest("hex"));22    } catch (e) {23      console.warn("hash impossible", f.name, e instanceof Error ? e.message : e);24    }25    if (++n % 200 === 0) console.log(`${n}/${files.length}`);26  });27  const groups = new Map<string, string[]>();28  for (const [id, h] of Array.from(hashes.entries())) { if (!groups.has(h)) groups.set(h, []); groups.get(h)!.push(id); }29  const byId = new Map(files.map((f) => [f.id, f]));30  const dupGroups = Array.from(groups.values()).filter((ids) => ids.length > 1).map((ids) => {31    // le « maître » = le plus ancien (createdAt), puis nom le plus court32    const sorted = ids.map((id) => byId.get(id)!).sort((a, b) => a.createdAt.localeCompare(b.createdAt) || a.name.length - b.name.length);33    return { master: sorted[0].id, duplicates: sorted.slice(1).map((f) => f.id), name: sorted[0].name, space: sorted[0].spaceName, size: sorted[0].size };34  });35  const out = { generatedAt: new Date().toISOString(), hashed: hashes.size, groups: dupGroups, duplicateIds: dupGroups.flatMap((g) => g.duplicates) };36  fs.writeFileSync(DUPES_PATH, JSON.stringify(out, null, 2));37  const bySpace = new Map<string, number>();38  for (const g of dupGroups) bySpace.set(g.space, (bySpace.get(g.space) ?? 0) + g.duplicates.length);39  console.log(`${hashes.size} fichiers hachés · ${dupGroups.length} groupes de doublons · ${out.duplicateIds.length} copies redondantes`);40  for (const [s, c] of Array.from(bySpace.entries())) console.log(`  ${s}: ${c}`);41  await prisma.$disconnect();42}4344main().catch((e) => { console.error(e); process.exit(1); });45