"""Content-addressable object store for raw observations, normalized text and block structures (spec §26, §96, §97). objects///.zst Objects are written once (dedupe by hash) and never mutated; many observations may point at the same key. zstd level 9 gives ~8× on HTML. `key` = sha256 hex of the *uncompressed* bytes; the relative path is derived, so the store can move as a whole. """ from __future__ import annotations import hashlib import os from pathlib import Path import zstandard as zstd from companyatlas.config import settings _cctx = zstd.ZstdCompressor(level=9) _dctx = zstd.ZstdDecompressor() def object_path(key: str, root: Path | None = None) -> Path: root = root or settings.objects_dir return root / key[:2] / key[2:4] / f"{key}.zst" def sha256_hex(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def put_bytes(data: bytes, *, key: str | None = None) -> tuple[str, int, bool]: """Store bytes; return (key, stored_size_bytes, created). Idempotent.""" key = key or sha256_hex(data) path = object_path(key) if path.exists(): return key, path.stat().st_size, False path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(f".zst.tmp{os.getpid()}") with open(tmp, "wb") as fh: fh.write(_cctx.compress(data)) tmp.replace(path) return key, path.stat().st_size, True def put_text(text: str) -> tuple[str, int, bool]: return put_bytes(text.encode("utf-8")) def get_bytes(key: str) -> bytes: with open(object_path(key), "rb") as fh: return _dctx.decompress(fh.read(), max_output_size=64 * 1024 * 1024) def get_text(key: str) -> str: return get_bytes(key).decode("utf-8", errors="replace") def exists(key: str) -> bool: return object_path(key).exists() def store_stats(root: Path | None = None) -> dict[str, int]: root = root or settings.objects_dir total = count = 0 if root.exists(): for dirpath, _dirs, files in os.walk(root): for f in files: if f.endswith(".zst"): try: total += os.stat(os.path.join(dirpath, f)).st_size count += 1 except OSError: pass return {"objects": count, "bytes": total} __all__ = ["exists", "get_bytes", "get_text", "object_path", "put_bytes", "put_text", "sha256_hex", "store_stats"]