spb/company-atlas
Public
Python 66.3%
TypeScript 22.7%
JavaScript 8.6%
HTML 1.4%
CSS 0.7%
1"""Mis-decoded bodies never become snapshots, entities or events (2026-09-12 Brotli incident)."""2from __future__ import annotations34import gzip56import brotli78from companyatlas.fetch import decode_text, looks_binary, text_quality9from companyatlas.sdk.models import ExtractedJob, ExtractedNewsItem, ExtractedPerson, Extraction10from companyatlas.services.pipeline import _drop_corrupt_entities1112HTML = "<html><head><meta charset='utf-8'><title>Société Générale — résultats</title></head><body>Économie 日本語 Zürich</body></html>".encode()131415def test_binary_bodies_are_flagged() -> None:16 assert looks_binary(brotli.compress(HTML), "text/html; charset=utf-8")17 assert looks_binary(gzip.compress(HTML), "text/html")18 assert not looks_binary(HTML, "text/html")19 assert not looks_binary(b"%PDF-1.7 binary", "application/pdf") # non-textual types are left to their connectors202122def test_decode_prefers_clean_codec() -> None:23 assert text_quality(decode_text(HTML, "text/html")) == 1.024 assert "bénéfices" in decode_text("Groupe Lactalis – bénéfices élevés".encode("cp1252"), "text/html")25 sjis = "<html><head><meta charset=shift_jis></head><body>トヨタ自動車</body></html>".encode("shift_jis")26 assert "トヨタ" in decode_text(sjis, "text/html")27 garbage = brotli.compress(HTML).decode("utf-8", errors="replace")28 assert text_quality(garbage) < 0.99293031def test_corrupt_entities_are_dropped() -> None:32 ex = Extraction(text="ok", blocks=[], title="News ��", jobs=[ExtractedJob(title="ML Engineer"), ExtractedJob(title="��7�c")],33 people=[ExtractedPerson(name="Jane Doe"), ExtractedPerson(name="1234")],34 news=[ExtractedNewsItem(title="Quarterly results", url="https://x.com/a"), ExtractedNewsItem(title="ng��t1]", url="https://x.com/b")])35 _drop_corrupt_entities(ex)36 assert [j.title for j in ex.jobs] == ["ML Engineer"]37 assert [p.name for p in ex.people] == ["Jane Doe"]38 assert [n.title for n in ex.news] == ["Quarterly results"]39