fundamentals: coverage_store.intern résout les ids localement (KeyError quand le cache est vidé à CACHE_MAX en plein appel — prod 2026-09-06)
2 changed files +20 −3
modified
hfmarketdata/api/fundamentals/coverage_store.py
+8 −3
@@ -56,8 +56,13 @@ def intern(docs: list[dict[str, Any] | None]) -> list[int]: | ||
| 56 | 56 | by_sha: dict[str, dict[str, Any] | None] = {} |
| 57 | 57 | for s, d in zip(shas, docs): |
| 58 | 58 | by_sha.setdefault(s, d) |
| 59 | + # Resolve into a local map: the process cache may be evicted (CACHE_MAX) between _remember() and the return. | |
| 60 | + found: dict[str, int] = {} | |
| 59 | 61 | with _lock: |
| 60 | − missing = [s for s in by_sha if s not in _id_by_sha] | |
| 62 | + for sha in by_sha: | |
| 63 | + if sha in _id_by_sha: | |
| 64 | + found[sha] = _id_by_sha[sha] | |
| 65 | + missing = [s for s in by_sha if s not in found] | |
| 61 | 66 | if missing: |
| 62 | 67 | with session() as s: |
| 63 | 68 | for i in range(0, len(missing), 500): |
@@ -65,9 +70,9 @@ def intern(docs: list[dict[str, Any] | None]) -> list[int]: | ||
| 65 | 70 | s.execute(sqlite_insert(FundCoverageBlob).on_conflict_do_nothing(index_elements=["sha1"]), |
| 66 | 71 | [{"sha1": sha, "json": canonical(by_sha[sha])} for sha in chunk]) |
| 67 | 72 | for row in s.execute(select(FundCoverageBlob.id, FundCoverageBlob.sha1).where(FundCoverageBlob.sha1.in_(chunk))): |
| 73 | + found[row.sha1] = int(row.id) | |
| 68 | 74 | _remember(row.sha1, int(row.id), by_sha[row.sha1] or {}) |
| 69 | − with _lock: | |
| 70 | − return [_id_by_sha[sha] for sha in shas] | |
| 75 | + return [found[sha] for sha in shas] | |
| 71 | 76 | |
| 72 | 77 | |
| 73 | 78 | def fetch(ids: set[int]) -> dict[int, dict[str, Any]]: |
modified
tests/test_fundamentals_ingest.py
+12 −0
@@ -279,6 +279,18 @@ def test_coverage_dictionary_interns_and_hydrates(app): | ||
| 279 | 279 | assert CS.sha_of_ids({ids[0]}) == {ids[0]: CS.digest(a)} |
| 280 | 280 | |
| 281 | 281 | |
| 282 | +def test_coverage_intern_survives_cache_eviction(app, monkeypatch): | |
| 283 | + """Prod 2026-09-06: the 50 533rd distinct document raised KeyError because _remember() wiped the cache | |
| 284 | + (CACHE_MAX) in the middle of intern(); ids must be resolved locally, not through the cache.""" | |
| 285 | + from fundamentals import coverage_store as CS | |
| 286 | + CS.reset_cache_for_tests() | |
| 287 | + monkeypatch.setattr(CS, "CACHE_MAX", 5) | |
| 288 | + docs = [{"k%d" % i: {"reason": "r%d" % i}} for i in range(20)] + [{"k0": {"reason": "r0"}}] | |
| 289 | + ids = CS.intern(docs) | |
| 290 | + assert len(ids) == 21 and len(set(ids[:20])) == 20 and ids[20] == ids[0] | |
| 291 | + assert CS.intern(docs) == ids # second pass: same ids, no KeyError | |
| 292 | + | |
| 293 | + | |
| 282 | 294 | def test_failure_samples_append_and_cap(app): |
| 283 | 295 | from core.db import session |
| 284 | 296 | from fundamentals import ingest |
| 285 | 297 | |