diff --git a/project-db/spid/store.py b/project-db/spid/store.py index 9b2d7d1..5265333 100644 --- a/project-db/spid/store.py +++ b/project-db/spid/store.py @@ -186,7 +186,9 @@ class Store: sql += " ORDER BY filing_date" if limit: sql += f" LIMIT {int(limit)}" - return self.con.execute(sql).fetch_arrow_table().to_pylist() + res = self.con.execute(sql) + table = res.to_arrow_table() if hasattr(res, "to_arrow_table") else res.fetch_arrow_table() # DuckDB >= 2.0 / < 2.0 + return table.to_pylist() # ---- write side --------------------------------------------------- def upsert_mentions(self, mentions: list[ProjectMention], ctx: dict) -> int: diff --git a/src/parsing/filing_parser.py b/src/parsing/filing_parser.py index 1d9f6cc..e1327ed 100644 --- a/src/parsing/filing_parser.py +++ b/src/parsing/filing_parser.py @@ -18,6 +18,7 @@ from __future__ import annotations import json import traceback +import uuid from concurrent.futures import ProcessPoolExecutor, as_completed from datetime import datetime, timezone from pathlib import Path @@ -78,6 +79,18 @@ class FilingParser: return self._error_result(accession_number, form_type, "empty_after_clean") sections = self._extractor.extract_sections(clean_text, form_type) + if not sections and clean_text: + # no Item markers found (e.g. exhibit-only 10-Q/10-K): keep the whole text as one + # real section so the chunks satisfy the filing_chunks -> filing_sections foreign key + sections = [{ + "section_id": str(uuid.uuid4()), + "section_name": "full_document", + "section_order": 0, + "item_number": "N/A", + "text": clean_text.strip(), + "word_count": len(clean_text.split()), + "char_count": len(clean_text), + }] for section in sections: section["accession_number"] = accession_number @@ -88,13 +101,6 @@ class FilingParser: chunk["accession_number"] = accession_number chunks.extend(section_chunks) - if not chunks and clean_text: - fallback_chunks = self._chunker.chunk_text(clean_text) - for chunk in fallback_chunks: - chunk["section_id"] = "full_document" - chunk["accession_number"] = accession_number - chunks.extend(fallback_chunks) - result = { "accession_number": accession_number, "form_type": form_type,