SPB Git

spb/firc0001_uqo

Public
2.5 KB · 61 lines diff
1diff --git a/project-db/spid/store.py b/project-db/spid/store.py2index 9b2d7d1..5265333 1006443--- a/project-db/spid/store.py4+++ b/project-db/spid/store.py5@@ -186,7 +186,9 @@ class Store:6         sql += " ORDER BY filing_date"7         if limit:8             sql += f" LIMIT {int(limit)}"9-        return self.con.execute(sql).fetch_arrow_table().to_pylist()10+        res = self.con.execute(sql)11+        table = res.to_arrow_table() if hasattr(res, "to_arrow_table") else res.fetch_arrow_table()  # DuckDB >= 2.0 / < 2.012+        return table.to_pylist()13 14     # ---- write side ---------------------------------------------------15     def upsert_mentions(self, mentions: list[ProjectMention], ctx: dict) -> int:16diff --git a/src/parsing/filing_parser.py b/src/parsing/filing_parser.py17index 1d9f6cc..e1327ed 10064418--- a/src/parsing/filing_parser.py19+++ b/src/parsing/filing_parser.py20@@ -18,6 +18,7 @@ from __future__ import annotations21 22 import json23 import traceback24+import uuid25 from concurrent.futures import ProcessPoolExecutor, as_completed26 from datetime import datetime, timezone27 from pathlib import Path28@@ -78,6 +79,18 @@ class FilingParser:29             return self._error_result(accession_number, form_type, "empty_after_clean")30 31         sections = self._extractor.extract_sections(clean_text, form_type)32+        if not sections and clean_text:33+            # no Item markers found (e.g. exhibit-only 10-Q/10-K): keep the whole text as one34+            # real section so the chunks satisfy the filing_chunks -> filing_sections foreign key35+            sections = [{36+                "section_id": str(uuid.uuid4()),37+                "section_name": "full_document",38+                "section_order": 0,39+                "item_number": "N/A",40+                "text": clean_text.strip(),41+                "word_count": len(clean_text.split()),42+                "char_count": len(clean_text),43+            }]44         for section in sections:45             section["accession_number"] = accession_number46 47@@ -88,13 +101,6 @@ class FilingParser:48                 chunk["accession_number"] = accession_number49             chunks.extend(section_chunks)50 51-        if not chunks and clean_text:52-            fallback_chunks = self._chunker.chunk_text(clean_text)53-            for chunk in fallback_chunks:54-                chunk["section_id"] = "full_document"55-                chunk["accession_number"] = accession_number56-            chunks.extend(fallback_chunks)57-58         result = {59             "accession_number": accession_number,60             "form_type": form_type,61
Files · 626579ecc632db9471ed812f29c10ca85bc6ed47
Menu

Theme