# ============================================================ # Project: EDGAR Research Database # Description: SEC EDGAR Research Infrastructure # # Authors: # - Manel Kammoun — manel.kammoun@uqo.ca # - Djerrycharli Tandjambianda — djerrycharli.tandjambianda@uqo.ca # - Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca # # Institution: # Université du Québec en Outaouais (UQO) # # License: # Research / Academic Use # ============================================================ from __future__ import annotations import re import uuid from pathlib import Path from src.parsing.html_parser import HTMLCleaner from src.utils.constants import FORM_10K_SECTIONS, FORM_10Q_SECTIONS from src.utils.logging import get_logger logger = get_logger("parsing.sections") _10K_ITEM_PATTERN = re.compile( r"(?:^|\n)\s*ITEM\s+(\d+[A-B]?)\s*[.\-:—–]?\s*", re.IGNORECASE, ) _10Q_PART_PATTERN = re.compile( r"(?:^|\n)\s*PART\s+(I{1,2})\s*[.\-:—–]?\s*", re.IGNORECASE, ) _10Q_ITEM_PATTERN = re.compile( r"(?:^|\n)\s*ITEM\s+(\d+[A-B]?)\s*[.\-:—–]?\s*", re.IGNORECASE, ) def _roman_to_int(roman: str) -> int: return {"I": 1, "II": 2}.get(roman.upper(), 0) class SectionExtractor: def __init__(self) -> None: self._cleaner = HTMLCleaner() def extract_sections(self, text: str, form_type: str) -> list[dict]: form_type = form_type.upper().replace("/A", "").strip() if form_type == "10-K": return self._extract_10k(text) if form_type == "10-Q": return self._extract_10q(text) if form_type == "8-K": return self._extract_8k(text) logger.warning("Unsupported form type for section extraction: %s", form_type) return self._extract_8k(text) def extract_from_file(self, file_path: Path, form_type: str) -> list[dict]: text = self._cleaner.clean_file(Path(file_path)) if not text: return [] return self.extract_sections(text, form_type) def _extract_10k(self, text: str) -> list[dict]: matches = list(_10K_ITEM_PATTERN.finditer(text)) if not matches: logger.info("No Item markers found in 10-K filing") return [] boundaries: list[tuple[str, int]] = [] for m in matches: item_num = m.group(1).upper() if item_num in FORM_10K_SECTIONS: boundaries.append((item_num, m.start())) boundaries = self._deduplicate_boundaries(boundaries, text) return self._build_sections(boundaries, text, FORM_10K_SECTIONS, "10-K") def _extract_10q(self, text: str) -> list[dict]: part_matches = list(_10Q_PART_PATTERN.finditer(text)) item_matches = list(_10Q_ITEM_PATTERN.finditer(text)) if not item_matches: logger.info("No Item markers found in 10-Q filing") return [] part_ranges: dict[int, tuple[int, int]] = {} for i, pm in enumerate(part_matches): part_num = _roman_to_int(pm.group(1)) start = pm.start() end = part_matches[i + 1].start() if i + 1 < len(part_matches) else len(text) part_ranges[part_num] = (start, end) boundaries: list[tuple[str, int]] = [] for im in item_matches: item_num = im.group(1).upper() part_num = self._determine_part(im.start(), part_ranges) section_key = f"P{part_num}_{item_num}" if section_key in FORM_10Q_SECTIONS: boundaries.append((section_key, im.start())) boundaries = self._deduplicate_boundaries(boundaries, text) return self._build_sections(boundaries, text, FORM_10Q_SECTIONS, "10-Q") def _extract_8k(self, text: str) -> list[dict]: if not text.strip(): return [] return [ { "section_id": str(uuid.uuid4()), "section_name": "full_body", "section_order": 0, "item_number": "N/A", "text": text.strip(), "word_count": len(text.split()), "char_count": len(text), } ] def _determine_part( self, position: int, part_ranges: dict[int, tuple[int, int]] ) -> int: for part_num, (start, end) in part_ranges.items(): if start <= position < end: return part_num if part_ranges: return min(part_ranges.keys()) return 1 def _deduplicate_boundaries( self, boundaries: list[tuple[str, int]], text: str ) -> list[tuple[str, int]]: if not boundaries: return boundaries seen: dict[str, list[tuple[str, int]]] = {} for key, pos in boundaries: seen.setdefault(key, []).append((key, pos)) result: list[tuple[str, int]] = [] for key, entries in seen.items(): if len(entries) == 1: result.append(entries[0]) continue # When duplicate markers exist, pick the one followed by the most # substantive text (heuristic: the last occurrence is typically the # real section start in SEC filings with a table-of-contents up top). result.append(entries[-1]) result.sort(key=lambda x: x[1]) return result def _build_sections( self, boundaries: list[tuple[str, int]], text: str, section_map: dict[str, str], form_type: str, ) -> list[dict]: sections: list[dict] = [] for i, (item_key, start) in enumerate(boundaries): end = boundaries[i + 1][1] if i + 1 < len(boundaries) else len(text) section_text = text[start:end].strip() header_match = re.match( r"(?:ITEM|PART)\s+\S+\s*[.\-:—–]?\s*\S*\s*\n?", section_text, re.IGNORECASE, ) if header_match: section_text = section_text[header_match.end():].strip() if len(section_text.split()) < 20: logger.debug( "Skipping near-empty section %s in %s", item_key, form_type ) continue sections.append( { "section_id": str(uuid.uuid4()), "section_name": section_map.get(item_key, item_key), "section_order": len(sections), "item_number": item_key, "text": section_text, "word_count": len(section_text.split()), "char_count": len(section_text), } ) return sections