spb/uqo-research-db

1# ============================================================2# Project: EDGAR Research Database3# Description: SEC EDGAR Research Infrastructure4#5# Authors:6# - Manel Kammoun — manel.kammoun@uqo.ca7# - Djerrycharli Tandjambianda — djerrycharli.tandjambianda@uqo.ca8# - Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca9#10# Institution:11# Université du Québec en Outaouais (UQO)12#13# License:14# Research / Academic Use15# ============================================================1617from __future__ import annotations1819import re20from pathlib import Path2122import chardet23from bs4 import BeautifulSoup, Tag2425from src.utils.config import get_parsing_config26from src.utils.logging import get_logger2728logger = get_logger("parsing.html")2930SGML_HEADER_PATTERN = re.compile(31    r"^.*?<(?:DOCUMENT|document)>.*?<(?:TEXT|text)>",32    re.DOTALL,33)3435SGML_HEADER_TAG_PATTERN = re.compile(36    r"^.*?(?=<(?:html|HTML|body|BODY|div|DIV|p|P|table|TABLE)\b)",37    re.DOTALL,38)394041class HTMLCleaner:42    def __init__(self, config: dict | None = None) -> None:43        cfg = config or get_parsing_config()44        html_cfg = cfg.get("html", {})45        self._parser: str = html_cfg.get("parser", "lxml")46        self._remove_tags: list[str] = html_cfg.get(47            "remove_tags", ["script", "style", "meta", "link"]48        )49        self._preserve_tables: bool = html_cfg.get("preserve_tables", True)50        self._preserve_lists: bool = html_cfg.get("preserve_lists", True)51        self._normalize_ws: bool = html_cfg.get("normalize_whitespace", True)52        self._max_file_size: int = html_cfg.get("max_file_size_mb", 50) * 1024 * 10245354    def clean(self, html_content: str) -> str:55        text = self._strip_sgml_header(html_content)56        soup = BeautifulSoup(text, self._parser)5758        for tag_name in self._remove_tags:59            for tag in soup.find_all(tag_name):60                tag.decompose()6162        if self._preserve_tables:63            self._convert_tables(soup)6465        if self._preserve_lists:66            self._convert_lists(soup)6768        raw_text = soup.get_text(separator="\n")6970        if self._normalize_ws:71            raw_text = self._normalize_whitespace(raw_text)7273        return raw_text.strip()7475    def clean_file(self, file_path: Path) -> str:76        file_path = Path(file_path)77        if not file_path.exists():78            raise FileNotFoundError(f"File not found: {file_path}")7980        file_size = file_path.stat().st_size81        if file_size > self._max_file_size:82            logger.warning(83                "Skipping file exceeding size limit: %s (%d MB)",84                file_path,85                file_size // (1024 * 1024),86            )87            return ""8889        raw_bytes = file_path.read_bytes()90        encoding = self._detect_encoding(raw_bytes)91        html_content = raw_bytes.decode(encoding, errors="replace")92        return self.clean(html_content)9394    def _detect_encoding(self, raw_bytes: bytes) -> str:95        for enc in ("utf-8", "latin-1", "windows-1252"):96            try:97                raw_bytes.decode(enc)98                return enc99            except (UnicodeDecodeError, LookupError):100                continue101        result = chardet.detect(raw_bytes)102        return result.get("encoding") or "utf-8"103104    def _strip_sgml_header(self, text: str) -> str:105        match = SGML_HEADER_PATTERN.search(text)106        if match:107            text = text[match.end():]108109        match = SGML_HEADER_TAG_PATTERN.search(text)110        if match and match.end() > 0:111            text = text[match.end():]112113        return text114115    def _convert_tables(self, soup: BeautifulSoup) -> None:116        for table in soup.find_all("table"):117            rows: list[str] = []118            for tr in table.find_all("tr"):119                cells = []120                for cell in tr.find_all(["td", "th"]):121                    cell_text = cell.get_text(strip=True)122                    cells.append(cell_text)123                if any(cells):124                    rows.append("\t".join(cells))125            table_text = "\n".join(rows)126            table.replace_with(BeautifulSoup(table_text, self._parser))127128    def _convert_lists(self, soup: BeautifulSoup) -> None:129        for li in soup.find_all("li"):130            li_text = li.get_text(strip=True)131            li.replace_with(f"- {li_text}\n")132133    def _normalize_whitespace(self, text: str) -> str:134        text = re.sub(r"[^\S\n]+", " ", text)135        text = re.sub(r" *\n *", "\n", text)136        text = re.sub(r"\n{3,}", "\n\n", text)137        lines = text.split("\n")138        cleaned = [line.strip() for line in lines]139        return "\n".join(cleaned)140