# ============================================================ # Project: EDGAR Research Database # Description: SEC EDGAR Research Infrastructure # # Authors: # - Manel Kammoun — manel.kammoun@uqo.ca # - Djerrycharli Tandjambianda — djerrycharli.tandjambianda@uqo.ca # - Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca # # Institution: # Université du Québec en Outaouais (UQO) # # License: # Research / Academic Use # ============================================================ from __future__ import annotations import os import time import httpx from src.utils.constants import SEC_ARCHIVES_URL, SEC_SUBMISSIONS_URL from src.utils.logging import get_logger logger = get_logger("acquisition.sec_client") MAX_RETRIES = 5 RETRY_BACKOFF_BASE = 2.0 MIN_REQUEST_INTERVAL = 0.1 class SECClient: def __init__(self) -> None: self._user_agent = os.environ.get( "SEC_USER_AGENT", "Research Agent research@example.com" ) self._last_request_time: float = 0.0 self._client = httpx.Client( timeout=30, headers={ "User-Agent": self._user_agent, "Accept": "application/json", }, ) def close(self) -> None: self._client.close() def __enter__(self) -> SECClient: return self def __exit__(self, *exc) -> None: self.close() def get_company_filings( self, cik: str, form_type: str = "10-K", start_date: str = "2010-01-01", ) -> list[dict]: padded_cik = cik.zfill(10) url = f"{SEC_SUBMISSIONS_URL}/CIK{padded_cik}.json" data = self._request_with_retry(url) if data is None: return [] filings = self._extract_filings(data, padded_cik, form_type, start_date) older_files = data.get("filings", {}).get("files", []) for file_entry in older_files: file_name = file_entry.get("name", "") if not file_name: continue older_url = f"{SEC_SUBMISSIONS_URL}/{file_name}" older_data = self._request_with_retry(older_url) if older_data is not None: filings.extend( self._extract_filings_from_block( older_data, padded_cik, form_type, start_date ) ) logger.info( "Found %d %s filings for CIK %s since %s", len(filings), form_type, padded_cik, start_date, ) return filings def _extract_filings( self, data: dict, cik: str, form_type: str, start_date: str, ) -> list[dict]: recent = data.get("filings", {}).get("recent", {}) return self._extract_filings_from_block(recent, cik, form_type, start_date) def _extract_filings_from_block( self, block: dict, cik: str, form_type: str, start_date: str, ) -> list[dict]: forms = block.get("form", []) accessions = block.get("accessionNumber", []) filing_dates = block.get("filingDate", []) report_dates = block.get("reportDate", []) primary_docs = block.get("primaryDocument", []) results: list[dict] = [] for i in range(len(forms)): if forms[i] != form_type: continue filing_date = filing_dates[i] if i < len(filing_dates) else "" if filing_date < start_date: continue accession = accessions[i] if i < len(accessions) else "" accession_clean = accession.replace("-", "") primary_doc = primary_docs[i] if i < len(primary_docs) else "" report_date = report_dates[i] if i < len(report_dates) else "" filing_url = f"{SEC_ARCHIVES_URL}/{cik.lstrip('0')}/{accession_clean}/{primary_doc}" results.append({ "accession_number": accession, "form_type": forms[i], "filing_date": filing_date, "report_date": report_date, "primary_document": primary_doc, "filing_url": filing_url, }) return results def _request_with_retry(self, url: str) -> dict | None: for attempt in range(MAX_RETRIES): self._rate_limit() try: response = self._client.get(url) if response.status_code == 404: logger.warning("Resource not found: %s", url) return None response.raise_for_status() return response.json() except httpx.HTTPStatusError as exc: wait = RETRY_BACKOFF_BASE ** attempt logger.warning( "HTTP %d on %s (attempt %d/%d), retrying in %.1fs", exc.response.status_code, url, attempt + 1, MAX_RETRIES, wait, ) time.sleep(wait) except httpx.RequestError as exc: wait = RETRY_BACKOFF_BASE ** attempt logger.warning( "Request error on %s: %s (attempt %d/%d), retrying in %.1fs", url, exc, attempt + 1, MAX_RETRIES, wait, ) time.sleep(wait) logger.error("All %d retries exhausted for %s", MAX_RETRIES, url) return None def _rate_limit(self) -> None: elapsed = time.monotonic() - self._last_request_time if elapsed < MIN_REQUEST_INTERVAL: time.sleep(MIN_REQUEST_INTERVAL - elapsed) self._last_request_time = time.monotonic()