spb/uqo-research-db
Public
1# ============================================================2# Project: EDGAR Research Database3# Description: SEC EDGAR Research Infrastructure4#5# Authors:6# - Manel Kammoun — manel.kammoun@uqo.ca7# - Djerrycharli Tandjambianda — djerrycharli.tandjambianda@uqo.ca8# - Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca9#10# Institution:11# Université du Québec en Outaouais (UQO)12#13# License:14# Research / Academic Use15# ============================================================1617from __future__ import annotations1819import io20import json21from datetime import datetime, timezone22from pathlib import Path2324import httpx25import pandas as pd2627from src.utils.constants import DATA_DIRS28from src.utils.logging import get_logger2930logger = get_logger("acquisition.sp500")3132WIKIPEDIA_SP500_URL = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"3334FALLBACK_TICKERS = [35 {"ticker": "AAPL", "company_name": "Apple Inc.", "cik": "", "sector": "Information Technology", "sub_industry": "Technology Hardware", "date_added": ""},36 {"ticker": "MSFT", "company_name": "Microsoft Corporation", "cik": "", "sector": "Information Technology", "sub_industry": "Systems Software", "date_added": ""},37 {"ticker": "GOOGL", "company_name": "Alphabet Inc.", "cik": "", "sector": "Communication Services", "sub_industry": "Interactive Media", "date_added": ""},38 {"ticker": "AMZN", "company_name": "Amazon.com Inc.", "cik": "", "sector": "Consumer Discretionary", "sub_industry": "Internet Retail", "date_added": ""},39 {"ticker": "META", "company_name": "Meta Platforms Inc.", "cik": "", "sector": "Communication Services", "sub_industry": "Interactive Media", "date_added": ""},40 {"ticker": "NVDA", "company_name": "NVIDIA Corporation", "cik": "", "sector": "Information Technology", "sub_industry": "Semiconductors", "date_added": ""},41 {"ticker": "TSLA", "company_name": "Tesla Inc.", "cik": "", "sector": "Consumer Discretionary", "sub_industry": "Automobile Manufacturers", "date_added": ""},42 {"ticker": "JPM", "company_name": "JPMorgan Chase & Co.", "cik": "", "sector": "Financials", "sub_industry": "Diversified Banks", "date_added": ""},43 {"ticker": "JNJ", "company_name": "Johnson & Johnson", "cik": "", "sector": "Health Care", "sub_industry": "Pharmaceuticals", "date_added": ""},44 {"ticker": "V", "company_name": "Visa Inc.", "cik": "", "sector": "Financials", "sub_industry": "Transaction Processing", "date_added": ""},45 {"ticker": "PG", "company_name": "Procter & Gamble Co.", "cik": "", "sector": "Consumer Staples", "sub_industry": "Personal Products", "date_added": ""},46 {"ticker": "UNH", "company_name": "UnitedHealth Group Inc.", "cik": "", "sector": "Health Care", "sub_industry": "Managed Health Care", "date_added": ""},47 {"ticker": "HD", "company_name": "The Home Depot Inc.", "cik": "", "sector": "Consumer Discretionary", "sub_industry": "Home Improvement Retail", "date_added": ""},48 {"ticker": "MA", "company_name": "Mastercard Inc.", "cik": "", "sector": "Financials", "sub_industry": "Transaction Processing", "date_added": ""},49 {"ticker": "DIS", "company_name": "The Walt Disney Company", "cik": "", "sector": "Communication Services", "sub_industry": "Movies & Entertainment", "date_added": ""},50 {"ticker": "BAC", "company_name": "Bank of America Corp.", "cik": "", "sector": "Financials", "sub_industry": "Diversified Banks", "date_added": ""},51 {"ticker": "XOM", "company_name": "Exxon Mobil Corporation", "cik": "", "sector": "Energy", "sub_industry": "Integrated Oil & Gas", "date_added": ""},52 {"ticker": "PFE", "company_name": "Pfizer Inc.", "cik": "", "sector": "Health Care", "sub_industry": "Pharmaceuticals", "date_added": ""},53 {"ticker": "CSCO", "company_name": "Cisco Systems Inc.", "cik": "", "sector": "Information Technology", "sub_industry": "Communications Equipment", "date_added": ""},54 {"ticker": "VZ", "company_name": "Verizon Communications Inc.", "cik": "", "sector": "Communication Services", "sub_industry": "Integrated Telecom", "date_added": ""},55]5657COLUMN_MAP = {58 "Symbol": "ticker",59 "Security": "company_name",60 "CIK": "cik",61 "GICS Sector": "sector",62 "GICS Sub-Industry": "sub_industry",63 "Date added": "date_added",64}656667class SP500Universe:6869 def __init__(self, cache_dir: Path | None = None) -> None:70 self._cache_dir = cache_dir or DATA_DIRS["metadata"]71 self._cache_path = self._cache_dir / "sp500_constituents.json"7273 def get_constituents(self) -> list[dict]:74 try:75 resp = httpx.get(76 WIKIPEDIA_SP500_URL,77 headers={"User-Agent": "Mozilla/5.0 (EDGAR-Research-DB)"},78 follow_redirects=True,79 timeout=30,80 )81 resp.raise_for_status()82 tables = pd.read_html(io.StringIO(resp.text))83 df = tables[0]84 df = df.rename(columns=COLUMN_MAP)8586 keep_cols = list(COLUMN_MAP.values())87 available = [c for c in keep_cols if c in df.columns]88 df = df[available]8990 df["cik"] = df["cik"].astype(str).str.strip()91 df["ticker"] = df["ticker"].astype(str).str.strip()92 df["date_added"] = df["date_added"].astype(str).str.strip()9394 constituents = df.to_dict(orient="records")95 logger.info("Fetched %d S&P 500 constituents from Wikipedia", len(constituents))9697 except Exception:98 logger.warning("Wikipedia scrape failed, using fallback ticker list", exc_info=True)99 constituents = [dict(entry) for entry in FALLBACK_TICKERS]100101 self._save_cache(constituents)102 return constituents103104 def load_cached(self) -> list[dict]:105 if not self._cache_path.exists():106 logger.info("No cached constituents found, fetching fresh data")107 return self.get_constituents()108109 with open(self._cache_path) as f:110 data = json.load(f)111112 logger.info(113 "Loaded %d cached constituents (fetched %s)",114 len(data["constituents"]),115 data.get("fetched_at", "unknown"),116 )117 return data["constituents"]118119 def get_tickers(self) -> list[str]:120 constituents = self.load_cached()121 return [c["ticker"] for c in constituents]122123 def _save_cache(self, constituents: list[dict]) -> None:124 self._cache_dir.mkdir(parents=True, exist_ok=True)125 payload = {126 "fetched_at": datetime.now(timezone.utc).isoformat(),127 "count": len(constituents),128 "constituents": constituents,129 }130 with open(self._cache_path, "w") as f:131 json.dump(payload, f, indent=2, default=str)132 logger.info("Cached %d constituents to %s", len(constituents), self._cache_path)133