spb/uqo-research-db
Public
1# ============================================================2# Project: EDGAR Research Database3# Description: SEC EDGAR Research Infrastructure4#5# Authors:6# - Manel Kammoun — manel.kammoun@uqo.ca7# - Djerrycharli Tandjambianda — djerrycharli.tandjambianda@uqo.ca8# - Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca9#10# Institution:11# Universite du Quebec en Outaouais (UQO)12#13# License:14# Research / Academic Use15# ============================================================1617from __future__ import annotations1819from typing import TYPE_CHECKING2021import numpy as np22import pyarrow as pa23import pyarrow.parquet as pq2425from src.embeddings.models import EmbeddingModelRegistry26from src.utils.config import get_config27from src.utils.constants import DATA_DIRS, PARQUET_COMPRESSION, PARQUET_COMPRESSION_LEVEL28from src.utils.logging import get_logger2930if TYPE_CHECKING:31 from src.storage.database import DatabaseManager3233logger = get_logger("factors.anchors")3435ANCHORS: dict[str, dict] = {36 "ai_adoption": {37 "label": "AI Internal Adoption",38 "category": "technology",39 "texts": [40 (41 "The company internally uses artificial intelligence, machine learning, "42 "generative AI, large language models, automation, predictive analytics, "43 "intelligent agents, and data-driven systems to improve productivity, "44 "operations, efficiency, decision-making, customer service, risk management, "45 "compliance, cybersecurity, supply chain, software development, finance, "46 "human resources, and other business processes."47 ),48 (49 "We have implemented generative AI, large language models, and natural "50 "language processing capabilities across our enterprise. Our AI strategy "51 "includes deploying intelligent automation, computer vision, and predictive "52 "analytics systems to transform operations."53 ),54 (55 "Investment in artificial intelligence research and development, including "56 "partnerships with AI technology providers, hiring of machine learning "57 "engineers, and deployment of AI infrastructure including GPU computing "58 "clusters for internal business applications."59 ),60 ],61 },62 "ai_disruption_risk": {63 "label": "AI Product Disruption Risk",64 "category": "risk",65 "texts": [66 (67 "Artificial intelligence may reduce the scarcity, defensibility, "68 "differentiation, pricing power, or competitive advantage of the company's "69 "products or services by enabling rapid replication, automation, synthetic "70 "generation, low-cost substitution, or commoditization of capabilities that "71 "were previously difficult, expensive, or specialized."72 ),73 (74 "The rapid advancement of AI technologies may render our current offerings "75 "less competitive. New market entrants leveraging AI could fundamentally "76 "alter industry dynamics, reduce barriers to entry, and erode our market "77 "position through AI-native products and services."78 ),79 (80 "Risks associated with artificial intelligence include potential competitive "81 "displacement by AI-powered alternatives, obsolescence of our existing "82 "products, regulatory uncertainty around AI governance, algorithmic bias, "83 "intellectual property challenges, and emergence of AI-native competitors."84 ),85 ],86 },87 "ai_infrastructure": {88 "label": "AI Infrastructure Exposure",89 "category": "technology",90 "texts": [91 (92 "Investment in AI computing infrastructure including data centers, GPU and "93 "TPU clusters, cloud computing resources, semiconductor chips, accelerator "94 "hardware, and high-performance computing systems dedicated to artificial "95 "intelligence and machine learning workloads."96 ),97 (98 "Capital expenditures for AI infrastructure, including networking equipment "99 "for distributed AI training, specialized computing platforms for large-scale "100 "model development, edge computing for AI inference, and data pipeline "101 "infrastructure for machine learning operations."102 ),103 (104 "Cloud infrastructure spending for AI workloads, including partnerships with "105 "hyperscale cloud providers, deployment of AI-optimized hardware, GPU supply "106 "chain management, and investment in next-generation computing architectures "107 "for artificial intelligence applications."108 ),109 ],110 },111 "esg_commitment": {112 "label": "ESG Exposure",113 "category": "sustainability",114 "texts": [115 (116 "Environmental, social, and governance initiatives including sustainability "117 "reporting, corporate social responsibility programs, diversity equity and "118 "inclusion efforts, stakeholder engagement, and adherence to ESG frameworks "119 "and standards such as SASB, GRI, and TCFD."120 ),121 (122 "The company is committed to ESG principles including reducing carbon "123 "emissions, improving energy efficiency, promoting workforce diversity, "124 "ensuring supply chain responsibility, ethical governance practices, and "125 "transparent reporting of environmental and social performance metrics."126 ),127 (128 "Board-level oversight of environmental and social risk management, executive "129 "compensation tied to sustainability metrics, corporate governance "130 "improvements, and integration of ESG factors into strategic decision-making "131 "and capital allocation processes."132 ),133 ],134 },135 "climate_risk": {136 "label": "Climate Risk",137 "category": "risk",138 "texts": [139 (140 "Climate change poses material risks to our operations including physical "141 "risks from extreme weather events, transition risks from evolving "142 "regulations, carbon pricing mechanisms, and shifting consumer preferences "143 "toward sustainable alternatives and low-carbon products."144 ),145 (146 "Exposure to climate-related financial risks including greenhouse gas "147 "emission regulations, carbon taxes, stranded asset risk, water scarcity, "148 "supply chain disruption from natural disasters, and reputational risk from "149 "insufficient climate action and environmental degradation."150 ),151 (152 "Climate risk assessment and scenario analysis aligned with TCFD "153 "recommendations, including evaluation of physical and transition risks "154 "under different warming scenarios, Scope 1, 2, and 3 greenhouse gas "155 "emissions measurement and reduction targets."156 ),157 ],158 },159 "cybersecurity": {160 "label": "Cybersecurity Risk",161 "category": "risk",162 "texts": [163 (164 "Cybersecurity risks including data breaches, ransomware attacks, "165 "unauthorized access to systems, intellectual property theft, and "166 "operational disruption from cyber incidents. Investment in information "167 "security controls, threat detection, and incident response capabilities."168 ),169 (170 "Data privacy and security compliance including GDPR, CCPA, and other data "171 "protection regulations. Cybersecurity governance, risk management "172 "frameworks, security operations centers, and employee security awareness "173 "training programs to protect critical infrastructure."174 ),175 (176 "Critical infrastructure protection, network security, endpoint security, "177 "identity and access management, zero-trust architecture implementation, "178 "penetration testing, cyber insurance, and board-level oversight of "179 "cybersecurity risk management and AI-powered threat detection."180 ),181 ],182 },183 "automation_labor": {184 "label": "Automation & Labor Replacement",185 "category": "operational",186 "texts": [187 (188 "Automation of business processes, robotic process automation, intelligent "189 "automation, and workforce transformation. Impact of automation on labor "190 "force, reskilling and upskilling programs, and the balance between human "191 "workers and automated systems in manufacturing and services."192 ),193 (194 "Workforce restructuring driven by technological automation, including "195 "reduction in manual processes, deployment of automated manufacturing and "196 "logistics systems, and investment in human-machine collaboration "197 "technologies and digital transformation of work."198 ),199 (200 "Labor market dynamics including talent acquisition challenges, employee "201 "retention, workforce planning in the context of automation, gig economy "202 "participation, remote work infrastructure, and the future of work "203 "transformation driven by AI and robotics."204 ),205 ],206 },207 "macro_risk": {208 "label": "Macroeconomic Risk",209 "category": "risk",210 "texts": [211 (212 "Macroeconomic risks including inflation, interest rate fluctuations, "213 "currency exchange rate volatility, recession risk, geopolitical tensions, "214 "trade policy uncertainty, tariffs, and supply chain disruptions affecting "215 "business performance and revenue."216 ),217 (218 "Economic uncertainty and market volatility, including sensitivity to GDP "219 "growth, consumer spending patterns, credit market conditions, commodity "220 "price fluctuations, and the impact of monetary and fiscal policy changes "221 "on business operations and profitability."222 ),223 (224 "Global economic risks including sovereign debt concerns, emerging market "225 "instability, trade war escalation, sanctions, energy price shocks, pandemic "226 "impacts on economic activity, and systemic financial market risks affecting "227 "corporate investment and capital markets."228 ),229 ],230 },231}232233234class AnchorRegistry:235236 def __init__(237 self,238 registry: EmbeddingModelRegistry | None = None,239 model_name: str | None = None,240 ) -> None:241 self._registry = registry or EmbeddingModelRegistry()242 cfg = get_config("factors").get("factors", {})243 self._model_name = model_name or cfg.get("model_name", "all-MiniLM-L6-v2")244 self._anchor_vectors: dict[str, np.ndarray] | None = None245246 @property247 def anchor_ids(self) -> list[str]:248 return list(ANCHORS.keys())249250 def embed_anchors(self) -> dict[str, np.ndarray]:251 if self._anchor_vectors is not None:252 return self._anchor_vectors253254 self._anchor_vectors = {}255 for anchor_id, info in ANCHORS.items():256 texts = info["texts"]257 vectors = self._registry.encode(258 texts,259 model_name=self._model_name,260 batch_size=len(texts),261 show_progress=False,262 )263 centroid = vectors.mean(axis=0)264 norm = np.linalg.norm(centroid)265 if norm > 0:266 centroid = centroid / norm267 self._anchor_vectors[anchor_id] = centroid.astype(np.float32)268 logger.info(269 "Embedded anchor '%s' (%d texts -> dim %d)",270 anchor_id, len(texts), centroid.shape[0],271 )272273 return self._anchor_vectors274275 def get_anchor_matrix(self) -> tuple[list[str], np.ndarray]:276 vectors = self.embed_anchors()277 ids = list(vectors.keys())278 matrix = np.stack([vectors[aid] for aid in ids])279 return ids, matrix280281 def save_to_duckdb(self, db: DatabaseManager) -> None:282 vectors = self.embed_anchors()283 db.execute("DROP TABLE IF EXISTS anchor_embeddings")284 db.execute("""285 CREATE TABLE anchor_embeddings (286 anchor_id VARCHAR PRIMARY KEY,287 label VARCHAR NOT NULL,288 category VARCHAR NOT NULL,289 model_name VARCHAR,290 dimension INTEGER,291 created_at TIMESTAMP DEFAULT current_timestamp292 )293 """)294 for anchor_id, vec in vectors.items():295 info = ANCHORS[anchor_id]296 db.execute(297 "INSERT INTO anchor_embeddings (anchor_id, label, category, model_name, dimension) "298 "VALUES (?, ?, ?, ?, ?)",299 [anchor_id, info["label"], info["category"], self._model_name, int(vec.shape[0])],300 )301 logger.info("Saved %d anchors to DuckDB", len(vectors))302303 def save_to_parquet(self) -> None:304 vectors = self.embed_anchors()305 records = []306 for anchor_id, vec in vectors.items():307 info = ANCHORS[anchor_id]308 records.append({309 "anchor_id": anchor_id,310 "label": info["label"],311 "category": info["category"],312 "model_name": self._model_name,313 "vector": vec.tolist(),314 })315 table = pa.Table.from_pylist(records)316 out_dir = DATA_DIRS["parquet"] / "anchor_embeddings"317 out_dir.mkdir(parents=True, exist_ok=True)318 pq.write_table(319 table,320 str(out_dir / "anchor_embeddings.parquet"),321 compression=PARQUET_COMPRESSION,322 compression_level=PARQUET_COMPRESSION_LEVEL,323 )324 logger.info("Saved anchor embeddings to parquet")325