"""Runtime settings (environment variables, `CA_` prefix). Never log `settings.model_dump()` — it contains secrets. Every tunable that the product spec calls a "magic number" (crawl intervals, thresholds, weights, retry counts) lives here or in `companyatlas.taxonomy` as typed configuration — never scattered as literals through the pipeline. """ from __future__ import annotations from functools import lru_cache from pathlib import Path from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=(".env",), env_file_encoding="utf-8", extra="ignore") app_env: str = Field("development", alias="APP_ENV") site_url: str = Field("https://www.company-atlas.co", alias="CA_SITE_URL") canonical_hosts: str = Field("www.company-atlas.co,www.company-atlas.com", alias="CA_CANONICAL_HOSTS") database_url: str = Field("postgresql+asyncpg://companyatlas:companyatlas@127.0.0.1:5432/companyatlas", alias="DATABASE_URL") db_pool_size: int = Field(8, alias="CA_DB_POOL_SIZE") db_max_overflow: int = Field(8, alias="CA_DB_MAX_OVERFLOW") data_dir: Path = Field(Path("./data"), alias="CA_DATA_DIR") api_host: str = Field("127.0.0.1", alias="CA_API_HOST") api_port: int = Field(8371, alias="CA_API_PORT") admin_token: str = Field("", alias="CA_ADMIN_TOKEN") log_json: bool = Field(True, alias="CA_LOG_JSON") tz: str = Field("America/Toronto", alias="CA_TZ") # ---------------------------------------------------------------- crawler / politeness user_agent: str = Field("CompanyAtlasBot/0.1 (+https://www.company-atlas.co/bot; contact@spboucher.ai)", alias="CA_USER_AGENT") http_timeout_s: float = Field(30.0, alias="CA_HTTP_TIMEOUT") max_body_bytes: int = Field(8 * 1024 * 1024, alias="CA_MAX_BODY_BYTES") default_rate_per_min: int = Field(20, alias="CA_DEFAULT_RATE_PER_MIN") domain_max_concurrency: int = Field(2, alias="CA_DOMAIN_MAX_CONCURRENCY") domain_daily_budget: int = Field(600, alias="CA_DOMAIN_DAILY_BUDGET") respect_robots: bool = Field(True, alias="CA_RESPECT_ROBOTS") fetch_concurrency: int = Field(16, alias="CA_FETCH_CONCURRENCY") browser_enabled: bool = Field(False, alias="CA_BROWSER_ENABLED") browser_concurrency: int = Field(2, alias="CA_BROWSER_CONCURRENCY") max_redirects: int = Field(5, alias="CA_MAX_REDIRECTS") fetch_retries: int = Field(2, alias="CA_FETCH_RETRIES") # ---------------------------------------------------------------- discovery discovery_max_pages: int = Field(12, alias="CA_DISCOVERY_MAX_PAGES") discovery_max_sitemap_urls: int = Field(5000, alias="CA_DISCOVERY_MAX_SITEMAP_URLS") discovery_min_confidence: float = Field(0.55, alias="CA_DISCOVERY_MIN_CONFIDENCE") discovery_max_sensors_per_company: int = Field(40, alias="CA_DISCOVERY_MAX_SENSORS") onboarding_concurrency: int = Field(12, alias="CA_ONBOARDING_CONCURRENCY") discovery_rate_per_min: int = Field(40, alias="CA_DISCOVERY_RATE_PER_MIN") # same-domain spacing during discovery (1.5 s) # ---------------------------------------------------------------- scheduling (seconds) scheduler_tick_s: int = Field(15, alias="CA_SCHEDULER_TICK_S") scheduler_claim_batch: int = Field(200, alias="CA_SCHEDULER_CLAIM_BATCH") min_interval_s: int = Field(15 * 60, alias="CA_MIN_INTERVAL_S") max_interval_s: int = Field(7 * 86400, alias="CA_MAX_INTERVAL_S") burst_interval_s: int = Field(15 * 60, alias="CA_BURST_INTERVAL_S") burst_decay: float = Field(1.6, alias="CA_BURST_DECAY") stability_growth: float = Field(1.25, alias="CA_STABILITY_GROWTH") failure_growth: float = Field(2.0, alias="CA_FAILURE_GROWTH") stale_after_failures: int = Field(6, alias="CA_STALE_AFTER_FAILURES") retire_after_failures: int = Field(30, alias="CA_RETIRE_AFTER_FAILURES") # ---------------------------------------------------------------- change detection noise_threshold: float = Field(0.20, alias="CA_NOISE_THRESHOLD") meaningful_threshold: float = Field(0.40, alias="CA_MEANINGFUL_THRESHOLD") major_threshold: float = Field(0.65, alias="CA_MAJOR_THRESHOLD") critical_threshold: float = Field(0.85, alias="CA_CRITICAL_THRESHOLD") keep_noise_snapshots: bool = Field(False, alias="CA_KEEP_NOISE_SNAPSHOTS") # ---------------------------------------------------------------- LLM enrichment (optional) llm_base_url: str = Field("", alias="CA_LLM_BASE_URL") llm_api_key: str = Field("", alias="CA_LLM_API_KEY") llm_small_model: str = Field("qwen3-4b-instruct-2507-4bit", alias="CA_LLM_SMALL_MODEL") llm_medium_model: str = Field("qwen3.6-35b-a3b-4bit", alias="CA_LLM_MEDIUM_MODEL") llm_large_model: str = Field("qwen3.8-27b-4bit", alias="CA_LLM_LARGE_MODEL") llm_timeout_s: float = Field(300.0, alias="CA_LLM_TIMEOUT") llm_enabled: bool = Field(True, alias="CA_LLM_ENABLED") llm_daily_budget: int = Field(1500, alias="CA_LLM_DAILY_BUDGET") llm_min_significance: float = Field(0.40, alias="CA_LLM_MIN_SIGNIFICANCE") llm_embedding_model: str = Field("qwen3-embedding-0.6b-8bit", alias="CA_LLM_EMBEDDING_MODEL") llm_max_tries: int = Field(6, alias="CA_LLM_MAX_TRIES") # 429/503 while the server swaps models llm_backoff_initial_s: float = Field(15.0, alias="CA_LLM_BACKOFF_INITIAL_S") llm_backoff_max_s: float = Field(120.0, alias="CA_LLM_BACKOFF_MAX_S") llm_job_max_attempts: int = Field(3, alias="CA_LLM_JOB_MAX_ATTEMPTS") llm_context_block_bytes: int = Field(3072, alias="CA_LLM_CONTEXT_BLOCK_BYTES") # before/after blocks sent to the model (≤ 3 kB each) prompts_dir: Path | None = Field(None, alias="CA_PROMPTS_DIR") # defaults to /prompts worker_concurrency: int = Field(1, alias="CA_WORKER_CONCURRENCY") # ---------------------------------------------------------------- company profile enrichment (services/enrichment.py) enrich_batch: int = Field(300, alias="CA_ENRICH_BATCH") # companies per periodic pass (every 10 min) enrich_concurrency: int = Field(4, alias="CA_ENRICH_CONCURRENCY") enrich_refresh_days: int = Field(30, alias="CA_ENRICH_REFRESH_DAYS") # re-enrich profiles older than this enrich_wikidata_rate_per_min: int = Field(300, alias="CA_ENRICH_WIKIDATA_RATE_PER_MIN") # ≤ 5 req/s (Wikimedia API etiquette) enrich_wikipedia_rate_per_min: int = Field(600, alias="CA_ENRICH_WIKIPEDIA_RATE_PER_MIN") # ≤ 10 req/s enrich_wikidata_entity_batch: int = Field(10, alias="CA_ENRICH_WIKIDATA_ENTITY_BATCH") # full entities per wbgetentities call enrich_wikidata_label_batch: int = Field(50, alias="CA_ENRICH_WIKIDATA_LABEL_BATCH") # API maximum for labels-only lookups enrich_wikidata_max_bytes: int = Field(48 * 1024 * 1024, alias="CA_ENRICH_WIKIDATA_MAX_BYTES") enrich_max_relationships_per_property: int = Field(40, alias="CA_ENRICH_MAX_RELATIONSHIPS") enrich_max_products: int = Field(12, alias="CA_ENRICH_MAX_PRODUCTS") enrich_description_max_chars: int = Field(1500, alias="CA_ENRICH_DESCRIPTION_MAX_CHARS") enrich_llm_min_text_chars: int = Field(400, alias="CA_ENRICH_LLM_MIN_TEXT_CHARS") enrich_llm_max_text_chars: int = Field(6000, alias="CA_ENRICH_LLM_MAX_TEXT_CHARS") enrich_http_timeout_s: float = Field(15.0, alias="CA_ENRICH_HTTP_TIMEOUT_S") # live homepage fetch when no snapshot exists (no retries) enrich_llm_timeout_s: float = Field(120.0, alias="CA_ENRICH_LLM_TIMEOUT_S") # per profile call (the gateway retries inside) enrich_llm_cooldown_s: float = Field(1800.0, alias="CA_ENRICH_LLM_COOLDOWN_S") # circuit breaker after a transport/timeout failure # ---------------------------------------------------------------- metrics / retention metrics_cron: str = Field("7 * * * *", alias="CA_METRICS_CRON") daily_cron: str = Field("20 0 * * *", alias="CA_DAILY_CRON") backup_cron: str = Field("35 4 * * *", alias="CA_BACKUP_CRON") baseline_window_days: int = Field(56, alias="CA_BASELINE_WINDOW_DAYS") anomaly_z: float = Field(2.5, alias="CA_ANOMALY_Z") metrics_active_window_days: int = Field(90, alias="CA_METRICS_ACTIVE_WINDOW_DAYS") # hourly pass = companies active within trends_min_companies: int = Field(3, alias="CA_TRENDS_MIN_COMPANIES") signals_window_days: int = Field(30, alias="CA_SIGNALS_WINDOW_DAYS") signals_ttl_days: int = Field(14, alias="CA_SIGNALS_TTL_DAYS") alerts_sweep_window_s: int = Field(1800, alias="CA_ALERTS_SWEEP_WINDOW_S") webhook_timeout_s: float = Field(10.0, alias="CA_WEBHOOK_TIMEOUT_S") retention_observations_days: int = Field(90, alias="CA_RETENTION_OBSERVATIONS_DAYS") retention_crawl_runs_days: int = Field(30, alias="CA_RETENTION_CRAWL_RUNS_DAYS") retention_llm_jobs_days: int = Field(60, alias="CA_RETENTION_LLM_JOBS_DAYS") seo_min_events: int = Field(1, alias="CA_SEO_MIN_EVENTS") seo_min_sensors: int = Field(3, alias="CA_SEO_MIN_SENSORS") @property def objects_dir(self) -> Path: return self.data_dir / "objects" @property def logs_dir(self) -> Path: return self.data_dir / "logs" @property def backups_dir(self) -> Path: return self.data_dir / "backups" @property def cache_dir(self) -> Path: return self.data_dir / "cache" @property def sync_database_url(self) -> str: return self.database_url.replace("+asyncpg", "") @property def llm_configured(self) -> bool: return bool(self.llm_enabled and self.llm_base_url) def ensure_dirs(self) -> None: for d in (self.objects_dir, self.logs_dir, self.backups_dir, self.cache_dir): d.mkdir(parents=True, exist_ok=True) @lru_cache def get_settings() -> Settings: return Settings() settings = get_settings() __all__ = ["Settings", "get_settings", "settings"]