spb/company-atlas

1"""Runtime settings (environment variables, `CA_` prefix). Never log `settings.model_dump()` — it contains secrets.23Every tunable that the product spec calls a "magic number" (crawl intervals, thresholds, weights, retry counts) lives here or in4`companyatlas.taxonomy` as typed configuration — never scattered as literals through the pipeline.5"""6from __future__ import annotations78from functools import lru_cache9from pathlib import Path1011from pydantic import Field12from pydantic_settings import BaseSettings, SettingsConfigDict131415class Settings(BaseSettings):16    model_config = SettingsConfigDict(env_file=(".env",), env_file_encoding="utf-8", extra="ignore")1718    app_env: str = Field("development", alias="APP_ENV")19    site_url: str = Field("https://www.company-atlas.co", alias="CA_SITE_URL")20    canonical_hosts: str = Field("www.company-atlas.co,www.company-atlas.com", alias="CA_CANONICAL_HOSTS")2122    database_url: str = Field("postgresql+asyncpg://companyatlas:companyatlas@127.0.0.1:5432/companyatlas", alias="DATABASE_URL")23    db_pool_size: int = Field(8, alias="CA_DB_POOL_SIZE")24    db_max_overflow: int = Field(8, alias="CA_DB_MAX_OVERFLOW")2526    data_dir: Path = Field(Path("./data"), alias="CA_DATA_DIR")27    api_host: str = Field("127.0.0.1", alias="CA_API_HOST")28    api_port: int = Field(8371, alias="CA_API_PORT")29    admin_token: str = Field("", alias="CA_ADMIN_TOKEN")30    log_json: bool = Field(True, alias="CA_LOG_JSON")31    tz: str = Field("America/Toronto", alias="CA_TZ")3233    # ---------------------------------------------------------------- crawler / politeness34    user_agent: str = Field("CompanyAtlasBot/0.1 (+https://www.company-atlas.co/bot; contact@spboucher.ai)", alias="CA_USER_AGENT")35    http_timeout_s: float = Field(30.0, alias="CA_HTTP_TIMEOUT")36    max_body_bytes: int = Field(8 * 1024 * 1024, alias="CA_MAX_BODY_BYTES")37    default_rate_per_min: int = Field(20, alias="CA_DEFAULT_RATE_PER_MIN")38    domain_max_concurrency: int = Field(2, alias="CA_DOMAIN_MAX_CONCURRENCY")39    domain_daily_budget: int = Field(600, alias="CA_DOMAIN_DAILY_BUDGET")40    respect_robots: bool = Field(True, alias="CA_RESPECT_ROBOTS")41    fetch_concurrency: int = Field(16, alias="CA_FETCH_CONCURRENCY")42    browser_enabled: bool = Field(False, alias="CA_BROWSER_ENABLED")43    browser_concurrency: int = Field(2, alias="CA_BROWSER_CONCURRENCY")44    max_redirects: int = Field(5, alias="CA_MAX_REDIRECTS")45    fetch_retries: int = Field(2, alias="CA_FETCH_RETRIES")4647    # ---------------------------------------------------------------- discovery48    discovery_max_pages: int = Field(12, alias="CA_DISCOVERY_MAX_PAGES")49    discovery_max_sitemap_urls: int = Field(5000, alias="CA_DISCOVERY_MAX_SITEMAP_URLS")50    discovery_min_confidence: float = Field(0.55, alias="CA_DISCOVERY_MIN_CONFIDENCE")51    discovery_max_sensors_per_company: int = Field(40, alias="CA_DISCOVERY_MAX_SENSORS")52    onboarding_concurrency: int = Field(12, alias="CA_ONBOARDING_CONCURRENCY")53    discovery_rate_per_min: int = Field(40, alias="CA_DISCOVERY_RATE_PER_MIN")   # same-domain spacing during discovery (1.5 s)5455    # ---------------------------------------------------------------- scheduling (seconds)56    scheduler_tick_s: int = Field(15, alias="CA_SCHEDULER_TICK_S")57    scheduler_claim_batch: int = Field(200, alias="CA_SCHEDULER_CLAIM_BATCH")58    min_interval_s: int = Field(15 * 60, alias="CA_MIN_INTERVAL_S")59    max_interval_s: int = Field(7 * 86400, alias="CA_MAX_INTERVAL_S")60    burst_interval_s: int = Field(15 * 60, alias="CA_BURST_INTERVAL_S")61    burst_decay: float = Field(1.6, alias="CA_BURST_DECAY")62    stability_growth: float = Field(1.25, alias="CA_STABILITY_GROWTH")63    failure_growth: float = Field(2.0, alias="CA_FAILURE_GROWTH")64    stale_after_failures: int = Field(6, alias="CA_STALE_AFTER_FAILURES")65    retire_after_failures: int = Field(30, alias="CA_RETIRE_AFTER_FAILURES")6667    # ---------------------------------------------------------------- change detection68    noise_threshold: float = Field(0.20, alias="CA_NOISE_THRESHOLD")69    meaningful_threshold: float = Field(0.40, alias="CA_MEANINGFUL_THRESHOLD")70    major_threshold: float = Field(0.65, alias="CA_MAJOR_THRESHOLD")71    critical_threshold: float = Field(0.85, alias="CA_CRITICAL_THRESHOLD")72    keep_noise_snapshots: bool = Field(False, alias="CA_KEEP_NOISE_SNAPSHOTS")7374    # ---------------------------------------------------------------- LLM enrichment (optional)75    llm_base_url: str = Field("", alias="CA_LLM_BASE_URL")76    llm_api_key: str = Field("", alias="CA_LLM_API_KEY")77    llm_small_model: str = Field("qwen3-4b-instruct-2507-4bit", alias="CA_LLM_SMALL_MODEL")78    llm_medium_model: str = Field("qwen3.6-35b-a3b-4bit", alias="CA_LLM_MEDIUM_MODEL")79    llm_large_model: str = Field("qwen3.8-27b-4bit", alias="CA_LLM_LARGE_MODEL")80    llm_timeout_s: float = Field(300.0, alias="CA_LLM_TIMEOUT")81    llm_enabled: bool = Field(True, alias="CA_LLM_ENABLED")82    llm_daily_budget: int = Field(1500, alias="CA_LLM_DAILY_BUDGET")83    llm_min_significance: float = Field(0.40, alias="CA_LLM_MIN_SIGNIFICANCE")84    llm_embedding_model: str = Field("qwen3-embedding-0.6b-8bit", alias="CA_LLM_EMBEDDING_MODEL")85    llm_max_tries: int = Field(6, alias="CA_LLM_MAX_TRIES")                       # 429/503 while the server swaps models86    llm_backoff_initial_s: float = Field(15.0, alias="CA_LLM_BACKOFF_INITIAL_S")87    llm_backoff_max_s: float = Field(120.0, alias="CA_LLM_BACKOFF_MAX_S")88    llm_job_max_attempts: int = Field(3, alias="CA_LLM_JOB_MAX_ATTEMPTS")89    llm_context_block_bytes: int = Field(3072, alias="CA_LLM_CONTEXT_BLOCK_BYTES")   # before/after blocks sent to the model (≤ 3 kB each)90    prompts_dir: Path | None = Field(None, alias="CA_PROMPTS_DIR")              # defaults to <repo>/prompts91    worker_concurrency: int = Field(1, alias="CA_WORKER_CONCURRENCY")9293    # ---------------------------------------------------------------- company profile enrichment (services/enrichment.py)94    enrich_batch: int = Field(300, alias="CA_ENRICH_BATCH")                          # companies per periodic pass (every 10 min)95    enrich_concurrency: int = Field(4, alias="CA_ENRICH_CONCURRENCY")96    enrich_refresh_days: int = Field(30, alias="CA_ENRICH_REFRESH_DAYS")              # re-enrich profiles older than this97    enrich_wikidata_rate_per_min: int = Field(300, alias="CA_ENRICH_WIKIDATA_RATE_PER_MIN")     # ≤ 5 req/s (Wikimedia API etiquette)98    enrich_wikipedia_rate_per_min: int = Field(600, alias="CA_ENRICH_WIKIPEDIA_RATE_PER_MIN")   # ≤ 10 req/s99    enrich_wikidata_entity_batch: int = Field(10, alias="CA_ENRICH_WIKIDATA_ENTITY_BATCH")      # full entities per wbgetentities call100    enrich_wikidata_label_batch: int = Field(50, alias="CA_ENRICH_WIKIDATA_LABEL_BATCH")        # API maximum for labels-only lookups101    enrich_wikidata_max_bytes: int = Field(48 * 1024 * 1024, alias="CA_ENRICH_WIKIDATA_MAX_BYTES")102    enrich_max_relationships_per_property: int = Field(40, alias="CA_ENRICH_MAX_RELATIONSHIPS")103    enrich_max_products: int = Field(12, alias="CA_ENRICH_MAX_PRODUCTS")104    enrich_description_max_chars: int = Field(1500, alias="CA_ENRICH_DESCRIPTION_MAX_CHARS")105    enrich_llm_min_text_chars: int = Field(400, alias="CA_ENRICH_LLM_MIN_TEXT_CHARS")106    enrich_llm_max_text_chars: int = Field(6000, alias="CA_ENRICH_LLM_MAX_TEXT_CHARS")107    enrich_http_timeout_s: float = Field(15.0, alias="CA_ENRICH_HTTP_TIMEOUT_S")       # live homepage fetch when no snapshot exists (no retries)108    enrich_llm_timeout_s: float = Field(120.0, alias="CA_ENRICH_LLM_TIMEOUT_S")          # per profile call (the gateway retries inside)109    enrich_llm_cooldown_s: float = Field(1800.0, alias="CA_ENRICH_LLM_COOLDOWN_S")       # circuit breaker after a transport/timeout failure110111    # ---------------------------------------------------------------- metrics / retention112    metrics_cron: str = Field("7 * * * *", alias="CA_METRICS_CRON")113    daily_cron: str = Field("20 0 * * *", alias="CA_DAILY_CRON")114    backup_cron: str = Field("35 4 * * *", alias="CA_BACKUP_CRON")115    baseline_window_days: int = Field(56, alias="CA_BASELINE_WINDOW_DAYS")116    anomaly_z: float = Field(2.5, alias="CA_ANOMALY_Z")117    metrics_active_window_days: int = Field(90, alias="CA_METRICS_ACTIVE_WINDOW_DAYS")   # hourly pass = companies active within118    trends_min_companies: int = Field(3, alias="CA_TRENDS_MIN_COMPANIES")119    signals_window_days: int = Field(30, alias="CA_SIGNALS_WINDOW_DAYS")120    signals_ttl_days: int = Field(14, alias="CA_SIGNALS_TTL_DAYS")121    alerts_sweep_window_s: int = Field(1800, alias="CA_ALERTS_SWEEP_WINDOW_S")122    webhook_timeout_s: float = Field(10.0, alias="CA_WEBHOOK_TIMEOUT_S")123    retention_observations_days: int = Field(90, alias="CA_RETENTION_OBSERVATIONS_DAYS")124    retention_crawl_runs_days: int = Field(30, alias="CA_RETENTION_CRAWL_RUNS_DAYS")125    retention_llm_jobs_days: int = Field(60, alias="CA_RETENTION_LLM_JOBS_DAYS")126    seo_min_events: int = Field(1, alias="CA_SEO_MIN_EVENTS")127    seo_min_sensors: int = Field(3, alias="CA_SEO_MIN_SENSORS")128129    @property130    def objects_dir(self) -> Path:131        return self.data_dir / "objects"132133    @property134    def logs_dir(self) -> Path:135        return self.data_dir / "logs"136137    @property138    def backups_dir(self) -> Path:139        return self.data_dir / "backups"140141    @property142    def cache_dir(self) -> Path:143        return self.data_dir / "cache"144145    @property146    def sync_database_url(self) -> str:147        return self.database_url.replace("+asyncpg", "")148149    @property150    def llm_configured(self) -> bool:151        return bool(self.llm_enabled and self.llm_base_url)152153    def ensure_dirs(self) -> None:154        for d in (self.objects_dir, self.logs_dir, self.backups_dir, self.cache_dir):155            d.mkdir(parents=True, exist_ok=True)156157158@lru_cache159def get_settings() -> Settings:160    return Settings()161162163settings = get_settings()164165__all__ = ["Settings", "get_settings", "settings"]166