spb/uqo-research-db

1# ============================================================2# Project: EDGAR Research Database3# Description: SEC EDGAR Research Infrastructure4#5# Authors:6# - Manel Kammoun — manel.kammoun@uqo.ca7# - Djerrycharli Tandjambianda — djerrycharli.tandjambianda@uqo.ca8# - Simon-Pierre Boucher — simon-pierre.boucher@uqo.ca9#10# Institution:11# Université du Québec en Outaouais (UQO)12#13# License:14# Research / Academic Use15# ============================================================1617"""Download 10-K / 10-Q / 8-K primary documents for a list of companies.1819    python scripts/download_filings.py --universe            # every company in universe.json20    python scripts/download_filings.py --ciks-file data/metadata/shards/M2U64.self.txt --rate 0.921    python scripts/download_filings.py --ciks 0000320193,0000789019 --raw-dir /tmp/raw --log /tmp/log.jsonl2223Resumable: already-logged filings whose file exists are skipped.24"""2526from __future__ import annotations2728import argparse29import sys30from pathlib import Path3132sys.path.insert(0, str(Path(__file__).resolve().parent.parent))3334from src.acquisition.bulk_downloader import BulkDownloader35from src.acquisition.universe import load_universe36from src.utils.constants import DATA_DIRS, SEC_FORM_TYPES37from src.utils.logging import setup_logging3839logger = setup_logging("download_filings")404142def parse_args() -> argparse.Namespace:43    p = argparse.ArgumentParser(description="Bulk EDGAR downloader")44    g = p.add_mutually_exclusive_group(required=True)45    g.add_argument("--universe", action="store_true", help="all companies in data/metadata/universe.json")46    g.add_argument("--ciks-file", type=str, help="text file with one CIK per line")47    g.add_argument("--ciks", type=str, help="comma-separated CIKs")48    p.add_argument("--forms", type=str, default=",".join(SEC_FORM_TYPES))49    p.add_argument("--start", type=str, default="2010-01-01")50    p.add_argument("--rate", type=float, default=9.0, help="requests per second for this process")51    p.add_argument("--workers", type=int, default=8)52    p.add_argument("--raw-dir", type=str, default=None)53    p.add_argument("--log", type=str, default=None, help="download log (JSONL)")54    p.add_argument("--submissions-dir", type=str, default=None)55    p.add_argument("--known-logs", type=str, default=None,56                   help="comma-separated earlier download logs whose filings are skipped without re-download")57    return p.parse_args()585960def main() -> None:61    args = parse_args()62    universe = {c["cik"]: c for c in load_universe()}63    if args.universe:64        ciks = list(universe)65    elif args.ciks_file:66        ciks = [l.strip() for l in Path(args.ciks_file).read_text().splitlines() if l.strip()]67    else:68        ciks = [c.strip() for c in args.ciks.split(",") if c.strip()]69    companies = [universe.get(c.zfill(10), {"cik": c.zfill(10), "ticker": ""}) for c in ciks]70    if not companies:71        logger.info("nothing to do")72        return73    dl = BulkDownloader(74        raw_dir=args.raw_dir, log_path=args.log, submissions_dir=args.submissions_dir,75        rate=args.rate, workers=args.workers,76        trusted_logs=[Path(x) for x in args.known_logs.split(",")] if args.known_logs else None,77    )78    stats = dl.download_many(companies, [f.strip() for f in args.forms.split(",")], args.start)79    logger.info("DONE %s", stats)80    if stats["failed"] > 0:81        sys.exit(2)828384if __name__ == "__main__":85    main()86