TypeScript 55.4%
Python 43.2%
SQL 1.2%
1#!/usr/bin/env python32"""Generate config/sources.d/49-edgar-issuers.yaml — SEC EDGAR `submissions` sensors for every S&P 500,3Nasdaq-100 and TSX 60 issuer not yet covered by the registry, plus the largest US-listed pharma/biotech4(SIC 2834/2836), semiconductor (3674), software (7372) and bank (6021/6022) issuers by market cap.56Builds on scripts/gen-edgar.py (apex/norm/slug/pretty_name/DOMAINS/MANUAL) but:7 * matches existing organizations across ALL registry files (founding file + every fragment, incl. 40–46) by8 manual map, ticker alias, apex domain, normalized name, then unambiguous name prefix;9 * skips every CIK already present anywhere in the registry (no duplicate sensor URLs);10 * resolves the company website from EDGAR `website`, Wikidata (P856 by ticker), then the DOMAINS maps;11 * sets `country:` from the business address (foreign private issuers keep their own country).1213Inputs (all fetched by `prepare`):14 /tmp/tickers.json https://www.sec.gov/files/company_tickers.json (ordered by market cap)15 /tmp/tickers_exch.json https://www.sec.gov/files/company_tickers_exchange.json16 /tmp/sp500.json ["MMM", …] (Wikipedia list of S&P 500 companies)17 /tmp/ndx.json ["AAPL", …] (api.nasdaq.com nasdaq100 list)18 /tmp/tsx60.json [["AEM", "Agnico Eagle Mines Limited"], …] (Wikipedia S&P/TSX 60)19 /tmp/wd.csv Wikidata SPARQL export: item,itemLabel,ticker,exchLabel,website2021Usage (repo root):22 python3 scripts/gen-edgar2.py fetch [--sector-limit 150] [--scan 900] # → /tmp/edgar2_meta.json (≤ 6 req/s)23 python3 scripts/gen-edgar2.py emit > config/sources.d/49-edgar-issuers.yaml24"""25import csv26import glob27import json28import re29import sys30import time31from urllib.parse import urlparse3233import requests34import yaml3536import importlib.util as _ilu # noqa: E4023738_spec = _ilu.spec_from_file_location("gen_edgar", __file__.rsplit("/", 1)[0] + "/gen-edgar.py")39_g = _ilu.module_from_spec(_spec)40_spec.loader.exec_module(_g)41DOMAINS1, MANUAL1, apex, norm, pretty_name, slug = _g.DOMAINS, _g.MANUAL, _g.apex, _g.norm, _g.pretty_name, _g.slug4243UA = "WebSensor registry generator (contact@websensor.io)"44META = "/tmp/edgar2_meta.json"45FORMS = "[8-K, 10-K, 10-Q, 6-K, 20-F, 40-F, S-1, SC 13D, DEF 14A]"46SECTOR_SICS = {"2834", "2836", "3674", "7372", "6021", "6022"}47US_STATES = set("AL AK AZ AR CA CO CT DE FL GA HI ID IL IN IA KS KY LA ME MD MA MI MN MS MO MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX UT VT VA WA WV WI WY DC PR GU VI X1".split())48COUNTRY_BY_DESC = {49 "canada": "CA", "netherlands": "NL", "united kingdom": "GB", "ireland": "IE", "switzerland": "CH", "germany": "DE",50 "france": "FR", "japan": "JP", "china": "CN", "taiwan": "TW", "korea": "KR", "india": "IN", "brazil": "BR",51 "israel": "IL", "singapore": "SG", "hong kong": "HK", "bermuda": "BM", "cayman islands": "KY", "luxembourg": "LU",52 "denmark": "DK", "sweden": "SE", "finland": "FI", "norway": "NO", "spain": "ES", "italy": "IT", "belgium": "BE",53 "australia": "AU", "mexico": "MX", "argentina": "AR", "chile": "CL", "colombia": "CO", "peru": "PE", "panama": "PA",54 "south africa": "ZA", "new zealand": "NZ", "austria": "AT", "portugal": "PT", "greece": "GR", "poland": "PL",55 "jersey": "JE", "guernsey": "GG", "isle of man": "IM", "monaco": "MC", "united arab emirates": "AE", "indonesia": "ID",56 "philippines": "PH", "thailand": "TH", "malaysia": "MY", "vietnam": "VN", "turkey": "TR", "russia": "RU",57 "puerto rico": "US", "virgin islands": "US", "marshall islands": "MH", "liberia": "LR", "cyprus": "CY", "malta": "MT",58 "czech": "CZ", "hungary": "HU", "uruguay": "UY", "macau": "MO", "kazakhstan": "KZ", "saudi arabia": "SA",59}60# TSX 60 ticker → SEC ticker (US listing) when the symbols differ; None = no SEC registration (skipped).61TSX_TO_SEC = {"ABX": "B", "BIP.UN": "BIP", "CCO": "CCJ", "CNR": "CNI", "GIB.A": "GIB", "K": "KGC", "MG": "MGA", "PPL": "PBA",62 "RCI.B": "RCI", "TECK.B": "TECK", "T": "TU", "ATD": None, "CTC.A": None, "CCL.B": None, "CSU": None, "DOL": None,63 "EMA": None, "FFH": None, "FM": None, "WN": None, "H": None, "IFC": None, "L": None, "MRU": None, "NA": None,64 "POW": None, "SAP": None, "TOU": None, "WSP": None}65# Ticker → existing registry id when domain/name matching cannot decide (checked against the real id set).66MANUAL2 = dict(MANUAL1)67MANUAL2.update({"GOOG": "google", "BRK-A": "berkshire-hathaway", "FOX": "fox-corporation", "FOXA": "fox-corporation", "NWS": "news-corp", "NWSA": "news-corp",68 "PARA": "paramount", "PSKY": "paramount-skydance", "WBD": "warner-bros-discovery", "EA": "electronic-arts", "TTWO": "take-two", "RBLX": "roblox",69 "U": "unity", "SPOT": "spotify", "ABNB": "airbnb", "BKNG": "booking-holdings", "EXPE": "expedia", "MAR": "marriott", "HLT": "hilton",70 "RY": "rbc", "TD": "td-bank", "BMO": "bmo", "BNS": "scotiabank", "CM": "cibc", "CNQ": "canadian-natural", "CNI": "cn", "CP": "cpkc",71 "ENB": "enbridge", "TRP": "tc-energy", "SU": "suncor", "BCE": "bce", "RCI": "rogers", "TU": "telus", "TRI": "thomson-reuters",72 "MFC": "manulife", "SLF": "sun-life", "OTEX": "opentext", "GIB": "cgi", "QSR": "restaurant-brands", "NTR": "nutrien", "WCN": "waste-connections",73 "CVE": "cenovus", "IMO": "imperial-oil", "FTS": "fortis", "CCJ": "cameco", "MGA": "magna", "CLS": "celestica", "CAE": "cae",74 "ORCL": "oracle", "IBM": "ibm", "CSCO": "cisco", "HPE": "hpe", "HPQ": "hp", "DELL": "dell", "NTAP": "netapp", "PSTG": "pure-storage",75 "CRWV": "coreweave", "NBIS": "nebius", "PLTR": "palantir", "SNPS": "synopsys", "CDNS": "cadence", "ANSS": "ansys", "ADP": "adp",76 "PAYX": "paychex", "OKTA": "okta", "TWLO": "twilio", "HUBS": "hubspot", "ZM": "zoom", "DOCU": "docusign", "BOX": "box",77 "DBX": "dropbox", "GTLB": "gitlab", "S": "sentinelone", "TENB": "tenable", "RPD": "rapid7", "CYBR": "cyberark", "QLYS": "qualys",78 "VRNS": "varonis", "CHKP": "check-point", "AKAM": "akamai", "FSLY": "fastly", "DOCN": "digitalocean", "GDDY": "godaddy", "WIX": "wix",79 "SQSP": "squarespace", "SHOP": "shopify", "SQ": "block", "XYZ": "block", "MELI": "mercadolibre", "SE": "sea-limited", "BABA": "alibaba",80 "JD": "jd-com", "PDD": "pdd-holdings", "BIDU": "baidu", "NTES": "netease", "TCEHY": "tencent", "SONY": "sony", "NTDOY": "nintendo",81 "TSM": "tsmc", "ASML": "asml", "AMAT": "applied-materials", "LRCX": "lam-research", "KLAC": "kla", "TXN": "texas-instruments",82 "ADI": "analog-devices", "MRVL": "marvell", "NXPI": "nxp", "ON": "onsemi", "MCHP": "microchip", "STM": "stmicroelectronics",83 "IFNNY": "infineon", "SWKS": "skyworks", "QRVO": "qorvo", "MPWR": "monolithic-power", "GFS": "globalfoundries", "SMCI": "supermicro",84 "WDC": "western-digital", "STX": "seagate", "SNDK": "sandisk", "AVGO": "broadcom", "MU": "micron", "ARM": "arm", "QCOM": "qualcomm",85 "INTC": "intel", "AMD": "amd", "NVDA": "nvidia", "AAPL": "apple", "MSFT": "microsoft", "GOOGL": "google", "META": "meta-ai",86 "AMZN": "amazon", "NFLX": "netflix", "TSLA": "tesla", "UBER": "uber", "LYFT": "lyft", "DASH": "doordash", "RDDT": "reddit",87 "PINS": "pinterest", "SNAP": "snap", "DUOL": "duolingo", "COIN": "coinbase", "HOOD": "robinhood", "CRCL": "circle", "MSTR": "strategy",88 "V": "visa", "MA": "mastercard", "AXP": "american-express", "PYPL": "paypal", "FI": "fiserv", "FIS": "fis", "GPN": "global-payments",89 "ADYEY": "adyen", "JPM": "jpmorgan", "BAC": "bank-of-america", "WFC": "wells-fargo", "C": "citi", "GS": "goldman-sachs",90 "MS": "morgan-stanley", "SCHW": "charles-schwab", "BLK": "blackrock", "BX": "blackstone", "KKR": "kkr", "APO": "apollo",91 "USB": "us-bancorp", "PNC": "pnc", "TFC": "truist", "COF": "capital-one", "BK": "bny", "STT": "state-street", "NTRS": "northern-trust",92 "ICE": "ice", "CME": "cme-group", "NDAQ": "nasdaq", "CBOE": "cboe", "SPGI": "sp-global", "MCO": "moodys", "MSCI": "msci",93 "HSBC": "hsbc", "BCS": "barclays", "UBS": "ubs", "DB": "deutsche-bank", "SAN": "santander", "ING": "ing", "BBVA": "bbva",94 "MUFG": "mufg", "SMFG": "smbc", "MFG": "mizuho", "NMR": "nomura", "LYG": "lloyds", "NWG": "natwest", "TM": "toyota", "HMC": "honda",95 "F": "ford", "GM": "gm", "STLA": "stellantis", "RIVN": "rivian", "LCID": "lucid", "PFE": "pfizer", "MRNA": "moderna", "MRK": "merck",96 "LLY": "eli-lilly", "AZN": "astrazeneca", "NVS": "novartis", "GSK": "gsk", "SNY": "sanofi", "NVO": "novo-nordisk", "RHHBY": "roche",97 "JNJ": "johnson-johnson", "ABBV": "abbvie", "AMGN": "amgen", "GILD": "gilead", "BMY": "bristol-myers-squibb", "REGN": "regeneron",98 "VRTX": "vertex", "BIIB": "biogen", "BNTX": "biontech", "TAK": "takeda", "BAYRY": "bayer", "ISRG": "intuitive-surgical",99 "MDT": "medtronic", "ABT": "abbott", "SYK": "stryker", "BSX": "boston-scientific", "TMO": "thermo-fisher", "DHR": "danaher",100 "UNH": "unitedhealth", "CVS": "cvs-health", "CI": "cigna", "HUM": "humana", "ELV": "elevance", "CNC": "centene", "MOH": "molina",101 "T": "att", "VZ": "verizon", "TMUS": "t-mobile", "CMCSA": "comcast", "CHTR": "charter", "DIS": "disney", "LUMN": "lumen",102 "DAL": "delta", "UAL": "united-airlines", "AAL": "american-airlines", "LUV": "southwest", "BA": "boeing", "LMT": "lockheed-martin",103 "RTX": "rtx", "NOC": "northrop-grumman", "GD": "general-dynamics", "GE": "ge-aerospace", "RKLB": "rocket-lab", "ASTS": "ast-spacemobile", "SPCX": "spacex",104 "XOM": "exxonmobil", "CVX": "chevron", "COP": "conocophillips", "SHEL": "shell", "BP": "bp", "TTE": "totalenergies", "EQNR": "equinor",105 "WMT": "walmart", "COST": "costco", "TGT": "target", "HD": "home-depot", "LOW": "lowes", "KR": "kroger", "NKE": "nike",106 "SBUX": "starbucks", "MCD": "mcdonalds", "KO": "coca-cola", "PEP": "pepsico", "PG": "procter-gamble", "UL": "unilever", "NSRGY": "nestle"})107# Tickers that must NOT be folded into a look-alike source (subsidiary / sister company with a similar name).108NEVER_MATCH = {"CCEP", "TXT"}109# Tickers skipped altogether (shell / successor registrants with an empty filing history).110SKIP_TICKERS = {"CONE"}111# Ticker → registry id when the slug of the display name is unusable (initials, accents, generic words).112IDS2 = {"FNB": "fnb-corporation", "ITUB": "itau-unibanco", "AOS": "a-o-smith", "NWSA": "news-corp", "SSNC": "ssc-technologies", "YOU": "clear-secure",113 "MSTR": "strategy-microstrategy", "WRB": "wr-berkley", "DHI": "dr-horton", "MTB": "mt-bank", "PCG": "pge-corporation", "GWW": "ww-grainger", "VMRK": "vivmark-residential"}114# Ticker → display name when EDGAR's registrant name is ugly (raw upper-case, legal suffixes, state fragments).115NAMES2 = {"SSNC": "SS&C Technologies", "BBD": "Banco Bradesco", "ITUB": "Itaú Unibanco", "AOS": "A. O. Smith", "DHI": "D.R. Horton", "WRB": "W. R. Berkley",116 "CRH": "CRH", "NWSA": "News Corp", "MTB": "M&T Bank", "PCG": "PG&E", "GWW": "W.W. Grainger", "ZION": "Zions Bancorporation", "KEY": "KeyCorp",117 "LEN": "Lennar", "OKE": "ONEOK", "PHM": "PulteGroup", "HBAN": "Huntington Bancshares", "GLW": "Corning", "CFG": "Citizens Financial Group",118 "UBSI": "United Bankshares", "ONB": "Old National Bancorp", "FNB": "F.N.B. Corporation", "CBSH": "Commerce Bancshares", "ABVX": "Abivax",119 "ASND": "Ascendis Pharma", "GMAB": "Genmab", "MFG": "Mizuho Financial Group", "SHG": "Shinhan Financial Group", "LVS": "Las Vegas Sands",120 "ODFL": "Old Dominion Freight Line", "MKC": "McCormick & Company", "MCK": "McKesson", "DVA": "DaVita", "COO": "The Cooper Companies",121 "EXPD": "Expeditors International", "IEX": "IDEX", "IDXX": "IDEXX Laboratories", "IQV": "IQVIA", "AME": "AMETEK", "MTD": "Mettler-Toledo",122 "STE": "STERIS", "CSGP": "CoStar Group", "SSB": "SouthState", "NTES": "NetEase", "GFS": "GlobalFoundries", "FORM": "FormFactor",123 "KDP": "Keurig Dr Pepper", "PM": "Philip Morris International", "MO": "Altria Group", "BF-B": "Brown-Forman", "BRO": "Brown & Brown",124 "MRSH": "Marsh McLennan", "FCNCA": "First Citizens BancShares", "EWBC": "East West Bancorp", "CFR": "Cullen/Frost Bankers", "HWC": "Hancock Whitney",125 "HOMB": "Home BancShares", "PNFP": "Pinnacle Financial Partners", "BPOP": "Popular, Inc.", "PB": "Prosperity Bancshares", "UMBF": "UMB Financial",126 "WAL": "Western Alliance Bancorporation", "WTFC": "Wintrust Financial", "BOKF": "BOK Financial", "ASB": "Associated Banc-Corp",127 "AUB": "Atlantic Union Bankshares", "ABCB": "Ameris Bancorp", "COLB": "Columbia Banking System", "GBCI": "Glacier Bancorp", "ALLY": "Ally Financial",128 "VLY": "Valley National Bancorp", "SYF": "Synchrony Financial", "ASX": "ASE Technology Holding", "UMC": "United Microelectronics", "STM": "STMicroelectronics",129 "TSEM": "Tower Semiconductor", "SIMO": "Silicon Motion", "MTSI": "MACOM Technology Solutions", "SLAB": "Silicon Labs", "AAOI": "Applied Optoelectronics",130 "YMM": "Full Truck Alliance", "GLBE": "Global-e", "APPF": "AppFolio", "BSY": "Bentley Systems", "YOU": "CLEAR (Clear Secure)", "GWRE": "Guidewire",131 "MANH": "Manhattan Associates", "MBLY": "Mobileye", "PAYC": "Paycom", "PCTY": "Paylocity", "PCOR": "Procore", "TTAN": "ServiceTitan", "SAIL": "SailPoint",132 "IFNNY": "Infineon Technologies", "ATEYY": "Advantest", "NBIS": "Nebius Group", "MSTR": "Strategy (MicroStrategy)", "IT": "Gartner", "J": "Jacobs",133 "L": "Loews Corporation", "TT": "Trane Technologies", "TEL": "TE Connectivity", "ARE": "Alexandria Real Estate Equities", "DOC": "Healthpeak Properties",134 "AES": "The AES Corporation", "APA": "APA Corporation", "EQT": "EQT Corporation", "PPL": "PPL Corporation", "NVR": "NVR, Inc.", "UDR": "UDR, Inc.",135 "BXP": "BXP, Inc.", "CDW": "CDW Corporation", "FICO": "Fair Isaac (FICO)", "GEN": "Gen Digital", "ON": "onsemi", "SMCI": "Supermicro", "SNDK": "Sandisk",136 "ECHO": "EchoStar", "PTC": "PTC Inc.", "B": "Barrick Mining", "MAA": "Mid-America Apartment Communities", "VMRK": "Vivmark Residential (Equity Residential)",137 "EXE": "Expand Energy", "FIX": "Comfort Systems USA", "EME": "EMCOR Group", "PWR": "Quanta Services", "TDY": "Teledyne Technologies", "GL": "Globe Life",138 "SW": "Smurfit Westrock", "ROP": "Roper Technologies", "HPE": "Hewlett Packard Enterprise", "FDXF": "FedEx Freight", "NXT": "Nextpower (Nextracker)",139 "Q": "Qnity Electronics", "TPL": "Texas Pacific Land", "AEM": "Agnico Eagle Mines", "FNV": "Franco-Nevada", "KGC": "Kinross Gold", "WPM": "Wheaton Precious Metals",140 "TECK": "Teck Resources", "PBA": "Pembina Pipeline", "MGA": "Magna International", "GIL": "Gildan Activewear", "FSV": "FirstService", "WCN": "Waste Connections",141 "FER": "Ferrovial", "CCEP": "Coca-Cola Europacific Partners", "WTW": "Willis Towers Watson", "PNR": "Pentair", "AMCR": "Amcor", "APTV": "Aptiv",142 "ALLE": "Allegion", "JCI": "Johnson Controls", "SPCX": "SpaceX"}143# Ticker → website domain when EDGAR, Wikidata and gen-edgar DOMAINS are silent.144DOMAINS2 = {"BRK-B": "berkshirehathaway.com", "GOOG": "abc.xyz", "PSKY": "paramount.com", "FOXA": "foxcorporation.com", "NWSA": "newscorp.com",145 "LEN-B": "lennar.com", "UHAL-B": "uhaul.com", "TKO": "tkogrp.com", "CRH": "crh.com", "TEL": "te.com", "APH": "amphenol.com",146 "TT": "tranetechnologies.com", "JCI": "johnsoncontrols.com", "ETN": "eaton.com", "ACN": "accenture.com", "MDT": "medtronic.com",147 "GRMN": "garmin.com", "STE": "steris.com", "ALLE": "allegion.com", "PNR": "pentair.com", "NVT": "nvent.com", "AON": "aon.com",148 "WTW": "wtwco.com", "CB": "chubb.com", "IR": "irco.com", "AER": "aercap.com", "ICLR": "iclr.com", "MPWR": "monolithicpower.com",149 "WAB": "wabteccorp.com", "FTV": "fortive.com", "HUBB": "hubbell.com", "L": "loews.com", "ERIE": "erieinsurance.com",150 "ABVX": "abivax.com", "ATEYY": "advantest.com", "ASND": "ascendispharma.com", "BLTE": "belitebio.com", "CGON": "cgoncology.com",151 "COGT": "cogentbio.com", "COLB": "columbiabankingsystem.com", "CRDO": "credosemi.com", "DNTH": "dianthustx.com", "FORM": "formfactor.com",152 "YMM": "fulltruckalliance.com", "GMAB": "genmab.com", "GLBE": "global-e.com", "IBRX": "immunitybio.com", "IMVT": "immunovant.com",153 "IFNNY": "infineon.com", "INSM": "insmed.com", "IONS": "ionis.com", "KNSA": "kiniksa.com", "LGND": "ligand.com", "MDGL": "madrigalpharma.com",154 "NXT": "nextracker.com", "ORKA": "orukatx.com", "PRAX": "praxismedicines.com", "PTGX": "protagonist-inc.com", "RVMD": "revmed.com",155 "RPRX": "royaltypharma.com", "SAIL": "sailpoint.com", "SRRK": "scholarrock.com", "TTAN": "servicetitan.com", "SYRE": "spyre.com",156 "TVTX": "travere.com", "UTHR": "unither.com", "VLY": "valley.com", "PCVX": "vaxcyte.com", "FER": "ferrovial.com", "FSV": "firstservice.com",157 "MRSH": "marshmclennan.com", "WCN": "wasteconnections.com", "WPM": "wheatonpm.com", "CCEP": "cocacolaep.com", "TXT": "textron.com",158 "ASX": "aseglobal.com", "AME": "ametek.com", "KDP": "keurigdrpepper.com", "GL": "globelifeinsurance.com", "DOC": "healthpeak.com",159 "SW": "smurfitwestrock.com", "MSTR": "strategy.com", "EXE": "expandenergy.com", "SYF": "synchrony.com", "ROP": "ropertech.com",160 "FNB": "fnb-online.com", "ZETA": "zetaglobal.com", "AES": "aes.com", "APA": "apacorp.com", "B": "barrick.com", "BXP": "bxp.com",161 "CDW": "cdw.com", "ECHO": "echostar.com", "EQT": "eqt.com", "FICO": "fico.com", "GEN": "gendigital.com", "KEY": "key.com",162 "MAA": "maac.com", "NVR": "nvrinc.com", "ON": "onsemi.com", "PCG": "pgecorp.com", "PPL": "pplweb.com", "PTC": "ptc.com",163 "SNDK": "sandisk.com", "SMCI": "supermicro.com", "UDR": "udr.com"}164165166def load_registry() -> dict:167 """Ids, apex domains, normalized names, aliases and CIKs across the founding file and every fragment."""168 by_domain: dict[str, str] = {}169 by_name: dict[str, str] = {}170 by_alias: dict[str, str] = {}171 ids: set[str] = set()172 ciks: set[str] = set()173 urls: set[str] = set()174 name_of: dict[str, str] = {}175 later_domains: dict[str, str] = {}176 later_names: dict[str, str] = {}177 files = ["config/sources.yaml"] + sorted(glob.glob("config/sources.d/*.yaml"))178 later_ids: set[str] = set() # declared in fragments merged AFTER this one — cannot be extended from here179 for f in files:180 if f.endswith("49-edgar-issuers.yaml"):181 continue182 text = open(f).read()183 ciks.update(f"{int(c):010d}" for c in re.findall(r"CIK(\d+)\.json", text))184 d = yaml.safe_load(text) or {}185 base = f.split("/")[-1]186 later = base[:2].isdigit() and (int(base[:2]) > 49 or base.startswith("49b"))187 for s in d.get("sources", []):188 for sen in s.get("sensors") or []:189 urls.add(sen.get("url", ""))190 if later:191 if not s.get("extend"):192 later_ids.add(s["id"])193 later_domains.setdefault(apex(s["domain"]), s["id"])194 later_names.setdefault(norm(s["name"]), s["id"])195 continue196 for a in s.get("aliases") or []:197 by_alias.setdefault(str(a).lower(), s["id"])198 if s.get("extend"):199 continue200 ids.add(s["id"])201 name_of[s["id"]] = norm(s["name"])202 by_domain.setdefault(apex(s["domain"]), s["id"])203 by_name.setdefault(norm(s["name"]), s["id"])204 return {"by_domain": by_domain, "by_name": by_name, "by_alias": by_alias, "ids": ids, "ciks": ciks, "urls": urls, "name_of": name_of, "later_ids": later_ids, "later_domains": later_domains, "later_names": later_names}205206207def sec_universe() -> tuple[list[dict], dict[str, dict]]:208 """company_tickers.json in market-cap order (list) + ticker → record (with exchange)."""209 ordered = [v for _, v in sorted(json.load(open("/tmp/tickers.json")).items(), key=lambda kv: int(kv[0]))]210 exch = {}211 for cik, name, ticker, ex in json.load(open("/tmp/tickers_exch.json"))["data"]:212 exch[ticker] = ex213 by_ticker = {}214 for v in ordered:215 v = dict(v)216 v["exchange"] = exch.get(v["ticker"])217 by_ticker.setdefault(v["ticker"], v)218 return ordered, by_ticker219220221def wikidata_sites() -> dict[str, list[tuple[str, str]]]:222 """ticker → [(normalized label, website)…] — several issuers can share a symbol across exchanges."""223 out: dict[str, list[tuple[str, str]]] = {}224 try:225 for r in csv.DictReader(open("/tmp/wd.csv")):226 t = r["ticker"].upper().replace(".", "-")227 if r.get("website"):228 out.setdefault(t, []).append((norm(r.get("itemLabel") or ""), r["website"]))229 except FileNotFoundError:230 pass231 return out232233234def wd_pick(t: str, name: str, wd: dict[str, list[tuple[str, str]]]) -> str | None:235 """Wikidata website for ticker t only when the Wikidata label shares a real word with the EDGAR name."""236 toks = {w for w in norm(name).split() if len(w) >= 4}237 for label, site in wd.get(t, []):238 if toks & {w for w in label.split() if len(w) >= 4}:239 return site240 return None241242243def fetch_one(sess: requests.Session, cik: str) -> dict | None:244 r = sess.get(f"https://data.sec.gov/submissions/CIK{cik}.json", headers={"User-Agent": UA, "Accept": "application/json", "Host": "data.sec.gov"}, timeout=30)245 if r.status_code != 200:246 print(f"CIK{cik}: HTTP {r.status_code}", file=sys.stderr)247 return None248 j = r.json()249 recent = (j.get("filings") or {}).get("recent") or {}250 forms = recent.get("form") or []251 biz = (j.get("addresses") or {}).get("business") or {}252 return {253 "cik": cik, "name": j.get("name"), "website": j.get("website") or j.get("investorWebsite") or "",254 "sic": str(j.get("sic") or ""), "sicDescription": j.get("sicDescription"), "exchanges": j.get("exchanges"), "tickers": j.get("tickers"),255 "state": j.get("stateOfIncorporation"), "stateDesc": j.get("stateOfIncorporationDescription"),256 "bizState": biz.get("stateOrCountry"), "bizStateDesc": biz.get("stateOrCountryDescription"),257 "recent": len(forms), "hasForeign": any(f in ("6-K", "20-F", "40-F") for f in forms[:200]),258 }259260261def fetch(argv: list[str]) -> None:262 sector_limit = int(argv[argv.index("--sector-limit") + 1]) if "--sector-limit" in argv else 150263 scan = int(argv[argv.index("--scan") + 1]) if "--scan" in argv else 900264 reg = load_registry()265 ordered, by_ticker = sec_universe()266 try:267 meta = json.load(open(META))268 except FileNotFoundError:269 meta = {"index": {}, "sector": {}}270 sess = requests.Session()271272 def polite_get(cik: str) -> dict | None:273 t0 = time.time()274 m = fetch_one(sess, cik)275 dt = time.time() - t0276 if dt < 0.17:277 time.sleep(0.17 - dt)278 return m279280 # 1) index universe: S&P 500 ∪ Nasdaq-100 ∪ TSX 60 (mapped to their SEC ticker)281 index_tickers: dict[str, str] = {}282 for t in json.load(open("/tmp/sp500.json")):283 index_tickers.setdefault(t.replace(".", "-"), "sp500")284 for t in json.load(open("/tmp/ndx.json")):285 index_tickers.setdefault(t.replace(".", "-"), "ndx")286 tsx_unmatched = []287 for t, name in json.load(open("/tmp/tsx60.json")):288 sec_t = TSX_TO_SEC.get(t, t)289 if sec_t is None:290 continue291 if sec_t not in by_ticker:292 # name match against SEC names (Canadian issuers are usually cross-listed under the same symbol)293 nm = norm(name)294 cands = [v["ticker"] for v in ordered if norm(v["title"]) == nm]295 if cands:296 sec_t = cands[0]297 else:298 tsx_unmatched.append((t, name))299 continue300 index_tickers.setdefault(sec_t, "tsx60")301 if tsx_unmatched:302 print(f"TSX 60 without SEC registration: {tsx_unmatched}", file=sys.stderr)303 todo = [t for t in index_tickers if t in by_ticker and f"{int(by_ticker[t]['cik_str']):010d}" not in reg["ciks"]]304 missing = [t for t in index_tickers if t not in by_ticker]305 if missing:306 print(f"index tickers not in company_tickers.json: {missing}", file=sys.stderr)307 print(f"index universe: {len(index_tickers)} tickers, {len(todo)} not yet covered by CIK", file=sys.stderr)308 seen_ciks: set[str] = set()309 for t in todo:310 cik = f"{int(by_ticker[t]['cik_str']):010d}"311 if cik in seen_ciks:312 continue313 seen_ciks.add(cik)314 if t in meta["index"]:315 continue316 m = polite_get(cik)317 if m:318 m["group"] = index_tickers[t]319 m["ticker"] = t320 m["exchange"] = by_ticker[t]["exchange"]321 meta["index"][t] = m322 json.dump(meta, open(META, "w"), indent=1)323 print(f"{len(meta['index'])} index issuers in {META}", file=sys.stderr)324325 # 2) sector universe: walk company_tickers.json (market-cap order) and keep SIC 2834/2836/3674/7372/6021/6022326 covered = set(reg["ciks"]) | {m["cik"] for m in meta["index"].values()}327 scanned = meta.setdefault("scanned", {})328 n_sector = len(meta["sector"])329 seen_here: set[str] = set()330 for v in ordered:331 if n_sector >= sector_limit or len(scanned) >= scan:332 break333 cik = f"{int(v['cik_str']):010d}"334 t = v["ticker"]335 if cik in covered or cik in seen_here or t in meta["sector"]:336 continue337 seen_here.add(cik)338 if cik in scanned:339 m = scanned[cik]340 if m is None or m.get("sic") not in SECTOR_SICS:341 continue342 else:343 m = polite_get(cik)344 scanned[cik] = {"sic": m["sic"], "name": m["name"]} if m else None345 if len(scanned) % 25 == 0:346 json.dump(meta, open(META, "w"), indent=1)347 print(f" scanned {len(scanned)} · sector picks {n_sector}", file=sys.stderr)348 if not m or m["sic"] not in SECTOR_SICS:349 continue350 m["group"] = "sector"351 m["ticker"] = t352 m["exchange"] = by_ticker.get(t, {}).get("exchange")353 meta["sector"][t] = m354 n_sector += 1355 json.dump(meta, open(META, "w"), indent=1)356 print(f"{len(meta['sector'])} sector issuers ({len(scanned)} scanned) in {META}", file=sys.stderr)357358359COUNTRY_OVERRIDE = {"BBD": "BR", "SHG": "KR", "NXPI": "NL", "KGC": "CA", "YMM": "CN", "NTES": "CN", "KNSA": "US", "TEL": "IE", "GIL": "CA",360 "ASX": "TW", "FER": "NL", "CRDO": "US", "SIMO": "TW", "ROIV": "US", "RPRX": "US", "AMCR": "CH", "ITUB": "BR", "MFG": "JP",361 "ATEYY": "JP", "IFNNY": "DE", "STM": "CH", "ABVX": "FR", "ASND": "DK", "GMAB": "DK", "GLBE": "IL", "TSEM": "IL", "UMC": "TW",362 "APTV": "IE", "CCEP": "GB", "WTW": "GB", "PNR": "GB", "ALLE": "IE", "CRH": "IE", "JCI": "IE", "STE": "IE", "TT": "IE",363 "ALKS": "IE", "FSV": "CA", "WCN": "CA", "MGA": "CA", "CLS": "CA", "AEM": "CA", "FNV": "CA", "WPM": "CA", "TECK": "CA", "PBA": "CA"}364365366def country_of(m: dict) -> str:367 if m.get("ticker") in COUNTRY_OVERRIDE:368 return COUNTRY_OVERRIDE[m["ticker"]]369 code = (m.get("bizState") or m.get("state") or "").upper()370 desc = (m.get("bizStateDesc") or m.get("stateDesc") or "").lower()371 if code in US_STATES:372 return "US"373 for k, v in COUNTRY_BY_DESC.items():374 if k in desc:375 return v376 if re.fullmatch(r"[AB][0-9]", code) or code == "Z4":377 return "CA" # EDGAR province codes A0–B0 / Z4 = Canada378 if desc or code:379 print(f"# unknown country {code!r} {desc!r} ({m.get('ticker')} {m.get('name')}) — defaulting to US", file=sys.stderr)380 return "US"381382383def categories(m: dict) -> str:384 """Registry categories from the SIC code / description (whole-word matching — "Toilet" is not "oil")."""385 sic = m.get("sic") or ""386 d = (m.get("sicDescription") or "").lower()387388 def has(*words: str) -> bool:389 return any(re.search(rf"\b{w}\b", d) for w in words)390391 def sic_in(*prefixes: str) -> bool:392 return any(sic.startswith(p) for p in prefixes)393394 if sic in ("2834", "2836", "2835", "8731") or has("pharmaceutical", "biological", "in vitro"):395 return "[pharma, filings]"396 if sic == "3674" or has("semiconductors"):397 return "[semiconductors, filings]"398 if sic in ("7372", "7371", "7370", "7373", "7374") or has("software", "computer programming", "data processing"):399 return "[enterprise, filings]"400 if sic_in("6798") or has("real estate investment trusts"):401 return "[real-estate, filings]"402 if sic_in("65") or has("real estate"):403 return "[real-estate, filings]"404 if sic_in("63", "64") or has("insurance"):405 return "[finance, insurance, filings]"406 if sic_in("60", "61", "62", "67") or has("bank", "banks", "finance", "investment", "loan", "credit", "brokers", "dealers"):407 return "[finance, filings]"408 if sic_in("5122") or has("wholesale-drugs", "drugs proprietaries"):409 return "[health, filings]"410 if sic_in("384", "385", "80") or has("medical", "surgical", "dental", "health", "hospitals", "orthopedic"):411 return "[health, filings]"412 if sic_in("2711", "2721", "2731", "2741", "4832", "4833", "4841", "7311", "7812", "7819", "7822") or has("newspapers", "periodicals", "books", "television", "radio broadcasting", "advertising", "motion picture", "cable"):413 return "[media, filings]"414 if sic_in("48") or has("telephone", "communications", "telegraph"):415 return "[telecom, filings]"416 if sic_in("4953", "4955", "4959") or has("refuse", "hazardous waste"):417 return "[infrastructure, filings]"418 if sic_in("13", "29", "46", "49", "5171", "5172") or has("petroleum", "natural gas", "gas transmission", "gas distribution", "electric", "energy", "crude", "power"):419 return "[energy, filings]"420 if sic_in("10", "12", "14") or has("mining", "gold", "silver", "ores", "quarrying", "coal"):421 return "[mining, filings]"422 if sic_in("2300", "23", "3021", "3140", "3021") or has("apparel", "footwear", "shoes"):423 return "[retail, filings]"424 if sic_in("52", "53", "54", "56", "57", "59", "5500", "5531") or has("retail", "stores", "convenience"):425 return "[retail, filings]"426 if sic_in("58") or has("eating places", "restaurants"):427 return "[retail, food, filings]"428 if sic_in("20", "21", "01", "02", "07") or has("food", "beverages", "tobacco", "cigarettes", "agricultural", "grain mill"):429 return "[food, filings]"430 if sic_in("70", "79") or has("hotels", "motels", "amusement", "gaming", "casino"):431 return "[entertainment, travel, filings]"432 if sic_in("37", "40", "42", "44", "45", "47", "3711", "3713", "3714", "3715", "3720", "3721", "3724", "3728", "3730") or has("motor vehicle", "motor vehicles", "aircraft", "air transportation", "railroad", "trucking", "ships", "shipping", "freight", "courier", "airports", "pipelines"):433 return "[transport, filings]"434 if sic_in("15", "16", "17") or has("construction", "contractors", "operative builders"):435 return "[real-estate, construction, filings]"436 if sic_in("28") or has("chemicals", "plastics", "fertilizers", "industrial gases", "cosmetics", "soap"):437 return "[chemicals, filings]"438 if sic_in("3571", "3572", "3576", "3577", "3578", "3579", "3661", "3663", "3669", "3670", "3672", "3677", "3678", "3679", "3812", "3821", "3822", "3823", "3824", "3825", "3826", "3827", "3829", "3861", "3357", "3679", "5045", "5065") or has("computer", "electronic", "instruments", "measuring", "optical", "photographic", "navigation", "communications equipment"):439 return "[technology, filings]"440 if sic_in("33", "34", "35", "36", "38", "39", "24", "25", "26", "30", "31", "32") or has("machinery", "equipment", "manufacturing", "industries", "products", "metal"):441 return "[manufacturing, filings]"442 if sic_in("50", "51") or has("wholesale"):443 return "[commerce, filings]"444 return "[enterprise, filings]"445446447def resolve_domain(t: str, m: dict, wd: dict[str, list[tuple[str, str]]]) -> tuple[str | None, str | None]:448 """→ (apex domain, host for homepage) from EDGAR website, Wikidata, DOMAINS maps."""449 cands = []450 if t in DOMAINS2:451 cands.append("https://" + DOMAINS2[t])452 if m.get("website"):453 cands.append(m["website"])454 for tk in [t] + list(m.get("tickers") or []):455 site = wd_pick(tk, m.get("name") or "", wd)456 if site and site not in cands:457 cands.append(site)458 for src in (DOMAINS2, DOMAINS1):459 if t in src:460 cands.append("https://" + src[t])461 for c in cands:462 host = urlparse(c if c.startswith("http") else "https://" + c).hostname463 if host and "." in host and not host.endswith(("sec.gov", "wikipedia.org")):464 host = host.lower()465 return apex(host), host466 return None, None467468469def emit() -> None:470 meta = json.load(open(META))471 reg = load_registry()472 wd = wikidata_sites()473 all_ids = set(reg["ids"])474 entries = list(meta["index"].values()) + list(meta["sector"].values())475 print("# config/sources.d/49-edgar-issuers.yaml — SEC EDGAR issuers (2026-09-11, generated by scripts/gen-edgar2.py from")476 print("# data.sec.gov submissions + company_tickers.json; validated live). One `edgar` sensor per issuer for every S&P 500,")477 print("# Nasdaq-100 and TSX 60 company not yet covered by an earlier fragment, plus the largest US-listed pharma/biotech")478 print("# (SIC 2834/2836), semiconductor (3674), software (7372) and bank (6021/6022) issuers by market cap. Forms watched:")479 print("# 8-K, 10-K, 10-Q, 6-K, 20-F, 40-F, S-1, SC 13D, DEF 14A (insider forms excluded). Organizations that already exist")480 print("# anywhere in the registry are extended (matched by ticker alias, apex domain or name); the others are declared with")481 print("# their own website as domain, `country:` from the business address (foreign private issuers keep theirs).")482 print("sources:")483 used: set[str] = set()484 used_ciks: set[str] = set(reg["ciks"])485 stats: dict[str, list[str]] = {}486 skipped = []487 for m in sorted(entries, key=lambda x: (x.get("group") != "sector", pretty_name(x["ticker"], x["name"] or "").lower())):488 t = m["ticker"]489 if m["cik"] in used_ciks or t in SKIP_TICKERS:490 continue491 ap, host = resolve_domain(t, m, wd)492 target = None493 why = ""494 nm = norm(m["name"] or "")495 toks = {w for w in nm.split() if len(w) >= 4}496 cand = MANUAL2.get(t)497 if cand and cand in all_ids:498 target, why = cand, "manual"499 if not target and ap and ap in reg["by_domain"]:500 target, why = reg["by_domain"][ap], f"domain {ap}"501 if not target and nm in reg["by_name"]:502 target, why = reg["by_name"][nm], "name"503 if not target and t.lower() in reg["by_alias"] and reg["by_alias"][t.lower()] in all_ids:504 # a ticker alias only counts when the source's own name shares a real word with the issuer's name505 sid = reg["by_alias"][t.lower()]506 if toks & {w for w in reg["name_of"].get(sid, "").split() if len(w) >= 4}:507 target, why = sid, f"alias {t.lower()}"508 if not target:509 cands = {sid for n, sid in reg["by_name"].items() if len(n) >= 6 and (nm.startswith(n + " ") or n.startswith(nm + " "))}510 if len(cands) == 1:511 target, why = cands.pop(), "name prefix"512 if not target and slug(m["name"] or "") in all_ids:513 target, why = slug(m["name"] or ""), "slug"514 if t in NEVER_MATCH:515 target = None516 if not target:517 later = (ap and reg["later_domains"].get(ap)) or reg["later_names"].get(nm) or (MANUAL2.get(t) if MANUAL2.get(t) in reg["later_ids"] else None)518 if later:519 skipped.append(f"{t} ({m['name']}): already declared as `{later}` in a later fragment — add its EDGAR sensor there")520 continue521 if target:522 print(f"# match {t:6} {m['name'][:40]:40} → {target} ({why})", file=sys.stderr)523 sensor = f' - {{ name: edgar filings, url: "https://data.sec.gov/submissions/CIK{m["cik"]}.json", type: REST_API, connector: edgar, tier: B, config: {{ forms: {FORMS} }} }}'524 sector = categories(m).strip("[]").split(",")[0]525 if target:526 if target in used:527 skipped.append(f"{t} → {target} already extended in this file")528 continue529 used.add(target)530 used_ciks.add(m["cik"])531 print(f" - id: {target}")532 print(" extend: true")533 print(" categories: [filings]")534 print(f" aliases: [{t.lower()}]")535 print(" sensors:")536 print(sensor)537 stats.setdefault(f"extended/{sector}", []).append(t)538 continue539 if not ap:540 skipped.append(f"{t} ({m['name']}): no website (EDGAR/Wikidata/DOMAINS)")541 continue542 name = NAMES2.get(t) or pretty_name(t, m["name"] or t)543 sid = IDS2.get(t) or slug(re.sub(r"\s*\(.*?\)|\b(s\.?a\.?|a/s|n\.?v\.?|plc|public limited company|l\.?p\.?|ag|se|inc\.?|corp\.?|co\.?|ltd\.?|the)\b|,.*$", " ", name, flags=re.I))544 if sid in all_ids or sid in used:545 sid = f"{sid}-{t.lower()}"546 if sid in all_ids or sid in used:547 skipped.append(f"{t}: id collision {sid}")548 continue549 used.add(sid)550 used_ciks.add(m["cik"])551 country = country_of(m)552 print(f" - id: {sid}")553 print(f" name: {json.dumps(name, ensure_ascii=False)}")554 print(f" domain: {ap}")555 print(f" homepage: https://{'www.' + host if host.count('.') == 1 else host}")556 print(f" categories: {categories(m)}")557 print(" tier: B")558 print(" entity_type: company")559 print(f" aliases: [{t.lower()}]")560 print(f" country: {country}")561 print(f" notes: \"SIC {m.get('sic') or 'n/a'} {m.get('sicDescription') or ''} · {', '.join(m.get('exchanges') or [])} · CIK {int(m['cik'])}{' · foreign private issuer' if m.get('hasForeign') else ''}\"")562 print(" discover: { rss: true }")563 print(" sensors:")564 print(sensor)565 stats.setdefault(f"new/{sector}", []).append(t)566 for k in sorted(stats):567 print(f"{k}: {len(stats[k])} — {' '.join(stats[k])}", file=sys.stderr)568 for s in skipped:569 print(f"# skipped: {s}", file=sys.stderr)570 print(f"total sensors: {sum(len(v) for v in stats.values())}", file=sys.stderr)571572573if __name__ == "__main__":574 if sys.argv[1:2] == ["fetch"]:575 fetch(sys.argv[2:])576 else:577 emit()578