TypeScript 55.4%
Python 43.2%
SQL 1.2%
1#!/usr/bin/env python32"""Generate config/sources.d/49b-us-agencies-fedreg.yaml — one Federal Register documents filter3(`jsonlist` connector) per US federal agency not yet covered by 44-governments.yaml / 46-transport….45Agency slugs come from https://www.federalregister.gov/api/v1/agencies (cached in /tmp/fr_agencies.json).6The curated table below maps each slug to an existing registry id (`extend: true`) or declares a new agency7source (domain = the agency's own website). Sub-offices without a site of their own (OMB, OSTP, DOL agencies,8Treasury offices, DOJ divisions…) become extra sensors on their parent department.910`check` queries each filter once (per_page=1, ≤ 2 req/s) and drops agencies with no document in the last11`--max-age` days (default 730) so the validator never sees an empty list.1213Usage (repo root):14 python3 scripts/gen-fedreg.py check # → /tmp/fedreg_check.json15 python3 scripts/gen-fedreg.py emit > config/sources.d/49b-us-agencies-fedreg.yaml16"""17import datetime as dt18import glob19import json20import sys21import time2223import requests24import yaml2526UA = "WebSensor registry generator (contact@websensor.io)"27CHECK = "/tmp/fedreg_check.json"28AGENCIES = "/tmp/fr_agencies.json"29CFG = "config: { itemsPath: results, keyField: document_number, titleField: title, urlField: html_url, summaryField: abstract, dateField: publication_date, maxItems: 40 }"3031# (slug, registry id, name or None (=extend), domain, categories, aliases, sensor label, tier)32# Slugs already present in 44/46: coast-guard, federal-aviation-administration, federal-communications-commission,33# federal-maritime-commission, federal-motor-carrier-safety-administration, federal-railroad-administration,34# foreign-assets-control-office, homeland-security-department, industry-and-security-bureau,35# national-highway-traffic-safety-administration, national-telecommunications-and-information-administration,36# national-transportation-safety-board, pipeline-and-hazardous-materials-safety-administration,37# surface-transportation-board, transportation-department, transportation-security-administration.38T = [39 # ── finance & markets ──40 ("securities-and-exchange-commission", "sec", None, None, None, None, "Federal Register documents", "B"),41 ("commodity-futures-trading-commission", "cftc", None, None, None, None, "Federal Register documents", "B"),42 ("federal-reserve-system", "federal-reserve", None, None, None, None, "Federal Register documents", "B"),43 ("treasury-department", "us-treasury", None, None, None, None, "Federal Register documents (Treasury)", "B"),44 ("financial-stability-oversight-council", "us-treasury", None, None, None, None, "FSOC documents in the Federal Register", "C"),45 ("investment-security-office", "us-treasury", None, None, None, None, "CFIUS (Office of Investment Security) documents in the Federal Register", "B"),46 ("financial-research-office", "us-treasury", None, None, None, None, "Office of Financial Research documents in the Federal Register", "C"),47 ("comptroller-of-the-currency", "occ", None, None, None, None, "Federal Register documents", "B"),48 ("federal-deposit-insurance-corporation", "fdic", None, None, None, None, "Federal Register documents", "B"),49 ("consumer-financial-protection-bureau", "cfpb", None, None, None, None, "Federal Register documents", "B"),50 ("national-credit-union-administration", "ncua", None, None, None, None, "Federal Register documents", "C"),51 ("financial-crimes-enforcement-network", "fincen", None, None, None, None, "Federal Register documents", "B"),52 ("internal-revenue-service", "irs", None, None, None, None, "Federal Register documents", "B"),53 ("federal-housing-finance-agency", "fhfa", None, None, None, None, "Federal Register documents", "C"),54 ("federal-trade-commission", "ftc", None, None, None, None, "Federal Register documents", "B"),55 ("federal-financial-institutions-examination-council", "ffiec", "Federal Financial Institutions Examination Council", "ffiec.gov", "[finance, government]", "[ffiec]", "Federal Register documents", "C"),56 ("farm-credit-administration", "farm-credit-administration", "Farm Credit Administration", "fca.gov", "[finance, agriculture, government]", "[fca, farm credit administration]", "Federal Register documents", "C"),57 ("community-development-financial-institutions-fund", "cdfi-fund", "CDFI Fund (U.S. Treasury)", "cdfifund.gov", "[finance, government]", "[cdfi fund, cdfi]", "Federal Register documents", "C"),58 ("bureau-of-the-fiscal-service", "fiscal-service", "Bureau of the Fiscal Service (U.S. Treasury)", "fiscal.treasury.gov", "[finance, government]", "[fiscal service, bureau of the fiscal service]", "Federal Register documents", "C"),59 ("united-states-mint", "us-mint", "United States Mint", "usmint.gov", "[finance, government]", "[us mint, united states mint]", "Federal Register documents", "C"),60 ("engraving-and-printing-bureau", "bep", "Bureau of Engraving and Printing", "bep.gov", "[finance, government]", "[bep, bureau of engraving and printing]", "Federal Register documents", "D"),61 ("alcohol-and-tobacco-tax-and-trade-bureau", "ttb", "Alcohol and Tobacco Tax and Trade Bureau", "ttb.gov", "[consumer-safety, food, government]", "[ttb]", "Federal Register documents", "C"),62 ("pension-benefit-guaranty-corporation", "pbgc", "Pension Benefit Guaranty Corporation", "pbgc.gov", "[finance, labour, government]", "[pbgc]", "Federal Register documents", "C"),63 ("railroad-retirement-board", "rrb", "U.S. Railroad Retirement Board", "rrb.gov", "[labour, government]", "[rrb, railroad retirement board]", "Federal Register documents", "D"),64 ("federal-retirement-thrift-investment-board", "frtib", "Federal Retirement Thrift Investment Board", "frtib.gov", "[finance, government]", "[frtib, thrift savings plan, tsp]", "Federal Register documents", "D"),65 ("export-import-bank", "exim", "Export-Import Bank of the United States", "exim.gov", "[finance, trade, government]", "[exim, ex-im bank, export-import bank]", "Federal Register documents", "C"),66 ("u-s-international-development-finance-corporation", "dfc", "U.S. International Development Finance Corporation", "dfc.gov", "[finance, international, government]", "[dfc]", "Federal Register documents", "C"),67 ("government-national-mortgage-association", "ginnie-mae", "Ginnie Mae", "ginniemae.gov", "[finance, real-estate, government]", "[ginnie mae, gnma]", "Federal Register documents", "D"),68 # ── trade & commerce ──69 ("commerce-department", "us-commerce", "U.S. Department of Commerce", "commerce.gov", "[government, commerce]", "[commerce department, doc, department of commerce]", "Federal Register documents", "B"),70 ("international-trade-administration", "ita", "International Trade Administration (U.S. Commerce)", "trade.gov", "[trade, government]", "[ita, international trade administration]", "Federal Register documents (antidumping, countervailing duties)", "B"),71 ("foreign-trade-zones-board", "ita", None, None, None, None, "Foreign-Trade Zones Board documents in the Federal Register", "C"),72 ("international-trade-commission", "usitc", "U.S. International Trade Commission", "usitc.gov", "[trade, government]", "[usitc, itc, international trade commission]", "Federal Register documents (Section 337, injury investigations)", "B"),73 ("trade-representative-office-of-united-states", "ustr", "Office of the U.S. Trade Representative", "ustr.gov", "[trade, politics, government]", "[ustr, trade representative]", "Federal Register documents (Section 301, tariffs)", "A"),74 ("patent-and-trademark-office", "uspto", "U.S. Patent and Trademark Office", "uspto.gov", "[legal, technology, government]", "[uspto, pto, patent office]", "Federal Register documents", "B"),75 ("economic-development-administration", "eda", "Economic Development Administration", "eda.gov", "[government, commerce]", "[eda]", "Federal Register documents", "D"),76 ("census-bureau", "census", None, None, None, None, "Federal Register documents", "C"),77 ("economic-analysis-bureau", "bea", None, None, None, None, "Federal Register documents", "D"),78 ("national-institute-of-standards-and-technology", "nist", None, None, None, None, "Federal Register documents", "C"),79 ("national-oceanic-and-atmospheric-administration", "noaa", None, None, None, None, "Federal Register documents (fisheries, marine rules)", "C"),80 ("trade-and-development-agency", "ustda", "U.S. Trade and Development Agency", "ustda.gov", "[trade, international, government]", "[ustda]", "Federal Register documents", "D"),81 # ── health ──82 ("health-and-human-services-department", "hhs", "U.S. Department of Health and Human Services", "hhs.gov", "[health, government]", "[hhs, department of health and human services]", "Federal Register documents", "B"),83 ("food-and-drug-administration", "fda", None, None, None, None, "Federal Register documents", "B"),84 ("centers-for-disease-control-and-prevention", "cdc", None, None, None, None, "Federal Register documents", "C"),85 ("centers-for-medicare-medicaid-services", "cms", None, None, None, None, "Federal Register documents (payment rules, coverage)", "B"),86 ("national-institutes-of-health", "nih", None, None, None, None, "Federal Register documents", "C"),87 ("substance-abuse-and-mental-health-services-administration", "samhsa", None, None, None, None, "Federal Register documents", "D"),88 ("health-resources-and-services-administration", "hrsa", "Health Resources and Services Administration", "hrsa.gov", "[health, government]", "[hrsa]", "Federal Register documents", "D"),89 ("agency-for-healthcare-research-and-quality", "ahrq", "Agency for Healthcare Research and Quality", "ahrq.gov", "[health, research, government]", "[ahrq]", "Federal Register documents", "D"),90 ("indian-health-service", "ihs", "Indian Health Service", "ihs.gov", "[health, government]", "[ihs, indian health service]", "Federal Register documents", "D"),91 ("children-and-families-administration", "acf", "Administration for Children and Families", "acf.hhs.gov", "[health, government]", "[acf]", "Federal Register documents", "D"),92 ("refugee-resettlement-office", "acf", None, None, None, None, "Office of Refugee Resettlement documents in the Federal Register", "D"),93 ("community-living-administration", "acl", "Administration for Community Living", "acl.gov", "[health, government]", "[acl, administration for community living]", "Federal Register documents", "D"),94 ("strategic-preparedness-and-response-administration", "aspr", "Administration for Strategic Preparedness and Response", "aspr.hhs.gov", "[health, government]", "[aspr]", "Federal Register documents", "C"),95 ("inspector-general-office-health-and-human-services-department", "hhs-oig", "HHS Office of Inspector General", "oig.hhs.gov", "[health, legal, government]", "[hhs oig, oig hhs]", "Federal Register documents", "D"),96 ("national-library-of-medicine", "nlm", "National Library of Medicine", "nlm.nih.gov", "[health, research, government]", "[nlm, national library of medicine]", "Federal Register documents", "D"),97 # ── food, agriculture, consumer safety ──98 ("agriculture-department", "usda", None, None, None, None, "Federal Register documents (USDA)", "C"),99 ("food-safety-and-inspection-service", "fsis", "Food Safety and Inspection Service (USDA)", "fsis.usda.gov", "[food, consumer-safety, government]", "[fsis]", "Federal Register documents", "B"),100 ("animal-and-plant-health-inspection-service", "aphis", "Animal and Plant Health Inspection Service (USDA)", "aphis.usda.gov", "[agriculture, food, government]", "[aphis]", "Federal Register documents", "B"),101 ("agricultural-marketing-service", "usda-ams", "Agricultural Marketing Service (USDA)", "ams.usda.gov", "[agriculture, food, government]", "[ams, agricultural marketing service]", "Federal Register documents", "C"),102 ("food-and-nutrition-service", "usda-fns", "Food and Nutrition Service (USDA)", "fns.usda.gov", "[food, government]", "[fns, food and nutrition service, snap]", "Federal Register documents", "C"),103 ("forest-service", "us-forest-service", "U.S. Forest Service", "fs.usda.gov", "[climate, agriculture, government]", "[forest service, usfs]", "Federal Register documents", "C"),104 ("farm-service-agency", "usda-fsa", "Farm Service Agency (USDA)", "fsa.usda.gov", "[agriculture, government]", "[farm service agency]", "Federal Register documents", "D"),105 ("commodity-credit-corporation", "usda-fsa", None, None, None, None, "Commodity Credit Corporation documents in the Federal Register", "D"),106 ("risk-management-agency", "usda-rma", "Risk Management Agency (USDA crop insurance)", "rma.usda.gov", "[agriculture, finance, government]", "[rma, risk management agency, federal crop insurance]", "Federal Register documents", "D"),107 ("federal-crop-insurance-corporation", "usda-rma", "Risk Management Agency (USDA crop insurance)", "rma.usda.gov", "[agriculture, finance, government]", "[rma, risk management agency, federal crop insurance]", "Federal Crop Insurance Corporation documents in the Federal Register", "D"),108 ("natural-resources-conservation-service", "usda-nrcs", "Natural Resources Conservation Service (USDA)", "nrcs.usda.gov", "[agriculture, climate, government]", "[nrcs]", "Federal Register documents", "D"),109 ("rural-utilities-service", "usda-rural-development", "USDA Rural Development", "rd.usda.gov", "[agriculture, telecom, energy, government]", "[rural development, rural utilities service, rus]", "Rural Utilities Service documents in the Federal Register", "D"),110 ("rural-housing-service", "usda-rural-development", None, None, None, None, "Rural Housing Service documents in the Federal Register", "D"),111 ("rural-business-cooperative-service", "usda-rural-development", None, None, None, None, "Rural Business-Cooperative Service documents in the Federal Register", "D"),112 ("economic-research-service", "usda-ers", "Economic Research Service (USDA)", "ers.usda.gov", "[agriculture, statistics, government]", "[ers, economic research service]", "Federal Register documents", "D"),113 ("national-agricultural-statistics-service", "usda-nass", "National Agricultural Statistics Service (USDA)", "nass.usda.gov", "[agriculture, statistics, government]", "[nass]", "Federal Register documents", "D"),114 ("foreign-agricultural-service", "usda-fas", "Foreign Agricultural Service (USDA)", "fas.usda.gov", "[agriculture, trade, government]", "[foreign agricultural service]", "Federal Register documents", "D"),115 ("national-institute-of-food-and-agriculture", "usda-nifa", "National Institute of Food and Agriculture (USDA)", "nifa.usda.gov", "[agriculture, research, government]", "[nifa]", "Federal Register documents", "D"),116 ("agricultural-research-service", "usda-ars", "Agricultural Research Service (USDA)", "ars.usda.gov", "[agriculture, research, government]", "[agricultural research service]", "Federal Register documents", "D"),117 ("consumer-product-safety-commission", "cpsc", None, None, None, None, "Federal Register documents", "B"),118 ("chemical-safety-and-hazard-investigation-board", "csb", "U.S. Chemical Safety and Hazard Investigation Board", "csb.gov", "[consumer-safety, energy, government]", "[csb, chemical safety board]", "Federal Register documents", "D"),119 # ── environment, energy, interior ──120 ("environmental-protection-agency", "epa", None, None, None, None, "Federal Register documents", "B"),121 ("energy-department", "us-doe", None, None, None, None, "Federal Register documents (DOE)", "B"),122 ("energy-efficiency-and-renewable-energy-office", "us-doe", None, None, None, None, "EERE documents in the Federal Register (efficiency standards)", "C"),123 ("national-nuclear-security-administration", "us-doe", None, None, None, None, "NNSA documents in the Federal Register", "C"),124 ("energy-information-administration", "eia", None, None, None, None, "Federal Register documents", "D"),125 ("federal-energy-regulatory-commission", "ferc", None, None, None, None, "Federal Register documents", "B"),126 ("nuclear-regulatory-commission", "us-nrc", None, None, None, None, "Federal Register documents", "B"),127 ("defense-nuclear-facilities-safety-board", "dnfsb", "Defense Nuclear Facilities Safety Board", "dnfsb.gov", "[energy, government]", "[dnfsb]", "Federal Register documents", "D"),128 ("bonneville-power-administration", "bpa", "Bonneville Power Administration", "bpa.gov", "[energy, government]", "[bpa, bonneville]", "Federal Register documents", "D"),129 ("western-area-power-administration", "wapa", "Western Area Power Administration", "wapa.gov", "[energy, government]", "[wapa]", "Federal Register documents", "D"),130 ("tennessee-valley-authority", "tva", "Tennessee Valley Authority", "tva.com", "[energy, government]", "[tva]", "Federal Register documents", "D"),131 ("interior-department", "us-doi", "U.S. Department of the Interior", "doi.gov", "[government, energy, climate]", "[doi, interior department, department of the interior]", "Federal Register documents", "B"),132 ("fish-and-wildlife-service", "fws", "U.S. Fish and Wildlife Service", "fws.gov", "[climate, government]", "[fws, fish and wildlife service, endangered species]", "Federal Register documents", "B"),133 ("land-management-bureau", "blm", "Bureau of Land Management", "blm.gov", "[energy, climate, government]", "[blm, bureau of land management]", "Federal Register documents", "B"),134 ("national-park-service", "nps", "National Park Service", "nps.gov", "[travel, climate, government]", "[nps, national park service]", "Federal Register documents", "C"),135 ("ocean-energy-management-bureau", "boem", "Bureau of Ocean Energy Management", "boem.gov", "[energy, government]", "[boem, offshore leasing]", "Federal Register documents", "B"),136 ("safety-and-environmental-enforcement-bureau", "bsee", "Bureau of Safety and Environmental Enforcement", "bsee.gov", "[energy, government]", "[bsee]", "Federal Register documents", "C"),137 ("indian-affairs-bureau", "bia", "Bureau of Indian Affairs", "bia.gov", "[government]", "[bia, bureau of indian affairs]", "Federal Register documents", "C"),138 ("reclamation-bureau", "usbr", "Bureau of Reclamation", "usbr.gov", "[energy, climate, government]", "[usbr, bureau of reclamation]", "Federal Register documents", "D"),139 ("surface-mining-reclamation-and-enforcement-office", "osmre", "Office of Surface Mining Reclamation and Enforcement", "osmre.gov", "[energy, government]", "[osmre]", "Federal Register documents", "D"),140 ("natural-resources-revenue-office", "onrr", "Office of Natural Resources Revenue", "onrr.gov", "[energy, finance, government]", "[onrr]", "Federal Register documents", "D"),141 ("geological-survey", "usgs", None, None, None, None, "Federal Register documents", "D"),142 ("national-indian-gaming-commission", "nigc", "National Indian Gaming Commission", "nigc.gov", "[gaming, government]", "[nigc]", "Federal Register documents", "D"),143 # ── labour ──144 ("labor-department", "us-dol", None, None, None, None, "Federal Register documents (DOL)", "B"),145 ("employee-benefits-security-administration", "us-dol", None, None, None, None, "EBSA documents in the Federal Register", "C"),146 ("employment-and-training-administration", "us-dol", None, None, None, None, "ETA documents in the Federal Register (H-1B/H-2 labor certification)", "C"),147 ("wage-and-hour-division", "us-dol", None, None, None, None, "Wage and Hour Division documents in the Federal Register", "C"),148 ("federal-contract-compliance-programs-office", "us-dol", None, None, None, None, "OFCCP documents in the Federal Register", "D"),149 ("workers-compensation-programs-office", "us-dol", None, None, None, None, "OWCP documents in the Federal Register", "D"),150 ("veterans-employment-and-training-service", "us-dol", None, None, None, None, "VETS documents in the Federal Register", "D"),151 ("occupational-safety-and-health-administration", "osha", "Occupational Safety and Health Administration", "osha.gov", "[labour, consumer-safety, government]", "[osha]", "Federal Register documents", "B"),152 ("mine-safety-and-health-administration", "msha", "Mine Safety and Health Administration", "msha.gov", "[labour, energy, government]", "[msha]", "Federal Register documents", "C"),153 ("occupational-safety-and-health-review-commission", "oshrc", "Occupational Safety and Health Review Commission", "oshrc.gov", "[labour, legal, government]", "[oshrc]", "Federal Register documents", "D"),154 ("labor-statistics-bureau", "bls", None, None, None, None, "Federal Register documents", "D"),155 ("national-labor-relations-board", "nlrb", None, None, None, None, "Federal Register documents", "C"),156 ("equal-employment-opportunity-commission", "eeoc", None, None, None, None, "Federal Register documents", "C"),157 ("national-mediation-board", "nmb", "National Mediation Board", "nmb.gov", "[labour, transport, government]", "[nmb, national mediation board]", "Federal Register documents", "D"),158 ("federal-mediation-and-conciliation-service", "fmcs", "Federal Mediation and Conciliation Service", "fmcs.gov", "[labour, government]", "[fmcs]", "Federal Register documents", "D"),159 ("federal-labor-relations-authority", "flra", "Federal Labor Relations Authority", "flra.gov", "[labour, government]", "[flra]", "Federal Register documents", "D"),160 ("merit-systems-protection-board", "mspb", "U.S. Merit Systems Protection Board", "mspb.gov", "[labour, legal, government]", "[mspb]", "Federal Register documents", "D"),161 ("personnel-management-office", "opm", "U.S. Office of Personnel Management", "opm.gov", "[labour, government]", "[opm, office of personnel management]", "Federal Register documents", "B"),162 # ── justice, security, immigration ──163 ("justice-department", "us-doj", None, None, None, None, "Federal Register documents (DOJ)", "B"),164 ("antitrust-division", "us-doj", None, None, None, None, "Antitrust Division documents in the Federal Register (Tunney Act notices)", "B"),165 ("executive-office-for-immigration-review", "us-doj", None, None, None, None, "EOIR documents in the Federal Register", "C"),166 ("federal-bureau-of-investigation", "fbi", None, None, None, None, "Federal Register documents", "D"),167 ("drug-enforcement-administration", "dea", "Drug Enforcement Administration", "dea.gov", "[health, legal, government]", "[dea, drug enforcement administration]", "Federal Register documents (scheduling, registrations)", "B"),168 ("alcohol-tobacco-firearms-and-explosives-bureau", "atf", "Bureau of Alcohol, Tobacco, Firearms and Explosives", "atf.gov", "[legal, consumer-safety, government]", "[atf]", "Federal Register documents", "C"),169 ("prisons-bureau", "bop", "Federal Bureau of Prisons", "bop.gov", "[legal, government]", "[bop, bureau of prisons]", "Federal Register documents", "D"),170 ("united-states-marshals-service", "usms", "U.S. Marshals Service", "usmarshals.gov", "[legal, government]", "[usms, marshals service]", "Federal Register documents", "D"),171 ("justice-programs-office", "ojp", "Office of Justice Programs", "ojp.gov", "[legal, government]", "[ojp, office of justice programs]", "Federal Register documents", "D"),172 ("united-states-sentencing-commission", "ussc", "U.S. Sentencing Commission", "ussc.gov", "[legal, government]", "[ussc, sentencing commission]", "Federal Register documents", "C"),173 ("u-s-citizenship-and-immigration-services", "uscis", None, None, None, None, "Federal Register documents", "B"),174 ("u-s-customs-and-border-protection", "us-cbp", None, None, None, None, "Federal Register documents", "B"),175 ("u-s-immigration-and-customs-enforcement", "us-ice", None, None, None, None, "Federal Register documents", "C"),176 ("federal-emergency-management-agency", "fema", None, None, None, None, "Federal Register documents (disaster declarations, flood maps)", "C"),177 ("secret-service", "us-secret-service", "U.S. Secret Service", "secretservice.gov", "[government, cyber]", "[secret service, usss]", "Federal Register documents", "D"),178 ("central-intelligence-agency", "cia", "Central Intelligence Agency", "cia.gov", "[government, politics]", "[cia]", "Federal Register documents", "D"),179 ("national-intelligence-office-of-the-national-director", "odni", "Office of the Director of National Intelligence", "dni.gov", "[government, politics, cyber]", "[odni, dni]", "Federal Register documents", "D"),180 ("privacy-and-civil-liberties-oversight-board", "pclob", "Privacy and Civil Liberties Oversight Board", "pclob.gov", "[legal, cyber, government]", "[pclob]", "Federal Register documents", "D"),181 ("civil-rights-commission", "usccr", "U.S. Commission on Civil Rights", "usccr.gov", "[legal, politics, government]", "[usccr, civil rights commission]", "Federal Register documents", "D"),182 ("state-department", "us-state-department", None, None, None, None, "Federal Register documents", "B"),183 ("agency-for-international-development", "usaid", "U.S. Agency for International Development", "usaid.gov", "[international, government]", "[usaid]", "Federal Register documents", "C"),184 ("millennium-challenge-corporation", "mcc", "Millennium Challenge Corporation", "mcc.gov", "[international, government]", "[mcc, millennium challenge corporation]", "Federal Register documents", "D"),185 ("united-states-agency-for-global-media", "usagm", "U.S. Agency for Global Media", "usagm.gov", "[media, government]", "[usagm, voice of america, voa]", "Federal Register documents", "D"),186 ("peace-corps", "peace-corps", "Peace Corps", "peacecorps.gov", "[international, government]", "[peace corps]", "Federal Register documents", "D"),187 ("u-s-china-economic-and-security-review-commission", "uscc", "U.S.-China Economic and Security Review Commission", "uscc.gov", "[politics, trade, government]", "[uscc]", "Federal Register documents", "D"),188 # ── defense ──189 ("defense-department", "us-dod", None, None, None, None, "Federal Register documents (DoD)", "B"),190 ("defense-acquisition-regulations-system", "us-dod", None, None, None, None, "DFARS documents in the Federal Register", "C"),191 ("army-department", "us-army", "U.S. Army", "army.mil", "[government]", "[us army, army]", "Federal Register documents", "D"),192 ("navy-department", "us-navy", "U.S. Navy", "navy.mil", "[government]", "[us navy, navy]", "Federal Register documents", "D"),193 ("air-force-department", "us-air-force", "U.S. Air Force", "af.mil", "[government, space]", "[us air force, air force, usaf]", "Federal Register documents", "D"),194 ("engineers-corps", "usace", "U.S. Army Corps of Engineers", "usace.army.mil", "[infrastructure, climate, government]", "[usace, army corps of engineers]", "Federal Register documents", "C"),195 ("defense-logistics-agency", "dla", "Defense Logistics Agency", "dla.mil", "[logistics, government]", "[dla]", "Federal Register documents", "D"),196 ("national-security-agency-central-security-service", "nsa", "National Security Agency", "nsa.gov", "[cyber, government]", "[nsa]", "Federal Register documents", "D"),197 ("national-geospatial-intelligence-agency", "nga", "National Geospatial-Intelligence Agency", "nga.mil", "[space, government]", "[nga]", "Federal Register documents", "D"),198 ("defense-information-systems-agency", "disa", "Defense Information Systems Agency", "disa.mil", "[cyber, telecom, government]", "[disa]", "Federal Register documents", "D"),199 # ── transport (not already in 46) ──200 ("federal-highway-administration", "fhwa", "Federal Highway Administration", "fhwa.dot.gov", "[transport, infrastructure, government]", "[fhwa]", "Federal Register documents", "C"),201 ("federal-transit-administration", "fta", "Federal Transit Administration", "transit.dot.gov", "[transport, government]", "[fta, federal transit administration]", "Federal Register documents", "C"),202 ("maritime-administration", "marad", "U.S. Maritime Administration", "maritime.dot.gov", "[transport, logistics, government]", "[marad]", "Federal Register documents", "C"),203 ("transportation-statistics-bureau", "bts", "Bureau of Transportation Statistics", "bts.gov", "[transport, statistics, government]", "[bts]", "Federal Register documents", "D"),204 ("commercial-space-transportation-office", "faa", None, None, None, None, "Office of Commercial Space Transportation documents in the Federal Register", "C"),205 ("national-railroad-passenger-corporation", "amtrak", None, None, None, None, "Federal Register documents", "D"),206 ("national-aeronautics-and-space-administration", "nasa", None, None, None, None, "Federal Register documents", "C"),207 # ── executive office, regulatory process, records ──208 ("executive-office-of-the-president", "white-house", None, None, None, None, "Executive Office of the President documents in the Federal Register", "B"),209 ("management-and-budget-office", "white-house", None, None, None, None, "OMB documents in the Federal Register", "B"),210 ("science-and-technology-policy-office", "white-house", None, None, None, None, "OSTP documents in the Federal Register", "B"),211 ("council-on-environmental-quality", "white-house", None, None, None, None, "CEQ documents in the Federal Register", "C"),212 ("office-of-national-drug-control-policy", "white-house", None, None, None, None, "ONDCP documents in the Federal Register", "D"),213 ("office-of-the-national-cyber-director", "white-house", None, None, None, None, "ONCD documents in the Federal Register", "C"),214 ("federal-register-office", "federal-register", None, None, None, None, "Office of the Federal Register documents", "D"),215 ("regulatory-information-service-center", "reginfo", "Regulatory Information Service Center (Unified Agenda)", "reginfo.gov", "[government, legal]", "[risc, reginfo, unified agenda]", "Federal Register documents", "D"),216 ("federal-acquisition-regulation-system", "far-council", "Federal Acquisition Regulation (FAR) Council", "acquisition.gov", "[government, legal]", "[far, federal acquisition regulation]", "Federal Register documents", "C"),217 ("general-services-administration", "gsa", "U.S. General Services Administration", "gsa.gov", "[government, infrastructure]", "[gsa]", "Federal Register documents", "C"),218 ("national-archives-and-records-administration", "nara", "National Archives and Records Administration", "archives.gov", "[government, legal]", "[nara, national archives]", "Federal Register documents", "D"),219 ("information-security-oversight-office", "nara", None, None, None, None, "ISOO documents in the Federal Register", "D"),220 ("government-publishing-office", "gpo", "U.S. Government Publishing Office", "gpo.gov", "[government]", "[gpo, government publishing office]", "Federal Register documents", "D"),221 ("library-of-congress", "library-of-congress", "Library of Congress", "loc.gov", "[education, government]", "[loc, library of congress]", "Federal Register documents", "D"),222 ("copyright-office-library-of-congress", "us-copyright-office", None, None, None, None, "Federal Register documents", "C"),223 ("copyright-royalty-board", "crb", "Copyright Royalty Board", "crb.gov", "[media, legal, government]", "[crb, copyright royalty board]", "Federal Register documents", "C"),224 ("postal-service", "usps", None, None, None, None, "Federal Register documents", "C"),225 ("postal-regulatory-commission", "prc-postal", "Postal Regulatory Commission", "prc.gov", "[logistics, government]", "[prc, postal regulatory commission]", "Federal Register documents", "C"),226 ("federal-election-commission", "fec", None, None, None, None, "Federal Register documents", "C"),227 ("election-assistance-commission", "eac", None, None, None, None, "Federal Register documents", "D"),228 ("government-ethics-office", "oge", None, None, None, None, "Federal Register documents", "D"),229 ("special-counsel-office", "us-osc", None, None, None, None, "Federal Register documents", "D"),230 ("social-security-administration", "ssa", "Social Security Administration", "ssa.gov", "[government, labour]", "[ssa, social security]", "Federal Register documents", "B"),231 ("education-department", "us-ed", "U.S. Department of Education", "ed.gov", "[education, government]", "[ed, department of education]", "Federal Register documents", "B"),232 ("veterans-affairs-department", "us-va", None, None, None, None, "Federal Register documents", "C"),233 ("housing-and-urban-development-department", "hud", None, None, None, None, "Federal Register documents", "B"),234 ("small-business-administration", "sba", None, None, None, None, "Federal Register documents", "C"),235 ("national-science-foundation", "nsf", None, None, None, None, "Federal Register documents", "D"),236 ("national-endowment-for-the-arts", "nea", "National Endowment for the Arts", "arts.gov", "[entertainment, government]", "[nea, national endowment for the arts]", "Federal Register documents", "D"),237 ("national-endowment-for-the-humanities", "neh", "National Endowment for the Humanities", "neh.gov", "[education, government]", "[neh]", "Federal Register documents", "D"),238 ("institute-of-museum-and-library-services", "imls", "Institute of Museum and Library Services", "imls.gov", "[education, government]", "[imls]", "Federal Register documents", "D"),239 ("smithsonian-institution", "smithsonian", "Smithsonian Institution", "si.edu", "[education, research, government]", "[smithsonian]", "Federal Register documents", "D"),240 ("corporation-for-national-and-community-service", "americorps", "AmeriCorps", "americorps.gov", "[government]", "[americorps, cncs]", "Federal Register documents", "D"),241 ("selective-service-system", "sss", "Selective Service System", "sss.gov", "[government]", "[selective service]", "Federal Register documents", "D"),242 ("legal-services-corporation", "lsc", "Legal Services Corporation", "lsc.gov", "[legal, government]", "[lsc, legal services corporation]", "Federal Register documents", "D"),243 ("administrative-conference-of-the-united-states", "acus", "Administrative Conference of the United States", "acus.gov", "[legal, government]", "[acus]", "Federal Register documents", "D"),244 ("architectural-and-transportation-barriers-compliance-board", "access-board", "U.S. Access Board", "access-board.gov", "[standards, government]", "[access board, atbcb]", "Federal Register documents", "D"),245 ("federal-permitting-improvement-steering-council", "permitting-council", "Federal Permitting Improvement Steering Council", "permits.performance.gov", "[infrastructure, energy, government]", "[permitting council, fpisc]", "Federal Register documents", "D"),246 ("first-responder-network-authority", "firstnet-authority", "First Responder Network Authority (FirstNet)", "firstnet.gov", "[telecom, government]", "[firstnet]", "Federal Register documents", "D"),247 ("international-joint-commission-united-states-and-canada", "ijc", "International Joint Commission (Canada–United States)", "ijc.org", "[international, climate, government]", "[ijc, international joint commission]", "Federal Register documents", "D"),248 ("judicial-conference-of-the-united-states", "us-courts", None, None, None, None, "Judicial Conference documents in the Federal Register", "D"),249 ("administrative-office-of-united-states-courts", "us-courts", None, None, None, None, "Administrative Office documents in the Federal Register", "D"),250 ("government-accountability-office", "gao", None, None, None, None, "Federal Register documents", "D"),251 ("congressional-budget-office", "cbo", None, None, None, None, "Federal Register documents", "D"),252]253254255def url_for(slug: str) -> str:256 return f"https://www.federalregister.gov/api/v1/documents.json?conditions[agencies][]={slug}&order=newest&per_page=40"257258259def registry_ids() -> tuple[set[str], set[str], set[str]]:260 ids, urls, extended = set(), set(), set()261 for f in ["config/sources.yaml"] + sorted(glob.glob("config/sources.d/*.yaml")):262 if f.endswith("49b-us-agencies-fedreg.yaml"):263 continue264 d = yaml.safe_load(open(f)) or {}265 for s in d.get("sources", []):266 (extended if s.get("extend") else ids).add(s["id"])267 for sen in s.get("sensors") or []:268 urls.add(sen.get("url", ""))269 return ids, urls, extended270271272def check(argv: list[str]) -> None:273 max_age = int(argv[argv.index("--max-age") + 1]) if "--max-age" in argv else 730274 slugs = {a["slug"] for a in json.load(open(AGENCIES))}275 try:276 out = json.load(open(CHECK))277 except FileNotFoundError:278 out = {}279 sess = requests.Session()280 for row in T:281 slug = row[0]282 if slug not in slugs:283 print(f"{slug}: not an agency slug", file=sys.stderr)284 out[slug] = {"ok": False, "why": "unknown slug"}285 continue286 if slug in out and "count" in out[slug]:287 continue288 r = sess.get(url_for(slug).replace("per_page=40", "per_page=1"), headers={"User-Agent": UA}, timeout=30)289 time.sleep(0.5)290 if r.status_code != 200:291 out[slug] = {"ok": False, "why": f"http {r.status_code}"}292 print(f"{slug}: HTTP {r.status_code}", file=sys.stderr)293 continue294 j = r.json()295 res = j.get("results") or []296 latest = res[0]["publication_date"] if res else None297 age = (dt.date.today() - dt.date.fromisoformat(latest)).days if latest else None298 ok = bool(res) and age is not None and age <= max_age299 out[slug] = {"ok": ok, "count": j.get("count", 0), "latest": latest, "age_days": age}300 print(f"{'OK ' if ok else 'DROP'} {slug:70} count={j.get('count', 0):>7} latest={latest}", file=sys.stderr)301 json.dump(out, open(CHECK, "w"), indent=1)302 json.dump(out, open(CHECK, "w"), indent=1)303 print(f"{sum(1 for v in out.values() if v.get('ok'))}/{len(out)} filters usable", file=sys.stderr)304305306def emit() -> None:307 chk = json.load(open(CHECK))308 ids, urls, _ = registry_ids()309 print("# config/sources.d/49b-us-agencies-fedreg.yaml — Federal Register documents per US federal agency (2026-09-11,")310 print("# generated by scripts/gen-fedreg.py from api/v1/agencies; every filter checked live for recent documents). Rules,")311 print("# proposed rules and notices of ~120 agencies not yet covered by 44-governments.yaml / 46-transport…: financial")312 print("# regulators, trade, health, agriculture, environment/energy/interior, labour, justice/security, defense, transport,")313 print("# the Executive Office and independent boards. Existing agency ids are extended; sub-offices without a site of their")314 print("# own (OMB, OSTP, DOL agencies, Treasury offices, DOJ divisions…) become extra sensors on the parent department.")315 print("# Federal Register API: 1000 req/h, no key; `jsonlist` keyed by document_number.")316 print("sources:")317 groups: dict[str, list[tuple]] = {}318 order: list[str] = []319 for row in T:320 slug, sid = row[0], row[1]321 if not chk.get(slug, {}).get("ok"):322 print(f"# dropped {slug}: {chk.get(slug)}", file=sys.stderr)323 continue324 if url_for(slug) in urls:325 print(f"# dropped {slug}: URL already in registry", file=sys.stderr)326 continue327 if sid not in groups:328 order.append(sid)329 groups.setdefault(sid, []).append(row)330 n_ext = n_new = n_sen = 0331 for sid in order:332 rows = groups[sid]333 decl = next((r for r in rows if r[2]), None)334 sensors = [f' - {{ name: {r[6]}, url: "{url_for(r[0])}", type: REST_API, connector: jsonlist, tier: {r[7]}, {CFG} }}' for r in rows]335 if sid in ids or decl is None:336 if decl is not None and sid in ids:337 print(f"# {sid} already declared elsewhere — extending instead of redeclaring", file=sys.stderr)338 if sid not in ids:339 print(f"# ERROR {sid} is not declared anywhere and has no declaration row", file=sys.stderr)340 continue341 print(f" - id: {sid}")342 print(" extend: true")343 print(" country: US")344 print(" sensors:")345 n_ext += 1346 else:347 _, _, name, domain, cats, aliases, _, tier = decl348 print(f" - id: {sid}")349 print(f" name: {json.dumps(name, ensure_ascii=False)}")350 print(f" domain: {domain}")351 print(f" homepage: https://{'www.' + domain if domain.count('.') == 1 else domain}")352 print(f" categories: {cats}")353 print(" tier: B")354 print(f" aliases: {aliases}")355 print(" country: US")356 print(f" notes: \"Federal Register agency slug: {', '.join(r[0] for r in rows)}\"")357 print(" discover: { rss: true }")358 print(" sensors:")359 n_new += 1360 for s in sensors:361 print(s)362 n_sen += 1363 print(f"{n_ext} extended, {n_new} new, {n_sen} sensors", file=sys.stderr)364365366if __name__ == "__main__":367 if sys.argv[1:2] == ["check"]:368 check(sys.argv[2:])369 else:370 emit()371