#!/usr/bin/env python3
"""Generate config/sources.d/49b-us-agencies-fedreg.yaml — one Federal Register documents filter
(`jsonlist` connector) per US federal agency not yet covered by 44-governments.yaml / 46-transport….

Agency slugs come from https://www.federalregister.gov/api/v1/agencies (cached in /tmp/fr_agencies.json).
The curated table below maps each slug to an existing registry id (`extend: true`) or declares a new agency
source (domain = the agency's own website). Sub-offices without a site of their own (OMB, OSTP, DOL agencies,
Treasury offices, DOJ divisions…) become extra sensors on their parent department.

`check` queries each filter once (per_page=1, ≤ 2 req/s) and drops agencies with no document in the last
`--max-age` days (default 730) so the validator never sees an empty list.

Usage (repo root):
  python3 scripts/gen-fedreg.py check          # → /tmp/fedreg_check.json
  python3 scripts/gen-fedreg.py emit > config/sources.d/49b-us-agencies-fedreg.yaml
"""
import datetime as dt
import glob
import json
import sys
import time

import requests
import yaml

UA = "WebSensor registry generator (contact@websensor.io)"
CHECK = "/tmp/fedreg_check.json"
AGENCIES = "/tmp/fr_agencies.json"
CFG = "config: { itemsPath: results, keyField: document_number, titleField: title, urlField: html_url, summaryField: abstract, dateField: publication_date, maxItems: 40 }"

# (slug, registry id, name or None (=extend), domain, categories, aliases, sensor label, tier)
# Slugs already present in 44/46: coast-guard, federal-aviation-administration, federal-communications-commission,
# federal-maritime-commission, federal-motor-carrier-safety-administration, federal-railroad-administration,
# foreign-assets-control-office, homeland-security-department, industry-and-security-bureau,
# national-highway-traffic-safety-administration, national-telecommunications-and-information-administration,
# national-transportation-safety-board, pipeline-and-hazardous-materials-safety-administration,
# surface-transportation-board, transportation-department, transportation-security-administration.
T = [
    # ── finance & markets ──
    ("securities-and-exchange-commission", "sec", None, None, None, None, "Federal Register documents", "B"),
    ("commodity-futures-trading-commission", "cftc", None, None, None, None, "Federal Register documents", "B"),
    ("federal-reserve-system", "federal-reserve", None, None, None, None, "Federal Register documents", "B"),
    ("treasury-department", "us-treasury", None, None, None, None, "Federal Register documents (Treasury)", "B"),
    ("financial-stability-oversight-council", "us-treasury", None, None, None, None, "FSOC documents in the Federal Register", "C"),
    ("investment-security-office", "us-treasury", None, None, None, None, "CFIUS (Office of Investment Security) documents in the Federal Register", "B"),
    ("financial-research-office", "us-treasury", None, None, None, None, "Office of Financial Research documents in the Federal Register", "C"),
    ("comptroller-of-the-currency", "occ", None, None, None, None, "Federal Register documents", "B"),
    ("federal-deposit-insurance-corporation", "fdic", None, None, None, None, "Federal Register documents", "B"),
    ("consumer-financial-protection-bureau", "cfpb", None, None, None, None, "Federal Register documents", "B"),
    ("national-credit-union-administration", "ncua", None, None, None, None, "Federal Register documents", "C"),
    ("financial-crimes-enforcement-network", "fincen", None, None, None, None, "Federal Register documents", "B"),
    ("internal-revenue-service", "irs", None, None, None, None, "Federal Register documents", "B"),
    ("federal-housing-finance-agency", "fhfa", None, None, None, None, "Federal Register documents", "C"),
    ("federal-trade-commission", "ftc", None, None, None, None, "Federal Register documents", "B"),
    ("federal-financial-institutions-examination-council", "ffiec", "Federal Financial Institutions Examination Council", "ffiec.gov", "[finance, government]", "[ffiec]", "Federal Register documents", "C"),
    ("farm-credit-administration", "farm-credit-administration", "Farm Credit Administration", "fca.gov", "[finance, agriculture, government]", "[fca, farm credit administration]", "Federal Register documents", "C"),
    ("community-development-financial-institutions-fund", "cdfi-fund", "CDFI Fund (U.S. Treasury)", "cdfifund.gov", "[finance, government]", "[cdfi fund, cdfi]", "Federal Register documents", "C"),
    ("bureau-of-the-fiscal-service", "fiscal-service", "Bureau of the Fiscal Service (U.S. Treasury)", "fiscal.treasury.gov", "[finance, government]", "[fiscal service, bureau of the fiscal service]", "Federal Register documents", "C"),
    ("united-states-mint", "us-mint", "United States Mint", "usmint.gov", "[finance, government]", "[us mint, united states mint]", "Federal Register documents", "C"),
    ("engraving-and-printing-bureau", "bep", "Bureau of Engraving and Printing", "bep.gov", "[finance, government]", "[bep, bureau of engraving and printing]", "Federal Register documents", "D"),
    ("alcohol-and-tobacco-tax-and-trade-bureau", "ttb", "Alcohol and Tobacco Tax and Trade Bureau", "ttb.gov", "[consumer-safety, food, government]", "[ttb]", "Federal Register documents", "C"),
    ("pension-benefit-guaranty-corporation", "pbgc", "Pension Benefit Guaranty Corporation", "pbgc.gov", "[finance, labour, government]", "[pbgc]", "Federal Register documents", "C"),
    ("railroad-retirement-board", "rrb", "U.S. Railroad Retirement Board", "rrb.gov", "[labour, government]", "[rrb, railroad retirement board]", "Federal Register documents", "D"),
    ("federal-retirement-thrift-investment-board", "frtib", "Federal Retirement Thrift Investment Board", "frtib.gov", "[finance, government]", "[frtib, thrift savings plan, tsp]", "Federal Register documents", "D"),
    ("export-import-bank", "exim", "Export-Import Bank of the United States", "exim.gov", "[finance, trade, government]", "[exim, ex-im bank, export-import bank]", "Federal Register documents", "C"),
    ("u-s-international-development-finance-corporation", "dfc", "U.S. International Development Finance Corporation", "dfc.gov", "[finance, international, government]", "[dfc]", "Federal Register documents", "C"),
    ("government-national-mortgage-association", "ginnie-mae", "Ginnie Mae", "ginniemae.gov", "[finance, real-estate, government]", "[ginnie mae, gnma]", "Federal Register documents", "D"),
    # ── trade & commerce ──
    ("commerce-department", "us-commerce", "U.S. Department of Commerce", "commerce.gov", "[government, commerce]", "[commerce department, doc, department of commerce]", "Federal Register documents", "B"),
    ("international-trade-administration", "ita", "International Trade Administration (U.S. Commerce)", "trade.gov", "[trade, government]", "[ita, international trade administration]", "Federal Register documents (antidumping, countervailing duties)", "B"),
    ("foreign-trade-zones-board", "ita", None, None, None, None, "Foreign-Trade Zones Board documents in the Federal Register", "C"),
    ("international-trade-commission", "usitc", "U.S. International Trade Commission", "usitc.gov", "[trade, government]", "[usitc, itc, international trade commission]", "Federal Register documents (Section 337, injury investigations)", "B"),
    ("trade-representative-office-of-united-states", "ustr", "Office of the U.S. Trade Representative", "ustr.gov", "[trade, politics, government]", "[ustr, trade representative]", "Federal Register documents (Section 301, tariffs)", "A"),
    ("patent-and-trademark-office", "uspto", "U.S. Patent and Trademark Office", "uspto.gov", "[legal, technology, government]", "[uspto, pto, patent office]", "Federal Register documents", "B"),
    ("economic-development-administration", "eda", "Economic Development Administration", "eda.gov", "[government, commerce]", "[eda]", "Federal Register documents", "D"),
    ("census-bureau", "census", None, None, None, None, "Federal Register documents", "C"),
    ("economic-analysis-bureau", "bea", None, None, None, None, "Federal Register documents", "D"),
    ("national-institute-of-standards-and-technology", "nist", None, None, None, None, "Federal Register documents", "C"),
    ("national-oceanic-and-atmospheric-administration", "noaa", None, None, None, None, "Federal Register documents (fisheries, marine rules)", "C"),
    ("trade-and-development-agency", "ustda", "U.S. Trade and Development Agency", "ustda.gov", "[trade, international, government]", "[ustda]", "Federal Register documents", "D"),
    # ── health ──
    ("health-and-human-services-department", "hhs", "U.S. Department of Health and Human Services", "hhs.gov", "[health, government]", "[hhs, department of health and human services]", "Federal Register documents", "B"),
    ("food-and-drug-administration", "fda", None, None, None, None, "Federal Register documents", "B"),
    ("centers-for-disease-control-and-prevention", "cdc", None, None, None, None, "Federal Register documents", "C"),
    ("centers-for-medicare-medicaid-services", "cms", None, None, None, None, "Federal Register documents (payment rules, coverage)", "B"),
    ("national-institutes-of-health", "nih", None, None, None, None, "Federal Register documents", "C"),
    ("substance-abuse-and-mental-health-services-administration", "samhsa", None, None, None, None, "Federal Register documents", "D"),
    ("health-resources-and-services-administration", "hrsa", "Health Resources and Services Administration", "hrsa.gov", "[health, government]", "[hrsa]", "Federal Register documents", "D"),
    ("agency-for-healthcare-research-and-quality", "ahrq", "Agency for Healthcare Research and Quality", "ahrq.gov", "[health, research, government]", "[ahrq]", "Federal Register documents", "D"),
    ("indian-health-service", "ihs", "Indian Health Service", "ihs.gov", "[health, government]", "[ihs, indian health service]", "Federal Register documents", "D"),
    ("children-and-families-administration", "acf", "Administration for Children and Families", "acf.hhs.gov", "[health, government]", "[acf]", "Federal Register documents", "D"),
    ("refugee-resettlement-office", "acf", None, None, None, None, "Office of Refugee Resettlement documents in the Federal Register", "D"),
    ("community-living-administration", "acl", "Administration for Community Living", "acl.gov", "[health, government]", "[acl, administration for community living]", "Federal Register documents", "D"),
    ("strategic-preparedness-and-response-administration", "aspr", "Administration for Strategic Preparedness and Response", "aspr.hhs.gov", "[health, government]", "[aspr]", "Federal Register documents", "C"),
    ("inspector-general-office-health-and-human-services-department", "hhs-oig", "HHS Office of Inspector General", "oig.hhs.gov", "[health, legal, government]", "[hhs oig, oig hhs]", "Federal Register documents", "D"),
    ("national-library-of-medicine", "nlm", "National Library of Medicine", "nlm.nih.gov", "[health, research, government]", "[nlm, national library of medicine]", "Federal Register documents", "D"),
    # ── food, agriculture, consumer safety ──
    ("agriculture-department", "usda", None, None, None, None, "Federal Register documents (USDA)", "C"),
    ("food-safety-and-inspection-service", "fsis", "Food Safety and Inspection Service (USDA)", "fsis.usda.gov", "[food, consumer-safety, government]", "[fsis]", "Federal Register documents", "B"),
    ("animal-and-plant-health-inspection-service", "aphis", "Animal and Plant Health Inspection Service (USDA)", "aphis.usda.gov", "[agriculture, food, government]", "[aphis]", "Federal Register documents", "B"),
    ("agricultural-marketing-service", "usda-ams", "Agricultural Marketing Service (USDA)", "ams.usda.gov", "[agriculture, food, government]", "[ams, agricultural marketing service]", "Federal Register documents", "C"),
    ("food-and-nutrition-service", "usda-fns", "Food and Nutrition Service (USDA)", "fns.usda.gov", "[food, government]", "[fns, food and nutrition service, snap]", "Federal Register documents", "C"),
    ("forest-service", "us-forest-service", "U.S. Forest Service", "fs.usda.gov", "[climate, agriculture, government]", "[forest service, usfs]", "Federal Register documents", "C"),
    ("farm-service-agency", "usda-fsa", "Farm Service Agency (USDA)", "fsa.usda.gov", "[agriculture, government]", "[farm service agency]", "Federal Register documents", "D"),
    ("commodity-credit-corporation", "usda-fsa", None, None, None, None, "Commodity Credit Corporation documents in the Federal Register", "D"),
    ("risk-management-agency", "usda-rma", "Risk Management Agency (USDA crop insurance)", "rma.usda.gov", "[agriculture, finance, government]", "[rma, risk management agency, federal crop insurance]", "Federal Register documents", "D"),
    ("federal-crop-insurance-corporation", "usda-rma", "Risk Management Agency (USDA crop insurance)", "rma.usda.gov", "[agriculture, finance, government]", "[rma, risk management agency, federal crop insurance]", "Federal Crop Insurance Corporation documents in the Federal Register", "D"),
    ("natural-resources-conservation-service", "usda-nrcs", "Natural Resources Conservation Service (USDA)", "nrcs.usda.gov", "[agriculture, climate, government]", "[nrcs]", "Federal Register documents", "D"),
    ("rural-utilities-service", "usda-rural-development", "USDA Rural Development", "rd.usda.gov", "[agriculture, telecom, energy, government]", "[rural development, rural utilities service, rus]", "Rural Utilities Service documents in the Federal Register", "D"),
    ("rural-housing-service", "usda-rural-development", None, None, None, None, "Rural Housing Service documents in the Federal Register", "D"),
    ("rural-business-cooperative-service", "usda-rural-development", None, None, None, None, "Rural Business-Cooperative Service documents in the Federal Register", "D"),
    ("economic-research-service", "usda-ers", "Economic Research Service (USDA)", "ers.usda.gov", "[agriculture, statistics, government]", "[ers, economic research service]", "Federal Register documents", "D"),
    ("national-agricultural-statistics-service", "usda-nass", "National Agricultural Statistics Service (USDA)", "nass.usda.gov", "[agriculture, statistics, government]", "[nass]", "Federal Register documents", "D"),
    ("foreign-agricultural-service", "usda-fas", "Foreign Agricultural Service (USDA)", "fas.usda.gov", "[agriculture, trade, government]", "[foreign agricultural service]", "Federal Register documents", "D"),
    ("national-institute-of-food-and-agriculture", "usda-nifa", "National Institute of Food and Agriculture (USDA)", "nifa.usda.gov", "[agriculture, research, government]", "[nifa]", "Federal Register documents", "D"),
    ("agricultural-research-service", "usda-ars", "Agricultural Research Service (USDA)", "ars.usda.gov", "[agriculture, research, government]", "[agricultural research service]", "Federal Register documents", "D"),
    ("consumer-product-safety-commission", "cpsc", None, None, None, None, "Federal Register documents", "B"),
    ("chemical-safety-and-hazard-investigation-board", "csb", "U.S. Chemical Safety and Hazard Investigation Board", "csb.gov", "[consumer-safety, energy, government]", "[csb, chemical safety board]", "Federal Register documents", "D"),
    # ── environment, energy, interior ──
    ("environmental-protection-agency", "epa", None, None, None, None, "Federal Register documents", "B"),
    ("energy-department", "us-doe", None, None, None, None, "Federal Register documents (DOE)", "B"),
    ("energy-efficiency-and-renewable-energy-office", "us-doe", None, None, None, None, "EERE documents in the Federal Register (efficiency standards)", "C"),
    ("national-nuclear-security-administration", "us-doe", None, None, None, None, "NNSA documents in the Federal Register", "C"),
    ("energy-information-administration", "eia", None, None, None, None, "Federal Register documents", "D"),
    ("federal-energy-regulatory-commission", "ferc", None, None, None, None, "Federal Register documents", "B"),
    ("nuclear-regulatory-commission", "us-nrc", None, None, None, None, "Federal Register documents", "B"),
    ("defense-nuclear-facilities-safety-board", "dnfsb", "Defense Nuclear Facilities Safety Board", "dnfsb.gov", "[energy, government]", "[dnfsb]", "Federal Register documents", "D"),
    ("bonneville-power-administration", "bpa", "Bonneville Power Administration", "bpa.gov", "[energy, government]", "[bpa, bonneville]", "Federal Register documents", "D"),
    ("western-area-power-administration", "wapa", "Western Area Power Administration", "wapa.gov", "[energy, government]", "[wapa]", "Federal Register documents", "D"),
    ("tennessee-valley-authority", "tva", "Tennessee Valley Authority", "tva.com", "[energy, government]", "[tva]", "Federal Register documents", "D"),
    ("interior-department", "us-doi", "U.S. Department of the Interior", "doi.gov", "[government, energy, climate]", "[doi, interior department, department of the interior]", "Federal Register documents", "B"),
    ("fish-and-wildlife-service", "fws", "U.S. Fish and Wildlife Service", "fws.gov", "[climate, government]", "[fws, fish and wildlife service, endangered species]", "Federal Register documents", "B"),
    ("land-management-bureau", "blm", "Bureau of Land Management", "blm.gov", "[energy, climate, government]", "[blm, bureau of land management]", "Federal Register documents", "B"),
    ("national-park-service", "nps", "National Park Service", "nps.gov", "[travel, climate, government]", "[nps, national park service]", "Federal Register documents", "C"),
    ("ocean-energy-management-bureau", "boem", "Bureau of Ocean Energy Management", "boem.gov", "[energy, government]", "[boem, offshore leasing]", "Federal Register documents", "B"),
    ("safety-and-environmental-enforcement-bureau", "bsee", "Bureau of Safety and Environmental Enforcement", "bsee.gov", "[energy, government]", "[bsee]", "Federal Register documents", "C"),
    ("indian-affairs-bureau", "bia", "Bureau of Indian Affairs", "bia.gov", "[government]", "[bia, bureau of indian affairs]", "Federal Register documents", "C"),
    ("reclamation-bureau", "usbr", "Bureau of Reclamation", "usbr.gov", "[energy, climate, government]", "[usbr, bureau of reclamation]", "Federal Register documents", "D"),
    ("surface-mining-reclamation-and-enforcement-office", "osmre", "Office of Surface Mining Reclamation and Enforcement", "osmre.gov", "[energy, government]", "[osmre]", "Federal Register documents", "D"),
    ("natural-resources-revenue-office", "onrr", "Office of Natural Resources Revenue", "onrr.gov", "[energy, finance, government]", "[onrr]", "Federal Register documents", "D"),
    ("geological-survey", "usgs", None, None, None, None, "Federal Register documents", "D"),
    ("national-indian-gaming-commission", "nigc", "National Indian Gaming Commission", "nigc.gov", "[gaming, government]", "[nigc]", "Federal Register documents", "D"),
    # ── labour ──
    ("labor-department", "us-dol", None, None, None, None, "Federal Register documents (DOL)", "B"),
    ("employee-benefits-security-administration", "us-dol", None, None, None, None, "EBSA documents in the Federal Register", "C"),
    ("employment-and-training-administration", "us-dol", None, None, None, None, "ETA documents in the Federal Register (H-1B/H-2 labor certification)", "C"),
    ("wage-and-hour-division", "us-dol", None, None, None, None, "Wage and Hour Division documents in the Federal Register", "C"),
    ("federal-contract-compliance-programs-office", "us-dol", None, None, None, None, "OFCCP documents in the Federal Register", "D"),
    ("workers-compensation-programs-office", "us-dol", None, None, None, None, "OWCP documents in the Federal Register", "D"),
    ("veterans-employment-and-training-service", "us-dol", None, None, None, None, "VETS documents in the Federal Register", "D"),
    ("occupational-safety-and-health-administration", "osha", "Occupational Safety and Health Administration", "osha.gov", "[labour, consumer-safety, government]", "[osha]", "Federal Register documents", "B"),
    ("mine-safety-and-health-administration", "msha", "Mine Safety and Health Administration", "msha.gov", "[labour, energy, government]", "[msha]", "Federal Register documents", "C"),
    ("occupational-safety-and-health-review-commission", "oshrc", "Occupational Safety and Health Review Commission", "oshrc.gov", "[labour, legal, government]", "[oshrc]", "Federal Register documents", "D"),
    ("labor-statistics-bureau", "bls", None, None, None, None, "Federal Register documents", "D"),
    ("national-labor-relations-board", "nlrb", None, None, None, None, "Federal Register documents", "C"),
    ("equal-employment-opportunity-commission", "eeoc", None, None, None, None, "Federal Register documents", "C"),
    ("national-mediation-board", "nmb", "National Mediation Board", "nmb.gov", "[labour, transport, government]", "[nmb, national mediation board]", "Federal Register documents", "D"),
    ("federal-mediation-and-conciliation-service", "fmcs", "Federal Mediation and Conciliation Service", "fmcs.gov", "[labour, government]", "[fmcs]", "Federal Register documents", "D"),
    ("federal-labor-relations-authority", "flra", "Federal Labor Relations Authority", "flra.gov", "[labour, government]", "[flra]", "Federal Register documents", "D"),
    ("merit-systems-protection-board", "mspb", "U.S. Merit Systems Protection Board", "mspb.gov", "[labour, legal, government]", "[mspb]", "Federal Register documents", "D"),
    ("personnel-management-office", "opm", "U.S. Office of Personnel Management", "opm.gov", "[labour, government]", "[opm, office of personnel management]", "Federal Register documents", "B"),
    # ── justice, security, immigration ──
    ("justice-department", "us-doj", None, None, None, None, "Federal Register documents (DOJ)", "B"),
    ("antitrust-division", "us-doj", None, None, None, None, "Antitrust Division documents in the Federal Register (Tunney Act notices)", "B"),
    ("executive-office-for-immigration-review", "us-doj", None, None, None, None, "EOIR documents in the Federal Register", "C"),
    ("federal-bureau-of-investigation", "fbi", None, None, None, None, "Federal Register documents", "D"),
    ("drug-enforcement-administration", "dea", "Drug Enforcement Administration", "dea.gov", "[health, legal, government]", "[dea, drug enforcement administration]", "Federal Register documents (scheduling, registrations)", "B"),
    ("alcohol-tobacco-firearms-and-explosives-bureau", "atf", "Bureau of Alcohol, Tobacco, Firearms and Explosives", "atf.gov", "[legal, consumer-safety, government]", "[atf]", "Federal Register documents", "C"),
    ("prisons-bureau", "bop", "Federal Bureau of Prisons", "bop.gov", "[legal, government]", "[bop, bureau of prisons]", "Federal Register documents", "D"),
    ("united-states-marshals-service", "usms", "U.S. Marshals Service", "usmarshals.gov", "[legal, government]", "[usms, marshals service]", "Federal Register documents", "D"),
    ("justice-programs-office", "ojp", "Office of Justice Programs", "ojp.gov", "[legal, government]", "[ojp, office of justice programs]", "Federal Register documents", "D"),
    ("united-states-sentencing-commission", "ussc", "U.S. Sentencing Commission", "ussc.gov", "[legal, government]", "[ussc, sentencing commission]", "Federal Register documents", "C"),
    ("u-s-citizenship-and-immigration-services", "uscis", None, None, None, None, "Federal Register documents", "B"),
    ("u-s-customs-and-border-protection", "us-cbp", None, None, None, None, "Federal Register documents", "B"),
    ("u-s-immigration-and-customs-enforcement", "us-ice", None, None, None, None, "Federal Register documents", "C"),
    ("federal-emergency-management-agency", "fema", None, None, None, None, "Federal Register documents (disaster declarations, flood maps)", "C"),
    ("secret-service", "us-secret-service", "U.S. Secret Service", "secretservice.gov", "[government, cyber]", "[secret service, usss]", "Federal Register documents", "D"),
    ("central-intelligence-agency", "cia", "Central Intelligence Agency", "cia.gov", "[government, politics]", "[cia]", "Federal Register documents", "D"),
    ("national-intelligence-office-of-the-national-director", "odni", "Office of the Director of National Intelligence", "dni.gov", "[government, politics, cyber]", "[odni, dni]", "Federal Register documents", "D"),
    ("privacy-and-civil-liberties-oversight-board", "pclob", "Privacy and Civil Liberties Oversight Board", "pclob.gov", "[legal, cyber, government]", "[pclob]", "Federal Register documents", "D"),
    ("civil-rights-commission", "usccr", "U.S. Commission on Civil Rights", "usccr.gov", "[legal, politics, government]", "[usccr, civil rights commission]", "Federal Register documents", "D"),
    ("state-department", "us-state-department", None, None, None, None, "Federal Register documents", "B"),
    ("agency-for-international-development", "usaid", "U.S. Agency for International Development", "usaid.gov", "[international, government]", "[usaid]", "Federal Register documents", "C"),
    ("millennium-challenge-corporation", "mcc", "Millennium Challenge Corporation", "mcc.gov", "[international, government]", "[mcc, millennium challenge corporation]", "Federal Register documents", "D"),
    ("united-states-agency-for-global-media", "usagm", "U.S. Agency for Global Media", "usagm.gov", "[media, government]", "[usagm, voice of america, voa]", "Federal Register documents", "D"),
    ("peace-corps", "peace-corps", "Peace Corps", "peacecorps.gov", "[international, government]", "[peace corps]", "Federal Register documents", "D"),
    ("u-s-china-economic-and-security-review-commission", "uscc", "U.S.-China Economic and Security Review Commission", "uscc.gov", "[politics, trade, government]", "[uscc]", "Federal Register documents", "D"),
    # ── defense ──
    ("defense-department", "us-dod", None, None, None, None, "Federal Register documents (DoD)", "B"),
    ("defense-acquisition-regulations-system", "us-dod", None, None, None, None, "DFARS documents in the Federal Register", "C"),
    ("army-department", "us-army", "U.S. Army", "army.mil", "[government]", "[us army, army]", "Federal Register documents", "D"),
    ("navy-department", "us-navy", "U.S. Navy", "navy.mil", "[government]", "[us navy, navy]", "Federal Register documents", "D"),
    ("air-force-department", "us-air-force", "U.S. Air Force", "af.mil", "[government, space]", "[us air force, air force, usaf]", "Federal Register documents", "D"),
    ("engineers-corps", "usace", "U.S. Army Corps of Engineers", "usace.army.mil", "[infrastructure, climate, government]", "[usace, army corps of engineers]", "Federal Register documents", "C"),
    ("defense-logistics-agency", "dla", "Defense Logistics Agency", "dla.mil", "[logistics, government]", "[dla]", "Federal Register documents", "D"),
    ("national-security-agency-central-security-service", "nsa", "National Security Agency", "nsa.gov", "[cyber, government]", "[nsa]", "Federal Register documents", "D"),
    ("national-geospatial-intelligence-agency", "nga", "National Geospatial-Intelligence Agency", "nga.mil", "[space, government]", "[nga]", "Federal Register documents", "D"),
    ("defense-information-systems-agency", "disa", "Defense Information Systems Agency", "disa.mil", "[cyber, telecom, government]", "[disa]", "Federal Register documents", "D"),
    # ── transport (not already in 46) ──
    ("federal-highway-administration", "fhwa", "Federal Highway Administration", "fhwa.dot.gov", "[transport, infrastructure, government]", "[fhwa]", "Federal Register documents", "C"),
    ("federal-transit-administration", "fta", "Federal Transit Administration", "transit.dot.gov", "[transport, government]", "[fta, federal transit administration]", "Federal Register documents", "C"),
    ("maritime-administration", "marad", "U.S. Maritime Administration", "maritime.dot.gov", "[transport, logistics, government]", "[marad]", "Federal Register documents", "C"),
    ("transportation-statistics-bureau", "bts", "Bureau of Transportation Statistics", "bts.gov", "[transport, statistics, government]", "[bts]", "Federal Register documents", "D"),
    ("commercial-space-transportation-office", "faa", None, None, None, None, "Office of Commercial Space Transportation documents in the Federal Register", "C"),
    ("national-railroad-passenger-corporation", "amtrak", None, None, None, None, "Federal Register documents", "D"),
    ("national-aeronautics-and-space-administration", "nasa", None, None, None, None, "Federal Register documents", "C"),
    # ── executive office, regulatory process, records ──
    ("executive-office-of-the-president", "white-house", None, None, None, None, "Executive Office of the President documents in the Federal Register", "B"),
    ("management-and-budget-office", "white-house", None, None, None, None, "OMB documents in the Federal Register", "B"),
    ("science-and-technology-policy-office", "white-house", None, None, None, None, "OSTP documents in the Federal Register", "B"),
    ("council-on-environmental-quality", "white-house", None, None, None, None, "CEQ documents in the Federal Register", "C"),
    ("office-of-national-drug-control-policy", "white-house", None, None, None, None, "ONDCP documents in the Federal Register", "D"),
    ("office-of-the-national-cyber-director", "white-house", None, None, None, None, "ONCD documents in the Federal Register", "C"),
    ("federal-register-office", "federal-register", None, None, None, None, "Office of the Federal Register documents", "D"),
    ("regulatory-information-service-center", "reginfo", "Regulatory Information Service Center (Unified Agenda)", "reginfo.gov", "[government, legal]", "[risc, reginfo, unified agenda]", "Federal Register documents", "D"),
    ("federal-acquisition-regulation-system", "far-council", "Federal Acquisition Regulation (FAR) Council", "acquisition.gov", "[government, legal]", "[far, federal acquisition regulation]", "Federal Register documents", "C"),
    ("general-services-administration", "gsa", "U.S. General Services Administration", "gsa.gov", "[government, infrastructure]", "[gsa]", "Federal Register documents", "C"),
    ("national-archives-and-records-administration", "nara", "National Archives and Records Administration", "archives.gov", "[government, legal]", "[nara, national archives]", "Federal Register documents", "D"),
    ("information-security-oversight-office", "nara", None, None, None, None, "ISOO documents in the Federal Register", "D"),
    ("government-publishing-office", "gpo", "U.S. Government Publishing Office", "gpo.gov", "[government]", "[gpo, government publishing office]", "Federal Register documents", "D"),
    ("library-of-congress", "library-of-congress", "Library of Congress", "loc.gov", "[education, government]", "[loc, library of congress]", "Federal Register documents", "D"),
    ("copyright-office-library-of-congress", "us-copyright-office", None, None, None, None, "Federal Register documents", "C"),
    ("copyright-royalty-board", "crb", "Copyright Royalty Board", "crb.gov", "[media, legal, government]", "[crb, copyright royalty board]", "Federal Register documents", "C"),
    ("postal-service", "usps", None, None, None, None, "Federal Register documents", "C"),
    ("postal-regulatory-commission", "prc-postal", "Postal Regulatory Commission", "prc.gov", "[logistics, government]", "[prc, postal regulatory commission]", "Federal Register documents", "C"),
    ("federal-election-commission", "fec", None, None, None, None, "Federal Register documents", "C"),
    ("election-assistance-commission", "eac", None, None, None, None, "Federal Register documents", "D"),
    ("government-ethics-office", "oge", None, None, None, None, "Federal Register documents", "D"),
    ("special-counsel-office", "us-osc", None, None, None, None, "Federal Register documents", "D"),
    ("social-security-administration", "ssa", "Social Security Administration", "ssa.gov", "[government, labour]", "[ssa, social security]", "Federal Register documents", "B"),
    ("education-department", "us-ed", "U.S. Department of Education", "ed.gov", "[education, government]", "[ed, department of education]", "Federal Register documents", "B"),
    ("veterans-affairs-department", "us-va", None, None, None, None, "Federal Register documents", "C"),
    ("housing-and-urban-development-department", "hud", None, None, None, None, "Federal Register documents", "B"),
    ("small-business-administration", "sba", None, None, None, None, "Federal Register documents", "C"),
    ("national-science-foundation", "nsf", None, None, None, None, "Federal Register documents", "D"),
    ("national-endowment-for-the-arts", "nea", "National Endowment for the Arts", "arts.gov", "[entertainment, government]", "[nea, national endowment for the arts]", "Federal Register documents", "D"),
    ("national-endowment-for-the-humanities", "neh", "National Endowment for the Humanities", "neh.gov", "[education, government]", "[neh]", "Federal Register documents", "D"),
    ("institute-of-museum-and-library-services", "imls", "Institute of Museum and Library Services", "imls.gov", "[education, government]", "[imls]", "Federal Register documents", "D"),
    ("smithsonian-institution", "smithsonian", "Smithsonian Institution", "si.edu", "[education, research, government]", "[smithsonian]", "Federal Register documents", "D"),
    ("corporation-for-national-and-community-service", "americorps", "AmeriCorps", "americorps.gov", "[government]", "[americorps, cncs]", "Federal Register documents", "D"),
    ("selective-service-system", "sss", "Selective Service System", "sss.gov", "[government]", "[selective service]", "Federal Register documents", "D"),
    ("legal-services-corporation", "lsc", "Legal Services Corporation", "lsc.gov", "[legal, government]", "[lsc, legal services corporation]", "Federal Register documents", "D"),
    ("administrative-conference-of-the-united-states", "acus", "Administrative Conference of the United States", "acus.gov", "[legal, government]", "[acus]", "Federal Register documents", "D"),
    ("architectural-and-transportation-barriers-compliance-board", "access-board", "U.S. Access Board", "access-board.gov", "[standards, government]", "[access board, atbcb]", "Federal Register documents", "D"),
    ("federal-permitting-improvement-steering-council", "permitting-council", "Federal Permitting Improvement Steering Council", "permits.performance.gov", "[infrastructure, energy, government]", "[permitting council, fpisc]", "Federal Register documents", "D"),
    ("first-responder-network-authority", "firstnet-authority", "First Responder Network Authority (FirstNet)", "firstnet.gov", "[telecom, government]", "[firstnet]", "Federal Register documents", "D"),
    ("international-joint-commission-united-states-and-canada", "ijc", "International Joint Commission (Canada–United States)", "ijc.org", "[international, climate, government]", "[ijc, international joint commission]", "Federal Register documents", "D"),
    ("judicial-conference-of-the-united-states", "us-courts", None, None, None, None, "Judicial Conference documents in the Federal Register", "D"),
    ("administrative-office-of-united-states-courts", "us-courts", None, None, None, None, "Administrative Office documents in the Federal Register", "D"),
    ("government-accountability-office", "gao", None, None, None, None, "Federal Register documents", "D"),
    ("congressional-budget-office", "cbo", None, None, None, None, "Federal Register documents", "D"),
]


def url_for(slug: str) -> str:
    return f"https://www.federalregister.gov/api/v1/documents.json?conditions[agencies][]={slug}&order=newest&per_page=40"


def registry_ids() -> tuple[set[str], set[str], set[str]]:
    ids, urls, extended = set(), set(), set()
    for f in ["config/sources.yaml"] + sorted(glob.glob("config/sources.d/*.yaml")):
        if f.endswith("49b-us-agencies-fedreg.yaml"):
            continue
        d = yaml.safe_load(open(f)) or {}
        for s in d.get("sources", []):
            (extended if s.get("extend") else ids).add(s["id"])
            for sen in s.get("sensors") or []:
                urls.add(sen.get("url", ""))
    return ids, urls, extended


def check(argv: list[str]) -> None:
    max_age = int(argv[argv.index("--max-age") + 1]) if "--max-age" in argv else 730
    slugs = {a["slug"] for a in json.load(open(AGENCIES))}
    try:
        out = json.load(open(CHECK))
    except FileNotFoundError:
        out = {}
    sess = requests.Session()
    for row in T:
        slug = row[0]
        if slug not in slugs:
            print(f"{slug}: not an agency slug", file=sys.stderr)
            out[slug] = {"ok": False, "why": "unknown slug"}
            continue
        if slug in out and "count" in out[slug]:
            continue
        r = sess.get(url_for(slug).replace("per_page=40", "per_page=1"), headers={"User-Agent": UA}, timeout=30)
        time.sleep(0.5)
        if r.status_code != 200:
            out[slug] = {"ok": False, "why": f"http {r.status_code}"}
            print(f"{slug}: HTTP {r.status_code}", file=sys.stderr)
            continue
        j = r.json()
        res = j.get("results") or []
        latest = res[0]["publication_date"] if res else None
        age = (dt.date.today() - dt.date.fromisoformat(latest)).days if latest else None
        ok = bool(res) and age is not None and age <= max_age
        out[slug] = {"ok": ok, "count": j.get("count", 0), "latest": latest, "age_days": age}
        print(f"{'OK  ' if ok else 'DROP'} {slug:70} count={j.get('count', 0):>7} latest={latest}", file=sys.stderr)
        json.dump(out, open(CHECK, "w"), indent=1)
    json.dump(out, open(CHECK, "w"), indent=1)
    print(f"{sum(1 for v in out.values() if v.get('ok'))}/{len(out)} filters usable", file=sys.stderr)


def emit() -> None:
    chk = json.load(open(CHECK))
    ids, urls, _ = registry_ids()
    print("# config/sources.d/49b-us-agencies-fedreg.yaml — Federal Register documents per US federal agency (2026-09-11,")
    print("# generated by scripts/gen-fedreg.py from api/v1/agencies; every filter checked live for recent documents). Rules,")
    print("# proposed rules and notices of ~120 agencies not yet covered by 44-governments.yaml / 46-transport…: financial")
    print("# regulators, trade, health, agriculture, environment/energy/interior, labour, justice/security, defense, transport,")
    print("# the Executive Office and independent boards. Existing agency ids are extended; sub-offices without a site of their")
    print("# own (OMB, OSTP, DOL agencies, Treasury offices, DOJ divisions…) become extra sensors on the parent department.")
    print("# Federal Register API: 1000 req/h, no key; `jsonlist` keyed by document_number.")
    print("sources:")
    groups: dict[str, list[tuple]] = {}
    order: list[str] = []
    for row in T:
        slug, sid = row[0], row[1]
        if not chk.get(slug, {}).get("ok"):
            print(f"# dropped {slug}: {chk.get(slug)}", file=sys.stderr)
            continue
        if url_for(slug) in urls:
            print(f"# dropped {slug}: URL already in registry", file=sys.stderr)
            continue
        if sid not in groups:
            order.append(sid)
        groups.setdefault(sid, []).append(row)
    n_ext = n_new = n_sen = 0
    for sid in order:
        rows = groups[sid]
        decl = next((r for r in rows if r[2]), None)
        sensors = [f'      - {{ name: {r[6]}, url: "{url_for(r[0])}", type: REST_API, connector: jsonlist, tier: {r[7]}, {CFG} }}' for r in rows]
        if sid in ids or decl is None:
            if decl is not None and sid in ids:
                print(f"# {sid} already declared elsewhere — extending instead of redeclaring", file=sys.stderr)
            if sid not in ids:
                print(f"# ERROR {sid} is not declared anywhere and has no declaration row", file=sys.stderr)
                continue
            print(f"  - id: {sid}")
            print("    extend: true")
            print("    country: US")
            print("    sensors:")
            n_ext += 1
        else:
            _, _, name, domain, cats, aliases, _, tier = decl
            print(f"  - id: {sid}")
            print(f"    name: {json.dumps(name, ensure_ascii=False)}")
            print(f"    domain: {domain}")
            print(f"    homepage: https://{'www.' + domain if domain.count('.') == 1 else domain}")
            print(f"    categories: {cats}")
            print("    tier: B")
            print(f"    aliases: {aliases}")
            print("    country: US")
            print(f"    notes: \"Federal Register agency slug: {', '.join(r[0] for r in rows)}\"")
            print("    discover: { rss: true }")
            print("    sensors:")
            n_new += 1
        for s in sensors:
            print(s)
            n_sen += 1
    print(f"{n_ext} extended, {n_new} new, {n_sen} sensors", file=sys.stderr)


if __name__ == "__main__":
    if sys.argv[1:2] == ["check"]:
        check(sys.argv[2:])
    else:
        emit()