TypeScript 55.4%
Python 43.2%
SQL 1.2%
1#!/usr/bin/env python32"""Generate config/sources.d/51-world-governments-regulators.yaml from a probed candidate list.34Usage: python3 scripts/gen-world-gov.py probe # curl every candidate → /tmp/wg-probe.json5 python3 scripts/gen-world-gov.py write # write the fragment from surviving candidates6Candidates live in scripts/gen_world_gov_data.py (SOURCES + CANDS). A candidate survives the probe when7the response is 200 and looks like what the connector expects (feed with items / JSON array / substantive HTML).8For every (source, sensor-name) only the first surviving alternative is kept.9"""10import json, os, re, subprocess, sys, glob11from concurrent.futures import ThreadPoolExecutor12from urllib.parse import urlparse1314sys.path.insert(0, os.path.dirname(__file__))15from gen_world_gov_data import SOURCES, CANDS # noqa: E40216import gen_world_gov_data2, gen_world_gov_data3, gen_world_gov_data4 # noqa: E402,F401 (register candidates)1718ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))19OUT = os.path.join(ROOT, "config/sources.d/51-world-governments-regulators.yaml")20PROBE = "/tmp/wg-probe.json"21UA = "WebSensorBot/0.1 (+https://www.websensor.io/bot; contact@websensor.io)"222324def expand(url):25 import datetime26 def rep(m):27 n, u = m.group(1), m.group(2)28 d = datetime.datetime.utcnow()29 if n:30 d -= datetime.timedelta(**{{"h": "hours", "d": "days", "m": "minutes"}[u]: int(n)})31 return d.strftime("%Y-%m-%dT%H:%M:%S.000")32 return re.sub(r"\{now(?:-(\d+)([hmd]))?\}", rep, url)333435def probe_one(c):36 url = expand(c["url"])37 body = f"/tmp/wg-body-{abs(hash(url))}.bin"38 try:39 r = subprocess.run(["curl", "-sSL", "--compressed", "--max-time", "30", "-A", UA, "-o", body,40 "-w", "%{http_code}\t%{content_type}\t%{url_effective}", url],41 capture_output=True, text=True, timeout=45)42 parts = r.stdout.strip().split("\t")43 code = int(parts[0] or 0); ctype = parts[1] if len(parts) > 1 else ""; final = parts[2] if len(parts) > 2 else ""44 except Exception as e: # noqa: BLE00145 return {**c, "code": 0, "note": str(e)[:80], "ok": False}46 data = b""47 if os.path.exists(body):48 data = open(body, "rb").read(); os.remove(body)49 txt = data[:400000].decode("utf-8", "ignore")50 kind = c["kind"]; ok = False; note = ""51 if code == 200:52 low = txt[:2000].lower()53 if kind == "rss":54 if "<rss" in low or "<feed" in low or "<rdf:rdf" in low or low.lstrip().startswith("{"):55 n = txt.count("<item") + txt.count("<entry")56 ok = n > 0; note = f"items={n}"57 else:58 note = "not-a-feed"59 elif kind == "json":60 try:61 j = json.loads(txt)62 arr = j63 for p in (c.get("cfg", {}).get("itemsPath") or "").split("."):64 if p: arr = arr[int(p)] if isinstance(arr, list) else arr[p]65 ok = isinstance(arr, list) and len(arr) > 0; note = f"items={len(arr) if isinstance(arr, list) else '?'}"66 except Exception as e: # noqa: BLE00167 note = f"json-error {str(e)[:40]}"68 else: # http (html)69 text = re.sub(r"<script.*?</script>|<style.*?</style>", " ", txt, flags=re.S | re.I)70 text = re.sub(r"<[^>]+>", " ", text); text = re.sub(r"\s+", " ", text)71 ok = len(text) > 1500 and "<html" in low; note = f"text={len(text)}"72 return {**c, "code": code, "ctype": ctype[:40], "final": final, "ok": ok, "note": note}737475def do_probe():76 with ThreadPoolExecutor(max_workers=16) as ex:77 res = list(ex.map(probe_one, CANDS))78 json.dump(res, open(PROBE, "w"), indent=1)79 okn = sum(r["ok"] for r in res)80 print(f"probed {len(res)} → ok {okn}")81 for r in res:82 print(f"{'OK ' if r['ok'] else '-- '} {r['code']:>3} {r['id']:28s} {r['name'][:28]:28s} {r['url'][:90]} {r['note']}")838485def registry_index():86 import yaml87 ids, urls, hosts = {}, set(), {}88 for f in [os.path.join(ROOT, "config/sources.yaml")] + sorted(glob.glob(os.path.join(ROOT, "config/sources.d/*.yaml"))):89 if f.endswith("51-world-governments-regulators.yaml"):90 continue91 d = yaml.safe_load(open(f))92 for s in d.get("sources", []):93 if not s.get("extend"):94 ids[s["id"]] = f95 for sen in s.get("sensors", []) or []:96 urls.add(sen["url"]); hosts.setdefault(urlparse(sen["url"]).hostname, set()).add(s["id"])97 return ids, urls, hosts9899100def q(s):101 return json.dumps(s, ensure_ascii=False)102103104# Sensor ids derive from ASCII letters of the name — give non-Latin names an English equivalent.105NAMES = {106 "новини": "news (Ukrainian)", "новости": "news (Russian)", "новини (VGP)": "news", "документи": "documents", "документы": "documents (Russian)",107 "постанови та розпорядження": "resolutions and orders", "рішення": "decisions", "експрес-випуски": "express releases",108 "оперативна інформація": "operational information", "оперативная информация": "operational information (Russian)",109 "пресс-релизы": "press releases (Russian)", "новые документы": "new documents (Russian)",110 "прессъобщения": "press releases (Bulgarian)", "новини (Bulgarian)": "news", "законопроекти": "bills (Bulgarian)", "последен брой": "latest issue (Bulgarian)",111 "ανακοινώσεις": "announcements (Greek)", "δελτία τύπου": "press releases (Greek)", "νομοσχέδια": "bills (Greek)", "τελευταία ΦΕΚ": "latest gazette issues (Greek)",112 "ข่าวรัฐบาล": "government news (Thai)", "ข่าวประชาสัมพันธ์": "news (Thai)", "ข่าวสาร": "news (Thai)", "ประกาศล่าสุด": "latest announcements (Thai)",113 "ข่าว ก.ล.ต.": "news (Thai)", "ข่าวกระทรวง": "ministry news (Thai)",114 "新聞稿": "press releases (Chinese)", "新聞": "news (Chinese)", "最新消息": "latest news (Chinese)", "最新公報": "latest gazette (Chinese)",115 "新聞資料": "news (Chinese)", "工作动态": "news (Chinese)", "政策法规": "policies and regulations (Chinese)", "总局动态": "news (Chinese)",116 "新闻发布": "press releases (Chinese)", "药品监管动态": "drug regulation news (Chinese)", "最新地震": "latest earthquakes (Chinese)",117 "tin tức": "news (Vietnamese)", "tin tức (VGP)": "news (Vietnamese, VGP)", "văn bản mới": "new legal documents", "công báo mới": "latest gazette issues",118 "hírek": "news (Hungarian)", "irományok": "parliamentary documents", "legfrissebb lapszámok": "latest issues", "gyorstájékoztatók": "first releases (Hungarian)",119 "határozatok": "decisions", "sajtóközlemények": "press releases (Hungarian)",120}121122123def ascii_name(name, lang):124 if name in NAMES:125 return NAMES[name]126 if not re.search(r"[a-z0-9]", name.lower()):127 return f"news ({lang or 'local'})"128 return name129130131def do_write():132 res = json.load(open(PROBE))133 ids, urls, hosts = registry_index()134 bad = set()135 if os.path.exists("/tmp/wg-bad.txt"):136 bad = {l.strip() for l in open("/tmp/wg-bad.txt") if l.strip()}137 keep = {}138 for r in res:139 if not r["ok"] or r["url"] in bad:140 continue141 if r["url"] in urls:142 print("DUP URL skipped:", r["url"]); continue143 k = (r["id"], r["name"])144 if k in keep:145 continue146 keep[k] = r147 by_src = {}148 for (sid, _), r in keep.items():149 by_src.setdefault(sid, []).append(r)150 lines = [151 "# config/sources.d/51-world-governments-regulators.yaml — world governments & regulators (authored 2026-09-11):",152 "# governments, parliaments, official gazettes, statistics offices, data-protection / competition / telecom / financial /",153 "# energy / medicines regulators, transport-safety boards and civil-protection alert feeds for the countries not covered by",154 "# 44-governments.yaml (Benelux, Ireland, Switzerland, Austria, Portugal, the Nordics, Central & Eastern Europe, the Baltics,",155 "# Ukraine, Türkiye, Israel, the Gulf, Egypt, Africa, Latin America, New Zealand, Singapore, Hong Kong, Taiwan, South-East Asia,",156 "# South Asia, China's English portals, Russia's official feeds) plus international bodies missing from 17. Every entry carries",157 "# `country:` (INT for international bodies) and `language:` when the feed is not in English. Every sensor was fetched and",158 "# parsed by apps/engine/src/validate.ts; blocked or client-rendered endpoints are recorded in `notes:`.",159 "sources:",160 ]161 cur_country = None162 secmap = {}163 for m in SOURCES.values():164 if m.get("section") and m["country"] not in secmap:165 secmap[m["country"]] = m["section"]166 for sid, meta in SOURCES.items():167 sens = by_src.get(sid, [])168 if not sens and not meta.get("keep_empty"):169 continue170 if meta.get("extend"):171 if sid not in ids:172 print("EXTEND of unknown id:", sid); continue173 elif sid in ids:174 print("ID COLLISION:", sid, ids[sid]); continue175 if meta["country"] != cur_country:176 cur_country = meta["country"]177 lines.append(f" # ───────────────────────────── {secmap.get(cur_country, cur_country)} ─────────────────────────────")178 lines.append(f" - id: {sid}")179 if meta.get("extend"):180 lines.append(" extend: true")181 else:182 lines.append(f" name: {meta['name']}")183 lines.append(f" domain: {meta['domain']}")184 if meta.get("homepage"):185 lines.append(f" homepage: {meta['homepage']}")186 lines.append(f" categories: [{', '.join(meta['categories'])}]")187 lines.append(f" tier: {meta.get('tier', 'B')}")188 if meta.get("weight"):189 lines.append(f" weight: {meta['weight']}")190 lines.append(f" aliases: [{', '.join(meta['aliases'])}]")191 if meta.get("llm") is False:192 lines.append(" llm: false")193 lines.append(f" country: {q(meta['country']) if meta['country'] in ('NO', 'ON', 'OFF', 'YES') else meta['country']}")194 if meta.get("language"):195 lines.append(f" language: {q(meta['language']) if meta['language'] in ('no', 'on', 'off', 'yes', 'y', 'n') else meta['language']}")196 if meta.get("notes"):197 lines.append(f" notes: {q(meta['notes'])}")198 if sens:199 lines.append(" sensors:")200 for r in sens:201 cfg = r.get("cfg")202 typ = {"rss": "RSS", "json": "REST_API", "http": "HTML"}[r["kind"]]203 if r["kind"] == "rss" and ("atom" in r["url"].lower() or "/feed" in r["url"].lower() and r["url"].endswith("feed")):204 typ = "ATOM" if "atom" in r["url"].lower() else "RSS"205 conn = {"rss": "rss", "json": "jsonlist", "http": "http"}[r["kind"]]206 nm = ascii_name(r["name"], meta.get("language"))207 s = f' - {{ name: {nm}, url: {q(r["url"])}, type: {typ}, connector: {conn}, tier: {r.get("tier") or meta.get("tier", "B")}'208 if cfg:209 s += ", config: " + yaml_flow(cfg)210 s += " }"211 lines.append(s)212 open(OUT, "w").write("\n".join(lines) + "\n")213 n = sum(len(v) for v in by_src.values())214 print(f"wrote {OUT}: {sum(1 for s in SOURCES if by_src.get(s) or SOURCES[s].get('keep_empty'))} sources, {n} sensors")215216217def yaml_flow(o):218 if isinstance(o, dict):219 return "{ " + ", ".join(f"{k}: {yaml_flow(v)}" for k, v in o.items()) + " }"220 if isinstance(o, list):221 return "[" + ", ".join(yaml_flow(v) for v in o) + "]"222 if isinstance(o, bool):223 return "true" if o else "false"224 if isinstance(o, (int, float)):225 return str(o)226 return json.dumps(o, ensure_ascii=False)227228229if __name__ == "__main__":230 {"probe": do_probe, "write": do_write}[sys.argv[1]]()231