SPB Git forge
spboucher.ai ↗

spb/websensor

Public
33commits 1branches 0releases
3.4 MBsize
maindefault branch
10 days agolast push
TypeScript 55.4% Python 43.2% SQL 1.2%

registry: +26 governments/public institutions, +45 news media (validated live); News feed channel; per-source llm switch (news = heuristics only)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 16 days ago (Sep 8, 2026) parent 3bb8d31

10 changed files +798 −7

modified CLAUDE.md +1 −1
@@ -34,7 +34,7 @@ design is `docs/PRODUCT-BRIEF.md`; this file is the working guide for the code.
34 34 redirect, and a reverse proxy to the Next.js app for everything else.
35 35 - `apps/web` — Next.js 16 frontend (:8261, loopback). Pages: live, breaking, explore, sources, entities,
36 36 timelines, silent changes, watchlists, alerts, event detail with diff viewer, health, API docs.
37 −- `config/sources.yaml` — the 200-organization registry (sources, curated sensors, product entities, discovery
37 +- `config/sources.yaml` — the 271-organization registry (incl. governments and news media; news sources run with `llm: false`, heuristics only) (sources, curated sensors, product entities, discovery
38 38 flags, fallbacks). `docs/connectors/*.md` document each connector family.
39 39
40 40 ## Rules
modified README.md +1 −1
@@ -13,7 +13,7 @@ Sources → Sensors/Connectors → Fetch (conditional) → Snapshot (immutable)
13 13 → Redis stream → WebSocket gateway → www.websensor.io
14 14 ```
15 15
16 −- 200 organizations, ~290 curated sensors at launch (`config/sources.yaml`), plus validated discovery.
16 +- 271 organizations (tech, cloud, cyber, finance, government, health, science, automotive, commerce, internet, 26 governments/public institutions, 45 news media), ~370 curated sensors at launch (`config/sources.yaml`), plus validated discovery.
17 17 - Connector families: HTTP (HTML/JSON/HEAD), RSS/Atom/JSON Feed, sitemaps, Statuspage, GitHub, JSON APIs
18 18 (CISA KEV, NVD, Federal Register, USGS, HIBP, Google Cloud incidents…), Scrapfly fallback.
19 19 - Every event is auditable: event → change → snapshot A/B → fetch run → sensor → source, with raw evidence.
modified apps/engine/src/pipeline.ts +1 −1
@@ -227,7 +227,7 @@ async function createEvent(ctx: { sensor: Sensor; source: Source; obs: Observati
227 227 }
228 228
229 229 let llm: Interpretation | null = null;
230 − if (llmAvailable() && prelim.score >= config.llm.minImportance) {
230 + if (llmAvailable() && source.llmEnabled && prelim.score >= config.llm.minImportance) {
231 231 llm = await interpretChange({ sourceName: source.name, sourceCategories: source.categories, url: sensor.url, sensorName: sensor.name, sensorType: sensor.type, heuristic, diff, prelimImportance: prelim.score, title: norm.title });
232 232 if (llm && !llm.meaningful) {
233 233 log.info({ sensor: sensor.id, type: llm.event_type }, "LLM judged change not meaningful");
modified apps/engine/src/registry.ts +4 −2
@@ -40,6 +40,8 @@ const sourceSchema = z.object({
40 40 sensors: z.array(sensorSchema).default([]),
41 41 notes: z.string().optional(),
42 42 enabled: z.boolean().default(true),
43 + /** false = heuristics only (high-volume feeds such as news wires) */
44 + llm: z.boolean().default(true),
43 45 });
44 46 export type SourceSeed = z.infer<typeof sourceSchema>;
45 47
@@ -62,8 +64,8 @@ export async function syncRegistry(seeds = loadSeeds()): Promise<{ sources: numb
62 64 for (const s of seeds) {
63 65 await db
64 66 .insert(sources)
65 − .values({ id: s.id, name: s.name, domain: s.domain, homepage: s.homepage ?? `https://${s.domain}`, description: s.description, categories: s.categories, tier: s.tier, importanceWeight: s.weight, discover: s.discover, fallback: s.fallback, notes: s.notes, enabled: s.enabled })
66 − .onConflictDoUpdate({ target: sources.id, set: { name: s.name, domain: s.domain, homepage: s.homepage ?? `https://${s.domain}`, description: s.description, categories: s.categories, tier: s.tier, importanceWeight: s.weight, discover: s.discover, fallback: s.fallback, notes: s.notes, enabled: s.enabled, updatedAt: new Date() } });
67 + .values({ id: s.id, name: s.name, domain: s.domain, homepage: s.homepage ?? `https://${s.domain}`, description: s.description, categories: s.categories, tier: s.tier, importanceWeight: s.weight, discover: s.discover, fallback: s.fallback, notes: s.notes, enabled: s.enabled, llmEnabled: s.llm })
68 + .onConflictDoUpdate({ target: sources.id, set: { name: s.name, domain: s.domain, homepage: s.homepage ?? `https://${s.domain}`, description: s.description, categories: s.categories, tier: s.tier, importanceWeight: s.weight, discover: s.discover, fallback: s.fallback, notes: s.notes, enabled: s.enabled, llmEnabled: s.llm, updatedAt: new Date() } });
67 69
68 70 // Organization entity + aliases
69 71 const entId = `org_${s.id}`;
modified apps/web/src/lib/format.ts +2 −1
@@ -137,7 +137,8 @@ export const CHANNELS: { key: string; label: string; query: Record<string, strin
137 137 { key: "science", label: "Science", query: { category: "science" }, ws: "events:science" },
138 138 { key: "products", label: "Products", query: { category: "products" }, ws: "events:products" },
139 139 { key: "infrastructure", label: "Infrastructure", query: { category: "infrastructure" }, ws: "events:infrastructure" },
140 + { key: "news", label: "News", query: { category: "news" }, ws: "events:news" },
140 141 { key: "silent", label: "Silent", query: { silent_change: true }, ws: "events:silent" },
141 142 ];
142 143
143 −export const CHANNEL_KEYS = ["ai", "cyber", "finance", "health", "government", "science", "products", "infrastructure"] as const;
144 +export const CHANNEL_KEYS = ["ai", "cyber", "finance", "health", "government", "science", "products", "infrastructure", "news"] as const;
modified config/sources.yaml +782 −0
@@ -2085,3 +2085,785 @@ sources:
2085 2085 discover: { rss: false }
2086 2086 sensors:
2087 2087 - { name: news feed, url: "https://letsencrypt.org/feed.xml", type: RSS, connector: rss, tier: B }
2088 +
2089 + # ───────────────────────── M · Governments & public institutions (added 2026-09-08) ─────────────────────────
2090 + - id: uk-government
2091 + name: UK Government
2092 + domain: gov.uk
2093 + homepage: https://www.gov.uk
2094 + categories: [government, politics]
2095 + tier: A
2096 + aliases: [gov.uk, hm government, uk gov]
2097 + discover: { rss: false }
2098 + sensors:
2099 + - { name: news and communications feed, url: "https://www.gov.uk/search/news-and-communications.atom", type: ATOM, connector: rss, tier: A }
2100 + - { name: statistics feed, url: "https://www.gov.uk/search/research-and-statistics.atom", type: ATOM, connector: rss, tier: B }
2101 + - id: european-commission
2102 + name: European Commission
2103 + domain: ec.europa.eu
2104 + homepage: https://commission.europa.eu
2105 + categories: [government, politics]
2106 + tier: A
2107 + weight: 1.2
2108 + aliases: [commission européenne, eu commission]
2109 + discover: { rss: false }
2110 + sensors:
2111 + - { name: press corner feed, url: "https://ec.europa.eu/commission/presscorner/api/rss?language=en", type: RSS, connector: rss, tier: A }
2112 + - id: european-parliament
2113 + name: European Parliament
2114 + domain: europarl.europa.eu
2115 + homepage: https://www.europarl.europa.eu
2116 + categories: [government, politics]
2117 + tier: B
2118 + aliases: [parlement européen]
2119 + discover: { rss: true }
2120 + notes: "RSS answers HTTP 202 (bot challenge) to non-browser clients — discovery only."
2121 + - id: bundesregierung
2122 + name: German Federal Government
2123 + domain: bundesregierung.de
2124 + homepage: https://www.bundesregierung.de
2125 + categories: [government, politics]
2126 + tier: B
2127 + aliases: [bundesregierung, german government]
2128 + discover: { rss: false }
2129 + sensors:
2130 + - { name: news feed, url: "https://www.bundesregierung.de/service/rss/breg-de/1151242/feed.xml", type: RSS, connector: rss, tier: B }
2131 + - id: pib-india
2132 + name: Press Information Bureau (India)
2133 + domain: pib.gov.in
2134 + homepage: https://www.pib.gov.in
2135 + categories: [government]
2136 + tier: B
2137 + aliases: [pib, government of india]
2138 + discover: { rss: false }
2139 + sensors:
2140 + - { name: all releases feed, url: "https://www.pib.gov.in/RssMain.aspx?ModId=6&Lang=1&Regid=3", type: RSS, connector: rss, tier: B }
2141 + - id: united-nations
2142 + name: United Nations
2143 + domain: un.org
2144 + homepage: https://news.un.org
2145 + categories: [government, politics]
2146 + tier: A
2147 + aliases: [un, nations unies, onu]
2148 + discover: { rss: false }
2149 + sensors:
2150 + - { name: un news feed, url: "https://news.un.org/feed/subscribe/en/news/all/rss.xml", type: RSS, connector: rss, tier: A }
2151 + - id: nato
2152 + name: NATO
2153 + domain: nato.int
2154 + homepage: https://www.nato.int
2155 + categories: [government, politics]
2156 + tier: B
2157 + aliases: [otan]
2158 + discover: { rss: true, pages: false }
2159 + sensors:
2160 + - { name: news, url: "https://www.nato.int/en/news-and-events/articles/news", type: HTML, connector: http, tier: B }
2161 + - id: us-state-department
2162 + name: U.S. Department of State
2163 + domain: state.gov
2164 + homepage: https://www.state.gov
2165 + categories: [government, politics]
2166 + tier: A
2167 + aliases: [state department]
2168 + discover: { rss: false }
2169 + sensors:
2170 + - { name: press releases feed, url: "https://www.state.gov/rss-feed/press-releases/feed/", type: RSS, connector: rss, tier: A }
2171 + - id: us-doj
2172 + name: U.S. Department of Justice
2173 + domain: justice.gov
2174 + homepage: https://www.justice.gov
2175 + categories: [government]
2176 + tier: A
2177 + aliases: [doj, justice department]
2178 + discover: { rss: false }
2179 + sensors:
2180 + - { name: press releases feed, url: "https://www.justice.gov/news/rss?type=press_release", type: RSS, connector: rss, tier: A }
2181 + - id: ftc
2182 + name: FTC
2183 + domain: ftc.gov
2184 + homepage: https://www.ftc.gov
2185 + categories: [government, commerce]
2186 + tier: A
2187 + aliases: [federal trade commission]
2188 + discover: { rss: false }
2189 + sensors:
2190 + - { name: press releases feed, url: "https://www.ftc.gov/feeds/press-release.xml", type: RSS, connector: rss, tier: A }
2191 + - id: us-dod
2192 + name: U.S. Department of Defense
2193 + domain: defense.gov
2194 + homepage: https://www.defense.gov
2195 + categories: [government]
2196 + tier: B
2197 + aliases: [pentagon, dod, department of war]
2198 + discover: { rss: false }
2199 + sensors:
2200 + - { name: releases feed, url: "https://www.defense.gov/DesktopModules/ArticleCS/RSS.ashx?ContentType=1&Site=945&max=20", type: RSS, connector: rss, tier: B }
2201 + - id: us-doe
2202 + name: U.S. Department of Energy
2203 + domain: energy.gov
2204 + homepage: https://www.energy.gov
2205 + categories: [government, science]
2206 + tier: B
2207 + aliases: [doe]
2208 + discover: { rss: false }
2209 + sensors:
2210 + - { name: news feed, url: "https://www.energy.gov/rss/news.xml", type: RSS, connector: rss, tier: B }
2211 + - id: gao
2212 + name: GAO
2213 + domain: gao.gov
2214 + homepage: https://www.gao.gov
2215 + categories: [government, finance]
2216 + tier: B
2217 + aliases: [government accountability office]
2218 + discover: { rss: false }
2219 + sensors:
2220 + - { name: reports feed, url: "https://www.gao.gov/rss/reports.xml", type: RSS, connector: rss, tier: B }
2221 + - id: cbo
2222 + name: CBO
2223 + domain: cbo.gov
2224 + homepage: https://www.cbo.gov
2225 + categories: [government, finance]
2226 + tier: B
2227 + aliases: [congressional budget office]
2228 + discover: { rss: false }
2229 + sensors:
2230 + - { name: publications feed, url: "https://www.cbo.gov/publications/all/rss.xml", type: RSS, connector: rss, tier: B }
2231 + - id: ontario
2232 + name: Government of Ontario
2233 + domain: ontario.ca
2234 + homepage: https://news.ontario.ca
2235 + categories: [government]
2236 + tier: B
2237 + aliases: [ontario]
2238 + discover: { rss: false }
2239 + sensors:
2240 + - { name: newsroom feed, url: "https://news.ontario.ca/newsroom/en/rss/allnews.rss", type: RSS, connector: rss, tier: B }
2241 + - id: british-columbia
2242 + name: Government of British Columbia
2243 + domain: gov.bc.ca
2244 + homepage: https://news.gov.bc.ca
2245 + categories: [government]
2246 + tier: B
2247 + aliases: [british columbia, bc government]
2248 + discover: { rss: false }
2249 + sensors:
2250 + - { name: news feed, url: "https://news.gov.bc.ca/feed", type: RSS, connector: rss, tier: B }
2251 + - id: alberta
2252 + name: Government of Alberta
2253 + domain: alberta.ca
2254 + homepage: https://www.alberta.ca
2255 + categories: [government]
2256 + tier: C
2257 + aliases: [alberta]
2258 + discover: { rss: true }
2259 + sensors:
2260 + - { name: news, url: "https://www.alberta.ca/news", type: HTML, connector: http, tier: B }
2261 + - id: finance-canada
2262 + name: Department of Finance Canada
2263 + domain: canada.ca
2264 + homepage: https://www.canada.ca/en/department-finance.html
2265 + categories: [government, finance]
2266 + tier: A
2267 + aliases: [finance canada, ministère des finances]
2268 + discover: { rss: false }
2269 + sensors:
2270 + - { name: news feed, url: "https://api.io.canada.ca/io-server/gc/news/en/v2?dept=financecanada&sort=publishedDate&orderBy=desc&pick=50&format=atom&atomtitle=Finance%20Canada", type: ATOM, connector: rss, tier: A }
2271 + - id: cra
2272 + name: Canada Revenue Agency
2273 + domain: canada.ca
2274 + homepage: https://www.canada.ca/en/revenue-agency.html
2275 + categories: [government, finance]
2276 + tier: B
2277 + aliases: [cra, agence du revenu du canada, arc]
2278 + discover: { rss: false }
2279 + sensors:
2280 + - { name: news feed, url: "https://api.io.canada.ca/io-server/gc/news/en/v2?dept=canadarevenueagency&sort=publishedDate&orderBy=desc&pick=50&format=atom&atomtitle=CRA", type: ATOM, connector: rss, tier: B }
2281 + - id: ircc
2282 + name: Immigration, Refugees and Citizenship Canada
2283 + domain: canada.ca
2284 + homepage: https://www.canada.ca/en/immigration-refugees-citizenship.html
2285 + categories: [government]
2286 + tier: B
2287 + aliases: [ircc, immigration canada]
2288 + discover: { rss: false }
2289 + sensors:
2290 + - { name: news feed, url: "https://api.io.canada.ca/io-server/gc/news/en/v2?dept=departmentofcitizenshipandimmigration&sort=publishedDate&orderBy=desc&pick=50&format=atom&atomtitle=IRCC", type: ATOM, connector: rss, tier: B }
2291 + - id: global-affairs-canada
2292 + name: Global Affairs Canada
2293 + domain: canada.ca
2294 + homepage: https://www.international.gc.ca
2295 + categories: [government, politics]
2296 + tier: B
2297 + aliases: [affaires mondiales canada, gac]
2298 + discover: { rss: false }
2299 + sensors:
2300 + - { name: news feed, url: "https://api.io.canada.ca/io-server/gc/news/en/v2?dept=departmentofforeignaffairstradeanddevelopment&sort=publishedDate&orderBy=desc&pick=50&format=atom&atomtitle=GAC", type: ATOM, connector: rss, tier: B }
2301 + - id: ised
2302 + name: Innovation, Science and Economic Development Canada
2303 + domain: canada.ca
2304 + homepage: https://ised-isde.canada.ca
2305 + categories: [government, technology]
2306 + tier: B
2307 + aliases: [ised, isde, innovation canada]
2308 + discover: { rss: false }
2309 + sensors:
2310 + - { name: news feed, url: "https://api.io.canada.ca/io-server/gc/news/en/v2?dept=innovationsciencedeveloppement&sort=publishedDate&orderBy=desc&pick=50&format=atom&atomtitle=ISED", type: ATOM, connector: rss, tier: B }
2311 + - id: eccc
2312 + name: Environment and Climate Change Canada
2313 + domain: canada.ca
2314 + homepage: https://www.canada.ca/en/environment-climate-change.html
2315 + categories: [government, science]
2316 + tier: B
2317 + aliases: [eccc, environnement canada]
2318 + discover: { rss: false }
2319 + sensors:
2320 + - { name: news feed, url: "https://api.io.canada.ca/io-server/gc/news/en/v2?dept=environmentandclimatechangecanada&sort=publishedDate&orderBy=desc&pick=50&format=atom&atomtitle=ECCC", type: ATOM, connector: rss, tier: B }
2321 + - id: dnd-canada
2322 + name: National Defence Canada
2323 + domain: canada.ca
2324 + homepage: https://www.canada.ca/en/department-national-defence.html
2325 + categories: [government]
2326 + tier: B
2327 + aliases: [dnd, défense nationale, canadian armed forces]
2328 + discover: { rss: false }
2329 + sensors:
2330 + - { name: news feed, url: "https://api.io.canada.ca/io-server/gc/news/en/v2?dept=nationaldefence&sort=publishedDate&orderBy=desc&pick=50&format=atom&atomtitle=DND", type: ATOM, connector: rss, tier: B }
2331 + - id: public-safety-canada
2332 + name: Public Safety Canada
2333 + domain: canada.ca
2334 + homepage: https://www.publicsafety.gc.ca
2335 + categories: [government, cyber]
2336 + tier: B
2337 + aliases: [sécurité publique canada, rcmp, grc]
2338 + discover: { rss: false }
2339 + sensors:
2340 + - { name: news feed, url: "https://api.io.canada.ca/io-server/gc/news/en/v2?dept=publicsafetycanada,royalcanadianmountedpolice,canadiancentreforcybersecurity&sort=publishedDate&orderBy=desc&pick=50&format=atom&atomtitle=Public%20Safety", type: ATOM, connector: rss, tier: B }
2341 + - id: prime-minister-canada
2342 + name: Prime Minister of Canada
2343 + domain: pm.gc.ca
2344 + homepage: https://www.pm.gc.ca
2345 + categories: [government, politics]
2346 + tier: A
2347 + aliases: [premier ministre du canada, pmo]
2348 + discover: { rss: true }
2349 + sensors:
2350 + - { name: news feed, url: "https://api.io.canada.ca/io-server/gc/news/en/v2?dept=primeministersoffice&sort=publishedDate&orderBy=desc&pick=50&format=atom&atomtitle=PMO", type: ATOM, connector: rss, tier: A }
2351 + - { name: news, url: "https://www.pm.gc.ca/en/news", type: HTML, connector: http, tier: A }
2352 +
2353 + # ───────────────────────── N · News & media (added 2026-09-08 — heuristics only, no LLM per item) ─────────────────────────
2354 + - id: bbc-news
2355 + name: BBC News
2356 + domain: bbc.co.uk
2357 + homepage: https://www.bbc.co.uk/news
2358 + categories: [news, media]
2359 + tier: A
2360 + weight: 0.8
2361 + llm: false
2362 + aliases: [bbc]
2363 + discover: { rss: false }
2364 + sensors:
2365 + - { name: top stories feed, url: "https://feeds.bbci.co.uk/news/rss.xml", type: RSS, connector: rss, tier: A }
2366 + - { name: world feed, url: "https://feeds.bbci.co.uk/news/world/rss.xml", type: RSS, connector: rss, tier: A }
2367 + - { name: technology feed, url: "https://feeds.bbci.co.uk/news/technology/rss.xml", type: RSS, connector: rss, tier: B }
2368 + - id: nytimes
2369 + name: The New York Times
2370 + domain: nytimes.com
2371 + homepage: https://www.nytimes.com
2372 + categories: [news, media]
2373 + tier: A
2374 + weight: 0.8
2375 + llm: false
2376 + aliases: [nyt, new york times]
2377 + discover: { rss: false }
2378 + sensors:
2379 + - { name: homepage feed, url: "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml", type: RSS, connector: rss, tier: A }
2380 + - { name: technology feed, url: "https://rss.nytimes.com/services/xml/rss/nyt/Technology.xml", type: RSS, connector: rss, tier: B }
2381 + - { name: business feed, url: "https://rss.nytimes.com/services/xml/rss/nyt/Business.xml", type: RSS, connector: rss, tier: B }
2382 + - id: the-guardian
2383 + name: The Guardian
2384 + domain: theguardian.com
2385 + homepage: https://www.theguardian.com
2386 + categories: [news, media]
2387 + tier: A
2388 + weight: 0.8
2389 + llm: false
2390 + aliases: [guardian]
2391 + discover: { rss: false }
2392 + sensors:
2393 + - { name: world feed, url: "https://www.theguardian.com/world/rss", type: RSS, connector: rss, tier: A }
2394 + - { name: technology feed, url: "https://www.theguardian.com/uk/technology/rss", type: RSS, connector: rss, tier: B }
2395 + - { name: business feed, url: "https://www.theguardian.com/uk/business/rss", type: RSS, connector: rss, tier: B }
2396 + - id: cnbc
2397 + name: CNBC
2398 + domain: cnbc.com
2399 + homepage: https://www.cnbc.com
2400 + categories: [news, media, finance]
2401 + tier: A
2402 + weight: 0.8
2403 + llm: false
2404 + discover: { rss: false }
2405 + sensors:
2406 + - { name: top news feed, url: "https://www.cnbc.com/id/100003114/device/rss/rss.html", type: RSS, connector: rss, tier: A }
2407 + - { name: markets feed, url: "https://www.cnbc.com/id/15839069/device/rss/rss.html", type: RSS, connector: rss, tier: A }
2408 + - id: cnn
2409 + name: CNN
2410 + domain: cnn.com
2411 + homepage: https://www.cnn.com
2412 + categories: [news, media]
2413 + tier: B
2414 + weight: 0.7
2415 + llm: false
2416 + discover: { rss: false }
2417 + sensors:
2418 + - { name: edition feed, url: "http://rss.cnn.com/rss/edition.rss", type: RSS, connector: rss, tier: A }
2419 + - id: al-jazeera
2420 + name: Al Jazeera
2421 + domain: aljazeera.com
2422 + homepage: https://www.aljazeera.com
2423 + categories: [news, media]
2424 + tier: B
2425 + weight: 0.7
2426 + llm: false
2427 + discover: { rss: false }
2428 + sensors:
2429 + - { name: all feed, url: "https://www.aljazeera.com/xml/rss/all.xml", type: RSS, connector: rss, tier: A }
2430 + - id: france24
2431 + name: France 24
2432 + domain: france24.com
2433 + homepage: https://www.france24.com
2434 + categories: [news, media]
2435 + tier: B
2436 + weight: 0.7
2437 + llm: false
2438 + discover: { rss: false }
2439 + sensors:
2440 + - { name: english feed, url: "https://www.france24.com/en/rss", type: RSS, connector: rss, tier: A }
2441 + - id: le-monde
2442 + name: Le Monde
2443 + domain: lemonde.fr
2444 + homepage: https://www.lemonde.fr
2445 + categories: [news, media]
2446 + tier: B
2447 + weight: 0.7
2448 + llm: false
2449 + discover: { rss: false }
2450 + sensors:
2451 + - { name: une feed, url: "https://www.lemonde.fr/rss/une.xml", type: RSS, connector: rss, tier: A }
2452 + - id: le-figaro
2453 + name: Le Figaro
2454 + domain: lefigaro.fr
2455 + homepage: https://www.lefigaro.fr
2456 + categories: [news, media]
2457 + tier: C
2458 + weight: 0.6
2459 + llm: false
2460 + discover: { rss: false }
2461 + sensors:
2462 + - { name: actualités feed, url: "https://www.lefigaro.fr/rss/figaro_actualites.xml", type: RSS, connector: rss, tier: B }
2463 + - id: radio-canada
2464 + name: Radio-Canada
2465 + domain: radio-canada.ca
2466 + homepage: https://ici.radio-canada.ca
2467 + categories: [news, media]
2468 + tier: A
2469 + weight: 0.8
2470 + llm: false
2471 + aliases: [ici radio-canada, rdi]
2472 + discover: { rss: false }
2473 + sensors:
2474 + - { name: à la une feed, url: "https://ici.radio-canada.ca/info/rss/info/a-la-une", type: RSS, connector: rss, tier: A }
2475 + - id: cbc
2476 + name: CBC News
2477 + domain: cbc.ca
2478 + homepage: https://www.cbc.ca/news
2479 + categories: [news, media]
2480 + tier: A
2481 + weight: 0.8
2482 + llm: false
2483 + aliases: [cbc]
2484 + discover: { rss: false }
2485 + sensors:
2486 + - { name: top stories feed, url: "https://www.cbc.ca/webfeed/rss/rss-topstories", type: RSS, connector: rss, tier: A }
2487 + - { name: canada feed, url: "https://www.cbc.ca/webfeed/rss/rss-canada", type: RSS, connector: rss, tier: B }
2488 + - id: la-presse
2489 + name: La Presse
2490 + domain: lapresse.ca
2491 + homepage: https://www.lapresse.ca
2492 + categories: [news, media]
2493 + tier: A
2494 + weight: 0.8
2495 + llm: false
2496 + discover: { rss: false }
2497 + sensors:
2498 + - { name: actualités feed, url: "https://www.lapresse.ca/actualites/rss", type: RSS, connector: rss, tier: A }
2499 + - id: le-devoir
2500 + name: Le Devoir
2501 + domain: ledevoir.com
2502 + homepage: https://www.ledevoir.com
2503 + categories: [news, media]
2504 + tier: A
2505 + weight: 0.8
2506 + llm: false
2507 + discover: { rss: false }
2508 + sensors:
2509 + - { name: manchettes feed, url: "https://www.ledevoir.com/rss/manchettes.xml", type: RSS, connector: rss, tier: A }
2510 + - id: journal-de-montreal
2511 + name: Le Journal de Montréal
2512 + domain: journaldemontreal.com
2513 + homepage: https://www.journaldemontreal.com
2514 + categories: [news, media]
2515 + tier: B
2516 + weight: 0.7
2517 + llm: false
2518 + aliases: [jdm]
2519 + discover: { rss: false }
2520 + sensors:
2521 + - { name: feed, url: "https://www.journaldemontreal.com/rss.xml", type: RSS, connector: rss, tier: A }
2522 + - id: tva-nouvelles
2523 + name: TVA Nouvelles
2524 + domain: tvanouvelles.ca
2525 + homepage: https://www.tvanouvelles.ca
2526 + categories: [news, media]
2527 + tier: B
2528 + weight: 0.7
2529 + llm: false
2530 + aliases: [tva]
2531 + discover: { rss: false }
2532 + sensors:
2533 + - { name: feed, url: "https://www.tvanouvelles.ca/rss.xml", type: RSS, connector: rss, tier: A }
2534 + - id: 24-heures
2535 + name: 24 heures
2536 + domain: 24heures.ca
2537 + homepage: https://www.24heures.ca
2538 + categories: [news, media]
2539 + tier: C
2540 + weight: 0.6
2541 + llm: false
2542 + discover: { rss: false }
2543 + sensors:
2544 + - { name: feed, url: "https://www.24heures.ca/rss.xml", type: RSS, connector: rss, tier: B }
2545 + - id: globe-and-mail
2546 + name: The Globe and Mail
2547 + domain: theglobeandmail.com
2548 + homepage: https://www.theglobeandmail.com
2549 + categories: [news, media]
2550 + tier: A
2551 + weight: 0.8
2552 + llm: false
2553 + aliases: [globe and mail]
2554 + discover: { rss: false }
2555 + sensors:
2556 + - { name: canada feed, url: "https://www.theglobeandmail.com/arc/outboundfeeds/rss/category/canada/", type: RSS, connector: rss, tier: A, config: { maxItems: 100 } }
2557 + - { name: business feed, url: "https://www.theglobeandmail.com/arc/outboundfeeds/rss/category/business/", type: RSS, connector: rss, tier: B, config: { maxItems: 100 } }
2558 + - id: national-post
2559 + name: National Post
2560 + domain: nationalpost.com
2561 + categories: [news, media]
2562 + tier: B
2563 + weight: 0.7
2564 + llm: false
2565 + discover: { rss: false }
2566 + sensors:
2567 + - { name: feed, url: "https://nationalpost.com/feed", type: RSS, connector: rss, tier: A }
2568 + - id: global-news
2569 + name: Global News
2570 + domain: globalnews.ca
2571 + categories: [news, media]
2572 + tier: B
2573 + weight: 0.7
2574 + llm: false
2575 + discover: { rss: false }
2576 + sensors:
2577 + - { name: feed, url: "https://globalnews.ca/feed/", type: RSS, connector: rss, tier: A }
2578 + - id: the-logic
2579 + name: The Logic
2580 + domain: thelogic.co
2581 + categories: [news, media, technology]
2582 + tier: B
2583 + weight: 0.7
2584 + llm: false
2585 + discover: { rss: false }
2586 + sensors:
2587 + - { name: feed, url: "https://thelogic.co/feed/", type: RSS, connector: rss, tier: B }
2588 + - id: wsj
2589 + name: The Wall Street Journal
2590 + domain: wsj.com
2591 + homepage: https://www.wsj.com
2592 + categories: [news, media, finance]
2593 + tier: A
2594 + weight: 0.8
2595 + llm: false
2596 + aliases: [wall street journal]
2597 + discover: { rss: false }
2598 + sensors:
2599 + - { name: world news feed, url: "https://feeds.a.dj.com/rss/RSSWorldNews.xml", type: RSS, connector: rss, tier: A }
2600 + - { name: markets feed, url: "https://feeds.a.dj.com/rss/RSSMarketsMain.xml", type: RSS, connector: rss, tier: A }
2601 + - id: financial-times
2602 + name: Financial Times
2603 + domain: ft.com
2604 + homepage: https://www.ft.com
2605 + categories: [news, media, finance]
2606 + tier: A
2607 + weight: 0.8
2608 + llm: false
2609 + aliases: [ft]
2610 + discover: { rss: false }
2611 + sensors:
2612 + - { name: home international feed, url: "https://www.ft.com/rss/home/international", type: RSS, connector: rss, tier: A }
2613 + - id: the-economist
2614 + name: The Economist
2615 + domain: economist.com
2616 + homepage: https://www.economist.com
2617 + categories: [news, media]
2618 + tier: B
2619 + weight: 0.7
2620 + llm: false
2621 + aliases: [economist]
2622 + discover: { rss: false }
2623 + sensors:
2624 + - { name: latest feed, url: "https://www.economist.com/latest/rss.xml", type: RSS, connector: rss, tier: B, config: { maxItems: 100 } }
2625 + - id: axios
2626 + name: Axios
2627 + domain: axios.com
2628 + homepage: https://www.axios.com
2629 + categories: [news, media]
2630 + tier: B
2631 + weight: 0.7
2632 + llm: false
2633 + discover: { rss: false }
2634 + sensors:
2635 + - { name: feed, url: "https://api.axios.com/feed/", type: RSS, connector: rss, tier: A, config: { maxItems: 100 } }
2636 + - id: npr
2637 + name: NPR
2638 + domain: npr.org
2639 + homepage: https://www.npr.org
2640 + categories: [news, media]
2641 + tier: B
2642 + weight: 0.7
2643 + llm: false
2644 + discover: { rss: false }
2645 + sensors:
2646 + - { name: news feed, url: "https://feeds.npr.org/1001/rss.xml", type: RSS, connector: rss, tier: A }
2647 + - id: washington-post
2648 + name: The Washington Post
2649 + domain: washingtonpost.com
2650 + homepage: https://www.washingtonpost.com
2651 + categories: [news, media]
2652 + tier: A
2653 + weight: 0.8
2654 + llm: false
2655 + aliases: [wapo, washington post]
2656 + discover: { rss: false }
2657 + sensors:
2658 + - { name: world feed, url: "https://feeds.washingtonpost.com/rss/world", type: RSS, connector: rss, tier: A }
2659 + - { name: politics feed, url: "https://feeds.washingtonpost.com/rss/politics", type: RSS, connector: rss, tier: A }
2660 + - id: scmp
2661 + name: South China Morning Post
2662 + domain: scmp.com
2663 + homepage: https://www.scmp.com
2664 + categories: [news, media]
2665 + tier: B
2666 + weight: 0.7
2667 + llm: false
2668 + aliases: [scmp]
2669 + discover: { rss: false }
2670 + sensors:
2671 + - { name: news feed, url: "https://www.scmp.com/rss/91/feed/", type: RSS, connector: rss, tier: B }
2672 + - id: dw
2673 + name: Deutsche Welle
2674 + domain: dw.com
2675 + homepage: https://www.dw.com
2676 + categories: [news, media]
2677 + tier: B
2678 + weight: 0.7
2679 + llm: false
2680 + aliases: [dw]
2681 + discover: { rss: false }
2682 + sensors:
2683 + - { name: all english feed, url: "https://rss.dw.com/rdf/rss-en-all", type: RSS, connector: rss, tier: B, config: { maxItems: 150 } }
2684 + - id: euronews
2685 + name: Euronews
2686 + domain: euronews.com
2687 + homepage: https://www.euronews.com
2688 + categories: [news, media]
2689 + tier: B
2690 + weight: 0.7
2691 + llm: false
2692 + discover: { rss: false }
2693 + sensors:
2694 + - { name: feed, url: "https://www.euronews.com/rss", type: RSS, connector: rss, tier: A }
2695 + - id: der-spiegel
2696 + name: Der Spiegel
2697 + domain: spiegel.de
2698 + homepage: https://www.spiegel.de
2699 + categories: [news, media]
2700 + tier: C
2701 + weight: 0.6
2702 + llm: false
2703 + aliases: [spiegel]
2704 + discover: { rss: false }
2705 + sensors:
2706 + - { name: international feed, url: "https://www.spiegel.de/international/index.rss", type: RSS, connector: rss, tier: B }
2707 + - id: el-pais
2708 + name: El País
2709 + domain: elpais.com
2710 + homepage: https://elpais.com
2711 + categories: [news, media]
2712 + tier: C
2713 + weight: 0.6
2714 + llm: false
2715 + discover: { rss: false }
2716 + sensors:
2717 + - { name: portada feed, url: "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/portada", type: RSS, connector: rss, tier: B, config: { maxItems: 150 } }
2718 + - id: semafor
2719 + name: Semafor
2720 + domain: semafor.com
2721 + homepage: https://www.semafor.com
2722 + categories: [news, media]
2723 + tier: C
2724 + weight: 0.6
2725 + llm: false
2726 + discover: { rss: false }
2727 + sensors:
2728 + - { name: feed, url: "https://www.semafor.com/rss.xml", type: RSS, connector: rss, tier: B, config: { maxItems: 150 } }
2729 + - id: google-news
2730 + name: Google News
2731 + domain: news.google.com
2732 + categories: [news, media]
2733 + tier: B
2734 + weight: 0.6
2735 + llm: false
2736 + discover: { rss: false }
2737 + sensors:
2738 + - { name: top stories (us) feed, url: "https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en", type: RSS, connector: rss, tier: A }
2739 + - { name: à la une (canada fr) feed, url: "https://news.google.com/rss?hl=fr-CA&gl=CA&ceid=CA:fr", type: RSS, connector: rss, tier: A }
2740 + - id: techcrunch
2741 + name: TechCrunch
2742 + domain: techcrunch.com
2743 + categories: [news, media, technology]
2744 + tier: A
2745 + weight: 0.8
2746 + llm: false
2747 + discover: { rss: false }
2748 + sensors:
2749 + - { name: feed, url: "https://techcrunch.com/feed/", type: RSS, connector: rss, tier: A }
2750 + - id: the-verge
2751 + name: The Verge
2752 + domain: theverge.com
2753 + homepage: https://www.theverge.com
2754 + categories: [news, media, technology]
2755 + tier: B
2756 + weight: 0.7
2757 + llm: false
2758 + aliases: [verge]
2759 + discover: { rss: false }
2760 + sensors:
2761 + - { name: feed, url: "https://www.theverge.com/rss/index.xml", type: ATOM, connector: rss, tier: A }
2762 + - id: ars-technica
2763 + name: Ars Technica
2764 + domain: arstechnica.com
2765 + categories: [news, media, technology]
2766 + tier: B
2767 + weight: 0.7
2768 + llm: false
2769 + aliases: [ars]
2770 + discover: { rss: false }
2771 + sensors:
2772 + - { name: feed, url: "https://feeds.arstechnica.com/arstechnica/index", type: RSS, connector: rss, tier: A }
2773 + - id: wired
2774 + name: WIRED
2775 + domain: wired.com
2776 + homepage: https://www.wired.com
2777 + categories: [news, media, technology]
2778 + tier: B
2779 + weight: 0.7
2780 + llm: false
2781 + discover: { rss: false }
2782 + sensors:
2783 + - { name: feed, url: "https://www.wired.com/feed/rss", type: RSS, connector: rss, tier: B }
2784 + - id: hacker-news
2785 + name: Hacker News
2786 + domain: news.ycombinator.com
2787 + categories: [news, media, technology, developer]
2788 + tier: B
2789 + weight: 0.7
2790 + llm: false
2791 + aliases: [hn, ycombinator]
2792 + discover: { rss: false }
2793 + sensors:
2794 + - { name: front page feed, url: "https://hnrss.org/frontpage", type: RSS, connector: rss, tier: A }
2795 + - id: the-register
2796 + name: The Register
2797 + domain: theregister.com
2798 + homepage: https://www.theregister.com
2799 + categories: [news, media, technology]
2800 + tier: B
2801 + weight: 0.7
2802 + llm: false
2803 + discover: { rss: false }
2804 + sensors:
2805 + - { name: headlines feed, url: "https://www.theregister.com/headlines.atom", type: ATOM, connector: rss, tier: A }
2806 + - id: bleepingcomputer
2807 + name: BleepingComputer
2808 + domain: bleepingcomputer.com
2809 + homepage: https://www.bleepingcomputer.com
2810 + categories: [news, media, cyber]
2811 + tier: A
2812 + weight: 0.8
2813 + llm: false
2814 + discover: { rss: false }
2815 + sensors:
2816 + - { name: feed, url: "https://www.bleepingcomputer.com/feed/", type: RSS, connector: rss, tier: A }
2817 + - id: krebs-on-security
2818 + name: Krebs on Security
2819 + domain: krebsonsecurity.com
2820 + categories: [news, media, cyber]
2821 + tier: B
2822 + weight: 0.8
2823 + llm: false
2824 + aliases: [krebs]
2825 + discover: { rss: false }
2826 + sensors:
2827 + - { name: feed, url: "https://krebsonsecurity.com/feed/", type: RSS, connector: rss, tier: B }
2828 + - id: the-hacker-news
2829 + name: The Hacker News
2830 + domain: thehackernews.com
2831 + categories: [news, media, cyber]
2832 + tier: B
2833 + weight: 0.7
2834 + llm: false
2835 + discover: { rss: false }
2836 + sensors:
2837 + - { name: feed, url: "https://feeds.feedburner.com/TheHackersNews", type: RSS, connector: rss, tier: A }
2838 + - id: the-record
2839 + name: The Record (Recorded Future News)
2840 + domain: therecord.media
2841 + categories: [news, media, cyber]
2842 + tier: B
2843 + weight: 0.7
2844 + llm: false
2845 + discover: { rss: false }
2846 + sensors:
2847 + - { name: feed, url: "https://therecord.media/feed", type: RSS, connector: rss, tier: A }
2848 + - id: 404-media
2849 + name: 404 Media
2850 + domain: 404media.co
2851 + homepage: https://www.404media.co
2852 + categories: [news, media, technology]
2853 + tier: C
2854 + weight: 0.6
2855 + llm: false
2856 + discover: { rss: false }
2857 + sensors:
2858 + - { name: feed, url: "https://www.404media.co/rss/", type: RSS, connector: rss, tier: B }
2859 + - id: mit-technology-review
2860 + name: MIT Technology Review
2861 + domain: technologyreview.com
2862 + homepage: https://www.technologyreview.com
2863 + categories: [news, media, technology, ai]
2864 + tier: B
2865 + weight: 0.7
2866 + llm: false
2867 + discover: { rss: false }
2868 + sensors:
2869 + - { name: feed, url: "https://www.technologyreview.com/feed/", type: RSS, connector: rss, tier: B }
modified docs/ARCHITECTURE.md +1 −1
@@ -27,7 +27,7 @@ events and entities (GIN). Migrations: `packages/db/migrations/*.sql`.
27 27 rules + sensor/URL priors → `signal`, `eventType`, `magnitude`. A `changes` row is always written.
28 28 6. If `signal ≥ WS_MEANINGFUL_SIGNAL` and extraction is trustworthy: **event candidate** — entity resolution
29 29 (aliases), novelty vs the 72 h window (shingle Jaccard; near-duplicates suppressed), preliminary importance,
30 − optional **Claude interpretation** (strict JSON: type, title, summary, why it matters, observed/inferred,
30 + optional **Claude interpretation** (skipped for sources with `llm_enabled = false`, e.g. news wires) (strict JSON: type, title, summary, why it matters, observed/inferred,
31 31 severity, confidence, announced?), final importance (8 stored components) + confidence, silent-change flag,
32 32 clustering (shared entity/source + textual similarity within 6 h), `events` row, Redis stream + pub/sub.
33 33 7. **Schedule** the next check adaptively (tier bounds, recency of change, change frequency, 304s, errors).
modified packages/core/src/taxonomy.ts +4 −0
@@ -77,6 +77,9 @@ export const CATEGORIES = [
77 77 "internet",
78 78 "standards",
79 79 "technology",
80 + "news",
81 + "media",
82 + "politics",
80 83 ] as const;
81 84 export type Category = (typeof CATEGORIES)[number] | string;
82 85
@@ -90,6 +93,7 @@ export const FEED_CHANNELS: Record<string, string[]> = {
90 93 science: ["science", "space"],
91 94 products: ["consumer-tech", "automotive", "semiconductors", "enterprise"],
92 95 infrastructure: ["cloud", "developer", "internet", "standards"],
96 + news: ["news", "media"],
93 97 };
94 98
95 99 export interface EventTypeSpec {
added packages/db/migrations/0004_source_llm.sql +1 −0
@@ -0,0 +1 @@
1 +alter table sources add column if not exists llm_enabled boolean not null default true;
modified packages/db/src/schema.ts +1 −0
@@ -18,6 +18,7 @@ export const sources = pgTable("sources", {
18 18 termsReviewedAt: ts("terms_reviewed_at"),
19 19 allowedMethods: text("allowed_methods").array().notNull().default([]),
20 20 rateLimitPerMin: integer("rate_limit_per_min"),
21 + llmEnabled: boolean("llm_enabled").notNull().default(true),
21 22 notes: text("notes"),
22 23 createdAt: ts("created_at").notNull().defaultNow(),
23 24 updatedAt: ts("updated_at").notNull().defaultNow(),
24 25