SPB Git
spboucher.ai

spb/social-runtime-crawler

Public

Research console (Next 16) + control-plane API + mld deployment for www.socialcrawl.co

- apps/dashboard: SocialCrawl console — observatory, sessions console (decisions, page state,
  entities, runtime APIs, media, world-model graph, live events), entity evidence pages,
  runtime API explorer, connectors, missions launcher; passcode gate + authenticated API proxy.
- apps/api: node:http control plane (overview, sessions, entities search, schemas, platforms,
  SSE stream, job runner spawning worker processes), token-protected mutations.
- worker: bounded action execution (60 s), PLAY_VIDEO no longer awaits play(); --job linking.
- storage: jsonb sanitiser, linkJob; util: unicode thousands separators, stopwords.
- deploy/social-runtime-crawler.mld.json: PM2 api 8350 + web 8351, MacLustr Tunnel route.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 12, 2026) parent 40fba6b Browse files

46 changed files +7,607 −105

46 changed files
Load all 46 files Unified Split
modified .gitignore +5 −0
@@ -14,3 +14,8 @@ data/platform_model/*/
14 14
15 # Compiled (learned) connectors are versioned intentionally; regression fixtures may be large. 15 # Compiled (learned) connectors are versioned intentionally; regression fixtures may be large.
16 connectors/*/regression_tests/large/ 16 connectors/*/regression_tests/large/
17+
18+# Next.js dashboard
19+apps/dashboard/.next/
20+apps/dashboard/next-env.d.ts
21+*.tsbuildinfo
modified apps/api/package.json +5 −1
@@ -5,6 +5,10 @@
5 "type": "module", 5 "type": "module",
6 "dependencies": { 6 "dependencies": {
7 "@src/shared": "workspace:*", 7 "@src/shared": "workspace:*",
8− "@src/storage": "workspace:*" 8+ "@src/storage": "workspace:*",
9+ "@src/platform-model": "workspace:*"
10+ },
11+ "scripts": {
12+ "start": "tsx src/server.ts"
9 } 13 }
10 } 14 }
added apps/api/src/jobs.ts +102 −0
@@ -0,0 +1,102 @@
1+import { spawn, type ChildProcess } from "node:child_process";
2+import fs from "node:fs";
3+import path from "node:path";
4+import { fileURLToPath } from "node:url";
5+import { createLogger, isPlatform, newId, type AgentMode, type AppConfig } from "@src/shared";
6+import type { PostgresStore } from "@src/storage";
7+
8+const log = createLogger("jobs");
9+const here = path.dirname(fileURLToPath(import.meta.url));
10+const repoRoot = path.resolve(here, "..", "..", "..");
11+
12+export interface JobRequest {
13+ platform: string;
14+ goal?: string;
15+ query?: string;
16+ mode?: AgentMode;
17+ seed?: string;
18+ minutes?: number;
19+ actions?: number;
20+ media?: number;
21+ headless?: boolean;
22+ account?: string;
23+}
24+
25+/**
26+ * Job runner (§63 control plane, single-node version): spawns the worker CLI as a child process,
27+ * streams its log to data/jobs/<id>.log and tracks status in crawl_jobs. One browser per job.
28+ */
29+export class JobRunner {
30+ private running = new Map<string, ChildProcess>();
31+ readonly logsDir: string;
32+ constructor(private readonly store: PostgresStore, private readonly cfg: AppConfig, private readonly maxConcurrent = 2) {
33+ this.logsDir = path.join(cfg.dataDir, "jobs");
34+ fs.mkdirSync(this.logsDir, { recursive: true });
35+ }
36+
37+ get activeCount() {
38+ return this.running.size;
39+ }
40+
41+ async start(req: JobRequest): Promise<{ job_id: string }> {
42+ if (!isPlatform(req.platform)) throw new Error("unknown platform");
43+ if (!["youtube", "reddit"].includes(req.platform)) throw new Error(`no adapter for ${req.platform} yet (Phase 1 = youtube, reddit)`);
44+ if (this.running.size >= this.maxConcurrent) throw new Error(`max ${this.maxConcurrent} concurrent jobs`);
45+ const job_id = newId("job");
46+ const mode = req.mode ?? "research";
47+ const minutes = clamp(req.minutes ?? 10, 1, 120);
48+ const actions = clamp(req.actions ?? 60, 1, 1000);
49+ const media = clamp(req.media ?? 1, 0, 4);
50+ const goal = req.goal?.trim() || (req.query ? `Discover public content about “${req.query}”` : `Observe the ${req.platform} feed`);
51+ const args = ["node_modules/tsx/dist/cli.mjs", "apps/worker/src/cli.ts", mode === "learn" ? "learn" : "crawl", req.platform, "--job", job_id, "--goal", goal, "--mode", mode, "--minutes", String(minutes), "--actions", String(actions), "--media", String(media), "--account", req.account?.trim() || "research-01"];
52+ if (req.query) args.push("--query", req.query);
53+ if (req.seed) args.push("--seed", req.seed);
54+ if (req.headless !== false) args.push("--headless");
55+
56+ await this.store.pool.query(`insert into crawl_jobs(job_id, platform, mode, goal, budget, status) values ($1,$2,$3,$4,$5,'queued')`, [job_id, req.platform, mode, goal, JSON.stringify({ minutes, actions, media, query: req.query ?? null, seed: req.seed ?? null, headless: req.headless !== false, account: req.account ?? "research-01" })]);
57+
58+ const logFile = path.join(this.logsDir, `${job_id}.log`);
59+ const out = fs.openSync(logFile, "a");
60+ const child = spawn(process.execPath, args, { cwd: repoRoot, env: { ...process.env, SRC_LOG_PRETTY: "1", SRC_LOG_LEVEL: "info" }, stdio: ["ignore", out, out] });
61+ this.running.set(job_id, child);
62+ await this.store.pool.query(`update crawl_jobs set status='running' where job_id=$1`, [job_id]);
63+ log.info("job started", { job_id, platform: req.platform, mode, pid: child.pid });
64+ child.on("exit", async (code, signal) => {
65+ this.running.delete(job_id);
66+ fs.closeSync(out);
67+ const status = signal ? "stopped" : code === 0 ? "done" : "failed";
68+ let result: unknown = null;
69+ try {
70+ const sid = (await this.store.pool.query(`select session_id from crawl_jobs where job_id=$1`, [job_id])).rows[0]?.session_id as string | undefined;
71+ if (sid) {
72+ const f = path.join(this.cfg.sessionsDir, sid, "summary.json");
73+ if (fs.existsSync(f)) result = JSON.parse(fs.readFileSync(f, "utf8"));
74+ }
75+ } catch {
76+ /* ignore */
77+ }
78+ await this.store.pool.query(`update crawl_jobs set status=$2, finished_at=now(), result=$3 where job_id=$1`, [job_id, status, result ? JSON.stringify(result) : null]).catch(() => {});
79+ log.info("job finished", { job_id, status, code, signal });
80+ });
81+ return { job_id };
82+ }
83+
84+ stop(job_id: string): boolean {
85+ const child = this.running.get(job_id);
86+ if (!child) return false;
87+ child.kill("SIGTERM");
88+ setTimeout(() => child.kill("SIGKILL"), 15_000).unref();
89+ return true;
90+ }
91+
92+ logTail(job_id: string, lines = 200): string {
93+ const f = path.join(this.logsDir, `${job_id}.log`);
94+ if (!fs.existsSync(f)) return "";
95+ const all = fs.readFileSync(f, "utf8").split("\n");
96+ return all.slice(-lines).join("\n");
97+ }
98+}
99+
100+function clamp(n: number, lo: number, hi: number) {
101+ return Math.max(lo, Math.min(hi, Number.isFinite(n) ? n : lo));
102+}
added apps/api/src/queries.ts +188 −0
@@ -0,0 +1,188 @@
1+import fs from "node:fs";
2+import path from "node:path";
3+import type { PostgresStore } from "@src/storage";
4+import { PLATFORMS, type AppConfig, type Platform } from "@src/shared";
5+import { PlatformModel } from "@src/platform-model";
6+
7+/** Read-side queries for the research console. All read PostgreSQL; files (models, manifests, session dirs) complete them. */
8+export class Queries {
9+ constructor(private readonly store: PostgresStore, private readonly cfg: AppConfig) {}
10+ private get q() {
11+ return this.store.pool;
12+ }
13+
14+ async overview() {
15+ const [totals, byType, running, recentSessions, series, surfaces, topActions, patterns] = await Promise.all([
16+ this.q.query(`select
17+ (select count(*) from sessions)::int as sessions,
18+ (select count(*) from entities)::int as entities,
19+ (select count(*) from media)::int as media,
20+ (select count(*) from schema_patterns)::int as schemas,
21+ (select count(*) from actions)::int as actions,
22+ (select count(*) from observations)::int as observations,
23+ (select count(*) from relationships)::int as relationships,
24+ (select count(*) from observations where event_type='CONNECTOR_PATTERN_LEARNED')::int as patterns_learned,
25+ (select count(*) from feed_items)::int as feed_items`),
26+ this.q.query(`select platform, entity_type, count(*)::int as n from entities group by 1,2 order by 3 desc`),
27+ this.q.query(`select session_id, platform, account_alias, started_at, health from sessions where ended_at is null and started_at > now() - interval '6 hours' order by started_at desc`),
28+ this.q.query(`select s.session_id, s.platform, s.goal, s.mode, s.started_at, s.ended_at, s.health,
29+ (select count(*) from actions a where a.session_id=s.session_id)::int as actions,
30+ (select count(distinct fingerprint) from entity_observations eo where eo.session_id=s.session_id)::int as entities
31+ from sessions s order by started_at desc limit 8`),
32+ this.q.query(`select date_trunc('hour', ts) + (floor(extract(minute from ts)/10)*10) * interval '1 minute' as bucket,
33+ count(*) filter (where event_type in ('VIDEO_DISCOVERED','POST_DISCOVERED','PROFILE_DISCOVERED','ENTITY_DISCOVERED'))::int as entities,
34+ count(*) filter (where event_type='NETWORK_RESPONSE_OBSERVED')::int as responses,
35+ count(*) filter (where event_type in ('ACTION_EXECUTED','ACTION_FAILED'))::int as actions,
36+ count(*) filter (where event_type='NETWORK_SCHEMA_DISCOVERED')::int as schemas
37+ from observations where ts > now() - interval '48 hours' group by 1 order by 1`),
38+ this.q.query(`select coalesce(sum((payload->>'dom_entities')::int),0)::int as dom, coalesce(sum((payload->>'network_entities')::int),0)::int as network,
39+ coalesce(sum((payload->>'both_surfaces')::int),0)::int as both, coalesce(sum((payload->>'field_agreements')::int),0)::int as agreements,
40+ coalesce(sum(jsonb_array_length(coalesce(payload->'field_conflicts','[]'::jsonb))),0)::int as conflicts
41+ from observations where event_type='PAGE_OPENED' and payload ? 'dom_entities'`),
42+ this.q.query(`select action_type, count(*)::int as n, avg(expected_gain)::float as avg_gain, avg(case when success then 1 else 0 end)::float as success_rate,
43+ avg((after_state->>'new_entities')::float) as avg_new_entities from actions group by 1 order by 2 desc`),
44+ this.q.query(`select platform, count(*)::int as n, avg((payload->>'confidence')::float) as conf from observations where event_type='CONNECTOR_PATTERN_LEARNED' group by 1`),
45+ ]);
46+ return {
47+ totals: totals.rows[0],
48+ by_type: byType.rows,
49+ running: running.rows,
50+ recent_sessions: recentSessions.rows,
51+ series: series.rows,
52+ surfaces: surfaces.rows[0],
53+ actions: topActions.rows,
54+ patterns_by_platform: patterns.rows,
55+ platforms: this.platforms(),
56+ };
57+ }
58+
59+ platforms() {
60+ return PLATFORMS.map((p) => {
61+ const file = path.join(this.cfg.platformModelDir, p, "platform_model.json");
62+ const manifest = path.join(process.cwd(), "connectors", p, "manifest.json");
63+ if (!fs.existsSync(file)) return { platform: p, learned: false, adapter: p === "youtube" || p === "reddit", confidence: 0, page_types: 0, entity_types: 0, navigation_actions: 0, network_schemas: 0, media_patterns: 0, sessions: 0, has_manifest: fs.existsSync(manifest), profiles: this.profilesFor(p) };
64+ const m = new PlatformModel(p, this.cfg.platformModelDir);
65+ return { learned: true, adapter: p === "youtube" || p === "reddit", has_manifest: fs.existsSync(manifest), profiles: this.profilesFor(p), ...m.summary() };
66+ });
67+ }
68+
69+ profilesFor(p: Platform) {
70+ if (!fs.existsSync(this.cfg.profilesDir)) return [];
71+ return fs
72+ .readdirSync(this.cfg.profilesDir)
73+ .filter((d) => d.startsWith(`${p}-`))
74+ .map((d) => ({ alias: d.slice(p.length + 1), has_state: fs.existsSync(path.join(this.cfg.profilesDir, d, "Default", "Cookies")) || fs.existsSync(path.join(this.cfg.profilesDir, d, "Default", "Network", "Cookies")) }));
75+ }
76+
77+ platformDetail(p: Platform) {
78+ const file = path.join(this.cfg.platformModelDir, p, "platform_model.json");
79+ const manifest = path.join(process.cwd(), "connectors", p, "manifest.json");
80+ return {
81+ platform: p,
82+ model: fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : null,
83+ manifest: fs.existsSync(manifest) ? JSON.parse(fs.readFileSync(manifest, "utf8")) : null,
84+ summary: fs.existsSync(file) ? new PlatformModel(p, this.cfg.platformModelDir).summary() : null,
85+ profiles: this.profilesFor(p),
86+ };
87+ }
88+
89+ async sessions(limit = 100) {
90+ return (
91+ await this.q.query(
92+ `select s.*, (select count(*) from actions a where a.session_id=s.session_id)::int as actions,
93+ (select count(distinct fingerprint) from entity_observations eo where eo.session_id=s.session_id)::int as entities,
94+ (select count(*) from observations o where o.session_id=s.session_id and o.event_type='NETWORK_SCHEMA_DISCOVERED')::int as schemas,
95+ (select count(*) from observations o where o.session_id=s.session_id and o.event_type='CONNECTOR_PATTERN_LEARNED')::int as patterns,
96+ (select payload->>'reason' from observations o where o.session_id=s.session_id and o.event_type='BUDGET_EXHAUSTED' limit 1) as ended_because
97+ from sessions s order by started_at desc limit $1`,
98+ [limit],
99+ )
100+ ).rows;
101+ }
102+
103+ async session(id: string) {
104+ const row = (await this.q.query(`select * from sessions where session_id=$1`, [id])).rows[0];
105+ if (!row) return null;
106+ const dir = path.join(this.cfg.sessionsDir, id);
107+ const read = (f: string) => {
108+ try {
109+ return JSON.parse(fs.readFileSync(path.join(dir, f), "utf8"));
110+ } catch {
111+ return null;
112+ }
113+ };
114+ const counts = (await this.q.query(`select event_type, count(*)::int as n from observations where session_id=$1 group by 1`, [id])).rows;
115+ return { ...row, job: read("job.json"), summary: read("summary.json"), event_counts: counts };
116+ }
117+
118+ async pages(id: string) {
119+ return (await this.q.query(`select step, ts, payload from observations where session_id=$1 and event_type='PAGE_OPENED' and payload ? 'page_type' order by ts`, [id])).rows;
120+ }
121+
122+ async world(id: string) {
123+ const dir = path.join(this.cfg.sessionsDir, id, "world_model.json");
124+ if (fs.existsSync(dir)) return JSON.parse(fs.readFileSync(dir, "utf8"));
125+ // Reconstruct from the database for sessions crawled on another machine.
126+ const nodes = (await this.q.query(`select e.fingerprint, e.entity_type as type, e.name, e.url, e.platform_id, min(eo.step) as first_seen_step, max(eo.step) as last_seen_step, count(*)::int as seen_count from entities e join entity_observations eo on eo.fingerprint=e.fingerprint where eo.session_id=$1 group by e.fingerprint limit 1500`, [id])).rows;
127+ const edges = (await this.q.query(`select from_fp as "from", to_fp as "to", rel_type as type, step from relationships where session_id=$1 limit 4000`, [id])).rows;
128+ return { nodes, edges };
129+ }
130+
131+ async searchEntities(o: { q?: string; type?: string; platform?: string; limit: number; offset: number; sort?: string }) {
132+ const params: unknown[] = [];
133+ const where: string[] = [];
134+ if (o.q) {
135+ params.push(`%${o.q}%`);
136+ where.push(`(name ilike $${params.length} or text_excerpt ilike $${params.length} or author ilike $${params.length} or platform_id ilike $${params.length})`);
137+ }
138+ if (o.type) {
139+ params.push(o.type);
140+ where.push(`entity_type = $${params.length}`);
141+ }
142+ if (o.platform) {
143+ params.push(o.platform);
144+ where.push(`platform = $${params.length}`);
145+ }
146+ const w = where.length ? `where ${where.join(" and ")}` : "";
147+ const order = o.sort === "views" ? `(metrics->>'views')::numeric desc nulls last` : o.sort === "seen" ? `seen_count desc` : `last_seen desc`;
148+ params.push(o.limit, o.offset);
149+ const rows = (await this.q.query(`select *, (select count(*) from entity_observations eo where eo.fingerprint=entities.fingerprint)::int as observations from entities ${w} order by ${order} limit $${params.length - 1} offset $${params.length}`, params)).rows;
150+ const total = (await this.q.query(`select count(*)::int as n from entities ${w}`, params.slice(0, -2))).rows[0]?.n ?? 0;
151+ return { rows, total };
152+ }
153+
154+ async entity(fp: string) {
155+ const e = (await this.q.query(`select * from entities where fingerprint=$1`, [fp])).rows[0];
156+ if (!e) return null;
157+ const observations = (await this.q.query(`select eo.session_id, eo.step, eo.ts, eo.surfaces, eo.snapshot, s.goal from entity_observations eo left join sessions s on s.session_id=eo.session_id where eo.fingerprint=$1 order by eo.ts desc limit 100`, [fp])).rows;
158+ const relations = (await this.q.query(`select r.rel_type, r.from_fp, r.to_fp, r.step, coalesce(a.name, a.platform_id) as from_name, coalesce(b.name, b.platform_id) as to_name, a.entity_type as from_type, b.entity_type as to_type from relationships r left join entities a on a.fingerprint=r.from_fp left join entities b on b.fingerprint=r.to_fp where r.from_fp=$1 or r.to_fp=$1 limit 200`, [fp])).rows;
159+ const media = (await this.q.query(`select * from media where fingerprint=$1`, [fp])).rows[0] ?? null;
160+ const feed = (await this.q.query(`select session_id, step, page_type, feed_position, visible, ts from feed_items where fingerprint=$1 order by ts desc limit 50`, [fp])).rows;
161+ return { entity: e, observations, relations, media, feed };
162+ }
163+
164+ async schemas(platform?: string) {
165+ const params: unknown[] = [];
166+ let w = "";
167+ if (platform) {
168+ params.push(platform);
169+ w = `where platform=$1`;
170+ }
171+ return (await this.q.query(`select * from schema_patterns ${w} order by observed_count desc limit 300`, params)).rows;
172+ }
173+
174+ async liveEvents(since: string, session?: string, limit = 200) {
175+ const params: unknown[] = [since];
176+ let w = `ts > $1`;
177+ if (session) {
178+ params.push(session);
179+ w += ` and session_id=$2`;
180+ }
181+ params.push(limit);
182+ return (await this.q.query(`select event_id, session_id, platform, event_type, step, ts, payload, provenance from observations where ${w} order by ts asc limit $${params.length}`, params)).rows;
183+ }
184+
185+ async jobs(limit = 50) {
186+ return (await this.q.query(`select j.*, s.health, (select count(*) from actions a where a.session_id=j.session_id)::int as actions, (select count(distinct fingerprint) from entity_observations eo where eo.session_id=j.session_id)::int as entities from crawl_jobs j left join sessions s on s.session_id=j.session_id order by created_at desc limit $1`, [limit])).rows;
187+ }
188+}
modified apps/api/src/server.ts +115 −74
@@ -2,108 +2,149 @@ import http from "node:http";
2 import fs from "node:fs"; 2 import fs from "node:fs";
3 import path from "node:path"; 3 import path from "node:path";
4 import { fileURLToPath } from "node:url"; 4 import { fileURLToPath } from "node:url";
5−import { createLogger, loadConfig } from "@src/shared"; 5+import { createLogger, isPlatform, loadConfig } from "@src/shared";
6−import { JsonlEventLog } from "@src/events";
7 import { PostgresStore } from "@src/storage"; 6 import { PostgresStore } from "@src/storage";
7+import { Queries } from "./queries.ts";
8+import { JobRunner } from "./jobs.ts";
8 9
9 /** 10 /**
10− * Research/Debug dashboard API (§53, §82). Zero framework: node:http + a static page. 11+ * Control-plane + research API (§53, §63, §82). Plain node:http, JSON, SSE.
11− * Reads PostgreSQL when configured, else the JSONL session logs. 12+ * Consumed by the Next.js console (apps/dashboard) through its authenticated proxy; mutations require SRC_API_TOKEN.
12 */ 13 */
13−const log = createLogger("dashboard"); 14+const log = createLogger("api");
14 const cfg = loadConfig(); 15 const cfg = loadConfig();
15 const here = path.dirname(fileURLToPath(import.meta.url)); 16 const here = path.dirname(fileURLToPath(import.meta.url));
16 const publicDir = path.join(here, "..", "public"); 17 const publicDir = path.join(here, "..", "public");
17−let store: PostgresStore | undefined; 18+const API_TOKEN = process.env.SRC_API_TOKEN ?? "";
18−try { 19+const PORT = Number(process.env.SRC_API_PORT ?? cfg.dashboardPort);
19− if (cfg.databaseUrl) store = await PostgresStore.connect(cfg.databaseUrl); 20+
20−} catch (err) { 21+if (!cfg.databaseUrl) {
21− log.warn("no database — serving JSONL logs", { err: (err as Error).message }); 22+ console.error("SRC_DATABASE_URL is required for the API");
23+ process.exit(1);
22 } 24 }
25+const store = await PostgresStore.connect(cfg.databaseUrl);
26+await store.migrate();
27+const queries = new Queries(store, cfg);
28+const jobs = new JobRunner(store, cfg, Number(process.env.SRC_MAX_JOBS ?? 2));
23 29
24 function json(res: http.ServerResponse, data: unknown, status = 200) { 30 function json(res: http.ServerResponse, data: unknown, status = 200) {
25− res.writeHead(status, { "content-type": "application/json; charset=utf-8", "access-control-allow-origin": "*" }); 31+ res.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" });
26 res.end(JSON.stringify(data)); 32 res.end(JSON.stringify(data));
27 } 33 }
28− 34+function readBody(req: http.IncomingMessage): Promise<string> {
29−function sessionDirs(): string[] { 35+ return new Promise((resolve, reject) => {
30− return fs.existsSync(cfg.sessionsDir) ? fs.readdirSync(cfg.sessionsDir).filter((d) => fs.existsSync(path.join(cfg.sessionsDir, d, "events.jsonl"))).sort().reverse() : []; 36+ let s = "";
31−} 37+ req.on("data", (c) => {
32− 38+ s += c;
33−function jsonlSessions() { 39+ if (s.length > 1e6) reject(new Error("body too large"));
34− return sessionDirs().map((id) => { 40+ });
35− const job = readJson(path.join(cfg.sessionsDir, id, "job.json")) as Record<string, unknown> | undefined; 41+ req.on("end", () => resolve(s));
36− const summary = readJson(path.join(cfg.sessionsDir, id, "summary.json")) as Record<string, unknown> | undefined; 42+ req.on("error", reject);
37− return { session_id: id, platform: job?.platform, mode: job?.mode, goal: job?.goal, account_alias: job?.account_alias, started_at: fs.statSync(path.join(cfg.sessionsDir, id)).birthtime, health: summary ? "stopped" : "running", actions: summary?.steps ?? null, entities: summary?.entities ?? null, ended_because: summary?.ended_because };
38 }); 43 });
39 } 44 }
40− 45+function authorized(req: http.IncomingMessage): boolean {
41−function readJson(file: string): unknown { 46+ if (!API_TOKEN) return true; // dev
42− try { 47+ return req.headers["x-src-token"] === API_TOKEN;
43− return JSON.parse(fs.readFileSync(file, "utf8"));
44− } catch {
45− return undefined;
46− }
47 } 48 }
48 49
49 const server = http.createServer(async (req, res) => { 50 const server = http.createServer(async (req, res) => {
50 const url = new URL(req.url ?? "/", "http://localhost"); 51 const url = new URL(req.url ?? "/", "http://localhost");
51 const parts = url.pathname.split("/").filter(Boolean); 52 const parts = url.pathname.split("/").filter(Boolean);
53+ const method = req.method ?? "GET";
52 try { 54 try {
53− if (parts[0] === "api") { 55+ if (parts[0] !== "api" && parts[0] !== "media") {
54− if (parts[1] === "sessions" && !parts[2]) return json(res, store ? await store.listSessions() : jsonlSessions()); 56+ // legacy static console (apps/api/public) — kept for local debugging
55− if (parts[1] === "sessions" && parts[2]) { 57+ const file = path.join(publicDir, parts.length ? parts.join("/") : "index.html");
56− const id = parts[2]; 58+ if (!file.startsWith(publicDir) || !fs.existsSync(file)) return json(res, { error: "not found" }, 404);
57− const dir = path.join(cfg.sessionsDir, id); 59+ res.writeHead(200, { "content-type": file.endsWith(".html") ? "text/html; charset=utf-8" : "application/octet-stream" });
58− const sub = parts[3]; 60+ return fs.createReadStream(file).pipe(res);
59− if (sub === "events") { 61+ }
60− const types = url.searchParams.get("types")?.split(",").filter(Boolean); 62+ if (parts[0] === "media") {
61− const limit = Number(url.searchParams.get("limit") ?? 300); 63+ const file = path.join(cfg.mediaDir, ...parts.slice(1).map(decodeURIComponent));
62− if (store) return json(res, await store.sessionEvents(id, { types, limit })); 64+ if (!file.startsWith(cfg.mediaDir) || !fs.existsSync(file)) return json(res, { error: "not found" }, 404);
63− const evs = JsonlEventLog.read(dir).filter((e) => !types?.length || types.includes(e.event_type)); 65+ res.writeHead(200, { "content-type": "image/jpeg", "cache-control": "public, max-age=86400" });
64− return json(res, evs.slice(-limit).reverse()); 66+ return fs.createReadStream(file).pipe(res);
65− } 67+ }
66− if (sub === "actions") return json(res, store ? await store.sessionActions(id) : JsonlEventLog.read(dir).filter((e) => e.event_type === "ACTION_PLANNED").map((e) => ({ step: e.step, ...(e.payload as Record<string, unknown>) }))); 68+ if (method !== "GET" && !authorized(req)) return json(res, { error: "unauthorized" }, 401);
67− if (sub === "entities") { 69+ const [, r1, r2, r3] = parts;
68− if (store) return json(res, await store.sessionEntities(id)); 70+
69− const seen = new Map<string, unknown>(); 71+ if (r1 === "health") return json(res, { ok: true, jobs_running: jobs.activeCount, db: true, ts: new Date().toISOString() });
70− for (const e of JsonlEventLog.read(dir)) if (/DISCOVERED|ENTITY_OBSERVED/.test(e.event_type) && (e.payload as { entity?: { fingerprint: string } }).entity) seen.set((e.payload as { entity: { fingerprint: string } }).entity.fingerprint, { ...(e.payload as { entity: object }).entity, last_step: e.step, provenance: e.provenance }); 72+ if (r1 === "overview") return json(res, await queries.overview());
71− return json(res, [...seen.values()].reverse()); 73+ if (r1 === "platforms" && !r2) return json(res, queries.platforms());
72− } 74+ if (r1 === "platforms" && r2) return isPlatform(r2) ? json(res, queries.platformDetail(r2)) : json(res, { error: "unknown platform" }, 404);
73− if (sub === "media") return json(res, store ? await store.sessionMedia(id) : JsonlEventLog.read(dir).filter((e) => e.event_type === "MEDIA_DISCOVERED").map((e) => e.payload)); 75+ if (r1 === "sessions" && !r2) return json(res, await queries.sessions(Number(url.searchParams.get("limit") ?? 100)));
74− if (sub === "world") return json(res, readJson(path.join(dir, "world_model.json")) ?? { nodes: [], edges: [] }); 76+ if (r1 === "sessions" && r2) {
75− if (sub === "summary") return json(res, { job: readJson(path.join(dir, "job.json")), summary: readJson(path.join(dir, "summary.json")) }); 77+ const id = r2;
76− if (sub === "schemas") { 78+ if (!r3) {
77− const platform = (readJson(path.join(dir, "job.json")) as { platform?: string } | undefined)?.platform; 79+ const s = await queries.session(id);
78− if (store) return json(res, await store.schemas(platform)); 80+ return s ? json(res, s) : json(res, { error: "not found" }, 404);
79− return json(res, JsonlEventLog.read(dir).filter((e) => e.event_type === "NETWORK_SCHEMA_DISCOVERED").map((e) => e.payload)); 81+ }
82+ if (r3 === "events") return json(res, await store.sessionEvents(id, { types: url.searchParams.get("types")?.split(",").filter(Boolean), limit: Number(url.searchParams.get("limit") ?? 300) }));
83+ if (r3 === "actions") return json(res, await store.sessionActions(id));
84+ if (r3 === "entities") return json(res, await store.sessionEntities(id, Number(url.searchParams.get("limit") ?? 500)));
85+ if (r3 === "media") return json(res, await store.sessionMedia(id));
86+ if (r3 === "pages") return json(res, await queries.pages(id));
87+ if (r3 === "world") return json(res, await queries.world(id));
88+ if (r3 === "schemas") {
89+ const s = await queries.session(id);
90+ return json(res, await queries.schemas(s?.platform));
91+ }
92+ if (r3 === "relationships") return json(res, await store.sessionRelationships(id));
93+ }
94+ if (r1 === "entities" && !r2) return json(res, await queries.searchEntities({ q: url.searchParams.get("q") ?? undefined, type: url.searchParams.get("type") ?? undefined, platform: url.searchParams.get("platform") ?? undefined, sort: url.searchParams.get("sort") ?? undefined, limit: Math.min(200, Number(url.searchParams.get("limit") ?? 50)), offset: Number(url.searchParams.get("offset") ?? 0) }));
95+ if (r1 === "entities" && r2) {
96+ const e = await queries.entity(decodeURIComponent(r2));
97+ return e ? json(res, e) : json(res, { error: "not found" }, 404);
98+ }
99+ if (r1 === "schemas") return json(res, await queries.schemas(url.searchParams.get("platform") ?? undefined));
100+ if (r1 === "stats") return json(res, await store.entityStats());
101+
102+ if (r1 === "stream") {
103+ // Server-Sent Events: poll the observations table and push new rows (§53 live panels).
104+ res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-store", connection: "keep-alive", "x-accel-buffering": "no" });
105+ let since = url.searchParams.get("since") ?? new Date(Date.now() - 60_000).toISOString();
106+ const session = url.searchParams.get("session") ?? undefined;
107+ const types = url.searchParams.get("types")?.split(",").filter(Boolean);
108+ let alive = true;
109+ req.on("close", () => (alive = false));
110+ res.write(`event: hello\ndata: ${JSON.stringify({ since })}\n\n`);
111+ while (alive) {
112+ try {
113+ const rows = await queries.liveEvents(since, session, 200);
114+ for (const row of rows) {
115+ since = new Date(row.ts).toISOString();
116+ if (types && !types.includes(row.event_type)) continue;
117+ res.write(`event: observation\ndata: ${JSON.stringify(row)}\n\n`);
118+ }
119+ res.write(`: ping ${Date.now()}\n\n`);
120+ } catch (err) {
121+ res.write(`event: error\ndata: ${JSON.stringify({ error: (err as Error).message })}\n\n`);
80 } 122 }
123+ await new Promise((r) => setTimeout(r, 1500));
81 } 124 }
82− if (parts[1] === "platform-model" && parts[2]) { 125+ return res.end();
83− const f = path.join(cfg.platformModelDir, parts[2], "platform_model.json"); 126+ }
84− return json(res, readJson(f) ?? { error: "no model yet" }); 127+
128+ if (r1 === "jobs" && method === "GET" && !r2) return json(res, await queries.jobs());
129+ if (r1 === "jobs" && method === "POST" && !r2) {
130+ const body = JSON.parse((await readBody(req)) || "{}");
131+ try {
132+ return json(res, await jobs.start(body), 201);
133+ } catch (err) {
134+ return json(res, { error: (err as Error).message }, 400);
85 } 135 }
86− if (parts[1] === "entity" && parts[2]) return json(res, store ? await store.entity(decodeURIComponent(parts[2])) : { error: "needs database" });
87− if (parts[1] === "stats") return json(res, store ? await store.entityStats() : []);
88− return json(res, { error: "not found" }, 404);
89 } 136 }
90− if (parts[0] === "media") { 137+ if (r1 === "jobs" && r2 && r3 === "log") {
91− // serve sampled frames 138+ res.writeHead(200, { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" });
92− const file = path.join(cfg.mediaDir, ...parts.slice(1)); 139+ return res.end(jobs.logTail(r2, Number(url.searchParams.get("lines") ?? 200)));
93− if (!file.startsWith(cfg.mediaDir) || !fs.existsSync(file)) return json(res, { error: "not found" }, 404);
94− res.writeHead(200, { "content-type": "image/jpeg" });
95− fs.createReadStream(file).pipe(res);
96− return;
97 } 140 }
98− const file = path.join(publicDir, parts.length ? parts.join("/") : "index.html"); 141+ if (r1 === "jobs" && r2 && r3 === "stop" && method === "POST") return json(res, { stopped: jobs.stop(r2) });
99− if (!file.startsWith(publicDir) || !fs.existsSync(file)) return json(res, { error: "not found" }, 404); 142+
100− const ext = path.extname(file); 143+ return json(res, { error: "not found" }, 404);
101− res.writeHead(200, { "content-type": ext === ".html" ? "text/html; charset=utf-8" : ext === ".js" ? "text/javascript" : ext === ".css" ? "text/css" : "application/octet-stream" });
102− fs.createReadStream(file).pipe(res);
103 } catch (err) { 144 } catch (err) {
104 log.error("request failed", { url: req.url, err: (err as Error).message }); 145 log.error("request failed", { url: req.url, err: (err as Error).message });
105 json(res, { error: (err as Error).message }, 500); 146 json(res, { error: (err as Error).message }, 500);
106 } 147 }
107 }); 148 });
108 149
109−server.listen(cfg.dashboardPort, () => log.info(`dashboard → http://localhost:${cfg.dashboardPort} (${store ? "postgres" : "jsonl"})`)); 150+server.listen(PORT, "127.0.0.1", () => log.info(`api → http://127.0.0.1:${PORT} (token ${API_TOKEN ? "on" : "off"})`));
added apps/dashboard/next.config.ts +13 −0
@@ -0,0 +1,13 @@
1+import type { NextConfig } from "next";
2+
3+const nextConfig: NextConfig = {
4+ reactStrictMode: true,
5+ poweredByHeader: false,
6+ images: { unoptimized: true },
7+ experimental: { serverActions: { allowedOrigins: ["www.socialcrawl.co", "socialcrawl.co", "localhost:8351"] } },
8+ async headers() {
9+ return [{ source: "/(.*)", headers: [{ key: "X-Frame-Options", value: "DENY" }, { key: "X-Content-Type-Options", value: "nosniff" }, { key: "Referrer-Policy", value: "same-origin" }] }];
10+ },
11+};
12+
13+export default nextConfig;
added apps/dashboard/package.json +31 −0
@@ -0,0 +1,31 @@
1+{
2+ "name": "@src/dashboard",
3+ "version": "0.1.0",
4+ "private": true,
5+ "type": "module",
6+ "scripts": {
7+ "dev": "next dev -p 8351",
8+ "build": "next build",
9+ "start": "next start -p 8351 -H 0.0.0.0",
10+ "typecheck": "tsc -p tsconfig.json --noEmit"
11+ },
12+ "dependencies": {
13+ "clsx": "^2.1.1",
14+ "d3-force": "^3.0.0",
15+ "framer-motion": "^12.23.0",
16+ "lucide-react": "^1.0.0",
17+ "next": "16.3.4",
18+ "react": "19.2.8",
19+ "react-dom": "19.2.8",
20+ "recharts": "^3.0.0"
21+ },
22+ "devDependencies": {
23+ "@tailwindcss/postcss": "^4",
24+ "@types/d3-force": "^3.0.10",
25+ "@types/node": "^24.0.0",
26+ "@types/react": "^19",
27+ "@types/react-dom": "^19",
28+ "tailwindcss": "^4",
29+ "typescript": "^5.9.3"
30+ }
31+}
added apps/dashboard/postcss.config.mjs +2 −0
@@ -0,0 +1,2 @@
1+const config = { plugins: { "@tailwindcss/postcss": {} } };
2+export default config;
added apps/dashboard/public/brand/icon.svg +1 −0
@@ -0,0 +1 @@
1+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48"><rect width="48" height="48" rx="10" fill="#05070b"/><defs><linearGradient id="g" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#38e1ff"/><stop offset="1" stop-color="#9d7bff"/></linearGradient></defs><ellipse cx="24" cy="24" rx="19" ry="10" fill="none" stroke="url(#g)" stroke-width="2.2" transform="rotate(-24 24 24)"/><ellipse cx="24" cy="24" rx="19" ry="10" fill="none" stroke="url(#g)" stroke-width="1.2" opacity=".5" transform="rotate(36 24 24)"/><circle cx="24" cy="24" r="5.5" fill="url(#g)"/><circle cx="40" cy="16" r="2.2" fill="#ffb347"/></svg>
added apps/dashboard/src/app/access/page.tsx +39 −0
@@ -0,0 +1,39 @@
1+import { cookies } from "next/headers";
2+import { redirect } from "next/navigation";
3+import { COOKIE, checkPasscode, gateEnabled, makeToken } from "@/lib/auth";
4+import { Brand } from "@/components/Brand";
5+
6+export const dynamic = "force-dynamic";
7+
8+async function login(formData: FormData) {
9+ "use server";
10+ const code = String(formData.get("passcode") ?? "");
11+ const next = String(formData.get("next") ?? "/");
12+ if (!checkPasscode(code)) redirect(`/access?error=1&next=${encodeURIComponent(next)}`);
13+ const jar = await cookies();
14+ jar.set(COOKIE, makeToken(), { httpOnly: true, sameSite: "lax", secure: process.env.NODE_ENV === "production", path: "/", maxAge: 30 * 24 * 3600 });
15+ redirect(next.startsWith("/") ? next : "/");
16+}
17+
18+export default async function AccessPage({ searchParams }: { searchParams: Promise<{ error?: string; next?: string }> }) {
19+ const sp = await searchParams;
20+ if (!gateEnabled()) redirect("/");
21+ return (
22+ <main className="min-h-screen grid place-items-center grid-bg px-6">
23+ <form action={login} className="panel w-full max-w-sm p-8 space-y-6">
24+ <Brand size="lg" />
25+ <div className="space-y-1">
26+ <h1 className="text-lg font-semibold">Research console</h1>
27+ <p className="text-sm text-fg-2">This console observes live social runtimes and can start browser sessions on the cluster. Enter the access code.</p>
28+ </div>
29+ <input type="hidden" name="next" value={sp.next ?? "/"} />
30+ <label className="block space-y-2">
31+ <span className="text-xs uppercase tracking-widest text-dim">Access code</span>
32+ <input name="passcode" type="password" autoFocus required className="w-full rounded-lg bg-bg-2 border border-line-2 px-3 py-2 mono text-fg outline-none focus:border-cyan/60 focus:glow-cyan" />
33+ </label>
34+ {sp.error && <p className="text-sm text-red">Wrong code.</p>}
35+ <button className="w-full rounded-lg bg-cyan/90 hover:bg-cyan text-bg font-semibold py-2 transition">Enter</button>
36+ </form>
37+ </main>
38+ );
39+}
added apps/dashboard/src/app/api/[...path]/route.ts +28 −0
@@ -0,0 +1,28 @@
1+import { cookies } from "next/headers";
2+import { NextResponse, type NextRequest } from "next/server";
3+import { COOKIE, verifyToken } from "@/lib/auth";
4+
5+export const dynamic = "force-dynamic";
6+const API_URL = process.env.API_URL ?? "http://127.0.0.1:8350";
7+const API_TOKEN = process.env.SRC_API_TOKEN ?? "";
8+
9+/** Authenticated pass-through to the control-plane API (adds the server token, streams SSE as-is). */
10+async function forward(req: NextRequest, path: string[]) {
11+ const jar = await cookies();
12+ if (!verifyToken(jar.get(COOKIE)?.value)) return NextResponse.json({ error: "unauthorized" }, { status: 401 });
13+ const target = new URL(`/api/${path.join("/")}`, API_URL);
14+ target.search = req.nextUrl.search;
15+ const headers: Record<string, string> = { "x-src-token": API_TOKEN };
16+ if (req.headers.get("content-type")) headers["content-type"] = req.headers.get("content-type")!;
17+ const upstream = await fetch(target, { method: req.method, headers, body: req.method === "GET" || req.method === "HEAD" ? undefined : await req.text(), cache: "no-store", // @ts-expect-error — undici option for streaming request bodies
18+ duplex: "half" });
19+ const ct = upstream.headers.get("content-type") ?? "application/json";
20+ return new Response(upstream.body, { status: upstream.status, headers: { "content-type": ct, "cache-control": "no-store", ...(ct.includes("event-stream") ? { connection: "keep-alive", "x-accel-buffering": "no" } : {}) } });
21+}
22+
23+export async function GET(req: NextRequest, ctx: { params: Promise<{ path: string[] }> }) {
24+ return forward(req, (await ctx.params).path);
25+}
26+export async function POST(req: NextRequest, ctx: { params: Promise<{ path: string[] }> }) {
27+ return forward(req, (await ctx.params).path);
28+}
added apps/dashboard/src/app/entities/[fp]/page.tsx +173 −0
@@ -0,0 +1,173 @@
1+"use client";
2+
3+import { use } from "react";
4+import Link from "next/link";
5+import { ExternalLink } from "lucide-react";
6+import { useApi } from "@/lib/api";
7+import { fmtDate, fmtDuration, fmtNum, truncate } from "@/lib/format";
8+import { Empty, Metric, Panel, Skeleton, SurfaceChips, Table, Tag, TypeTag } from "@/components/ui";
9+import type { EntityRow } from "@/components/EntityTable";
10+
11+interface Detail {
12+ entity: EntityRow & { first_seen: string; last_seen: string; seen_count: number; first_session: string };
13+ observations: { session_id: string; step: number; ts: string; surfaces: string[]; snapshot: { name?: string; author?: string; metrics?: Record<string, number>; context?: string; provenance?: { surface: string; confidence: number; detail?: string }[] }; goal: string | null }[];
14+ relations: { rel_type: string; from_fp: string; to_fp: string; from_name: string | null; to_name: string | null; from_type: string; to_type: string; step: number }[];
15+ media: { duration_s: number | null; width: number | null; height: number | null; delivery: { kind: string; hostnames: string[] } | null; frames: string[] | null; thumbnail_url: string | null } | null;
16+ feed: { session_id: string; step: number; page_type: string; feed_position: number; visible: boolean; ts: string }[];
17+}
18+
19+export default function EntityPage({ params }: { params: Promise<{ fp: string }> }) {
20+ const { fp } = use(params);
21+ const { data, loading, error } = useApi<Detail>(`/entities/${encodeURIComponent(decodeURIComponent(fp))}`);
22+ if (loading) return <Skeleton rows={10} />;
23+ if (error || !data) return <Empty>{error ?? "not found"}</Empty>;
24+ const e = data.entity;
25+ const type = e.entity_type ?? e.type ?? "?";
26+ return (
27+ <div className="space-y-4">
28+ <div className="text-xs text-dim mono">
29+ <Link href="/entities" className="hover:text-cyan">entities</Link> / {e.fingerprint}
30+ </div>
31+ <div className="flex flex-wrap items-start gap-4">
32+ <div className="min-w-0">
33+ <div className="flex items-center gap-2">
34+ <TypeTag type={type} />
35+ <Tag>{e.platform}</Tag>
36+ {e.url && (
37+ <a href={e.url} target="_blank" rel="noreferrer" className="text-dim hover:text-cyan flex items-center gap-1 text-xs">
38+ open on platform <ExternalLink className="h-3.5 w-3.5" />
39+ </a>
40+ )}
41+ </div>
42+ <h1 className="text-xl font-semibold tracking-tight mt-2 max-w-3xl">{e.name ?? e.text_excerpt ?? e.platform_id}</h1>
43+ {e.author && <div className="text-sm text-fg-2 mt-1">by {e.author}</div>}
44+ {e.text_excerpt && e.name && <p className="text-sm text-fg-2 mt-2 max-w-3xl">{e.text_excerpt}</p>}
45+ </div>
46+ </div>
47+
48+ <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">
49+ <Panel title="Evidence per field" sub="every canonical value points back to the surface that produced it (§38)" className="xl:col-span-2" pad={false}>
50+ <div className="p-4">
51+ <Table head={["field", "value", "evidence"]} dense>
52+ {Object.entries(e.fields ?? {}).map(([k, f]) => (
53+ <tr key={k}>
54+ <td className="mono text-fg-2 whitespace-nowrap">{k}</td>
55+ <td className="max-w-[480px] break-words">{typeof f.value === "string" ? f.value : JSON.stringify(f.value)}</td>
56+ <td>
57+ <SurfaceChips provenance={f.provenance} />
58+ {f.provenance?.[0]?.detail && <div className="text-[10px] text-dim mono truncate max-w-[280px]">{f.provenance[0].detail}</div>}
59+ </td>
60+ </tr>
61+ ))}
62+ </Table>
63+ </div>
64+ </Panel>
65+ <div className="space-y-4">
66+ <Panel title="Canonical record">
67+ <Metric label="platform id" value={e.platform_id ?? "—"} />
68+ <Metric label="confidence" value={e.confidence ? Number(e.confidence).toFixed(2) : "—"} />
69+ <Metric label="observed" value={`${e.seen_count}× in ${new Set(data.observations.map((o) => o.session_id)).size} session(s)`} />
70+ <Metric label="first seen" value={fmtDate(e.first_seen)} />
71+ <Metric label="last seen" value={fmtDate(e.last_seen)} />
72+ {e.metrics &&
73+ Object.entries(e.metrics).map(([k, v]) => (
74+ <Metric key={k} label={k} value={fmtNum(v)} />
75+ ))}
76+ {data.media && (
77+ <>
78+ <Metric label="duration" value={fmtDuration(data.media.duration_s)} />
79+ {data.media.width ? <Metric label="resolution" value={`${data.media.width}×${data.media.height}`} /> : null}
80+ {data.media.delivery && <Metric label="delivery" value={`${data.media.delivery.kind} · ${data.media.delivery.hostnames.slice(0, 2).join(", ")}`} />}
81+ </>
82+ )}
83+ </Panel>
84+ {data.media?.frames?.length ? (
85+ <Panel title="Sampled frames" sub="0 / 25 / 50 / 75 / 100 % (LEVEL 2)">
86+ <div className="grid grid-cols-5 gap-1">
87+ {data.media.frames.map((f) => (
88+ // eslint-disable-next-line @next/next/no-img-element
89+ <img key={f} src={`/media/${f.split("/media/")[1] ?? ""}`} alt="" className="aspect-video object-cover rounded" />
90+ ))}
91+ </div>
92+ </Panel>
93+ ) : data.media?.thumbnail_url || e.media?.thumbnail_url ? (
94+ <Panel title="Thumbnail">
95+ {/* eslint-disable-next-line @next/next/no-img-element */}
96+ <img src={data.media?.thumbnail_url ?? e.media?.thumbnail_url ?? ""} alt="" className="rounded w-full" />
97+ </Panel>
98+ ) : null}
99+ </div>
100+ </div>
101+
102+ <div className="grid grid-cols-1 xl:grid-cols-2 gap-4">
103+ <Panel title="Observations" sub="raw sightings, never overwritten (§37)" pad={false}>
104+ <div className="p-4">
105+ <Table head={["when", "session", "step", "surfaces", "context", "name as seen"]} dense>
106+ {data.observations.map((o, i) => (
107+ <tr key={i}>
108+ <td className="mono text-dim whitespace-nowrap">{fmtDate(o.ts)}</td>
109+ <td>
110+ <Link href={`/sessions/${o.session_id}`} className="mono text-[11px] hover:text-cyan">
111+ {truncate(o.goal ?? o.session_id, 32)}
112+ </Link>
113+ </td>
114+ <td className="mono">#{o.step}</td>
115+ <td>
116+ <SurfaceChips provenance={o.snapshot.provenance ?? o.surfaces.map((s) => ({ surface: s, confidence: 0 }))} />
117+ </td>
118+ <td className="text-dim text-[11px]">{o.snapshot.context}</td>
119+ <td className="text-fg-2 max-w-[260px] truncate">{o.snapshot.name}</td>
120+ </tr>
121+ ))}
122+ </Table>
123+ </div>
124+ </Panel>
125+ <div className="space-y-4">
126+ <Panel title="Relationships" pad={false}>
127+ <div className="p-4">
128+ {data.relations.length ? (
129+ <Table head={["from", "relation", "to"]} dense>
130+ {data.relations.map((r, i) => (
131+ <tr key={i}>
132+ <td className="max-w-[240px] truncate">
133+ <TypeTag type={r.from_type ?? "?"} /> <Link href={`/entities/${encodeURIComponent(r.from_fp)}`} className="hover:text-cyan">{truncate(r.from_name ?? r.from_fp, 40)}</Link>
134+ </td>
135+ <td>
136+ <Tag color="#9d7bff">{r.rel_type}</Tag>
137+ </td>
138+ <td className="max-w-[240px] truncate">
139+ <TypeTag type={r.to_type ?? "?"} /> <Link href={`/entities/${encodeURIComponent(r.to_fp)}`} className="hover:text-cyan">{truncate(r.to_name ?? r.to_fp, 40)}</Link>
140+ </td>
141+ </tr>
142+ ))}
143+ </Table>
144+ ) : (
145+ <Empty>no relationships recorded</Empty>
146+ )}
147+ </div>
148+ </Panel>
149+ <Panel title="Feed exposure" sub="where it ranked when the crawler saw it (§44)" pad={false}>
150+ <div className="p-4">
151+ {data.feed.length ? (
152+ <Table head={["when", "page", "position", "visible"]} dense>
153+ {data.feed.map((f, i) => (
154+ <tr key={i}>
155+ <td className="mono text-dim whitespace-nowrap">{fmtDate(f.ts)}</td>
156+ <td>
157+ <Tag color="#38e1ff">{f.page_type}</Tag>
158+ </td>
159+ <td className="mono">#{f.feed_position + 1}</td>
160+ <td className="mono">{f.visible ? "yes" : "below fold"}</td>
161+ </tr>
162+ ))}
163+ </Table>
164+ ) : (
165+ <Empty>not seen in a feed</Empty>
166+ )}
167+ </div>
168+ </Panel>
169+ </div>
170+ </div>
171+ </div>
172+ );
173+}
added apps/dashboard/src/app/entities/page.tsx +85 −0
@@ -0,0 +1,85 @@
1+"use client";
2+
3+import { useState } from "react";
4+import { Search } from "lucide-react";
5+import { useApi } from "@/lib/api";
6+import { fmtNum } from "@/lib/format";
7+import { Empty, Panel, Skeleton } from "@/components/ui";
8+import { EntityTable, type EntityRow } from "@/components/EntityTable";
9+
10+const TYPES = ["", "video", "channel", "profile", "post", "comment", "community", "hashtag", "page"];
11+const PLATFORMS = ["", "youtube", "reddit", "facebook", "instagram", "tiktok", "x"];
12+
13+export default function Entities() {
14+ const [q, setQ] = useState("");
15+ const [type, setType] = useState("");
16+ const [platform, setPlatform] = useState("");
17+ const [sort, setSort] = useState("last_seen");
18+ const [page, setPage] = useState(0);
19+ const limit = 50;
20+ const params = new URLSearchParams({ limit: String(limit), offset: String(page * limit), sort });
21+ if (q) params.set("q", q);
22+ if (type) params.set("type", type);
23+ if (platform) params.set("platform", platform);
24+ const { data, loading } = useApi<{ rows: EntityRow[]; total: number }>(`/entities?${params}`);
25+ const sel = "rounded-lg bg-bg-2 border border-line-2 px-2 py-1.5 text-sm text-fg outline-none focus:border-cyan/60";
26+ return (
27+ <div className="space-y-4">
28+ <div className="flex flex-wrap items-end gap-3">
29+ <div>
30+ <h1 className="text-2xl font-semibold tracking-tight">Entities</h1>
31+ <p className="text-sm text-fg-2">Canonical entities with per-field evidence. {data ? <span className="mono">{fmtNum(data.total)} match</span> : null}</p>
32+ </div>
33+ <div className="ml-auto flex flex-wrap gap-2 items-center">
34+ <label className="relative">
35+ <Search className="h-4 w-4 absolute left-2 top-2 text-dim" />
36+ <input
37+ value={q}
38+ onChange={(e) => {
39+ setQ(e.target.value);
40+ setPage(0);
41+ }}
42+ placeholder="search name, text, author, id…"
43+ className={`${sel} pl-8 w-72 mono`}
44+ />
45+ </label>
46+ <select value={type} onChange={(e) => (setType(e.target.value), setPage(0))} className={sel}>
47+ {TYPES.map((t) => (
48+ <option key={t} value={t}>
49+ {t || "all types"}
50+ </option>
51+ ))}
52+ </select>
53+ <select value={platform} onChange={(e) => (setPlatform(e.target.value), setPage(0))} className={sel}>
54+ {PLATFORMS.map((p) => (
55+ <option key={p} value={p}>
56+ {p || "all platforms"}
57+ </option>
58+ ))}
59+ </select>
60+ <select value={sort} onChange={(e) => setSort(e.target.value)} className={sel}>
61+ <option value="last_seen">recently seen</option>
62+ <option value="seen">most observed</option>
63+ <option value="views">most views</option>
64+ </select>
65+ </div>
66+ </div>
67+ <Panel pad={false}>
68+ <div className="p-4">{loading ? <Skeleton rows={10} /> : data?.rows.length ? <EntityTable rows={data.rows} showPlatform /> : <Empty>Nothing matches.</Empty>}</div>
69+ {data && data.total > limit && (
70+ <div className="flex items-center gap-3 px-4 py-3 border-t border-line text-xs">
71+ <button disabled={page === 0} onClick={() => setPage((p) => p - 1)} className="px-2 py-1 rounded border border-line disabled:opacity-40 hover:border-cyan/50">
72+ ← prev
73+ </button>
74+ <span className="mono text-dim">
75+ {page * limit + 1}–{Math.min(data.total, (page + 1) * limit)} of {data.total}
76+ </span>
77+ <button disabled={(page + 1) * limit >= data.total} onClick={() => setPage((p) => p + 1)} className="px-2 py-1 rounded border border-line disabled:opacity-40 hover:border-cyan/50">
78+ next →
79+ </button>
80+ </div>
81+ )}
82+ </Panel>
83+ </div>
84+ );
85+}
added apps/dashboard/src/app/globals.css +87 −0
@@ -0,0 +1,87 @@
1+@import "tailwindcss";
2+
3+@theme {
4+ --color-bg: #05070b;
5+ --color-bg-2: #0a0e15;
6+ --color-panel: #0d1219;
7+ --color-panel-2: #111823;
8+ --color-line: #1a2331;
9+ --color-line-2: #243042;
10+ --color-fg: #e6edf5;
11+ --color-fg-2: #a9b6c6;
12+ --color-dim: #6b7a8c;
13+ --color-cyan: #38e1ff;
14+ --color-amber: #ffb347;
15+ --color-green: #43e69a;
16+ --color-red: #ff5c7a;
17+ --color-violet: #9d7bff;
18+ --color-pink: #ff7ad9;
19+ --font-sans: var(--font-sans), ui-sans-serif, system-ui, sans-serif;
20+ --font-mono: var(--font-mono), ui-monospace, "SF Mono", Menlo, monospace;
21+}
22+
23+:root {
24+ color-scheme: dark;
25+}
26+
27+html,
28+body {
29+ background: var(--color-bg);
30+ color: var(--color-fg);
31+ font-family: var(--font-sans);
32+ -webkit-font-smoothing: antialiased;
33+}
34+
35+body {
36+ background-image:
37+ radial-gradient(1200px 600px at 10% -10%, rgba(56, 225, 255, 0.07), transparent 60%),
38+ radial-gradient(900px 500px at 100% 0%, rgba(157, 123, 255, 0.07), transparent 60%);
39+ background-attachment: fixed;
40+}
41+
42+.grid-bg {
43+ background-image:
44+ linear-gradient(rgba(255, 255, 255, 0.025) 1px, transparent 1px),
45+ linear-gradient(90deg, rgba(255, 255, 255, 0.025) 1px, transparent 1px);
46+ background-size: 28px 28px;
47+ mask-image: radial-gradient(ellipse at top, black 40%, transparent 85%);
48+}
49+
50+.panel {
51+ background: linear-gradient(180deg, rgba(255, 255, 255, 0.025), rgba(255, 255, 255, 0.005)), var(--color-panel);
52+ border: 1px solid var(--color-line);
53+ border-radius: 14px;
54+ box-shadow: 0 1px 0 rgba(255, 255, 255, 0.03) inset, 0 10px 30px -20px rgba(0, 0, 0, 0.8);
55+}
56+
57+.glow-cyan { box-shadow: 0 0 0 1px rgba(56, 225, 255, 0.25), 0 0 24px -6px rgba(56, 225, 255, 0.5); }
58+
59+.mono { font-family: var(--font-mono); font-feature-settings: "tnum" 1, "zero" 1; }
60+
61+.scrollbar-thin::-webkit-scrollbar { width: 8px; height: 8px; }
62+.scrollbar-thin::-webkit-scrollbar-thumb { background: var(--color-line-2); border-radius: 8px; }
63+.scrollbar-thin::-webkit-scrollbar-track { background: transparent; }
64+
65+@keyframes pulse-dot {
66+ 0%, 100% { transform: scale(1); opacity: 1; }
67+ 50% { transform: scale(1.6); opacity: 0.35; }
68+}
69+.live-dot::after {
70+ content: "";
71+ position: absolute;
72+ inset: 0;
73+ border-radius: 9999px;
74+ background: currentColor;
75+ animation: pulse-dot 1.6s ease-in-out infinite;
76+}
77+
78+@keyframes shimmer { from { background-position: -200% 0; } to { background-position: 200% 0; } }
79+.skeleton {
80+ background: linear-gradient(90deg, rgba(255,255,255,0.03) 25%, rgba(255,255,255,0.08) 50%, rgba(255,255,255,0.03) 75%);
81+ background-size: 200% 100%;
82+ animation: shimmer 1.4s linear infinite;
83+ border-radius: 6px;
84+}
85+
86+a { color: inherit; }
87+::selection { background: rgba(56, 225, 255, 0.3); }
added apps/dashboard/src/app/jobs/page.tsx +213 −0
@@ -0,0 +1,213 @@
1+"use client";
2+
3+import { useEffect, useState } from "react";
4+import Link from "next/link";
5+import { Play, Square } from "lucide-react";
6+import { api, useApi } from "@/lib/api";
7+import { ago, truncate } from "@/lib/format";
8+import { Empty, Panel, Skeleton, StatusPill, Table, Tag } from "@/components/ui";
9+
10+interface Job {
11+ job_id: string;
12+ session_id: string | null;
13+ platform: string;
14+ mode: string;
15+ goal: string;
16+ budget: { minutes: number; actions: number; media: number; query?: string | null; headless: boolean; account: string };
17+ status: string;
18+ created_at: string;
19+ finished_at: string | null;
20+ result: { steps?: number; entities?: number; videos?: number; ended_because?: string } | null;
21+ actions: number;
22+ entities: number;
23+}
24+
25+const PRESETS = [
26+ { label: "Topic · AI Québec on YouTube", platform: "youtube", mode: "topic", query: "intelligence artificielle Québec", goal: "Discover public Quebec creators and channels discussing artificial intelligence", minutes: 10, actions: 60, media: 2 },
27+ { label: "Learn · YouTube runtime", platform: "youtube", mode: "learn", query: "", goal: "", minutes: 15, actions: 80, media: 1 },
28+ { label: "Observe · YouTube home feed", platform: "youtube", mode: "observe", query: "", goal: "Observe the recommendation feed without navigating", minutes: 5, actions: 30, media: 0 },
29+ { label: "Research · Reddit r/Quebec AI", platform: "reddit", mode: "research", query: "intelligence artificielle Québec", goal: "Discover public Reddit posts and users discussing AI in Quebec", minutes: 10, actions: 60, media: 1 },
30+];
31+
32+export default function Missions() {
33+ const jobs = useApi<Job[]>("/jobs", { refreshMs: 5000 });
34+ const [form, setForm] = useState({ platform: "youtube", mode: "research", query: "intelligence artificielle Québec", goal: "Discover public Quebec creators discussing AI", minutes: 10, actions: 60, media: 1, headless: true, account: "research-01" });
35+ const [busy, setBusy] = useState(false);
36+ const [msg, setMsg] = useState<string | null>(null);
37+ const [logJob, setLogJob] = useState<string | null>(null);
38+ const input = "w-full rounded-lg bg-bg-2 border border-line-2 px-3 py-2 text-sm outline-none focus:border-cyan/60";
39+
40+ async function launch() {
41+ setBusy(true);
42+ setMsg(null);
43+ try {
44+ const r = await api<{ job_id: string }>("/jobs", { method: "POST", body: JSON.stringify(form) });
45+ setMsg(`mission ${r.job_id} launched`);
46+ setLogJob(r.job_id);
47+ jobs.refresh();
48+ } catch (e) {
49+ setMsg((e as Error).message);
50+ } finally {
51+ setBusy(false);
52+ }
53+ }
54+ async function stop(id: string) {
55+ await api(`/jobs/${id}/stop`, { method: "POST", body: "{}" });
56+ jobs.refresh();
57+ }
58+
59+ return (
60+ <div className="space-y-4">
61+ <div>
62+ <h1 className="text-2xl font-semibold tracking-tight">Missions</h1>
63+ <p className="text-sm text-fg-2 max-w-2xl">A mission opens one browser on the cluster node, gives the agent a goal and a budget, and lets it explore. Read-only: the crawler never likes, follows, comments or subscribes.</p>
64+ </div>
65+ <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">
66+ <Panel title="New mission" className="xl:col-span-1">
67+ <div className="space-y-3">
68+ <div className="flex flex-wrap gap-1">
69+ {PRESETS.map((p) => (
70+ <button key={p.label} onClick={() => setForm((f) => ({ ...f, ...p }))} className="text-[11px] rounded-md border border-line px-2 py-1 hover:border-cyan/50 text-fg-2 hover:text-fg">
71+ {p.label}
72+ </button>
73+ ))}
74+ </div>
75+ <div className="grid grid-cols-2 gap-2">
76+ <label className="text-xs text-dim">
77+ platform
78+ <select value={form.platform} onChange={(e) => setForm({ ...form, platform: e.target.value })} className={input}>
79+ <option value="youtube">youtube</option>
80+ <option value="reddit">reddit</option>
81+ </select>
82+ </label>
83+ <label className="text-xs text-dim">
84+ mode
85+ <select value={form.mode} onChange={(e) => setForm({ ...form, mode: e.target.value })} className={input}>
86+ {["research", "topic", "observe", "profile", "learn"].map((m) => (
87+ <option key={m}>{m}</option>
88+ ))}
89+ </select>
90+ </label>
91+ </div>
92+ <label className="text-xs text-dim block">
93+ topic / search query
94+ <input value={form.query} onChange={(e) => setForm({ ...form, query: e.target.value })} className={input} placeholder="e.g. intelligence artificielle Québec" />
95+ </label>
96+ <label className="text-xs text-dim block">
97+ goal (what the agent optimises for)
98+ <textarea value={form.goal} onChange={(e) => setForm({ ...form, goal: e.target.value })} rows={2} className={input} />
99+ </label>
100+ <div className="grid grid-cols-3 gap-2">
101+ <label className="text-xs text-dim">
102+ minutes
103+ <input type="number" min={1} max={120} value={form.minutes} onChange={(e) => setForm({ ...form, minutes: Number(e.target.value) })} className={`${input} mono`} />
104+ </label>
105+ <label className="text-xs text-dim">
106+ actions
107+ <input type="number" min={1} max={1000} value={form.actions} onChange={(e) => setForm({ ...form, actions: Number(e.target.value) })} className={`${input} mono`} />
108+ </label>
109+ <label className="text-xs text-dim">
110+ media level
111+ <select value={form.media} onChange={(e) => setForm({ ...form, media: Number(e.target.value) })} className={input}>
112+ <option value={0}>0 · metadata</option>
113+ <option value={1}>1 · + thumbnail</option>
114+ <option value={2}>2 · + frames</option>
115+ </select>
116+ </label>
117+ </div>
118+ <div className="grid grid-cols-2 gap-2">
119+ <label className="text-xs text-dim">
120+ account profile
121+ <input value={form.account} onChange={(e) => setForm({ ...form, account: e.target.value })} className={`${input} mono`} />
122+ </label>
123+ <label className="text-xs text-dim flex items-end gap-2 pb-2">
124+ <input type="checkbox" checked={form.headless} onChange={(e) => setForm({ ...form, headless: e.target.checked })} /> headless browser
125+ </label>
126+ </div>
127+ <button disabled={busy} onClick={launch} className="w-full rounded-lg bg-cyan/90 hover:bg-cyan disabled:opacity-50 text-bg font-semibold py-2 flex items-center justify-center gap-2">
128+ <Play className="h-4 w-4" /> Launch mission
129+ </button>
130+ {msg && <div className="text-xs text-fg-2 mono">{msg}</div>}
131+ </div>
132+ </Panel>
133+ <Panel title="Missions" className="xl:col-span-2" pad={false} right={jobs.data ? `${jobs.data.filter((j) => j.status === "running").length} running` : ""}>
134+ <div className="p-4">
135+ {jobs.loading ? (
136+ <Skeleton rows={6} />
137+ ) : jobs.data?.length ? (
138+ <Table head={["status", "platform", "goal", "mode", "budget", "steps", "entities", "created", ""]} dense>
139+ {jobs.data.map((j) => (
140+ <tr key={j.job_id}>
141+ <td>
142+ <StatusPill status={j.status} />
143+ </td>
144+ <td>
145+ <Tag>{j.platform}</Tag>
146+ </td>
147+ <td className="max-w-[360px]">
148+ {j.session_id ? (
149+ <Link href={`/sessions/${j.session_id}`} className="hover:text-cyan">
150+ {truncate(j.goal, 80)}
151+ </Link>
152+ ) : (
153+ truncate(j.goal, 80)
154+ )}
155+ {j.budget?.query && <div className="text-[10.5px] text-dim">“{j.budget.query}”</div>}
156+ </td>
157+ <td className="text-fg-2">{j.mode}</td>
158+ <td className="mono text-[11px] text-dim">
159+ {j.budget?.minutes}m · {j.budget?.actions}a · L{j.budget?.media}
160+ </td>
161+ <td className="mono">{j.actions || j.result?.steps || 0}</td>
162+ <td className="mono">{j.entities || j.result?.entities || 0}</td>
163+ <td className="text-dim whitespace-nowrap">{ago(j.created_at)}</td>
164+ <td className="whitespace-nowrap">
165+ <button onClick={() => setLogJob(j.job_id)} className="text-[11px] text-dim hover:text-cyan mr-2">
166+ log
167+ </button>
168+ {j.status === "running" && (
169+ <button onClick={() => stop(j.job_id)} className="text-[11px] text-red hover:text-red/80 inline-flex items-center gap-1">
170+ <Square className="h-3 w-3" /> stop
171+ </button>
172+ )}
173+ </td>
174+ </tr>
175+ ))}
176+ </Table>
177+ ) : (
178+ <Empty>No missions yet — launch one on the left.</Empty>
179+ )}
180+ </div>
181+ </Panel>
182+ </div>
183+ {logJob && <JobLog id={logJob} onClose={() => setLogJob(null)} />}
184+ </div>
185+ );
186+}
187+
188+function JobLog({ id, onClose }: { id: string; onClose: () => void }) {
189+ const [text, setText] = useState("");
190+ useEffect(() => {
191+ let alive = true;
192+ let timer: ReturnType<typeof setTimeout> | undefined;
193+ const tick = async () => {
194+ try {
195+ const r = await fetch(`/api/jobs/${id}/log?lines=300`, { cache: "no-store" });
196+ if (alive) setText(await r.text());
197+ } catch {
198+ /* ignore */
199+ }
200+ if (alive) timer = setTimeout(tick, 3000);
201+ };
202+ tick();
203+ return () => {
204+ alive = false;
205+ if (timer) clearTimeout(timer);
206+ };
207+ }, [id]);
208+ return (
209+ <Panel title={`Worker log · ${id}`} right={<button onClick={onClose} className="hover:text-cyan">close</button>}>
210+ <pre className="mono text-[11px] leading-5 text-fg-2 whitespace-pre-wrap max-h-[420px] overflow-auto scrollbar-thin">{text || "…"}</pre>
211+ </Panel>
212+ );
213+}
added apps/dashboard/src/app/layout.tsx +24 −0
@@ -0,0 +1,24 @@
1+import type { Metadata } from "next";
2+import { Inter, JetBrains_Mono } from "next/font/google";
3+import "./globals.css";
4+import { Shell } from "@/components/Shell";
5+
6+const sans = Inter({ subsets: ["latin"], variable: "--font-sans", display: "swap" });
7+const mono = JetBrains_Mono({ subsets: ["latin"], variable: "--font-mono", display: "swap" });
8+
9+export const metadata: Metadata = {
10+ title: { default: "SocialCrawl — Social Runtime Mining console", template: "%s · SocialCrawl" },
11+ description: "Research console of the Social Runtime Crawler: live observation of social-media runtimes, information-gain navigation, learned connectors.",
12+ robots: { index: false, follow: false },
13+ icons: { icon: "/brand/icon.svg" },
14+};
15+
16+export default function RootLayout({ children }: { children: React.ReactNode }) {
17+ return (
18+ <html lang="en" className={`${sans.variable} ${mono.variable}`}>
19+ <body className="min-h-screen">
20+ <Shell>{children}</Shell>
21+ </body>
22+ </html>
23+ );
24+}
added apps/dashboard/src/app/media/[...path]/route.ts +15 −0
@@ -0,0 +1,15 @@
1+import { cookies } from "next/headers";
2+import { NextResponse, type NextRequest } from "next/server";
3+import { COOKIE, verifyToken } from "@/lib/auth";
4+
5+export const dynamic = "force-dynamic";
6+const API_URL = process.env.API_URL ?? "http://127.0.0.1:8350";
7+
8+/** Sampled video frames, served through the gate. */
9+export async function GET(req: NextRequest, ctx: { params: Promise<{ path: string[] }> }) {
10+ const jar = await cookies();
11+ if (!verifyToken(jar.get(COOKIE)?.value)) return NextResponse.json({ error: "unauthorized" }, { status: 401 });
12+ const { path } = await ctx.params;
13+ const upstream = await fetch(new URL(`/media/${path.map(encodeURIComponent).join("/")}`, API_URL), { cache: "no-store" });
14+ return new Response(upstream.body, { status: upstream.status, headers: { "content-type": upstream.headers.get("content-type") ?? "image/jpeg", "cache-control": "private, max-age=3600" } });
15+}
added apps/dashboard/src/app/page.tsx +126 −0
@@ -0,0 +1,126 @@
1+"use client";
2+
3+import Link from "next/link";
4+import { ArrowRight } from "lucide-react";
5+import { useApi } from "@/lib/api";
6+import { ago, fmtNum, truncate } from "@/lib/format";
7+import { Bar, Empty, Kpi, Panel, Skeleton, StatusPill, Tag } from "@/components/ui";
8+import { ActivityChart, SurfaceBars, TypeDonut, YieldBars } from "@/components/charts";
9+import { LiveFeed } from "@/components/LiveFeed";
10+
11+interface Overview {
12+ totals: { sessions: number; entities: number; media: number; schemas: number; actions: number; observations: number; relationships: number; patterns_learned: number; feed_items: number };
13+ by_type: { platform: string; entity_type: string; n: number }[];
14+ running: { session_id: string; platform: string; started_at: string }[];
15+ recent_sessions: { session_id: string; platform: string; goal: string | null; mode: string | null; started_at: string; ended_at: string | null; health: string; actions: number; entities: number }[];
16+ series: { bucket: string; entities: number; responses: number; actions: number; schemas: number }[];
17+ surfaces: { dom: number; network: number; both: number; agreements: number; conflicts: number };
18+ actions: { action_type: string; n: number; avg_gain: number | null; success_rate: number | null; avg_new_entities: number | null }[];
19+ platforms: { platform: string; learned: boolean; adapter: boolean; confidence: number; page_types: number; entity_types: number; network_schemas: number; sessions: number; has_manifest: boolean }[];
20+}
21+
22+export default function Observatory() {
23+ const { data, loading } = useApi<Overview>("/overview", { refreshMs: 8000 });
24+ const t = data?.totals;
25+ const s = data?.surfaces;
26+ const agreement = s && s.agreements + s.conflicts > 0 ? s.agreements / (s.agreements + s.conflicts) : null;
27+ const spark = (k: keyof Overview["series"][number]) => (data?.series ?? []).slice(-30).map((r) => Number(r[k]));
28+ return (
29+ <div className="space-y-6">
30+ <div className="flex flex-wrap items-end gap-4">
31+ <div>
32+ <h1 className="text-2xl font-semibold tracking-tight">Observatory</h1>
33+ <p className="text-sm text-fg-2 mt-1 max-w-2xl">The crawler does not download pages. It watches social applications run inside an authenticated browser, fuses what the network and the DOM reveal, and learns the platform as it goes.</p>
34+ </div>
35+ <div className="ml-auto flex gap-2">
36+ {data?.running.map((r) => (
37+ <Link key={r.session_id} href={`/sessions/${r.session_id}`} className="panel px-3 py-1.5 text-xs flex items-center gap-2 hover:border-amber/50">
38+ <StatusPill status="running" /> {r.platform} <span className="text-dim">{ago(r.started_at)}</span>
39+ </Link>
40+ ))}
41+ <Link href="/jobs" className="rounded-lg bg-cyan/90 hover:bg-cyan text-bg text-sm font-semibold px-3 py-1.5 flex items-center gap-1">
42+ New mission <ArrowRight className="h-4 w-4" />
43+ </Link>
44+ </div>
45+ </div>
46+
47+ <div className="grid grid-cols-2 md:grid-cols-3 xl:grid-cols-6 gap-3">
48+ <Kpi label="entities" value={fmtNum(t?.entities)} hint={`${fmtNum(t?.relationships)} relationships`} spark={spark("entities")} />
49+ <Kpi label="media" value={fmtNum(t?.media)} hint="videos & images with metadata" accent="pink" />
50+ <Kpi label="runtime schemas" value={fmtNum(t?.schemas)} hint={`${fmtNum(t?.patterns_learned)} patterns learned`} accent="violet" spark={spark("schemas")} />
51+ <Kpi label="actions" value={fmtNum(t?.actions)} hint={`${fmtNum(t?.sessions)} sessions`} accent="amber" spark={spark("actions")} />
52+ <Kpi label="observations" value={fmtNum(t?.observations)} hint="raw, append-only" accent="fg" spark={spark("responses")} />
53+ <Kpi label="surface agreement" value={agreement === null ? "—" : `${Math.round(agreement * 100)}%`} hint={s ? `${fmtNum(s.both)} entities seen on both surfaces` : ""} accent="green" />
54+ </div>
55+
56+ <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">
57+ <Panel title="Activity" sub="10-minute buckets, last 48 h" className="xl:col-span-2">
58+ {loading ? <Skeleton rows={6} /> : data?.series.length ? <ActivityChart data={data.series} /> : <Empty>No activity yet — launch a mission.</Empty>}
59+ </Panel>
60+ <Panel title="Entity mix" sub="canonical entities by type">
61+ {data?.by_type.length ? <TypeDonut data={data.by_type} /> : <Empty>—</Empty>}
62+ </Panel>
63+ </div>
64+
65+ <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">
66+ <Panel title="Live observation stream" sub="every event, as the browser sees it" className="xl:col-span-2" pad={false}>
67+ <div className="p-2">
68+ <LiveFeed max={80} height={380} />
69+ </div>
70+ </Panel>
71+ <div className="space-y-4">
72+ <Panel title="Network vs DOM" sub="where entities were confirmed">
73+ {s ? <SurfaceBars dom={s.dom} network={s.network} both={s.both} /> : <Skeleton />}
74+ <div className="text-xs text-dim mt-2">
75+ field agreements <span className="mono text-fg">{fmtNum(s?.agreements)}</span> · conflicts <span className="mono text-fg">{fmtNum(s?.conflicts)}</span>
76+ </div>
77+ </Panel>
78+ <Panel title="Action yield" sub="average new entities per action type">
79+ {data?.actions.length ? <YieldBars data={data.actions} height={190} /> : <Empty>—</Empty>}
80+ </Panel>
81+ </div>
82+ </div>
83+
84+ <div className="grid grid-cols-1 xl:grid-cols-2 gap-4">
85+ <Panel title="Recent sessions" right={<Link href="/sessions" className="hover:text-cyan">all →</Link>} pad={false}>
86+ {data?.recent_sessions.length ? (
87+ <ul className="divide-y divide-line/60">
88+ {data.recent_sessions.map((r) => (
89+ <li key={r.session_id}>
90+ <Link href={`/sessions/${r.session_id}`} className="flex items-center gap-3 px-4 py-2.5 hover:bg-white/[0.03]">
91+ <StatusPill status={r.ended_at ? "done" : r.health} />
92+ <Tag>{r.platform}</Tag>
93+ <span className="text-sm truncate">{truncate(r.goal ?? "(no goal)", 70)}</span>
94+ <span className="ml-auto mono text-xs text-dim whitespace-nowrap">
95+ {r.actions} steps · {r.entities} ent · {ago(r.started_at)}
96+ </span>
97+ </Link>
98+ </li>
99+ ))}
100+ </ul>
101+ ) : (
102+ <Empty>No sessions yet.</Empty>
103+ )}
104+ </Panel>
105+ <Panel title="Connectors" sub="learned platform knowledge" right={<Link href="/platforms" className="hover:text-cyan">details →</Link>}>
106+ <div className="space-y-3">
107+ {data?.platforms
108+ .filter((p) => p.adapter || p.learned)
109+ .map((p) => (
110+ <Link key={p.platform} href={`/platforms/${p.platform}`} className="block group">
111+ <div className="flex items-center gap-3 text-sm">
112+ <span className="w-20 capitalize group-hover:text-cyan">{p.platform}</span>
113+ <Bar value={p.confidence / 100} color={p.confidence >= 70 ? "#43e69a" : p.confidence >= 40 ? "#38e1ff" : "#ffb347"} className="flex-1" />
114+ <span className="mono text-xs w-10 text-right">{p.confidence}%</span>
115+ </div>
116+ <div className="text-[10.5px] text-dim mono ml-[92px]">
117+ {p.page_types} page types · {p.entity_types} entity types · {p.network_schemas} schemas · {p.sessions} sessions{p.has_manifest ? " · compiled" : ""}
118+ </div>
119+ </Link>
120+ ))}
121+ </div>
122+ </Panel>
123+ </div>
124+ </div>
125+ );
126+}
added apps/dashboard/src/app/platforms/[platform]/page.tsx +198 −0
@@ -0,0 +1,198 @@
1+"use client";
2+
3+import { use } from "react";
4+import Link from "next/link";
5+import { useApi } from "@/lib/api";
6+import { fmtDate } from "@/lib/format";
7+import { Bar, Empty, Metric, Panel, Skeleton, Table, Tag } from "@/components/ui";
8+
9+interface Detail {
10+ platform: string;
11+ summary: { confidence: number; page_types: number; entity_types: number; navigation_actions: number; network_schemas: number; media_patterns: number; sessions: number } | null;
12+ model: {
13+ updated_at: string;
14+ response_patterns: Record<string, { shape_hash: string; hostname: string; path_pattern: string; method: string; graphql_operation?: string; likely_entity_types: Record<string, number>; observed_count: number; entity_yield_total: number; confidence: number; triggered_by: Record<string, number>; first_seen: string; last_seen: string }>;
15+ action_patterns: Record<string, { action_type: string; from_page_type: string; observed_count: number; avg_new_entities: number; avg_network_responses: number; to_page_types: Record<string, number>; failures: number }>;
16+ page_types: Record<string, { page_type: string; observed_count: number; url_patterns: Record<string, number>; avg_entities: number; entity_types: Record<string, number>; avg_confidence: number }>;
17+ media_patterns: Record<string, { hostname: string; kind: string; observed_count: number }>;
18+ sessions_learned: string[];
19+ } | null;
20+ manifest: { compiled_at: string; confidence: number; network_patterns: unknown[] } | null;
21+ profiles: { alias: string; has_state: boolean }[];
22+}
23+
24+export default function PlatformPage({ params }: { params: Promise<{ platform: string }> }) {
25+ const { platform } = use(params);
26+ const { data, loading } = useApi<Detail>(`/platforms/${platform}`, { refreshMs: 15_000 });
27+ if (loading) return <Skeleton rows={10} />;
28+ if (!data) return <Empty>unknown platform</Empty>;
29+ const m = data.model;
30+ const rps = m ? Object.values(m.response_patterns).filter((r) => Object.keys(r.likely_entity_types).length).sort((a, b) => b.confidence - a.confidence) : [];
31+ const aps = m ? Object.values(m.action_patterns).filter((a) => a.from_page_type !== "ANY").sort((a, b) => b.observed_count - a.observed_count) : [];
32+ const pts = m ? Object.values(m.page_types).sort((a, b) => b.observed_count - a.observed_count) : [];
33+ return (
34+ <div className="space-y-4">
35+ <div className="text-xs text-dim mono">
36+ <Link href="/platforms" className="hover:text-cyan">connectors</Link> / {platform}
37+ </div>
38+ <div className="flex flex-wrap items-end gap-4">
39+ <h1 className="text-2xl font-semibold tracking-tight capitalize">{platform}</h1>
40+ {data.summary && (
41+ <div className="ml-auto flex items-center gap-3 w-72">
42+ <span className="text-xs text-dim">confidence</span>
43+ <Bar value={data.summary.confidence / 100} className="flex-1" color={data.summary.confidence >= 70 ? "#43e69a" : "#38e1ff"} />
44+ <span className="mono">{data.summary.confidence}%</span>
45+ </div>
46+ )}
47+ </div>
48+ {!m ? (
49+ <Panel>
50+ <Empty>
51+ Nothing learned yet on {platform}. {["youtube", "reddit"].includes(platform) ? "Launch a learning mission from the Missions page." : "No adapter yet — this platform is scheduled for a later phase."}
52+ </Empty>
53+ </Panel>
54+ ) : (
55+ <>
56+ <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">
57+ <Panel title="Learned model">
58+ <Metric label="page types" value={data.summary?.page_types} />
59+ <Metric label="entity types" value={data.summary?.entity_types} />
60+ <Metric label="navigation actions" value={data.summary?.navigation_actions} />
61+ <Metric label="network schemas" value={data.summary?.network_schemas} />
62+ <Metric label="media patterns" value={data.summary?.media_patterns} />
63+ <Metric label="sessions learned from" value={m.sessions_learned.length} />
64+ <Metric label="updated" value={fmtDate(m.updated_at)} />
65+ <Metric label="compiled manifest" value={data.manifest ? `${fmtDate(data.manifest.compiled_at)} · ${data.manifest.network_patterns.length} patterns` : "not yet (needs ≥ 40 %)"} />
66+ <Metric label="browser profiles" value={data.profiles.length ? data.profiles.map((p) => `${p.alias}${p.has_state ? " ✓" : ""}`).join(", ") : "none — run pnpm login"} />
67+ </Panel>
68+ <Panel title="Page types" sub="what the crawler recognises, and what each page usually yields" className="xl:col-span-2" pad={false}>
69+ <div className="p-4">
70+ <Table head={["page type", "seen", "avg entities", "avg conf", "url patterns", "entity mix"]} dense>
71+ {pts.map((p) => (
72+ <tr key={p.page_type}>
73+ <td>
74+ <Tag color="#38e1ff">{p.page_type}</Tag>
75+ </td>
76+ <td className="mono">{p.observed_count}</td>
77+ <td className="mono">{p.avg_entities.toFixed(1)}</td>
78+ <td className="mono">{p.avg_confidence.toFixed(2)}</td>
79+ <td className="mono text-[11px] text-fg-2">
80+ {Object.entries(p.url_patterns)
81+ .sort((a, b) => b[1] - a[1])
82+ .slice(0, 3)
83+ .map(([k, v]) => `${k} (${v})`)
84+ .join(" ")}
85+ </td>
86+ <td className="text-[11px]">
87+ {Object.entries(p.entity_types)
88+ .sort((a, b) => b[1] - a[1])
89+ .slice(0, 4)
90+ .map(([k, v]) => (
91+ <Tag key={k} className="mr-1">
92+ {k} {v}
93+ </Tag>
94+ ))}
95+ </td>
96+ </tr>
97+ ))}
98+ </Table>
99+ </div>
100+ </Panel>
101+ </div>
102+ <Panel title="Runtime API map" sub="action → response shape → entity types (§71) — the learned frontend API of the platform" pad={false}>
103+ <div className="p-4">
104+ {rps.length ? (
105+ <Table head={["endpoint", "entity types", "triggered by", "seen", "yield", "confidence"]} dense>
106+ {rps.map((r) => (
107+ <tr key={r.shape_hash}>
108+ <td className="mono text-[11.5px]">
109+ <Tag color="#9d7bff" className="mr-1">
110+ {r.method}
111+ </Tag>
112+ {r.hostname}
113+ <span className="text-fg-2">{r.path_pattern}</span>
114+ {r.graphql_operation && <Tag color="#ff7ad9" className="ml-1">{r.graphql_operation}</Tag>}
115+ <div className="text-[10px] text-dim">#{r.shape_hash.slice(0, 10)}</div>
116+ </td>
117+ <td>
118+ {Object.entries(r.likely_entity_types).map(([t, n]) => (
119+ <Tag key={t} color="#38e1ff" className="mr-1">
120+ {t} {n}
121+ </Tag>
122+ ))}
123+ </td>
124+ <td>
125+ {Object.entries(r.triggered_by).map(([a, n]) => (
126+ <Tag key={a} color="#ffb347" className="mr-1">
127+ {a.toLowerCase()} {n}
128+ </Tag>
129+ ))}
130+ </td>
131+ <td className="mono">{r.observed_count}</td>
132+ <td className="mono">{(r.entity_yield_total / Math.max(1, r.observed_count)).toFixed(1)} ent/resp</td>
133+ <td className="w-32">
134+ <Bar value={r.confidence} color={r.confidence >= 0.5 ? "#43e69a" : "#ffb347"} />
135+ <div className="mono text-[10px] text-dim">{r.confidence.toFixed(2)}</div>
136+ </td>
137+ </tr>
138+ ))}
139+ </Table>
140+ ) : (
141+ <Empty>no entity-bearing schema learned yet</Empty>
142+ )}
143+ </div>
144+ </Panel>
145+ <div className="grid grid-cols-1 xl:grid-cols-2 gap-4">
146+ <Panel title="Navigation patterns" sub="action from a page type → where it leads and what it yields (§30)" pad={false}>
147+ <div className="p-4">
148+ <Table head={["action", "from", "→ to", "seen", "avg new ent", "avg resp", "fail"]} dense>
149+ {aps.map((a) => (
150+ <tr key={a.action_type + a.from_page_type}>
151+ <td>
152+ <Tag color="#ffb347">{a.action_type.toLowerCase()}</Tag>
153+ </td>
154+ <td>
155+ <Tag color="#38e1ff">{a.from_page_type}</Tag>
156+ </td>
157+ <td className="text-[11px]">
158+ {Object.entries(a.to_page_types)
159+ .sort((x, y) => y[1] - x[1])
160+ .map(([k, v]) => `${k} ${v}`)
161+ .join(", ")}
162+ </td>
163+ <td className="mono">{a.observed_count}</td>
164+ <td className="mono">{a.avg_new_entities.toFixed(1)}</td>
165+ <td className="mono">{a.avg_network_responses.toFixed(1)}</td>
166+ <td className="mono">{a.failures}</td>
167+ </tr>
168+ ))}
169+ </Table>
170+ </div>
171+ </Panel>
172+ <Panel title="Media delivery patterns" pad={false}>
173+ <div className="p-4">
174+ {Object.values(m.media_patterns).length ? (
175+ <Table head={["host", "kind", "seen"]} dense>
176+ {Object.values(m.media_patterns)
177+ .sort((a, b) => b.observed_count - a.observed_count)
178+ .map((mp) => (
179+ <tr key={mp.hostname + mp.kind}>
180+ <td className="mono text-[11.5px]">{mp.hostname}</td>
181+ <td>
182+ <Tag color="#ff7ad9">{mp.kind}</Tag>
183+ </td>
184+ <td className="mono">{mp.observed_count}</td>
185+ </tr>
186+ ))}
187+ </Table>
188+ ) : (
189+ <Empty>none yet</Empty>
190+ )}
191+ </div>
192+ </Panel>
193+ </div>
194+ </>
195+ )}
196+ </div>
197+ );
198+}
added apps/dashboard/src/app/platforms/page.tsx +82 −0
Large diff (82 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/app/schemas/page.tsx +97 −0
Large diff (97 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/app/sessions/[id]/page.tsx +405 −0
Large diff (405 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/app/sessions/page.tsx +69 −0
Large diff (69 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/Brand.tsx +33 −0
Large diff (33 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/EntityTable.tsx +79 −0
Large diff (79 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/LiveFeed.tsx +78 −0
Large diff (78 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/Shell.tsx +78 −0
Large diff (78 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/WorldGraph.tsx +237 −0
Large diff (237 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/charts.tsx +110 −0
Large diff (110 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/ui.tsx +135 −0
Large diff (135 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/lib/api.ts +94 −0
Large diff (94 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/lib/auth.ts +36 −0
Large diff (36 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/lib/format.ts +84 −0
Large diff (84 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/proxy.ts +24 −0
Large diff (24 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/tsconfig.json +41 −0
Large diff (41 lines) is not loaded by default to keep the page fast.Load diff
added apps/worker/qa/screenshots.mjs +12 −0
Large diff (12 lines) is not loaded by default to keep the page fast.Load diff
modified apps/worker/src/cli.ts +1 −1
Large diff (8 lines) is not loaded by default to keep the page fast.Load diff
modified apps/worker/src/engine.ts +1 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
modified apps/worker/src/executor.ts +0 −0

Binary file not shown.

modified connectors/youtube/manifest.json +2993 −14
Large diff (3,065 lines) is not loaded by default to keep the page fast.Load diff
added deploy/social-runtime-crawler.mld.json +102 −0
Large diff (102 lines) is not loaded by default to keep the page fast.Load diff
modified package.json +5 −2
Large diff (16 lines) is not loaded by default to keep the page fast.Load diff
modified packages/media/src/index.ts +2 −0
Large diff (9 lines) is not loaded by default to keep the page fast.Load diff
modified packages/storage/src/index.ts +0 −0

Binary file not shown.

modified pnpm-lock.yaml +1358 −12
Large diff (1,692 lines) is not loaded by default to keep the page fast.Load diff
modified tsconfig.json +1 −1
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere