SPB Git
spboucher.ai

spb/social-runtime-crawler

Public

Research console (Next 16) + control-plane API + mld deployment for www.socialcrawl.co

- apps/dashboard: SocialCrawl console — observatory, sessions console (decisions, page state,
  entities, runtime APIs, media, world-model graph, live events), entity evidence pages,
  runtime API explorer, connectors, missions launcher; passcode gate + authenticated API proxy.
- apps/api: node:http control plane (overview, sessions, entities search, schemas, platforms,
  SSE stream, job runner spawning worker processes), token-protected mutations.
- worker: bounded action execution (60 s), PLAY_VIDEO no longer awaits play(); --job linking.
- storage: jsonb sanitiser, linkJob; util: unicode thousands separators, stopwords.
- deploy/social-runtime-crawler.mld.json: PM2 api 8350 + web 8351, MacLustr Tunnel route.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 12, 2026) parent 40fba6b Browse files

46 changed files +7,607 −105

46 changed files
Load all 46 files Unified Split
modified .gitignore +5 −0
@@ -14,3 +14,8 @@ data/platform_model/*/
14 14
15 15 # Compiled (learned) connectors are versioned intentionally; regression fixtures may be large.
16 16 connectors/*/regression_tests/large/
17 +
18 +# Next.js dashboard
19 +apps/dashboard/.next/
20 +apps/dashboard/next-env.d.ts
21 +*.tsbuildinfo
modified apps/api/package.json +5 −1
@@ -5,6 +5,10 @@
5 5 "type": "module",
6 6 "dependencies": {
7 7 "@src/shared": "workspace:*",
8 − "@src/storage": "workspace:*"
8 + "@src/storage": "workspace:*",
9 + "@src/platform-model": "workspace:*"
10 + },
11 + "scripts": {
12 + "start": "tsx src/server.ts"
9 13 }
10 14 }
added apps/api/src/jobs.ts +102 −0
@@ -0,0 +1,102 @@
1 +import { spawn, type ChildProcess } from "node:child_process";
2 +import fs from "node:fs";
3 +import path from "node:path";
4 +import { fileURLToPath } from "node:url";
5 +import { createLogger, isPlatform, newId, type AgentMode, type AppConfig } from "@src/shared";
6 +import type { PostgresStore } from "@src/storage";
7 +
8 +const log = createLogger("jobs");
9 +const here = path.dirname(fileURLToPath(import.meta.url));
10 +const repoRoot = path.resolve(here, "..", "..", "..");
11 +
12 +export interface JobRequest {
13 + platform: string;
14 + goal?: string;
15 + query?: string;
16 + mode?: AgentMode;
17 + seed?: string;
18 + minutes?: number;
19 + actions?: number;
20 + media?: number;
21 + headless?: boolean;
22 + account?: string;
23 +}
24 +
25 +/**
26 + * Job runner (§63 control plane, single-node version): spawns the worker CLI as a child process,
27 + * streams its log to data/jobs/<id>.log and tracks status in crawl_jobs. One browser per job.
28 + */
29 +export class JobRunner {
30 + private running = new Map<string, ChildProcess>();
31 + readonly logsDir: string;
32 + constructor(private readonly store: PostgresStore, private readonly cfg: AppConfig, private readonly maxConcurrent = 2) {
33 + this.logsDir = path.join(cfg.dataDir, "jobs");
34 + fs.mkdirSync(this.logsDir, { recursive: true });
35 + }
36 +
37 + get activeCount() {
38 + return this.running.size;
39 + }
40 +
41 + async start(req: JobRequest): Promise<{ job_id: string }> {
42 + if (!isPlatform(req.platform)) throw new Error("unknown platform");
43 + if (!["youtube", "reddit"].includes(req.platform)) throw new Error(`no adapter for ${req.platform} yet (Phase 1 = youtube, reddit)`);
44 + if (this.running.size >= this.maxConcurrent) throw new Error(`max ${this.maxConcurrent} concurrent jobs`);
45 + const job_id = newId("job");
46 + const mode = req.mode ?? "research";
47 + const minutes = clamp(req.minutes ?? 10, 1, 120);
48 + const actions = clamp(req.actions ?? 60, 1, 1000);
49 + const media = clamp(req.media ?? 1, 0, 4);
50 + const goal = req.goal?.trim() || (req.query ? `Discover public content about “${req.query}”` : `Observe the ${req.platform} feed`);
51 + const args = ["node_modules/tsx/dist/cli.mjs", "apps/worker/src/cli.ts", mode === "learn" ? "learn" : "crawl", req.platform, "--job", job_id, "--goal", goal, "--mode", mode, "--minutes", String(minutes), "--actions", String(actions), "--media", String(media), "--account", req.account?.trim() || "research-01"];
52 + if (req.query) args.push("--query", req.query);
53 + if (req.seed) args.push("--seed", req.seed);
54 + if (req.headless !== false) args.push("--headless");
55 +
56 + await this.store.pool.query(`insert into crawl_jobs(job_id, platform, mode, goal, budget, status) values ($1,$2,$3,$4,$5,'queued')`, [job_id, req.platform, mode, goal, JSON.stringify({ minutes, actions, media, query: req.query ?? null, seed: req.seed ?? null, headless: req.headless !== false, account: req.account ?? "research-01" })]);
57 +
58 + const logFile = path.join(this.logsDir, `${job_id}.log`);
59 + const out = fs.openSync(logFile, "a");
60 + const child = spawn(process.execPath, args, { cwd: repoRoot, env: { ...process.env, SRC_LOG_PRETTY: "1", SRC_LOG_LEVEL: "info" }, stdio: ["ignore", out, out] });
61 + this.running.set(job_id, child);
62 + await this.store.pool.query(`update crawl_jobs set status='running' where job_id=$1`, [job_id]);
63 + log.info("job started", { job_id, platform: req.platform, mode, pid: child.pid });
64 + child.on("exit", async (code, signal) => {
65 + this.running.delete(job_id);
66 + fs.closeSync(out);
67 + const status = signal ? "stopped" : code === 0 ? "done" : "failed";
68 + let result: unknown = null;
69 + try {
70 + const sid = (await this.store.pool.query(`select session_id from crawl_jobs where job_id=$1`, [job_id])).rows[0]?.session_id as string | undefined;
71 + if (sid) {
72 + const f = path.join(this.cfg.sessionsDir, sid, "summary.json");
73 + if (fs.existsSync(f)) result = JSON.parse(fs.readFileSync(f, "utf8"));
74 + }
75 + } catch {
76 + /* ignore */
77 + }
78 + await this.store.pool.query(`update crawl_jobs set status=$2, finished_at=now(), result=$3 where job_id=$1`, [job_id, status, result ? JSON.stringify(result) : null]).catch(() => {});
79 + log.info("job finished", { job_id, status, code, signal });
80 + });
81 + return { job_id };
82 + }
83 +
84 + stop(job_id: string): boolean {
85 + const child = this.running.get(job_id);
86 + if (!child) return false;
87 + child.kill("SIGTERM");
88 + setTimeout(() => child.kill("SIGKILL"), 15_000).unref();
89 + return true;
90 + }
91 +
92 + logTail(job_id: string, lines = 200): string {
93 + const f = path.join(this.logsDir, `${job_id}.log`);
94 + if (!fs.existsSync(f)) return "";
95 + const all = fs.readFileSync(f, "utf8").split("\n");
96 + return all.slice(-lines).join("\n");
97 + }
98 +}
99 +
100 +function clamp(n: number, lo: number, hi: number) {
101 + return Math.max(lo, Math.min(hi, Number.isFinite(n) ? n : lo));
102 +}
added apps/api/src/queries.ts +188 −0
@@ -0,0 +1,188 @@
1 +import fs from "node:fs";
2 +import path from "node:path";
3 +import type { PostgresStore } from "@src/storage";
4 +import { PLATFORMS, type AppConfig, type Platform } from "@src/shared";
5 +import { PlatformModel } from "@src/platform-model";
6 +
7 +/** Read-side queries for the research console. All read PostgreSQL; files (models, manifests, session dirs) complete them. */
8 +export class Queries {
9 + constructor(private readonly store: PostgresStore, private readonly cfg: AppConfig) {}
10 + private get q() {
11 + return this.store.pool;
12 + }
13 +
14 + async overview() {
15 + const [totals, byType, running, recentSessions, series, surfaces, topActions, patterns] = await Promise.all([
16 + this.q.query(`select
17 + (select count(*) from sessions)::int as sessions,
18 + (select count(*) from entities)::int as entities,
19 + (select count(*) from media)::int as media,
20 + (select count(*) from schema_patterns)::int as schemas,
21 + (select count(*) from actions)::int as actions,
22 + (select count(*) from observations)::int as observations,
23 + (select count(*) from relationships)::int as relationships,
24 + (select count(*) from observations where event_type='CONNECTOR_PATTERN_LEARNED')::int as patterns_learned,
25 + (select count(*) from feed_items)::int as feed_items`),
26 + this.q.query(`select platform, entity_type, count(*)::int as n from entities group by 1,2 order by 3 desc`),
27 + this.q.query(`select session_id, platform, account_alias, started_at, health from sessions where ended_at is null and started_at > now() - interval '6 hours' order by started_at desc`),
28 + this.q.query(`select s.session_id, s.platform, s.goal, s.mode, s.started_at, s.ended_at, s.health,
29 + (select count(*) from actions a where a.session_id=s.session_id)::int as actions,
30 + (select count(distinct fingerprint) from entity_observations eo where eo.session_id=s.session_id)::int as entities
31 + from sessions s order by started_at desc limit 8`),
32 + this.q.query(`select date_trunc('hour', ts) + (floor(extract(minute from ts)/10)*10) * interval '1 minute' as bucket,
33 + count(*) filter (where event_type in ('VIDEO_DISCOVERED','POST_DISCOVERED','PROFILE_DISCOVERED','ENTITY_DISCOVERED'))::int as entities,
34 + count(*) filter (where event_type='NETWORK_RESPONSE_OBSERVED')::int as responses,
35 + count(*) filter (where event_type in ('ACTION_EXECUTED','ACTION_FAILED'))::int as actions,
36 + count(*) filter (where event_type='NETWORK_SCHEMA_DISCOVERED')::int as schemas
37 + from observations where ts > now() - interval '48 hours' group by 1 order by 1`),
38 + this.q.query(`select coalesce(sum((payload->>'dom_entities')::int),0)::int as dom, coalesce(sum((payload->>'network_entities')::int),0)::int as network,
39 + coalesce(sum((payload->>'both_surfaces')::int),0)::int as both, coalesce(sum((payload->>'field_agreements')::int),0)::int as agreements,
40 + coalesce(sum(jsonb_array_length(coalesce(payload->'field_conflicts','[]'::jsonb))),0)::int as conflicts
41 + from observations where event_type='PAGE_OPENED' and payload ? 'dom_entities'`),
42 + this.q.query(`select action_type, count(*)::int as n, avg(expected_gain)::float as avg_gain, avg(case when success then 1 else 0 end)::float as success_rate,
43 + avg((after_state->>'new_entities')::float) as avg_new_entities from actions group by 1 order by 2 desc`),
44 + this.q.query(`select platform, count(*)::int as n, avg((payload->>'confidence')::float) as conf from observations where event_type='CONNECTOR_PATTERN_LEARNED' group by 1`),
45 + ]);
46 + return {
47 + totals: totals.rows[0],
48 + by_type: byType.rows,
49 + running: running.rows,
50 + recent_sessions: recentSessions.rows,
51 + series: series.rows,
52 + surfaces: surfaces.rows[0],
53 + actions: topActions.rows,
54 + patterns_by_platform: patterns.rows,
55 + platforms: this.platforms(),
56 + };
57 + }
58 +
59 + platforms() {
60 + return PLATFORMS.map((p) => {
61 + const file = path.join(this.cfg.platformModelDir, p, "platform_model.json");
62 + const manifest = path.join(process.cwd(), "connectors", p, "manifest.json");
63 + if (!fs.existsSync(file)) return { platform: p, learned: false, adapter: p === "youtube" || p === "reddit", confidence: 0, page_types: 0, entity_types: 0, navigation_actions: 0, network_schemas: 0, media_patterns: 0, sessions: 0, has_manifest: fs.existsSync(manifest), profiles: this.profilesFor(p) };
64 + const m = new PlatformModel(p, this.cfg.platformModelDir);
65 + return { learned: true, adapter: p === "youtube" || p === "reddit", has_manifest: fs.existsSync(manifest), profiles: this.profilesFor(p), ...m.summary() };
66 + });
67 + }
68 +
69 + profilesFor(p: Platform) {
70 + if (!fs.existsSync(this.cfg.profilesDir)) return [];
71 + return fs
72 + .readdirSync(this.cfg.profilesDir)
73 + .filter((d) => d.startsWith(`${p}-`))
74 + .map((d) => ({ alias: d.slice(p.length + 1), has_state: fs.existsSync(path.join(this.cfg.profilesDir, d, "Default", "Cookies")) || fs.existsSync(path.join(this.cfg.profilesDir, d, "Default", "Network", "Cookies")) }));
75 + }
76 +
77 + platformDetail(p: Platform) {
78 + const file = path.join(this.cfg.platformModelDir, p, "platform_model.json");
79 + const manifest = path.join(process.cwd(), "connectors", p, "manifest.json");
80 + return {
81 + platform: p,
82 + model: fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : null,
83 + manifest: fs.existsSync(manifest) ? JSON.parse(fs.readFileSync(manifest, "utf8")) : null,
84 + summary: fs.existsSync(file) ? new PlatformModel(p, this.cfg.platformModelDir).summary() : null,
85 + profiles: this.profilesFor(p),
86 + };
87 + }
88 +
89 + async sessions(limit = 100) {
90 + return (
91 + await this.q.query(
92 + `select s.*, (select count(*) from actions a where a.session_id=s.session_id)::int as actions,
93 + (select count(distinct fingerprint) from entity_observations eo where eo.session_id=s.session_id)::int as entities,
94 + (select count(*) from observations o where o.session_id=s.session_id and o.event_type='NETWORK_SCHEMA_DISCOVERED')::int as schemas,
95 + (select count(*) from observations o where o.session_id=s.session_id and o.event_type='CONNECTOR_PATTERN_LEARNED')::int as patterns,
96 + (select payload->>'reason' from observations o where o.session_id=s.session_id and o.event_type='BUDGET_EXHAUSTED' limit 1) as ended_because
97 + from sessions s order by started_at desc limit $1`,
98 + [limit],
99 + )
100 + ).rows;
101 + }
102 +
103 + async session(id: string) {
104 + const row = (await this.q.query(`select * from sessions where session_id=$1`, [id])).rows[0];
105 + if (!row) return null;
106 + const dir = path.join(this.cfg.sessionsDir, id);
107 + const read = (f: string) => {
108 + try {
109 + return JSON.parse(fs.readFileSync(path.join(dir, f), "utf8"));
110 + } catch {
111 + return null;
112 + }
113 + };
114 + const counts = (await this.q.query(`select event_type, count(*)::int as n from observations where session_id=$1 group by 1`, [id])).rows;
115 + return { ...row, job: read("job.json"), summary: read("summary.json"), event_counts: counts };
116 + }
117 +
118 + async pages(id: string) {
119 + return (await this.q.query(`select step, ts, payload from observations where session_id=$1 and event_type='PAGE_OPENED' and payload ? 'page_type' order by ts`, [id])).rows;
120 + }
121 +
122 + async world(id: string) {
123 + const dir = path.join(this.cfg.sessionsDir, id, "world_model.json");
124 + if (fs.existsSync(dir)) return JSON.parse(fs.readFileSync(dir, "utf8"));
125 + // Reconstruct from the database for sessions crawled on another machine.
126 + const nodes = (await this.q.query(`select e.fingerprint, e.entity_type as type, e.name, e.url, e.platform_id, min(eo.step) as first_seen_step, max(eo.step) as last_seen_step, count(*)::int as seen_count from entities e join entity_observations eo on eo.fingerprint=e.fingerprint where eo.session_id=$1 group by e.fingerprint limit 1500`, [id])).rows;
127 + const edges = (await this.q.query(`select from_fp as "from", to_fp as "to", rel_type as type, step from relationships where session_id=$1 limit 4000`, [id])).rows;
128 + return { nodes, edges };
129 + }
130 +
131 + async searchEntities(o: { q?: string; type?: string; platform?: string; limit: number; offset: number; sort?: string }) {
132 + const params: unknown[] = [];
133 + const where: string[] = [];
134 + if (o.q) {
135 + params.push(`%${o.q}%`);
136 + where.push(`(name ilike $${params.length} or text_excerpt ilike $${params.length} or author ilike $${params.length} or platform_id ilike $${params.length})`);
137 + }
138 + if (o.type) {
139 + params.push(o.type);
140 + where.push(`entity_type = $${params.length}`);
141 + }
142 + if (o.platform) {
143 + params.push(o.platform);
144 + where.push(`platform = $${params.length}`);
145 + }
146 + const w = where.length ? `where ${where.join(" and ")}` : "";
147 + const order = o.sort === "views" ? `(metrics->>'views')::numeric desc nulls last` : o.sort === "seen" ? `seen_count desc` : `last_seen desc`;
148 + params.push(o.limit, o.offset);
149 + const rows = (await this.q.query(`select *, (select count(*) from entity_observations eo where eo.fingerprint=entities.fingerprint)::int as observations from entities ${w} order by ${order} limit $${params.length - 1} offset $${params.length}`, params)).rows;
150 + const total = (await this.q.query(`select count(*)::int as n from entities ${w}`, params.slice(0, -2))).rows[0]?.n ?? 0;
151 + return { rows, total };
152 + }
153 +
154 + async entity(fp: string) {
155 + const e = (await this.q.query(`select * from entities where fingerprint=$1`, [fp])).rows[0];
156 + if (!e) return null;
157 + const observations = (await this.q.query(`select eo.session_id, eo.step, eo.ts, eo.surfaces, eo.snapshot, s.goal from entity_observations eo left join sessions s on s.session_id=eo.session_id where eo.fingerprint=$1 order by eo.ts desc limit 100`, [fp])).rows;
158 + const relations = (await this.q.query(`select r.rel_type, r.from_fp, r.to_fp, r.step, coalesce(a.name, a.platform_id) as from_name, coalesce(b.name, b.platform_id) as to_name, a.entity_type as from_type, b.entity_type as to_type from relationships r left join entities a on a.fingerprint=r.from_fp left join entities b on b.fingerprint=r.to_fp where r.from_fp=$1 or r.to_fp=$1 limit 200`, [fp])).rows;
159 + const media = (await this.q.query(`select * from media where fingerprint=$1`, [fp])).rows[0] ?? null;
160 + const feed = (await this.q.query(`select session_id, step, page_type, feed_position, visible, ts from feed_items where fingerprint=$1 order by ts desc limit 50`, [fp])).rows;
161 + return { entity: e, observations, relations, media, feed };
162 + }
163 +
164 + async schemas(platform?: string) {
165 + const params: unknown[] = [];
166 + let w = "";
167 + if (platform) {
168 + params.push(platform);
169 + w = `where platform=$1`;
170 + }
171 + return (await this.q.query(`select * from schema_patterns ${w} order by observed_count desc limit 300`, params)).rows;
172 + }
173 +
174 + async liveEvents(since: string, session?: string, limit = 200) {
175 + const params: unknown[] = [since];
176 + let w = `ts > $1`;
177 + if (session) {
178 + params.push(session);
179 + w += ` and session_id=$2`;
180 + }
181 + params.push(limit);
182 + return (await this.q.query(`select event_id, session_id, platform, event_type, step, ts, payload, provenance from observations where ${w} order by ts asc limit $${params.length}`, params)).rows;
183 + }
184 +
185 + async jobs(limit = 50) {
186 + return (await this.q.query(`select j.*, s.health, (select count(*) from actions a where a.session_id=j.session_id)::int as actions, (select count(distinct fingerprint) from entity_observations eo where eo.session_id=j.session_id)::int as entities from crawl_jobs j left join sessions s on s.session_id=j.session_id order by created_at desc limit $1`, [limit])).rows;
187 + }
188 +}
modified apps/api/src/server.ts +115 −74
@@ -2,108 +2,149 @@ import http from "node:http";
2 2 import fs from "node:fs";
3 3 import path from "node:path";
4 4 import { fileURLToPath } from "node:url";
5 −import { createLogger, loadConfig } from "@src/shared";
6 −import { JsonlEventLog } from "@src/events";
5 +import { createLogger, isPlatform, loadConfig } from "@src/shared";
7 6 import { PostgresStore } from "@src/storage";
7 +import { Queries } from "./queries.ts";
8 +import { JobRunner } from "./jobs.ts";
8 9
9 10 /**
10 − * Research/Debug dashboard API (§53, §82). Zero framework: node:http + a static page.
11 − * Reads PostgreSQL when configured, else the JSONL session logs.
11 + * Control-plane + research API (§53, §63, §82). Plain node:http, JSON, SSE.
12 + * Consumed by the Next.js console (apps/dashboard) through its authenticated proxy; mutations require SRC_API_TOKEN.
12 13 */
13 −const log = createLogger("dashboard");
14 +const log = createLogger("api");
14 15 const cfg = loadConfig();
15 16 const here = path.dirname(fileURLToPath(import.meta.url));
16 17 const publicDir = path.join(here, "..", "public");
17 −let store: PostgresStore | undefined;
18 −try {
19 − if (cfg.databaseUrl) store = await PostgresStore.connect(cfg.databaseUrl);
20 −} catch (err) {
21 − log.warn("no database — serving JSONL logs", { err: (err as Error).message });
18 +const API_TOKEN = process.env.SRC_API_TOKEN ?? "";
19 +const PORT = Number(process.env.SRC_API_PORT ?? cfg.dashboardPort);
20 +
21 +if (!cfg.databaseUrl) {
22 + console.error("SRC_DATABASE_URL is required for the API");
23 + process.exit(1);
22 24 }
25 +const store = await PostgresStore.connect(cfg.databaseUrl);
26 +await store.migrate();
27 +const queries = new Queries(store, cfg);
28 +const jobs = new JobRunner(store, cfg, Number(process.env.SRC_MAX_JOBS ?? 2));
23 29
24 30 function json(res: http.ServerResponse, data: unknown, status = 200) {
25 − res.writeHead(status, { "content-type": "application/json; charset=utf-8", "access-control-allow-origin": "*" });
31 + res.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" });
26 32 res.end(JSON.stringify(data));
27 33 }
28 −
29 −function sessionDirs(): string[] {
30 − return fs.existsSync(cfg.sessionsDir) ? fs.readdirSync(cfg.sessionsDir).filter((d) => fs.existsSync(path.join(cfg.sessionsDir, d, "events.jsonl"))).sort().reverse() : [];
31 −}
32 −
33 −function jsonlSessions() {
34 − return sessionDirs().map((id) => {
35 − const job = readJson(path.join(cfg.sessionsDir, id, "job.json")) as Record<string, unknown> | undefined;
36 − const summary = readJson(path.join(cfg.sessionsDir, id, "summary.json")) as Record<string, unknown> | undefined;
37 − return { session_id: id, platform: job?.platform, mode: job?.mode, goal: job?.goal, account_alias: job?.account_alias, started_at: fs.statSync(path.join(cfg.sessionsDir, id)).birthtime, health: summary ? "stopped" : "running", actions: summary?.steps ?? null, entities: summary?.entities ?? null, ended_because: summary?.ended_because };
34 +function readBody(req: http.IncomingMessage): Promise<string> {
35 + return new Promise((resolve, reject) => {
36 + let s = "";
37 + req.on("data", (c) => {
38 + s += c;
39 + if (s.length > 1e6) reject(new Error("body too large"));
40 + });
41 + req.on("end", () => resolve(s));
42 + req.on("error", reject);
38 43 });
39 44 }
40 −
41 −function readJson(file: string): unknown {
42 − try {
43 − return JSON.parse(fs.readFileSync(file, "utf8"));
44 − } catch {
45 − return undefined;
46 − }
45 +function authorized(req: http.IncomingMessage): boolean {
46 + if (!API_TOKEN) return true; // dev
47 + return req.headers["x-src-token"] === API_TOKEN;
47 48 }
48 49
49 50 const server = http.createServer(async (req, res) => {
50 51 const url = new URL(req.url ?? "/", "http://localhost");
51 52 const parts = url.pathname.split("/").filter(Boolean);
53 + const method = req.method ?? "GET";
52 54 try {
53 − if (parts[0] === "api") {
54 − if (parts[1] === "sessions" && !parts[2]) return json(res, store ? await store.listSessions() : jsonlSessions());
55 − if (parts[1] === "sessions" && parts[2]) {
56 − const id = parts[2];
57 − const dir = path.join(cfg.sessionsDir, id);
58 − const sub = parts[3];
59 − if (sub === "events") {
60 − const types = url.searchParams.get("types")?.split(",").filter(Boolean);
61 − const limit = Number(url.searchParams.get("limit") ?? 300);
62 − if (store) return json(res, await store.sessionEvents(id, { types, limit }));
63 − const evs = JsonlEventLog.read(dir).filter((e) => !types?.length || types.includes(e.event_type));
64 − return json(res, evs.slice(-limit).reverse());
65 − }
66 − if (sub === "actions") return json(res, store ? await store.sessionActions(id) : JsonlEventLog.read(dir).filter((e) => e.event_type === "ACTION_PLANNED").map((e) => ({ step: e.step, ...(e.payload as Record<string, unknown>) })));
67 − if (sub === "entities") {
68 − if (store) return json(res, await store.sessionEntities(id));
69 − const seen = new Map<string, unknown>();
70 − for (const e of JsonlEventLog.read(dir)) if (/DISCOVERED|ENTITY_OBSERVED/.test(e.event_type) && (e.payload as { entity?: { fingerprint: string } }).entity) seen.set((e.payload as { entity: { fingerprint: string } }).entity.fingerprint, { ...(e.payload as { entity: object }).entity, last_step: e.step, provenance: e.provenance });
71 − return json(res, [...seen.values()].reverse());
72 − }
73 − if (sub === "media") return json(res, store ? await store.sessionMedia(id) : JsonlEventLog.read(dir).filter((e) => e.event_type === "MEDIA_DISCOVERED").map((e) => e.payload));
74 − if (sub === "world") return json(res, readJson(path.join(dir, "world_model.json")) ?? { nodes: [], edges: [] });
75 − if (sub === "summary") return json(res, { job: readJson(path.join(dir, "job.json")), summary: readJson(path.join(dir, "summary.json")) });
76 − if (sub === "schemas") {
77 − const platform = (readJson(path.join(dir, "job.json")) as { platform?: string } | undefined)?.platform;
78 − if (store) return json(res, await store.schemas(platform));
79 − return json(res, JsonlEventLog.read(dir).filter((e) => e.event_type === "NETWORK_SCHEMA_DISCOVERED").map((e) => e.payload));
55 + if (parts[0] !== "api" && parts[0] !== "media") {
56 + // legacy static console (apps/api/public) — kept for local debugging
57 + const file = path.join(publicDir, parts.length ? parts.join("/") : "index.html");
58 + if (!file.startsWith(publicDir) || !fs.existsSync(file)) return json(res, { error: "not found" }, 404);
59 + res.writeHead(200, { "content-type": file.endsWith(".html") ? "text/html; charset=utf-8" : "application/octet-stream" });
60 + return fs.createReadStream(file).pipe(res);
61 + }
62 + if (parts[0] === "media") {
63 + const file = path.join(cfg.mediaDir, ...parts.slice(1).map(decodeURIComponent));
64 + if (!file.startsWith(cfg.mediaDir) || !fs.existsSync(file)) return json(res, { error: "not found" }, 404);
65 + res.writeHead(200, { "content-type": "image/jpeg", "cache-control": "public, max-age=86400" });
66 + return fs.createReadStream(file).pipe(res);
67 + }
68 + if (method !== "GET" && !authorized(req)) return json(res, { error: "unauthorized" }, 401);
69 + const [, r1, r2, r3] = parts;
70 +
71 + if (r1 === "health") return json(res, { ok: true, jobs_running: jobs.activeCount, db: true, ts: new Date().toISOString() });
72 + if (r1 === "overview") return json(res, await queries.overview());
73 + if (r1 === "platforms" && !r2) return json(res, queries.platforms());
74 + if (r1 === "platforms" && r2) return isPlatform(r2) ? json(res, queries.platformDetail(r2)) : json(res, { error: "unknown platform" }, 404);
75 + if (r1 === "sessions" && !r2) return json(res, await queries.sessions(Number(url.searchParams.get("limit") ?? 100)));
76 + if (r1 === "sessions" && r2) {
77 + const id = r2;
78 + if (!r3) {
79 + const s = await queries.session(id);
80 + return s ? json(res, s) : json(res, { error: "not found" }, 404);
81 + }
82 + if (r3 === "events") return json(res, await store.sessionEvents(id, { types: url.searchParams.get("types")?.split(",").filter(Boolean), limit: Number(url.searchParams.get("limit") ?? 300) }));
83 + if (r3 === "actions") return json(res, await store.sessionActions(id));
84 + if (r3 === "entities") return json(res, await store.sessionEntities(id, Number(url.searchParams.get("limit") ?? 500)));
85 + if (r3 === "media") return json(res, await store.sessionMedia(id));
86 + if (r3 === "pages") return json(res, await queries.pages(id));
87 + if (r3 === "world") return json(res, await queries.world(id));
88 + if (r3 === "schemas") {
89 + const s = await queries.session(id);
90 + return json(res, await queries.schemas(s?.platform));
91 + }
92 + if (r3 === "relationships") return json(res, await store.sessionRelationships(id));
93 + }
94 + if (r1 === "entities" && !r2) return json(res, await queries.searchEntities({ q: url.searchParams.get("q") ?? undefined, type: url.searchParams.get("type") ?? undefined, platform: url.searchParams.get("platform") ?? undefined, sort: url.searchParams.get("sort") ?? undefined, limit: Math.min(200, Number(url.searchParams.get("limit") ?? 50)), offset: Number(url.searchParams.get("offset") ?? 0) }));
95 + if (r1 === "entities" && r2) {
96 + const e = await queries.entity(decodeURIComponent(r2));
97 + return e ? json(res, e) : json(res, { error: "not found" }, 404);
98 + }
99 + if (r1 === "schemas") return json(res, await queries.schemas(url.searchParams.get("platform") ?? undefined));
100 + if (r1 === "stats") return json(res, await store.entityStats());
101 +
102 + if (r1 === "stream") {
103 + // Server-Sent Events: poll the observations table and push new rows (§53 live panels).
104 + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-store", connection: "keep-alive", "x-accel-buffering": "no" });
105 + let since = url.searchParams.get("since") ?? new Date(Date.now() - 60_000).toISOString();
106 + const session = url.searchParams.get("session") ?? undefined;
107 + const types = url.searchParams.get("types")?.split(",").filter(Boolean);
108 + let alive = true;
109 + req.on("close", () => (alive = false));
110 + res.write(`event: hello\ndata: ${JSON.stringify({ since })}\n\n`);
111 + while (alive) {
112 + try {
113 + const rows = await queries.liveEvents(since, session, 200);
114 + for (const row of rows) {
115 + since = new Date(row.ts).toISOString();
116 + if (types && !types.includes(row.event_type)) continue;
117 + res.write(`event: observation\ndata: ${JSON.stringify(row)}\n\n`);
118 + }
119 + res.write(`: ping ${Date.now()}\n\n`);
120 + } catch (err) {
121 + res.write(`event: error\ndata: ${JSON.stringify({ error: (err as Error).message })}\n\n`);
80 122 }
123 + await new Promise((r) => setTimeout(r, 1500));
81 124 }
82 − if (parts[1] === "platform-model" && parts[2]) {
83 − const f = path.join(cfg.platformModelDir, parts[2], "platform_model.json");
84 − return json(res, readJson(f) ?? { error: "no model yet" });
125 + return res.end();
126 + }
127 +
128 + if (r1 === "jobs" && method === "GET" && !r2) return json(res, await queries.jobs());
129 + if (r1 === "jobs" && method === "POST" && !r2) {
130 + const body = JSON.parse((await readBody(req)) || "{}");
131 + try {
132 + return json(res, await jobs.start(body), 201);
133 + } catch (err) {
134 + return json(res, { error: (err as Error).message }, 400);
85 135 }
86 − if (parts[1] === "entity" && parts[2]) return json(res, store ? await store.entity(decodeURIComponent(parts[2])) : { error: "needs database" });
87 − if (parts[1] === "stats") return json(res, store ? await store.entityStats() : []);
88 − return json(res, { error: "not found" }, 404);
89 136 }
90 − if (parts[0] === "media") {
91 − // serve sampled frames
92 − const file = path.join(cfg.mediaDir, ...parts.slice(1));
93 − if (!file.startsWith(cfg.mediaDir) || !fs.existsSync(file)) return json(res, { error: "not found" }, 404);
94 − res.writeHead(200, { "content-type": "image/jpeg" });
95 − fs.createReadStream(file).pipe(res);
96 − return;
137 + if (r1 === "jobs" && r2 && r3 === "log") {
138 + res.writeHead(200, { "content-type": "text/plain; charset=utf-8", "cache-control": "no-store" });
139 + return res.end(jobs.logTail(r2, Number(url.searchParams.get("lines") ?? 200)));
97 140 }
98 − const file = path.join(publicDir, parts.length ? parts.join("/") : "index.html");
99 − if (!file.startsWith(publicDir) || !fs.existsSync(file)) return json(res, { error: "not found" }, 404);
100 − const ext = path.extname(file);
101 − res.writeHead(200, { "content-type": ext === ".html" ? "text/html; charset=utf-8" : ext === ".js" ? "text/javascript" : ext === ".css" ? "text/css" : "application/octet-stream" });
102 − fs.createReadStream(file).pipe(res);
141 + if (r1 === "jobs" && r2 && r3 === "stop" && method === "POST") return json(res, { stopped: jobs.stop(r2) });
142 +
143 + return json(res, { error: "not found" }, 404);
103 144 } catch (err) {
104 145 log.error("request failed", { url: req.url, err: (err as Error).message });
105 146 json(res, { error: (err as Error).message }, 500);
106 147 }
107 148 });
108 149
109 −server.listen(cfg.dashboardPort, () => log.info(`dashboard → http://localhost:${cfg.dashboardPort} (${store ? "postgres" : "jsonl"})`));
150 +server.listen(PORT, "127.0.0.1", () => log.info(`api → http://127.0.0.1:${PORT} (token ${API_TOKEN ? "on" : "off"})`));
added apps/dashboard/next.config.ts +13 −0
@@ -0,0 +1,13 @@
1 +import type { NextConfig } from "next";
2 +
3 +const nextConfig: NextConfig = {
4 + reactStrictMode: true,
5 + poweredByHeader: false,
6 + images: { unoptimized: true },
7 + experimental: { serverActions: { allowedOrigins: ["www.socialcrawl.co", "socialcrawl.co", "localhost:8351"] } },
8 + async headers() {
9 + return [{ source: "/(.*)", headers: [{ key: "X-Frame-Options", value: "DENY" }, { key: "X-Content-Type-Options", value: "nosniff" }, { key: "Referrer-Policy", value: "same-origin" }] }];
10 + },
11 +};
12 +
13 +export default nextConfig;
added apps/dashboard/package.json +31 −0
@@ -0,0 +1,31 @@
1 +{
2 + "name": "@src/dashboard",
3 + "version": "0.1.0",
4 + "private": true,
5 + "type": "module",
6 + "scripts": {
7 + "dev": "next dev -p 8351",
8 + "build": "next build",
9 + "start": "next start -p 8351 -H 0.0.0.0",
10 + "typecheck": "tsc -p tsconfig.json --noEmit"
11 + },
12 + "dependencies": {
13 + "clsx": "^2.1.1",
14 + "d3-force": "^3.0.0",
15 + "framer-motion": "^12.23.0",
16 + "lucide-react": "^1.0.0",
17 + "next": "16.3.4",
18 + "react": "19.2.8",
19 + "react-dom": "19.2.8",
20 + "recharts": "^3.0.0"
21 + },
22 + "devDependencies": {
23 + "@tailwindcss/postcss": "^4",
24 + "@types/d3-force": "^3.0.10",
25 + "@types/node": "^24.0.0",
26 + "@types/react": "^19",
27 + "@types/react-dom": "^19",
28 + "tailwindcss": "^4",
29 + "typescript": "^5.9.3"
30 + }
31 +}
added apps/dashboard/postcss.config.mjs +2 −0
@@ -0,0 +1,2 @@
1 +const config = { plugins: { "@tailwindcss/postcss": {} } };
2 +export default config;
added apps/dashboard/public/brand/icon.svg +1 −0
@@ -0,0 +1 @@
1 +<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48"><rect width="48" height="48" rx="10" fill="#05070b"/><defs><linearGradient id="g" x1="0" x2="1" y1="0" y2="1"><stop offset="0" stop-color="#38e1ff"/><stop offset="1" stop-color="#9d7bff"/></linearGradient></defs><ellipse cx="24" cy="24" rx="19" ry="10" fill="none" stroke="url(#g)" stroke-width="2.2" transform="rotate(-24 24 24)"/><ellipse cx="24" cy="24" rx="19" ry="10" fill="none" stroke="url(#g)" stroke-width="1.2" opacity=".5" transform="rotate(36 24 24)"/><circle cx="24" cy="24" r="5.5" fill="url(#g)"/><circle cx="40" cy="16" r="2.2" fill="#ffb347"/></svg>
added apps/dashboard/src/app/access/page.tsx +39 −0
@@ -0,0 +1,39 @@
1 +import { cookies } from "next/headers";
2 +import { redirect } from "next/navigation";
3 +import { COOKIE, checkPasscode, gateEnabled, makeToken } from "@/lib/auth";
4 +import { Brand } from "@/components/Brand";
5 +
6 +export const dynamic = "force-dynamic";
7 +
8 +async function login(formData: FormData) {
9 + "use server";
10 + const code = String(formData.get("passcode") ?? "");
11 + const next = String(formData.get("next") ?? "/");
12 + if (!checkPasscode(code)) redirect(`/access?error=1&next=${encodeURIComponent(next)}`);
13 + const jar = await cookies();
14 + jar.set(COOKIE, makeToken(), { httpOnly: true, sameSite: "lax", secure: process.env.NODE_ENV === "production", path: "/", maxAge: 30 * 24 * 3600 });
15 + redirect(next.startsWith("/") ? next : "/");
16 +}
17 +
18 +export default async function AccessPage({ searchParams }: { searchParams: Promise<{ error?: string; next?: string }> }) {
19 + const sp = await searchParams;
20 + if (!gateEnabled()) redirect("/");
21 + return (
22 + <main className="min-h-screen grid place-items-center grid-bg px-6">
23 + <form action={login} className="panel w-full max-w-sm p-8 space-y-6">
24 + <Brand size="lg" />
25 + <div className="space-y-1">
26 + <h1 className="text-lg font-semibold">Research console</h1>
27 + <p className="text-sm text-fg-2">This console observes live social runtimes and can start browser sessions on the cluster. Enter the access code.</p>
28 + </div>
29 + <input type="hidden" name="next" value={sp.next ?? "/"} />
30 + <label className="block space-y-2">
31 + <span className="text-xs uppercase tracking-widest text-dim">Access code</span>
32 + <input name="passcode" type="password" autoFocus required className="w-full rounded-lg bg-bg-2 border border-line-2 px-3 py-2 mono text-fg outline-none focus:border-cyan/60 focus:glow-cyan" />
33 + </label>
34 + {sp.error && <p className="text-sm text-red">Wrong code.</p>}
35 + <button className="w-full rounded-lg bg-cyan/90 hover:bg-cyan text-bg font-semibold py-2 transition">Enter</button>
36 + </form>
37 + </main>
38 + );
39 +}
added apps/dashboard/src/app/api/[...path]/route.ts +28 −0
@@ -0,0 +1,28 @@
1 +import { cookies } from "next/headers";
2 +import { NextResponse, type NextRequest } from "next/server";
3 +import { COOKIE, verifyToken } from "@/lib/auth";
4 +
5 +export const dynamic = "force-dynamic";
6 +const API_URL = process.env.API_URL ?? "http://127.0.0.1:8350";
7 +const API_TOKEN = process.env.SRC_API_TOKEN ?? "";
8 +
9 +/** Authenticated pass-through to the control-plane API (adds the server token, streams SSE as-is). */
10 +async function forward(req: NextRequest, path: string[]) {
11 + const jar = await cookies();
12 + if (!verifyToken(jar.get(COOKIE)?.value)) return NextResponse.json({ error: "unauthorized" }, { status: 401 });
13 + const target = new URL(`/api/${path.join("/")}`, API_URL);
14 + target.search = req.nextUrl.search;
15 + const headers: Record<string, string> = { "x-src-token": API_TOKEN };
16 + if (req.headers.get("content-type")) headers["content-type"] = req.headers.get("content-type")!;
17 + const upstream = await fetch(target, { method: req.method, headers, body: req.method === "GET" || req.method === "HEAD" ? undefined : await req.text(), cache: "no-store", // @ts-expect-error — undici option for streaming request bodies
18 + duplex: "half" });
19 + const ct = upstream.headers.get("content-type") ?? "application/json";
20 + return new Response(upstream.body, { status: upstream.status, headers: { "content-type": ct, "cache-control": "no-store", ...(ct.includes("event-stream") ? { connection: "keep-alive", "x-accel-buffering": "no" } : {}) } });
21 +}
22 +
23 +export async function GET(req: NextRequest, ctx: { params: Promise<{ path: string[] }> }) {
24 + return forward(req, (await ctx.params).path);
25 +}
26 +export async function POST(req: NextRequest, ctx: { params: Promise<{ path: string[] }> }) {
27 + return forward(req, (await ctx.params).path);
28 +}
added apps/dashboard/src/app/entities/[fp]/page.tsx +173 −0
@@ -0,0 +1,173 @@
1 +"use client";
2 +
3 +import { use } from "react";
4 +import Link from "next/link";
5 +import { ExternalLink } from "lucide-react";
6 +import { useApi } from "@/lib/api";
7 +import { fmtDate, fmtDuration, fmtNum, truncate } from "@/lib/format";
8 +import { Empty, Metric, Panel, Skeleton, SurfaceChips, Table, Tag, TypeTag } from "@/components/ui";
9 +import type { EntityRow } from "@/components/EntityTable";
10 +
11 +interface Detail {
12 + entity: EntityRow & { first_seen: string; last_seen: string; seen_count: number; first_session: string };
13 + observations: { session_id: string; step: number; ts: string; surfaces: string[]; snapshot: { name?: string; author?: string; metrics?: Record<string, number>; context?: string; provenance?: { surface: string; confidence: number; detail?: string }[] }; goal: string | null }[];
14 + relations: { rel_type: string; from_fp: string; to_fp: string; from_name: string | null; to_name: string | null; from_type: string; to_type: string; step: number }[];
15 + media: { duration_s: number | null; width: number | null; height: number | null; delivery: { kind: string; hostnames: string[] } | null; frames: string[] | null; thumbnail_url: string | null } | null;
16 + feed: { session_id: string; step: number; page_type: string; feed_position: number; visible: boolean; ts: string }[];
17 +}
18 +
19 +export default function EntityPage({ params }: { params: Promise<{ fp: string }> }) {
20 + const { fp } = use(params);
21 + const { data, loading, error } = useApi<Detail>(`/entities/${encodeURIComponent(decodeURIComponent(fp))}`);
22 + if (loading) return <Skeleton rows={10} />;
23 + if (error || !data) return <Empty>{error ?? "not found"}</Empty>;
24 + const e = data.entity;
25 + const type = e.entity_type ?? e.type ?? "?";
26 + return (
27 + <div className="space-y-4">
28 + <div className="text-xs text-dim mono">
29 + <Link href="/entities" className="hover:text-cyan">entities</Link> / {e.fingerprint}
30 + </div>
31 + <div className="flex flex-wrap items-start gap-4">
32 + <div className="min-w-0">
33 + <div className="flex items-center gap-2">
34 + <TypeTag type={type} />
35 + <Tag>{e.platform}</Tag>
36 + {e.url && (
37 + <a href={e.url} target="_blank" rel="noreferrer" className="text-dim hover:text-cyan flex items-center gap-1 text-xs">
38 + open on platform <ExternalLink className="h-3.5 w-3.5" />
39 + </a>
40 + )}
41 + </div>
42 + <h1 className="text-xl font-semibold tracking-tight mt-2 max-w-3xl">{e.name ?? e.text_excerpt ?? e.platform_id}</h1>
43 + {e.author && <div className="text-sm text-fg-2 mt-1">by {e.author}</div>}
44 + {e.text_excerpt && e.name && <p className="text-sm text-fg-2 mt-2 max-w-3xl">{e.text_excerpt}</p>}
45 + </div>
46 + </div>
47 +
48 + <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">
49 + <Panel title="Evidence per field" sub="every canonical value points back to the surface that produced it (§38)" className="xl:col-span-2" pad={false}>
50 + <div className="p-4">
51 + <Table head={["field", "value", "evidence"]} dense>
52 + {Object.entries(e.fields ?? {}).map(([k, f]) => (
53 + <tr key={k}>
54 + <td className="mono text-fg-2 whitespace-nowrap">{k}</td>
55 + <td className="max-w-[480px] break-words">{typeof f.value === "string" ? f.value : JSON.stringify(f.value)}</td>
56 + <td>
57 + <SurfaceChips provenance={f.provenance} />
58 + {f.provenance?.[0]?.detail && <div className="text-[10px] text-dim mono truncate max-w-[280px]">{f.provenance[0].detail}</div>}
59 + </td>
60 + </tr>
61 + ))}
62 + </Table>
63 + </div>
64 + </Panel>
65 + <div className="space-y-4">
66 + <Panel title="Canonical record">
67 + <Metric label="platform id" value={e.platform_id ?? "—"} />
68 + <Metric label="confidence" value={e.confidence ? Number(e.confidence).toFixed(2) : "—"} />
69 + <Metric label="observed" value={`${e.seen_count}× in ${new Set(data.observations.map((o) => o.session_id)).size} session(s)`} />
70 + <Metric label="first seen" value={fmtDate(e.first_seen)} />
71 + <Metric label="last seen" value={fmtDate(e.last_seen)} />
72 + {e.metrics &&
73 + Object.entries(e.metrics).map(([k, v]) => (
74 + <Metric key={k} label={k} value={fmtNum(v)} />
75 + ))}
76 + {data.media && (
77 + <>
78 + <Metric label="duration" value={fmtDuration(data.media.duration_s)} />
79 + {data.media.width ? <Metric label="resolution" value={`${data.media.width}×${data.media.height}`} /> : null}
80 + {data.media.delivery && <Metric label="delivery" value={`${data.media.delivery.kind} · ${data.media.delivery.hostnames.slice(0, 2).join(", ")}`} />}
81 + </>
82 + )}
83 + </Panel>
84 + {data.media?.frames?.length ? (
85 + <Panel title="Sampled frames" sub="0 / 25 / 50 / 75 / 100 % (LEVEL 2)">
86 + <div className="grid grid-cols-5 gap-1">
87 + {data.media.frames.map((f) => (
88 + // eslint-disable-next-line @next/next/no-img-element
89 + <img key={f} src={`/media/${f.split("/media/")[1] ?? ""}`} alt="" className="aspect-video object-cover rounded" />
90 + ))}
91 + </div>
92 + </Panel>
93 + ) : data.media?.thumbnail_url || e.media?.thumbnail_url ? (
94 + <Panel title="Thumbnail">
95 + {/* eslint-disable-next-line @next/next/no-img-element */}
96 + <img src={data.media?.thumbnail_url ?? e.media?.thumbnail_url ?? ""} alt="" className="rounded w-full" />
97 + </Panel>
98 + ) : null}
99 + </div>
100 + </div>
101 +
102 + <div className="grid grid-cols-1 xl:grid-cols-2 gap-4">
103 + <Panel title="Observations" sub="raw sightings, never overwritten (§37)" pad={false}>
104 + <div className="p-4">
105 + <Table head={["when", "session", "step", "surfaces", "context", "name as seen"]} dense>
106 + {data.observations.map((o, i) => (
107 + <tr key={i}>
108 + <td className="mono text-dim whitespace-nowrap">{fmtDate(o.ts)}</td>
109 + <td>
110 + <Link href={`/sessions/${o.session_id}`} className="mono text-[11px] hover:text-cyan">
111 + {truncate(o.goal ?? o.session_id, 32)}
112 + </Link>
113 + </td>
114 + <td className="mono">#{o.step}</td>
115 + <td>
116 + <SurfaceChips provenance={o.snapshot.provenance ?? o.surfaces.map((s) => ({ surface: s, confidence: 0 }))} />
117 + </td>
118 + <td className="text-dim text-[11px]">{o.snapshot.context}</td>
119 + <td className="text-fg-2 max-w-[260px] truncate">{o.snapshot.name}</td>
120 + </tr>
121 + ))}
122 + </Table>
123 + </div>
124 + </Panel>
125 + <div className="space-y-4">
126 + <Panel title="Relationships" pad={false}>
127 + <div className="p-4">
128 + {data.relations.length ? (
129 + <Table head={["from", "relation", "to"]} dense>
130 + {data.relations.map((r, i) => (
131 + <tr key={i}>
132 + <td className="max-w-[240px] truncate">
133 + <TypeTag type={r.from_type ?? "?"} /> <Link href={`/entities/${encodeURIComponent(r.from_fp)}`} className="hover:text-cyan">{truncate(r.from_name ?? r.from_fp, 40)}</Link>
134 + </td>
135 + <td>
136 + <Tag color="#9d7bff">{r.rel_type}</Tag>
137 + </td>
138 + <td className="max-w-[240px] truncate">
139 + <TypeTag type={r.to_type ?? "?"} /> <Link href={`/entities/${encodeURIComponent(r.to_fp)}`} className="hover:text-cyan">{truncate(r.to_name ?? r.to_fp, 40)}</Link>
140 + </td>
141 + </tr>
142 + ))}
143 + </Table>
144 + ) : (
145 + <Empty>no relationships recorded</Empty>
146 + )}
147 + </div>
148 + </Panel>
149 + <Panel title="Feed exposure" sub="where it ranked when the crawler saw it (§44)" pad={false}>
150 + <div className="p-4">
151 + {data.feed.length ? (
152 + <Table head={["when", "page", "position", "visible"]} dense>
153 + {data.feed.map((f, i) => (
154 + <tr key={i}>
155 + <td className="mono text-dim whitespace-nowrap">{fmtDate(f.ts)}</td>
156 + <td>
157 + <Tag color="#38e1ff">{f.page_type}</Tag>
158 + </td>
159 + <td className="mono">#{f.feed_position + 1}</td>
160 + <td className="mono">{f.visible ? "yes" : "below fold"}</td>
161 + </tr>
162 + ))}
163 + </Table>
164 + ) : (
165 + <Empty>not seen in a feed</Empty>
166 + )}
167 + </div>
168 + </Panel>
169 + </div>
170 + </div>
171 + </div>
172 + );
173 +}
added apps/dashboard/src/app/entities/page.tsx +85 −0
@@ -0,0 +1,85 @@
1 +"use client";
2 +
3 +import { useState } from "react";
4 +import { Search } from "lucide-react";
5 +import { useApi } from "@/lib/api";
6 +import { fmtNum } from "@/lib/format";
7 +import { Empty, Panel, Skeleton } from "@/components/ui";
8 +import { EntityTable, type EntityRow } from "@/components/EntityTable";
9 +
10 +const TYPES = ["", "video", "channel", "profile", "post", "comment", "community", "hashtag", "page"];
11 +const PLATFORMS = ["", "youtube", "reddit", "facebook", "instagram", "tiktok", "x"];
12 +
13 +export default function Entities() {
14 + const [q, setQ] = useState("");
15 + const [type, setType] = useState("");
16 + const [platform, setPlatform] = useState("");
17 + const [sort, setSort] = useState("last_seen");
18 + const [page, setPage] = useState(0);
19 + const limit = 50;
20 + const params = new URLSearchParams({ limit: String(limit), offset: String(page * limit), sort });
21 + if (q) params.set("q", q);
22 + if (type) params.set("type", type);
23 + if (platform) params.set("platform", platform);
24 + const { data, loading } = useApi<{ rows: EntityRow[]; total: number }>(`/entities?${params}`);
25 + const sel = "rounded-lg bg-bg-2 border border-line-2 px-2 py-1.5 text-sm text-fg outline-none focus:border-cyan/60";
26 + return (
27 + <div className="space-y-4">
28 + <div className="flex flex-wrap items-end gap-3">
29 + <div>
30 + <h1 className="text-2xl font-semibold tracking-tight">Entities</h1>
31 + <p className="text-sm text-fg-2">Canonical entities with per-field evidence. {data ? <span className="mono">{fmtNum(data.total)} match</span> : null}</p>
32 + </div>
33 + <div className="ml-auto flex flex-wrap gap-2 items-center">
34 + <label className="relative">
35 + <Search className="h-4 w-4 absolute left-2 top-2 text-dim" />
36 + <input
37 + value={q}
38 + onChange={(e) => {
39 + setQ(e.target.value);
40 + setPage(0);
41 + }}
42 + placeholder="search name, text, author, id…"
43 + className={`${sel} pl-8 w-72 mono`}
44 + />
45 + </label>
46 + <select value={type} onChange={(e) => (setType(e.target.value), setPage(0))} className={sel}>
47 + {TYPES.map((t) => (
48 + <option key={t} value={t}>
49 + {t || "all types"}
50 + </option>
51 + ))}
52 + </select>
53 + <select value={platform} onChange={(e) => (setPlatform(e.target.value), setPage(0))} className={sel}>
54 + {PLATFORMS.map((p) => (
55 + <option key={p} value={p}>
56 + {p || "all platforms"}
57 + </option>
58 + ))}
59 + </select>
60 + <select value={sort} onChange={(e) => setSort(e.target.value)} className={sel}>
61 + <option value="last_seen">recently seen</option>
62 + <option value="seen">most observed</option>
63 + <option value="views">most views</option>
64 + </select>
65 + </div>
66 + </div>
67 + <Panel pad={false}>
68 + <div className="p-4">{loading ? <Skeleton rows={10} /> : data?.rows.length ? <EntityTable rows={data.rows} showPlatform /> : <Empty>Nothing matches.</Empty>}</div>
69 + {data && data.total > limit && (
70 + <div className="flex items-center gap-3 px-4 py-3 border-t border-line text-xs">
71 + <button disabled={page === 0} onClick={() => setPage((p) => p - 1)} className="px-2 py-1 rounded border border-line disabled:opacity-40 hover:border-cyan/50">
72 + ← prev
73 + </button>
74 + <span className="mono text-dim">
75 + {page * limit + 1}–{Math.min(data.total, (page + 1) * limit)} of {data.total}
76 + </span>
77 + <button disabled={(page + 1) * limit >= data.total} onClick={() => setPage((p) => p + 1)} className="px-2 py-1 rounded border border-line disabled:opacity-40 hover:border-cyan/50">
78 + next →
79 + </button>
80 + </div>
81 + )}
82 + </Panel>
83 + </div>
84 + );
85 +}
added apps/dashboard/src/app/globals.css +87 −0
@@ -0,0 +1,87 @@
1 +@import "tailwindcss";
2 +
3 +@theme {
4 + --color-bg: #05070b;
5 + --color-bg-2: #0a0e15;
6 + --color-panel: #0d1219;
7 + --color-panel-2: #111823;
8 + --color-line: #1a2331;
9 + --color-line-2: #243042;
10 + --color-fg: #e6edf5;
11 + --color-fg-2: #a9b6c6;
12 + --color-dim: #6b7a8c;
13 + --color-cyan: #38e1ff;
14 + --color-amber: #ffb347;
15 + --color-green: #43e69a;
16 + --color-red: #ff5c7a;
17 + --color-violet: #9d7bff;
18 + --color-pink: #ff7ad9;
19 + --font-sans: var(--font-sans), ui-sans-serif, system-ui, sans-serif;
20 + --font-mono: var(--font-mono), ui-monospace, "SF Mono", Menlo, monospace;
21 +}
22 +
23 +:root {
24 + color-scheme: dark;
25 +}
26 +
27 +html,
28 +body {
29 + background: var(--color-bg);
30 + color: var(--color-fg);
31 + font-family: var(--font-sans);
32 + -webkit-font-smoothing: antialiased;
33 +}
34 +
35 +body {
36 + background-image:
37 + radial-gradient(1200px 600px at 10% -10%, rgba(56, 225, 255, 0.07), transparent 60%),
38 + radial-gradient(900px 500px at 100% 0%, rgba(157, 123, 255, 0.07), transparent 60%);
39 + background-attachment: fixed;
40 +}
41 +
42 +.grid-bg {
43 + background-image:
44 + linear-gradient(rgba(255, 255, 255, 0.025) 1px, transparent 1px),
45 + linear-gradient(90deg, rgba(255, 255, 255, 0.025) 1px, transparent 1px);
46 + background-size: 28px 28px;
47 + mask-image: radial-gradient(ellipse at top, black 40%, transparent 85%);
48 +}
49 +
50 +.panel {
51 + background: linear-gradient(180deg, rgba(255, 255, 255, 0.025), rgba(255, 255, 255, 0.005)), var(--color-panel);
52 + border: 1px solid var(--color-line);
53 + border-radius: 14px;
54 + box-shadow: 0 1px 0 rgba(255, 255, 255, 0.03) inset, 0 10px 30px -20px rgba(0, 0, 0, 0.8);
55 +}
56 +
57 +.glow-cyan { box-shadow: 0 0 0 1px rgba(56, 225, 255, 0.25), 0 0 24px -6px rgba(56, 225, 255, 0.5); }
58 +
59 +.mono { font-family: var(--font-mono); font-feature-settings: "tnum" 1, "zero" 1; }
60 +
61 +.scrollbar-thin::-webkit-scrollbar { width: 8px; height: 8px; }
62 +.scrollbar-thin::-webkit-scrollbar-thumb { background: var(--color-line-2); border-radius: 8px; }
63 +.scrollbar-thin::-webkit-scrollbar-track { background: transparent; }
64 +
65 +@keyframes pulse-dot {
66 + 0%, 100% { transform: scale(1); opacity: 1; }
67 + 50% { transform: scale(1.6); opacity: 0.35; }
68 +}
69 +.live-dot::after {
70 + content: "";
71 + position: absolute;
72 + inset: 0;
73 + border-radius: 9999px;
74 + background: currentColor;
75 + animation: pulse-dot 1.6s ease-in-out infinite;
76 +}
77 +
78 +@keyframes shimmer { from { background-position: -200% 0; } to { background-position: 200% 0; } }
79 +.skeleton {
80 + background: linear-gradient(90deg, rgba(255,255,255,0.03) 25%, rgba(255,255,255,0.08) 50%, rgba(255,255,255,0.03) 75%);
81 + background-size: 200% 100%;
82 + animation: shimmer 1.4s linear infinite;
83 + border-radius: 6px;
84 +}
85 +
86 +a { color: inherit; }
87 +::selection { background: rgba(56, 225, 255, 0.3); }
added apps/dashboard/src/app/jobs/page.tsx +213 −0
@@ -0,0 +1,213 @@
1 +"use client";
2 +
3 +import { useEffect, useState } from "react";
4 +import Link from "next/link";
5 +import { Play, Square } from "lucide-react";
6 +import { api, useApi } from "@/lib/api";
7 +import { ago, truncate } from "@/lib/format";
8 +import { Empty, Panel, Skeleton, StatusPill, Table, Tag } from "@/components/ui";
9 +
10 +interface Job {
11 + job_id: string;
12 + session_id: string | null;
13 + platform: string;
14 + mode: string;
15 + goal: string;
16 + budget: { minutes: number; actions: number; media: number; query?: string | null; headless: boolean; account: string };
17 + status: string;
18 + created_at: string;
19 + finished_at: string | null;
20 + result: { steps?: number; entities?: number; videos?: number; ended_because?: string } | null;
21 + actions: number;
22 + entities: number;
23 +}
24 +
25 +const PRESETS = [
26 + { label: "Topic · AI Québec on YouTube", platform: "youtube", mode: "topic", query: "intelligence artificielle Québec", goal: "Discover public Quebec creators and channels discussing artificial intelligence", minutes: 10, actions: 60, media: 2 },
27 + { label: "Learn · YouTube runtime", platform: "youtube", mode: "learn", query: "", goal: "", minutes: 15, actions: 80, media: 1 },
28 + { label: "Observe · YouTube home feed", platform: "youtube", mode: "observe", query: "", goal: "Observe the recommendation feed without navigating", minutes: 5, actions: 30, media: 0 },
29 + { label: "Research · Reddit r/Quebec AI", platform: "reddit", mode: "research", query: "intelligence artificielle Québec", goal: "Discover public Reddit posts and users discussing AI in Quebec", minutes: 10, actions: 60, media: 1 },
30 +];
31 +
32 +export default function Missions() {
33 + const jobs = useApi<Job[]>("/jobs", { refreshMs: 5000 });
34 + const [form, setForm] = useState({ platform: "youtube", mode: "research", query: "intelligence artificielle Québec", goal: "Discover public Quebec creators discussing AI", minutes: 10, actions: 60, media: 1, headless: true, account: "research-01" });
35 + const [busy, setBusy] = useState(false);
36 + const [msg, setMsg] = useState<string | null>(null);
37 + const [logJob, setLogJob] = useState<string | null>(null);
38 + const input = "w-full rounded-lg bg-bg-2 border border-line-2 px-3 py-2 text-sm outline-none focus:border-cyan/60";
39 +
40 + async function launch() {
41 + setBusy(true);
42 + setMsg(null);
43 + try {
44 + const r = await api<{ job_id: string }>("/jobs", { method: "POST", body: JSON.stringify(form) });
45 + setMsg(`mission ${r.job_id} launched`);
46 + setLogJob(r.job_id);
47 + jobs.refresh();
48 + } catch (e) {
49 + setMsg((e as Error).message);
50 + } finally {
51 + setBusy(false);
52 + }
53 + }
54 + async function stop(id: string) {
55 + await api(`/jobs/${id}/stop`, { method: "POST", body: "{}" });
56 + jobs.refresh();
57 + }
58 +
59 + return (
60 + <div className="space-y-4">
61 + <div>
62 + <h1 className="text-2xl font-semibold tracking-tight">Missions</h1>
63 + <p className="text-sm text-fg-2 max-w-2xl">A mission opens one browser on the cluster node, gives the agent a goal and a budget, and lets it explore. Read-only: the crawler never likes, follows, comments or subscribes.</p>
64 + </div>
65 + <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">
66 + <Panel title="New mission" className="xl:col-span-1">
67 + <div className="space-y-3">
68 + <div className="flex flex-wrap gap-1">
69 + {PRESETS.map((p) => (
70 + <button key={p.label} onClick={() => setForm((f) => ({ ...f, ...p }))} className="text-[11px] rounded-md border border-line px-2 py-1 hover:border-cyan/50 text-fg-2 hover:text-fg">
71 + {p.label}
72 + </button>
73 + ))}
74 + </div>
75 + <div className="grid grid-cols-2 gap-2">
76 + <label className="text-xs text-dim">
77 + platform
78 + <select value={form.platform} onChange={(e) => setForm({ ...form, platform: e.target.value })} className={input}>
79 + <option value="youtube">youtube</option>
80 + <option value="reddit">reddit</option>
81 + </select>
82 + </label>
83 + <label className="text-xs text-dim">
84 + mode
85 + <select value={form.mode} onChange={(e) => setForm({ ...form, mode: e.target.value })} className={input}>
86 + {["research", "topic", "observe", "profile", "learn"].map((m) => (
87 + <option key={m}>{m}</option>
88 + ))}
89 + </select>
90 + </label>
91 + </div>
92 + <label className="text-xs text-dim block">
93 + topic / search query
94 + <input value={form.query} onChange={(e) => setForm({ ...form, query: e.target.value })} className={input} placeholder="e.g. intelligence artificielle Québec" />
95 + </label>
96 + <label className="text-xs text-dim block">
97 + goal (what the agent optimises for)
98 + <textarea value={form.goal} onChange={(e) => setForm({ ...form, goal: e.target.value })} rows={2} className={input} />
99 + </label>
100 + <div className="grid grid-cols-3 gap-2">
101 + <label className="text-xs text-dim">
102 + minutes
103 + <input type="number" min={1} max={120} value={form.minutes} onChange={(e) => setForm({ ...form, minutes: Number(e.target.value) })} className={`${input} mono`} />
104 + </label>
105 + <label className="text-xs text-dim">
106 + actions
107 + <input type="number" min={1} max={1000} value={form.actions} onChange={(e) => setForm({ ...form, actions: Number(e.target.value) })} className={`${input} mono`} />
108 + </label>
109 + <label className="text-xs text-dim">
110 + media level
111 + <select value={form.media} onChange={(e) => setForm({ ...form, media: Number(e.target.value) })} className={input}>
112 + <option value={0}>0 · metadata</option>
113 + <option value={1}>1 · + thumbnail</option>
114 + <option value={2}>2 · + frames</option>
115 + </select>
116 + </label>
117 + </div>
118 + <div className="grid grid-cols-2 gap-2">
119 + <label className="text-xs text-dim">
120 + account profile
121 + <input value={form.account} onChange={(e) => setForm({ ...form, account: e.target.value })} className={`${input} mono`} />
122 + </label>
123 + <label className="text-xs text-dim flex items-end gap-2 pb-2">
124 + <input type="checkbox" checked={form.headless} onChange={(e) => setForm({ ...form, headless: e.target.checked })} /> headless browser
125 + </label>
126 + </div>
127 + <button disabled={busy} onClick={launch} className="w-full rounded-lg bg-cyan/90 hover:bg-cyan disabled:opacity-50 text-bg font-semibold py-2 flex items-center justify-center gap-2">
128 + <Play className="h-4 w-4" /> Launch mission
129 + </button>
130 + {msg && <div className="text-xs text-fg-2 mono">{msg}</div>}
131 + </div>
132 + </Panel>
133 + <Panel title="Missions" className="xl:col-span-2" pad={false} right={jobs.data ? `${jobs.data.filter((j) => j.status === "running").length} running` : ""}>
134 + <div className="p-4">
135 + {jobs.loading ? (
136 + <Skeleton rows={6} />
137 + ) : jobs.data?.length ? (
138 + <Table head={["status", "platform", "goal", "mode", "budget", "steps", "entities", "created", ""]} dense>
139 + {jobs.data.map((j) => (
140 + <tr key={j.job_id}>
141 + <td>
142 + <StatusPill status={j.status} />
143 + </td>
144 + <td>
145 + <Tag>{j.platform}</Tag>
146 + </td>
147 + <td className="max-w-[360px]">
148 + {j.session_id ? (
149 + <Link href={`/sessions/${j.session_id}`} className="hover:text-cyan">
150 + {truncate(j.goal, 80)}
151 + </Link>
152 + ) : (
153 + truncate(j.goal, 80)
154 + )}
155 + {j.budget?.query && <div className="text-[10.5px] text-dim">“{j.budget.query}”</div>}
156 + </td>
157 + <td className="text-fg-2">{j.mode}</td>
158 + <td className="mono text-[11px] text-dim">
159 + {j.budget?.minutes}m · {j.budget?.actions}a · L{j.budget?.media}
160 + </td>
161 + <td className="mono">{j.actions || j.result?.steps || 0}</td>
162 + <td className="mono">{j.entities || j.result?.entities || 0}</td>
163 + <td className="text-dim whitespace-nowrap">{ago(j.created_at)}</td>
164 + <td className="whitespace-nowrap">
165 + <button onClick={() => setLogJob(j.job_id)} className="text-[11px] text-dim hover:text-cyan mr-2">
166 + log
167 + </button>
168 + {j.status === "running" && (
169 + <button onClick={() => stop(j.job_id)} className="text-[11px] text-red hover:text-red/80 inline-flex items-center gap-1">
170 + <Square className="h-3 w-3" /> stop
171 + </button>
172 + )}
173 + </td>
174 + </tr>
175 + ))}
176 + </Table>
177 + ) : (
178 + <Empty>No missions yet — launch one on the left.</Empty>
179 + )}
180 + </div>
181 + </Panel>
182 + </div>
183 + {logJob && <JobLog id={logJob} onClose={() => setLogJob(null)} />}
184 + </div>
185 + );
186 +}
187 +
188 +function JobLog({ id, onClose }: { id: string; onClose: () => void }) {
189 + const [text, setText] = useState("");
190 + useEffect(() => {
191 + let alive = true;
192 + let timer: ReturnType<typeof setTimeout> | undefined;
193 + const tick = async () => {
194 + try {
195 + const r = await fetch(`/api/jobs/${id}/log?lines=300`, { cache: "no-store" });
196 + if (alive) setText(await r.text());
197 + } catch {
198 + /* ignore */
199 + }
200 + if (alive) timer = setTimeout(tick, 3000);
201 + };
202 + tick();
203 + return () => {
204 + alive = false;
205 + if (timer) clearTimeout(timer);
206 + };
207 + }, [id]);
208 + return (
209 + <Panel title={`Worker log · ${id}`} right={<button onClick={onClose} className="hover:text-cyan">close</button>}>
210 + <pre className="mono text-[11px] leading-5 text-fg-2 whitespace-pre-wrap max-h-[420px] overflow-auto scrollbar-thin">{text || "…"}</pre>
211 + </Panel>
212 + );
213 +}
added apps/dashboard/src/app/layout.tsx +24 −0
@@ -0,0 +1,24 @@
1 +import type { Metadata } from "next";
2 +import { Inter, JetBrains_Mono } from "next/font/google";
3 +import "./globals.css";
4 +import { Shell } from "@/components/Shell";
5 +
6 +const sans = Inter({ subsets: ["latin"], variable: "--font-sans", display: "swap" });
7 +const mono = JetBrains_Mono({ subsets: ["latin"], variable: "--font-mono", display: "swap" });
8 +
9 +export const metadata: Metadata = {
10 + title: { default: "SocialCrawl — Social Runtime Mining console", template: "%s · SocialCrawl" },
11 + description: "Research console of the Social Runtime Crawler: live observation of social-media runtimes, information-gain navigation, learned connectors.",
12 + robots: { index: false, follow: false },
13 + icons: { icon: "/brand/icon.svg" },
14 +};
15 +
16 +export default function RootLayout({ children }: { children: React.ReactNode }) {
17 + return (
18 + <html lang="en" className={`${sans.variable} ${mono.variable}`}>
19 + <body className="min-h-screen">
20 + <Shell>{children}</Shell>
21 + </body>
22 + </html>
23 + );
24 +}
added apps/dashboard/src/app/media/[...path]/route.ts +15 −0
@@ -0,0 +1,15 @@
1 +import { cookies } from "next/headers";
2 +import { NextResponse, type NextRequest } from "next/server";
3 +import { COOKIE, verifyToken } from "@/lib/auth";
4 +
5 +export const dynamic = "force-dynamic";
6 +const API_URL = process.env.API_URL ?? "http://127.0.0.1:8350";
7 +
8 +/** Sampled video frames, served through the gate. */
9 +export async function GET(req: NextRequest, ctx: { params: Promise<{ path: string[] }> }) {
10 + const jar = await cookies();
11 + if (!verifyToken(jar.get(COOKIE)?.value)) return NextResponse.json({ error: "unauthorized" }, { status: 401 });
12 + const { path } = await ctx.params;
13 + const upstream = await fetch(new URL(`/media/${path.map(encodeURIComponent).join("/")}`, API_URL), { cache: "no-store" });
14 + return new Response(upstream.body, { status: upstream.status, headers: { "content-type": upstream.headers.get("content-type") ?? "image/jpeg", "cache-control": "private, max-age=3600" } });
15 +}
added apps/dashboard/src/app/page.tsx +126 −0
@@ -0,0 +1,126 @@
1 +"use client";
2 +
3 +import Link from "next/link";
4 +import { ArrowRight } from "lucide-react";
5 +import { useApi } from "@/lib/api";
6 +import { ago, fmtNum, truncate } from "@/lib/format";
7 +import { Bar, Empty, Kpi, Panel, Skeleton, StatusPill, Tag } from "@/components/ui";
8 +import { ActivityChart, SurfaceBars, TypeDonut, YieldBars } from "@/components/charts";
9 +import { LiveFeed } from "@/components/LiveFeed";
10 +
11 +interface Overview {
12 + totals: { sessions: number; entities: number; media: number; schemas: number; actions: number; observations: number; relationships: number; patterns_learned: number; feed_items: number };
13 + by_type: { platform: string; entity_type: string; n: number }[];
14 + running: { session_id: string; platform: string; started_at: string }[];
15 + recent_sessions: { session_id: string; platform: string; goal: string | null; mode: string | null; started_at: string; ended_at: string | null; health: string; actions: number; entities: number }[];
16 + series: { bucket: string; entities: number; responses: number; actions: number; schemas: number }[];
17 + surfaces: { dom: number; network: number; both: number; agreements: number; conflicts: number };
18 + actions: { action_type: string; n: number; avg_gain: number | null; success_rate: number | null; avg_new_entities: number | null }[];
19 + platforms: { platform: string; learned: boolean; adapter: boolean; confidence: number; page_types: number; entity_types: number; network_schemas: number; sessions: number; has_manifest: boolean }[];
20 +}
21 +
22 +export default function Observatory() {
23 + const { data, loading } = useApi<Overview>("/overview", { refreshMs: 8000 });
24 + const t = data?.totals;
25 + const s = data?.surfaces;
26 + const agreement = s && s.agreements + s.conflicts > 0 ? s.agreements / (s.agreements + s.conflicts) : null;
27 + const spark = (k: keyof Overview["series"][number]) => (data?.series ?? []).slice(-30).map((r) => Number(r[k]));
28 + return (
29 + <div className="space-y-6">
30 + <div className="flex flex-wrap items-end gap-4">
31 + <div>
32 + <h1 className="text-2xl font-semibold tracking-tight">Observatory</h1>
33 + <p className="text-sm text-fg-2 mt-1 max-w-2xl">The crawler does not download pages. It watches social applications run inside an authenticated browser, fuses what the network and the DOM reveal, and learns the platform as it goes.</p>
34 + </div>
35 + <div className="ml-auto flex gap-2">
36 + {data?.running.map((r) => (
37 + <Link key={r.session_id} href={`/sessions/${r.session_id}`} className="panel px-3 py-1.5 text-xs flex items-center gap-2 hover:border-amber/50">
38 + <StatusPill status="running" /> {r.platform} <span className="text-dim">{ago(r.started_at)}</span>
39 + </Link>
40 + ))}
41 + <Link href="/jobs" className="rounded-lg bg-cyan/90 hover:bg-cyan text-bg text-sm font-semibold px-3 py-1.5 flex items-center gap-1">
42 + New mission <ArrowRight className="h-4 w-4" />
43 + </Link>
44 + </div>
45 + </div>
46 +
47 + <div className="grid grid-cols-2 md:grid-cols-3 xl:grid-cols-6 gap-3">
48 + <Kpi label="entities" value={fmtNum(t?.entities)} hint={`${fmtNum(t?.relationships)} relationships`} spark={spark("entities")} />
49 + <Kpi label="media" value={fmtNum(t?.media)} hint="videos & images with metadata" accent="pink" />
50 + <Kpi label="runtime schemas" value={fmtNum(t?.schemas)} hint={`${fmtNum(t?.patterns_learned)} patterns learned`} accent="violet" spark={spark("schemas")} />
51 + <Kpi label="actions" value={fmtNum(t?.actions)} hint={`${fmtNum(t?.sessions)} sessions`} accent="amber" spark={spark("actions")} />
52 + <Kpi label="observations" value={fmtNum(t?.observations)} hint="raw, append-only" accent="fg" spark={spark("responses")} />
53 + <Kpi label="surface agreement" value={agreement === null ? "—" : `${Math.round(agreement * 100)}%`} hint={s ? `${fmtNum(s.both)} entities seen on both surfaces` : ""} accent="green" />
54 + </div>
55 +
56 + <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">
57 + <Panel title="Activity" sub="10-minute buckets, last 48 h" className="xl:col-span-2">
58 + {loading ? <Skeleton rows={6} /> : data?.series.length ? <ActivityChart data={data.series} /> : <Empty>No activity yet — launch a mission.</Empty>}
59 + </Panel>
60 + <Panel title="Entity mix" sub="canonical entities by type">
61 + {data?.by_type.length ? <TypeDonut data={data.by_type} /> : <Empty>—</Empty>}
62 + </Panel>
63 + </div>
64 +
65 + <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">
66 + <Panel title="Live observation stream" sub="every event, as the browser sees it" className="xl:col-span-2" pad={false}>
67 + <div className="p-2">
68 + <LiveFeed max={80} height={380} />
69 + </div>
70 + </Panel>
71 + <div className="space-y-4">
72 + <Panel title="Network vs DOM" sub="where entities were confirmed">
73 + {s ? <SurfaceBars dom={s.dom} network={s.network} both={s.both} /> : <Skeleton />}
74 + <div className="text-xs text-dim mt-2">
75 + field agreements <span className="mono text-fg">{fmtNum(s?.agreements)}</span> · conflicts <span className="mono text-fg">{fmtNum(s?.conflicts)}</span>
76 + </div>
77 + </Panel>
78 + <Panel title="Action yield" sub="average new entities per action type">
79 + {data?.actions.length ? <YieldBars data={data.actions} height={190} /> : <Empty>—</Empty>}
80 + </Panel>
81 + </div>
82 + </div>
83 +
84 + <div className="grid grid-cols-1 xl:grid-cols-2 gap-4">
85 + <Panel title="Recent sessions" right={<Link href="/sessions" className="hover:text-cyan">all →</Link>} pad={false}>
86 + {data?.recent_sessions.length ? (
87 + <ul className="divide-y divide-line/60">
88 + {data.recent_sessions.map((r) => (
89 + <li key={r.session_id}>
90 + <Link href={`/sessions/${r.session_id}`} className="flex items-center gap-3 px-4 py-2.5 hover:bg-white/[0.03]">
91 + <StatusPill status={r.ended_at ? "done" : r.health} />
92 + <Tag>{r.platform}</Tag>
93 + <span className="text-sm truncate">{truncate(r.goal ?? "(no goal)", 70)}</span>
94 + <span className="ml-auto mono text-xs text-dim whitespace-nowrap">
95 + {r.actions} steps · {r.entities} ent · {ago(r.started_at)}
96 + </span>
97 + </Link>
98 + </li>
99 + ))}
100 + </ul>
101 + ) : (
102 + <Empty>No sessions yet.</Empty>
103 + )}
104 + </Panel>
105 + <Panel title="Connectors" sub="learned platform knowledge" right={<Link href="/platforms" className="hover:text-cyan">details →</Link>}>
106 + <div className="space-y-3">
107 + {data?.platforms
108 + .filter((p) => p.adapter || p.learned)
109 + .map((p) => (
110 + <Link key={p.platform} href={`/platforms/${p.platform}`} className="block group">
111 + <div className="flex items-center gap-3 text-sm">
112 + <span className="w-20 capitalize group-hover:text-cyan">{p.platform}</span>
113 + <Bar value={p.confidence / 100} color={p.confidence >= 70 ? "#43e69a" : p.confidence >= 40 ? "#38e1ff" : "#ffb347"} className="flex-1" />
114 + <span className="mono text-xs w-10 text-right">{p.confidence}%</span>
115 + </div>
116 + <div className="text-[10.5px] text-dim mono ml-[92px]">
117 + {p.page_types} page types · {p.entity_types} entity types · {p.network_schemas} schemas · {p.sessions} sessions{p.has_manifest ? " · compiled" : ""}
118 + </div>
119 + </Link>
120 + ))}
121 + </div>
122 + </Panel>
123 + </div>
124 + </div>
125 + );
126 +}
added apps/dashboard/src/app/platforms/[platform]/page.tsx +198 −0
@@ -0,0 +1,198 @@
1 +"use client";
2 +
3 +import { use } from "react";
4 +import Link from "next/link";
5 +import { useApi } from "@/lib/api";
6 +import { fmtDate } from "@/lib/format";
7 +import { Bar, Empty, Metric, Panel, Skeleton, Table, Tag } from "@/components/ui";
8 +
9 +interface Detail {
10 + platform: string;
11 + summary: { confidence: number; page_types: number; entity_types: number; navigation_actions: number; network_schemas: number; media_patterns: number; sessions: number } | null;
12 + model: {
13 + updated_at: string;
14 + response_patterns: Record<string, { shape_hash: string; hostname: string; path_pattern: string; method: string; graphql_operation?: string; likely_entity_types: Record<string, number>; observed_count: number; entity_yield_total: number; confidence: number; triggered_by: Record<string, number>; first_seen: string; last_seen: string }>;
15 + action_patterns: Record<string, { action_type: string; from_page_type: string; observed_count: number; avg_new_entities: number; avg_network_responses: number; to_page_types: Record<string, number>; failures: number }>;
16 + page_types: Record<string, { page_type: string; observed_count: number; url_patterns: Record<string, number>; avg_entities: number; entity_types: Record<string, number>; avg_confidence: number }>;
17 + media_patterns: Record<string, { hostname: string; kind: string; observed_count: number }>;
18 + sessions_learned: string[];
19 + } | null;
20 + manifest: { compiled_at: string; confidence: number; network_patterns: unknown[] } | null;
21 + profiles: { alias: string; has_state: boolean }[];
22 +}
23 +
24 +export default function PlatformPage({ params }: { params: Promise<{ platform: string }> }) {
25 + const { platform } = use(params);
26 + const { data, loading } = useApi<Detail>(`/platforms/${platform}`, { refreshMs: 15_000 });
27 + if (loading) return <Skeleton rows={10} />;
28 + if (!data) return <Empty>unknown platform</Empty>;
29 + const m = data.model;
30 + const rps = m ? Object.values(m.response_patterns).filter((r) => Object.keys(r.likely_entity_types).length).sort((a, b) => b.confidence - a.confidence) : [];
31 + const aps = m ? Object.values(m.action_patterns).filter((a) => a.from_page_type !== "ANY").sort((a, b) => b.observed_count - a.observed_count) : [];
32 + const pts = m ? Object.values(m.page_types).sort((a, b) => b.observed_count - a.observed_count) : [];
33 + return (
34 + <div className="space-y-4">
35 + <div className="text-xs text-dim mono">
36 + <Link href="/platforms" className="hover:text-cyan">connectors</Link> / {platform}
37 + </div>
38 + <div className="flex flex-wrap items-end gap-4">
39 + <h1 className="text-2xl font-semibold tracking-tight capitalize">{platform}</h1>
40 + {data.summary && (
41 + <div className="ml-auto flex items-center gap-3 w-72">
42 + <span className="text-xs text-dim">confidence</span>
43 + <Bar value={data.summary.confidence / 100} className="flex-1" color={data.summary.confidence >= 70 ? "#43e69a" : "#38e1ff"} />
44 + <span className="mono">{data.summary.confidence}%</span>
45 + </div>
46 + )}
47 + </div>
48 + {!m ? (
49 + <Panel>
50 + <Empty>
51 + Nothing learned yet on {platform}. {["youtube", "reddit"].includes(platform) ? "Launch a learning mission from the Missions page." : "No adapter yet — this platform is scheduled for a later phase."}
52 + </Empty>
53 + </Panel>
54 + ) : (
55 + <>
56 + <div className="grid grid-cols-1 xl:grid-cols-3 gap-4">
57 + <Panel title="Learned model">
58 + <Metric label="page types" value={data.summary?.page_types} />
59 + <Metric label="entity types" value={data.summary?.entity_types} />
60 + <Metric label="navigation actions" value={data.summary?.navigation_actions} />
61 + <Metric label="network schemas" value={data.summary?.network_schemas} />
62 + <Metric label="media patterns" value={data.summary?.media_patterns} />
63 + <Metric label="sessions learned from" value={m.sessions_learned.length} />
64 + <Metric label="updated" value={fmtDate(m.updated_at)} />
65 + <Metric label="compiled manifest" value={data.manifest ? `${fmtDate(data.manifest.compiled_at)} · ${data.manifest.network_patterns.length} patterns` : "not yet (needs ≥ 40 %)"} />
66 + <Metric label="browser profiles" value={data.profiles.length ? data.profiles.map((p) => `${p.alias}${p.has_state ? " ✓" : ""}`).join(", ") : "none — run pnpm login"} />
67 + </Panel>
68 + <Panel title="Page types" sub="what the crawler recognises, and what each page usually yields" className="xl:col-span-2" pad={false}>
69 + <div className="p-4">
70 + <Table head={["page type", "seen", "avg entities", "avg conf", "url patterns", "entity mix"]} dense>
71 + {pts.map((p) => (
72 + <tr key={p.page_type}>
73 + <td>
74 + <Tag color="#38e1ff">{p.page_type}</Tag>
75 + </td>
76 + <td className="mono">{p.observed_count}</td>
77 + <td className="mono">{p.avg_entities.toFixed(1)}</td>
78 + <td className="mono">{p.avg_confidence.toFixed(2)}</td>
79 + <td className="mono text-[11px] text-fg-2">
80 + {Object.entries(p.url_patterns)
81 + .sort((a, b) => b[1] - a[1])
82 + .slice(0, 3)
83 + .map(([k, v]) => `${k} (${v})`)
84 + .join(" ")}
85 + </td>
86 + <td className="text-[11px]">
87 + {Object.entries(p.entity_types)
88 + .sort((a, b) => b[1] - a[1])
89 + .slice(0, 4)
90 + .map(([k, v]) => (
91 + <Tag key={k} className="mr-1">
92 + {k} {v}
93 + </Tag>
94 + ))}
95 + </td>
96 + </tr>
97 + ))}
98 + </Table>
99 + </div>
100 + </Panel>
101 + </div>
102 + <Panel title="Runtime API map" sub="action → response shape → entity types (§71) — the learned frontend API of the platform" pad={false}>
103 + <div className="p-4">
104 + {rps.length ? (
105 + <Table head={["endpoint", "entity types", "triggered by", "seen", "yield", "confidence"]} dense>
106 + {rps.map((r) => (
107 + <tr key={r.shape_hash}>
108 + <td className="mono text-[11.5px]">
109 + <Tag color="#9d7bff" className="mr-1">
110 + {r.method}
111 + </Tag>
112 + {r.hostname}
113 + <span className="text-fg-2">{r.path_pattern}</span>
114 + {r.graphql_operation && <Tag color="#ff7ad9" className="ml-1">{r.graphql_operation}</Tag>}
115 + <div className="text-[10px] text-dim">#{r.shape_hash.slice(0, 10)}</div>
116 + </td>
117 + <td>
118 + {Object.entries(r.likely_entity_types).map(([t, n]) => (
119 + <Tag key={t} color="#38e1ff" className="mr-1">
120 + {t} {n}
121 + </Tag>
122 + ))}
123 + </td>
124 + <td>
125 + {Object.entries(r.triggered_by).map(([a, n]) => (
126 + <Tag key={a} color="#ffb347" className="mr-1">
127 + {a.toLowerCase()} {n}
128 + </Tag>
129 + ))}
130 + </td>
131 + <td className="mono">{r.observed_count}</td>
132 + <td className="mono">{(r.entity_yield_total / Math.max(1, r.observed_count)).toFixed(1)} ent/resp</td>
133 + <td className="w-32">
134 + <Bar value={r.confidence} color={r.confidence >= 0.5 ? "#43e69a" : "#ffb347"} />
135 + <div className="mono text-[10px] text-dim">{r.confidence.toFixed(2)}</div>
136 + </td>
137 + </tr>
138 + ))}
139 + </Table>
140 + ) : (
141 + <Empty>no entity-bearing schema learned yet</Empty>
142 + )}
143 + </div>
144 + </Panel>
145 + <div className="grid grid-cols-1 xl:grid-cols-2 gap-4">
146 + <Panel title="Navigation patterns" sub="action from a page type → where it leads and what it yields (§30)" pad={false}>
147 + <div className="p-4">
148 + <Table head={["action", "from", "→ to", "seen", "avg new ent", "avg resp", "fail"]} dense>
149 + {aps.map((a) => (
150 + <tr key={a.action_type + a.from_page_type}>
151 + <td>
152 + <Tag color="#ffb347">{a.action_type.toLowerCase()}</Tag>
153 + </td>
154 + <td>
155 + <Tag color="#38e1ff">{a.from_page_type}</Tag>
156 + </td>
157 + <td className="text-[11px]">
158 + {Object.entries(a.to_page_types)
159 + .sort((x, y) => y[1] - x[1])
160 + .map(([k, v]) => `${k} ${v}`)
161 + .join(", ")}
162 + </td>
163 + <td className="mono">{a.observed_count}</td>
164 + <td className="mono">{a.avg_new_entities.toFixed(1)}</td>
165 + <td className="mono">{a.avg_network_responses.toFixed(1)}</td>
166 + <td className="mono">{a.failures}</td>
167 + </tr>
168 + ))}
169 + </Table>
170 + </div>
171 + </Panel>
172 + <Panel title="Media delivery patterns" pad={false}>
173 + <div className="p-4">
174 + {Object.values(m.media_patterns).length ? (
175 + <Table head={["host", "kind", "seen"]} dense>
176 + {Object.values(m.media_patterns)
177 + .sort((a, b) => b.observed_count - a.observed_count)
178 + .map((mp) => (
179 + <tr key={mp.hostname + mp.kind}>
180 + <td className="mono text-[11.5px]">{mp.hostname}</td>
181 + <td>
182 + <Tag color="#ff7ad9">{mp.kind}</Tag>
183 + </td>
184 + <td className="mono">{mp.observed_count}</td>
185 + </tr>
186 + ))}
187 + </Table>
188 + ) : (
189 + <Empty>none yet</Empty>
190 + )}
191 + </div>
192 + </Panel>
193 + </div>
194 + </>
195 + )}
196 + </div>
197 + );
198 +}
added apps/dashboard/src/app/platforms/page.tsx +82 −0
Large diff (82 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/app/schemas/page.tsx +97 −0
Large diff (97 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/app/sessions/[id]/page.tsx +405 −0
Large diff (405 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/app/sessions/page.tsx +69 −0
Large diff (69 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/Brand.tsx +33 −0
Large diff (33 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/EntityTable.tsx +79 −0
Large diff (79 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/LiveFeed.tsx +78 −0
Large diff (78 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/Shell.tsx +78 −0
Large diff (78 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/WorldGraph.tsx +237 −0
Large diff (237 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/charts.tsx +110 −0
Large diff (110 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/components/ui.tsx +135 −0
Large diff (135 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/lib/api.ts +94 −0
Large diff (94 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/lib/auth.ts +36 −0
Large diff (36 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/lib/format.ts +84 −0
Large diff (84 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/src/proxy.ts +24 −0
Large diff (24 lines) is not loaded by default to keep the page fast.Load diff
added apps/dashboard/tsconfig.json +41 −0
Large diff (41 lines) is not loaded by default to keep the page fast.Load diff
added apps/worker/qa/screenshots.mjs +12 −0
Large diff (12 lines) is not loaded by default to keep the page fast.Load diff
modified apps/worker/src/cli.ts +1 −1
Large diff (8 lines) is not loaded by default to keep the page fast.Load diff
modified apps/worker/src/engine.ts +1 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
modified apps/worker/src/executor.ts +0 −0

Binary file not shown.

modified connectors/youtube/manifest.json +2993 −14
Large diff (3,065 lines) is not loaded by default to keep the page fast.Load diff
added deploy/social-runtime-crawler.mld.json +102 −0
Large diff (102 lines) is not loaded by default to keep the page fast.Load diff
modified package.json +5 −2
Large diff (16 lines) is not loaded by default to keep the page fast.Load diff
modified packages/media/src/index.ts +2 −0
Large diff (9 lines) is not loaded by default to keep the page fast.Load diff
modified packages/storage/src/index.ts +0 −0

Binary file not shown.

modified pnpm-lock.yaml +1358 −12
Large diff (1,692 lines) is not loaded by default to keep the page fast.Load diff
modified tsconfig.json +1 −1
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere