spb/social-runtime-crawler
Public
TypeScript 91.8%
HTML 3.2%
JavaScript 3%
SQL 1.4%
CSS 0.7%
1<!doctype html>2<html lang="en">3<head>4<meta charset="utf-8" />5<title>Social Runtime Crawler — research console</title>6<meta name="viewport" content="width=device-width, initial-scale=1" />7<style>8 :root { --bg:#0b0d10; --panel:#12161b; --line:#1f262e; --fg:#d7dee6; --dim:#7f8b97; --acc:#4cc2ff; --ok:#41d18d; --warn:#ffb454; --bad:#ff6b6b; }9 * { box-sizing:border-box; }10 body { margin:0; background:var(--bg); color:var(--fg); font:13px/1.45 "SF Mono", ui-monospace, Menlo, monospace; }11 header { display:flex; gap:16px; align-items:center; padding:10px 16px; border-bottom:1px solid var(--line); position:sticky; top:0; background:var(--bg); z-index:2; }12 header h1 { font-size:14px; margin:0; letter-spacing:.04em; color:var(--acc); }13 header select, header button { background:var(--panel); color:var(--fg); border:1px solid var(--line); padding:5px 8px; font:inherit; border-radius:4px; }14 main { display:grid; grid-template-columns: repeat(12, 1fr); gap:10px; padding:10px 16px; }15 section { background:var(--panel); border:1px solid var(--line); border-radius:6px; min-height:120px; display:flex; flex-direction:column; overflow:hidden; }16 section h2 { margin:0; padding:7px 10px; font-size:11px; text-transform:uppercase; letter-spacing:.08em; color:var(--dim); border-bottom:1px solid var(--line); display:flex; justify-content:space-between; }17 section .body { padding:8px 10px; overflow:auto; max-height:420px; white-space:pre-wrap; word-break:break-word; }18 .c3 { grid-column: span 3; } .c4 { grid-column: span 4; } .c6 { grid-column: span 6; } .c8 { grid-column: span 8; } .c12 { grid-column: span 12; }19 table { width:100%; border-collapse:collapse; } td, th { padding:3px 6px; border-bottom:1px solid var(--line); text-align:left; vertical-align:top; } th { color:var(--dim); font-weight:normal; }20 .tag { display:inline-block; padding:0 6px; border-radius:3px; background:#1a222b; color:var(--acc); margin-right:4px; font-size:11px; }21 .ok { color:var(--ok);} .warn { color:var(--warn);} .bad { color:var(--bad);} .dim { color:var(--dim);}22 .kpi { display:flex; gap:18px; flex-wrap:wrap; } .kpi div b { display:block; font-size:20px; color:var(--fg); } .kpi div { color:var(--dim); }23 .bar { height:6px; background:#1a222b; border-radius:3px; overflow:hidden; } .bar i { display:block; height:100%; background:var(--acc); }24 a { color:var(--acc); text-decoration:none; }25 .frames img { height:70px; margin:2px; border:1px solid var(--line); border-radius:3px; }26</style>27</head>28<body>29<header>30 <h1>SOCIAL RUNTIME CRAWLER</h1>31 <select id="session"></select>32 <button id="refresh">refresh</button>33 <label><input type="checkbox" id="auto" checked /> auto (5 s)</label>34 <span id="status" class="dim"></span>35</header>36<main>37 <section class="c12"><h2>Session <span id="goal" class="dim"></span></h2><div class="body kpi" id="kpi"></div></section>38 <section class="c6"><h2>Agent decisions <span class="dim">why did it do that?</span></h2><div class="body" id="decisions"></div></section>39 <section class="c6"><h2>Current page state <span class="dim">semantic DOM</span></h2><div class="body" id="page"></div></section>40 <section class="c6"><h2>Visible / recent entities <span id="entcount" class="dim"></span></h2><div class="body" id="entities"></div></section>41 <section class="c6"><h2>Detected network schemas <span class="dim">runtime API discovery</span></h2><div class="body" id="schemas"></div></section>42 <section class="c4"><h2>Media queue</h2><div class="body" id="media"></div></section>43 <section class="c4"><h2>Connector confidence <span class="dim">learned platform model</span></h2><div class="body" id="model"></div></section>44 <section class="c4"><h2>World model</h2><div class="body" id="world"></div></section>45 <section class="c12"><h2>Recent observations <span class="dim">raw event stream</span></h2><div class="body" id="events"></div></section>46</main>47<script>48const $ = (id) => document.getElementById(id);49const esc = (s) => String(s ?? "").replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ }[c]));50const api = (p) => fetch("/api" + p).then((r) => r.json());51let current = null;5253async function loadSessions() {54 const sessions = await api("/sessions");55 const sel = $("session");56 const prev = sel.value;57 sel.innerHTML = sessions.map((s) => `<option value="${esc(s.session_id)}">${esc(s.session_id)} · ${esc(s.platform)} · ${esc(s.mode ?? "")} · ${esc(s.health ?? "")}</option>`).join("");58 if (prev && sessions.some((s) => s.session_id === prev)) sel.value = prev;59 current = sel.value || null;60}6162async function render() {63 if (!current) return;64 $("status").textContent = "loading…";65 const [summary, actions, entities, schemas, media, events, world] = await Promise.all([66 api(`/sessions/${current}/summary`), api(`/sessions/${current}/actions`), api(`/sessions/${current}/entities`),67 api(`/sessions/${current}/schemas`), api(`/sessions/${current}/media`), api(`/sessions/${current}/events?limit=120`), api(`/sessions/${current}/world`),68 ]);69 const job = summary.job ?? {}; const sum = summary.summary;70 $("goal").textContent = `${job.platform ?? ""} · ${job.mode ?? ""} · ${job.goal ?? ""}`;71 const pages = events.filter((e) => e.event_type === "PAGE_OPENED");72 const last = pages[0];73 $("kpi").innerHTML = [74 ["steps", actions.length], ["entities", entities.length], ["schemas", schemas.length], ["media", media.length],75 ["events", events.length + (events.length >= 120 ? "+" : "")], ["status", sum ? `ended: ${sum.ended_because}` : "running"],76 ["planner", actions[actions.length - 1]?.planner ?? "—"],77 ].map(([k, v]) => `<div><b>${esc(v)}</b>${k}</div>`).join("");7879 $("decisions").innerHTML = `<table><tr><th>#</th><th>action</th><th>gain</th><th>nov</th><th>rel</th><th>planner</th><th>reason</th></tr>` +80 actions.slice().reverse().slice(0, 40).map((a) => {81 const act = a.action ?? { type: a.action_type, label: a.label };82 const gain = a.expected_information_gain ?? a.expected_gain ?? 0;83 return `<tr><td>${a.step}</td><td><span class="tag">${esc(act.type)}</span>${esc((act.label || "").slice(0, 70))}</td><td>${Number(gain).toFixed(3)}</td><td>${Number(a.novelty ?? 0).toFixed(2)}</td><td>${Number(a.relevance ?? 0).toFixed(2)}</td><td>${esc(a.planner)}</td><td class="dim">${esc(a.reason)}</td></tr>`;84 }).join("") + `</table>`;8586 if (last) {87 const p = last.payload;88 $("page").innerHTML = `<div><span class="tag">${esc(p.page_type)}</span> conf ${Number(p.confidence).toFixed(2)} · <a href="${esc(p.url)}" target="_blank">${esc((p.url || "").slice(0, 90))}</a></div>89 <div class="dim">entities ${p.entities} · dom ${p.dom_entities} · network ${p.network_entities} · both ${p.both_surfaces} · field agreements ${p.field_agreements} · conflicts ${(p.field_conflicts || []).length} · videos ${p.videos}</div>90 <div class="dim">signals: ${esc((p.signals || []).join(", "))}</div><hr style="border-color:var(--line)"/>${esc(p.summary)}`;91 } else $("page").textContent = "no page yet";9293 $("entcount").textContent = `${entities.length}`;94 $("entities").innerHTML = `<table><tr><th>type</th><th>name</th><th>author</th><th>metrics</th><th>sources</th><th>conf</th></tr>` +95 entities.slice(0, 80).map((e) => {96 const prov = e.provenance || (e.fields ? Object.values(e.fields).flatMap((f) => f.provenance || []) : []);97 const surfaces = [...new Set(prov.map((p) => p.surface))];98 const conf = e.confidence ?? Math.max(0, ...prov.map((p) => p.confidence));99 return `<tr><td><span class="tag">${esc(e.type ?? e.entity_type)}</span></td><td>${e.url ? `<a href="${esc(e.url)}" target="_blank">` : ""}${esc((e.name || e.text || e.text_excerpt || e.platform_id || "").slice(0, 80))}${e.url ? "</a>" : ""}</td><td class="dim">${esc((e.author || "").slice(0, 30))}</td><td class="dim">${esc(JSON.stringify(e.metrics || {}))}</td><td>${surfaces.map((s) => `<span class="tag">${esc(s)}</span>`).join("")}</td><td>${Number(conf).toFixed(2)}</td></tr>`;100 }).join("") + `</table>`;101102 $("schemas").innerHTML = `<table><tr><th>endpoint</th><th>shape</th><th>entities</th><th>seen</th></tr>` +103 schemas.slice(0, 60).map((s) => {104 const fp = s.fingerprint || {}; const sc = s.schema || {};105 const types = (sc.candidate_entity_types || []).map((c) => `${c.type} ${Number(c.confidence).toFixed(2)}`).join(", ");106 return `<tr><td>${esc(fp.method)} ${esc(fp.hostname)}<span class="dim">${esc(fp.path_pattern)}</span>${fp.graphql_operation ? ` <span class="tag">${esc(fp.graphql_operation)}</span>` : ""}</td><td class="dim">${esc((s.shape_hash || fp.response_shape_hash || "").slice(0, 10))}</td><td>${esc(types)}</td><td>${esc(s.observed_count ?? "")}</td></tr>`;107 }).join("") + `</table>`;108109 $("media").innerHTML = media.length ? media.slice(0, 30).map((m) => `<div><span class="tag">${esc(m.media_type)}</span>${esc((m.title || m.platform_media_id || "").slice(0, 60))} <span class="dim">${m.duration_s ? Math.round(m.duration_s) + "s" : ""} ${m.width ? m.width + "×" + m.height : ""} ${m.delivery?.kind ?? ""}</span>${(m.frames || []).length ? `<div class="frames">${m.frames.map((f) => `<img src="/media/${esc(f.split("/media/")[1] || "")}" />`).join("")}</div>` : ""}</div>`).join("") : "<span class='dim'>no media yet</span>";110111 if (job.platform) {112 const pm = await api(`/platform-model/${job.platform}`);113 if (pm && pm.response_patterns) {114 const rps = Object.values(pm.response_patterns).filter((r) => Object.keys(r.likely_entity_types || {}).length).sort((a, b) => b.confidence - a.confidence);115 const conf = rps.length ? rps.reduce((s, r) => s + r.confidence, 0) / rps.length : 0;116 $("model").innerHTML = `<div class="kpi"><div><b>${Math.round(conf * 100)}%</b>schema confidence</div><div><b>${Object.keys(pm.page_types || {}).length}</b>page types</div><div><b>${rps.length}</b>entity schemas</div><div><b>${(pm.sessions_learned || []).length}</b>sessions</div></div>117 ${rps.slice(0, 12).map((r) => `<div style="margin-top:6px"><span class="dim">${esc(r.hostname)}${esc(r.path_pattern)}</span> ${Object.keys(r.likely_entity_types).map((t) => `<span class="tag">${esc(t)}</span>`).join("")}<div class="bar"><i style="width:${Math.round(r.confidence * 100)}%"></i></div></div>`).join("")}`;118 } else $("model").textContent = "no learned model yet";119 }120121 const nodes = world.nodes || []; const byType = {};122 for (const n of nodes) byType[n.type] = (byType[n.type] || 0) + 1;123 $("world").innerHTML = `<div class="kpi"><div><b>${nodes.length}</b>nodes</div><div><b>${(world.edges || []).length}</b>edges</div><div><b>${nodes.filter((n) => n.visited).length}</b>visited</div></div>124 <div style="margin-top:8px">${Object.entries(byType).map(([t, n]) => `<span class="tag">${esc(t)} ${n}</span>`).join(" ")}</div>125 <div class="dim" style="margin-top:8px">${nodes.filter((n) => (n.surfaces || []).length > 1).length} nodes confirmed by ≥2 surfaces</div>`;126127 $("events").innerHTML = `<table>` + events.slice(0, 120).map((e) => {128 const p = e.payload || {}; let brief = "";129 if (e.event_type === "NETWORK_RESPONSE_OBSERVED") brief = `${p.kind} ${p.hostname}${p.path_pattern} · ${p.entity_count} entities · ${p.body_size} B`;130 else if (e.event_type === "DOM_CHANGED") brief = `+${p.added} −${p.removed} ${JSON.stringify(p.regions)}${p.modal_opened ? " modal" : ""}`;131 else if (/DISCOVERED|OBSERVED/.test(e.event_type) && p.entity) brief = `${p.entity.type}: ${(p.entity.name || p.entity.platform_id || "").slice(0, 70)}`;132 else if (e.event_type === "ACTION_PLANNED") brief = `${p.action?.type} · ${p.reason}`;133 else brief = JSON.stringify(p).slice(0, 140);134 const cls = /FAILED|DEGRADED|LOOP|AUTH|ERROR/.test(e.event_type) ? "bad" : /LEARNED|REPAIRED|SCHEMA/.test(e.event_type) ? "ok" : "";135 return `<tr><td class="dim">${esc((e.ts || e.timestamp || "").slice(11, 19))}</td><td>#${e.step ?? 0}</td><td class="${cls}">${esc(e.event_type)}</td><td class="dim">${esc(brief)}</td></tr>`;136 }).join("") + `</table>`;137 $("status").textContent = `updated ${new Date().toLocaleTimeString()}`;138}139140$("session").addEventListener("change", () => { current = $("session").value; render(); });141$("refresh").addEventListener("click", () => loadSessions().then(render));142setInterval(() => { if ($("auto").checked) loadSessions().then(render); }, 5000);143loadSessions().then(render);144</script>145</body>146</html>147