spb/social-runtime-crawler

1import { clamp01, jaccard, tokenize, type ActionScore, type AgentMode, type ObservedEntity, type PageState, type SemanticAction } from "@src/shared";2import type { WorldModel } from "./WorldModel.ts";34/**5 * Information Gain Engine (§26–§27):6 *   gain = novelty × relevance × expected_entity_yield × confidence × source_quality ÷ cost7 * with penalties for already-seen / duplicate / loop / low-quality / low-relevance.8 */9export interface GainContext {10  goal: string;11  mode: AgentMode;12  world: WorldModel;13  state: PageState;14  recentActionTypes: string[]; // last N action types15  recentUrls: string[];16  stepsWithoutNewEntities: number;17  /** Learned expectations: action type → average new entities produced (from the platform model). */18  learnedYield?: Record<string, number>;19  /** Labels (type + target) of actions that failed in the last few steps — retrying them blindly is a loop in disguise. */20  recentFailures?: Set<string>;21}2223export function actionKey(a: SemanticAction): string {24  return `${a.type}:${a.target_url ?? a.query ?? a.label}`;25}2627const TYPE_YIELD: Record<string, number> = {28  OPEN_CHANNEL: 0.8,29  OPEN_PROFILE: 0.8,30  OPEN_PAGE: 0.75,31  OPEN_VIDEO: 0.65,32  OPEN_POST: 0.65,33  OPEN_COMMENTS: 0.55,34  OPEN_ENTITY: 0.6,35  SCROLL_DOWN: 0.6,36  SEARCH: 0.85,37  EXPAND: 0.4,38  PLAY_VIDEO: 0.3,39  SCROLL_UP: 0.05,40  BACK: 0.2,41  RETURN_TO_FEED: 0.3,42  FILTER: 0.4,43  WAIT_FOR_CONTENT: 0.1,44  END_SESSION: 0.0,45  FORWARD: 0.1,46  PAUSE_VIDEO: 0.0,47  COLLAPSE: 0.0,48};4950export function relevanceOf(text: string | undefined, goal: string): number {51  if (!goal.trim()) return 0.6;52  if (!text) return 0.35;53  const g = tokenize(goal);54  const t = tokenize(text);55  if (g.size === 0 || t.size === 0) return 0.4;56  let hits = 0;57  for (const tok of g) if (t.has(tok) || [...t].some((x) => x.startsWith(tok.slice(0, 5)) && tok.length > 4)) hits++;58  const coverage = hits / g.size;59  return clamp01(0.3 + 0.7 * coverage + 0.2 * jaccard(g, t));60}6162function entityOf(ctx: GainContext, a: SemanticAction): ObservedEntity | undefined {63  return a.target_ref ? ctx.state.entities.find((e) => e.ref === a.target_ref) : undefined;64}6566export function scoreActions(ctx: GainContext, actions: SemanticAction[]): ActionScore[] {67  const scores: ActionScore[] = [];68  const currentRel = relevanceOf(ctx.state.summary_text.slice(0, 1500), ctx.goal);69  for (const a of actions) {70    const penalties: string[] = [];71    const e = entityOf(ctx, a);72    let novelty = 0.5;73    let relevance = 0.5;74    let confidence = 0.7;75    let source_quality = 0.7;76    let yieldExp = ctx.learnedYield?.[a.type] !== undefined ? clamp01(ctx.learnedYield[a.type]! / 10) : TYPE_YIELD[a.type] ?? 0.3;7778    if (e) {79      novelty = ctx.world.novelty(e);80      relevance = relevanceOf(`${e.name ?? ""} ${e.text ?? ""} ${e.author ?? ""}`, ctx.goal);81      confidence = Math.max(0, ...e.provenance.map((p) => p.confidence));82      source_quality = e.provenance.some((p) => p.surface === "network") && e.provenance.some((p) => p.surface === "dom") ? 0.95 : 0.75;83      if (ctx.world.isVisited(e)) penalties.push("already_seen");84      if (/^(navigation|header|sidebar)/.test(e.context ?? "") && ctx.mode !== "learn") penalties.push("low_quality_source");85      if (!e.name && !e.text) penalties.push("low_confidence_entity");86      // Profiles/channels are hubs: raise yield when the goal is about people/orgs.87      if ((e.type === "channel" || e.type === "profile") && /personalit|people|person|creator|influenc|chaîne|channel|profil/i.test(ctx.goal)) yieldExp = Math.min(1, yieldExp + 0.15);88    } else {89      switch (a.type) {90        case "SCROLL_DOWN":91          relevance = currentRel;92          novelty = ctx.stepsWithoutNewEntities > 2 ? 0.2 : 0.6;93          if (ctx.recentActionTypes.slice(-4).every((t) => t === "SCROLL_DOWN") && ctx.recentActionTypes.length >= 4) penalties.push("navigation_loop");94          break;95        case "SEARCH":96          relevance = 0.9;97          novelty = ctx.recentActionTypes.includes("SEARCH") ? 0.15 : 0.9;98          if (ctx.state.classification.page_type === "SEARCH_RESULTS") penalties.push("already_seen");99          break;100        case "EXPAND":101          relevance = currentRel;102          novelty = 0.5;103          break;104        case "PLAY_VIDEO":105          relevance = currentRel;106          novelty = ctx.state.classification.page_type === "VIDEO_DETAIL" ? 0.6 : 0.3;107          break;108        case "BACK":109        case "RETURN_TO_FEED":110          relevance = 0.4;111          novelty = ctx.stepsWithoutNewEntities > 1 ? 0.6 : 0.2;112          if (ctx.recentActionTypes.slice(-2).includes(a.type)) penalties.push("navigation_loop");113          break;114        case "END_SESSION":115          relevance = 0.1;116          novelty = ctx.stepsWithoutNewEntities > 6 ? 0.9 : 0.05;117          break;118        default:119          break;120      }121    }122    if (ctx.mode === "observe" && a.type !== "SCROLL_DOWN" && a.type !== "WAIT_FOR_CONTENT" && a.type !== "END_SESSION") penalties.push("mode_observe_no_navigation");123    if (relevance < 0.35 && e) penalties.push("low_relevance");124    if (ctx.recentFailures?.has(actionKey(a))) penalties.push("recently_failed");125126    let gain = (novelty * relevance * yieldExp * confidence * source_quality) / Math.max(0.25, a.cost);127    for (const p of penalties) gain *= p === "already_seen" ? 0.05 : p === "navigation_loop" ? 0.15 : p === "mode_observe_no_navigation" ? 0.02 : p === "recently_failed" ? 0.1 : 0.5;128129    scores.push({ action_id: a.id, novelty, relevance, expected_entity_yield: yieldExp, confidence, source_quality, cost: a.cost, penalties, information_gain: gain });130  }131  return scores.sort((x, y) => y.information_gain - x.information_gain);132}133