spb/social-runtime-crawler

1import fs from "node:fs";2import path from "node:path";3import { clamp01, createLogger, nowIso, type EntityType, type NetworkFingerprint, type PageType, type Platform } from "@src/shared";4import type { EventBus, SocialEvent } from "@src/events";56const log = createLogger("platform-model");78/**9 * Connector Learning Engine (§29–§31, §61, §71): persists what the crawler learns about a platform10 * into data/platform_model/<platform>/*.json and turns repeated observations into confidence.11 */12export interface ResponsePattern {13  shape_hash: string;14  hostname: string;15  path_pattern: string;16  method: string;17  graphql_operation?: string;18  likely_entity_types: Partial<Record<EntityType, number>>; // type → observed count19  observed_count: number;20  entity_yield_total: number;21  first_seen: string;22  last_seen: string;23  confidence: number;24  triggered_by: Record<string, number>; // action type → count (runtime API discovery, §71)25  sample_url?: string;26  fields?: { path: string; semantic?: { kind: string; confidence: number }; example?: string }[];27}2829export interface ActionPattern {30  action_type: string;31  from_page_type: PageType | "ANY";32  observed_count: number;33  avg_new_entities: number;34  avg_network_responses: number;35  to_page_types: Record<string, number>;36  usual_shape_hashes: Record<string, number>;37  failures: number;38}3940export interface PageTypeKnowledge {41  page_type: PageType;42  observed_count: number;43  url_patterns: Record<string, number>;44  avg_entities: number;45  entity_types: Record<string, number>;46  avg_confidence: number;47}4849export interface PlatformModelFile {50  platform: Platform;51  version: number;52  updated_at: string;53  response_patterns: Record<string, ResponsePattern>;54  action_patterns: Record<string, ActionPattern>; // key: `${action_type}@${page_type}`55  page_types: Record<string, PageTypeKnowledge>;56  media_patterns: Record<string, { hostname: string; kind: string; observed_count: number }>;57  sessions_learned: string[];58}5960export interface StepOutcome {61  step: number;62  action_type: string;63  action_id: string;64  from_page_type: PageType;65  to_page_type: PageType;66  to_url: string;67  page_confidence: number;68  entities_total: number;69  new_entities: number;70  entity_types: Record<string, number>;71  responses: { fingerprint: NetworkFingerprint; entity_count: number; kind: string; url: string; fields?: ResponsePattern["fields"] }[];72  failed: boolean;73}7475export class PlatformModel {76  readonly file: string;77  private model: PlatformModelFile;78  private dirty = false;7980  constructor(readonly platform: Platform, modelDir: string) {81    const dir = path.join(modelDir, platform);82    fs.mkdirSync(dir, { recursive: true });83    this.file = path.join(dir, "platform_model.json");84    this.model = fs.existsSync(this.file)85      ? (JSON.parse(fs.readFileSync(this.file, "utf8")) as PlatformModelFile)86      : { platform, version: 1, updated_at: nowIso(), response_patterns: {}, action_patterns: {}, page_types: {}, media_patterns: {}, sessions_learned: [] };87  }8889  get data(): PlatformModelFile {90    return this.model;91  }9293  /** Learn from one executed step (action → observation, §30). */94  learnStep(o: StepOutcome, sessionId: string): { newPatterns: string[] } {95    const m = this.model;96    if (!m.sessions_learned.includes(sessionId)) m.sessions_learned.push(sessionId);97    const newPatterns: string[] = [];98    const now = nowIso();99100    // Action patterns101    for (const key of [`${o.action_type}@${o.from_page_type}`, `${o.action_type}@ANY`]) {102      const ap: ActionPattern = m.action_patterns[key] ?? { action_type: o.action_type, from_page_type: key.endsWith("@ANY") ? "ANY" : o.from_page_type, observed_count: 0, avg_new_entities: 0, avg_network_responses: 0, to_page_types: {}, usual_shape_hashes: {}, failures: 0 };103      const n = ap.observed_count;104      ap.avg_new_entities = (ap.avg_new_entities * n + o.new_entities) / (n + 1);105      ap.avg_network_responses = (ap.avg_network_responses * n + o.responses.length) / (n + 1);106      ap.observed_count = n + 1;107      ap.to_page_types[o.to_page_type] = (ap.to_page_types[o.to_page_type] ?? 0) + 1;108      if (o.failed) ap.failures++;109      for (const r of o.responses) if (r.entity_count > 0) ap.usual_shape_hashes[r.fingerprint.response_shape_hash] = (ap.usual_shape_hashes[r.fingerprint.response_shape_hash] ?? 0) + 1;110      m.action_patterns[key] = ap;111    }112113    // Response patterns (runtime API discovery)114    for (const r of o.responses) {115      const h = r.fingerprint.response_shape_hash;116      if (!h) continue;117      if (r.kind === "media_manifest" || r.kind === "media_segment" || r.kind === "image") {118        const mk = `${r.fingerprint.hostname}:${r.kind}`;119        const mp = m.media_patterns[mk] ?? { hostname: r.fingerprint.hostname, kind: r.kind, observed_count: 0 };120        mp.observed_count++;121        m.media_patterns[mk] = mp;122        continue;123      }124      let rp = m.response_patterns[h];125      if (!rp) {126        rp = { shape_hash: h, hostname: r.fingerprint.hostname, path_pattern: r.fingerprint.path_pattern, method: r.fingerprint.method, graphql_operation: r.fingerprint.graphql_operation, likely_entity_types: {}, observed_count: 0, entity_yield_total: 0, first_seen: now, last_seen: now, confidence: 0, triggered_by: {}, sample_url: r.url, fields: r.fields };127        m.response_patterns[h] = rp;128        if (r.entity_count > 0 || r.fingerprint.observed_entity_types.length) newPatterns.push(h);129      }130      rp.observed_count++;131      rp.last_seen = now;132      rp.entity_yield_total += r.entity_count;133      rp.triggered_by[o.action_type] = (rp.triggered_by[o.action_type] ?? 0) + 1;134      for (const t of r.fingerprint.observed_entity_types) rp.likely_entity_types[t] = (rp.likely_entity_types[t] ?? 0) + 1;135      if (!rp.fields && r.fields) rp.fields = r.fields;136      // confidence grows with repeated observation and consistent entity yield137      const consistency = rp.entity_yield_total > 0 ? Math.min(1, rp.entity_yield_total / rp.observed_count / 5) : 0.1;138      rp.confidence = clamp01(1 - Math.exp(-rp.observed_count / 4)) * (0.5 + 0.5 * consistency);139    }140141    // Page types142    const pk: PageTypeKnowledge = m.page_types[o.to_page_type] ?? { page_type: o.to_page_type, observed_count: 0, url_patterns: {}, avg_entities: 0, entity_types: {}, avg_confidence: 0 };143    const n = pk.observed_count;144    pk.avg_entities = (pk.avg_entities * n + o.entities_total) / (n + 1);145    pk.avg_confidence = (pk.avg_confidence * n + o.page_confidence) / (n + 1);146    pk.observed_count = n + 1;147    try {148      const u = new URL(o.to_url);149      const pat = u.pathname.split("/").map((s) => (/^[A-Za-z0-9_-]{8,}$/.test(s) || /^\d+$/.test(s) ? "*" : s)).join("/") || "/";150      pk.url_patterns[pat] = (pk.url_patterns[pat] ?? 0) + 1;151    } catch {152      /* ignore */153    }154    for (const [t, c] of Object.entries(o.entity_types)) pk.entity_types[t] = (pk.entity_types[t] ?? 0) + c;155    m.page_types[o.to_page_type] = pk;156157    m.updated_at = now;158    this.dirty = true;159    return { newPatterns };160  }161162  /** Learned average entity yield per action type — feeds the information-gain engine. */163  learnedYield(): Record<string, number> {164    const out: Record<string, number> = {};165    for (const ap of Object.values(this.model.action_patterns)) if (ap.from_page_type === "ANY" && ap.observed_count >= 2) out[ap.action_type] = ap.avg_new_entities;166    return out;167  }168169  /**170   * Degradation check (§32): on a page type we know well, observing far fewer entities than usual171   * (and than the adapter expects) signals a broken connector.172   */173  isDegraded(pageType: PageType, observed: number, adapterExpected: number): boolean {174    const pk = this.model.page_types[pageType];175    const expected = Math.max(adapterExpected, pk && pk.observed_count >= 5 ? pk.avg_entities * 0.3 : 0);176    return expected >= 3 && observed === 0;177  }178179  /** Overall connector confidence (§74 summary). */180  summary() {181    const rps = Object.values(this.model.response_patterns).filter((r) => Object.keys(r.likely_entity_types).length);182    const entityTypes = new Set<string>();183    for (const r of rps) for (const t of Object.keys(r.likely_entity_types)) entityTypes.add(t);184    for (const p of Object.values(this.model.page_types)) for (const t of Object.keys(p.entity_types)) entityTypes.add(t);185    const conf = rps.length ? rps.reduce((s, r) => s + r.confidence, 0) / rps.length : 0;186    const pageConf = Object.values(this.model.page_types).reduce((s, p) => s + p.avg_confidence, 0) / Math.max(1, Object.keys(this.model.page_types).length);187    return {188      platform: this.platform,189      page_types: Object.keys(this.model.page_types).length,190      entity_types: entityTypes.size,191      navigation_actions: new Set(Object.values(this.model.action_patterns).map((a) => a.action_type)).size,192      network_schemas: rps.length,193      media_patterns: Object.keys(this.model.media_patterns).length,194      confidence: Math.round(100 * clamp01(0.6 * conf + 0.4 * pageConf)),195      sessions: this.model.sessions_learned.length,196    };197  }198199  save(): void {200    if (!this.dirty) return;201    fs.writeFileSync(this.file, JSON.stringify(this.model, null, 2));202    // Also split into the documented files (§29) for humans / other workers.203    const dir = path.dirname(this.file);204    fs.writeFileSync(path.join(dir, "response_patterns.json"), JSON.stringify(Object.values(this.model.response_patterns).sort((a, b) => b.confidence - a.confidence), null, 2));205    fs.writeFileSync(path.join(dir, "action_patterns.json"), JSON.stringify(Object.values(this.model.action_patterns), null, 2));206    fs.writeFileSync(path.join(dir, "page_types.json"), JSON.stringify(Object.values(this.model.page_types), null, 2));207    fs.writeFileSync(path.join(dir, "media_patterns.json"), JSON.stringify(Object.values(this.model.media_patterns), null, 2));208    this.dirty = false;209    log.debug("platform model saved", { file: this.file });210  }211212  /** Emit CONNECTOR_PATTERN_LEARNED for freshly learned response shapes. */213  announce(bus: EventBus, sessionId: string, step: number, hashes: string[]): void {214    for (const h of hashes) {215      const rp = this.model.response_patterns[h];216      if (!rp) continue;217      bus.emit({218        event_type: "CONNECTOR_PATTERN_LEARNED",219        platform: this.platform,220        session_id: sessionId,221        step,222        payload: { pattern: "network_response", shape_hash: h, hostname: rp.hostname, path_pattern: rp.path_pattern, graphql_operation: rp.graphql_operation, likely_entity_types: rp.likely_entity_types, observed_count: rp.observed_count, confidence: rp.confidence },223      });224    }225  }226}227228/** Compile a learned model into a connector manifest (§31/§74) — first version: declarative YAML-ish JSON. */229export function compileConnector(model: PlatformModel, outDir: string): string {230  const s = model.summary();231  const d = model.data;232  fs.mkdirSync(outDir, { recursive: true });233  const manifest = {234    platform: model.platform,235    compiled_at: nowIso(),236    learned_from_sessions: d.sessions_learned.length,237    confidence: s.confidence,238    page_types: Object.values(d.page_types).map((p) => ({ page_type: p.page_type, url_patterns: Object.entries(p.url_patterns).sort((a, b) => b[1] - a[1]).slice(0, 5).map(([k]) => k), avg_entities: Math.round(p.avg_entities * 10) / 10 })),239    network_patterns: Object.values(d.response_patterns)240      .filter((r) => r.confidence >= 0.2 && Object.keys(r.likely_entity_types).length)241      .sort((a, b) => b.confidence - a.confidence)242      .map((r) => ({ shape_hash: r.shape_hash, status: r.confidence >= 0.5 ? "stable" : "provisional", hostname: r.hostname, path_pattern: r.path_pattern, method: r.method, graphql_operation: r.graphql_operation, likely_entity_types: r.likely_entity_types, confidence: Math.round(r.confidence * 100) / 100, triggered_by: r.triggered_by, key_fields: (r.fields ?? []).filter((f) => f.semantic && f.semantic.confidence >= 0.7).slice(0, 25) })),243    navigation: Object.values(d.action_patterns).filter((a) => a.from_page_type !== "ANY").map((a) => ({ action: a.action_type, from: a.from_page_type, to: a.to_page_types, avg_new_entities: Math.round(a.avg_new_entities * 10) / 10, observed: a.observed_count })),244    media: Object.values(d.media_patterns),245  };246  const file = path.join(outDir, "manifest.json");247  fs.writeFileSync(file, JSON.stringify(manifest, null, 2));248  return file;249}250251/** Subscribe to the bus for lightweight bookkeeping (pattern announcements are produced by the engine). */252export function attachPlatformModel(model: PlatformModel, bus: EventBus): () => void {253  return bus.on("SESSION_ENDED", (_ev: SocialEvent) => model.save());254}255