Bounded network collection (streaming/long-poll bodies), observe timeout, recently_failed penalty, .mldignore
- NetworkObserver: 8 s body timeout, 6 s step collection cap, skip streaming/presence URLs and event-streams, fewer WS frames - engine: observation bounded to 45 s; failed actions penalised for 8 steps (no blind retries) - .mldignore keeps data/ (browser profiles) out of mld stage; README notes Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
6 changed files +7,790 −12
added
.mldignore
+9 −0
@@ -0,0 +1,9 @@ | ||
| 1 | +# Excluded from `mld stage` (laptop → gateway staging). Never ship runtime data: browser profiles hold live sessions. | |
| 2 | +data/ | |
| 3 | +apps/dashboard/.next/ | |
| 4 | +apps/dashboard/next-env.d.ts | |
| 5 | +*.tsbuildinfo | |
| 6 | +.env | |
| 7 | +.env.* | |
| 8 | +!.env.example | |
| 9 | +.claude/ | |
modified
README.md
+4 −1
@@ -21,10 +21,13 @@ Implemented (Phases 0 → 8 of §75, plus media detection and platform learning) | ||
| 21 | 21 | | Storage | PostgreSQL (raw observations, entities, media, actions, schemas, feed items, relationships) + JSONL replay log | |
| 22 | 22 | | Console | **www.socialcrawl.co** — Next 16 research console (`apps/dashboard`): observatory, session console (decisions, page state, entities, runtime APIs, media, world-model graph, live events), entity evidence, runtime API explorer, connectors, mission launcher. Passcode gate. | |
| 23 | 23 | | Control plane | `apps/api` (node:http, port 8350): overview/sessions/entities/schemas/platforms queries, SSE `/api/stream`, job runner (spawns worker processes, `crawl_jobs`). Mutations need `SRC_API_TOKEN`. | |
| 24 | −| Adapters | YouTube, Reddit (Phase 1). Facebook/Instagram/TikTok/X/LinkedIn/Threads: not yet | | |
| 24 | +| Adapters | YouTube, Reddit (Phase 1), Facebook (Phase 2 — §52 prototype validated 2026-09-12 with an authenticated profile: search → group → public page → posts → profiles, GraphQL operations learned automatically). Instagram/TikTok/X/LinkedIn/Threads: not yet | | |
| 25 | 25 | |
| 26 | 26 | Read-only by construction: no like/follow/comment/share/subscribe code path exists. |
| 27 | 27 | |
| 28 | +Runtime data (`data/`: browser profiles, sessions, media) is excluded from git **and** from `mld stage` through `.mldignore` — an authenticated | |
| 29 | +profile must never leave the machine it was created on. On the laptop, `SRC_DATA_DIR` can point outside the repository. | |
| 30 | + | |
| 28 | 31 | ## Quick start |
| 29 | 32 | |
| 30 | 33 | ```bash |
modified
apps/worker/src/engine.ts
+13 −3
@@ -16,7 +16,7 @@ import { SocialBrowserSession } from "@src/browser"; | ||
| 16 | 16 | import { NetworkObserver, DomObserver, snapshotDom, classifyPage, type ClassifiedResponse } from "@src/observers"; |
| 17 | 17 | import { mergeSurfaces } from "@src/entities"; |
| 18 | 18 | import { detectVideos, sampleVideoFrames } from "@src/media"; |
| 19 | −import { WorldModel, LoopDetector, HeuristicPlanner, LlmPlanner, createLlmClient, type Planner, type GainContext } from "@src/agent"; | |
| 19 | +import { WorldModel, LoopDetector, HeuristicPlanner, LlmPlanner, createLlmClient, actionKey, type Planner, type GainContext } from "@src/agent"; | |
| 20 | 20 | import { PlatformModel, compileConnector } from "@src/platform-model"; |
| 21 | 21 | import { openStore, type PostgresStore } from "@src/storage"; |
| 22 | 22 | import { getAdapter, entitiesFromDom, buildActions, pageFingerprint, type SocialPlatformAdapter } from "@src/connectors"; |
@@ -59,6 +59,7 @@ export class CrawlEngine { | ||
| 59 | 59 | private sessionDir!: string; |
| 60 | 60 | private step = 0; |
| 61 | 61 | private stepsWithoutNew = 0; |
| 62 | + private recentFailures = new Map<string, number>(); // action key → step of last failure | |
| 62 | 63 | private counts = { entities: 0, videos: 0, profiles: 0, posts: 0, videos_seen: 0, profiles_seen: 0, posts_seen: 0, patterns: 0, schemas: 0 }; |
| 63 | 64 | private degraded = false; |
| 64 | 65 | |
@@ -133,6 +134,8 @@ export class CrawlEngine { | ||
| 133 | 134 | this.dom.beginStep(this.step); |
| 134 | 135 | const target = decision.chosen_action.target_ref ? state.entities.find((e) => e.ref === decision.chosen_action.target_ref) : undefined; |
| 135 | 136 | const result = await this.executor.execute(decision.chosen_action, { entityUrl: target?.url }); |
| 137 | + if (!result.ok) this.recentFailures.set(actionKey(decision.chosen_action), this.step); | |
| 138 | + for (const [k, s] of this.recentFailures) if (this.step - s > 8) this.recentFailures.delete(k); | |
| 136 | 139 | this.session.recordAction(decision.chosen_action.type, result.navigated && /^OPEN_/.test(decision.chosen_action.type) ? 1 : decision.chosen_action.type === "BACK" ? -1 : 0); |
| 137 | 140 | if (target) this.world.markVisited(target.fingerprint, target.url); |
| 138 | 141 | if (result.navigated) this.bus.emit({ event_type: "NAVIGATION_COMPLETED", platform: job.platform, session_id: this.session.id, step: this.step, payload: { from: result.url_before, to: result.url_after, action: decision.chosen_action.type, duration_ms: result.duration_ms } }); |
@@ -140,7 +143,14 @@ export class CrawlEngine { | ||
| 140 | 143 | const responses = await this.net.collectStep(); |
| 141 | 144 | const domDeltas = this.dom.collectStep(); |
| 142 | 145 | const prevState = state; |
| 143 | − state = await this.observe(responses, decision.chosen_action.id, target?.fingerprint); | |
| 146 | + // Observation is bounded too (§66): a page that never settles must not stall the loop. | |
| 147 | + state = await Promise.race([ | |
| 148 | + this.observe(responses, decision.chosen_action.id, target?.fingerprint), | |
| 149 | + new Promise<PageState>((_, rej) => setTimeout(() => rej(new Error("observe timeout")), 45_000)), | |
| 150 | + ]).catch((err: Error) => { | |
| 151 | + log.warn("observation failed", { err: err.message }); | |
| 152 | + return prevState; | |
| 153 | + }); | |
| 144 | 154 | const newEntities = this.world.observe(state.entities, this.step, target?.fingerprint); |
| 145 | 155 | this.stepsWithoutNew = newEntities.length ? 0 : this.stepsWithoutNew + 1; |
| 146 | 156 | this.tally(newEntities); |
@@ -231,7 +241,7 @@ export class CrawlEngine { | ||
| 231 | 241 | private async plan(state: PageState): Promise<AgentDecision> { |
| 232 | 242 | // Relevance is measured against the goal *and* the topic query (the query carries the domain words). |
| 233 | 243 | const goal = this.job.query ? `${this.job.goal} ${this.job.query}` : this.job.goal; |
| 234 | − const ctx: GainContext = { goal, mode: this.job.mode, world: this.world, state, recentActionTypes: this.loops.recentActionTypes(), recentUrls: [...this.world.visitedUrls].slice(-10), stepsWithoutNewEntities: this.stepsWithoutNew, learnedYield: this.model.learnedYield() }; | |
| 244 | + const ctx: GainContext = { goal, mode: this.job.mode, world: this.world, state, recentActionTypes: this.loops.recentActionTypes(), recentUrls: [...this.world.visitedUrls].slice(-10), stepsWithoutNewEntities: this.stepsWithoutNew, learnedYield: this.model.learnedYield(), recentFailures: new Set(this.recentFailures.keys()) }; | |
| 235 | 245 | return this.planner.plan(ctx, this.step); |
| 236 | 246 | } |
| 237 | 247 | |
added
connectors/facebook/manifest.json
+7739 −0
Large diff (7,740 lines) is not loaded by default to keep the page fast.Load diff
modified
packages/agent/src/InformationGain.ts
+8 −1
Large diff (23 lines) is not loaded by default to keep the page fast.Load diff
modified
packages/observers/src/network/NetworkObserver.ts
+17 −7
Large diff (56 lines) is not loaded by default to keep the page fast.Load diff