Bounded network collection (streaming/long-poll bodies), observe timeout, recently_failed penalty, .mldignore

- NetworkObserver: 8 s body timeout, 6 s step collection cap, skip streaming/presence URLs and event-streams, fewer WS frames
- engine: observation bounded to 45 s; failed actions penalised for 8 steps (no blind retries)
- .mldignore keeps data/ (browser profiles) out of mld stage; README notes

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 1 mo ago (Sep 12, 2026) parent ad4d2d0 Browse files

6 changed files +7,790 −12

added .mldignore +9 −0

++# Excluded from `mld stage` (laptop → gateway staging). Never ship runtime data: browser profiles hold live sessions.
++data/
++apps/dashboard/.next/
++apps/dashboard/next-env.d.ts
++*.tsbuildinfo
++.env
++.env.*
++!.env.example
++.claude/

modified README.md +4 −1

				`@@ -21,10 +21,13 @@ Implemented (Phases 0 → 8 of §75, plus media detection and platform learning)`
21	\| Storage \| PostgreSQL (raw observations, entities, media, actions, schemas, feed items, relationships) + JSONL replay log \|	21	\| Storage \| PostgreSQL (raw observations, entities, media, actions, schemas, feed items, relationships) + JSONL replay log \|
22	\| Console \| www.socialcrawl.co — Next 16 research console (`apps/dashboard`): observatory, session console (decisions, page state, entities, runtime APIs, media, world-model graph, live events), entity evidence, runtime API explorer, connectors, mission launcher. Passcode gate. \|	22	\| Console \| www.socialcrawl.co — Next 16 research console (`apps/dashboard`): observatory, session console (decisions, page state, entities, runtime APIs, media, world-model graph, live events), entity evidence, runtime API explorer, connectors, mission launcher. Passcode gate. \|
23	\| Control plane \| `apps/api` (node:http, port 8350): overview/sessions/entities/schemas/platforms queries, SSE `/api/stream`, job runner (spawns worker processes, `crawl_jobs`). Mutations need `SRC_API_TOKEN`. \|	23	\| Control plane \| `apps/api` (node:http, port 8350): overview/sessions/entities/schemas/platforms queries, SSE `/api/stream`, job runner (spawns worker processes, `crawl_jobs`). Mutations need `SRC_API_TOKEN`. \|
24	−\| Adapters \| YouTube, Reddit (Phase 1). Facebook/Instagram/TikTok/X/LinkedIn/Threads: not yet \|	24	+\| Adapters \| YouTube, Reddit (Phase 1), Facebook (Phase 2 — §52 prototype validated 2026-09-12 with an authenticated profile: search → group → public page → posts → profiles, GraphQL operations learned automatically). Instagram/TikTok/X/LinkedIn/Threads: not yet \|
25		25
26	Read-only by construction: no like/follow/comment/share/subscribe code path exists.	26	Read-only by construction: no like/follow/comment/share/subscribe code path exists.
27		27
		28	+Runtime data (`data/`: browser profiles, sessions, media) is excluded from git and from `mld stage` through `.mldignore` — an authenticated
		29	+profile must never leave the machine it was created on. On the laptop, `SRC_DATA_DIR` can point outside the repository.
		30	+
28	## Quick start	31	## Quick start
29		32
30	```bash	33	```bash

modified apps/worker/src/engine.ts +13 −3

				`@@ -16,7 +16,7 @@ import { SocialBrowserSession } from "@src/browser";`
16	import { NetworkObserver, DomObserver, snapshotDom, classifyPage, type ClassifiedResponse } from "@src/observers";	16	import { NetworkObserver, DomObserver, snapshotDom, classifyPage, type ClassifiedResponse } from "@src/observers";
17	import { mergeSurfaces } from "@src/entities";	17	import { mergeSurfaces } from "@src/entities";
18	import { detectVideos, sampleVideoFrames } from "@src/media";	18	import { detectVideos, sampleVideoFrames } from "@src/media";
19	−import { WorldModel, LoopDetector, HeuristicPlanner, LlmPlanner, createLlmClient, type Planner, type GainContext } from "@src/agent";	19	+import { WorldModel, LoopDetector, HeuristicPlanner, LlmPlanner, createLlmClient, actionKey, type Planner, type GainContext } from "@src/agent";
20	import { PlatformModel, compileConnector } from "@src/platform-model";	20	import { PlatformModel, compileConnector } from "@src/platform-model";
21	import { openStore, type PostgresStore } from "@src/storage";	21	import { openStore, type PostgresStore } from "@src/storage";
22	import { getAdapter, entitiesFromDom, buildActions, pageFingerprint, type SocialPlatformAdapter } from "@src/connectors";	22	import { getAdapter, entitiesFromDom, buildActions, pageFingerprint, type SocialPlatformAdapter } from "@src/connectors";
				`@@ -59,6 +59,7 @@ export class CrawlEngine {`
59	private sessionDir!: string;	59	private sessionDir!: string;
60	private step = 0;	60	private step = 0;
61	private stepsWithoutNew = 0;	61	private stepsWithoutNew = 0;
		62	+ private recentFailures = new Map<string, number>(); // action key → step of last failure
62	private counts = { entities: 0, videos: 0, profiles: 0, posts: 0, videos_seen: 0, profiles_seen: 0, posts_seen: 0, patterns: 0, schemas: 0 };	63	private counts = { entities: 0, videos: 0, profiles: 0, posts: 0, videos_seen: 0, profiles_seen: 0, posts_seen: 0, patterns: 0, schemas: 0 };
63	private degraded = false;	64	private degraded = false;
64		65
				`@@ -133,6 +134,8 @@ export class CrawlEngine {`
133	this.dom.beginStep(this.step);	134	this.dom.beginStep(this.step);
134	const target = decision.chosen_action.target_ref ? state.entities.find((e) => e.ref === decision.chosen_action.target_ref) : undefined;	135	const target = decision.chosen_action.target_ref ? state.entities.find((e) => e.ref === decision.chosen_action.target_ref) : undefined;
135	const result = await this.executor.execute(decision.chosen_action, { entityUrl: target?.url });	136	const result = await this.executor.execute(decision.chosen_action, { entityUrl: target?.url });
		137	+ if (!result.ok) this.recentFailures.set(actionKey(decision.chosen_action), this.step);
		138	+ for (const [k, s] of this.recentFailures) if (this.step - s > 8) this.recentFailures.delete(k);
136	this.session.recordAction(decision.chosen_action.type, result.navigated && /^OPEN_/.test(decision.chosen_action.type) ? 1 : decision.chosen_action.type === "BACK" ? -1 : 0);	139	this.session.recordAction(decision.chosen_action.type, result.navigated && /^OPEN_/.test(decision.chosen_action.type) ? 1 : decision.chosen_action.type === "BACK" ? -1 : 0);
137	if (target) this.world.markVisited(target.fingerprint, target.url);	140	if (target) this.world.markVisited(target.fingerprint, target.url);
138	if (result.navigated) this.bus.emit({ event_type: "NAVIGATION_COMPLETED", platform: job.platform, session_id: this.session.id, step: this.step, payload: { from: result.url_before, to: result.url_after, action: decision.chosen_action.type, duration_ms: result.duration_ms } });	141	if (result.navigated) this.bus.emit({ event_type: "NAVIGATION_COMPLETED", platform: job.platform, session_id: this.session.id, step: this.step, payload: { from: result.url_before, to: result.url_after, action: decision.chosen_action.type, duration_ms: result.duration_ms } });
				`@@ -140,7 +143,14 @@ export class CrawlEngine {`
140	const responses = await this.net.collectStep();	143	const responses = await this.net.collectStep();
141	const domDeltas = this.dom.collectStep();	144	const domDeltas = this.dom.collectStep();
142	const prevState = state;	145	const prevState = state;
143	− state = await this.observe(responses, decision.chosen_action.id, target?.fingerprint);	146	+ // Observation is bounded too (§66): a page that never settles must not stall the loop.
		147	+ state = await Promise.race([
		148	+ this.observe(responses, decision.chosen_action.id, target?.fingerprint),
		149	+ new Promise<PageState>((_, rej) => setTimeout(() => rej(new Error("observe timeout")), 45_000)),
		150	+ ]).catch((err: Error) => {
		151	+ log.warn("observation failed", { err: err.message });
		152	+ return prevState;
		153	+ });
144	const newEntities = this.world.observe(state.entities, this.step, target?.fingerprint);	154	const newEntities = this.world.observe(state.entities, this.step, target?.fingerprint);
145	this.stepsWithoutNew = newEntities.length ? 0 : this.stepsWithoutNew + 1;	155	this.stepsWithoutNew = newEntities.length ? 0 : this.stepsWithoutNew + 1;
146	this.tally(newEntities);	156	this.tally(newEntities);
				`@@ -231,7 +241,7 @@ export class CrawlEngine {`
231	private async plan(state: PageState): Promise<AgentDecision> {	241	private async plan(state: PageState): Promise<AgentDecision> {
232	// Relevance is measured against the goal and the topic query (the query carries the domain words).	242	// Relevance is measured against the goal and the topic query (the query carries the domain words).
233	const goal = this.job.query ? `${this.job.goal} ${this.job.query}` : this.job.goal;	243	const goal = this.job.query ? `${this.job.goal} ${this.job.query}` : this.job.goal;
234	− const ctx: GainContext = { goal, mode: this.job.mode, world: this.world, state, recentActionTypes: this.loops.recentActionTypes(), recentUrls: [...this.world.visitedUrls].slice(-10), stepsWithoutNewEntities: this.stepsWithoutNew, learnedYield: this.model.learnedYield() };	244	+ const ctx: GainContext = { goal, mode: this.job.mode, world: this.world, state, recentActionTypes: this.loops.recentActionTypes(), recentUrls: [...this.world.visitedUrls].slice(-10), stepsWithoutNewEntities: this.stepsWithoutNew, learnedYield: this.model.learnedYield(), recentFailures: new Set(this.recentFailures.keys()) };
235	return this.planner.plan(ctx, this.step);	245	return this.planner.plan(ctx, this.step);
236	}	246	}
237		247

added connectors/facebook/manifest.json +7739 −0

Large diff (7,740 lines) is not loaded by default to keep the page fast.Load diff

modified packages/agent/src/InformationGain.ts +8 −1

Large diff (23 lines) is not loaded by default to keep the page fast.Load diff

modified packages/observers/src/network/NetworkObserver.ts +17 −7

Large diff (56 lines) is not loaded by default to keep the page fast.Load diff

spb/social-runtime-crawler

Bounded network collection (streaming/long-poll bodies), observe timeout, recently_failed penalty, .mldignore

Theme