/** * Core vocabulary of the Social Runtime Crawler. * Everything here is platform-independent. Platform quirks live in adapters. */ export const PLATFORMS = [ "youtube", "reddit", "facebook", "instagram", "tiktok", "x", "linkedin", "threads", ] as const; export type Platform = (typeof PLATFORMS)[number]; export function isPlatform(v: string): v is Platform { return (PLATFORMS as readonly string[]).includes(v); } /** Observation surfaces (§9). */ export type Surface = | "network" | "dom" | "runtime_state" | "accessibility" | "visual" | "media" | "navigation"; export interface Provenance { surface: Surface; confidence: number; // 0..1 observation_id?: string; detail?: string; } /** Page classification (§58). */ export type PageType = | "HOME_FEED" | "SEARCH_RESULTS" | "PROFILE" | "PUBLIC_PAGE" | "POST_DETAIL" | "VIDEO_DETAIL" | "CHANNEL" | "GROUP" | "COMMENT_VIEW" | "LOGIN" | "UNKNOWN"; export interface PageClassification { page_type: PageType; confidence: number; signals: string[]; } /** Entity ontology (§22). */ export type EntityType = | "person" | "organization" | "page" | "profile" | "channel" | "account" | "post" | "comment" | "video" | "image" | "topic" | "hashtag" | "url" | "event" | "location" | "product" | "organization_role" | "community"; export type RelationType = | "AUTHORED" | "MENTIONED" | "REPLIED_TO" | "POSTED_BY" | "BELONGS_TO" | "LINKS_TO" | "FEATURES" | "HAS_PROFILE" | "REPRESENTS" | "DISCOVERED_FROM"; /** A value with provenance (§9, §38). */ export interface Evidenced { value: T; provenance: Provenance[]; } /** * An entity as observed on a page during one step. * `ref` is a short, stable-within-a-step handle exposed to the planner (E1, E2…). */ export interface ObservedEntity { ref: string; type: EntityType; platform: Platform; platform_id?: string; // e.g. YouTube videoId, Reddit t3_xxx url?: string; name?: string; // display name / title text?: string; // caption / body excerpt author?: string; author_url?: string; metrics?: Partial>; media?: { has_video: boolean; has_image: boolean; duration_s?: number; thumbnail_url?: string }; published_text?: string; context?: string; // e.g. "feed item #12", "search result", "comment author" fields: Record; provenance: Provenance[]; /** Deterministic fingerprint used for dedup across steps (platform + id or url). */ fingerprint: string; } /** Semantic action vocabulary (§25). */ export type ActionType = | "OPEN_ENTITY" | "OPEN_POST" | "OPEN_PROFILE" | "OPEN_PAGE" | "OPEN_CHANNEL" | "OPEN_VIDEO" | "OPEN_COMMENTS" | "SCROLL_DOWN" | "SCROLL_UP" | "SEARCH" | "FILTER" | "PLAY_VIDEO" | "PAUSE_VIDEO" | "EXPAND" | "COLLAPSE" | "BACK" | "FORWARD" | "RETURN_TO_FEED" | "WAIT_FOR_CONTENT" | "END_SESSION"; export interface SemanticAction { id: string; // A1, A2… type: ActionType; target_ref?: string; // entity ref for OPEN_* actions target_url?: string; query?: string; // for SEARCH label: string; // human/LLM readable cost: number; // relative exploration cost (1 = scroll) } export interface ActionScore { action_id: string; novelty: number; relevance: number; expected_entity_yield: number; confidence: number; source_quality: number; cost: number; penalties: string[]; information_gain: number; } export interface AgentDecision { step: number; goal: string; chosen_action: SemanticAction; expected_information_gain: number; novelty: number; relevance: number; reason: string; // concise explanation only — never chain-of-thought planner: "heuristic" | "llm" | "fallback"; scores: ActionScore[]; } export type AgentMode = "observe" | "research" | "profile" | "topic" | "learn"; export interface CrawlBudget { max_minutes: number; max_actions: number; max_profiles: number; max_posts: number; max_videos: number; max_depth: number; max_llm_tokens: number; max_storage_mb: number; } export const DEFAULT_BUDGET: CrawlBudget = { max_minutes: 10, max_actions: 60, max_profiles: 25, max_posts: 200, max_videos: 100, max_depth: 5, max_llm_tokens: 200_000, max_storage_mb: 500, }; export type MediaLevel = 0 | 1 | 2 | 3 | 4; export interface CrawlJob { job_id: string; platform: Platform; account_alias: string; mode: AgentMode; goal: string; seed_url?: string; query?: string; budget: CrawlBudget; media_level: MediaLevel; stay_on_platform: boolean; read_only: true; // §46 — prototype is strictly read-only } /** Compact page state handed to the planner (§14). */ export interface PageState { url: string; title: string; platform: Platform; classification: PageClassification; entities: ObservedEntity[]; actions: SemanticAction[]; media: ObservedMedia[]; summary_text: string; // the textual semantic DOM representation fingerprint: string; // page fingerprint (url + visible entity ids) for loop detection captured_at: string; } export interface ObservedMedia { media_type: "video" | "image" | "audio"; platform: Platform; platform_media_id?: string; url?: string; page_url?: string; title?: string; author?: string; duration_s?: number; width?: number; height?: number; thumbnail_url?: string; delivery?: { kind: "progressive" | "hls" | "dash" | "unknown"; manifest_url?: string; hostnames: string[] }; fingerprint: string; provenance: Provenance[]; } /** Network fingerprint (§10). */ export interface NetworkFingerprint { hostname: string; path_pattern: string; // path with numeric / id-like segments replaced by * method: string; content_type: string; response_shape_hash: string; is_graphql: boolean; graphql_operation?: string; observed_entity_types: EntityType[]; } export interface SchemaField { path: string; // dotted path, arrays as [] types: string[]; semantic: SemanticFieldGuess[]; examples: string[]; frequency: number; // 0..1 across profiled objects } export type SemanticFieldKind = | "identifier" | "username" | "display_name" | "title" | "text" | "url" | "media_url" | "thumbnail_url" | "timestamp" | "count" | "duration" | "cursor" | "boolean" | "unknown"; export interface SemanticFieldGuess { kind: SemanticFieldKind; confidence: number; } export interface SchemaProfile { shape_hash: string; root_type: "object" | "array" | "scalar"; repeated_object_paths: string[]; // arrays of similar objects → candidate entity lists fields: SchemaField[]; candidate_entity_types: { type: EntityType; confidence: number; path: string }[]; object_count: number; } export interface SessionInfo { session_id: string; platform: Platform; account_alias: string; profile_path: string; started_at: string; current_url?: string; current_entity?: string; navigation_depth: number; last_action?: string; health: "starting" | "healthy" | "degraded" | "auth_required" | "crashed" | "stopped"; }