spb/social-runtime-crawler

1/**2 * Core vocabulary of the Social Runtime Crawler.3 * Everything here is platform-independent. Platform quirks live in adapters.4 */56export const PLATFORMS = [7  "youtube",8  "reddit",9  "facebook",10  "instagram",11  "tiktok",12  "x",13  "linkedin",14  "threads",15] as const;16export type Platform = (typeof PLATFORMS)[number];1718export function isPlatform(v: string): v is Platform {19  return (PLATFORMS as readonly string[]).includes(v);20}2122/** Observation surfaces (§9). */23export type Surface =24  | "network"25  | "dom"26  | "runtime_state"27  | "accessibility"28  | "visual"29  | "media"30  | "navigation";3132export interface Provenance {33  surface: Surface;34  confidence: number; // 0..135  observation_id?: string;36  detail?: string;37}3839/** Page classification (§58). */40export type PageType =41  | "HOME_FEED"42  | "SEARCH_RESULTS"43  | "PROFILE"44  | "PUBLIC_PAGE"45  | "POST_DETAIL"46  | "VIDEO_DETAIL"47  | "CHANNEL"48  | "GROUP"49  | "COMMENT_VIEW"50  | "LOGIN"51  | "UNKNOWN";5253export interface PageClassification {54  page_type: PageType;55  confidence: number;56  signals: string[];57}5859/** Entity ontology (§22). */60export type EntityType =61  | "person"62  | "organization"63  | "page"64  | "profile"65  | "channel"66  | "account"67  | "post"68  | "comment"69  | "video"70  | "image"71  | "topic"72  | "hashtag"73  | "url"74  | "event"75  | "location"76  | "product"77  | "organization_role"78  | "community";7980export type RelationType =81  | "AUTHORED"82  | "MENTIONED"83  | "REPLIED_TO"84  | "POSTED_BY"85  | "BELONGS_TO"86  | "LINKS_TO"87  | "FEATURES"88  | "HAS_PROFILE"89  | "REPRESENTS"90  | "DISCOVERED_FROM";9192/** A value with provenance (§9, §38). */93export interface Evidenced<T = unknown> {94  value: T;95  provenance: Provenance[];96}9798/**99 * An entity as observed on a page during one step.100 * `ref` is a short, stable-within-a-step handle exposed to the planner (E1, E2…).101 */102export interface ObservedEntity {103  ref: string;104  type: EntityType;105  platform: Platform;106  platform_id?: string; // e.g. YouTube videoId, Reddit t3_xxx107  url?: string;108  name?: string; // display name / title109  text?: string; // caption / body excerpt110  author?: string;111  author_url?: string;112  metrics?: Partial<Record<"views" | "likes" | "comments" | "shares" | "score" | "subscribers" | "followers", number>>;113  media?: { has_video: boolean; has_image: boolean; duration_s?: number; thumbnail_url?: string };114  published_text?: string;115  context?: string; // e.g. "feed item #12", "search result", "comment author"116  fields: Record<string, Evidenced>;117  provenance: Provenance[];118  /** Deterministic fingerprint used for dedup across steps (platform + id or url). */119  fingerprint: string;120}121122/** Semantic action vocabulary (§25). */123export type ActionType =124  | "OPEN_ENTITY"125  | "OPEN_POST"126  | "OPEN_PROFILE"127  | "OPEN_PAGE"128  | "OPEN_CHANNEL"129  | "OPEN_VIDEO"130  | "OPEN_COMMENTS"131  | "SCROLL_DOWN"132  | "SCROLL_UP"133  | "SEARCH"134  | "FILTER"135  | "PLAY_VIDEO"136  | "PAUSE_VIDEO"137  | "EXPAND"138  | "COLLAPSE"139  | "BACK"140  | "FORWARD"141  | "RETURN_TO_FEED"142  | "WAIT_FOR_CONTENT"143  | "END_SESSION";144145export interface SemanticAction {146  id: string; // A1, A2…147  type: ActionType;148  target_ref?: string; // entity ref for OPEN_* actions149  target_url?: string;150  query?: string; // for SEARCH151  label: string; // human/LLM readable152  cost: number; // relative exploration cost (1 = scroll)153}154155export interface ActionScore {156  action_id: string;157  novelty: number;158  relevance: number;159  expected_entity_yield: number;160  confidence: number;161  source_quality: number;162  cost: number;163  penalties: string[];164  information_gain: number;165}166167export interface AgentDecision {168  step: number;169  goal: string;170  chosen_action: SemanticAction;171  expected_information_gain: number;172  novelty: number;173  relevance: number;174  reason: string; // concise explanation only — never chain-of-thought175  planner: "heuristic" | "llm" | "fallback";176  scores: ActionScore[];177}178179export type AgentMode = "observe" | "research" | "profile" | "topic" | "learn";180181export interface CrawlBudget {182  max_minutes: number;183  max_actions: number;184  max_profiles: number;185  max_posts: number;186  max_videos: number;187  max_depth: number;188  max_llm_tokens: number;189  max_storage_mb: number;190}191192export const DEFAULT_BUDGET: CrawlBudget = {193  max_minutes: 10,194  max_actions: 60,195  max_profiles: 25,196  max_posts: 200,197  max_videos: 100,198  max_depth: 5,199  max_llm_tokens: 200_000,200  max_storage_mb: 500,201};202203export type MediaLevel = 0 | 1 | 2 | 3 | 4;204205export interface CrawlJob {206  job_id: string;207  platform: Platform;208  account_alias: string;209  mode: AgentMode;210  goal: string;211  seed_url?: string;212  query?: string;213  budget: CrawlBudget;214  media_level: MediaLevel;215  stay_on_platform: boolean;216  read_only: true; // §46 — prototype is strictly read-only217}218219/** Compact page state handed to the planner (§14). */220export interface PageState {221  url: string;222  title: string;223  platform: Platform;224  classification: PageClassification;225  entities: ObservedEntity[];226  actions: SemanticAction[];227  media: ObservedMedia[];228  summary_text: string; // the textual semantic DOM representation229  fingerprint: string; // page fingerprint (url + visible entity ids) for loop detection230  captured_at: string;231}232233export interface ObservedMedia {234  media_type: "video" | "image" | "audio";235  platform: Platform;236  platform_media_id?: string;237  url?: string;238  page_url?: string;239  title?: string;240  author?: string;241  duration_s?: number;242  width?: number;243  height?: number;244  thumbnail_url?: string;245  delivery?: { kind: "progressive" | "hls" | "dash" | "unknown"; manifest_url?: string; hostnames: string[] };246  fingerprint: string;247  provenance: Provenance[];248}249250/** Network fingerprint (§10). */251export interface NetworkFingerprint {252  hostname: string;253  path_pattern: string; // path with numeric / id-like segments replaced by *254  method: string;255  content_type: string;256  response_shape_hash: string;257  is_graphql: boolean;258  graphql_operation?: string;259  observed_entity_types: EntityType[];260}261262export interface SchemaField {263  path: string; // dotted path, arrays as []264  types: string[];265  semantic: SemanticFieldGuess[];266  examples: string[];267  frequency: number; // 0..1 across profiled objects268}269270export type SemanticFieldKind =271  | "identifier"272  | "username"273  | "display_name"274  | "title"275  | "text"276  | "url"277  | "media_url"278  | "thumbnail_url"279  | "timestamp"280  | "count"281  | "duration"282  | "cursor"283  | "boolean"284  | "unknown";285286export interface SemanticFieldGuess {287  kind: SemanticFieldKind;288  confidence: number;289}290291export interface SchemaProfile {292  shape_hash: string;293  root_type: "object" | "array" | "scalar";294  repeated_object_paths: string[]; // arrays of similar objects → candidate entity lists295  fields: SchemaField[];296  candidate_entity_types: { type: EntityType; confidence: number; path: string }[];297  object_count: number;298}299300export interface SessionInfo {301  session_id: string;302  platform: Platform;303  account_alias: string;304  profile_path: string;305  started_at: string;306  current_url?: string;307  current_entity?: string;308  navigation_depth: number;309  last_action?: string;310  health: "starting" | "healthy" | "degraded" | "auth_required" | "crashed" | "stopped";311}312