TypeScript 55.4%
Python 43.2%
SQL 1.2%
1import { bigint, bigserial, boolean, date, integer, jsonb, pgTable, primaryKey, real, smallint, text, timestamp } from "drizzle-orm/pg-core";23const ts = (name: string) => timestamp(name, { withTimezone: true, mode: "date" });45export const sources = pgTable("sources", {6 id: text("id").primaryKey(),7 name: text("name").notNull(),8 domain: text("domain").notNull(),9 homepage: text("homepage"),10 description: text("description"),11 categories: text("categories").array().notNull().default([]),12 tier: text("tier").notNull().default("B"),13 importanceWeight: real("importance_weight").notNull().default(1),14 discover: jsonb("discover").$type<Record<string, unknown>>().notNull().default({}),15 fallback: jsonb("fallback").$type<Record<string, unknown>>().notNull().default({}),16 enabled: boolean("enabled").notNull().default(true),17 robotsCheckedAt: ts("robots_checked_at"),18 termsReviewedAt: ts("terms_reviewed_at"),19 allowedMethods: text("allowed_methods").array().notNull().default([]),20 rateLimitPerMin: integer("rate_limit_per_min"),21 llmEnabled: boolean("llm_enabled").notNull().default(true),22 notes: text("notes"),23 firstParty: boolean("first_party").notNull().default(true),24 country: text("country"),25 language: text("language"),26 kind: text("kind").notNull().default("registry"),27 ownerToken: text("owner_token"),28 /** seed (YAML registry) · import (admin) · factory (Source Factory) */29 origin: text("origin").notNull().default("seed"),30 sector: text("sector"),31 createdAt: ts("created_at").notNull().defaultNow(),32 updatedAt: ts("updated_at").notNull().defaultNow(),33});3435export const sensors = pgTable("sensors", {36 id: text("id").primaryKey(),37 sourceId: text("source_id").notNull(),38 name: text("name").notNull(),39 url: text("url").notNull(),40 type: text("type").notNull(),41 connector: text("connector").notNull(),42 tier: text("tier").notNull().default("B"),43 importanceWeight: real("importance_weight").notNull().default(1),44 config: jsonb("config").$type<Record<string, unknown>>().notNull().default({}),45 baseIntervalSeconds: integer("base_interval_seconds"),46 enabled: boolean("enabled").notNull().default(true),47 health: text("health").notNull().default("UP"),48 nextCheckAt: ts("next_check_at").notNull().defaultNow(),49 lastCheckAt: ts("last_check_at"),50 lastChangeAt: ts("last_change_at"),51 lastEventAt: ts("last_event_at"),52 lastStatus: integer("last_status"),53 lastError: text("last_error"),54 etag: text("etag"),55 lastModified: text("last_modified"),56 state: jsonb("state").$type<Record<string, unknown> | null>(),57 lastSnapshotId: text("last_snapshot_id"),58 consecutiveErrors: integer("consecutive_errors").notNull().default(0),59 totalRuns: integer("total_runs").notNull().default(0),60 totalNotModified: integer("total_not_modified").notNull().default(0),61 rawChanges: integer("raw_changes").notNull().default(0),62 meaningfulChanges: integer("meaningful_changes").notNull().default(0),63 avgLatencyMs: integer("avg_latency_ms"),64 status: text("status").notNull().default("ACTIVE"),65 validatedAt: ts("validated_at"),66 priority: smallint("priority").notNull().default(2),67 createdAt: ts("created_at").notNull().defaultNow(),68 updatedAt: ts("updated_at").notNull().defaultNow(),69});7071export const sensorRuns = pgTable("sensor_runs", {72 id: text("id").primaryKey(),73 sensorId: text("sensor_id").notNull(),74 startedAt: ts("started_at").notNull(),75 finishedAt: ts("finished_at"),76 httpStatus: integer("http_status"),77 outcome: text("outcome").notNull(),78 error: text("error"),79 durationMs: integer("duration_ms"),80 bytes: integer("bytes"),81 fetchMethod: text("fetch_method"),82 snapshotId: text("snapshot_id"),83});8485export const snapshots = pgTable("snapshots", {86 id: text("id").primaryKey(),87 sensorId: text("sensor_id").notNull(),88 url: text("url").notNull(),89 capturedAt: ts("captured_at").notNull(),90 httpStatus: integer("http_status"),91 contentType: text("content_type"),92 contentLength: integer("content_length"),93 contentHash: text("content_hash").notNull(),94 canonicalHash: text("canonical_hash").notNull(),95 semanticHash: text("semantic_hash"),96 etag: text("etag"),97 lastModified: text("last_modified"),98 storageKey: text("storage_key"),99 canonicalStorageKey: text("canonical_storage_key"),100 parserVersion: text("parser_version").notNull(),101 fetchDurationMs: integer("fetch_duration_ms"),102 fetchMethod: text("fetch_method"),103 mode: text("mode").notNull(),104 title: text("title"),105 publishedAt: ts("published_at"),106 extractionConfidence: real("extraction_confidence"),107 extra: jsonb("extra").$type<Record<string, unknown> | null>(),108});109110export const changes = pgTable("changes", {111 id: text("id").primaryKey(),112 sensorId: text("sensor_id").notNull(),113 oldSnapshotId: text("old_snapshot_id"),114 newSnapshotId: text("new_snapshot_id").notNull(),115 detectedAt: ts("detected_at").notNull(),116 kind: text("kind").notNull(),117 diff: jsonb("diff").$type<Record<string, unknown>>().notNull(),118 diffStorageKey: text("diff_storage_key"),119 signal: real("signal").notNull(),120 noiseRatio: real("noise_ratio").notNull(),121 magnitude: real("magnitude").notNull(),122 heuristic: jsonb("heuristic").$type<Record<string, unknown>>().notNull(),123 meaningful: boolean("meaningful").notNull().default(false),124 eventId: text("event_id"),125 changeClass: text("change_class"),126 fieldChanges: jsonb("field_changes").$type<FieldChange[] | null>(),127});128129/** A field-level difference extracted from a diff (price, number, date, version, status word…). */130export interface FieldChange {131 label: string;132 kind: "price" | "percent" | "number" | "date" | "version" | "status" | "text";133 before: string | null;134 after: string | null;135 /** relative change in % for numeric kinds */136 deltaPct?: number | null;137}138139export const eventClusters = pgTable("event_clusters", {140 id: text("id").primaryKey(),141 slug: text("slug"),142 title: text("title").notNull(),143 summary: text("summary"),144 primaryEventId: text("primary_event_id"),145 entityIds: text("entity_ids").array().notNull().default([]),146 categories: text("categories").array().notNull().default([]),147 eventCount: integer("event_count").notNull().default(0),148 maxImportance: real("max_importance").notNull().default(0),149 firstAt: ts("first_at").notNull(),150 lastAt: ts("last_at").notNull(),151 sourceCount: integer("source_count").notNull().default(1),152 firstPartyCount: integer("first_party_count").notNull().default(0),153 externalCount: integer("external_count").notNull().default(0),154 velocity: real("velocity").notNull().default(0),155 state: text("state").notNull().default("watching"),156 leadTimeMs: bigint("lead_time_ms", { mode: "number" }),157 firstPartyAt: ts("first_party_at"),158 firstExternalAt: ts("first_external_at"),159 timeline: jsonb("timeline").$type<ClusterTimelineStep[]>().notNull().default([]),160});161162export interface ClusterTimelineStep {163 at: string;164 eventId: string;165 sourceId: string;166 sourceName?: string;167 sensorType?: string;168 firstParty: boolean;169 eventType: string;170 importance: number;171}172173export const events = pgTable("events", {174 id: text("id").primaryKey(),175 slug: text("slug").notNull(),176 sensorId: text("sensor_id").notNull(),177 sourceId: text("source_id").notNull(),178 clusterId: text("cluster_id"),179 changeId: text("change_id"),180 oldSnapshotId: text("old_snapshot_id"),181 newSnapshotId: text("new_snapshot_id"),182 url: text("url").notNull(),183 eventType: text("event_type").notNull(),184 title: text("title").notNull(),185 summary: text("summary").notNull(),186 whyItMatters: text("why_it_matters"),187 importance: real("importance").notNull(),188 importanceComponents: jsonb("importance_components").$type<Record<string, number>>().notNull().default({}),189 confidence: real("confidence").notNull(),190 novelty: real("novelty").notNull(),191 categories: text("categories").array().notNull().default([]),192 keywords: text("keywords").array().notNull().default([]),193 silentChange: boolean("silent_change").notNull().default(false),194 evidenceLabel: text("evidence_label").notNull().default("OBSERVED"),195 publishedAt: ts("published_at"),196 observedFrom: ts("observed_from"),197 detectedAt: ts("detected_at").notNull(),198 processedAt: ts("processed_at").notNull(),199 publishedToFeedAt: ts("published_to_feed_at"),200 detectionLatencyMs: integer("detection_latency_ms"),201 processingLatencyMs: integer("processing_latency_ms"),202 processingVersion: text("processing_version").notNull(),203 interpretation: jsonb("interpretation").$type<Record<string, unknown>>().notNull().default({}),204 fingerprint: text("fingerprint"),205 signalScore: real("signal_score"),206 velocityScore: real("velocity_score").notNull().default(0),207 impactScore: real("impact_score").notNull().default(0),208 anomalyScore: real("anomaly_score").notNull().default(0),209 changeClass: text("change_class"),210 firstParty: boolean("first_party").notNull().default(true),211 country: text("country"),212 language: text("language"),213 canonicalUrl: text("canonical_url"),214 fieldChanges: jsonb("field_changes").$type<FieldChange[] | null>(),215 scoreReasons: jsonb("score_reasons").$type<ScoreReason[]>().notNull().default([]),216});217218export interface ScoreReason {219 /** + raises the score, - lowers it */220 sign: "+" | "-";221 text: string;222 /** contribution in points (approximate, for the transparency panel) */223 points?: number;224}225226export const entityDaily = pgTable(227 "entity_daily",228 {229 entityId: text("entity_id").notNull(),230 day: date("day", { mode: "string" }).notNull(),231 events: integer("events").notNull().default(0),232 silent: integer("silent").notNull().default(0),233 breaking: integer("breaking").notNull().default(0),234 maxImportance: real("max_importance").notNull().default(0),235 },236 (t) => [primaryKey({ columns: [t.entityId, t.day] })],237);238239export const sourceDaily = pgTable(240 "source_daily",241 {242 sourceId: text("source_id").notNull(),243 day: date("day", { mode: "string" }).notNull(),244 checks: integer("checks").notNull().default(0),245 notModified: integer("not_modified").notNull().default(0),246 errors: integer("errors").notNull().default(0),247 rawChanges: integer("raw_changes").notNull().default(0),248 events: integer("events").notNull().default(0),249 },250 (t) => [primaryKey({ columns: [t.sourceId, t.day] })],251);252253export const bookmarks = pgTable(254 "bookmarks",255 {256 ownerToken: text("owner_token").notNull(),257 eventId: text("event_id").notNull(),258 note: text("note"),259 createdAt: ts("created_at").notNull().defaultNow(),260 },261 (t) => [primaryKey({ columns: [t.ownerToken, t.eventId] })],262);263264export const savedViews = pgTable("saved_views", {265 id: text("id").primaryKey(),266 ownerToken: text("owner_token").notNull(),267 name: text("name").notNull(),268 query: text("query").notNull(),269 createdAt: ts("created_at").notNull().defaultNow(),270});271272export const interpretations = pgTable("interpretations", {273 id: bigserial("id", { mode: "number" }).primaryKey(),274 eventId: text("event_id").notNull(),275 version: integer("version").notNull(),276 model: text("model").notNull(),277 payload: jsonb("payload").$type<Record<string, unknown>>().notNull(),278 createdAt: ts("created_at").notNull().defaultNow(),279});280281export const entities = pgTable("entities", {282 id: text("id").primaryKey(),283 name: text("name").notNull(),284 type: text("type").notNull(),285 description: text("description"),286 domain: text("domain"),287 homepage: text("homepage"),288 importance: real("importance").notNull().default(50),289 categories: text("categories").array().notNull().default([]),290 parentId: text("parent_id"),291 metadata: jsonb("metadata").$type<Record<string, unknown>>().notNull().default({}),292 eventCount: integer("event_count").notNull().default(0),293 lastEventAt: ts("last_event_at"),294 createdAt: ts("created_at").notNull().defaultNow(),295});296297export const entityAliases = pgTable("entity_aliases", {298 alias: text("alias").primaryKey(),299 entityId: text("entity_id").notNull(),300});301302export const eventEntities = pgTable(303 "event_entities",304 {305 eventId: text("event_id").notNull(),306 entityId: text("entity_id").notNull(),307 role: text("role").notNull().default("subject"),308 },309 (t) => [primaryKey({ columns: [t.eventId, t.entityId] })],310);311312export const sourceEntities = pgTable(313 "source_entities",314 {315 sourceId: text("source_id").notNull(),316 entityId: text("entity_id").notNull(),317 },318 (t) => [primaryKey({ columns: [t.sourceId, t.entityId] })],319);320321export const entityRelations = pgTable(322 "entity_relations",323 {324 fromId: text("from_id").notNull(),325 relation: text("relation").notNull(),326 toId: text("to_id").notNull(),327 metadata: jsonb("metadata").$type<Record<string, unknown>>().notNull().default({}),328 },329 (t) => [primaryKey({ columns: [t.fromId, t.relation, t.toId] })],330);331332export const urls = pgTable("urls", {333 url: text("url").primaryKey(),334 domain: text("domain").notNull(),335 sourceId: text("source_id"),336 sensorId: text("sensor_id"),337 firstSeenAt: ts("first_seen_at").notNull().defaultNow(),338 lastSeenAt: ts("last_seen_at").notNull().defaultNow(),339 status: text("status").notNull().default("active"),340 missingCount: integer("missing_count").notNull().default(0),341 snapshotCount: integer("snapshot_count").notNull().default(0),342 changeCount: integer("change_count").notNull().default(0),343});344345export const urlHistory = pgTable("url_history", {346 id: bigserial("id", { mode: "number" }).primaryKey(),347 url: text("url").notNull(),348 at: ts("at").notNull().defaultNow(),349 kind: text("kind").notNull(),350 snapshotId: text("snapshot_id"),351 changeId: text("change_id"),352 eventId: text("event_id"),353 note: text("note"),354});355356export const watchlists = pgTable("watchlists", {357 id: text("id").primaryKey(),358 ownerToken: text("owner_token").notNull(),359 name: text("name").notNull(),360 createdAt: ts("created_at").notNull().defaultNow(),361});362363export const watchlistItems = pgTable(364 "watchlist_items",365 {366 watchlistId: text("watchlist_id").notNull(),367 kind: text("kind").notNull(),368 value: text("value").notNull(),369 addedAt: ts("added_at").notNull().defaultNow(),370 },371 (t) => [primaryKey({ columns: [t.watchlistId, t.kind, t.value] })],372);373374export const alerts = pgTable("alerts", {375 id: text("id").primaryKey(),376 ownerToken: text("owner_token").notNull(),377 name: text("name").notNull(),378 rule: jsonb("rule").$type<Record<string, unknown>>().notNull(),379 channel: text("channel").notNull().default("web"),380 channelConfig: jsonb("channel_config").$type<Record<string, unknown>>().notNull().default({}),381 enabled: boolean("enabled").notNull().default(true),382 createdAt: ts("created_at").notNull().defaultNow(),383 lastFiredAt: ts("last_fired_at"),384 firedCount: integer("fired_count").notNull().default(0),385});386387export const notifications = pgTable("notifications", {388 id: bigserial("id", { mode: "number" }).primaryKey(),389 alertId: text("alert_id").notNull(),390 eventId: text("event_id").notNull(),391 createdAt: ts("created_at").notNull().defaultNow(),392 readAt: ts("read_at"),393 channel: text("channel").notNull().default("web"),394 status: text("status").notNull().default("queued"),395 deliveredAt: ts("delivered_at"),396 error: text("error"),397});398399export const connectorHealth = pgTable("connector_health", {400 connector: text("connector").primaryKey(),401 status: text("status").notNull().default("UP"),402 runs24h: integer("runs_24h").notNull().default(0),403 errors24h: integer("errors_24h").notNull().default(0),404 successRate: real("success_rate"),405 avgLatencyMs: integer("avg_latency_ms"),406 changes24h: integer("changes_24h").notNull().default(0),407 events24h: integer("events_24h").notNull().default(0),408 lastSuccessAt: ts("last_success_at"),409 lastErrorAt: ts("last_error_at"),410 lastError: text("last_error"),411 httpCodes: jsonb("http_codes").$type<Record<string, number>>().notNull().default({}),412 rateLimitUntil: ts("rate_limit_until"),413 updatedAt: ts("updated_at").notNull().defaultNow(),414});415416export const discoveryCandidates = pgTable("discovery_candidates", {417 id: text("id").primaryKey(),418 sourceId: text("source_id").notNull(),419 url: text("url").notNull(),420 kind: text("kind").notNull(),421 evidence: text("evidence"),422 score: jsonb("score").$type<Record<string, unknown>>().notNull().default({}),423 status: text("status").notNull().default("candidate"),424 foundAt: ts("found_at").notNull().defaultNow(),425 seedId: text("seed_id"),426 connector: text("connector"),427 type: text("type"),428 name: text("name"),429 kindClass: text("kind_class"),430 config: jsonb("config").$type<Record<string, unknown>>().notNull().default({}),431 tier: text("tier"),432 scoreValue: real("score_value"),433 reason: text("reason"),434 shadowSensorId: text("shadow_sensor_id"),435 decidedAt: ts("decided_at"),436 updatedAt: ts("updated_at").notNull().defaultNow(),437});438439export const factorySeeds = pgTable("factory_seeds", {440 id: text("id").primaryKey(),441 name: text("name").notNull(),442 domain: text("domain").notNull(),443 homepage: text("homepage"),444 categories: text("categories").array().notNull().default([]),445 country: text("country"),446 language: text("language"),447 tier: text("tier").notNull().default("B"),448 weight: real("weight").notNull().default(1),449 importance: smallint("importance").notNull().default(2),450 aliases: text("aliases").array().notNull().default([]),451 firstParty: boolean("first_party").notNull().default(true),452 sector: text("sector"),453 universe: text("universe"),454 hints: jsonb("hints").$type<Record<string, unknown>>().notNull().default({}),455 status: text("status").notNull().default("queued"),456 attempts: integer("attempts").notNull().default(0),457 sourceId: text("source_id"),458 candidates: integer("candidates").notNull().default(0),459 shadow: integer("shadow").notNull().default(0),460 accepted: integer("accepted").notNull().default(0),461 rejected: integer("rejected").notNull().default(0),462 lastError: text("last_error"),463 discoveredAt: ts("discovered_at"),464 createdAt: ts("created_at").notNull().defaultNow(),465 updatedAt: ts("updated_at").notNull().defaultNow(),466});467export type FactorySeed = typeof factorySeeds.$inferSelect;468export type DiscoveryCandidate = typeof discoveryCandidates.$inferSelect;469470export const metricsDaily = pgTable("metrics_daily", {471 day: date("day", { mode: "string" }).primaryKey(),472 checks: bigint("checks", { mode: "number" }).notNull().default(0),473 notModified: bigint("not_modified", { mode: "number" }).notNull().default(0),474 bytes: bigint("bytes", { mode: "number" }).notNull().default(0),475 rawChanges: bigint("raw_changes", { mode: "number" }).notNull().default(0),476 events: bigint("events", { mode: "number" }).notNull().default(0),477 silentEvents: bigint("silent_events", { mode: "number" }).notNull().default(0),478 errors: bigint("errors", { mode: "number" }).notNull().default(0),479 llmCalls: bigint("llm_calls", { mode: "number" }).notNull().default(0),480 llmInputTokens: bigint("llm_input_tokens", { mode: "number" }).notNull().default(0),481 llmOutputTokens: bigint("llm_output_tokens", { mode: "number" }).notNull().default(0),482 scrapflyCalls: bigint("scrapfly_calls", { mode: "number" }).notNull().default(0),483});484485export const llmUsage = pgTable("llm_usage", {486 id: bigserial("id", { mode: "number" }).primaryKey(),487 at: ts("at").notNull().defaultNow(),488 model: text("model").notNull(),489 purpose: text("purpose").notNull(),490 inputTokens: integer("input_tokens").notNull(),491 outputTokens: integer("output_tokens").notNull(),492 eventId: text("event_id"),493 ok: boolean("ok").notNull().default(true),494});495496export type Source = typeof sources.$inferSelect;497export type Sensor = typeof sensors.$inferSelect;498export type Snapshot = typeof snapshots.$inferSelect;499export type Change = typeof changes.$inferSelect;500export type Event = typeof events.$inferSelect;501export type EventCluster = typeof eventClusters.$inferSelect;502export type Entity = typeof entities.$inferSelect;503