import { pgTable, text, timestamp, boolean, integer, bigint, doublePrecision, jsonb, index, uniqueIndex, primaryKey, } from "drizzle-orm/pg-core"; import { relations, sql } from "drizzle-orm"; const ts = (name: string) => timestamp(name, { withTimezone: true, mode: "date" }); const now = () => sql`now()`; // --------------------------------------------------------------------------- // Auth (Better Auth compatible) // --------------------------------------------------------------------------- export const users = pgTable( "users", { id: text("id").primaryKey(), name: text("name").notNull().default(""), email: text("email").notNull(), emailVerified: boolean("email_verified").notNull().default(false), image: text("image"), role: text("role").notNull().default("user"), // user | admin banned: boolean("banned").notNull().default(false), banReason: text("ban_reason"), onboardingCompletedAt: ts("onboarding_completed_at"), createdAt: ts("created_at").notNull().default(now()), updatedAt: ts("updated_at").notNull().default(now()), }, (t) => [uniqueIndex("users_email_uq").on(t.email)], ); export const sessions = pgTable( "sessions", { id: text("id").primaryKey(), expiresAt: ts("expires_at").notNull(), token: text("token").notNull(), createdAt: ts("created_at").notNull().default(now()), updatedAt: ts("updated_at").notNull().default(now()), ipAddress: text("ip_address"), userAgent: text("user_agent"), userId: text("user_id") .notNull() .references(() => users.id, { onDelete: "cascade" }), }, (t) => [uniqueIndex("sessions_token_uq").on(t.token), index("sessions_user_idx").on(t.userId)], ); export const accounts = pgTable( "accounts", { id: text("id").primaryKey(), accountId: text("account_id").notNull(), providerId: text("provider_id").notNull(), userId: text("user_id") .notNull() .references(() => users.id, { onDelete: "cascade" }), accessToken: text("access_token"), refreshToken: text("refresh_token"), idToken: text("id_token"), accessTokenExpiresAt: ts("access_token_expires_at"), refreshTokenExpiresAt: ts("refresh_token_expires_at"), scope: text("scope"), password: text("password"), createdAt: ts("created_at").notNull().default(now()), updatedAt: ts("updated_at").notNull().default(now()), }, (t) => [index("accounts_user_idx").on(t.userId)], ); export const verifications = pgTable( "verifications", { id: text("id").primaryKey(), identifier: text("identifier").notNull(), value: text("value").notNull(), expiresAt: ts("expires_at").notNull(), createdAt: ts("created_at").notNull().default(now()), updatedAt: ts("updated_at").notNull().default(now()), }, (t) => [index("verifications_identifier_idx").on(t.identifier)], ); /** * Signup allowlist: Fetcha is invitation-only. An administrator adds an email here (optionally * sending an invitation), and only then can that address create an account. */ export const signupAllowlist = pgTable( "signup_allowlist", { email: text("email").primaryKey(), // lower-cased note: text("note"), invitedByUserId: text("invited_by_user_id").references(() => users.id, { onDelete: "set null" }), /** Last time an invitation email was sent. */ invitedAt: ts("invited_at"), /** Set when the account was created. */ usedAt: ts("used_at"), userId: text("user_id").references(() => users.id, { onDelete: "set null" }), createdAt: ts("created_at").notNull().default(now()), }, (t) => [index("signup_allowlist_user_idx").on(t.userId)], ); // --------------------------------------------------------------------------- // Organizations & projects // --------------------------------------------------------------------------- export const organizations = pgTable( "organizations", { id: text("id").primaryKey(), name: text("name").notNull(), slug: text("slug").notNull(), plan: text("plan").notNull().default("unlimited"), ownerUserId: text("owner_user_id") .notNull() .references(() => users.id, { onDelete: "restrict" }), /** Show upstream provider names in debug output (admin-enabled). */ providerVisibility: boolean("provider_visibility").notNull().default(false), softLimitUsd: doublePrecision("soft_limit_usd"), hardLimitUsd: doublePrecision("hard_limit_usd"), stripeCustomerId: text("stripe_customer_id"), suspended: boolean("suspended").notNull().default(false), createdAt: ts("created_at").notNull().default(now()), updatedAt: ts("updated_at").notNull().default(now()), }, (t) => [uniqueIndex("organizations_slug_uq").on(t.slug), index("organizations_owner_idx").on(t.ownerUserId)], ); export const organizationMembers = pgTable( "organization_members", { organizationId: text("organization_id") .notNull() .references(() => organizations.id, { onDelete: "cascade" }), userId: text("user_id") .notNull() .references(() => users.id, { onDelete: "cascade" }), role: text("role").notNull().default("owner"), // owner | admin | developer | billing | viewer createdAt: ts("created_at").notNull().default(now()), }, (t) => [primaryKey({ columns: [t.organizationId, t.userId] }), index("org_members_user_idx").on(t.userId)], ); export const projects = pgTable( "projects", { id: text("id").primaryKey(), organizationId: text("organization_id") .notNull() .references(() => organizations.id, { onDelete: "cascade" }), name: text("name").notNull(), slug: text("slug").notNull(), description: text("description"), environment: text("environment").notNull().default("production"), // production | development defaultCountry: text("default_country"), defaultNetwork: text("default_network").notNull().default("auto"), logLevel: text("log_level").notNull().default("metadata"), // none | metadata | headers | full softLimitUsd: doublePrecision("soft_limit_usd"), hardLimitUsd: doublePrecision("hard_limit_usd"), monthlyRequestLimit: integer("monthly_request_limit"), archivedAt: ts("archived_at"), createdAt: ts("created_at").notNull().default(now()), updatedAt: ts("updated_at").notNull().default(now()), }, (t) => [uniqueIndex("projects_org_slug_uq").on(t.organizationId, t.slug), index("projects_org_idx").on(t.organizationId)], ); export const apiKeys = pgTable( "api_keys", { id: text("id").primaryKey(), organizationId: text("organization_id") .notNull() .references(() => organizations.id, { onDelete: "cascade" }), projectId: text("project_id") .notNull() .references(() => projects.id, { onDelete: "cascade" }), createdByUserId: text("created_by_user_id").references(() => users.id, { onDelete: "set null" }), name: text("name").notNull(), keyHash: text("key_hash").notNull(), keyPrefix: text("key_prefix").notNull(), last4: text("last4").notNull(), mode: text("mode").notNull().default("live"), // live | test scopes: jsonb("scopes").$type().notNull().default(sql`'["fetch:execute","sessions:write","usage:read"]'::jsonb`), expiresAt: ts("expires_at"), lastUsedAt: ts("last_used_at"), revokedAt: ts("revoked_at"), rotatedFromId: text("rotated_from_id"), createdAt: ts("created_at").notNull().default(now()), }, (t) => [uniqueIndex("api_keys_hash_uq").on(t.keyHash), index("api_keys_project_idx").on(t.projectId), index("api_keys_org_idx").on(t.organizationId)], ); // --------------------------------------------------------------------------- // Requests // --------------------------------------------------------------------------- export const fetchRequests = pgTable( "fetch_requests", { id: text("id").primaryKey(), // req_xxx organizationId: text("organization_id") .notNull() .references(() => organizations.id, { onDelete: "cascade" }), projectId: text("project_id") .notNull() .references(() => projects.id, { onDelete: "cascade" }), apiKeyId: text("api_key_id").references(() => apiKeys.id, { onDelete: "set null" }), source: text("source").notNull().default("api"), // api | playground | sdk url: text("url").notNull(), finalUrl: text("final_url"), domain: text("domain").notNull(), method: text("method").notNull().default("GET"), requestedNetwork: text("requested_network").notNull().default("auto"), network: text("network"), // resolved concrete network country: text("country"), region: text("region"), city: text("city"), sessionId: text("session_id"), browser: boolean("browser").notNull().default(false), /** Execution mode of the final attempt: http | browser. */ mode: text("mode"), format: text("format").notNull().default("html"), status: text("status").notNull().default("pending"), // pending | success | failed httpStatus: integer("http_status"), errorCode: text("error_code"), errorMessage: text("error_message"), attempts: integer("attempts").notNull().default(0), latencyMs: integer("latency_ms"), bytesIn: bigint("bytes_in", { mode: "number" }).notNull().default(0), bytesOut: bigint("bytes_out", { mode: "number" }).notNull().default(0), costUsd: doublePrecision("cost_usd").notNull().default(0), // upstream cost priceUsd: doublePrecision("price_usd").notNull().default(0), // billed to customer cached: boolean("cached").notNull().default(false), requestHeaders: jsonb("request_headers").$type>(), responseHeaders: jsonb("response_headers").$type>(), timing: jsonb("timing").$type>(), clientIp: text("client_ip"), userAgent: text("user_agent"), createdAt: ts("created_at").notNull().default(now()), completedAt: ts("completed_at"), }, (t) => [ index("fetch_requests_project_created_idx").on(t.projectId, t.createdAt), index("fetch_requests_org_created_idx").on(t.organizationId, t.createdAt), index("fetch_requests_domain_idx").on(t.domain), index("fetch_requests_created_idx").on(t.createdAt), ], ); export const requestAttempts = pgTable( "request_attempts", { id: text("id").primaryKey(), // att_xxx requestId: text("request_id") .notNull() .references(() => fetchRequests.id, { onDelete: "cascade" }), attemptNo: integer("attempt_no").notNull(), provider: text("provider").notNull(), // oxylabs | decodo | soax | direct network: text("network").notNull(), mode: text("mode").notNull().default("http"), // http | browser country: text("country"), sessionKey: text("session_key"), outcome: text("outcome").notNull(), // success | blocked | timeout | error | provider_error httpStatus: integer("http_status"), errorCode: text("error_code"), errorDetail: text("error_detail"), // internal only, may contain provider text blockReason: text("block_reason"), durationMs: integer("duration_ms").notNull().default(0), bytesIn: bigint("bytes_in", { mode: "number" }).notNull().default(0), bytesOut: bigint("bytes_out", { mode: "number" }).notNull().default(0), unitPricePerGb: doublePrecision("unit_price_per_gb").notNull().default(0), costUsd: doublePrecision("cost_usd").notNull().default(0), routingScore: doublePrecision("routing_score"), timing: jsonb("timing").$type>(), createdAt: ts("created_at").notNull().default(now()), }, (t) => [index("request_attempts_request_idx").on(t.requestId), index("request_attempts_provider_created_idx").on(t.provider, t.createdAt)], ); export const proxySessions = pgTable( "proxy_sessions", { id: text("id").primaryKey(), // sess_xxx organizationId: text("organization_id") .notNull() .references(() => organizations.id, { onDelete: "cascade" }), projectId: text("project_id") .notNull() .references(() => projects.id, { onDelete: "cascade" }), label: text("label"), provider: text("provider").notNull(), network: text("network").notNull(), country: text("country"), region: text("region"), city: text("city"), /** Provider-side sticky key; internal only. */ stickyKey: text("sticky_key").notNull(), cookies: jsonb("cookies").$type>().notNull().default(sql`'[]'::jsonb`), status: text("status").notNull().default("active"), // active | expired | closed requestCount: integer("request_count").notNull().default(0), lastUsedAt: ts("last_used_at"), expiresAt: ts("expires_at").notNull(), createdAt: ts("created_at").notNull().default(now()), }, (t) => [index("proxy_sessions_project_idx").on(t.projectId), index("proxy_sessions_expires_idx").on(t.expiresAt)], ); // --------------------------------------------------------------------------- // Usage & billing // --------------------------------------------------------------------------- export const usageEvents = pgTable( "usage_events", { id: text("id").primaryKey(), organizationId: text("organization_id") .notNull() .references(() => organizations.id, { onDelete: "cascade" }), projectId: text("project_id").references(() => projects.id, { onDelete: "set null" }), requestId: text("request_id"), metric: text("metric").notNull(), // request | bandwidth | residential_bandwidth | mobile_bandwidth | browser_seconds | extraction quantity: doublePrecision("quantity").notNull(), unit: text("unit").notNull(), // count | bytes | seconds costUsd: doublePrecision("cost_usd").notNull().default(0), // customer price upstreamCostUsd: doublePrecision("upstream_cost_usd").notNull().default(0), createdAt: ts("created_at").notNull().default(now()), }, (t) => [index("usage_events_org_created_idx").on(t.organizationId, t.createdAt), index("usage_events_project_created_idx").on(t.projectId, t.createdAt)], ); export const subscriptions = pgTable("subscriptions", { id: text("id").primaryKey(), organizationId: text("organization_id") .notNull() .references(() => organizations.id, { onDelete: "cascade" }), plan: text("plan").notNull(), status: text("status").notNull().default("active"), // active | trialing | past_due | canceled stripeSubscriptionId: text("stripe_subscription_id"), currentPeriodStart: ts("current_period_start"), currentPeriodEnd: ts("current_period_end"), cancelAtPeriodEnd: boolean("cancel_at_period_end").notNull().default(false), createdAt: ts("created_at").notNull().default(now()), updatedAt: ts("updated_at").notNull().default(now()), }); export const billingEvents = pgTable( "billing_events", { id: text("id").primaryKey(), organizationId: text("organization_id").references(() => organizations.id, { onDelete: "set null" }), type: text("type").notNull(), stripeEventId: text("stripe_event_id"), payload: jsonb("payload").$type>(), createdAt: ts("created_at").notNull().default(now()), }, (t) => [index("billing_events_org_idx").on(t.organizationId)], ); // --------------------------------------------------------------------------- // Crawl jobs // --------------------------------------------------------------------------- export const crawlJobs = pgTable( "crawl_jobs", { id: text("id").primaryKey(), // crawl_xxx organizationId: text("organization_id") .notNull() .references(() => organizations.id, { onDelete: "cascade" }), projectId: text("project_id") .notNull() .references(() => projects.id, { onDelete: "cascade" }), apiKeyId: text("api_key_id").references(() => apiKeys.id, { onDelete: "set null" }), source: text("source").notNull().default("api"), // api | playground | sdk label: text("label"), seedUrl: text("seed_url").notNull(), domain: text("domain").notNull(), options: jsonb("options").$type>().notNull(), status: text("status").notNull().default("queued"), // queued | running | completed | failed | cancelled workerId: text("worker_id"), pagesDiscovered: integer("pages_discovered").notNull().default(0), pagesFetched: integer("pages_fetched").notNull().default(0), pagesOk: integer("pages_ok").notNull().default(0), pagesBlocked: integer("pages_blocked").notNull().default(0), pagesFailed: integer("pages_failed").notNull().default(0), bytes: bigint("bytes", { mode: "number" }).notNull().default(0), costUsd: doublePrecision("cost_usd").notNull().default(0), errorCode: text("error_code"), errorMessage: text("error_message"), webhookStatus: text("webhook_status"), createdAt: ts("created_at").notNull().default(now()), startedAt: ts("started_at"), heartbeatAt: ts("heartbeat_at"), completedAt: ts("completed_at"), }, (t) => [index("crawl_jobs_project_created_idx").on(t.projectId, t.createdAt), index("crawl_jobs_status_idx").on(t.status)], ); export const crawlPages = pgTable( "crawl_pages", { id: text("id").primaryKey(), // cpg_xxx jobId: text("job_id") .notNull() .references(() => crawlJobs.id, { onDelete: "cascade" }), url: text("url").notNull(), finalUrl: text("final_url"), depth: integer("depth").notNull().default(0), parentUrl: text("parent_url"), status: text("status").notNull().default("pending"), // pending | success | blocked | failed | skipped httpStatus: integer("http_status"), errorCode: text("error_code"), requestId: text("request_id"), title: text("title"), description: text("description"), contentType: text("content_type"), /** Converted content (markdown / text / html) — bounded by the job's per-page cap. */ content: text("content"), linksCount: integer("links_count").notNull().default(0), bytes: bigint("bytes", { mode: "number" }).notNull().default(0), durationMs: integer("duration_ms"), mode: text("mode"), // http | browser createdAt: ts("created_at").notNull().default(now()), fetchedAt: ts("fetched_at"), }, (t) => [index("crawl_pages_job_idx").on(t.jobId, t.createdAt), uniqueIndex("crawl_pages_job_url_uq").on(t.jobId, t.url)], ); // --------------------------------------------------------------------------- // Providers, routing intelligence // --------------------------------------------------------------------------- export const providerConfigs = pgTable("provider_configs", { id: text("id").primaryKey(), // oxylabs | decodo | soax label: text("label").notNull(), enabled: boolean("enabled").notNull().default(true), networks: jsonb("networks").$type().notNull().default(sql`'["residential"]'::jsonb`), pricePerGbUsd: jsonb("price_per_gb_usd").$type>().notNull().default(sql`'{}'::jsonb`), weight: doublePrecision("weight").notNull().default(1), maxConcurrency: integer("max_concurrency").notNull().default(200), notes: text("notes"), updatedAt: ts("updated_at").notNull().default(now()), }); export const providerHealth = pgTable( "provider_health", { id: text("id").primaryKey(), provider: text("provider").notNull(), network: text("network").notNull(), status: text("status").notNull(), // healthy | degraded | down | unconfigured latencyMs: integer("latency_ms"), successRate: doublePrecision("success_rate"), detail: text("detail"), checkedAt: ts("checked_at").notNull().default(now()), }, (t) => [index("provider_health_provider_checked_idx").on(t.provider, t.checkedAt)], ); export const domainProfiles = pgTable( "domain_profiles", { domain: text("domain").primaryKey(), preferredNetwork: text("preferred_network"), preferredProvider: text("preferred_provider"), requests: integer("requests").notNull().default(0), successes: integer("successes").notNull().default(0), blocks: integer("blocks").notNull().default(0), captchas: integer("captchas").notNull().default(0), browserRequired: integer("browser_required").notNull().default(0), avgLatencyMs: doublePrecision("avg_latency_ms").notNull().default(0), /** { "oxylabs:residential": { n, ok, blocked, latency_ms_avg, cost_usd } } */ routeStats: jsonb("route_stats").$type>().notNull().default(sql`'{}'::jsonb`), policy: jsonb("policy").$type<{ order?: string[]; force_network?: string; browser?: boolean }>(), lastSeenAt: ts("last_seen_at").notNull().default(now()), updatedAt: ts("updated_at").notNull().default(now()), }, ); export const routingMetrics = pgTable( "routing_metrics", { id: text("id").primaryKey(), bucket: ts("bucket").notNull(), // hourly bucket provider: text("provider").notNull(), network: text("network").notNull(), country: text("country"), requests: integer("requests").notNull().default(0), successes: integer("successes").notNull().default(0), blocked: integer("blocked").notNull().default(0), errors: integer("errors").notNull().default(0), latencySumMs: bigint("latency_sum_ms", { mode: "number" }).notNull().default(0), bytes: bigint("bytes", { mode: "number" }).notNull().default(0), costUsd: doublePrecision("cost_usd").notNull().default(0), }, (t) => [uniqueIndex("routing_metrics_bucket_uq").on(t.bucket, t.provider, t.network, t.country)], ); // --------------------------------------------------------------------------- // Webhooks, audit, abuse, flags, legal // --------------------------------------------------------------------------- export const webhooks = pgTable("webhooks", { id: text("id").primaryKey(), organizationId: text("organization_id") .notNull() .references(() => organizations.id, { onDelete: "cascade" }), projectId: text("project_id").references(() => projects.id, { onDelete: "cascade" }), url: text("url").notNull(), secret: text("secret").notNull(), events: jsonb("events").$type().notNull().default(sql`'[]'::jsonb`), enabled: boolean("enabled").notNull().default(true), createdAt: ts("created_at").notNull().default(now()), }); export const webhookDeliveries = pgTable( "webhook_deliveries", { id: text("id").primaryKey(), webhookId: text("webhook_id") .notNull() .references(() => webhooks.id, { onDelete: "cascade" }), event: text("event").notNull(), payload: jsonb("payload").$type>().notNull(), status: text("status").notNull().default("pending"), responseStatus: integer("response_status"), attempts: integer("attempts").notNull().default(0), nextAttemptAt: ts("next_attempt_at"), createdAt: ts("created_at").notNull().default(now()), }, (t) => [index("webhook_deliveries_webhook_idx").on(t.webhookId)], ); export const auditLogs = pgTable( "audit_logs", { id: text("id").primaryKey(), organizationId: text("organization_id").references(() => organizations.id, { onDelete: "cascade" }), userId: text("user_id").references(() => users.id, { onDelete: "set null" }), action: text("action").notNull(), // login | password.changed | api_key.created | ... target: text("target"), metadata: jsonb("metadata").$type>(), ipAddress: text("ip_address"), userAgent: text("user_agent"), createdAt: ts("created_at").notNull().default(now()), }, (t) => [index("audit_logs_org_created_idx").on(t.organizationId, t.createdAt), index("audit_logs_user_idx").on(t.userId)], ); export const abuseEvents = pgTable( "abuse_events", { id: text("id").primaryKey(), organizationId: text("organization_id").references(() => organizations.id, { onDelete: "cascade" }), projectId: text("project_id"), requestId: text("request_id"), kind: text("kind").notNull(), // ssrf_attempt | prohibited_target | rate_abuse | manual severity: text("severity").notNull().default("low"), detail: text("detail"), resolved: boolean("resolved").notNull().default(false), createdAt: ts("created_at").notNull().default(now()), }, (t) => [index("abuse_events_org_idx").on(t.organizationId)], ); export const featureFlags = pgTable("feature_flags", { key: text("key").primaryKey(), enabled: boolean("enabled").notNull().default(false), description: text("description"), plans: jsonb("plans").$type(), organizationIds: jsonb("organization_ids").$type(), updatedAt: ts("updated_at").notNull().default(now()), }); export const legalAcceptances = pgTable( "legal_acceptances", { id: text("id").primaryKey(), userId: text("user_id") .notNull() .references(() => users.id, { onDelete: "cascade" }), document: text("document").notNull(), // terms | privacy | acceptable_use version: text("version").notNull(), ipAddress: text("ip_address"), acceptedAt: ts("accepted_at").notNull().default(now()), }, (t) => [index("legal_acceptances_user_idx").on(t.userId)], ); export const statusIncidents = pgTable("status_incidents", { id: text("id").primaryKey(), component: text("component").notNull(), title: text("title").notNull(), body: text("body"), severity: text("severity").notNull().default("minor"), startedAt: ts("started_at").notNull().default(now()), resolvedAt: ts("resolved_at"), }); // --------------------------------------------------------------------------- // Relations // --------------------------------------------------------------------------- export const usersRelations = relations(users, ({ many }) => ({ memberships: many(organizationMembers), ownedOrganizations: many(organizations), })); export const organizationsRelations = relations(organizations, ({ many, one }) => ({ owner: one(users, { fields: [organizations.ownerUserId], references: [users.id] }), members: many(organizationMembers), projects: many(projects), apiKeys: many(apiKeys), })); export const organizationMembersRelations = relations(organizationMembers, ({ one }) => ({ organization: one(organizations, { fields: [organizationMembers.organizationId], references: [organizations.id] }), user: one(users, { fields: [organizationMembers.userId], references: [users.id] }), })); export const projectsRelations = relations(projects, ({ one, many }) => ({ organization: one(organizations, { fields: [projects.organizationId], references: [organizations.id] }), apiKeys: many(apiKeys), requests: many(fetchRequests), })); export const apiKeysRelations = relations(apiKeys, ({ one }) => ({ project: one(projects, { fields: [apiKeys.projectId], references: [projects.id] }), organization: one(organizations, { fields: [apiKeys.organizationId], references: [organizations.id] }), })); export const fetchRequestsRelations = relations(fetchRequests, ({ one, many }) => ({ project: one(projects, { fields: [fetchRequests.projectId], references: [projects.id] }), attempts: many(requestAttempts), })); export const requestAttemptsRelations = relations(requestAttempts, ({ one }) => ({ request: one(fetchRequests, { fields: [requestAttempts.requestId], references: [fetchRequests.id] }), })); export type User = typeof users.$inferSelect; export type Organization = typeof organizations.$inferSelect; export type Project = typeof projects.$inferSelect; export type ApiKey = typeof apiKeys.$inferSelect; export type FetchRequestRow = typeof fetchRequests.$inferSelect; export type RequestAttemptRow = typeof requestAttempts.$inferSelect; export type ProxySession = typeof proxySessions.$inferSelect; export type UsageEvent = typeof usageEvents.$inferSelect; export type DomainProfile = typeof domainProfiles.$inferSelect; export type ProviderHealthRow = typeof providerHealth.$inferSelect; export type AuditLog = typeof auditLogs.$inferSelect; export type SignupAllowlistRow = typeof signupAllowlist.$inferSelect; export type CrawlJob = typeof crawlJobs.$inferSelect; export type CrawlPage = typeof crawlPages.$inferSelect;