CREATE TABLE "crawl_jobs" (
	"id" text PRIMARY KEY NOT NULL,
	"organization_id" text NOT NULL,
	"project_id" text NOT NULL,
	"api_key_id" text,
	"source" text DEFAULT 'api' NOT NULL,
	"label" text,
	"seed_url" text NOT NULL,
	"domain" text NOT NULL,
	"options" jsonb NOT NULL,
	"status" text DEFAULT 'queued' NOT NULL,
	"worker_id" text,
	"pages_discovered" integer DEFAULT 0 NOT NULL,
	"pages_fetched" integer DEFAULT 0 NOT NULL,
	"pages_ok" integer DEFAULT 0 NOT NULL,
	"pages_blocked" integer DEFAULT 0 NOT NULL,
	"pages_failed" integer DEFAULT 0 NOT NULL,
	"bytes" bigint DEFAULT 0 NOT NULL,
	"cost_usd" double precision DEFAULT 0 NOT NULL,
	"error_code" text,
	"error_message" text,
	"webhook_status" text,
	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
	"started_at" timestamp with time zone,
	"heartbeat_at" timestamp with time zone,
	"completed_at" timestamp with time zone
);
--> statement-breakpoint
CREATE TABLE "crawl_pages" (
	"id" text PRIMARY KEY NOT NULL,
	"job_id" text NOT NULL,
	"url" text NOT NULL,
	"final_url" text,
	"depth" integer DEFAULT 0 NOT NULL,
	"parent_url" text,
	"status" text DEFAULT 'pending' NOT NULL,
	"http_status" integer,
	"error_code" text,
	"request_id" text,
	"title" text,
	"description" text,
	"content_type" text,
	"content" text,
	"links_count" integer DEFAULT 0 NOT NULL,
	"bytes" bigint DEFAULT 0 NOT NULL,
	"duration_ms" integer,
	"mode" text,
	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
	"fetched_at" timestamp with time zone
);
--> statement-breakpoint
CREATE TABLE "signup_allowlist" (
	"email" text PRIMARY KEY NOT NULL,
	"note" text,
	"invited_by_user_id" text,
	"invited_at" timestamp with time zone,
	"used_at" timestamp with time zone,
	"user_id" text,
	"created_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
ALTER TABLE "organizations" ALTER COLUMN "plan" SET DEFAULT 'unlimited';--> statement-breakpoint
ALTER TABLE "fetch_requests" ADD COLUMN "mode" text;--> statement-breakpoint
ALTER TABLE "request_attempts" ADD COLUMN "mode" text DEFAULT 'http' NOT NULL;--> statement-breakpoint
ALTER TABLE "crawl_jobs" ADD CONSTRAINT "crawl_jobs_organization_id_organizations_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organizations"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "crawl_jobs" ADD CONSTRAINT "crawl_jobs_project_id_projects_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."projects"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "crawl_jobs" ADD CONSTRAINT "crawl_jobs_api_key_id_api_keys_id_fk" FOREIGN KEY ("api_key_id") REFERENCES "public"."api_keys"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "crawl_pages" ADD CONSTRAINT "crawl_pages_job_id_crawl_jobs_id_fk" FOREIGN KEY ("job_id") REFERENCES "public"."crawl_jobs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "signup_allowlist" ADD CONSTRAINT "signup_allowlist_invited_by_user_id_users_id_fk" FOREIGN KEY ("invited_by_user_id") REFERENCES "public"."users"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "signup_allowlist" ADD CONSTRAINT "signup_allowlist_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
CREATE INDEX "crawl_jobs_project_created_idx" ON "crawl_jobs" USING btree ("project_id","created_at");--> statement-breakpoint
CREATE INDEX "crawl_jobs_status_idx" ON "crawl_jobs" USING btree ("status");--> statement-breakpoint
CREATE INDEX "crawl_pages_job_idx" ON "crawl_pages" USING btree ("job_id","created_at");--> statement-breakpoint
CREATE UNIQUE INDEX "crawl_pages_job_url_uq" ON "crawl_pages" USING btree ("job_id","url");--> statement-breakpoint
CREATE INDEX "signup_allowlist_user_idx" ON "signup_allowlist" USING btree ("user_id");--> statement-breakpoint
UPDATE "organizations" SET "plan" = 'unlimited' WHERE "plan" <> 'unlimited';