spb/vquant Public MIT
VibeQuant — AI-powered institutional-grade financial intelligence platform.
TypeScript 84.3%
Python 11.7%
JavaScript 1.6%
CSS 1.5%
HTML 0.7%
1/*2 * =============================================================================3 * VibeQuant (vquant) — AI-Powered Financial Intelligence Platform4 * -----------------------------------------------------------------------------5 * File: server/services/firecrawlService.ts6 *7 * Author: Simon-Pierre Boucher8 * Contact: contact@spboucher.ai9 * Website: https://www.spboucher.ai10 * Demo: https://www.vquant.ai11 * License: MIT (see LICENSE)12 *13 * Copyright © 2026 Simon-Pierre Boucher. All rights reserved.14 * =============================================================================15 */1617import axios from 'axios';18import { summarizeContent, shouldSummarize } from './contentSummarizerService';1920const FIRECRAWL_API_KEY = process.env.FIRECRAWL_API_KEY;21const FIRECRAWL_BASE_URL = 'https://api.firecrawl.dev/v2';2223// ==================== COMPLETE INTERFACES FROM DOCUMENTATION ====================2425export interface FirecrawlLocation {26 country?: string;27 languages?: string[];28}2930export interface FirecrawlAction {31 type: 'wait' | 'click' | 'scroll' | 'screenshot' | 'write' | 'press';32 selector?: string;33 milliseconds?: number;34 key?: string;35 text?: string;36}3738export interface FirecrawlWebhook {39 url: string;40 headers?: Record<string, string>;41}4243export interface FirecrawlScrapeOptions {44 url: string;45 formats?: Array<string | { type: 'json'; schema: any }>;46 onlyMainContent?: boolean;47 includeTags?: string[];48 excludeTags?: string[];49 maxAge?: number;50 headers?: Record<string, string>;51 waitFor?: number;52 mobile?: boolean;53 skipTlsVerification?: boolean;54 timeout?: number;55 parsers?: string[];56 actions?: FirecrawlAction[];57 location?: FirecrawlLocation;58 removeBase64Images?: boolean;59 blockAds?: boolean;60 proxy?: 'basic' | 'stealth' | 'auto';61 storeInCache?: boolean;62 zeroDataRetention?: boolean;63}6465export interface FirecrawlCrawlOptions {66 url: string;67 prompt?: string;68 excludePaths?: string[];69 includePaths?: string[];70 maxDiscoveryDepth?: number;71 sitemap?: 'skip' | 'include';72 ignoreQueryParameters?: boolean;73 limit?: number;74 crawlEntireDomain?: boolean;75 allowExternalLinks?: boolean;76 allowSubdomains?: boolean;77 delay?: number;78 maxConcurrency?: number;79 webhook?: FirecrawlWebhook;80 scrapeOptions?: Omit<FirecrawlScrapeOptions, 'url'>;81 zeroDataRetention?: boolean;82}8384export interface FirecrawlExtractOptions {85 urls: string[];86 prompt?: string;87 schema?: any;88 enableWebSearch?: boolean;89 ignoreSitemap?: boolean;90 includeSubdomains?: boolean;91 showSources?: boolean;92 scrapeOptions?: Omit<FirecrawlScrapeOptions, 'url'>;93 ignoreInvalidURLs?: boolean;94}9596export interface FirecrawlSearchOptions {97 query: string;98 limit?: number;99 sources?: Array<'web' | 'images' | 'news'>;100 categories?: Array<'github' | 'research' | 'pdf'>;101 tbs?: string;102 location?: string;103 country?: string;104 timeout?: number;105 ignoreInvalidURLs?: boolean;106 scrapeOptions?: Omit<FirecrawlScrapeOptions, 'url'>;107}108109export interface FirecrawlMapOptions {110 url: string;111 search?: string;112 sitemap?: 'skip' | 'include' | 'only';113 includeSubdomains?: boolean;114 ignoreQueryParameters?: boolean;115 limit?: number;116 timeout?: number;117 location?: FirecrawlLocation;118}119120export interface FirecrawlAgentOptions {121 prompt: string;122 urls?: string[];123 schema?: any;124 maxCredits?: number;125 strictConstrainToURLs?: boolean;126 model?: 'spark-1-mini' | 'spark-1-pro';127}128129export interface FirecrawlBatchScrapeOptions {130 urls: string[];131 webhook?: FirecrawlWebhook;132 maxConcurrency?: number;133 ignoreInvalidURLs?: boolean;134 formats?: Array<string | { type: 'json'; schema: any }>;135 onlyMainContent?: boolean;136 includeTags?: string[];137 excludeTags?: string[];138 maxAge?: number;139 headers?: Record<string, string>;140 waitFor?: number;141 mobile?: boolean;142 skipTlsVerification?: boolean;143 timeout?: number;144 parsers?: string[];145 actions?: FirecrawlAction[];146 location?: FirecrawlLocation;147 removeBase64Images?: boolean;148 blockAds?: boolean;149 proxy?: 'basic' | 'stealth' | 'auto';150 storeInCache?: boolean;151 zeroDataRetention?: boolean;152}153154export interface FirecrawlSearchResult {155 url: string;156 title: string;157 description: string;158 content?: string;159 markdown?: string;160}161162export interface FirecrawlScrapeResult {163 url: string;164 content: string;165 markdown?: string;166 metadata?: Record<string, any>;167}168169export interface FirecrawlCrawlResult {170 success: boolean;171 data: Array<{172 url: string;173 content: string;174 markdown?: string;175 }>;176}177178// Fonction helper pour requêtes authentifiées179async function firecrawlRequest(endpoint: string, data: any, timeoutMs: number = 90000): Promise<any> {180 if (!FIRECRAWL_API_KEY) {181 throw new Error('FIRECRAWL_API_KEY not configured in environment variables');182 }183184 try {185 console.log(`[Firecrawl] Requesting ${endpoint} with timeout ${timeoutMs}ms...`);186187 const response = await axios.post(`${FIRECRAWL_BASE_URL}${endpoint}`, data, {188 headers: {189 'Authorization': `Bearer ${FIRECRAWL_API_KEY}`,190 'Content-Type': 'application/json',191 },192 timeout: timeoutMs, // Configurable timeout193 maxContentLength: 50 * 1024 * 1024, // 50MB max response194 maxBodyLength: 50 * 1024 * 1024,195 });196197 console.log(`[Firecrawl] ✓ Request successful (${JSON.stringify(response.data).length} chars)`);198 return response.data;199 } catch (error) {200 if (axios.isAxiosError(error)) {201 if (error.code === 'ECONNABORTED' || error.message.includes('timeout')) {202 console.error(`[Firecrawl] Timeout after ${timeoutMs}ms for ${endpoint}`);203 throw new Error(`Firecrawl timeout: Request took longer than ${timeoutMs / 1000} seconds. Try reducing the scope or increasing timeout.`);204 }205 if (error.response) {206 console.error(`[Firecrawl] HTTP ${error.response.status}:`, error.response.data);207 throw new Error(`Firecrawl API error: ${error.response.status} - ${JSON.stringify(error.response.data)}`);208 }209 }210 console.error(`[Firecrawl] ${endpoint} error:`, error);211 throw new Error(`Firecrawl request failed: ${(error as Error).message}`);212 }213}214215// ==================== UPDATED FUNCTIONS WITH ALL PARAMETERS ====================216217// Scrape - Single URL with full parameter support218export async function scrapeFirecrawl(options: FirecrawlScrapeOptions): Promise<FirecrawlScrapeResult> {219 const requestData: any = {220 url: options.url,221 formats: options.formats || ['markdown'],222 onlyMainContent: options.onlyMainContent ?? true,223 parsers: options.parsers || ['pdf'],224 };225226 // Add optional parameters227 if (options.includeTags) requestData.includeTags = options.includeTags;228 if (options.excludeTags) requestData.excludeTags = options.excludeTags;229 if (options.maxAge !== undefined) requestData.maxAge = options.maxAge;230 if (options.headers) requestData.headers = options.headers;231 if (options.waitFor !== undefined) requestData.waitFor = options.waitFor;232 if (options.mobile !== undefined) requestData.mobile = options.mobile;233 if (options.skipTlsVerification !== undefined) requestData.skipTlsVerification = options.skipTlsVerification;234 if (options.timeout !== undefined) requestData.timeout = options.timeout;235 if (options.actions) requestData.actions = options.actions;236 if (options.location) requestData.location = options.location;237 if (options.removeBase64Images !== undefined) requestData.removeBase64Images = options.removeBase64Images;238 if (options.blockAds !== undefined) requestData.blockAds = options.blockAds;239 if (options.proxy) requestData.proxy = options.proxy;240 if (options.storeInCache !== undefined) requestData.storeInCache = options.storeInCache;241 if (options.zeroDataRetention !== undefined) requestData.zeroDataRetention = options.zeroDataRetention;242243 // Use longer timeout for PDFs244 const isPDF = options.url.toLowerCase().endsWith('.pdf') || options.url.toLowerCase().includes('.pdf');245 const timeout = isPDF ? 120000 : (options.timeout || 90000); // 2 min for PDFs, 90s for others246247 const response = await firecrawlRequest('/scrape', requestData, timeout);248 const data = response.data || {};249250 // Log result size251 const resultSize = JSON.stringify(data).length;252 console.log(`[Firecrawl Scrape] Result size: ${resultSize} chars for ${options.url}`);253254 // Detect if this is a PDF or large document255 const content = data.markdown || data.content || '';256 const detectedIsPDF = isPDF || data.metadata?.contentType?.includes('pdf');257258 // For PDFs, use aggressive summarization threshold (3000 chars = ~2 pages)259 // For other content, use 8000 chars threshold260 const summarizationThreshold = detectedIsPDF ? 3000 : 8000;261262 if (content.length > summarizationThreshold) {263 const docType = detectedIsPDF ? 'PDF' : 'document';264 console.log(`[Firecrawl Scrape] Large ${docType} detected (${content.length} chars), summarizing...`);265266 try {267 const summaryResult = await summarizeContent(content, 'pdf', {268 focusOn: detectedIsPDF269 ? ['revenue', 'earnings', 'EPS', 'guidance', 'margins', 'growth', 'cash flow', 'outlook', 'risks', 'strategy']270 : undefined271 });272273 console.log(`[Firecrawl Scrape] ✓ Summarized: ${summaryResult.metadata.compressionRatio.toFixed(1)}x compression (${summaryResult.metadata.model})`);274275 return {276 ...data,277 markdown: summaryResult.summary,278 content: summaryResult.summary,279 metadata: {280 ...data.metadata,281 summarized: true,282 originalLength: summaryResult.metadata.originalLength,283 summaryLength: summaryResult.metadata.summaryLength,284 compressionRatio: summaryResult.metadata.compressionRatio,285 summaryModel: summaryResult.metadata.model,286 keyPoints: summaryResult.keyPoints,287 documentType: docType288 }289 };290 } catch (error) {291 console.error(`[Firecrawl Scrape] Summarization failed:`, error);292293 // If summarization fails completely, return truncated version with clear message294 const truncated = content.substring(0, 12000);295 console.warn(`[Firecrawl Scrape] Returning truncated content (12K chars) due to summarization failure`);296297 return {298 ...data,299 markdown: truncated + '\n\n[⚠️ Content truncated - original document was too large to process fully. Showing first 12,000 characters.]',300 content: truncated,301 metadata: {302 ...data.metadata,303 truncated: true,304 originalLength: content.length,305 truncatedLength: truncated.length,306 reason: 'summarization_failed'307 }308 };309 }310 }311312 return data;313}314315// Crawl - Multiple URLs with full parameter support316export async function crawlFirecrawl(options: FirecrawlCrawlOptions): Promise<FirecrawlCrawlResult> {317 const requestData: any = {318 url: options.url,319 limit: Math.min(options.limit || 10, 20), // Cap for performance320 };321322 // Add optional parameters323 if (options.prompt) requestData.prompt = options.prompt;324 if (options.excludePaths) requestData.excludePaths = options.excludePaths;325 if (options.includePaths) requestData.includePaths = options.includePaths;326 if (options.maxDiscoveryDepth !== undefined) requestData.maxDiscoveryDepth = options.maxDiscoveryDepth;327 if (options.sitemap) requestData.sitemap = options.sitemap;328 if (options.ignoreQueryParameters !== undefined) requestData.ignoreQueryParameters = options.ignoreQueryParameters;329 if (options.crawlEntireDomain !== undefined) requestData.crawlEntireDomain = options.crawlEntireDomain;330 if (options.allowExternalLinks !== undefined) requestData.allowExternalLinks = options.allowExternalLinks;331 if (options.allowSubdomains !== undefined) requestData.allowSubdomains = options.allowSubdomains;332 if (options.delay !== undefined) requestData.delay = options.delay;333 if (options.maxConcurrency !== undefined) requestData.maxConcurrency = options.maxConcurrency;334 if (options.webhook) requestData.webhook = options.webhook;335 if (options.scrapeOptions) requestData.scrapeOptions = options.scrapeOptions;336 if (options.zeroDataRetention !== undefined) requestData.zeroDataRetention = options.zeroDataRetention;337338 const response = await firecrawlRequest('/crawl', requestData);339 const pageCount = response.data?.length || 0;340 console.log(`[Firecrawl Crawl] Crawled ${pageCount} pages from ${options.url}`);341 return response;342}343344// Search - Search and scrape results with full parameter support345export async function searchFirecrawl(options: FirecrawlSearchOptions): Promise<FirecrawlSearchResult[]> {346 const requestData: any = {347 query: options.query,348 limit: Math.min(options.limit || 20, 50),349 };350351 // Add optional parameters352 if (options.sources) requestData.sources = options.sources;353 if (options.categories) requestData.categories = options.categories;354 if (options.tbs) requestData.tbs = options.tbs;355 if (options.location) requestData.location = options.location;356 if (options.country) requestData.country = options.country;357 if (options.timeout !== undefined) requestData.timeout = options.timeout;358 if (options.ignoreInvalidURLs !== undefined) requestData.ignoreInvalidURLs = options.ignoreInvalidURLs;359 if (options.scrapeOptions) requestData.scrapeOptions = options.scrapeOptions;360361 const response = await firecrawlRequest('/search', requestData);362363 // Handle different response formats364 let results = [];365 if (response.data) {366 results = response.data;367 } else if (Array.isArray(response)) {368 results = response;369 } else if (response.results) {370 results = response.results;371 }372373 console.log(`[Firecrawl Search] Found ${results.length} results for: "${options.query}"`);374 return results;375}376377// ==================== NEW ENDPOINTS ====================378379// Extract - Extract structured data using LLM380export async function extractFirecrawl(options: FirecrawlExtractOptions): Promise<any> {381 const requestData: any = {382 urls: options.urls,383 };384385 // Add optional parameters386 if (options.prompt) requestData.prompt = options.prompt;387 if (options.schema) requestData.schema = options.schema;388 if (options.enableWebSearch !== undefined) requestData.enableWebSearch = options.enableWebSearch;389 if (options.ignoreSitemap !== undefined) requestData.ignoreSitemap = options.ignoreSitemap;390 if (options.includeSubdomains !== undefined) requestData.includeSubdomains = options.includeSubdomains;391 if (options.showSources !== undefined) requestData.showSources = options.showSources;392 if (options.scrapeOptions) requestData.scrapeOptions = options.scrapeOptions;393 if (options.ignoreInvalidURLs !== undefined) requestData.ignoreInvalidURLs = options.ignoreInvalidURLs;394395 const response = await firecrawlRequest('/extract', requestData);396 console.log(`[Firecrawl Extract] Extracted data from ${options.urls.length} URL(s)`);397 return response.data || response;398}399400// Map - Map all URLs from a website401export async function mapFirecrawl(options: FirecrawlMapOptions): Promise<{ links: string[] }> {402 const requestData: any = {403 url: options.url,404 };405406 // Add optional parameters407 if (options.search) requestData.search = options.search;408 if (options.sitemap) requestData.sitemap = options.sitemap;409 if (options.includeSubdomains !== undefined) requestData.includeSubdomains = options.includeSubdomains;410 if (options.ignoreQueryParameters !== undefined) requestData.ignoreQueryParameters = options.ignoreQueryParameters;411 if (options.limit !== undefined) requestData.limit = options.limit;412 if (options.timeout !== undefined) requestData.timeout = options.timeout;413 if (options.location) requestData.location = options.location;414415 const response = await firecrawlRequest('/map', requestData);416 const links = response.links || response.data?.links || [];417 console.log(`[Firecrawl Map] Found ${links.length} links from ${options.url}`);418 return { links };419}420421// Agent - Autonomous data extraction agent422export async function agentFirecrawl(options: FirecrawlAgentOptions): Promise<any> {423 const requestData: any = {424 prompt: options.prompt,425 };426427 // Add optional parameters428 if (options.urls) requestData.urls = options.urls;429 if (options.schema) requestData.schema = options.schema;430 if (options.maxCredits !== undefined) requestData.maxCredits = options.maxCredits;431 if (options.strictConstrainToURLs !== undefined) requestData.strictConstrainToURLs = options.strictConstrainToURLs;432 if (options.model) requestData.model = options.model;433434 const response = await firecrawlRequest('/agent', requestData);435 console.log(`[Firecrawl Agent] Task completed with prompt: "${options.prompt.substring(0, 50)}..."`);436 return response.data || response;437}438439// Batch Scrape - Scrape multiple URLs in batch440export async function batchScrapeFirecrawl(options: FirecrawlBatchScrapeOptions): Promise<any> {441 const requestData: any = {442 urls: options.urls,443 };444445 // Add optional parameters446 if (options.webhook) requestData.webhook = options.webhook;447 if (options.maxConcurrency !== undefined) requestData.maxConcurrency = options.maxConcurrency;448 if (options.ignoreInvalidURLs !== undefined) requestData.ignoreInvalidURLs = options.ignoreInvalidURLs;449 if (options.formats) requestData.formats = options.formats;450 if (options.onlyMainContent !== undefined) requestData.onlyMainContent = options.onlyMainContent;451 if (options.includeTags) requestData.includeTags = options.includeTags;452 if (options.excludeTags) requestData.excludeTags = options.excludeTags;453 if (options.maxAge !== undefined) requestData.maxAge = options.maxAge;454 if (options.headers) requestData.headers = options.headers;455 if (options.waitFor !== undefined) requestData.waitFor = options.waitFor;456 if (options.mobile !== undefined) requestData.mobile = options.mobile;457 if (options.skipTlsVerification !== undefined) requestData.skipTlsVerification = options.skipTlsVerification;458 if (options.timeout !== undefined) requestData.timeout = options.timeout;459 if (options.parsers) requestData.parsers = options.parsers;460 if (options.actions) requestData.actions = options.actions;461 if (options.location) requestData.location = options.location;462 if (options.removeBase64Images !== undefined) requestData.removeBase64Images = options.removeBase64Images;463 if (options.blockAds !== undefined) requestData.blockAds = options.blockAds;464 if (options.proxy) requestData.proxy = options.proxy;465 if (options.storeInCache !== undefined) requestData.storeInCache = options.storeInCache;466 if (options.zeroDataRetention !== undefined) requestData.zeroDataRetention = options.zeroDataRetention;467468 const response = await firecrawlRequest('/batch/scrape', requestData);469 console.log(`[Firecrawl Batch Scrape] Scraped ${options.urls.length} URL(s)`);470 return response.data || response;471}472