TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import type { FastifyInstance } from 'fastify';2import { z } from 'zod';3import { createHmac } from 'node:crypto';4import { connectorsForUrl, createRouter, createCrawlContext, listConnectorMeta } from '@rareindex/connectors';5import type { NormalizedRecord } from '@rareindex/shared';6import { problem } from '../lib/envelope.js';78/**9 * Internal (loopback-only) endpoints used by the web app. Connector modules are loaded dynamically10 * (`import(path)`), which the Next.js bundler cannot do — so URL lookups run here, where tsx can.11 * Auth: `x-internal-token` = HMAC(SESSION_SECRET, "internal") — both processes share SESSION_SECRET.12 */13export function internalToken(secret = process.env.SESSION_SECRET ?? 'dev-only'): string {14 return createHmac('sha256', secret).update('internal').digest('hex');15}1617export async function internalRoutes(app: FastifyInstance) {18 app.addHook('onRequest', async (req, reply) => {19 if (!req.url.startsWith('/internal/')) return;20 const loopback = req.ip === '127.0.0.1' || req.ip === '::1' || req.ip === '::ffff:127.0.0.1';21 const ok = loopback && req.headers['x-internal-token'] === internalToken();22 if (!ok) return problem(reply, 403, 'Forbidden', 'Internal endpoint');23 });2425 app.get('/internal/lookup/sources', async () => ({ data: listConnectorMeta({ enabled: true }).filter((m) => m.supportsLookup).map((m) => ({ id: m.id, sourceName: m.sourceName, sourceUrl: m.sourceUrl })) }));2627 app.post('/internal/lookup', async (req, reply) => {28 const body = z.object({ url: z.string().url().max(2000) }).safeParse(req.body);29 if (!body.success) return problem(reply, 400, 'Invalid body', 'url required');30 const url = body.data.url;31 const connectors = await connectorsForUrl(url);32 const connector = connectors[0];33 if (!connector?.lookup) return { data: null, meta: { supported: false } };34 const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });35 const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: 1 } });36 const started = Date.now();37 const raws = await connector.lookup(url, ctx);38 const records: NormalizedRecord[] = [];39 for (const raw of raws) records.push(...(await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() })));40 return { data: { connectorId: connector.meta.id, sourceId: connector.meta.sourceId, records }, meta: { supported: true, durationMs: Date.now() - started, engineStats: ctx.engineStats } };41 });42}43