/** * Connector dev tool — draft model, conversions to/from the YAML config schema (packages/connectors/src/config.ts), * template builder and localStorage persistence. Pure functions, no React. */ import type { DevtoolFetchResult, DevtoolPreviewResult, RunSampleResult, SaveConfigResult, ValidateConfigResult } from "@/lib/admin-api"; import { toYaml } from "../yaml"; export type RuleType = "selector" | "attr" | "jsonld" | "meta" | "jsonPath" | "regex" | "constant"; export const RULE_TYPES: Array<{ value: RuleType; label: string; hint: string }> = [ { value: "selector", label: "CSS selector", hint: "text of the first (or all) matching elements" }, { value: "attr", label: "Attribute", hint: "attribute value of a selected element (href, src, data-*)" }, { value: "jsonld", label: "JSON-LD", hint: "value inside a JSON-LD object of a given @type, via JSON path" }, { value: "meta", label: "Meta tag", hint: " content" }, { value: "jsonPath", label: "Embedded JSON", hint: "JSON path over embedded JSON blobs (__NEXT_DATA__, drupal-settings…)" }, { value: "regex", label: "Regex", hint: "regular expression over the element text (or the whole page)" }, { value: "constant", label: "Constant", hint: "fixed default value" }, ]; export const TRANSFORMS = ["", "mw", "sqm", "ha", "money", "date", "status", "type", "country", "int", "float", "trim", "lower", "url", "bool"] as const; export const EXTRACT_KINDS = ["facility", "operator", "campus", "cloud_region", "ixp", "project", "news_event", "country"] as const; export const SOURCE_KINDS_LIST = ["operator", "government", "filing", "utility", "dataset", "community", "secondary", "news", "cloud_provider", "registry"] as const; export const COMMON_FIELDS: Record = { facility: ["name", "address", "city", "regionName", "countryIso2", "postalCode", "lat", "lng", "status", "facilityType", "itCapacityMw", "totalPowerMw", "plannedPowerMw", "buildingSqm", "siteAreaHa", "openedOn", "operatorName", "website", "description", "tier", "pue", "coolingType", "renewableClaim", "certifications"], operator: ["name", "kind", "website", "hqCountryIso2", "description"], campus: ["name", "address", "city", "countryIso2", "lat", "lng", "operatorName", "plannedPowerMw"], cloud_region: ["code", "name", "city", "countryIso2", "lat", "lng", "availabilityZones", "launchedOn", "status", "providerName"], ixp: ["name", "nameLong", "city", "countryIso2", "website", "networkCount"], project: ["name", "operatorName", "city", "countryIso2", "status", "announcedOn", "expectedOpening", "plannedMw", "investmentUsd", "description"], news_event: ["title", "summary", "publishedAt", "url", "operatorName", "countryIso2", "mw", "eventType"], country: ["iso2", "name"], }; export interface FieldRule { id: string; name: string; type: RuleType; selector: string; attr: string; regex: string; regexFlags: string; group: string; all: boolean; jsonPath: string; jsonld: string; meta: string; transform: string; default: string; } export interface ExtractorDraft { name: string; kind: string; key: string; each: string; minCertainty: string; parser: string; match: string; fields: FieldRule[]; } export interface FetchDraft { url: string; level: number; maxLevel: number; renderJs: boolean; userAgent: "bot" | "browser"; waitForSelector: string; } export interface MetaDraft { id: string; name: string; domain: string; kind: string; license: string; attribution: string; homepage: string; scheduleGroup: string; scheduleInterval: string; rpm: string; maxCreditsPerRun: string; } export interface DefaultsDraft { operatorName: string; operatorKind: string; countryIso2: string; facilityType: string; status: string; } export interface Draft { fetch: FetchDraft; extractor: ExtractorDraft; defaults: DefaultsDraft; meta: MetaDraft; /** full YAML in the editor (step 4); `yamlDirty` = edited by hand since the last regeneration */ yaml: string; yamlDirty: boolean; sampleUrls: string; savedAt: number | null; } export interface DevtoolShared { draft: Draft; update: (patch: Partial | ((d: Draft) => Draft)) => void; fetchResult: DevtoolFetchResult | null; setFetchResult: (r: DevtoolFetchResult | null) => void; previewResult: DevtoolPreviewResult | null; setPreviewResult: (r: DevtoolPreviewResult | null) => void; validation: ValidateConfigResult | null; setValidation: (r: ValidateConfigResult | null) => void; saveResult: SaveConfigResult | null; setSaveResult: (r: SaveConfigResult | null) => void; sampleResult: RunSampleResult | null; setSampleResult: (r: RunSampleResult | null) => void; goto: (step: number) => void; } let seq = 0; export function uid(): string { seq += 1; return `${Date.now().toString(36)}${seq.toString(36)}`; } export function newFieldRule(partial: Partial = {}): FieldRule { return { id: uid(), name: "", type: "selector", selector: "", attr: "", regex: "", regexFlags: "", group: "", all: false, jsonPath: "", jsonld: "", meta: "", transform: "", default: "", ...partial }; } export const EMPTY_DRAFT: Draft = { fetch: { url: "", level: 1, maxLevel: 2, renderJs: false, userAgent: "bot", waitForSelector: "" }, extractor: { name: "facility", kind: "facility", key: "{url}", each: "", minCertainty: "", parser: "", match: "", fields: [newFieldRule({ name: "name", type: "selector", selector: "h1" })] }, defaults: { operatorName: "", operatorKind: "", countryIso2: "", facilityType: "", status: "" }, meta: { id: "", name: "", domain: "", kind: "operator", license: "", attribution: "", homepage: "", scheduleGroup: "seed", scheduleInterval: "weekly", rpm: "20", maxCreditsPerRun: "40" }, yaml: "", yamlDirty: false, sampleUrls: "", savedAt: null, }; /* ------------------------------------------------------------------ helpers */ export function domainOf(url: string): string { try { return new URL(url).hostname; } catch { return ""; } } export function slugify(s: string): string { return s .toLowerCase() .replace(/^www\./, "") .replace(/\.[a-z]{2,}(\.[a-z]{2})?$/, "") .replace(/[^a-z0-9]+/g, "-") .replace(/^-+|-+$/g, "") .slice(0, 40); } export function parseUrls(text: string): string[] { const out: string[] = []; for (const line of text.split(/\r?\n|,|\s+/)) { const s = line.trim(); if (!s) continue; try { const u = new URL(s); if (u.protocol === "http:" || u.protocol === "https:") out.push(u.toString()); } catch { /* skip */ } } return [...new Set(out)]; } function numOrUndef(s: string): number | undefined { const n = Number(s); return s.trim() !== "" && Number.isFinite(n) ? n : undefined; } function coerceDefault(s: string): unknown { const t = s.trim(); if (t === "") return undefined; if (t === "true") return true; if (t === "false") return false; if (t === "null") return null; if (/^-?\d+(\.\d+)?$/.test(t)) return Number(t); return s; } /* ------------------------------------------------------------------ rule ↔ config */ /** One field rule → schema `selectorRule` (bare string when it is just a selector). */ export function ruleToConfig(r: FieldRule): string | Record | null { const o: Record = {}; switch (r.type) { case "selector": if (!r.selector.trim()) return null; o.selector = r.selector.trim(); if (r.all) o.all = true; if (r.regex.trim()) { o.regex = r.regex; if (r.regexFlags.trim()) o.regexFlags = r.regexFlags.trim(); if (numOrUndef(r.group) !== undefined) o.group = numOrUndef(r.group); } break; case "attr": if (!r.selector.trim() || !r.attr.trim()) return null; o.selector = r.selector.trim(); o.attr = r.attr.trim(); if (r.all) o.all = true; break; case "jsonld": if (!r.jsonld.trim() && !r.jsonPath.trim()) return null; if (r.jsonld.trim()) o.jsonld = r.jsonld.trim(); if (r.jsonPath.trim()) o.jsonPath = r.jsonPath.trim(); break; case "meta": if (!r.meta.trim()) return null; o.meta = r.meta.trim(); break; case "jsonPath": if (!r.jsonPath.trim()) return null; o.jsonPath = r.jsonPath.trim(); break; case "regex": if (!r.regex.trim()) return null; o.regex = r.regex; if (r.regexFlags.trim()) o.regexFlags = r.regexFlags.trim(); if (numOrUndef(r.group) !== undefined) o.group = numOrUndef(r.group); if (r.selector.trim()) o.selector = r.selector.trim(); if (r.all) o.all = true; break; case "constant": if (r.default.trim() === "") return null; break; } if (r.transform) o.transform = r.transform; const def = coerceDefault(r.default); if (def !== undefined) o.default = def; const keys = Object.keys(o); if (keys.length === 1 && keys[0] === "selector") return o.selector as string; return o; } /** schema `selectorRule` → field rule (best effort classification). */ export function configToRule(name: string, rule: unknown): FieldRule { if (typeof rule === "string") return newFieldRule({ name, type: "selector", selector: rule }); const o = (rule ?? {}) as Record; const str = (k: string) => (typeof o[k] === "string" ? (o[k] as string) : typeof o[k] === "number" ? String(o[k]) : ""); const base: Partial = { name, selector: str("selector"), attr: str("attr"), regex: str("regex"), regexFlags: str("regexFlags"), group: str("group"), all: o.all === true, jsonPath: str("jsonPath"), jsonld: str("jsonld"), meta: str("meta"), transform: str("transform"), default: o.default === undefined ? "" : typeof o.default === "string" ? o.default : JSON.stringify(o.default) }; let type: RuleType = "selector"; if (o.jsonld) type = "jsonld"; else if (o.meta) type = "meta"; else if (o.jsonPath) type = "jsonPath"; else if (o.attr) type = "attr"; else if (o.regex && !o.selector) type = "regex"; else if (o.selector) type = "selector"; else if (o.default !== undefined) type = "constant"; return newFieldRule({ ...base, type }); } export function extractorToConfig(ex: ExtractorDraft, opts: { includeMatch?: boolean } = {}): Record { const fields: Record = {}; for (const f of ex.fields) { const name = f.name.trim(); if (!name) continue; const cfg = ruleToConfig(f); if (cfg !== null) fields[name] = cfg; } const o: Record = { kind: ex.kind }; if (opts.includeMatch && ex.match.trim()) o.match = ex.match.split(/\r?\n/).map((s) => s.trim()).filter(Boolean); if (ex.parser.trim()) o.parser = ex.parser.trim(); if (ex.key.trim()) o.key = ex.key.trim(); if (ex.each.trim()) o.each = ex.each.trim(); if (Object.keys(fields).length) o.fields = fields; const mc = numOrUndef(ex.minCertainty); if (mc !== undefined) o.minCertainty = mc; return o; } export function defaultsToConfig(d: DefaultsDraft): Record | undefined { const o: Record = {}; for (const [k, v] of Object.entries(d)) if (v.trim()) o[k] = v.trim(); return Object.keys(o).length ? o : undefined; } /** Full connector config object from the draft (what the YAML template serializes). */ export function buildConfigObject(draft: Draft): Record { const { meta, fetch, extractor, defaults } = draft; const id = meta.id.trim() || slugify(domainOf(fetch.url) || "new-connector") || "new-connector"; const domain = meta.domain.trim() || domainOf(fetch.url) || "example.com"; const exName = extractor.name.trim() || extractor.kind; const seeds = fetch.url ? [{ url: fetch.url, group: meta.scheduleGroup.trim() || "seed", pageType: extractor.kind === "facility" ? "facility_page" : extractor.kind === "cloud_region" ? "cloud_region" : extractor.kind === "news_event" ? "press_release" : undefined }] : []; const cfg: Record = { id, name: meta.name.trim() || domain, domain, kind: meta.kind || "operator", mode: "html", priority: 3, enabled: false, license: meta.license.trim() || undefined, attribution: meta.attribution.trim() || undefined, homepage: meta.homepage.trim() || (fetch.url ? `${new URL(fetch.url).origin}/` : undefined), fetch: { level: fetch.level, maxLevel: Math.max(fetch.level, fetch.maxLevel), renderJs: fetch.renderJs || undefined, userAgent: fetch.userAgent === "browser" ? "browser" : undefined, waitForSelector: fetch.waitForSelector.trim() || undefined, rpm: numOrUndef(meta.rpm) ?? 20, concurrency: 2, respectRobots: true, maxCreditsPerRun: numOrUndef(meta.maxCreditsPerRun) ?? 40 }, discovery: { sitemap: false, seeds, include: domain ? [`^https?://${domain.replace(/\./g, "\\.")}/`] : [], exclude: ["\\?"], maxUrlsPerRun: 500 }, schedule: { [meta.scheduleGroup.trim() || "seed"]: meta.scheduleInterval.trim() || "weekly" }, extractors: { [exName]: extractorToConfig(extractor, { includeMatch: true }) }, defaults: defaultsToConfig(defaults), parserVersion: "v1", }; return cfg; } export function buildYaml(draft: Draft): string { const cfg = buildConfigObject(draft); const header = `# ${cfg.name} — generated by the DataCenterIndex connector dev tool\n# Set enabled: true once \`pnpm dci run ${cfg.id} --dry-run --limit 5\` yields valid entities. Document the source in docs/connectors/${cfg.id}.md.\n`; return header + toYaml(cfg); } /** Populate a draft from a parsed connector config (as returned by devtool/validate-config) + its raw YAML text. */ export function draftFromConfig(cfg: Record, yaml: string, prev: Draft = EMPTY_DRAFT): Draft { const str = (v: unknown, d = "") => (typeof v === "string" ? v : typeof v === "number" ? String(v) : d); const fetch = (cfg.fetch ?? {}) as Record; const discovery = (cfg.discovery ?? {}) as Record; const seeds = Array.isArray(discovery.seeds) ? (discovery.seeds as Array) : []; const firstSeed = seeds[0]; const seedUrl = typeof firstSeed === "string" ? firstSeed : (firstSeed?.url ?? ""); const schedule = (cfg.schedule ?? {}) as Record; const [group, interval] = Object.entries(schedule)[0] ?? ["seed", "weekly"]; const extractors = (cfg.extractors ?? {}) as Record>; const exEntries = Object.entries(extractors); const chosen = exEntries.find(([, e]) => e.fields && Object.keys(e.fields as object).length) ?? exEntries[0]; const [exName, ex] = chosen ?? ["facility", {}]; const fields = Object.entries((ex.fields ?? {}) as Record).map(([n, r]) => configToRule(n, r)); const defaults = (cfg.defaults ?? {}) as Record; return { ...prev, fetch: { url: prev.fetch.url || seedUrl, level: Number(fetch.level ?? 1) || 1, maxLevel: Number(fetch.maxLevel ?? 2) || 2, renderJs: fetch.renderJs === true, userAgent: fetch.userAgent === "browser" ? "browser" : "bot", waitForSelector: str(fetch.waitForSelector) }, extractor: { name: exName, kind: str(ex.kind, "facility"), key: str(ex.key), each: str(ex.each), minCertainty: str(ex.minCertainty), parser: str(ex.parser), match: Array.isArray(ex.match) ? (ex.match as string[]).join("\n") : "", fields: fields.length ? fields : [newFieldRule({ name: "name", selector: "h1" })] }, defaults: { operatorName: str(defaults.operatorName), operatorKind: str(defaults.operatorKind), countryIso2: str(defaults.countryIso2), facilityType: str(defaults.facilityType), status: str(defaults.status) }, meta: { id: str(cfg.id), name: str(cfg.name), domain: str(cfg.domain), kind: str(cfg.kind, "operator"), license: str(cfg.license), attribution: str(cfg.attribution), homepage: str(cfg.homepage), scheduleGroup: group ?? "seed", scheduleInterval: interval ?? "weekly", rpm: str(fetch.rpm, "20"), maxCreditsPerRun: str(fetch.maxCreditsPerRun, "40") }, yaml, yamlDirty: false, }; } /* ------------------------------------------------------------------ persistence */ export const DRAFT_KEY = "dci:devtool:draft:v1"; export function loadDraft(): Draft | null { try { const raw = window.localStorage.getItem(DRAFT_KEY); if (!raw) return null; const parsed = JSON.parse(raw) as Partial; return { ...EMPTY_DRAFT, ...parsed, fetch: { ...EMPTY_DRAFT.fetch, ...(parsed.fetch ?? {}) }, extractor: { ...EMPTY_DRAFT.extractor, ...(parsed.extractor ?? {}), fields: (parsed.extractor?.fields ?? EMPTY_DRAFT.extractor.fields).map((f) => newFieldRule(f)) }, defaults: { ...EMPTY_DRAFT.defaults, ...(parsed.defaults ?? {}) }, meta: { ...EMPTY_DRAFT.meta, ...(parsed.meta ?? {}) } }; } catch { return null; } } export function saveDraft(d: Draft): void { try { window.localStorage.setItem(DRAFT_KEY, JSON.stringify({ ...d, savedAt: Date.now() })); } catch { /* quota — ignore */ } } /** Union of entity field names present (non-empty) across a list of entities, in a stable order. */ export function fieldUnion(entities: Array>, preferred: string[] = []): string[] { const seen = new Set(); const out: string[] = []; for (const p of preferred) { if (entities.some((e) => p in e)) { seen.add(p); out.push(p); } } for (const e of entities) { for (const k of Object.keys(e)) { if (!seen.has(k) && k !== "provenance") { seen.add(k); out.push(k); } } } return out; } export function isEmptyValue(v: unknown): boolean { return v === null || v === undefined || v === "" || (Array.isArray(v) && v.length === 0); }