spb/datacenterindex
Public
HTML 53.9%
TypeScript 44.5%
JavaScript 0.6%
SQL 0.5%
1/**2 * Connector dev tool — draft model, conversions to/from the YAML config schema (packages/connectors/src/config.ts),3 * template builder and localStorage persistence. Pure functions, no React.4 */5import type { DevtoolFetchResult, DevtoolPreviewResult, RunSampleResult, SaveConfigResult, ValidateConfigResult } from "@/lib/admin-api";6import { toYaml } from "../yaml";78export type RuleType = "selector" | "attr" | "jsonld" | "meta" | "jsonPath" | "regex" | "constant";910export const RULE_TYPES: Array<{ value: RuleType; label: string; hint: string }> = [11 { value: "selector", label: "CSS selector", hint: "text of the first (or all) matching elements" },12 { value: "attr", label: "Attribute", hint: "attribute value of a selected element (href, src, data-*)" },13 { value: "jsonld", label: "JSON-LD", hint: "value inside a JSON-LD object of a given @type, via JSON path" },14 { value: "meta", label: "Meta tag", hint: "<meta name|property=…> content" },15 { value: "jsonPath", label: "Embedded JSON", hint: "JSON path over embedded JSON blobs (__NEXT_DATA__, drupal-settings…)" },16 { value: "regex", label: "Regex", hint: "regular expression over the element text (or the whole page)" },17 { value: "constant", label: "Constant", hint: "fixed default value" },18];1920export const TRANSFORMS = ["", "mw", "sqm", "ha", "money", "date", "status", "type", "country", "int", "float", "trim", "lower", "url", "bool"] as const;21export const EXTRACT_KINDS = ["facility", "operator", "campus", "cloud_region", "ixp", "project", "news_event", "country"] as const;22export const SOURCE_KINDS_LIST = ["operator", "government", "filing", "utility", "dataset", "community", "secondary", "news", "cloud_provider", "registry"] as const;23export const COMMON_FIELDS: Record<string, string[]> = {24 facility: ["name", "address", "city", "regionName", "countryIso2", "postalCode", "lat", "lng", "status", "facilityType", "itCapacityMw", "totalPowerMw", "plannedPowerMw", "buildingSqm", "siteAreaHa", "openedOn", "operatorName", "website", "description", "tier", "pue", "coolingType", "renewableClaim", "certifications"],25 operator: ["name", "kind", "website", "hqCountryIso2", "description"],26 campus: ["name", "address", "city", "countryIso2", "lat", "lng", "operatorName", "plannedPowerMw"],27 cloud_region: ["code", "name", "city", "countryIso2", "lat", "lng", "availabilityZones", "launchedOn", "status", "providerName"],28 ixp: ["name", "nameLong", "city", "countryIso2", "website", "networkCount"],29 project: ["name", "operatorName", "city", "countryIso2", "status", "announcedOn", "expectedOpening", "plannedMw", "investmentUsd", "description"],30 news_event: ["title", "summary", "publishedAt", "url", "operatorName", "countryIso2", "mw", "eventType"],31 country: ["iso2", "name"],32};3334export interface FieldRule {35 id: string;36 name: string;37 type: RuleType;38 selector: string;39 attr: string;40 regex: string;41 regexFlags: string;42 group: string;43 all: boolean;44 jsonPath: string;45 jsonld: string;46 meta: string;47 transform: string;48 default: string;49}5051export interface ExtractorDraft {52 name: string;53 kind: string;54 key: string;55 each: string;56 minCertainty: string;57 parser: string;58 match: string;59 fields: FieldRule[];60}6162export interface FetchDraft {63 url: string;64 level: number;65 maxLevel: number;66 renderJs: boolean;67 userAgent: "bot" | "browser";68 waitForSelector: string;69}7071export interface MetaDraft {72 id: string;73 name: string;74 domain: string;75 kind: string;76 license: string;77 attribution: string;78 homepage: string;79 scheduleGroup: string;80 scheduleInterval: string;81 rpm: string;82 maxCreditsPerRun: string;83}8485export interface DefaultsDraft {86 operatorName: string;87 operatorKind: string;88 countryIso2: string;89 facilityType: string;90 status: string;91}9293export interface Draft {94 fetch: FetchDraft;95 extractor: ExtractorDraft;96 defaults: DefaultsDraft;97 meta: MetaDraft;98 /** full YAML in the editor (step 4); `yamlDirty` = edited by hand since the last regeneration */99 yaml: string;100 yamlDirty: boolean;101 sampleUrls: string;102 savedAt: number | null;103}104105export interface DevtoolShared {106 draft: Draft;107 update: (patch: Partial<Draft> | ((d: Draft) => Draft)) => void;108 fetchResult: DevtoolFetchResult | null;109 setFetchResult: (r: DevtoolFetchResult | null) => void;110 previewResult: DevtoolPreviewResult | null;111 setPreviewResult: (r: DevtoolPreviewResult | null) => void;112 validation: ValidateConfigResult | null;113 setValidation: (r: ValidateConfigResult | null) => void;114 saveResult: SaveConfigResult | null;115 setSaveResult: (r: SaveConfigResult | null) => void;116 sampleResult: RunSampleResult | null;117 setSampleResult: (r: RunSampleResult | null) => void;118 goto: (step: number) => void;119}120121let seq = 0;122export function uid(): string {123 seq += 1;124 return `${Date.now().toString(36)}${seq.toString(36)}`;125}126127export function newFieldRule(partial: Partial<FieldRule> = {}): FieldRule {128 return { id: uid(), name: "", type: "selector", selector: "", attr: "", regex: "", regexFlags: "", group: "", all: false, jsonPath: "", jsonld: "", meta: "", transform: "", default: "", ...partial };129}130131export const EMPTY_DRAFT: Draft = {132 fetch: { url: "", level: 1, maxLevel: 2, renderJs: false, userAgent: "bot", waitForSelector: "" },133 extractor: { name: "facility", kind: "facility", key: "{url}", each: "", minCertainty: "", parser: "", match: "", fields: [newFieldRule({ name: "name", type: "selector", selector: "h1" })] },134 defaults: { operatorName: "", operatorKind: "", countryIso2: "", facilityType: "", status: "" },135 meta: { id: "", name: "", domain: "", kind: "operator", license: "", attribution: "", homepage: "", scheduleGroup: "seed", scheduleInterval: "weekly", rpm: "20", maxCreditsPerRun: "40" },136 yaml: "",137 yamlDirty: false,138 sampleUrls: "",139 savedAt: null,140};141142/* ------------------------------------------------------------------ helpers */143144export function domainOf(url: string): string {145 try {146 return new URL(url).hostname;147 } catch {148 return "";149 }150}151152export function slugify(s: string): string {153 return s154 .toLowerCase()155 .replace(/^www\./, "")156 .replace(/\.[a-z]{2,}(\.[a-z]{2})?$/, "")157 .replace(/[^a-z0-9]+/g, "-")158 .replace(/^-+|-+$/g, "")159 .slice(0, 40);160}161162export function parseUrls(text: string): string[] {163 const out: string[] = [];164 for (const line of text.split(/\r?\n|,|\s+/)) {165 const s = line.trim();166 if (!s) continue;167 try {168 const u = new URL(s);169 if (u.protocol === "http:" || u.protocol === "https:") out.push(u.toString());170 } catch {171 /* skip */172 }173 }174 return [...new Set(out)];175}176177function numOrUndef(s: string): number | undefined {178 const n = Number(s);179 return s.trim() !== "" && Number.isFinite(n) ? n : undefined;180}181182function coerceDefault(s: string): unknown {183 const t = s.trim();184 if (t === "") return undefined;185 if (t === "true") return true;186 if (t === "false") return false;187 if (t === "null") return null;188 if (/^-?\d+(\.\d+)?$/.test(t)) return Number(t);189 return s;190}191192/* ------------------------------------------------------------------ rule ↔ config */193194/** One field rule → schema `selectorRule` (bare string when it is just a selector). */195export function ruleToConfig(r: FieldRule): string | Record<string, unknown> | null {196 const o: Record<string, unknown> = {};197 switch (r.type) {198 case "selector":199 if (!r.selector.trim()) return null;200 o.selector = r.selector.trim();201 if (r.all) o.all = true;202 if (r.regex.trim()) {203 o.regex = r.regex;204 if (r.regexFlags.trim()) o.regexFlags = r.regexFlags.trim();205 if (numOrUndef(r.group) !== undefined) o.group = numOrUndef(r.group);206 }207 break;208 case "attr":209 if (!r.selector.trim() || !r.attr.trim()) return null;210 o.selector = r.selector.trim();211 o.attr = r.attr.trim();212 if (r.all) o.all = true;213 break;214 case "jsonld":215 if (!r.jsonld.trim() && !r.jsonPath.trim()) return null;216 if (r.jsonld.trim()) o.jsonld = r.jsonld.trim();217 if (r.jsonPath.trim()) o.jsonPath = r.jsonPath.trim();218 break;219 case "meta":220 if (!r.meta.trim()) return null;221 o.meta = r.meta.trim();222 break;223 case "jsonPath":224 if (!r.jsonPath.trim()) return null;225 o.jsonPath = r.jsonPath.trim();226 break;227 case "regex":228 if (!r.regex.trim()) return null;229 o.regex = r.regex;230 if (r.regexFlags.trim()) o.regexFlags = r.regexFlags.trim();231 if (numOrUndef(r.group) !== undefined) o.group = numOrUndef(r.group);232 if (r.selector.trim()) o.selector = r.selector.trim();233 if (r.all) o.all = true;234 break;235 case "constant":236 if (r.default.trim() === "") return null;237 break;238 }239 if (r.transform) o.transform = r.transform;240 const def = coerceDefault(r.default);241 if (def !== undefined) o.default = def;242 const keys = Object.keys(o);243 if (keys.length === 1 && keys[0] === "selector") return o.selector as string;244 return o;245}246247/** schema `selectorRule` → field rule (best effort classification). */248export function configToRule(name: string, rule: unknown): FieldRule {249 if (typeof rule === "string") return newFieldRule({ name, type: "selector", selector: rule });250 const o = (rule ?? {}) as Record<string, unknown>;251 const str = (k: string) => (typeof o[k] === "string" ? (o[k] as string) : typeof o[k] === "number" ? String(o[k]) : "");252 const base: Partial<FieldRule> = { name, selector: str("selector"), attr: str("attr"), regex: str("regex"), regexFlags: str("regexFlags"), group: str("group"), all: o.all === true, jsonPath: str("jsonPath"), jsonld: str("jsonld"), meta: str("meta"), transform: str("transform"), default: o.default === undefined ? "" : typeof o.default === "string" ? o.default : JSON.stringify(o.default) };253 let type: RuleType = "selector";254 if (o.jsonld) type = "jsonld";255 else if (o.meta) type = "meta";256 else if (o.jsonPath) type = "jsonPath";257 else if (o.attr) type = "attr";258 else if (o.regex && !o.selector) type = "regex";259 else if (o.selector) type = "selector";260 else if (o.default !== undefined) type = "constant";261 return newFieldRule({ ...base, type });262}263264export function extractorToConfig(ex: ExtractorDraft, opts: { includeMatch?: boolean } = {}): Record<string, unknown> {265 const fields: Record<string, unknown> = {};266 for (const f of ex.fields) {267 const name = f.name.trim();268 if (!name) continue;269 const cfg = ruleToConfig(f);270 if (cfg !== null) fields[name] = cfg;271 }272 const o: Record<string, unknown> = { kind: ex.kind };273 if (opts.includeMatch && ex.match.trim()) o.match = ex.match.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);274 if (ex.parser.trim()) o.parser = ex.parser.trim();275 if (ex.key.trim()) o.key = ex.key.trim();276 if (ex.each.trim()) o.each = ex.each.trim();277 if (Object.keys(fields).length) o.fields = fields;278 const mc = numOrUndef(ex.minCertainty);279 if (mc !== undefined) o.minCertainty = mc;280 return o;281}282283export function defaultsToConfig(d: DefaultsDraft): Record<string, unknown> | undefined {284 const o: Record<string, unknown> = {};285 for (const [k, v] of Object.entries(d)) if (v.trim()) o[k] = v.trim();286 return Object.keys(o).length ? o : undefined;287}288289/** Full connector config object from the draft (what the YAML template serializes). */290export function buildConfigObject(draft: Draft): Record<string, unknown> {291 const { meta, fetch, extractor, defaults } = draft;292 const id = meta.id.trim() || slugify(domainOf(fetch.url) || "new-connector") || "new-connector";293 const domain = meta.domain.trim() || domainOf(fetch.url) || "example.com";294 const exName = extractor.name.trim() || extractor.kind;295 const seeds = fetch.url ? [{ url: fetch.url, group: meta.scheduleGroup.trim() || "seed", pageType: extractor.kind === "facility" ? "facility_page" : extractor.kind === "cloud_region" ? "cloud_region" : extractor.kind === "news_event" ? "press_release" : undefined }] : [];296 const cfg: Record<string, unknown> = {297 id,298 name: meta.name.trim() || domain,299 domain,300 kind: meta.kind || "operator",301 mode: "html",302 priority: 3,303 enabled: false,304 license: meta.license.trim() || undefined,305 attribution: meta.attribution.trim() || undefined,306 homepage: meta.homepage.trim() || (fetch.url ? `${new URL(fetch.url).origin}/` : undefined),307 fetch: { level: fetch.level, maxLevel: Math.max(fetch.level, fetch.maxLevel), renderJs: fetch.renderJs || undefined, userAgent: fetch.userAgent === "browser" ? "browser" : undefined, waitForSelector: fetch.waitForSelector.trim() || undefined, rpm: numOrUndef(meta.rpm) ?? 20, concurrency: 2, respectRobots: true, maxCreditsPerRun: numOrUndef(meta.maxCreditsPerRun) ?? 40 },308 discovery: { sitemap: false, seeds, include: domain ? [`^https?://${domain.replace(/\./g, "\\.")}/`] : [], exclude: ["\\?"], maxUrlsPerRun: 500 },309 schedule: { [meta.scheduleGroup.trim() || "seed"]: meta.scheduleInterval.trim() || "weekly" },310 extractors: { [exName]: extractorToConfig(extractor, { includeMatch: true }) },311 defaults: defaultsToConfig(defaults),312 parserVersion: "v1",313 };314 return cfg;315}316317export function buildYaml(draft: Draft): string {318 const cfg = buildConfigObject(draft);319 const header = `# ${cfg.name} — generated by the DataCenterIndex connector dev tool\n# Set enabled: true once \`pnpm dci run ${cfg.id} --dry-run --limit 5\` yields valid entities. Document the source in docs/connectors/${cfg.id}.md.\n`;320 return header + toYaml(cfg);321}322323/** Populate a draft from a parsed connector config (as returned by devtool/validate-config) + its raw YAML text. */324export function draftFromConfig(cfg: Record<string, unknown>, yaml: string, prev: Draft = EMPTY_DRAFT): Draft {325 const str = (v: unknown, d = "") => (typeof v === "string" ? v : typeof v === "number" ? String(v) : d);326 const fetch = (cfg.fetch ?? {}) as Record<string, unknown>;327 const discovery = (cfg.discovery ?? {}) as Record<string, unknown>;328 const seeds = Array.isArray(discovery.seeds) ? (discovery.seeds as Array<string | { url?: string; group?: string }>) : [];329 const firstSeed = seeds[0];330 const seedUrl = typeof firstSeed === "string" ? firstSeed : (firstSeed?.url ?? "");331 const schedule = (cfg.schedule ?? {}) as Record<string, string>;332 const [group, interval] = Object.entries(schedule)[0] ?? ["seed", "weekly"];333 const extractors = (cfg.extractors ?? {}) as Record<string, Record<string, unknown>>;334 const exEntries = Object.entries(extractors);335 const chosen = exEntries.find(([, e]) => e.fields && Object.keys(e.fields as object).length) ?? exEntries[0];336 const [exName, ex] = chosen ?? ["facility", {}];337 const fields = Object.entries((ex.fields ?? {}) as Record<string, unknown>).map(([n, r]) => configToRule(n, r));338 const defaults = (cfg.defaults ?? {}) as Record<string, unknown>;339 return {340 ...prev,341 fetch: { url: prev.fetch.url || seedUrl, level: Number(fetch.level ?? 1) || 1, maxLevel: Number(fetch.maxLevel ?? 2) || 2, renderJs: fetch.renderJs === true, userAgent: fetch.userAgent === "browser" ? "browser" : "bot", waitForSelector: str(fetch.waitForSelector) },342 extractor: { name: exName, kind: str(ex.kind, "facility"), key: str(ex.key), each: str(ex.each), minCertainty: str(ex.minCertainty), parser: str(ex.parser), match: Array.isArray(ex.match) ? (ex.match as string[]).join("\n") : "", fields: fields.length ? fields : [newFieldRule({ name: "name", selector: "h1" })] },343 defaults: { operatorName: str(defaults.operatorName), operatorKind: str(defaults.operatorKind), countryIso2: str(defaults.countryIso2), facilityType: str(defaults.facilityType), status: str(defaults.status) },344 meta: { id: str(cfg.id), name: str(cfg.name), domain: str(cfg.domain), kind: str(cfg.kind, "operator"), license: str(cfg.license), attribution: str(cfg.attribution), homepage: str(cfg.homepage), scheduleGroup: group ?? "seed", scheduleInterval: interval ?? "weekly", rpm: str(fetch.rpm, "20"), maxCreditsPerRun: str(fetch.maxCreditsPerRun, "40") },345 yaml,346 yamlDirty: false,347 };348}349350/* ------------------------------------------------------------------ persistence */351352export const DRAFT_KEY = "dci:devtool:draft:v1";353354export function loadDraft(): Draft | null {355 try {356 const raw = window.localStorage.getItem(DRAFT_KEY);357 if (!raw) return null;358 const parsed = JSON.parse(raw) as Partial<Draft>;359 return { ...EMPTY_DRAFT, ...parsed, fetch: { ...EMPTY_DRAFT.fetch, ...(parsed.fetch ?? {}) }, extractor: { ...EMPTY_DRAFT.extractor, ...(parsed.extractor ?? {}), fields: (parsed.extractor?.fields ?? EMPTY_DRAFT.extractor.fields).map((f) => newFieldRule(f)) }, defaults: { ...EMPTY_DRAFT.defaults, ...(parsed.defaults ?? {}) }, meta: { ...EMPTY_DRAFT.meta, ...(parsed.meta ?? {}) } };360 } catch {361 return null;362 }363}364365export function saveDraft(d: Draft): void {366 try {367 window.localStorage.setItem(DRAFT_KEY, JSON.stringify({ ...d, savedAt: Date.now() }));368 } catch {369 /* quota — ignore */370 }371}372373/** Union of entity field names present (non-empty) across a list of entities, in a stable order. */374export function fieldUnion(entities: Array<Record<string, unknown>>, preferred: string[] = []): string[] {375 const seen = new Set<string>();376 const out: string[] = [];377 for (const p of preferred) {378 if (entities.some((e) => p in e)) {379 seen.add(p);380 out.push(p);381 }382 }383 for (const e of entities) {384 for (const k of Object.keys(e)) {385 if (!seen.has(k) && k !== "provenance") {386 seen.add(k);387 out.push(k);388 }389 }390 }391 return out;392}393394export function isEmptyValue(v: unknown): boolean {395 return v === null || v === undefined || v === "" || (Array.isArray(v) && v.length === 0);396}397