/**
* Connector dev tool — draft model, conversions to/from the YAML config schema (packages/connectors/src/config.ts),
* template builder and localStorage persistence. Pure functions, no React.
*/
import type { DevtoolFetchResult, DevtoolPreviewResult, RunSampleResult, SaveConfigResult, ValidateConfigResult } from "@/lib/admin-api";
import { toYaml } from "../yaml";
export type RuleType = "selector" | "attr" | "jsonld" | "meta" | "jsonPath" | "regex" | "constant";
export const RULE_TYPES: Array<{ value: RuleType; label: string; hint: string }> = [
{ value: "selector", label: "CSS selector", hint: "text of the first (or all) matching elements" },
{ value: "attr", label: "Attribute", hint: "attribute value of a selected element (href, src, data-*)" },
{ value: "jsonld", label: "JSON-LD", hint: "value inside a JSON-LD object of a given @type, via JSON path" },
{ value: "meta", label: "Meta tag", hint: " content" },
{ value: "jsonPath", label: "Embedded JSON", hint: "JSON path over embedded JSON blobs (__NEXT_DATA__, drupal-settings…)" },
{ value: "regex", label: "Regex", hint: "regular expression over the element text (or the whole page)" },
{ value: "constant", label: "Constant", hint: "fixed default value" },
];
export const TRANSFORMS = ["", "mw", "sqm", "ha", "money", "date", "status", "type", "country", "int", "float", "trim", "lower", "url", "bool"] as const;
export const EXTRACT_KINDS = ["facility", "operator", "campus", "cloud_region", "ixp", "project", "news_event", "country"] as const;
export const SOURCE_KINDS_LIST = ["operator", "government", "filing", "utility", "dataset", "community", "secondary", "news", "cloud_provider", "registry"] as const;
export const COMMON_FIELDS: Record = {
facility: ["name", "address", "city", "regionName", "countryIso2", "postalCode", "lat", "lng", "status", "facilityType", "itCapacityMw", "totalPowerMw", "plannedPowerMw", "buildingSqm", "siteAreaHa", "openedOn", "operatorName", "website", "description", "tier", "pue", "coolingType", "renewableClaim", "certifications"],
operator: ["name", "kind", "website", "hqCountryIso2", "description"],
campus: ["name", "address", "city", "countryIso2", "lat", "lng", "operatorName", "plannedPowerMw"],
cloud_region: ["code", "name", "city", "countryIso2", "lat", "lng", "availabilityZones", "launchedOn", "status", "providerName"],
ixp: ["name", "nameLong", "city", "countryIso2", "website", "networkCount"],
project: ["name", "operatorName", "city", "countryIso2", "status", "announcedOn", "expectedOpening", "plannedMw", "investmentUsd", "description"],
news_event: ["title", "summary", "publishedAt", "url", "operatorName", "countryIso2", "mw", "eventType"],
country: ["iso2", "name"],
};
export interface FieldRule {
id: string;
name: string;
type: RuleType;
selector: string;
attr: string;
regex: string;
regexFlags: string;
group: string;
all: boolean;
jsonPath: string;
jsonld: string;
meta: string;
transform: string;
default: string;
}
export interface ExtractorDraft {
name: string;
kind: string;
key: string;
each: string;
minCertainty: string;
parser: string;
match: string;
fields: FieldRule[];
}
export interface FetchDraft {
url: string;
level: number;
maxLevel: number;
renderJs: boolean;
userAgent: "bot" | "browser";
waitForSelector: string;
}
export interface MetaDraft {
id: string;
name: string;
domain: string;
kind: string;
license: string;
attribution: string;
homepage: string;
scheduleGroup: string;
scheduleInterval: string;
rpm: string;
maxCreditsPerRun: string;
}
export interface DefaultsDraft {
operatorName: string;
operatorKind: string;
countryIso2: string;
facilityType: string;
status: string;
}
export interface Draft {
fetch: FetchDraft;
extractor: ExtractorDraft;
defaults: DefaultsDraft;
meta: MetaDraft;
/** full YAML in the editor (step 4); `yamlDirty` = edited by hand since the last regeneration */
yaml: string;
yamlDirty: boolean;
sampleUrls: string;
savedAt: number | null;
}
export interface DevtoolShared {
draft: Draft;
update: (patch: Partial | ((d: Draft) => Draft)) => void;
fetchResult: DevtoolFetchResult | null;
setFetchResult: (r: DevtoolFetchResult | null) => void;
previewResult: DevtoolPreviewResult | null;
setPreviewResult: (r: DevtoolPreviewResult | null) => void;
validation: ValidateConfigResult | null;
setValidation: (r: ValidateConfigResult | null) => void;
saveResult: SaveConfigResult | null;
setSaveResult: (r: SaveConfigResult | null) => void;
sampleResult: RunSampleResult | null;
setSampleResult: (r: RunSampleResult | null) => void;
goto: (step: number) => void;
}
let seq = 0;
export function uid(): string {
seq += 1;
return `${Date.now().toString(36)}${seq.toString(36)}`;
}
export function newFieldRule(partial: Partial = {}): FieldRule {
return { id: uid(), name: "", type: "selector", selector: "", attr: "", regex: "", regexFlags: "", group: "", all: false, jsonPath: "", jsonld: "", meta: "", transform: "", default: "", ...partial };
}
export const EMPTY_DRAFT: Draft = {
fetch: { url: "", level: 1, maxLevel: 2, renderJs: false, userAgent: "bot", waitForSelector: "" },
extractor: { name: "facility", kind: "facility", key: "{url}", each: "", minCertainty: "", parser: "", match: "", fields: [newFieldRule({ name: "name", type: "selector", selector: "h1" })] },
defaults: { operatorName: "", operatorKind: "", countryIso2: "", facilityType: "", status: "" },
meta: { id: "", name: "", domain: "", kind: "operator", license: "", attribution: "", homepage: "", scheduleGroup: "seed", scheduleInterval: "weekly", rpm: "20", maxCreditsPerRun: "40" },
yaml: "",
yamlDirty: false,
sampleUrls: "",
savedAt: null,
};
/* ------------------------------------------------------------------ helpers */
export function domainOf(url: string): string {
try {
return new URL(url).hostname;
} catch {
return "";
}
}
export function slugify(s: string): string {
return s
.toLowerCase()
.replace(/^www\./, "")
.replace(/\.[a-z]{2,}(\.[a-z]{2})?$/, "")
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "")
.slice(0, 40);
}
export function parseUrls(text: string): string[] {
const out: string[] = [];
for (const line of text.split(/\r?\n|,|\s+/)) {
const s = line.trim();
if (!s) continue;
try {
const u = new URL(s);
if (u.protocol === "http:" || u.protocol === "https:") out.push(u.toString());
} catch {
/* skip */
}
}
return [...new Set(out)];
}
function numOrUndef(s: string): number | undefined {
const n = Number(s);
return s.trim() !== "" && Number.isFinite(n) ? n : undefined;
}
function coerceDefault(s: string): unknown {
const t = s.trim();
if (t === "") return undefined;
if (t === "true") return true;
if (t === "false") return false;
if (t === "null") return null;
if (/^-?\d+(\.\d+)?$/.test(t)) return Number(t);
return s;
}
/* ------------------------------------------------------------------ rule ↔ config */
/** One field rule → schema `selectorRule` (bare string when it is just a selector). */
export function ruleToConfig(r: FieldRule): string | Record | null {
const o: Record = {};
switch (r.type) {
case "selector":
if (!r.selector.trim()) return null;
o.selector = r.selector.trim();
if (r.all) o.all = true;
if (r.regex.trim()) {
o.regex = r.regex;
if (r.regexFlags.trim()) o.regexFlags = r.regexFlags.trim();
if (numOrUndef(r.group) !== undefined) o.group = numOrUndef(r.group);
}
break;
case "attr":
if (!r.selector.trim() || !r.attr.trim()) return null;
o.selector = r.selector.trim();
o.attr = r.attr.trim();
if (r.all) o.all = true;
break;
case "jsonld":
if (!r.jsonld.trim() && !r.jsonPath.trim()) return null;
if (r.jsonld.trim()) o.jsonld = r.jsonld.trim();
if (r.jsonPath.trim()) o.jsonPath = r.jsonPath.trim();
break;
case "meta":
if (!r.meta.trim()) return null;
o.meta = r.meta.trim();
break;
case "jsonPath":
if (!r.jsonPath.trim()) return null;
o.jsonPath = r.jsonPath.trim();
break;
case "regex":
if (!r.regex.trim()) return null;
o.regex = r.regex;
if (r.regexFlags.trim()) o.regexFlags = r.regexFlags.trim();
if (numOrUndef(r.group) !== undefined) o.group = numOrUndef(r.group);
if (r.selector.trim()) o.selector = r.selector.trim();
if (r.all) o.all = true;
break;
case "constant":
if (r.default.trim() === "") return null;
break;
}
if (r.transform) o.transform = r.transform;
const def = coerceDefault(r.default);
if (def !== undefined) o.default = def;
const keys = Object.keys(o);
if (keys.length === 1 && keys[0] === "selector") return o.selector as string;
return o;
}
/** schema `selectorRule` → field rule (best effort classification). */
export function configToRule(name: string, rule: unknown): FieldRule {
if (typeof rule === "string") return newFieldRule({ name, type: "selector", selector: rule });
const o = (rule ?? {}) as Record;
const str = (k: string) => (typeof o[k] === "string" ? (o[k] as string) : typeof o[k] === "number" ? String(o[k]) : "");
const base: Partial = { name, selector: str("selector"), attr: str("attr"), regex: str("regex"), regexFlags: str("regexFlags"), group: str("group"), all: o.all === true, jsonPath: str("jsonPath"), jsonld: str("jsonld"), meta: str("meta"), transform: str("transform"), default: o.default === undefined ? "" : typeof o.default === "string" ? o.default : JSON.stringify(o.default) };
let type: RuleType = "selector";
if (o.jsonld) type = "jsonld";
else if (o.meta) type = "meta";
else if (o.jsonPath) type = "jsonPath";
else if (o.attr) type = "attr";
else if (o.regex && !o.selector) type = "regex";
else if (o.selector) type = "selector";
else if (o.default !== undefined) type = "constant";
return newFieldRule({ ...base, type });
}
export function extractorToConfig(ex: ExtractorDraft, opts: { includeMatch?: boolean } = {}): Record {
const fields: Record = {};
for (const f of ex.fields) {
const name = f.name.trim();
if (!name) continue;
const cfg = ruleToConfig(f);
if (cfg !== null) fields[name] = cfg;
}
const o: Record = { kind: ex.kind };
if (opts.includeMatch && ex.match.trim()) o.match = ex.match.split(/\r?\n/).map((s) => s.trim()).filter(Boolean);
if (ex.parser.trim()) o.parser = ex.parser.trim();
if (ex.key.trim()) o.key = ex.key.trim();
if (ex.each.trim()) o.each = ex.each.trim();
if (Object.keys(fields).length) o.fields = fields;
const mc = numOrUndef(ex.minCertainty);
if (mc !== undefined) o.minCertainty = mc;
return o;
}
export function defaultsToConfig(d: DefaultsDraft): Record | undefined {
const o: Record = {};
for (const [k, v] of Object.entries(d)) if (v.trim()) o[k] = v.trim();
return Object.keys(o).length ? o : undefined;
}
/** Full connector config object from the draft (what the YAML template serializes). */
export function buildConfigObject(draft: Draft): Record {
const { meta, fetch, extractor, defaults } = draft;
const id = meta.id.trim() || slugify(domainOf(fetch.url) || "new-connector") || "new-connector";
const domain = meta.domain.trim() || domainOf(fetch.url) || "example.com";
const exName = extractor.name.trim() || extractor.kind;
const seeds = fetch.url ? [{ url: fetch.url, group: meta.scheduleGroup.trim() || "seed", pageType: extractor.kind === "facility" ? "facility_page" : extractor.kind === "cloud_region" ? "cloud_region" : extractor.kind === "news_event" ? "press_release" : undefined }] : [];
const cfg: Record = {
id,
name: meta.name.trim() || domain,
domain,
kind: meta.kind || "operator",
mode: "html",
priority: 3,
enabled: false,
license: meta.license.trim() || undefined,
attribution: meta.attribution.trim() || undefined,
homepage: meta.homepage.trim() || (fetch.url ? `${new URL(fetch.url).origin}/` : undefined),
fetch: { level: fetch.level, maxLevel: Math.max(fetch.level, fetch.maxLevel), renderJs: fetch.renderJs || undefined, userAgent: fetch.userAgent === "browser" ? "browser" : undefined, waitForSelector: fetch.waitForSelector.trim() || undefined, rpm: numOrUndef(meta.rpm) ?? 20, concurrency: 2, respectRobots: true, maxCreditsPerRun: numOrUndef(meta.maxCreditsPerRun) ?? 40 },
discovery: { sitemap: false, seeds, include: domain ? [`^https?://${domain.replace(/\./g, "\\.")}/`] : [], exclude: ["\\?"], maxUrlsPerRun: 500 },
schedule: { [meta.scheduleGroup.trim() || "seed"]: meta.scheduleInterval.trim() || "weekly" },
extractors: { [exName]: extractorToConfig(extractor, { includeMatch: true }) },
defaults: defaultsToConfig(defaults),
parserVersion: "v1",
};
return cfg;
}
export function buildYaml(draft: Draft): string {
const cfg = buildConfigObject(draft);
const header = `# ${cfg.name} — generated by the DataCenterIndex connector dev tool\n# Set enabled: true once \`pnpm dci run ${cfg.id} --dry-run --limit 5\` yields valid entities. Document the source in docs/connectors/${cfg.id}.md.\n`;
return header + toYaml(cfg);
}
/** Populate a draft from a parsed connector config (as returned by devtool/validate-config) + its raw YAML text. */
export function draftFromConfig(cfg: Record, yaml: string, prev: Draft = EMPTY_DRAFT): Draft {
const str = (v: unknown, d = "") => (typeof v === "string" ? v : typeof v === "number" ? String(v) : d);
const fetch = (cfg.fetch ?? {}) as Record;
const discovery = (cfg.discovery ?? {}) as Record;
const seeds = Array.isArray(discovery.seeds) ? (discovery.seeds as Array) : [];
const firstSeed = seeds[0];
const seedUrl = typeof firstSeed === "string" ? firstSeed : (firstSeed?.url ?? "");
const schedule = (cfg.schedule ?? {}) as Record;
const [group, interval] = Object.entries(schedule)[0] ?? ["seed", "weekly"];
const extractors = (cfg.extractors ?? {}) as Record>;
const exEntries = Object.entries(extractors);
const chosen = exEntries.find(([, e]) => e.fields && Object.keys(e.fields as object).length) ?? exEntries[0];
const [exName, ex] = chosen ?? ["facility", {}];
const fields = Object.entries((ex.fields ?? {}) as Record).map(([n, r]) => configToRule(n, r));
const defaults = (cfg.defaults ?? {}) as Record;
return {
...prev,
fetch: { url: prev.fetch.url || seedUrl, level: Number(fetch.level ?? 1) || 1, maxLevel: Number(fetch.maxLevel ?? 2) || 2, renderJs: fetch.renderJs === true, userAgent: fetch.userAgent === "browser" ? "browser" : "bot", waitForSelector: str(fetch.waitForSelector) },
extractor: { name: exName, kind: str(ex.kind, "facility"), key: str(ex.key), each: str(ex.each), minCertainty: str(ex.minCertainty), parser: str(ex.parser), match: Array.isArray(ex.match) ? (ex.match as string[]).join("\n") : "", fields: fields.length ? fields : [newFieldRule({ name: "name", selector: "h1" })] },
defaults: { operatorName: str(defaults.operatorName), operatorKind: str(defaults.operatorKind), countryIso2: str(defaults.countryIso2), facilityType: str(defaults.facilityType), status: str(defaults.status) },
meta: { id: str(cfg.id), name: str(cfg.name), domain: str(cfg.domain), kind: str(cfg.kind, "operator"), license: str(cfg.license), attribution: str(cfg.attribution), homepage: str(cfg.homepage), scheduleGroup: group ?? "seed", scheduleInterval: interval ?? "weekly", rpm: str(fetch.rpm, "20"), maxCreditsPerRun: str(fetch.maxCreditsPerRun, "40") },
yaml,
yamlDirty: false,
};
}
/* ------------------------------------------------------------------ persistence */
export const DRAFT_KEY = "dci:devtool:draft:v1";
export function loadDraft(): Draft | null {
try {
const raw = window.localStorage.getItem(DRAFT_KEY);
if (!raw) return null;
const parsed = JSON.parse(raw) as Partial;
return { ...EMPTY_DRAFT, ...parsed, fetch: { ...EMPTY_DRAFT.fetch, ...(parsed.fetch ?? {}) }, extractor: { ...EMPTY_DRAFT.extractor, ...(parsed.extractor ?? {}), fields: (parsed.extractor?.fields ?? EMPTY_DRAFT.extractor.fields).map((f) => newFieldRule(f)) }, defaults: { ...EMPTY_DRAFT.defaults, ...(parsed.defaults ?? {}) }, meta: { ...EMPTY_DRAFT.meta, ...(parsed.meta ?? {}) } };
} catch {
return null;
}
}
export function saveDraft(d: Draft): void {
try {
window.localStorage.setItem(DRAFT_KEY, JSON.stringify({ ...d, savedAt: Date.now() }));
} catch {
/* quota — ignore */
}
}
/** Union of entity field names present (non-empty) across a list of entities, in a stable order. */
export function fieldUnion(entities: Array>, preferred: string[] = []): string[] {
const seen = new Set();
const out: string[] = [];
for (const p of preferred) {
if (entities.some((e) => p in e)) {
seen.add(p);
out.push(p);
}
}
for (const e of entities) {
for (const k of Object.keys(e)) {
if (!seen.has(k) && k !== "provenance") {
seen.add(k);
out.push(k);
}
}
}
return out;
}
export function isEmptyValue(v: unknown): boolean {
return v === null || v === undefined || v === "" || (Array.isArray(v) && v.length === 0);
}