spb/datacenterindex
Public
HTML 53.9%
TypeScript 44.5%
JavaScript 0.6%
SQL 0.5%
1import { load, type CheerioAPI } from "cheerio";2import type { ExtractedRecord, RawDocument } from "@dci/connectors";3import { metaDescription } from "@dci/connectors";4import { cleanText, countryFromText, htmlToText, parseMw, parseAreaHa, parseAreaSqm } from "@dci/core";56/**7 * Shared helpers for operator connectors (batch 1). Pure functions over page HTML/text; no network.8 * Rule: we only surface what the page publishes. No geocoding, no guessed capacities.9 */1011export const US_STATES = new Set(["AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA", "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY", "DC"]);12export const CA_PROVINCES = new Set(["AB", "BC", "MB", "NB", "NL", "NS", "NT", "NU", "ON", "PE", "QC", "SK", "YT"]);1314const US_STATE_NAMES: Record<string, string> = { alabama: "AL", alaska: "AK", arizona: "AZ", arkansas: "AR", california: "CA", colorado: "CO", connecticut: "CT", delaware: "DE", florida: "FL", georgia: "GA", hawaii: "HI", idaho: "ID", illinois: "IL", indiana: "IN", iowa: "IA", kansas: "KS", kentucky: "KY", louisiana: "LA", maine: "ME", maryland: "MD", massachusetts: "MA", michigan: "MI", minnesota: "MN", mississippi: "MS", missouri: "MO", montana: "MT", nebraska: "NE", nevada: "NV", "new hampshire": "NH", "new jersey": "NJ", "new mexico": "NM", "new york": "NY", "north carolina": "NC", "north dakota": "ND", ohio: "OH", oklahoma: "OK", oregon: "OR", pennsylvania: "PA", "rhode island": "RI", "south carolina": "SC", "south dakota": "SD", tennessee: "TN", texas: "TX", utah: "UT", vermont: "VT", virginia: "VA", washington: "WA", "west virginia": "WV", wisconsin: "WI", wyoming: "WY" };1516/** "Nevada" → "NV"; "VA" → "VA"; unknown → null. */17export function usStateCode(s: string | null | undefined): string | null {18 if (!s) return null;19 const t = s.trim();20 if (US_STATES.has(t.toUpperCase()) && t.length === 2) return t.toUpperCase();21 return US_STATE_NAMES[t.toLowerCase()] ?? null;22}2324export interface ParsedAddress { address: string | null; city: string | null; region: string | null; postal: string | null; countryIso2: string | null }2526const STREET_SUFFIX = /^(.*\b(?:Drive|Dr|Road|Rd|Parkway|Pkwy|Court|Ct|Boulevard|Blvd|Street|St|Avenue|Ave|Way|Lane|Ln|Place|Pl|Circle|Cir|Trail|Trl|Highway|Hwy|Loop|Terrace|Ter|Plaza|Pike|Turnpike|Tpke|Expressway|Expy|Freeway|Fwy|Route|Rte|Square|Sq|Row|Run|Path|Alley|Center|Centre)\.?)(?:\s*\([^)]*\))?\s+(.+)$/i;2728/** North-American postal address: "21715 Filigree Court, Ashburn, VA 20147" / "22435 Glenn Drive Sterling, VA 20164 USA" / "123 Rue X, Montréal, QC H1A 1A1". */29export function parseNaAddress(raw: string | null | undefined): ParsedAddress | null {30 const s = cleanText(raw?.replace(/<[^>]+>/g, " "));31 if (!s) return null;32 // "Itasca,IL 60143" (a <br> collapsed to nothing after the comma) is accepted: the state is anchored by the postal code that follows33 const m = s.match(/^(.+?)(?:,\s*|\s+)([A-Z]{2})[,\s]+(\d{5}(?:-\d{4})?|[A-Z]\d[A-Z]\s?\d[A-Z]\d)\b(?:[\s,]*(?:USA|U\.S\.A\.|United States|US|Canada))?\.?$/);34 if (!m) return null;35 const head = m[1]!.trim();36 const region = m[2]!;37 const postal = m[3]!;38 const isCa = CA_PROVINCES.has(region) && /^[A-Z]\d[A-Z]/.test(postal);39 if (!isCa && !US_STATES.has(region)) return null;40 let street: string | null = head;41 let city: string | null = null;42 const lastComma = head.lastIndexOf(",");43 if (lastComma > 0) { street = head.slice(0, lastComma).trim(); city = head.slice(lastComma + 1).trim(); }44 else { const sm = head.match(STREET_SUFFIX); if (sm) { street = sm[1]!.trim(); city = sm[2]!.trim(); } }45 return { address: street || null, city: city || null, region, postal, countryIso2: isCa ? "CA" : "US" };46}4748/** European-style: "Hanauer Landstrasse 302, 60314 Frankfurt am Main, Germany" / "Huibertgatweg 2 9979 XZ Eemshaven , Netherlands". */49export function parseEuAddress(raw: string | null | undefined): ParsedAddress | null {50 const s = cleanText(raw?.replace(/<[^>]+>/g, " "));51 if (!s) return null;52 const m = s.match(/^(.+?),?\s+((?:[A-Z]{1,2}-)?\d{4,6}(?:\s?[A-Z]{2})?)\s+([^,]+?)\s*,\s*([A-Za-z .'-]+)$/);53 if (m) {54 const countryIso2 = countryFromText(m[4]!);55 if (countryIso2) return { address: m[1]!.trim(), city: m[3]!.trim(), region: null, postal: m[2]!.trim(), countryIso2 };56 }57 // Comma-separated with the country last: "…, Vienna, 1210, Austria" / "665 Ajax Avenue, Slough Trading Estate, Slough, SL1 4BG, United Kingdom" / "Street 1, City, Country"58 const parts = s.split(",").map((p) => p.trim()).filter(Boolean);59 if (parts.length >= 3) {60 const countryIso2 = countryFromText(parts[parts.length - 1]!);61 if (countryIso2 && !/\d/.test(parts[parts.length - 1]!)) {62 const rest = parts.slice(0, -1);63 const isPostal = (p: string) => /\d/.test(p) && p.length <= 10 && !/^\d+\s+[A-Za-z]/.test(p);64 let postal: string | null = null, city: string | null = null;65 if (rest.length >= 2 && isPostal(rest[rest.length - 1]!)) { postal = rest.pop()!; city = rest.pop() ?? null; }66 else if (rest.length >= 2) { city = rest.pop() ?? null; }67 // "Frankfurt am Main 60486" / "60486 Frankfurt am Main" inside the city token68 const cm = city?.match(/^(?:((?:[A-Z]{1,2}-)?\d{4,6})\s+(.+)|(.+?)\s+((?:[A-Z]{1,2}-)?\d{4,6}))$/);69 if (cm) { postal ??= (cm[1] ?? cm[4]) ?? null; city = (cm[2] ?? cm[3]) ?? city; }70 if (city && /\d/.test(city) && !/^\d/.test(city)) { /* keep */ }71 const street = rest.join(", ");72 if (city && /\d/.test(street + (postal ?? ""))) return { address: street || null, city, region: null, postal, countryIso2 };73 }74 }75 return null;76}7778/** Best-effort structured address; falls back to the raw string + country detected in it. */79export function parseAddress(raw: string | null | undefined): ParsedAddress | null {80 const s = cleanText(raw?.replace(/<[^>]+>/g, " "));81 if (!s) return null;82 return parseNaAddress(s) ?? parseEuAddress(s) ?? { address: s, city: null, region: null, postal: null, countryIso2: countryFromText(s) };83}8485/** "Sterling, VA" / "Frankfurt, Germany" / "London, UK" → city, region, country. */86export function parseCityLine(raw: string | null | undefined): { city: string | null; region: string | null; countryIso2: string | null } {87 const s = cleanText(raw);88 if (!s) return { city: null, region: null, countryIso2: null };89 const parts = s.split(",").map((p) => p.trim()).filter(Boolean);90 const city = parts[0] ?? null;91 const tail = parts[parts.length - 1] ?? "";92 if (parts.length >= 2) {93 const st = usStateCode(tail);94 if (st) return { city, region: st, countryIso2: "US" };95 if (CA_PROVINCES.has(tail.toUpperCase()) && tail.length === 2) return { city, region: tail.toUpperCase(), countryIso2: "CA" };96 const c = countryFromText(tail);97 if (c) return { city, region: parts.length >= 3 ? parts[1] ?? null : null, countryIso2: c };98 }99 return { city, region: null, countryIso2: countryFromText(s) };100}101102/** First MW figure appearing within `window` chars after a label. */103export function mwAfter(text: string, label: RegExp, window = 160): number | null {104 const m = text.match(label);105 if (!m || m.index == null) return null;106 const seg = text.slice(m.index + m[0].length, m.index + m[0].length + window);107 return parseMw(seg);108}109110/** First MW figure immediately followed by `ctx` (anchored right after the figure, e.g. /\s*of critical IT load/). */111export function mwWithContext(text: string, ctx: RegExp, after = 60): number | null {112 const anchored = new RegExp(`^(?:${ctx.source})`, ctx.flags.replace("g", ""));113 for (const m of text.matchAll(/(\d{1,3}(?:[.,]\d{3})+|\d+(?:[.,]\d+)?)\s*(?:MW|megawatts?)\b\+?/gi)) {114 const i = m.index ?? 0;115 const tail = text.slice(i + m[0].length, i + m[0].length + after);116 if (anchored.test(tail)) return parseMw(m[0]);117 }118 return null;119}120121/** "800K square feet" / "2.4 million sqft" / "105,000 SQ FT" / "18,000 sq. m." → m². */122export function areaSqm(text: string | null | undefined): number | null {123 if (!text) return null;124 const k = text.match(/(\d+(?:\.\d+)?)\s*[Kk]\b\s*(?:square\s*feet|sq\.?\s*ft|sqft|SF)\b/i);125 if (k) return Math.round(Number(k[1]) * 1000 * 0.09290304);126 const mm = text.match(/(\d+(?:\.\d+)?)\s*[Mm]\b\s*(?:square\s*feet|sq\.?\s*ft|sqft|SF)\b/);127 if (mm) return Math.round(Number(mm[1]) * 1_000_000 * 0.09290304);128 const sf = text.match(/(\d{1,3}(?:,\d{3})+|\d+)\s*SF\b/);129 if (sf) return Math.round(Number(sf[1]!.replace(/,/g, "")) * 0.09290304);130 return parseAreaSqm(text.replace(/sq\.\s*m\./i, "sqm"));131}132133export function areaHa(text: string | null | undefined): number | null { return parseAreaHa(text ?? ""); }134135/** Facility code like DC2, FRA5, IAD12, NVA8, ATL01A. */136export const CODE_RE = /^[A-Z]{2,4}\d{1,3}[A-Z]?$/;137export function extractCode(s: string | null | undefined): string | null {138 if (!s) return null;139 const m = s.toUpperCase().match(/\b([A-Z]{2,4}\d{1,3}[A-Z]?)\b/);140 return m ? m[1]! : null;141}142143export function lastSlug(url: string): string {144 const u = url.split(/[?#]/)[0]!.replace(/\/+$/, "");145 return decodeURIComponent(u.slice(u.lastIndexOf("/") + 1)).toLowerCase();146}147148export function pageHtml(doc: RawDocument): string { return doc.text; }149export function $of(doc: RawDocument): CheerioAPI { return load(doc.text); }150export function bodyText(html: string): string { return htmlToText(html); }151export function description(html: string): string | null { return metaDescription(html); }152153/** Standard "coming soon / under construction" detection on a facility page's headline area only. */154export function statusFromText(text: string | null | undefined): "announced" | "under_construction" | null {155 if (!text) return null;156 if (/\b(under construction|currently being built|breaking ground|broke ground)\b/i.test(text)) return "under_construction";157 if (/\b(coming soon|opening (in|soon)|future (campus|data center|facility)|planned (campus|data center|facility)|now pre-?leasing|is developing|under development|in development)\b/i.test(text)) return "announced";158 return null;159}160161/** Drop nulls / empty strings so `methods` only lists fields that exist. */162export function compact(data: Record<string, unknown>, methods: Record<string, string>): { data: Record<string, unknown>; methods: Record<string, string> } {163 const d: Record<string, unknown> = {};164 const m: Record<string, string> = {};165 for (const [k, v] of Object.entries(data)) {166 if (v == null || v === "" || (Array.isArray(v) && !v.length)) continue;167 d[k] = v;168 if (methods[k]) m[k] = methods[k]!;169 }170 return { data: d, methods: m };171}172173export function record(kind: ExtractedRecord["kind"], key: string, url: string, data: Record<string, unknown>, methods: Record<string, string>, certainty = 0.9): ExtractedRecord {174 const c = compact(data, methods);175 return { kind, key, url, data: { ...c.data, website: url }, methods: c.methods, certainty, pageType: "facility_page" };176}177178/** Certification-looking tokens from a list of strings (drops "target"/"pending" items). */179export function certificationsFrom(items: string[]): string[] {180 const out = new Set<string>();181 for (const raw of items) {182 const t = cleanText(raw);183 if (!t || /\b(target|pending|planned|in progress)\b/i.test(t)) continue;184 const m = t.match(/\b(ISO\/IEC\s?\d{4,5}(?:-\d)?|ISO\s?\d{4,5}(?:-\d)?|SOC\s?[123](?:\s?Type\s?(?:I{1,2}|[12]))?|PCI[- ]?DSS|HIPAA|HITRUST|FedRAMP(?:\s\w+)?|NIST\s?800-53(?:\/FISMA\s\w+)?|FISMA(?:\s\w+)?|LEED(?:\s\w+)?|BREEAM(?:\s"?\w+"?)?|Uptime(?:\sInstitute)?\s(?:Tier\s(?:I{1,3}V?|[1-4])[\w\s]*)|HECVAT|ITAR|GDPR|C5|TISAX|EN\s?50600|SS\s?564|MTCS(?:\s\w+)?|OSPAR|CSA\sSTAR|ENERGY\sSTAR|ISAE\s?3402|IRAP|TIA-942(?:\s\w+)?)\b/i);185 if (m) out.add(m[1]!.replace(/\s+/g, " ").trim());186 }187 return [...out];188}189190export const KNOWN_CLOUDS = [/\bAWS\b|Amazon Web Services/i, /\bAzure\b|Microsoft/i, /Google Cloud|\bGCP\b|\bGoogle\b/i, /IBM Cloud/i, /Oracle(?:\sCloud|\sFastConnect)?/i, /Alibaba Cloud/i, /Salesforce/i, /SAP/i, /Tencent/i, /Huawei Cloud/i];191export const CLOUD_NAMES = ["AWS", "Microsoft Azure", "Google Cloud", "IBM Cloud", "Oracle Cloud", "Alibaba Cloud", "Salesforce", "SAP", "Tencent Cloud", "Huawei Cloud"];192export function cloudsIn(text: string | null | undefined): string[] {193 if (!text) return [];194 const out: string[] = [];195 KNOWN_CLOUDS.forEach((re, i) => { if (re.test(text)) out.push(CLOUD_NAMES[i]!); });196 return out;197}198199export function ixpsIn(text: string | null | undefined): string[] {200 if (!text) return [];201 const out = new Set<string>();202 for (const m of text.matchAll(/\b(Any2(?:\s?Exchange)?(?:\s(?:West|East|Denver|Chicago|Boston|Miami|Atlanta|New York|Los Angeles|Silicon Valley|Northern Virginia))?|LINX(?:\s\w+)?|DE-CIX(?:\s\w+)?|AMS-IX(?:\s\w+)?|NYIIX|SIX(?:\sSeattle)?|Equinix IX|Equinix Internet Exchange|MICE|France-IX|LONAP|NL-ix|JPNAP|JPIX|BBIX|HKIX|SGIX|Megaport IX|NAPAfrica|JINX|Community IX|MIX(?:\sMilan)?|ECIX|Netnod|SwissIX|CIXP|VIX|ESPANIX|GigaPIX|PTT|IX\.br)\b/g)) out.add(m[1]!.replace(/\s+/g, " "));203 return [...out];204}205206/** Text of the first element matching any selector. */207export function firstText($: CheerioAPI, ...selectors: string[]): string | null {208 for (const s of selectors) { const el = $(s).first(); if (el.length) { const t = cleanText(el.text()); if (t) return t; } }209 return null;210}211212/** Cooling keywords → short claim string (only when explicit). */213export function coolingFrom(text: string | null | undefined): string | null {214 if (!text) return null;215 const m = text.match(/\b(direct[- ]to[- ]chip liquid cooling|liquid[- ]cooling|immersion cooling|water[- ]free cooling|zero[- ]water cooling|closed[- ]loop (?:chilled[- ]water|water[- ]cooling|cooling)(?: systems?)?|air[- ]cooled chillers?|free cooling|evaporative cooling|adiabatic cooling|rear[- ]door heat exchangers?|air[- ]side economiz(?:ation|ers?))\b/i);216 return m ? m[1]!.replace(/\s+/g, " ") : null;217}218219/** The clause (bounded by ". ", ", " or " and ") that carries a renewable / carbon-free energy claim; null when none. */220export function renewableFrom(text: string | null | undefined): string | null {221 if (!text) return null;222 const re = /\b(100%\s+(?:renewable|carbon[- ]free|green)|powered by (?:100% )?renewable|renewable (?:energy|power|electricity)(?:\s+(?:credits|options|certificates|sourc\w+|match\w*|procurement|supply))?|carbon[- ]free (?:energy|electricity)|wind (?:RECs?|power|energy)|solar (?:power|PPA|energy)|hydro(?:electric)?(?: power)?|green[- ]e\s+RECs?|RECs\s+from|vPPAs?|power purchase agreement)/i;223 const m = text.match(re);224 if (!m || m.index == null) return null;225 const bounds = [". ", ", ", " and ", "; "].map((sep) => { const p = text.lastIndexOf(sep, m.index!); return p < 0 ? 0 : p + sep.length; });226 const start = Math.max(0, m.index - 80, ...bounds);227 let end = text.length;228 for (const sep of [". ", "; ", ", "]) { const p = text.indexOf(sep, m.index + m[0].length); if (p > 0 && p < end) end = p; }229 end = Math.min(end, m.index + m[0].length + 120);230 const clause = cleanText(text.slice(start, end));231 return clause ? clause.slice(0, 200) : null;232}233234export { cleanText, countryFromText, parseMw };235