API: grid-cluster facilities without country on the world map; word-bounded operator matching in search; country backfill script
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 changed files +84 −5
modified
apps/api/src/repositories/map.ts
+22 −0
@@ -70,6 +70,28 @@ async function countryClusters(sql: Sql, q: MapQuery): Promise<MapCluster[]> { | ||
| 70 | 70 | const clat = num(r.clat), clng = num(r.clng); |
| 71 | 71 | if (clat != null && clng != null) { c.lat = clat; c.lng = clng; } |
| 72 | 72 | } |
| 73 | + // Facilities without a resolved country must not collapse into one meaningless blob at their mean coordinate: | |
| 74 | + // re-cluster them on a coarse grid (zoom-4 cells) instead. | |
| 75 | + const unknown = by.get("unknown"); | |
| 76 | + if (unknown) { | |
| 77 | + by.delete("unknown"); | |
| 78 | + const size = cellSize(4); | |
| 79 | + const cells = await sql<Row[]>` | |
| 80 | + select floor((f.lng + 180) / ${size})::int as i, floor((f.lat + 90) / ${size})::int as j, f.status, count(*)::int as n, sum(${mwExpr(sql)})::float as mw, avg(f.lat)::float as alat, avg(f.lng)::float as alng | |
| 81 | + from facilities f ${facilityJoins(sql)} | |
| 82 | + where ${conds(sql, q)} and f.country_iso2 is null | |
| 83 | + group by 1, 2, 3`; | |
| 84 | + for (const r of cells) { | |
| 85 | + const key = `cell:${int(r.i)}:${int(r.j)}`; | |
| 86 | + let c = by.get(key); | |
| 87 | + if (!c) { c = { key, lat: num(r.alat) ?? 0, lng: num(r.alng) ?? 0, count: 0, mw: null, statuses: {}, label: null, _slat: 0, _slng: 0, _hasMw: false }; by.set(key, c); } | |
| 88 | + const n = int(r.n); | |
| 89 | + c.count += n; | |
| 90 | + const mw = num(r.mw); | |
| 91 | + if (mw != null) { c.mw = (c.mw ?? 0) + mw; c._hasMw = true; } | |
| 92 | + addStatus(c.statuses, r.status, n); | |
| 93 | + } | |
| 94 | + } | |
| 73 | 95 | return [...by.values()].map((c) => { |
| 74 | 96 | const { _slat, _slng, _hasMw, ...rest } = c; |
| 75 | 97 | if (!rest.lat && !rest.lng && rest.count) { rest.lat = _slat / rest.count; rest.lng = _slng / rest.count; } |
modified
apps/api/src/repositories/operators.ts
+20 −4
@@ -126,10 +126,26 @@ export async function getOperatorDetail(idOrSlug: string, fPage = 1): Promise<{ | ||
| 126 | 126 | /** Operator candidates for query interpretation (trigram). */ |
| 127 | 127 | export async function operatorCandidates(q: string, limit = 5): Promise<Array<{ id: string; slug: string; name: string; similarity: number }>> { |
| 128 | 128 | const sql = pg(); |
| 129 | + // SQL does the cheap prefilter (trigram or substring); word-boundary verification happens here so short aliases | |
| 130 | + // ("IO") never match inside unrelated words ("operatIOnal"). | |
| 129 | 131 | const rows = await sql<Row[]>` |
| 130 | − select o.id, o.slug, o.name, greatest(similarity(o.name, ${q}), case when ${q} ilike '%' || o.name || '%' then 0.95 else 0 end, case when exists (select 1 from unnest(o.aliases) a where ${q} ilike '%' || a || '%') then 0.9 else 0 end) as sim | |
| 132 | + select o.id, o.slug, o.name, o.aliases, similarity(o.name, ${q}) as sim | |
| 131 | 133 | from operators o |
| 132 | − where o.name % ${q} or ${q} ilike '%' || o.name || '%' or exists (select 1 from unnest(o.aliases) a where ${q} ilike '%' || a || '%') | |
| 133 | − order by sim desc, length(o.name) desc limit ${limit}`; | |
| 134 | − return rows.map((r) => ({ id: reqStr(r.id), slug: reqStr(r.slug), name: reqStr(r.name), similarity: num(r.sim) ?? 0 })); | |
| 134 | + where (o.name % ${q} and similarity(o.name, ${q}) >= 0.45) or position(lower(o.name) in lower(${q})) > 0 | |
| 135 | + or exists (select 1 from unnest(o.aliases) a where length(a) >= 3 and position(lower(a) in lower(${q})) > 0) | |
| 136 | + order by sim desc, length(o.name) desc limit ${limit * 4}`; | |
| 137 | + const esc = (x: string) => x.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); | |
| 138 | + const bounded = (needle: string) => new RegExp(`(^|[^\\p{L}\\p{N}])${esc(needle)}([^\\p{L}\\p{N}]|$)`, "iu").test(q); | |
| 139 | + return rows | |
| 140 | + .map((r) => { | |
| 141 | + const name = reqStr(r.name); | |
| 142 | + const aliases = Array.isArray(r.aliases) ? (r.aliases as string[]) : []; | |
| 143 | + let similarity = num(r.sim) ?? 0; | |
| 144 | + if (bounded(name)) similarity = Math.max(similarity, 0.95); | |
| 145 | + else if (aliases.some((a) => a.length >= 3 && bounded(a))) similarity = Math.max(similarity, 0.9); | |
| 146 | + return { id: reqStr(r.id), slug: reqStr(r.slug), name, similarity }; | |
| 147 | + }) | |
| 148 | + .filter((r) => r.similarity >= 0.45) | |
| 149 | + .sort((a, b) => b.similarity - a.similarity || b.name.length - a.name.length) | |
| 150 | + .slice(0, limit); | |
| 135 | 151 | } |
modified
apps/api/src/search/interpret.ts
+1 −1
@@ -70,7 +70,7 @@ export function interpretQuery(raw: string, opts: InterpretOptions = {}): Interp | ||
| 70 | 70 | // 2. Operator (candidates supplied by the caller from a trigram lookup on the full query) |
| 71 | 71 | for (const op of opts.operators ?? []) { |
| 72 | 72 | const re = new RegExp(`(^|[^a-z0-9])${esc(op.name)}([^a-z0-9]|$)`, "i"); |
| 73 | − if (re.test(text) || (op.similarity ?? 0) >= 0.6) { | |
| 73 | + if (re.test(text) || (op.similarity ?? 0) >= 0.8) { | |
| 74 | 74 | out.operator = op.slug; |
| 75 | 75 | if (re.test(text)) text = strip(text, re); |
| 76 | 76 | else if ((op.similarity ?? 0) >= 0.85) text = ""; |
added
scripts/backfill-country.ts
+41 −0
@@ -0,0 +1,41 @@ | ||
| 1 | +/** | |
| 2 | + * Backfill facilities.country_iso2 from coordinates (offline Natural Earth polygons) for rows ingested before | |
| 3 | + * ingest/country-lookup.ts existed, then assign metros where missing. Idempotent; writes provenance rows. | |
| 4 | + * DATABASE_URL=… node node_modules/tsx/dist/cli.mjs scripts/backfill-country.ts [--dry-run] | |
| 5 | + */ | |
| 6 | +import { getSql, closeDb } from "@dci/db"; | |
| 7 | +import { stableId } from "@dci/core"; | |
| 8 | +import { countryFromPoint } from "../apps/worker/src/ingest/country-lookup.js"; | |
| 9 | + | |
| 10 | +const dry = process.argv.includes("--dry-run"); | |
| 11 | +const sql = getSql(); | |
| 12 | +const rows = await sql<{ id: string; lat: number; lng: number; slug: string }[]>`select id, lat, lng, slug from facilities where country_iso2 is null and lat is not null and merged_into is null`; | |
| 13 | +const known = new Set((await sql<{ iso2: string }[]>`select iso2 from countries`).map((r) => r.iso2)); | |
| 14 | +let fixed = 0, unresolved = 0; | |
| 15 | +const byCountry = new Map<string, number>(); | |
| 16 | +for (const r of rows) { | |
| 17 | + const iso = countryFromPoint(Number(r.lat), Number(r.lng)); | |
| 18 | + if (!iso || !known.has(iso)) { unresolved++; continue; } | |
| 19 | + byCountry.set(iso, (byCountry.get(iso) ?? 0) + 1); | |
| 20 | + if (!dry) { | |
| 21 | + await sql`update facilities set country_iso2 = ${iso}, updated_at = now() where id = ${r.id}`; | |
| 22 | + await sql`insert into provenance (id, entity_type, entity_id, field, value, source_id, connector_id, url, confidence, is_estimate, method, note) | |
| 23 | + values (${stableId("provenance", `facility|${r.id}|country_iso2|derived-geo`)}, 'facility', ${r.id}, 'country_iso2', ${sql.json(iso)}, 'src_derived', 'derived', ${"https://www.datacenterindex.io/methodology#geolocation"}, 'moderate', false, 'derived:point-in-country', 'Natural Earth 1:50m admin-0 polygons (public domain)') | |
| 24 | + on conflict do nothing`; | |
| 25 | + } | |
| 26 | + fixed++; | |
| 27 | +} | |
| 28 | +if (!dry) { | |
| 29 | + await sql`insert into sources (id, connector_id, name, domain, kind, priority, license, attribution) values ('src_derived', 'derived', 'Derived (DataCenterIndex)', 'www.datacenterindex.io', 'registry', 4, 'Derived from indexed coordinates + Natural Earth (public domain)', 'DataCenterIndex') on conflict do nothing`; | |
| 30 | + // metro assignment for rows without metro: nearest metro within its radius | |
| 31 | + const m = await sql`update facilities f set metro_id = ( | |
| 32 | + select mm.id from metros mm where mm.country_iso2 = f.country_iso2 | |
| 33 | + and 6371.0088 * 2 * asin(sqrt(power(sin(radians(mm.lat - f.lat) / 2), 2) + cos(radians(f.lat)) * cos(radians(mm.lat)) * power(sin(radians(mm.lng - f.lng) / 2), 2))) <= mm.radius_km | |
| 34 | + order by 6371.0088 * 2 * asin(sqrt(power(sin(radians(mm.lat - f.lat) / 2), 2) + cos(radians(f.lat)) * cos(radians(mm.lat)) * power(sin(radians(mm.lng - f.lng) / 2), 2))) limit 1) | |
| 35 | + where f.metro_id is null and f.lat is not null and f.country_iso2 is not null and f.merged_into is null | |
| 36 | + and exists (select 1 from metros mm where mm.country_iso2 = f.country_iso2)`; | |
| 37 | + console.log(`metros assigned: ${m.count}`); | |
| 38 | +} | |
| 39 | +console.log(`candidates ${rows.length} · fixed ${fixed} · unresolved ${unresolved}${dry ? " (dry run)" : ""}`); | |
| 40 | +console.log([...byCountry.entries()].sort((a, b) => b[1] - a[1]).slice(0, 12).map(([k, v]) => `${k}:${v}`).join(" ")); | |
| 41 | +await closeDb(); | |
| 42 | ||