| 1 |
|
−import { newId, type SensorType } from "@websensor/core"; |
|
1 |
+import { canonicalizeHtml, jaccard, newId, shingles, type SensorType } from "@websensor/core"; |
| 2 |
2 |
import { httpFetch } from "./fetcher"; |
| 3 |
3 |
import { parseFeed } from "./rss"; |
| 4 |
4 |
import { parseSitemap } from "./sitemap"; |
| 111 |
111 |
} |
| 112 |
112 |
|
| 113 |
113 |
if (opts.probePages) { |
|
114 |
+ // Catch-all sites answer 200 for any path: a page candidate must be a real, distinct, non-thin document |
|
115 |
+ // (GET, no redirect back to the homepage, canonical text different from the homepage's). |
|
116 |
+ const homeCanon = homeHtml ? canonicalizeHtml(homeHtml, home.meta.finalUrl || base) : null; |
| 114 |
117 |
await parallel(PAGE_PATHS, 4, async ([path, kind]) => { |
| 115 |
118 |
const url = base + path; |
| 116 |
|
− const o = await httpFetch(id, url, { method: "HEAD", timeoutMs: 10_000 }); |
| 117 |
|
− if (o.meta.status === 200 || o.meta.status === 405) add({ url, type: "HTML", connector: "http", evidence: `HEAD ${o.meta.status}`, value: kind === "pricing" || kind === "changelog" || kind === "security" ? 0.6 : 0.4 }); |
|
119 |
+ const o = await httpFetch(id, url, { timeoutMs: 15_000, maxBytes: 3 * 1024 * 1024 }); |
|
120 |
+ if (o.meta.status !== 200 || !o.body) return; |
|
121 |
+ const finalPath = safePath(o.meta.finalUrl); |
|
122 |
+ if (finalPath === "/" || finalPath === "") return; // redirected home |
|
123 |
+ const c = canonicalizeHtml(o.body.toString("utf8"), o.meta.finalUrl); |
|
124 |
+ if (c.text.length < 200 || (homeCanon && (c.canonicalHash === homeCanon.canonicalHash || (c.title && c.title === homeCanon.title && jaccard(shingles(c.text), shingles(homeCanon.text)) > 0.8)))) return; |
|
125 |
+ add({ url: o.meta.finalUrl || url, type: "HTML", connector: "http", evidence: `GET 200, ${c.text.length} chars, distinct from homepage`, value: kind === "pricing" || kind === "changelog" || kind === "security" ? 0.6 : 0.4, title: c.title ?? undefined }); |
| 118 |
126 |
}); |
| 119 |
127 |
} |
| 120 |
128 |
|
| 121 |
129 |
return [...found.values()].sort((a, b) => b.value - a.value); |
| 122 |
130 |
} |
| 123 |
131 |
|
|
132 |
+function safePath(u: string): string { |
|
133 |
+ try { |
|
134 |
+ return new URL(u).pathname.replace(/\/$/, ""); |
|
135 |
+ } catch { |
|
136 |
+ return ""; |
|
137 |
+ } |
|
138 |
+} |
|
139 |
+ |
| 124 |
140 |
function safeAbs(h: string, base: string): string | null { |
| 125 |
141 |
try { |
| 126 |
142 |
const u = new URL(h, base); |
| 127 |
143 |
|