markdown: keep content-bearing nav landmarks (portal pages); qa: wait for request id in playground

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 16 days ago (Sep 8, 2026) parent 7a8260e

2 changed files +23 −6

modified packages/core/src/markdown.ts +17 −6

		`@@ -263,10 +263,21 @@ function esc(s: string): string {`
263	263	return s.replace(/([\\`*_{}[\]<>])/g, "\\$1");
264	264	}
265	265
	266	+type WriterOpts = Required<Pick<MarkdownOptions, "images" \| "links" \| "stripNoise">> & { baseUrl?: string; /** Text length of the rendered scope, used to keep "noise" landmarks that actually hold the content (e.g. a <nav> of language links on a portal). */ scopeLen?: number };
	267	+
266	268	class MdWriter {
267	269	out: string[] = [];
268		− private listStack: Array<{ ordered: boolean; index: number }> = [];
269		− constructor(private readonly opts: Required<Pick<MarkdownOptions, "images" \| "links" \| "stripNoise">> & { baseUrl?: string }) {}
	270	+ constructor(private readonly opts: WriterOpts) {}
	271	+
	272	+ /** Noise check that spares landmark elements carrying a large share of the scope's text. */
	273	+ private noise(n: HNode): boolean {
	274	+ if (!this.opts.stripNoise \|\| !isNoise(n)) return false;
	275	+ if (n.type === "element" && (n.tag === "nav" \|\| n.tag === "aside" \|\| n.tag === "header" \|\| n.tag === "footer" \|\| n.tag === "form") && this.opts.scopeLen) {
	276	+ const t = textOf(n).length;
	277	+ if (t >= 120 && t >= this.opts.scopeLen * 0.35) return false;
	278	+ }
	279	+ return true;
	280	+ }
270	281
271	282	block(s: string) {
272	283	const t = s.replace(/\n{3,}/g, "\n\n").trim();
		`@@ -287,7 +298,7 @@ class MdWriter {`
287	298	s += decodeEntities(c.text).replace(/\s+/g, " ");
288	299	continue;
289	300	}
290		− if (this.opts.stripNoise && isNoise(c)) continue;
	301	+ if (this.noise(c)) continue;
291	302	switch (c.tag) {
292	303	case "br":
293	304	s += " \n";
		`@@ -375,7 +386,7 @@ class MdWriter {`
375	386	inlineBuf += decodeEntities(c.text).replace(/\s+/g, " ");
376	387	continue;
377	388	}
378		− if (this.opts.stripNoise && isNoise(c)) continue;
	389	+ if (this.noise(c)) continue;
379	390	if (!BLOCK.has(c.tag) && !["table", "ul", "ol", "dl"].includes(c.tag)) {
380	391	inlineBuf += this.inline({ ...c, children: [c] } as HNode);
381	392	continue;
		`@@ -535,7 +546,7 @@ class MdWriter {`
535	546	export function htmlToMarkdown(html: string, opts: MarkdownOptions = {}): string {
536	547	const root = parseHtml(html);
537	548	const scope = opts.mainContent === false ? (findAll(root, (n) => n.tag === "body")[0] ?? root) : mainContent(root);
538		− const writer = new MdWriter({ images: opts.images ?? true, links: opts.links ?? true, stripNoise: opts.stripNoise ?? true, baseUrl: opts.baseUrl });
	549	+ const writer = new MdWriter({ images: opts.images ?? true, links: opts.links ?? true, stripNoise: opts.stripNoise ?? true, baseUrl: opts.baseUrl, scopeLen: textOf(scope).length });
539	550	let md = writer.render(scope);
540	551	// Prepend the document title when the content does not already start with a heading.
541	552	const title = findAll(root, (n) => n.tag === "title")[0];
		`@@ -549,7 +560,7 @@ export function htmlToMarkdown(html: string, opts: MarkdownOptions = {}): string`
549	560	export function htmlToMainText(html: string): string {
550	561	const root = parseHtml(html);
551	562	const scope = mainContent(root);
552		− const w = new MdWriter({ images: false, links: false, stripNoise: true });
	563	+ const w = new MdWriter({ images: false, links: false, stripNoise: true, scopeLen: textOf(scope).length });
553	564	return w
554	565	.render(scope)
555	566	.replace(/^#+\s*/gm, "")

modified packages/core/test/markdown.test.ts +6 −0

		`@@ -32,6 +32,12 @@ describe("markdown", () => {`
32	32	expect(md).not.toContain("Nav A");
33	33	expect(md).not.toContain("var a = 1");
34	34	});
	35	+ it("keeps a <nav> that carries the page's actual content (portal pages)", () => {
	36	+ const portal = `<html><head><title>Portal</title></head><body><main><h1>Portal</h1><nav class="central-featured">${Array.from({ length: 10 }, (_, i) => `<div class="lang"><a href="/l${i}"><strong>Language ${i}</strong> <small>${i}00 000+ articles</small></a></div>`).join("")}</nav></main><footer><a href="/terms">Terms</a></footer></body></html>`;
	37	+ const md = htmlToMarkdown(portal, { baseUrl: "https://p.example/" });
	38	+ expect(md).toContain("[Language 3 300 000+ articles](https://p.example/l3)");
	39	+ expect(md).not.toContain("Terms");
	40	+ });
35	41	it("extracts readable main text", () => {
36	42	const t = htmlToMainText(PAGE);
37	43	expect(t).toContain("Bonjour");
38	44