Repository Public
1/**2 * Markdown → page. Front-matter → title/icon/tags/properties; `[[Wiki links]]` → mentions when the title (or archive path) is3 * known, otherwise plain text; `` and `[file](relative)` → attachments through `resolveFile`.4 */5import { markdownToDoc } from "@/lib/editor/markdown-client";6import type { PMNode } from "@/lib/editor/text";7import { docText } from "@/lib/editor/text";8import { createPage, updatePage, findDuplicatePage, getPageById, type ImportCtx, type Page } from "./pages";910export interface FrontMatter { data: Record<string, unknown>; body: string }11const yamlValue = (raw: string): unknown => {12 const s = raw.trim();13 if (s === "" ) return "";14 if (s === "null" || s === "~") return null; if (s === "true") return true; if (s === "false") return false;15 if (/^-?\d+(\.\d+)?$/.test(s)) return Number(s);16 if (/^\[.*\]$/.test(s)) { try { return JSON.parse(s); } catch { return s.slice(1, -1).split(",").map((x) => yamlValue(x)).filter((x) => x !== ""); } }17 if (/^\{.*\}$/.test(s)) { try { return JSON.parse(s); } catch { return s; } }18 if (/^".*"$/.test(s)) { try { return JSON.parse(s); } catch { return s.slice(1, -1); } }19 if (/^'.*'$/.test(s)) return s.slice(1, -1).replace(/''/g, "'");20 return s;21};22/** Minimal YAML subset: `key: scalar`, `key: [a, b]`, block lists (`- x`) and one level of nested maps. */23export function parseFrontMatter(src: string): FrontMatter {24 const m = src.replace(/^/, "").match(/^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/);25 if (!m) return { data: {}, body: src };26 const data: Record<string, unknown> = {}; const lines = m[1].split(/\r?\n/);27 let i = 0;28 while (i < lines.length) {29 const line = lines[i]; i++;30 const kv = line.match(/^([A-Za-z0-9_ .-]+?):\s*(.*)$/); if (!kv || /^\s/.test(line)) continue;31 const key = kv[1].trim(); const rest = kv[2];32 if (rest !== "") { data[key] = yamlValue(rest); continue; }33 // block: list or nested map34 const list: unknown[] = []; const map: Record<string, unknown> = {}; let isList = false, isMap = false;35 while (i < lines.length && (/^\s+\S/.test(lines[i]) || lines[i].trim() === "")) {36 const l = lines[i]; i++; if (!l.trim()) continue;37 const li = l.match(/^\s+-\s*(.*)$/); if (li) { isList = true; list.push(yamlValue(li[1])); continue; }38 const mk = l.match(/^\s+([A-Za-z0-9_ .-]+?):\s*(.*)$/); if (mk) { isMap = true; map[mk[1].trim()] = yamlValue(mk[2]); }39 }40 data[key] = isList ? list : isMap ? map : null;41 }42 return { data, body: src.slice(m[0].length) };43}44/** Notion-style header: `# Title` then `Key: value` lines until the first blank line. */45export function parseNotionHeader(body: string): { title: string | null; props: Record<string, string>; body: string } {46 let rest = body.replace(/^\s*\n/, ""); let title: string | null = null;47 const h = rest.match(/^#\s+(.+?)\s*\r?\n/); if (h) { title = h[1].trim(); rest = rest.slice(h[0].length); }48 const props: Record<string, string> = {};49 const lines = rest.split(/\r?\n/); let j = 0;50 while (j < lines.length && /^[A-Za-z][A-Za-z0-9 _()/#&-]{0,60}:\s/.test(lines[j]) && !/^https?:/i.test(lines[j])) { const mm = lines[j].match(/^([^:]+):\s+(.*)$/); if (!mm) break; props[mm[1].trim()] = mm[2].trim(); j++; }51 if (j > 0 && (j >= lines.length || lines[j].trim() === "")) rest = lines.slice(j).join("\n"); else Object.keys(props).forEach((k) => delete props[k]);52 return { title, props, body: rest.replace(/^\s*\n/, "") };53}5455export type FileResolver = (ref: string) => Promise<{ id: string; filename: string; mime: string; size: number } | null>;56export interface MdOptions { resolveFile?: FileResolver; resolvePath?: (ref: string) => string | null }57const IMG_RE = /!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;58const WIKI_RE = /\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|([^\]]+))?\]\]/g;59const TOKEN_RE = /⦃(img|ref):(\d+)⦄/g;60const tok = (k: string, n: number) => `⦃${k}:${n}⦄`;6162/** Markdown → ProseMirror doc, resolving images/attachments and wiki links. */63export async function mdToDoc(ctx: ImportCtx, md: string, opts: MdOptions = {}): Promise<PMNode> {64 const imgs: { alt: string; src: string }[] = []; const refs: { target: string; alias: string | null }[] = [];65 // only transform outside fenced code blocks66 const parts = md.replace(/\r\n/g, "\n").split(/(^```[\s\S]*?^```$)/m);67 const pre = parts.map((seg, i) => (i % 2 ? seg : seg.replace(IMG_RE, (_m, alt, src) => { imgs.push({ alt, src }); return tok("img", imgs.length - 1); }).replace(WIKI_RE, (_m, t, a) => { refs.push({ target: t.trim(), alias: a?.trim() ?? null }); return tok("ref", refs.length - 1); }))).join("");68 const nodes = markdownToDoc(pre);69 const files = new Map<number, { src: string; alt: string } | null>(); const links = new Map<string, string>();70 const resolveSrc = async (src: string): Promise<string | null> => {71 if (/^(https?:)?\/\//i.test(src) || src.startsWith("data:") || src.startsWith("/api/files/")) return src;72 if (!opts.resolveFile) return null;73 const f = await opts.resolveFile(safeDecode(src)); return f ? `/api/files/${f.id}` : null;74 };75 for (let i = 0; i < imgs.length; i++) { const s = await resolveSrc(imgs[i].src); files.set(i, s ? { src: s, alt: imgs[i].alt } : null); }76 const resolveRef = (r: { target: string; alias: string | null }) => { const id = ctx.titles.get(r.target.toLowerCase()) ?? opts.resolvePath?.(r.target) ?? opts.resolvePath?.(r.target + ".md") ?? null; return { id, label: r.alias ?? r.target }; };77 const resolveHref = async (href: string): Promise<string> => {78 if (links.has(href)) return links.get(href)!;79 let out = href;80 if (!/^(https?:|mailto:|#|\/|data:)/i.test(href)) {81 const dec = safeDecode(href.replace(/#.*$/, ""));82 const pid = opts.resolvePath?.(dec); if (pid) out = `/p/${pid}`;83 else if (opts.resolveFile && !/\.md$/i.test(dec)) { const f = await opts.resolveFile(dec); if (f) out = `/api/files/${f.id}`; }84 }85 links.set(href, out); return out;86 };87 // collect hrefs first (async), then rewrite synchronously88 const hrefs = new Set<string>(); const collect = (n: PMNode) => { for (const m of n.marks ?? []) if (m.type === "link" && typeof m.attrs?.href === "string") hrefs.add(m.attrs.href); n.content?.forEach(collect); }; nodes.forEach(collect);89 for (const h of hrefs) await resolveHref(h);90 const splitText = (n: PMNode): PMNode[] => {91 const text = n.text ?? ""; if (!TOKEN_RE.test(text)) { TOKEN_RE.lastIndex = 0; return [n]; } TOKEN_RE.lastIndex = 0;92 const out: PMNode[] = []; let last = 0; let m: RegExpExecArray | null;93 while ((m = TOKEN_RE.exec(text))) {94 if (m.index > last) out.push({ ...n, text: text.slice(last, m.index) });95 const idx = Number(m[2]);96 if (m[1] === "img") { const f = files.get(idx); if (f) out.push({ type: "image", attrs: { src: f.src, alt: f.alt || null, title: null } }); else out.push({ ...n, text: imgs[idx]?.alt ? `[image: ${imgs[idx].alt}]` : `[image: ${imgs[idx]?.src ?? ""}]` }); }97 else { const { id, label } = resolveRef(refs[idx]); if (id) out.push({ type: "mention", attrs: { id, label } }); else out.push({ ...n, text: label }); }98 last = m.index + m[0].length;99 }100 if (last < text.length) out.push({ ...n, text: text.slice(last) });101 return out;102 };103 const walk = (n: PMNode): PMNode[] => {104 if (n.type === "text") { const nn = n.marks?.some((m) => m.type === "link") ? { ...n, marks: n.marks!.map((m) => (m.type === "link" ? { ...m, attrs: { ...m.attrs, href: links.get(String(m.attrs?.href)) ?? m.attrs?.href } } : m)) } : n; return splitText(nn); }105 if (!n.content) return [n];106 let children = n.content.flatMap(walk);107 // marked emits the task checkbox as a stray "[x] " paragraph before the item text — drop it108 if ((n.type === "taskItem" || n.type === "listItem") && children.length > 1 && children[0].type === "paragraph" && children[0].content?.length === 1 && /^\[( |x|X)\]\s*$/.test(children[0].content[0].text ?? "")) children = children.slice(1);109 if (n.type === "paragraph" || n.type === "heading") {110 // images are block nodes: split the paragraph around them111 const out: PMNode[] = []; let buf: PMNode[] = [];112 const flush = () => { if (buf.length) out.push({ ...n, content: buf }); buf = []; };113 for (const c of children) { if (c.type === "image") { flush(); out.push(c); } else buf.push(c); }114 flush(); return out.length ? out : [{ ...n, content: [] }];115 }116 return [{ ...n, content: children }];117 };118 const content = nodes.flatMap(walk);119 return { type: "doc", content: content.length ? content : [{ type: "paragraph" }] };120}121const safeDecode = (s: string) => { try { return decodeURIComponent(s); } catch { return s; } };122123/** Relative (archive-local) targets of a Markdown body: images and links that are neither URLs nor anchors. Used by the import preview. */124export function relativeRefs(md: string): { images: string[]; links: string[] } {125 const images: string[] = []; const links: string[] = []; const rel = (s: string) => !/^([a-z][a-z0-9+.-]*:|\/\/|#|\/)/i.test(s);126 const body = md.replace(/\r\n/g, "\n").split(/(^```[\s\S]*?^```$)/m).filter((_s, i) => i % 2 === 0).join("\n");127 for (const m of body.matchAll(IMG_RE)) if (rel(m[2])) images.push(safeDecode(m[2]));128 for (const m of body.replace(IMG_RE, " ").matchAll(/\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g)) if (rel(m[1])) links.push(safeDecode(m[1].replace(/#.*$/, "")));129 return { images, links };130}131/** Plain text the imported page would hold — computed without touching the database or storing files (duplicate detection, preview). */132export function previewText(md: string): string {133 const parts = md.replace(/\r\n/g, "\n").split(/(^```[\s\S]*?^```$)/m);134 const pre = parts.map((seg, i) => (i % 2 ? seg : seg.replace(IMG_RE, " ").replace(WIKI_RE, (_m, t: string, a?: string) => a?.trim() ?? t.trim()))).join("");135 return docText({ type: "doc", content: markdownToDoc(pre) });136}137/** Title, properties and body of a Markdown file, exactly as importMarkdown reads them. */138export function readMarkdown(src: string, filename?: string) {139 const { data, body } = parseFrontMatter(src);140 const nh = parseNotionHeader(body);141 const title = String(data.title ?? nh.title ?? (filename ?? "Untitled").replace(/^.*[\\/]/, "").replace(/\.(md|markdown|txt)$/i, "")).trim() || "Untitled";142 return { data, nh, title, body: nh.body };143}144145/**146 * Full Markdown file → page (front-matter + Notion header aware). `existingId` updates a page created in a first pass.147 * With ctx.duplicates = "skip", a page with the same title and text is not created again: the existing page is returned.148 */149export async function importMarkdown(ctx: ImportCtx, src: string, opts: MdOptions & { filename?: string; parentId?: string | null; existingId?: string; section?: string; key?: string } = {}): Promise<Page> {150 const { data, nh, title } = readMarkdown(src, opts.filename);151 if (!opts.existingId && !(opts.key && ctx.ledger?.has(`page:${opts.key}`))) {152 const dup = findDuplicatePage(ctx, title, previewText(nh.body));153 if (dup) { const p = await getPageById(ctx, dup.id); if (p) { ctx.stats.duplicates++; ctx.stats.skipped.push(`“${p.title}” already exists with the same content — not imported again`); return p; } }154 }155 const props: Record<string, unknown> = { ...(typeof data.properties === "object" && data.properties ? (data.properties as Record<string, unknown>) : {}), ...nh.props };156 for (const [k, v] of Object.entries(data)) if (!["title", "id", "icon", "tags", "properties", "created", "updated", "type", "section", "database", "cover"].includes(k)) props[k] = v;157 const tags = Array.isArray(data.tags) ? data.tags.map(String) : typeof data.tags === "string" ? data.tags.split(/[,;]\s*/).filter(Boolean) : typeof nh.props.Tags === "string" ? nh.props.Tags.split(/,\s*/).filter(Boolean) : [];158 if (tags.length) props.tags = tags;159 const doc = await mdToDoc(ctx, nh.body, opts);160 const icon = typeof data.icon === "string" ? data.icon.slice(0, 16) : null;161 const dates = { createdAt: data.created ? new Date(String(data.created)) : undefined, updatedAt: data.updated ? new Date(String(data.updated)) : undefined };162 const valid = (d?: Date) => (d && !isNaN(d.getTime()) ? d : undefined);163 if (opts.existingId) return updatePage(ctx, opts.existingId, { title, content: doc, icon, properties: props, updatedAt: valid(dates.updatedAt) });164 return createPage(ctx, { key: opts.key, title, parentId: opts.parentId ?? null, section: opts.section ?? (typeof data.section === "string" ? data.section : "notes"), content: doc, icon, properties: props, createdAt: valid(dates.createdAt), updatedAt: valid(dates.updatedAt) });165}166