SPB Git

Repository Public

code-highlight.ts 6.1 KB 97 lines typescript

1import type { BlockOf } from "@/schema";23/** A small dependency-free tokenizer: comments, strings, numbers and keywords. Enough to make code readable. */4export type CodeLanguage = BlockOf<"code">["language"];5export type TokenKind = "plain" | "comment" | "string" | "number" | "keyword";6export interface Token { kind: TokenKind; text: string }78const words = (s: string): ReadonlySet<string> => new Set(s.split(" "));9const JS = words("const let var function return if else for while do switch case break continue new class extends import export from default async await try catch finally throw typeof instanceof in of null undefined true false this super yield static interface type enum implements readonly public private protected as satisfies keyof void never unknown any");10const C_LIKE = words("int long short char float double void unsigned signed const static struct union enum typedef return if else for while do switch case break continue sizeof include define NULL true false extern inline");11const KEYWORDS: Record<CodeLanguage, ReadonlySet<string>> = {12  python: words("def return if elif else for while in not and or is None True False import from as class with try except finally raise lambda yield pass break continue global nonlocal assert async await del"),13  typescript: JS, javascript: JS,14  r: words("function return if else for while repeat in next break TRUE FALSE NULL NA NaN Inf library require"),15  sql: words("select from where and or not in is null as join left right inner outer full cross on group by order having limit offset insert into values update set delete create table index view with union all distinct case when then else end asc desc primary key references default true false"),16  bash: words("if then else elif fi for in do done while until case esac function return export local readonly echo exit set unset source"),17  json: words("true false null"),18  yaml: words("true false null yes no on off"),19  latex: words(""),20  julia: words("function end return if elseif else for while in do begin let struct mutable module using import export const true false nothing try catch finally macro abstract type where"),21  rust: words("fn let mut const static struct enum impl trait pub use mod crate self Self super match if else for while loop in return break continue as ref move async await dyn where type unsafe true false Some None Ok Err"),22  go: words("func var const type struct interface package import return if else for range switch case default break continue go defer chan map select fallthrough nil true false"),23  c: C_LIKE,24  pseudo: words("function procedure return if then else end for each while do repeat until in to and or not input output true false"),25  text: words(""),26};27const LINE_COMMENT: Record<CodeLanguage, string[]> = {28  python: ["#"], typescript: ["//"], javascript: ["//"], r: ["#"], sql: ["--"], bash: ["#"], json: [], yaml: ["#"], latex: ["%"],29  julia: ["#"], rust: ["//"], go: ["//"], c: ["//"], pseudo: ["//", "#"], text: [],30};31const BLOCK_COMMENT: ReadonlySet<CodeLanguage> = new Set<CodeLanguage>(["typescript", "javascript", "rust", "go", "c", "sql"]);32const QUOTES: Record<CodeLanguage, string> = {33  python: "\"'", typescript: "\"'`", javascript: "\"'`", r: "\"'", sql: "'", bash: "\"'", json: "\"", yaml: "\"'", latex: "",34  julia: "\"", rust: "\"", go: "\"`", c: "\"'", pseudo: "\"", text: "",35};3637export function tokenize(code: string, language: CodeLanguage): Token[] {38  if (language === "text") return [{ kind: "plain", text: code }];39  const out: Token[] = [];40  const push = (kind: TokenKind, text: string) => {41    if (!text) return;42    const last = out[out.length - 1];43    if (last && last.kind === kind) last.text += text; else out.push({ kind, text });44  };45  const keywords = KEYWORDS[language], insensitive = language === "sql";46  let i = 0;47  while (i < code.length) {48    const c = code[i];49    const lc = LINE_COMMENT[language].find((m) => code.startsWith(m, i));50    if (lc && !(language === "bash" && lc === "#" && i > 0 && /[\w$]/.test(code[i - 1]))) {51      let end = code.indexOf("\n", i); if (end < 0) end = code.length;52      push("comment", code.slice(i, end)); i = end; continue;53    }54    if (BLOCK_COMMENT.has(language) && code.startsWith("/*", i)) {55      let end = code.indexOf("*/", i + 2); end = end < 0 ? code.length : end + 2;56      push("comment", code.slice(i, end)); i = end; continue;57    }58    if (language === "latex" && c === "\\") {59      const m = /^\\([a-zA-Z@]+|.)/.exec(code.slice(i, i + 40));60      if (m) { push("keyword", m[0]); i += m[0].length; continue; }61    }62    if (QUOTES[language].includes(c)) {63      const triple = language === "python" && code.startsWith(c.repeat(3), i);64      const close = triple ? c.repeat(3) : c;65      let j = i + close.length;66      while (j < code.length) {67        if (code[j] === "\\") { j += 2; continue; }68        if (code.startsWith(close, j)) { j += close.length; break; }69        if (!triple && c !== "`" && code[j] === "\n") break;70        j++;71      }72      push("string", code.slice(i, Math.min(j, code.length))); i = Math.min(j, code.length); continue;73    }74    if (/[0-9]/.test(c) && (i === 0 || !/[\w$]/.test(code[i - 1]))) {75      const m = /^(0[xX][0-9a-fA-F_]+|[0-9][0-9_]*(\.[0-9_]+)?([eE][+-]?[0-9]+)?)/.exec(code.slice(i, i + 40));76      if (m) { push("number", m[0]); i += m[0].length; continue; }77    }78    if (/[A-Za-z_]/.test(c)) {79      const m = /^[A-Za-z_][A-Za-z0-9_]*/.exec(code.slice(i, i + 80));80      const word = m ? m[0] : c;81      push(keywords.has(insensitive ? word.toLowerCase() : word) ? "keyword" : "plain", word); i += word.length; continue;82    }83    push("plain", c); i++;84  }85  return out;86}8788/** Tokens split into lines (a multi-line string or comment continues on the next line with the same kind). */89export function tokenizeLines(code: string, language: CodeLanguage): Token[][] {90  const lines: Token[][] = [[]];91  for (const tok of tokenize(code.replace(/\r\n?/g, "\n").replace(/\n+$/, ""), language)) {92    const parts = tok.text.split("\n");93    parts.forEach((part, k) => { if (k > 0) lines.push([]); if (part) lines[lines.length - 1].push({ kind: tok.kind, text: part }); });94  }95  return lines;96}97
Files main
Menu

Theme