Merge pull request '[codex] Add searchable speeches archive' (#2) from codex/searchable-speeches-archive into main

Reviewed-on: https://git.bcgen.ie/familyfedie/familyfedie-website/pulls/2

src/lib/speeches.ts

@@ -0,0 +1,224 @@
+import fs from "node:fs";
+import path from "node:path";
+
+const repoRoot = process.cwd();
+const speechesRoot = path.join(repoRoot, "content", "speeches");
+
+type FrontmatterValue = string | string[];
+
+type Frontmatter = {
+  title?: string;
+  published?: string;
+  updated?: string;
+  source?: string;
+  speechYear?: string;
+  speechCollection?: string;
+  categories?: string[];
+  tags?: string[];
+};
+
+export type SpeechArchiveItem = {
+  id: string;
+  title: string;
+  url: string;
+  year: string;
+  speaker: string;
+  category: string;
+  categories: string[];
+  tags: string[];
+  excerpt: string;
+  published: string;
+  searchText: string;
+};
+
+const entityMap: Record<string, string> = {
+  amp: "&",
+  apos: "'",
+  hellip: "...",
+  laquo: "<<",
+  nbsp: " ",
+  quot: '"',
+  raquo: ">>",
+  rsquo: "'",
+  lsquo: "'",
+  rdquo: '"',
+  ldquo: '"',
+};
+
+function walkMarkdownFiles(dir = speechesRoot): string[] {
+  if (!fs.existsSync(dir)) {
+    return [];
+  }
+
+  return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      return walkMarkdownFiles(fullPath);
+    }
+    return entry.isFile() && entry.name.endsWith(".md") ? [fullPath] : [];
+  });
+}
+
+function decodeEntities(value = ""): string {
+  return value.replace(/&(#x?[0-9a-f]+|[a-z]+);/gi, (match, entity) => {
+    const key = entity.toLowerCase();
+    if (key.startsWith("#x")) {
+      return String.fromCodePoint(Number.parseInt(key.slice(2), 16));
+    }
+    if (key.startsWith("#")) {
+      return String.fromCodePoint(Number.parseInt(key.slice(1), 10));
+    }
+    return entityMap[key] ?? match;
+  });
+}
+
+function textFromHtml(html: string): string {
+  return decodeEntities(
+    html
+      .replace(/<script[\s\S]*?<\/script>/gi, " ")
+      .replace(/<style[\s\S]*?<\/style>/gi, " ")
+      .replace(/<[^>]+>/g, " "),
+  )
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function parseScalar(value: string): string {
+  const trimmed = value.trim();
+  if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
+    return JSON.parse(trimmed);
+  }
+  return trimmed;
+}
+
+function parseFrontmatter(markdown: string): { frontmatter: Frontmatter; body: string } {
+  const match = markdown.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+  if (!match) {
+    return { frontmatter: {}, body: markdown };
+  }
+
+  const parsed: Record<string, FrontmatterValue> = {};
+  let currentArrayKey = "";
+
+  for (const line of match[1].split("\n")) {
+    const arrayItem = line.match(/^\s+-\s+(.+)$/);
+    if (arrayItem && currentArrayKey) {
+      const current = parsed[currentArrayKey];
+      parsed[currentArrayKey] = [...(Array.isArray(current) ? current : []), parseScalar(arrayItem[1])];
+      continue;
+    }
+
+    const keyValue = line.match(/^([A-Za-z0-9_-]+):(?:\s*(.*))?$/);
+    if (!keyValue) {
+      continue;
+    }
+
+    const [, key, rawValue = ""] = keyValue;
+    currentArrayKey = "";
+
+    if (rawValue.trim() === "[]") {
+      parsed[key] = [];
+    } else if (rawValue.trim() === "") {
+      parsed[key] = [];
+      currentArrayKey = key;
+    } else {
+      parsed[key] = parseScalar(rawValue);
+    }
+  }
+
+  return { frontmatter: parsed as Frontmatter, body: match[2] };
+}
+
+function normalizeSearchText(value: string): string {
+  return value
+    .normalize("NFKD")
+    .replace(/[\u0300-\u036f]/g, "")
+    .toLowerCase();
+}
+
+function inferSpeaker(frontmatter: Frontmatter, bodyText: string): string {
+  const categories = frontmatter.categories?.join(" ") ?? "";
+  const collection = frontmatter.speechCollection ?? "";
+  const searchable = `${categories} ${collection} ${bodyText.slice(0, 500)}`.toLowerCase();
+
+  if (searchable.includes("hak ja han") || collection === "mrs-hak-ja-han-moon") {
+    return "Hak Ja Han Moon";
+  }
+  if (searchable.includes("sun myung moon") || /^\d{4}$/.test(collection)) {
+    return "Rev. Sun Myung Moon";
+  }
+  return "Unknown";
+}
+
+function inferYear(frontmatter: Frontmatter, filePath: string, bodyText: string): string {
+  if (frontmatter.speechYear) {
+    return frontmatter.speechYear;
+  }
+
+  const relativeParts = path.relative(speechesRoot, filePath).split(path.sep);
+  const directoryYear = relativeParts.find((part) => /^(1\d{3}|20\d{2})$/.test(part));
+  if (directoryYear) {
+    return directoryYear;
+  }
+
+  const categoryYear = frontmatter.categories?.join(" ").match(/\b(1\d{3}|20\d{2})\b/)?.[1];
+  if (categoryYear) {
+    return categoryYear;
+  }
+
+  return bodyText.slice(0, 1400).match(/\b(1\d{3}|20\d{2})\b/)?.[1] ?? "Unknown";
+}
+
+function excerptFromText(bodyText: string): string {
+  const excerpt = bodyText.replace(/^(REVEREND|REV\.|DR\.|MRS\.) [A-Z .]+/i, "").trim();
+  return excerpt.length > 230 ? `${excerpt.slice(0, 230).trim()}...` : excerpt;
+}
+
+function itemFromFile(filePath: string): SpeechArchiveItem {
+  const markdown = fs.readFileSync(filePath, "utf8");
+  const { frontmatter, body } = parseFrontmatter(markdown);
+  const bodyText = textFromHtml(body);
+  const categories = frontmatter.categories ?? [];
+  const tags = frontmatter.tags ?? [];
+  const relativePath = path.relative(speechesRoot, filePath).split(path.sep).join("/");
+  const title = frontmatter.title ?? path.basename(filePath, ".md");
+  const year = inferYear(frontmatter, filePath, bodyText);
+  const speaker = inferSpeaker(frontmatter, bodyText);
+  const category = categories[0] ?? (speaker === "Unknown" ? "Uncategorized speeches" : `Speeches of ${speaker}`);
+  const searchText = normalizeSearchText([title, year, speaker, category, ...categories, ...tags, bodyText].join(" "));
+
+  return {
+    id: relativePath.replace(/\.md$/, ""),
+    title,
+    url: frontmatter.source ?? "#",
+    year,
+    speaker,
+    category,
+    categories,
+    tags,
+    excerpt: excerptFromText(bodyText),
+    published: frontmatter.published ?? "",
+    searchText,
+  };
+}
+
+export function getSpeechArchive(): SpeechArchiveItem[] {
+  return walkMarkdownFiles()
+    .map(itemFromFile)
+    .sort((a, b) => {
+      const yearSort = Number.parseInt(b.year, 10) - Number.parseInt(a.year, 10);
+      if (Number.isFinite(yearSort) && yearSort !== 0) {
+        return yearSort;
+      }
+      return a.title.localeCompare(b.title);
+    });
+}
+
+export function uniqueSorted(values: string[], direction: "asc" | "desc" = "asc"): string[] {
+  const unique = Array.from(new Set(values.filter(Boolean)));
+  return unique.sort((a, b) => {
+    const numeric = Number.parseInt(a, 10) - Number.parseInt(b, 10);
+    const comparison = Number.isFinite(numeric) && numeric !== 0 ? numeric : a.localeCompare(b);
+    return direction === "desc" ? comparison * -1 : comparison;
+  });
+}

src/lib/static-pages.ts

@@ -9,6 +9,7 @@ export const migratedHtmlPages = new Set([
   "events.html",
   "index.html",
   "services.html",
+  "speeches/index.html",
   "the-founders.html",
   "videos.html",
 ]);