From 561b28bebfa941ba25fe97f98d0d2b3d997f8212 Mon Sep 17 00:00:00 2001 From: Loyyd Date: Sun, 5 Jul 2026 14:37:47 +0200 Subject: [PATCH] Optimize speech search and admin payloads --- src/lib/admin-speeches.ts | 52 ++++++++++++++ src/lib/speeches.ts | 41 +++++++++++ src/pages/admin/content/[...id].json.ts | 23 ++++++ src/pages/admin/index.astro | 48 ++++++------- src/pages/speeches/index.astro | 93 ++++++++++++++++++++++--- src/pages/speeches/search-index.json.ts | 4 +- 6 files changed, 223 insertions(+), 38 deletions(-) create mode 100644 src/lib/admin-speeches.ts create mode 100644 src/pages/admin/content/[...id].json.ts diff --git a/src/lib/admin-speeches.ts b/src/lib/admin-speeches.ts new file mode 100644 index 00000000..37e47e9c --- /dev/null +++ b/src/lib/admin-speeches.ts @@ -0,0 +1,52 @@ +import type { ArchiveEntry } from "./archive"; + +export type SpeechAdminEntry = { + id: string; + path: string; + title: string; + publishedDate: string; + speechYear: string; + speaker: "rev" | "mrs" | "other"; + source: string; + categories: string[]; + tags: string[]; + bodyUrl: string; +}; + +function encodePath(value: string): string { + return value.split("/").map(encodeURIComponent).join("/"); +} + +export function speechAdminEntryFromArchiveEntry(entry: ArchiveEntry, fallbackDate: string): SpeechAdminEntry { + const categories = entry.data.categories ?? []; + const tags = entry.data.tags ?? []; + const publishedDate = entry.data.published?.slice(0, 10) ?? fallbackDate; + const speechYear = entry.data.speechYear ?? entry.id.match(/\b(1\d{3}|20\d{2})\b/)?.[1] ?? new Date().getFullYear().toString(); + const categoryText = categories.join(" ").toLowerCase(); + const collection = entry.data.speechCollection ?? ""; + const speaker = categoryText.includes("hak ja han") || collection === "mrs-hak-ja-han-moon" + ? "mrs" + : categoryText.includes("rev sun myung moon") || /^\d{4}$/.test(collection) || /^\d{4}$/.test(speechYear) + ? "rev" + : "other"; + + return { + id: entry.id, + path: `content/${entry.id}.md`, + title: entry.data.title ?? entry.id, + publishedDate, + speechYear, + speaker, + source: entry.data.source ?? "", + categories, + tags, + bodyUrl: `/admin/content/${encodePath(entry.id)}.json`, + }; +} + +export function speechAdminEntryWithBody(entry: ArchiveEntry, fallbackDate: string) { + return { + ...speechAdminEntryFromArchiveEntry(entry, fallbackDate), + body: entry.body ?? "", + }; +} diff --git a/src/lib/speeches.ts b/src/lib/speeches.ts index 21e696c0..21fd9511 100644 --- a/src/lib/speeches.ts +++ b/src/lib/speeches.ts @@ -32,6 +32,14 @@ export type SpeechArchiveItem = { searchText: string; }; +export type SpeechSearchIndexItem = Omit; + +export type CompactSpeechSearchIndex = { + version: 2; + items: SpeechSearchIndexItem[]; + index: Record; +}; + const entityMap: Record = { amp: "&", apos: "'", @@ -215,6 +223,39 @@ export function getSpeechArchive(): SpeechArchiveItem[] { }); } +function packDocumentIds(documentIds: number[]): string { + return documentIds.map((documentId) => documentId.toString(36)).join(","); +} + +export function getCompactSpeechSearchIndex(): CompactSpeechSearchIndex { + const items = getSpeechArchive(); + const index = new Map(); + + items.forEach((item, documentId) => { + const terms = new Set(item.searchText.match(/[a-z0-9]+/g) ?? []); + + terms.forEach((term) => { + if (term.length < 2) { + return; + } + + const documentIds = index.get(term) ?? []; + documentIds.push(documentId); + index.set(term, documentIds); + }); + }); + + return { + version: 2, + items: items.map(({ searchText, ...item }) => item), + index: Object.fromEntries( + Array.from(index) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([term, documentIds]) => [term, packDocumentIds(documentIds)]), + ), + }; +} + export function uniqueSorted(values: string[], direction: "asc" | "desc" = "asc"): string[] { const unique = Array.from(new Set(values.filter(Boolean))); return unique.sort((a, b) => { diff --git a/src/pages/admin/content/[...id].json.ts b/src/pages/admin/content/[...id].json.ts new file mode 100644 index 00000000..b5c4d997 --- /dev/null +++ b/src/pages/admin/content/[...id].json.ts @@ -0,0 +1,23 @@ +import { getArchiveEntries } from "../../../lib/archive"; +import { speechAdminEntryWithBody } from "../../../lib/admin-speeches"; + +export async function getStaticPaths() { + const today = new Date().toISOString().slice(0, 10); + const entries = await getArchiveEntries("speech"); + + return entries.map((entry) => ({ + params: { id: entry.id }, + props: { + speech: speechAdminEntryWithBody(entry, today), + }, + })); +} + +export function GET({ props }: { props: { speech: ReturnType } }) { + return new Response(JSON.stringify(props.speech), { + headers: { + "content-type": "application/json; charset=utf-8", + "cache-control": "public, max-age=300", + }, + }); +} diff --git a/src/pages/admin/index.astro b/src/pages/admin/index.astro index 267e55c2..6bdea2c6 100644 --- a/src/pages/admin/index.astro +++ b/src/pages/admin/index.astro @@ -1,5 +1,6 @@ --- import { getArchiveEntries } from "../../lib/archive"; +import { speechAdminEntryFromArchiveEntry } from "../../lib/admin-speeches"; import { calendarEvents, recurringCalendarEvents } from "../../data/events"; const entries = await getArchiveEntries(); @@ -17,32 +18,7 @@ const latestEntryDate = latestEntry?.data.published : "No date"; const totalEvents = calendarEvents.length + recurringCalendarEvents.length; const configuredServices = [contactFormEndpoint, plausibleDomain].filter(Boolean).length; -const speechAdminEntries = speechEntries.map((entry) => { - const categories = entry.data.categories ?? []; - const tags = entry.data.tags ?? []; - const publishedDate = entry.data.published?.slice(0, 10) ?? today; - const speechYear = entry.data.speechYear ?? entry.id.match(/\b(1\d{3}|20\d{2})\b/)?.[1] ?? new Date().getFullYear().toString(); - const categoryText = categories.join(" ").toLowerCase(); - const collection = entry.data.speechCollection ?? ""; - const speaker = categoryText.includes("hak ja han") || collection === "mrs-hak-ja-han-moon" - ? "mrs" - : categoryText.includes("rev sun myung moon") || /^\d{4}$/.test(collection) || /^\d{4}$/.test(speechYear) - ? "rev" - : "other"; - - return { - id: entry.id, - path: `content/${entry.id}.md`, - title: entry.data.title ?? entry.id, - publishedDate, - speechYear, - speaker, - source: entry.data.source ?? "", - categories, - tags, - body: entry.body ?? "", - }; -}); +const speechAdminEntries = speechEntries.map((entry) => speechAdminEntryFromArchiveEntry(entry, today)); const adminConfig = { analyticsDashboardUrl, contactEndpointConfigured: Boolean(contactFormEndpoint), @@ -1265,11 +1241,29 @@ const adminConfig = { } } - function loadSpeechEditor(speech) { + async function loadSpeechEditor(speech) { if (!speechForm) { return; } + const message = document.querySelector("[data-speech-message]"); + setMessage(message, "Loading speech body..."); + + if (!speech.body && speech.bodyUrl) { + try { + const response = await fetch(speech.bodyUrl); + + if (!response.ok) { + throw new Error(`Speech body request failed: ${response.status}`); + } + + speech = { ...speech, ...(await response.json()) }; + } catch { + setMessage(message, "Could not load that speech body. Try refreshing the admin page.", "error"); + return; + } + } + setFormValue(speechForm, "title", speech.title); setFormValue(speechForm, "publishedDate", speech.publishedDate); setFormValue(speechForm, "speechYear", speech.speechYear); diff --git a/src/pages/speeches/index.astro b/src/pages/speeches/index.astro index d37f001b..7d041b6f 100644 --- a/src/pages/speeches/index.astro +++ b/src/pages/speeches/index.astro @@ -92,6 +92,9 @@ const speakers = uniqueSorted(speeches.map((speech) => speech.speaker)); } let speeches = []; + let searchIndex = {}; + let searchTerms = []; + const termCache = new Map(); let renderTimer = 0; function normalize(value) { @@ -129,19 +132,82 @@ const speakers = uniqueSorted(speeches.map((speech) => speech.speaker)); speakerSelect.value = params.get("speaker") || ""; } - function matchesSpeech(speech, filters) { + function decodeDocumentIds(value) { + return new Set(String(value || "").split(",").filter(Boolean).map((documentId) => Number.parseInt(documentId, 36))); + } + + function unionSets(sets) { + const union = new Set(); + sets.forEach((set) => { + set.forEach((value) => union.add(value)); + }); + return union; + } + + function intersectSets(left, right) { + const smaller = left.size <= right.size ? left : right; + const larger = left.size <= right.size ? right : left; + const intersection = new Set(); + + smaller.forEach((value) => { + if (larger.has(value)) { + intersection.add(value); + } + }); + + return intersection; + } + + function documentsForTerm(term) { + if (termCache.has(term)) { + return termCache.get(term); + } + + const matches = []; + + if (searchIndex[term]) { + matches.push(decodeDocumentIds(searchIndex[term])); + } + + if (term.length >= 3) { + searchTerms.forEach((indexedTerm) => { + if (indexedTerm !== term && indexedTerm.startsWith(term)) { + matches.push(decodeDocumentIds(searchIndex[indexedTerm])); + } + }); + } + + const documentIds = matches.length ? unionSets(matches) : new Set(); + termCache.set(term, documentIds); + return documentIds; + } + + function searchCandidates(query) { + const terms = normalize(query).match(/[a-z0-9]+/g)?.filter((term) => term.length >= 2) ?? []; + + if (!terms.length) { + return null; + } + + return terms.reduce((candidateIds, term) => { + const termIds = documentsForTerm(term); + + if (!candidateIds) { + return termIds; + } + + return intersectSets(candidateIds, termIds); + }, null); + } + + function matchesSpeechMetadata(speech, filters) { if (filters.year && speech.year !== filters.year) { return false; } if (filters.speaker && speech.speaker !== filters.speaker) { return false; } - if (!filters.q) { - return true; - } - - const terms = normalize(filters.q).split(/\s+/).filter(Boolean); - return terms.every((term) => speech.searchText.includes(term)); + return true; } function appendMeta(parent, values) { @@ -176,7 +242,13 @@ const speakers = uniqueSorted(speeches.map((speech) => speech.speaker)); function render() { const filters = selectedFilters(); - const matched = speeches.filter((speech) => matchesSpeech(speech, filters)); + const candidateIds = searchCandidates(filters.q); + const matched = speeches.filter((speech, index) => { + if (candidateIds && !candidateIds.has(index)) { + return false; + } + return matchesSpeechMetadata(speech, filters); + }); const fragment = document.createDocumentFragment(); if (matched.length) { @@ -208,7 +280,10 @@ const speakers = uniqueSorted(speeches.map((speech) => speech.speaker)); return response.json(); }) .then((data) => { - speeches = data; + speeches = Array.isArray(data) ? data : data.items ?? []; + searchIndex = Array.isArray(data) ? {} : data.index ?? {}; + searchTerms = Object.keys(searchIndex); + termCache.clear(); render(); form.addEventListener("input", queueRender); form.addEventListener("change", render); diff --git a/src/pages/speeches/search-index.json.ts b/src/pages/speeches/search-index.json.ts index cf129d44..a287811c 100644 --- a/src/pages/speeches/search-index.json.ts +++ b/src/pages/speeches/search-index.json.ts @@ -1,7 +1,7 @@ -import { getSpeechArchive } from "../../lib/speeches"; +import { getCompactSpeechSearchIndex } from "../../lib/speeches"; export function GET() { - return new Response(JSON.stringify(getSpeechArchive()), { + return new Response(JSON.stringify(getCompactSpeechSearchIndex()), { headers: { "content-type": "application/json; charset=utf-8", "cache-control": "public, max-age=300",