200 lines
9.0 KiB
JavaScript
200 lines
9.0 KiB
JavaScript
// ── Audiobook span segmentation ─────────────────────────────────────────────
|
||
// Deterministic split of a book's text into narration and dialogue spans from
|
||
// its own quotation typography. The LLM is then only asked WHO speaks each
|
||
// dialogue span, never to re-emit the text: that removes the fusion, dropped
|
||
// and duplicated text an LLM introduces when it rewrites a whole passage.
|
||
//
|
||
// Pure functions, no DOM: loaded in the browser and importable from Node so
|
||
// the segmentation can be scored against the hand-corrected references.
|
||
(function (root) {
|
||
'use strict';
|
||
|
||
const STYLES = [
|
||
{ name: 'guillemets-inward', open: '»', close: '«' }, // German »…«
|
||
{ name: 'low-high', open: '„', close: '“' }, // German „…“
|
||
{ name: 'curly', open: '“', close: '”' }, // English “…”
|
||
{ name: 'guillemets-outward', open: '«', close: '»' }, // French «…»
|
||
];
|
||
const MIN_PAIRS = 5;
|
||
const MAX_SPEECH_CHARS = 12000; // a speech longer than this is treated as a lost closer
|
||
|
||
// Counts well-formed open…close pairs per style; the style with the most wins.
|
||
// One style per document: nested/secondary marks (›‹, ‘’) are deliberately ignored.
|
||
function detectStyle(text) {
|
||
let best = null, bestPairs = 0;
|
||
for (const st of STYLES) {
|
||
let pairs = 0, inside = false;
|
||
for (let i = 0; i < text.length; i++) {
|
||
const ch = text[i];
|
||
if (ch === st.open) inside = true;
|
||
else if (ch === st.close && inside) { pairs++; inside = false; }
|
||
}
|
||
if (pairs > bestPairs) { best = st; bestPairs = pairs; }
|
||
}
|
||
return bestPairs >= MIN_PAIRS ? best : null;
|
||
}
|
||
|
||
// Sentences of text[a,b) as [start,end) ranges, each ending at . ! ? … plus
|
||
// any trailing quote/bracket marks.
|
||
function sentenceRanges(text, a, b) {
|
||
const out = [];
|
||
const re = /[^.!?…]*[.!?…]+[»«“”"’)\]]*\s*|[^.!?…]+$/g;
|
||
const slice = text.slice(a, b);
|
||
let m;
|
||
while (true) {
|
||
m = re.exec(slice);
|
||
if (!m || !m[0].length) break;
|
||
out.push([a + m.index, a + m.index + m[0].length]);
|
||
}
|
||
return out;
|
||
}
|
||
|
||
// First/second-person pronouns and possessives are strong speech evidence in
|
||
// prose; third-person narration almost never contains them. `ihr`/`sie` are
|
||
// excluded (3rd-person possessive/pronoun) and polite `Ihr/Euch` only count
|
||
// when not sentence-initial.
|
||
const SPEECH_PRONOUN = /\b(?:ich|mich|mir|mein\w*|du|dich|dir|dein\w*|wir|uns|unser\w*|euch|euer|eure\w*|Euch|Euer|Eure\w*|I|me|my|you|your|we|us|our)\b/;
|
||
function looksLikeSpeech(sentence) {
|
||
const s = sentence.trim();
|
||
if (!s) return false;
|
||
if (SPEECH_PRONOUN.test(s)) return true;
|
||
if (/\?["»«”’)]*$/.test(s)) return true;
|
||
if (/!["»«”’)]*$/.test(s) && s.split(/\s+/).length <= 12) return true;
|
||
return false;
|
||
}
|
||
|
||
// A closing mark with no opener: the opener was lost in extraction. The
|
||
// speech starts at the earliest run of speech-like sentences directly before
|
||
// the mark, never before `floor` (the end of the previous span).
|
||
function recoverStart(text, floor, closeIdx) {
|
||
const sents = sentenceRanges(text, floor, closeIdx);
|
||
if (!sents.length) return closeIdx;
|
||
let k = sents.length - 1;
|
||
while (k > 0 && looksLikeSpeech(text.slice(sents[k - 1][0], sents[k - 1][1])) &&
|
||
!/\n\s*\n/.test(text.slice(sents[k - 1][1], sents[k][0]))) k--;
|
||
return sents[k][0];
|
||
}
|
||
|
||
// An opening mark whose closer is missing: the speech ends after the last of
|
||
// the leading run of speech-like sentences.
|
||
function recoverEnd(text, openEnd, limit) {
|
||
const sents = sentenceRanges(text, openEnd, limit);
|
||
if (!sents.length) return limit;
|
||
let k = 0;
|
||
while (k + 1 < sents.length && looksLikeSpeech(text.slice(sents[k + 1][0], sents[k + 1][1]))) k++;
|
||
return sents[k][1];
|
||
}
|
||
|
||
// A quote that is really a quoted term or title inside a narration sentence
|
||
// (»den gewundenen Weg der Krieger« handelte…) is not speech.
|
||
function isQuotedTerm(text, a, b) {
|
||
const inner = text.slice(a, b).trim();
|
||
if (!inner) return true;
|
||
if (/[.!?…]["»«”’)]*$/.test(inner)) return false;
|
||
const before = text.slice(0, a).trimEnd();
|
||
const after = text.slice(b).replace(/^[»«“”"’)\s]+/, '');
|
||
const beforeOpen = before && !/[.!?…:]["»«”’)]*$/.test(before);
|
||
const afterCont = /^[a-zäöüß,;]/.test(after);
|
||
return beforeOpen && afterCont;
|
||
}
|
||
|
||
function segmentByQuotes(text, opts) {
|
||
const style = (opts && opts.style) || detectStyle(text);
|
||
if (!style) return null;
|
||
const spans = [];
|
||
const push = (kind, a, b) => { if (text.slice(a, b).trim()) spans.push({ kind, start: a, end: b }); };
|
||
const paraBreakNext = (i) => /^\s*\n\s*\n/.test(text.slice(i, i + 6));
|
||
let narrStart = 0, qStart = 0, inQuote = false;
|
||
|
||
for (let i = 0; i < text.length; i++) {
|
||
const ch = text[i];
|
||
if (!inQuote) {
|
||
if (ch === style.open) { push('narration', narrStart, i); inQuote = true; qStart = i + 1; }
|
||
else if (ch === style.close) {
|
||
const s = recoverStart(text, Math.max(narrStart, 0), i);
|
||
push('narration', narrStart, s); push('dialogue', s, i); narrStart = i + 1;
|
||
}
|
||
} else if (ch === style.close) {
|
||
push('dialogue', qStart, i); inQuote = false; narrStart = i + 1;
|
||
} else if (ch === style.open) {
|
||
// A second opener with no closer between: the first quote lost its closer.
|
||
const end = recoverEnd(text, qStart, i);
|
||
push('dialogue', qStart, end); push('narration', end, i);
|
||
qStart = i + 1; narrStart = i + 1;
|
||
} else if (ch === '\n' && paraBreakNext(i)) {
|
||
// Paragraph break inside a quote. A long speech simply continues across
|
||
// paragraphs (some typesetting re-opens each one, some only closes the
|
||
// last), so keep going while a closing mark still follows before the next
|
||
// opener; only when it never does was the closer lost.
|
||
const rest = text.slice(i).replace(/^\s+/, '');
|
||
if (rest[0] !== style.open) {
|
||
const nextClose = text.indexOf(style.close, i), nextOpen = text.indexOf(style.open, i);
|
||
const closesFirst = nextClose !== -1 && (nextOpen === -1 || nextClose < nextOpen) && nextClose - i <= MAX_SPEECH_CHARS;
|
||
if (!closesFirst) {
|
||
const end = recoverEnd(text, qStart, i);
|
||
push('dialogue', qStart, end); push('narration', end, i);
|
||
inQuote = false; narrStart = i;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if (inQuote) {
|
||
const end = recoverEnd(text, qStart, text.length);
|
||
push('dialogue', qStart, end); push('narration', end, text.length);
|
||
} else push('narration', narrStart, text.length);
|
||
|
||
// Quoted terms/titles are narration; then merge neighbouring narration.
|
||
for (const sp of spans) if (sp.kind === 'dialogue' && isQuotedTerm(text, sp.start, sp.end)) sp.kind = 'narration';
|
||
const merged = [];
|
||
for (const sp of spans) {
|
||
const last = merged[merged.length - 1];
|
||
if (last && last.kind === 'narration' && sp.kind === 'narration') last.end = sp.end;
|
||
else merged.push(Object.assign({}, sp));
|
||
}
|
||
return merged.map((sp, index) => ({
|
||
index, kind: sp.kind, start: sp.start, end: sp.end,
|
||
text: text.slice(sp.start, sp.end).replace(/[»«]/g, '').replace(/[ \t]+/g, ' ').trim(),
|
||
})).filter(sp => sp.text);
|
||
}
|
||
|
||
// Groups consecutive spans into passages for one LLM call each. Cuts prefer a
|
||
// paragraph boundary once a window is reasonably full; a window never exceeds
|
||
// `hardChars` unless a single span alone does. Ids are global to the book
|
||
// (N/D + running index) so a window can be re-sent or resumed unchanged.
|
||
function buildWindows(spans, text, opts) {
|
||
const target = (opts && opts.targetChars) || 2400;
|
||
const hard = (opts && opts.hardChars) || 3600;
|
||
const windows = [];
|
||
let cur = [], chars = 0;
|
||
const flush = () => { if (cur.length) windows.push(cur); cur = []; chars = 0; };
|
||
for (let i = 0; i < spans.length; i++) {
|
||
const sp = spans[i];
|
||
if (cur.length && chars + sp.text.length > hard) flush();
|
||
cur.push(sp); chars += sp.text.length;
|
||
const next = spans[i + 1];
|
||
if (next && chars >= target) {
|
||
const gap = text.slice(sp.end, next.start);
|
||
if (/\n\s*\n/.test(gap) || chars >= hard * 0.9) flush();
|
||
}
|
||
}
|
||
flush();
|
||
// Attribution only needs the edges of a long span (speech tags sit there), so
|
||
// what is sent to the LLM is abbreviated; the caller keeps the full text.
|
||
const forLlm = (t, max) => t.length <= max ? t : t.slice(0, Math.ceil(max * 0.6)).trimEnd() + ' [...] ' + t.slice(-Math.floor(max * 0.4)).trimStart();
|
||
return windows.map((w, index) => ({
|
||
index,
|
||
spans: w.map(sp => ({
|
||
id: (sp.kind === 'dialogue' ? 'D' : 'N') + sp.index, kind: sp.kind, text: sp.text,
|
||
llmText: forLlm(sp.text, sp.kind === 'dialogue' ? 1500 : 900), start: sp.start, end: sp.end,
|
||
})),
|
||
hasDialogue: w.some(sp => sp.kind === 'dialogue'),
|
||
}));
|
||
}
|
||
|
||
const spansApi = { detectStyle, segmentByQuotes, buildWindows, looksLikeSpeech, recoverStart, recoverEnd, isQuotedTerm };
|
||
root.TTVC = root.TTVC || {};
|
||
root.TTVC.audiobook = root.TTVC.audiobook || {};
|
||
root.TTVC.audiobook.spans = spansApi;
|
||
if (typeof module !== 'undefined' && module.exports) module.exports = spansApi;
|
||
})(typeof window !== 'undefined' ? window : globalThis);
|