diff --git a/server/src/db/schema.ts b/server/src/db/schema.ts index f48bd94..a02428a 100644 --- a/server/src/db/schema.ts +++ b/server/src/db/schema.ts @@ -203,6 +203,14 @@ const MIGRATIONS: string[] = [ PRIMARY KEY (day, key) ); `, + /* 2 — long-lived ledger of every article URL ever seen, so an item that is still in a feed + after its article was pruned (e.g. rejected ones after 14 days) is never processed again */ ` + CREATE TABLE seen ( + url_hash TEXT PRIMARY KEY, + first_seen TEXT NOT NULL + ); + INSERT OR IGNORE INTO seen (url_hash, first_seen) SELECT url_hash, fetched_at FROM article; + `, ]; export function migrate(db: Database): number { diff --git a/server/src/http/activity.ts b/server/src/http/activity.ts index f8681ba..9013573 100644 --- a/server/src/http/activity.ts +++ b/server/src/http/activity.ts @@ -19,6 +19,16 @@ export function buildActivity(ctx: Ctx): ActivityDTO { const calls = db.prepare(`SELECT ok, ms, tokens_in, tokens_out FROM llm_call WHERE at >= ?`).all<{ ok: number; ms: number; tokens_in: number | null; tokens_out: number | null }>(since24h); const okMs = calls.filter((c) => c.ok).map((c) => c.ms).sort((a, b) => a - b); const breaker = ctx.llm.breakerInfo(); + const byPurpose = db + .prepare( + `SELECT purpose, COUNT(*) AS calls, SUM(CASE WHEN ok = 0 THEN 1 ELSE 0 END) AS errors, + COALESCE(SUM(tokens_in), 0) AS tokens_in, COALESCE(SUM(tokens_out), 0) AS tokens_out + FROM llm_call WHERE at >= ? GROUP BY purpose ORDER BY calls DESC`, + ) + .all<{ purpose: string; calls: number; errors: number; tokens_in: number; tokens_out: number }>(since24h) + .map((r) => ({ purpose: r.purpose, calls: r.calls, errors: r.errors, tokensIn: r.tokens_in, tokensOut: r.tokens_out })); + const pipelineCalls = byPurpose.filter((p) => !['test', 'playground', 'dry-run'].includes(p.purpose)).reduce((s, p) => s + p.calls, 0); + const published24h = count(`SELECT COUNT(*) AS n FROM story WHERE first_published_at >= ?`, since24h); const failures: ActivityFailure[] = [ ...db @@ -87,6 +97,15 @@ export function buildActivity(ctx: Ctx): ActivityDTO { p95LatencyMs: okMs.length ? okMs[Math.min(okMs.length - 1, Math.floor(okMs.length * 0.95))]! : null, tokensIn24h: calls.reduce((s, c) => s + (c.tokens_in ?? 0), 0), tokensOut24h: calls.reduce((s, c) => s + (c.tokens_out ?? 0), 0), + error: ctx.llm.errorMessage(), + byPurpose, + callsPerStory: published24h ? Math.round((pipelineCalls / published24h) * 10) / 10 : null, + }, + savings7d: { + duplicatesSkipped: statSum('duplicates_skipped'), + groupedByHeadline: statSum('grouped_by_headline'), + matchCallsSkipped: statSum('match_calls_skipped'), + triageBatched: statSum('triage_batched'), }, failures: failures.slice(0, 60), importance7d, diff --git a/server/src/http/api.ts b/server/src/http/api.ts index 249802e..2d8635b 100644 --- a/server/src/http/api.ts +++ b/server/src/http/api.ts @@ -319,6 +319,8 @@ async function playgroundRun(ctx: Ctx, url: string, models: string[]): Promise

) => { res.json({ llm: ctx.llm.state(), + llmError: ctx.llm.errorMessage(), waiting: waitingCount(ctx), version: ctx.config.version, latestDigestId: latestPublishedDigest(ctx)?.id ?? null, diff --git a/server/src/llm/client.ts b/server/src/llm/client.ts index ad27ac8..80f9e79 100644 --- a/server/src/llm/client.ts +++ b/server/src/llm/client.ts @@ -113,6 +113,8 @@ export class LlmClient { private failures = 0; private nextProbe = 0; private probing = false; + /** Set when the server rejects requests as misconfigured (401/403/404); pauses LLM work. */ + private configError: { message: string; until: number } | null = null; private timer: NodeJS.Timeout | null = null; private readonly listeners: (() => void)[] = []; @@ -140,12 +142,28 @@ export class LlmClient { this.sem.limit = this.getSettings().concurrency; this.modeCache.clear(); // New connection settings deserve a fresh chance. + const hadConfigError = this.configError !== null; + this.configError = null; if (this.failures > 0) this.reset(); + else if (hadConfigError) for (const l of this.listeners) l(); + } + + /** Why LLM work is paused because the server rejected requests, if it is. */ + errorMessage(): string | null { + if (!this.configError) return null; + if (Date.now() >= this.configError.until) { + // Try again after a while (the server may have been fixed without a settings change). + this.configError = null; + for (const l of this.listeners) l(); + return null; + } + return this.configError.message; } state(): LlmState { const s = this.getSettings(); if (!s.baseUrl || !s.summaryModel) return 'unconfigured'; + if (this.errorMessage()) return 'offline'; return this.failures >= BREAKER_THRESHOLD ? 'offline' : 'ok'; } @@ -252,6 +270,7 @@ export class LlmClient { const s = this.conn(opts.override); const affectBreaker = !opts.override; if (affectBreaker && this.failures >= BREAKER_THRESHOLD) throw new LlmUnavailableError('LLM is offline'); + if (affectBreaker && this.errorMessage()) throw new LlmUnavailableError(this.errorMessage()!); if (!opts.model) throw new LlmUnavailableError('No model configured'); const release = await this.sem.acquire(); const started = Date.now(); @@ -273,6 +292,7 @@ export class LlmClient { max_tokens: opts.maxTokens ?? 2048, stream: false, }; + if (s.reasoningEffort) body.reasoning_effort = s.reasoningEffort; if (mode === 'json_schema') { body.response_format = { type: 'json_schema', json_schema: { name: opts.schemaName, strict: true, schema: opts.schema } }; } else if (mode === 'json_object') { @@ -282,9 +302,11 @@ export class LlmClient { try { data = await this.request('/chat/completions', body, s, s.timeoutSec * 1000); } catch (err) { - if (err instanceof LlmRequestError && mode !== 'prompt') { + // Only a rejected request *shape* (400/422) is worth retrying in a simpler output mode; + // a bad key, unknown model or wrong URL would just fail three times. + if (err instanceof LlmRequestError && (err.status === 400 || err.status === 422) && mode !== 'prompt') { firstError ??= err; - continue; // try a simpler structured-output mode + continue; } throw err; } @@ -298,6 +320,14 @@ export class LlmClient { const e = err as Error; this.record(opts.purpose, opts.model, false, Date.now() - started, null, null, e.message); if (affectBreaker && err instanceof LlmUnavailableError) this.recordFailure(e); + if (affectBreaker && err instanceof LlmRequestError && [401, 403, 404].includes(err.status)) { + // Misconfiguration (bad key, unknown model, wrong URL): every request would fail the same + // way, so pause all LLM work instead of burning attempts on each article. + const message = `The LLM server rejected requests for "${opts.model}" (HTTP ${err.status}). Check Settings → LLM connection. ${err.message.slice(0, 200)}`; + if (!this.configError) log.warn('pausing LLM work: the server rejected the request', { model: opts.model, status: err.status }); + this.configError = { message, until: Date.now() + 10 * 60_000 }; + throw new LlmUnavailableError(message); + } throw err; } finally { release(); @@ -341,6 +371,7 @@ export class LlmClient { // One repair attempt with the model's own answer in context. const repairBody: Record = { + ...(s.reasoningEffort ? { reasoning_effort: s.reasoningEffort } : {}), model: opts.model, messages: [ ...messages, diff --git a/server/src/llm/prompts.ts b/server/src/llm/prompts.ts index fc5a903..673f8e4 100644 --- a/server/src/llm/prompts.ts +++ b/server/src/llm/prompts.ts @@ -203,6 +203,66 @@ export function validateTriage(v: unknown): TriageOutput | string { }; } +// ───────────────────────────── triage, batched ───────────────────────────── + +export const TRIAGE_BATCH_SCHEMA: JsonSchema = { + type: 'object', + properties: { + results: { + type: 'array', + items: { + type: 'object', + properties: { + n: { type: 'integer' }, + decision: { type: 'string', enum: ['keep', 'reject'] }, + rule_id: { type: ['integer', 'null'] }, + reason: { type: 'string' }, + confidence: { type: 'number' }, + }, + required: ['n', 'decision', 'rule_id', 'reason', 'confidence'], + additionalProperties: false, + }, + }, + }, + required: ['results'], + additionalProperties: false, +}; + +/** Several articles from one source in one call: the rules and instructions are sent once. */ +export function buildTriageBatchPrompt( + instructions: string, + negatives: TriageRuleRef[], + exceptions: TriageRuleRef[], + examples: TriageExample[], + articles: (ArticleForPrompt & { n: number })[], +): { system: string; user: string } { + const single = buildTriagePrompt(instructions, negatives, exceptions, examples, articles[0]!); + const system = single.system.replace( + /\n\nOutput format:[\s\S]*$/, + `\n\nYou get several numbered articles. Judge each one on its own. + +Output format: reply with ONLY a JSON object: +{"results": [{"n":

, "decision": "keep" or "reject", "rule_id": , "reason": "", "confidence": }]} +Include every article number exactly once.`, + ); + const user = articles.map((a) => articleBlock(a, `article_${a.n}`)).join('\n\n'); + return { system, user }; +} + +export function validateTriageBatch(v: unknown): (TriageOutput & { n: number })[] | string { + if (!v || typeof v !== 'object') return 'expected a JSON object'; + const results = (v as Record).results; + if (!Array.isArray(results)) return 'results must be an array'; + const out: (TriageOutput & { n: number })[] = []; + for (const r of results) { + if (!r || typeof r !== 'object') continue; + const n = Number((r as Record).n); + const t = validateTriage(r); + if (Number.isInteger(n) && typeof t !== 'string') out.push({ ...t, n }); + } + return out; +} + // ───────────────────────────── dry run (batched) ───────────────────────────── export const DRY_RUN_SCHEMA: JsonSchema = { diff --git a/server/src/pipeline/cluster.ts b/server/src/pipeline/cluster.ts index 7b27348..2d46513 100644 --- a/server/src/pipeline/cluster.ts +++ b/server/src/pipeline/cluster.ts @@ -1,4 +1,5 @@ import type { Ctx } from '../context.ts'; +import { bumpStat } from '../context.ts'; import { HOUR, isoMinus } from '../lib/time.ts'; import { truncate } from '../lib/text.ts'; import { log } from '../lib/log.ts'; @@ -157,7 +158,47 @@ function sourceName(ctx: Ctx, id: number): string { return ctx.db.prepare(`SELECT name FROM source WHERE id = ?`).get<{ name: string }>(id)?.name ?? 'Unknown'; } -const MAX_LLM_CANDIDATES = 60; +/** Recent stories scanned for a possible match, and the most sent to the model. */ +const SCAN_STORIES = 200; +const MAX_LLM_CANDIDATES = 12; + +/** Capitalised words that start sentences or headlines in Croatian and English and say nothing about the event. */ +const COMMON = new Set( + ( + 'the a an this that these those there here in on at for from with after before when how why what who new his her its their our your ' + + 'we you they it he she said says says breaking live update watch video photo photos exclusive report reports ' + + 'u na za iz od do po kako sto zasto gdje tko koji koja koje novi nova novo ovo ova ovaj taj ta to nakon prema ' + + 'danas jucer sutra foto video uzivo pogledajte evo ovo ovako sve svi vijesti hrvatska hrvatski hrvatske ' + + // Dates are shared by unrelated stories of the same day. + 'monday tuesday wednesday thursday friday saturday sunday january february march april may june july august ' + + 'september october november december ponedjeljak utorak srijeda srijedu cetvrtak petak subota subotu nedjelja nedjelju ' + + 'mr mrs ms dr' + ).split(/\s+/), +); + +function fold(s: string): string { + return s.toLowerCase().normalize('NFKD').replace(/\p{M}+/gu, '').replace(/đ/g, 'd'); +} + +/** + * Cheap "could these be the same event?" signal that works across Croatian and English: names + * (capitalised words, cut to 4 letters so "Trump"/"Trumpov" and "Bahrain"/"Bahrein" agree) and + * numbers. A story that shares none of them with the new article is not worth asking the model about. + */ +export function eventStems(text: string): Set { + const out = new Set(); + for (const m of text.matchAll(/(? { @@ -170,13 +211,38 @@ async function clusterByLlm(ctx: Ctx, a: ArticleRow): Promise { WHERE s.first_reported_at >= ? AND s.hidden = 0 AND s.state != 'failed' ORDER BY s.last_reported_at DESC LIMIT ?`, ) - .all<{ id: number; title: string | null; lead_title: string | null }>(since, MAX_LLM_CANDIDATES); - if (!stories.length) { + .all<{ id: number; title: string | null; lead_title: string | null }>(since, SCAN_STORIES); + + // Free pre-filter: only stories that share a name or number with the new article are candidates. + const mine = eventStems(`${a.title}\n${articleText(a).slice(0, 600)}`); + const memberTitles = new Map(); + if (stories.length) { + const ids = stories.map((x) => x.id); + for (const r of ctx.db + .prepare(`SELECT story_id, title FROM article WHERE story_id IN (${ids.map(() => '?').join(',')})`) + .all<{ story_id: number; title: string }>(...ids)) { + const list = memberTitles.get(r.story_id) ?? []; + list.push(r.title); + memberTitles.set(r.story_id, list); + } + } + const scored = stories + .map((x) => { + const theirs = eventStems([x.title ?? '', ...(memberTitles.get(x.id) ?? [])].join('\n')); + let shared = 0; + for (const st of mine) if (theirs.has(st)) shared++; + return { story: x, shared }; + }) + .filter((x) => x.shared > 0) + .sort((p, q) => q.shared - p.shared) + .slice(0, MAX_LLM_CANDIDATES); + if (!scored.length) { + if (stories.length) bumpStat(ctx.db, 'match_calls_skipped'); createStoryForArticle(ctx, a); ctx.wake(); return; } - const candidates = stories.map((x) => ({ id: x.id, title: x.title ?? x.lead_title ?? '', originalTitle: x.lead_title })); + const candidates = scored.map(({ story: x }) => ({ id: x.id, title: x.title ?? x.lead_title ?? '', originalTitle: x.lead_title })); const prompt = buildStoryMatchPrompt( promptText(s, 'sameEvent'), { sourceName: sourceName(ctx, a.source_id), title: a.title, publishedAt: a.published_at, categories: parseJsonArray(a.categories), text: truncate(articleText(a), 800) }, diff --git a/server/src/pipeline/fetcher.ts b/server/src/pipeline/fetcher.ts index 7473e90..68a62b1 100644 --- a/server/src/pipeline/fetcher.ts +++ b/server/src/pipeline/fetcher.ts @@ -7,6 +7,7 @@ import { normalizeLang, normalizeTitle, sha1 } from '../lib/text.ts'; import { HOUR, MINUTE, isoMinus, isoPlus, nowIso } from '../lib/time.ts'; import { log, errorMessage } from '../lib/log.ts'; import { setMeta } from '../db/schema.ts'; +import { attachArticle } from './stories.ts'; export interface SourceRow { id: number; @@ -80,6 +81,19 @@ export async function fetchSource(ctx: Ctx, sourceId: number): Promise<{ newItem } } +interface Twin { + id: number; + source_id: number; + state: string; + story_id: number | null; +} + +/** + * Stores new feed items. Everything here is free (no AI): an item is dropped when its URL was + * ever seen before, when its source already delivered the same GUID or headline, and an + * identical headline from another outlet (wire copy) joins the existing story, or is filtered + * the same way, without triage, grouping or a summary. + */ function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType['items'], feedLang: string | null): number { const s = ctx.settings.get().pipeline; const now = new Date(); @@ -87,31 +101,60 @@ function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType= ?`); + const byTitle = ctx.db.prepare( + `SELECT id, source_id, state, story_id FROM article WHERE title_hash = ? AND fetched_at >= ? ORDER BY id LIMIT 1`, + ); const insert = ctx.db.prepare( `INSERT INTO article (source_id, guid, url, url_hash, title_hash, title, excerpt, categories, text, partial, author, image_url, language, published_at, fetched_at, state, state_changed_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, ); + const copyVerdict = ctx.db.prepare( + `INSERT OR REPLACE INTO verdict (article_id, decision, decided_by, rule_id, rule_text, reason, confidence, model, prompt_version, overridden, decided_at) + SELECT ?, decision, decided_by, rule_id, rule_text, 'Same headline as an article already filtered. ' || reason, confidence, model, prompt_version, 0, ? + FROM verdict WHERE article_id = ?`, + ); let fresh = 0; + let duplicates = 0; + const attachLater: { storyId: number; article: { id: number; published_at: string | null; fetched_at: string } }[] = []; + ctx.db.transaction(() => { for (const item of items) { const url = item.link ? canonicalizeUrl(item.link) : null; if (!url) continue; const urlHash = sha1(url); - if (byUrl.get(urlHash)) continue; - if (item.guid && byGuid.get(src.id, item.guid)) continue; + if (seen.get(urlHash) || byUrl.get(urlHash)) continue; + markSeen.run(urlHash, fetchedAt); + if (item.guid && byGuid.get(src.id, item.guid)) { + duplicates++; + continue; + } const normTitle = normalizeTitle(item.title); const titleHash = sha1(normTitle); - if (normTitle.length > 12 && byTitle.get(titleHash, titleWindow)) continue; // Feeds sometimes publish future dates; never trust a date later than now. const published = item.publishedAt && item.publishedAt.getTime() <= now.getTime() ? item.publishedAt.toISOString() : fetchedAt; const tooOld = published < maxAgeCutoff || (!src.initial_import_done && published < firstAddCutoff); const fullText = src.full_text === 1; - const state = tooOld ? 'skipped' : fullText ? 'new' : 'extracted'; - insert.run( + + let state = tooOld ? 'skipped' : fullText ? 'new' : 'extracted'; + let twin: Twin | undefined; + if (!tooOld && normTitle.length > 12) { + twin = byTitle.get(titleHash, titleWindow); + if (twin) { + if (twin.source_id === src.id || (twin.state !== 'grouped' && twin.state !== 'rejected') || (twin.state === 'grouped' && !twin.story_id)) { + // Same outlet under a new URL, or the twin is still being processed: nothing to add. + duplicates++; + continue; + } + // Identical headline from another outlet: reuse the twin's outcome instead of the AI. + state = twin.state === 'rejected' ? 'rejected' : 'kept'; + } + } + const id = insert.run( src.id, item.guid, url, @@ -120,8 +163,8 @@ function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType): void { +/** + * Adds an article to an existing story and triggers regeneration / feed re-evaluation as needed. + * `rewrite: false` for wire copy: the same text adds a source but nothing worth a new summary. + */ +export function attachArticle( + ctx: Ctx, + storyId: number, + a: Pick, + opts: { rewrite?: boolean } = {}, +): void { const now = nowIso(); ctx.db.transaction(() => { ctx.db.prepare(`UPDATE article SET story_id = ?, state = 'grouped', state_changed_at = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ?`).run(storyId, now, a.id); @@ -268,7 +276,7 @@ export function attachArticle(ctx: Ctx, storyId: number, a: Pick= c.regenAtSources; const written = s.version_written_at ? Date.parse(s.version_written_at) : Date.now(); const gap = c.regenGapHours > 0 && Date.parse(reportedAt(a)) - written > c.regenGapHours * HOUR; diff --git a/server/src/pipeline/summarize.ts b/server/src/pipeline/summarize.ts index 3282fb2..53d6055 100644 --- a/server/src/pipeline/summarize.ts +++ b/server/src/pipeline/summarize.ts @@ -124,7 +124,8 @@ export async function summarizeStory(ctx: Ctx, storyId: number): Promise { const lead = all.find((a) => a.id === story.lead_article_id) ?? all[0]!; const others = all.filter((a) => a.id !== lead.id).sort((x, y) => x.partial - y.partial); const chosen = [lead, ...others].slice(0, 4); - const perArticle = chosen.length > 1 ? Math.max(2000, Math.floor(settings.llm.maxArticleChars / chosen.length)) : settings.llm.maxArticleChars; + // A rewrite from several sources shares one article-sized budget instead of multiplying it. + const perArticle = chosen.length > 1 ? Math.max(1500, Math.floor(settings.llm.maxArticleChars / chosen.length)) : settings.llm.maxArticleChars; const written = await writeTitleAndDescription(ctx.llm, settings, chosen.map((a) => articleForPrompt(ctx, a, perArticle))); if (written.formatRetries) bumpStat(ctx.db, 'format_retries', written.formatRetries); if (!written.languageOk) { diff --git a/server/src/pipeline/triage.ts b/server/src/pipeline/triage.ts index ff759b7..ef81e55 100644 --- a/server/src/pipeline/triage.ts +++ b/server/src/pipeline/triage.ts @@ -3,14 +3,19 @@ import type { DryRunHit, DryRunStatus, RuleDTO, RuleInput } from '../../../share import type { Ctx } from '../context.ts'; import { buildDryRunPrompt, + buildTriageBatchPrompt, buildTriagePrompt, DRY_RUN_SCHEMA, promptText, promptVersion, + TRIAGE_BATCH_SCHEMA, TRIAGE_SCHEMA, validateDryRun, validateTriage, + validateTriageBatch, } from '../llm/prompts.ts'; +import type { TriageOutput, TriageRuleRef } from '../llm/prompts.ts'; +import { bumpStat } from '../context.ts'; import { nowIso } from '../lib/time.ts'; import { truncate } from '../lib/text.ts'; import { errorMessage } from '../lib/log.ts'; @@ -120,13 +125,9 @@ function writeVerdict(ctx: Ctx, articleId: number, v: VerdictInput): void { .run(articleId, v.decision, v.decidedBy, v.ruleId, v.ruleText, v.reason, v.confidence, v.model, v.promptVersion, nowIso()); } -/** Whether triage for this article needs the LLM (any enabled AI negative rule applies). */ +/** Whether triage for this article needs the LLM (no keyword rule decides it and an AI negative applies). */ export function triageNeedsLlm(ctx: Ctx, a: Pick): boolean { - const rules = activeRules(ctx, a.source_id); - const hay = keywordHaystack(a); - if (rules.some((r) => r.kind === 'keyword' && r.polarity === 'exception' && keywordMatcher(r.text)(hay))) return false; - if (rules.some((r) => r.kind === 'keyword' && r.polarity === 'negative' && keywordMatcher(r.text)(hay))) return false; - return rules.some((r) => r.kind === 'ai' && r.polarity === 'negative'); + return freeVerdict(activeRules(ctx, a.source_id), a) === null; } function sourceName(ctx: Ctx, sourceId: number): string { @@ -148,44 +149,85 @@ export interface TriageVerdict extends VerdictInput { ms: number | null; } -/** Decides keep/reject without touching the database (used by triage and the playground). */ -export async function evaluateTriage(ctx: Ctx, input: TriageInput): Promise { - const rules = - input.sourceId === null - ? listRules(ctx).filter((r) => r.enabled) - : activeRules(ctx, input.sourceId); - const hay = keywordHaystack(input); - const base = { model: null, promptVersion: null, confidence: null, ms: null }; +/** Characters of article text the triage model sees: headline + lede is enough to judge the subject. */ +const TRIAGE_TEXT_CHARS = 800; +/** Articles from one source judged in one call. */ +export const TRIAGE_BATCH_SIZE = 8; +const NO_LLM = { model: null, promptVersion: null, confidence: null, ms: null }; + +/** Keyword exceptions, keyword negatives, and "no AI rule applies" — all free. Null = needs the LLM. */ +function freeVerdict(rules: RuleDTO[], input: Pick): TriageVerdict | null { + const hay = keywordHaystack(input); for (const r of rules.filter((x) => x.kind === 'keyword' && x.polarity === 'exception')) { const hit = keywordMatcher(r.text)(hay); - if (hit) return { ...base, decision: 'keep', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Kept by exception: matched "${hit}"`, hitRuleId: r.id }; + if (hit) return { ...NO_LLM, decision: 'keep', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Kept by exception: matched "${hit}"`, hitRuleId: r.id }; } for (const r of rules.filter((x) => x.kind === 'keyword' && x.polarity === 'negative')) { const hit = keywordMatcher(r.text)(hay); - if (hit) return { ...base, decision: 'reject', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Matched keyword "${hit}"`, hitRuleId: r.id }; + if (hit) return { ...NO_LLM, decision: 'reject', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Matched keyword "${hit}"`, hitRuleId: r.id }; } + if (!rules.some((r) => r.kind === 'ai' && r.polarity === 'negative')) { + return { ...NO_LLM, decision: 'keep', decidedBy: 'none', ruleId: null, ruleText: null, reason: 'No rule applies', hitRuleId: null }; + } + return null; +} +/** Turns the model's answer into a verdict: reject only with a valid rule at or above the threshold. */ +function interpretTriage( + out: TriageOutput, + rules: RuleDTO[], + threshold: number, + common: { model: string; promptVersion: string; ms: number | null }, +): TriageVerdict { const negatives = rules.filter((r) => r.kind === 'ai' && r.polarity === 'negative'); const exceptions = rules.filter((r) => r.kind === 'ai' && r.polarity === 'exception'); - if (negatives.length === 0) { - return { ...base, decision: 'keep', decidedBy: 'none', ruleId: null, ruleText: null, reason: 'No rule applies', hitRuleId: null }; + const negRule = negatives.find((r) => r.id === out.ruleId); + const excRule = exceptions.find((r) => r.id === out.ruleId); + const base = { ...common, decidedBy: 'ai' as const, confidence: out.confidence }; + if (out.decision === 'reject' && negRule && (out.confidence ?? 0) >= threshold) { + return { ...base, decision: 'reject', ruleId: negRule.id, ruleText: negRule.text, reason: out.reason, hitRuleId: negRule.id }; } + let reason = out.reason; + if (out.decision === 'reject') { + reason = negRule + ? `Kept: confidence ${Math.round((out.confidence ?? 0) * 100)}% is below the ${Math.round(threshold * 100)}% threshold. ${out.reason}` + : `Kept: the model did not name a valid rule. ${out.reason}`; + } + return { + ...base, + decision: 'keep', + ruleId: excRule?.id ?? negRule?.id ?? null, + ruleText: excRule?.text ?? negRule?.text ?? null, + reason, + hitRuleId: excRule?.id ?? null, + }; +} + +function ruleRefs(rules: RuleDTO[], polarity: 'negative' | 'exception'): TriageRuleRef[] { + return rules.filter((r) => r.kind === 'ai' && r.polarity === polarity).map((r) => ({ id: r.id, text: r.text })); +} + +/** Decides keep/reject without touching the database (used by triage and the playground). */ +export async function evaluateTriage(ctx: Ctx, input: TriageInput): Promise { + const rules = input.sourceId === null ? listRules(ctx).filter((r) => r.enabled) : activeRules(ctx, input.sourceId); + const free = freeVerdict(rules, input); + if (free) return free; const settings = ctx.settings.get(); const model = settings.llm.triageModel || settings.llm.summaryModel; const instructions = promptText(settings, 'triage'); const prompt = buildTriagePrompt( instructions, - negatives.map((r) => ({ id: r.id, text: r.text })), - exceptions.map((r) => ({ id: r.id, text: r.text })), + ruleRefs(rules, 'negative'), + ruleRefs(rules, 'exception'), filterExamples(ctx.db, settings.pipeline.examplesInPrompt), { sourceName: input.sourceName, title: input.title, publishedAt: input.publishedAt, categories: parseJsonArray(input.categories), - text: truncate(input.text, 1500), + text: truncate(input.text, TRIAGE_TEXT_CHARS), }, ); const res = await ctx.llm.chatJson({ @@ -198,46 +240,96 @@ export async function evaluateTriage(ctx: Ctx, input: TriageInput): Promise r.id === out.ruleId); - const excRule = exceptions.find((r) => r.id === out.ruleId); - const threshold = settings.pipeline.rejectThreshold; - const common = { decidedBy: 'ai' as const, confidence: out.confidence, model, promptVersion: pv, ms: res.ms }; - if (out.decision === 'reject' && negRule && (out.confidence ?? 0) >= threshold) { - return { ...common, decision: 'reject', ruleId: negRule.id, ruleText: negRule.text, reason: out.reason, hitRuleId: negRule.id }; - } - let reason = out.reason; - if (out.decision === 'reject') { - reason = negRule - ? `Kept: confidence ${Math.round((out.confidence ?? 0) * 100)}% is below the ${Math.round(threshold * 100)}% threshold. ${out.reason}` - : `Kept: the model did not name a valid rule. ${out.reason}`; - } - return { - ...common, - decision: 'keep', - ruleId: excRule?.id ?? negRule?.id ?? null, - ruleText: excRule?.text ?? negRule?.text ?? null, - reason, - hitRuleId: excRule?.id ?? null, - }; + return interpretTriage(res.value, rules, settings.pipeline.rejectThreshold, { model, promptVersion: promptVersion(instructions), ms: res.ms }); +} + +function inputOf(ctx: Ctx, a: ArticleRow): TriageInput { + return { sourceId: a.source_id, sourceName: sourceName(ctx, a.source_id), title: a.title, categories: a.categories, text: articleText(a), publishedAt: a.published_at }; +} + +function record(ctx: Ctx, articleId: number, v: TriageVerdict): void { + if (v.hitRuleId !== null) bumpRule(ctx, v.hitRuleId); + writeVerdict(ctx, articleId, v); } /** Decides keep/reject for one stored article and writes the verdict. */ export async function triageArticle(ctx: Ctx, a: ArticleRow): Promise<'keep' | 'reject'> { - const v = await evaluateTriage(ctx, { - sourceId: a.source_id, - sourceName: sourceName(ctx, a.source_id), - title: a.title, - categories: a.categories, - text: articleText(a), - publishedAt: a.published_at, - }); - if (v.hitRuleId !== null) bumpRule(ctx, v.hitRuleId); - writeVerdict(ctx, a.id, v); + const v = await evaluateTriage(ctx, inputOf(ctx, a)); + record(ctx, a.id, v); return v.decision; } +/** + * Triage for several articles from one source. Free decisions (keyword rules, no AI rule) are + * made first; the rest share one LLM call, so the rules and instructions are sent once. + */ +export async function triageBatch(ctx: Ctx, articles: ArticleRow[]): Promise> { + const out = new Map(); + if (!articles.length) return out; + const rules = activeRules(ctx, articles[0]!.source_id); + const needAi: ArticleRow[] = []; + for (const a of articles) { + const free = freeVerdict(rules, a); + if (free) { + record(ctx, a.id, free); + out.set(a.id, free.decision); + } else { + needAi.push(a); + } + } + if (needAi.length === 0) return out; + if (needAi.length === 1) { + out.set(needAi[0]!.id, await triageArticle(ctx, needAi[0]!)); + return out; + } + + const settings = ctx.settings.get(); + const model = settings.llm.triageModel || settings.llm.summaryModel; + const instructions = promptText(settings, 'triage'); + const name = sourceName(ctx, needAi[0]!.source_id); + const prompt = buildTriageBatchPrompt( + instructions, + ruleRefs(rules, 'negative'), + ruleRefs(rules, 'exception'), + filterExamples(ctx.db, settings.pipeline.examplesInPrompt), + needAi.map((a, i) => ({ + n: i + 1, + sourceName: name, + title: a.title, + publishedAt: a.published_at, + categories: parseJsonArray(a.categories), + text: truncate(articleText(a), TRIAGE_TEXT_CHARS), + })), + ); + const res = await ctx.llm.chatJson({ + purpose: 'triage', + model, + system: prompt.system, + user: prompt.user, + schemaName: 'triage_batch', + schema: TRIAGE_BATCH_SCHEMA, + validate: validateTriageBatch, + maxTokens: 512 + 160 * needAi.length, + }); + const byN = new Map(res.value.map((r) => [r.n, r])); + const common = { model, promptVersion: promptVersion(instructions), ms: res.ms }; + const missing: ArticleRow[] = []; + needAi.forEach((a, i) => { + const r = byN.get(i + 1); + if (!r) { + missing.push(a); + return; + } + const v = interpretTriage(r, rules, settings.pipeline.rejectThreshold, common); + record(ctx, a.id, v); + out.set(a.id, v.decision); + }); + bumpStat(ctx.db, 'triage_batched', needAi.length - missing.length - 1); + // The model skipped some: judge those one by one rather than guessing. + for (const a of missing) out.set(a.id, await triageArticle(ctx, a)); + return out; +} + // ───────────────────────────── rule CRUD ───────────────────────────── function normalizeRuleInput(input: Partial): Partial { diff --git a/server/src/pipeline/worker.ts b/server/src/pipeline/worker.ts index b696e5a..d232a31 100644 --- a/server/src/pipeline/worker.ts +++ b/server/src/pipeline/worker.ts @@ -4,7 +4,7 @@ import { log, errorMessage } from '../lib/log.ts'; import { LlmUnavailableError } from '../llm/client.ts'; import { normalizeLang } from '../lib/text.ts'; import { extractArticle } from './extract.ts'; -import { triageArticle, triageNeedsLlm } from './triage.ts'; +import { TRIAGE_BATCH_SIZE, triageBatch, triageNeedsLlm } from './triage.ts'; import { clusterArticle, clusteringActive } from './cluster.ts'; import { PermanentError, summarizeStory } from './summarize.ts'; import { nextPendingDigest, processDigest } from './digest.ts'; @@ -14,7 +14,15 @@ const EXTRACT_SLOTS = 3; const MAX_ATTEMPTS = 3; const BACKOFF = [1 * MINUTE, 5 * MINUTE, 15 * MINUTE]; -type Task = { key: string; lane: 'extract' | 'llm'; label: string; run: () => Promise; onError: (err: unknown) => void }; +type Task = { + key: string; + /** Extra items this task owns (batched triage), claimed alongside `key`. */ + extraKeys?: string[]; + lane: 'extract' | 'llm'; + label: string; + run: () => Promise; + onError: (err: unknown) => void; +}; /** * Background processing. State lives in the database (article.state / story.state), so a @@ -78,7 +86,8 @@ export class Worker { } private launch(t: Task): void { - this.claimed.add(t.key); + const keys = [t.key, ...(t.extraKeys ?? [])]; + for (const k of keys) this.claimed.add(k); this.active[t.lane]++; void t .run() @@ -90,7 +99,7 @@ export class Worker { } }) .finally(() => { - this.claimed.delete(t.key); + for (const k of keys) this.claimed.delete(k); this.active[t.lane]--; this.wake(); }); @@ -112,6 +121,10 @@ export class Worker { return (a.published_at ?? a.fetched_at) < isoMinus(maxAge * HOUR); } + private sourceStillExtracting(sourceId: number): boolean { + return !!this.ctx.db.prepare(`SELECT 1 FROM article WHERE source_id = ? AND state = 'new' LIMIT 1`).get(sourceId); + } + private skip(a: ArticleRow): void { this.ctx.db.prepare(`UPDATE article SET state = 'skipped', state_changed_at = ? WHERE id = ?`).run(nowIso(), a.id); } @@ -166,15 +179,37 @@ export class Worker { } } - for (const a of this.dueArticles('extracted', 50)) { + const extracted = this.dueArticles('extracted', 100); + for (const a of extracted) { const key = `a${a.id}`; if (this.claimed.has(key)) continue; if (this.tooOld(a)) { this.skip(a); continue; } - if (!llmUp && triageNeedsLlm(ctx, a)) continue; - return { key, lane: 'llm', label: 'triage', run: () => this.runTriage(a), onError: (err) => this.articleFailed(a, err) }; + const needsLlm = triageNeedsLlm(ctx, a); + if (!llmUp && needsLlm) continue; + // Give a fetch's other articles a moment to finish extracting so they can join this call. + if (needsLlm && (a.state_changed_at > isoMinus(60_000)) && this.sourceStillExtracting(a.source_id)) continue; + // Articles from the same source share the rules, so they can share one triage call. + const batch = [a]; + if (needsLlm) { + for (const b of extracted) { + if (batch.length >= TRIAGE_BATCH_SIZE) break; + if (b.id === a.id || b.source_id !== a.source_id || this.claimed.has(`a${b.id}`) || this.tooOld(b)) continue; + if (triageNeedsLlm(ctx, b)) batch.push(b); + } + } + return { + key, + extraKeys: batch.slice(1).map((b) => `a${b.id}`), + lane: 'llm', + label: 'triage', + run: () => this.runTriage(batch), + onError: (err) => { + for (const b of batch) this.articleFailed(b, err); + }, + }; } return null; } @@ -193,11 +228,12 @@ export class Worker { .run(ex.text || null, ex.partial ? 1 : 0, ex.imageUrl, ex.author, language, nowIso(), ex.error, a.id); } - private async runTriage(a: ArticleRow): Promise { - const decision = await triageArticle(this.ctx, a); - this.ctx.db - .prepare(`UPDATE article SET state = ?, state_changed_at = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ?`) - .run(decision === 'reject' ? 'rejected' : 'kept', nowIso(), a.id); + private async runTriage(batch: ArticleRow[]): Promise { + const decisions = await triageBatch(this.ctx, batch); + const update = this.ctx.db.prepare( + `UPDATE article SET state = ?, state_changed_at = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ? AND state = 'extracted'`, + ); + for (const [id, decision] of decisions) update.run(decision === 'reject' ? 'rejected' : 'kept', nowIso(), id); } // ───────────────────────────── failure handling ───────────────────────────── diff --git a/server/src/savings.test.ts b/server/src/savings.test.ts new file mode 100644 index 0000000..18e9992 --- /dev/null +++ b/server/src/savings.test.ts @@ -0,0 +1,183 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { AddressInfo } from 'node:net'; +import { loadConfig } from './config.ts'; +import { startRssFeed } from './server.ts'; +import type { RunningApp } from './server.ts'; +import { startMockLlm } from './testing/mock-llm.ts'; +import type { MockLlm } from './testing/mock-llm.ts'; +import { eventStems } from './pipeline/cluster.ts'; +import type { ActivityDTO, StatusDTO, StoryListResponse } from '../../shared/types.ts'; + +/** Everything that keeps AI calls down: duplicate ledger, free grouping, pre-filter, batching, pausing on bad config. */ + +const BODY = (s: string) => `${s} `.repeat(8); +const ARTICLES: Record = { + '/a/quake': ['Earthquake hits Turkey, hundreds injured', BODY('A strong earthquake hit eastern Turkey on Sunday, injuring hundreds of people in Malatya.')], + '/a/bank': ['Central bank keeps interest rates unchanged', BODY('The European Central Bank kept rates unchanged on Thursday in Frankfurt.')], + '/a/celeb': ['Celebrity couple announces divorce after reality show', BODY('The celebrity couple from a reality show announced a divorce; showbiz fans reacted.')], + '/a/fish': ['Scientists discover new species of deep sea fish', BODY('Marine biologists found a new fish species in the Pacific Ocean near Hawaii.')], + // Same headline as /a/quake from another outlet (wire copy), different URL. + '/b/quake': ['Earthquake hits Turkey, hundreds injured', BODY('A strong earthquake hit eastern Turkey on Sunday, injuring hundreds of people in Malatya.')], + '/b/match': ['Dinamo wins derby against Hajduk in Zagreb', BODY('Dinamo beat Hajduk 3-1 in Zagreb on Saturday evening.')], +}; + +interface Harness { + app: RunningApp; + mock: MockLlm; + feeds: Record; + base: string; + call(method: string, p: string, body?: unknown): Promise; + idle(): Promise; + close(): Promise; +} + +async function harness(feeds: Record): Promise { + let base = ''; + const fixture = http.createServer((req, res) => { + const u = (req.url ?? '').replace(/\?.*$/, ''); + const feed = feeds[u]; + if (feed) { + return res.writeHead(200, { 'content-type': 'application/rss+xml' }).end( + `F${feed + .map((k, i) => `${ARTICLES[k]![0]}${base}${k}${k}${new Date(Date.now() - (i + 1) * 60_000).toUTCString()}`) + .join('')}`, + ); + } + const a = ARTICLES[u]; + if (a) return res.writeHead(200, { 'content-type': 'text/html' }).end(`${a[0]}

${a[0]}

${a[1]}

${a[1]}

`); + res.writeHead(404).end(); + }); + await new Promise((r) => fixture.listen(0, '127.0.0.1', () => r())); + base = `http://127.0.0.1:${(fixture.address() as AddressInfo).port}`; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'rssfeed-sav-')); + const mock = await startMockLlm(0, { embeddings: false }); + const config = { ...loadConfig({ RSSFEED_DATA_DIR: dir, TZ: 'Europe/Zagreb', RSSFEED_ALLOW_PRIVATE_URLS: '1', RSSFEED_WEB_DIST: path.join(dir, 'none') }), port: 0, host: '127.0.0.1' }; + const app = await startRssFeed(config, { scheduler: false }); + const call = async (method: string, p: string, body?: unknown): Promise => { + const res = await fetch(`http://127.0.0.1:${app.port}${p}`, { method, headers: { 'content-type': 'application/json' }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); + const text = await res.text(); + assert.ok(res.ok, `${method} ${p} → ${res.status}: ${text}`); + return (text ? JSON.parse(text) : null) as T; + }; + const idle = async () => { + // Sources added via the API are fetched in the background: wait until every one has been fetched. + for (let i = 0; i < 200; i++) { + const sources = await call<{ lastFetchAt: string | null }[]>('GET', '/api/sources'); + if (sources.every((s) => s.lastFetchAt)) break; + await new Promise((r) => setTimeout(r, 100)); + } + for (let i = 0; i < 200; i++) { + const q = (await call('GET', '/api/activity')).queue; + if (q.articlesNew + q.articlesExtracted + q.articlesKept + q.storiesPending + q.storiesUpdating === 0 && !app.worker.busy) return; + await new Promise((r) => setTimeout(r, 150)); + } + throw new Error('pipeline did not go idle'); + }; + return { + app, + mock, + feeds, + base, + call, + idle, + async close() { + await app.close(); + await mock.close(); + await new Promise((r) => fixture.close(() => r())); + fs.rmSync(dir, { recursive: true, force: true }); + }, + }; +} + +const chatCalls = (m: MockLlm, schema?: string) => m.calls.filter((c) => c.path.endsWith('/chat/completions') && (!schema || c.schema === schema)).length; + +test('eventStems: names cut to 4 letters, numbers, years and sentence starters ignored', () => { + const hr = eventStems('Trumpov plan za golf teren u Washingtonu propao, sud blokirao zatvaranje 15 rupa'); + const en = eventStems('Judge blocks Trump plan to close Washington golf course with 15 holes in 2026'); + assert.ok(hr.has('trum') && en.has('trum')); + assert.ok(hr.has('wash') && en.has('wash')); + assert.ok(hr.has('#15') && en.has('#15')); + assert.ok(!en.has('#2026')); + assert.ok(!eventStems('The new report says').size); +}); + +test('AI savings: batched triage, free wire-copy grouping, match pre-filter, seen ledger', { timeout: 90_000 }, async () => { + const h = await harness({ '/one.xml': ['/a/quake', '/a/bank', '/a/celeb', '/a/fish'], '/two.xml': [] }); + try { + await h.call('PUT', '/api/settings', { llm: { baseUrl: h.mock.url, triageModel: 'mock-small', summaryModel: 'mock-large', embeddingModel: '' } }); + await h.call('POST', '/api/rules', { kind: 'ai', polarity: 'negative', text: 'Celebrity gossip and showbiz news' }); + const one = await h.call<{ id: number }>('POST', '/api/sources', { feedUrl: `${h.base}/one.xml`, name: 'One' }); + await h.idle(); + + // 4 articles from one source → one batched triage call, not four. + assert.equal(chatCalls(h.mock, 'triage_batch'), 1, JSON.stringify(h.mock.calls.map((c) => c.schema))); + assert.equal(chatCalls(h.mock, 'triage'), 0); + // Unrelated articles share no names/numbers with earlier stories → no story-match calls at all. + assert.equal(chatCalls(h.mock, 'story_match'), 0); + const first = await h.call('GET', '/api/stories?view=all'); + assert.equal(first.total, 3, 'celebrity article filtered, three stories'); + + // Wire copy: identical headline from a second outlet joins the story without any AI call. + h.feeds['/two.xml'] = ['/b/quake']; + const before = chatCalls(h.mock); + const two = await h.call<{ id: number }>('POST', '/api/sources', { feedUrl: `${h.base}/two.xml`, name: 'Two' }); + await h.idle(); + const quake = (await h.call('GET', '/api/stories?view=all')).items.find((s) => s.sourceCount === 2); + assert.ok(quake, 'wire copy joined the earthquake story'); + assert.equal(chatCalls(h.mock), before, 'no AI calls for wire copy'); + + // A related article (shares "Zagreb"? no: shares nothing) vs one that shares names gets a match call. + h.feeds['/two.xml'] = ['/b/quake', '/b/match']; + await h.call('POST', `/api/sources/${two.id}/fetch`); + await h.idle(); + assert.equal(chatCalls(h.mock, 'story_match'), 0, 'Dinamo/Hajduk shares nothing with earlier stories'); + + // Seen ledger: an article pruned from the database is not processed again if the feed still lists it. + h.app.ctx.db.prepare(`DELETE FROM article WHERE url LIKE '%/a/celeb'`).run(); + const callsBefore = chatCalls(h.mock); + await h.call('POST', `/api/sources/${one.id}/fetch`); + await h.idle(); + assert.equal(chatCalls(h.mock), callsBefore, 'pruned article was not re-triaged'); + assert.equal(h.app.ctx.db.prepare(`SELECT COUNT(*) AS n FROM article WHERE url LIKE '%/a/celeb'`).get<{ n: number }>()!.n, 0); + + const act = await h.call('GET', '/api/activity'); + assert.equal(act.savings7d.groupedByHeadline, 1); + assert.ok(act.savings7d.triageBatched >= 2); + assert.ok(act.savings7d.matchCallsSkipped >= 3); + assert.ok(act.llm.byPurpose.some((p) => p.purpose === 'triage')); + } finally { + await h.close(); + } +}); + +test('a misconfigured model pauses LLM work instead of failing every article', { timeout: 60_000 }, async () => { + const h = await harness({ '/one.xml': ['/a/quake', '/a/bank'] }); + try { + await h.call('PUT', '/api/settings', { llm: { baseUrl: h.mock.url, triageModel: 'mock-small', summaryModel: 'no-such-model', embeddingModel: '' } }); + await h.call('POST', '/api/sources', { feedUrl: `${h.base}/one.xml`, name: 'One' }); + for (let i = 0; i < 100; i++) { + if ((await h.call('GET', '/api/status')).llmError) break; + await new Promise((r) => setTimeout(r, 150)); + } + const st = await h.call('GET', '/api/status'); + assert.equal(st.llm, 'offline'); + assert.match(st.llmError ?? '', /no-such-model/); + await new Promise((r) => setTimeout(r, 1500)); + // One rejected request, not one per article per output format. + assert.equal(h.mock.calls.filter((c) => c.model === 'no-such-model').length, 1); + const act = await h.call('GET', '/api/activity'); + assert.equal(act.failures.length, 0, 'articles wait instead of failing'); + + // Fixing the setting resumes work. + await h.call('PUT', '/api/settings', { llm: { summaryModel: 'mock-large' } }); + await h.idle(); + assert.equal((await h.call('GET', '/api/stories?view=all')).total, 2); + } finally { + await h.close(); + } +}); diff --git a/server/src/settings.ts b/server/src/settings.ts index 959769a..6879c4c 100644 --- a/server/src/settings.ts +++ b/server/src/settings.ts @@ -1,4 +1,4 @@ -import { CATEGORIES, PROMPT_KEYS } from '../../shared/types.ts'; +import { CATEGORIES, PROMPT_KEYS, REASONING_EFFORTS } from '../../shared/types.ts'; import type { Category, CategoryControl, DeepPartial, FeedControls, Settings } from '../../shared/types.ts'; import type { Config } from './config.ts'; import type { Database } from './db/database.ts'; @@ -40,8 +40,10 @@ export function defaultSettings(): Settings { embeddingModel: '', concurrency: 1, timeoutSec: 120, - maxArticleChars: 12_000, + // Two sentences need the lede and the key paragraphs, not the whole article. + maxArticleChars: 6_000, temperature: 0.2, + reasoningEffort: '', }, pipeline: { defaultPollMinutes: 15, @@ -115,6 +117,7 @@ export function sanitizeSettings(input: unknown): Settings { timeoutSec: clampInt(llm.timeoutSec, 10, 900, d.llm.timeoutSec), maxArticleChars: clampInt(llm.maxArticleChars, 1000, 200_000, d.llm.maxArticleChars), temperature: clamp(llm.temperature, 0, 2, d.llm.temperature), + reasoningEffort: oneOf(llm.reasoningEffort, REASONING_EFFORTS, ''), }, pipeline: { defaultPollMinutes: clampInt(s.pipeline.defaultPollMinutes, 1, 1440, d.pipeline.defaultPollMinutes), diff --git a/server/src/testing/mock-llm.ts b/server/src/testing/mock-llm.ts index 7f658c4..8ac574f 100644 --- a/server/src/testing/mock-llm.ts +++ b/server/src/testing/mock-llm.ts @@ -22,6 +22,7 @@ export interface MockLlm { close(): Promise; } +const KNOWN_MODELS = ['mock-small', 'mock-large', 'mock-embed']; const WORD = /[\p{L}\p{N}]+/gu; const STOP = new Set(['the', 'a', 'an', 'of', 'in', 'on', 'to', 'and', 'for', 'is', 'at', 'by', 'with', 'from', 'je', 'i', 'u', 'na', 'za', 'se', 'od', 'da']); @@ -66,6 +67,17 @@ function answer(schema: string | null, system: string, user: string): unknown { ? { decision: 'reject', rule_id: ruleId, reason: 'The article is celebrity gossip.', confidence: 0.93 } : { decision: 'keep', rule_id: null, reason: 'No rule matches the article.', confidence: 0.9 }; } + case 'triage_batch': { + const ruleId = Number(/\[(\d+)\]/.exec(system.split('Negative rules')[1] ?? '')?.[1] ?? 0) || null; + return { + results: [...user.matchAll(/([\s\S]*?)<\/article_\d+>/g)].map((b) => { + const celeb = /celebrit|showbiz|slavn|influencer|reality/i.test(b[2] ?? ''); + return celeb + ? { n: Number(b[1]), decision: 'reject', rule_id: ruleId, reason: 'The article is celebrity gossip.', confidence: 0.93 } + : { n: Number(b[1]), decision: 'keep', rule_id: null, reason: 'No rule matches the article.', confidence: 0.9 }; + }), + }; + } case 'same_event': { const [a, b] = user.split(''); const sim = jaccard(headline(a ?? ''), headline(b ?? '')); @@ -130,7 +142,7 @@ export async function startMockLlm(port = 0, opts: MockOptions = {}): Promise ({ id, object: 'model' })) }); + return send(200, { object: 'list', data: KNOWN_MODELS.map((id) => ({ id, object: 'model' })) }); } let body: Record = {}; try { @@ -153,9 +165,14 @@ export async function startMockLlm(port = 0, opts: MockOptions = {}): Promise m.role === 'system')?.content ?? ''; const user = msgs.find((m) => m.role === 'user')?.content ?? ''; + if (!KNOWN_MODELS.includes(String(body.model))) { + calls.push({ path, schema: null, model: String(body.model) }); + return send(404, { error: { message: `model '${String(body.model)}' not found` } }); + } let schema = rf?.json_schema?.name ?? null; if (!schema) { - if (/"decision"/.test(system)) schema = 'triage'; + if (/"results"/.test(system) && /"decision"/.test(system)) schema = 'triage_batch'; + else if (/"decision"/.test(system)) schema = 'triage'; else if (/same_event/.test(system)) schema = 'same_event'; else if (/title_hr/.test(system)) schema = 'story_summary'; else if (/"opening"/.test(system)) schema = 'digest_opening'; diff --git a/shared/types.ts b/shared/types.ts index ba30360..9fc4137 100644 --- a/shared/types.ts +++ b/shared/types.ts @@ -357,6 +357,9 @@ export interface DigestDTO { // ───────────────────────────── Settings ───────────────────────────── +export const REASONING_EFFORTS = ['', 'minimal', 'low', 'medium', 'high'] as const; +export type ReasoningEffort = (typeof REASONING_EFFORTS)[number]; + export interface LlmSettings { baseUrl: string; apiKey: string; @@ -367,6 +370,8 @@ export interface LlmSettings { timeoutSec: number; maxArticleChars: number; temperature: number; + /** Sent as `reasoning_effort`; '' = don't send (server default). Lower = fewer hidden thinking tokens. */ + reasoningEffort: ReasoningEffort; } export interface PipelineSettings { @@ -535,6 +540,9 @@ export interface LlmTestResult { export interface PlaygroundSummary { model: string; ok: boolean; + /** Tokens billed for this summary (including any retry and hidden reasoning). */ + tokensIn: number | null; + tokensOut: number | null; title: string | null; description: string | null; category: Category | null; @@ -601,7 +609,15 @@ export interface ActivityDTO { p95LatencyMs: number | null; tokensIn24h: number; tokensOut24h: number; + /** Why LLM work is paused, when the server rejected requests (bad key, unknown model). */ + error: string | null; + /** Last 24 h, per pipeline step (triage, story-match, summarize, …). */ + byPurpose: { purpose: string; calls: number; errors: number; tokensIn: number; tokensOut: number }[]; + /** LLM calls in the last 24 h per story published in the last 24 h. */ + callsPerStory: number | null; }; + /** Work saved without AI in the last 7 days. */ + savings7d: { duplicatesSkipped: number; groupedByHeadline: number; matchCallsSkipped: number; triageBatched: number }; failures: ActivityFailure[]; importance7d: { score: number; count: number }[]; grouping7d: { articles: number; inMultiSourceStories: number; share: number }; @@ -621,6 +637,8 @@ export interface ActivityDTO { */ export interface StatusDTO { llm: LlmState; + /** Set when LLM work is paused because the server rejected requests (bad key, unknown model). */ + llmError: string | null; /** Articles and stories waiting on the LLM. */ waiting: number; version: string; diff --git a/web/src/App.tsx b/web/src/App.tsx index 02ac84a..12855cb 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -31,6 +31,19 @@ function StatusBanner() { ); } if (!status) return null; + if (status.llm === 'offline' && status.llmError) { + return ( +
+
+ LLM paused + + {status.llmError} {status.waiting > 0 ? `${plural(status.waiting, 'item')} waiting; nothing is lost.` : ''} + + Fix it +
+
+ ); + } if (status.llm === 'offline') { return (
diff --git a/web/src/screens/Activity.tsx b/web/src/screens/Activity.tsx index f17f0b0..27c2d31 100644 --- a/web/src/screens/Activity.tsx +++ b/web/src/screens/Activity.tsx @@ -32,6 +32,20 @@ function LlmStateBadge({ state }: { state: ActivityDTO['llm']['state'] }) { return Not configured; } +const PURPOSE_LABELS: Record = { + triage: 'Triage (negatives)', + 'story-match': 'Grouping (triage model)', + 'same-event': 'Grouping check', + cluster: 'Grouping (embeddings)', + summarize: 'Title and description', + regenerate: 'Rewrite from several sources', + digest: 'Digest opening', + 'dry-run': 'Rule dry runs', + playground: 'Playground', + test: 'Connection tests', + embed: 'Embeddings', +}; + function ActivityBody({ data, onChanged }: { data: ActivityDTO; onChanged: () => void }) { const [busy, setBusy] = useState(null); const q = data.queue; @@ -93,18 +107,61 @@ function ActivityBody({ data, onChanged }: { data: ActivityDTO; onChanged: () => Connect a server in Settings → LLM connection.

)} - {l.state === 'offline' && ( -

- {plural(l.consecutiveFailures, 'failure')} in a row. {l.nextProbeAt ? `Next check ${relTime(l.nextProbeAt)}.` : ''} Nothing is lost while - it's offline. -

- )} + {l.state === 'offline' && + (l.error ? ( +

+ {l.error} Work is paused and nothing is lost; it resumes when you change the LLM settings, or by itself within 10 minutes. +

+ ) : ( +

+ {plural(l.consecutiveFailures, 'failure')} in a row. {l.nextProbeAt ? `Next check ${relTime(l.nextProbeAt)}.` : ''} Nothing is lost while + it's offline. +

+ ))}
+ - - + +
+ {l.byPurpose.length > 0 && ( +
+ + + + + + + + + + + + + {l.byPurpose.map((p) => ( + + + + + + + + + ))} + +
Step, last 24 hCallsErrorsTokens inTokens outOut per call
{PURPOSE_LABELS[p.purpose] ?? p.purpose}{fmtNumber(p.calls)}{fmtNumber(p.errors)}{fmtNumber(p.tokensIn)}{fmtNumber(p.tokensOut)}{p.calls - p.errors > 0 ? fmtNumber(Math.round(p.tokensOut / Math.max(1, p.calls - p.errors))) : '—'}
+
+ )} + + +
+

Saved without AI, last 7 days

+
+ + + +
diff --git a/web/src/screens/Playground.tsx b/web/src/screens/Playground.tsx index 980ba8a..fb05e4f 100644 --- a/web/src/screens/Playground.tsx +++ b/web/src/screens/Playground.tsx @@ -166,6 +166,11 @@ function SummaryCard({ s }: { s: PlaygroundSummary }) { {s.languageOk ? 'Croatian OK' : 'Language check failed'} {s.formatRetries > 0 && · {s.formatRetries} format retries}

+ {(s.tokensIn !== null || s.tokensOut !== null) && ( +

+ {s.tokensIn ?? '—'} tokens in · {s.tokensOut ?? '—'} tokens out (including hidden reasoning) +

+ )}
); } diff --git a/web/src/screens/settings/LlmSection.tsx b/web/src/screens/settings/LlmSection.tsx index d7b571f..e44f40f 100644 --- a/web/src/screens/settings/LlmSection.tsx +++ b/web/src/screens/settings/LlmSection.tsx @@ -1,6 +1,6 @@ import { useState, type ReactNode } from 'react'; import { api } from '../../api.ts'; -import type { LlmSettings, LlmTestResult } from '../../contract.ts'; +import type { LlmSettings, LlmTestResult, ReasoningEffort } from '../../contract.ts'; import { useDraft } from '../../hooks.ts'; import { Badge, ErrorNote, Field, NumberField, Spinner, TextField } from '../../components.tsx'; import { errorMessage, fmtMs } from '../../util.ts'; @@ -122,7 +122,7 @@ export function LlmSection({ data, onSaved }: SectionProps) { set('timeoutSec', Math.round(v))} /> set('maxArticleChars', Math.round(v))} /> set('temperature', v)} /> + + +