Cut AI usage: free dedupe and grouping, batched triage, fewer tokens
Release / build (push) Successful in 1m10s

Before the AI (free):
- Seen-URL ledger: every article URL is remembered for 180+ days, so an
  item still listed in a feed after its article was pruned (rejected ones
  go after 14 days) is never fetched or judged again.
- Wire copy: an identical headline from another outlet within 48 h joins
  the existing story (or is filtered like its twin) with no triage,
  grouping or summary call, and does not trigger a rewrite.

Triage:
- Up to 8 articles from one source are judged in one call, so rules,
  instructions and examples are sent once; the worker waits up to a
  minute for a fetch's other articles to finish extracting. Articles the
  model skips are judged singly.
- 800 characters of article text instead of 1500.

Grouping without embeddings:
- Free pre-filter on shared names (4-letter stems, so Trump/Trumpov and
  Bahrain/Bahrein agree) and numbers; no call when no recent story shares
  one, and at most 12 candidates instead of 60.

Title and description:
- Max article characters default 6000 (was 12000); a multi-source rewrite
  shares that budget.
- New "Reasoning effort" setting, sent as reasoning_effort. Measured on
  production: Haiku 5.5 spends ~590 hidden reasoning tokens per summary.

No wasted retries:
- 401/403/404 from the chat endpoint (bad key, unknown model, wrong URL)
  pauses LLM work with a banner instead of failing every article; it
  resumes on a settings change or after 10 minutes.
- Simpler structured-output modes are tried only on 400/422, so a
  misconfiguration costs one request, not three.

Visibility:
- Activity: calls and tokens per pipeline step, calls per published
  story, and work saved without AI. Playground: tokens per model.

Tests for each saving and for the pause on misconfiguration.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
TMCandClaude Opus 5.5 committed 2026-11-15 12:10:00 +01:00
1 parent 4789767103
commit 29036127a9
20 files changed
+786 -100

No files matched your search

+8
View File
@@ -203,6 +203,14 @@ const MIGRATIONS: string[] = [
PRIMARY KEY (day, key)
);
`,
/* 2 — long-lived ledger of every article URL ever seen, so an item that is still in a feed
after its article was pruned (e.g. rejected ones after 14 days) is never processed again */ `
CREATE TABLE seen (
url_hash TEXT PRIMARY KEY,
first_seen TEXT NOT NULL
);
INSERT OR IGNORE INTO seen (url_hash, first_seen) SELECT url_hash, fetched_at FROM article;
`,
];
export function migrate(db: Database): number {
+19
View File
@@ -19,6 +19,16 @@ export function buildActivity(ctx: Ctx): ActivityDTO {
const calls = db.prepare(`SELECT ok, ms, tokens_in, tokens_out FROM llm_call WHERE at >= ?`).all<{ ok: number; ms: number; tokens_in: number | null; tokens_out: number | null }>(since24h);
const okMs = calls.filter((c) => c.ok).map((c) => c.ms).sort((a, b) => a - b);
const breaker = ctx.llm.breakerInfo();
const byPurpose = db
.prepare(
`SELECT purpose, COUNT(*) AS calls, SUM(CASE WHEN ok = 0 THEN 1 ELSE 0 END) AS errors,
COALESCE(SUM(tokens_in), 0) AS tokens_in, COALESCE(SUM(tokens_out), 0) AS tokens_out
FROM llm_call WHERE at >= ? GROUP BY purpose ORDER BY calls DESC`,
)
.all<{ purpose: string; calls: number; errors: number; tokens_in: number; tokens_out: number }>(since24h)
.map((r) => ({ purpose: r.purpose, calls: r.calls, errors: r.errors, tokensIn: r.tokens_in, tokensOut: r.tokens_out }));
const pipelineCalls = byPurpose.filter((p) => !['test', 'playground', 'dry-run'].includes(p.purpose)).reduce((s, p) => s + p.calls, 0);
const published24h = count(`SELECT COUNT(*) AS n FROM story WHERE first_published_at >= ?`, since24h);
const failures: ActivityFailure[] = [
...db
@@ -87,6 +97,15 @@ export function buildActivity(ctx: Ctx): ActivityDTO {
p95LatencyMs: okMs.length ? okMs[Math.min(okMs.length - 1, Math.floor(okMs.length * 0.95))]! : null,
tokensIn24h: calls.reduce((s, c) => s + (c.tokens_in ?? 0), 0),
tokensOut24h: calls.reduce((s, c) => s + (c.tokens_out ?? 0), 0),
error: ctx.llm.errorMessage(),
byPurpose,
callsPerStory: published24h ? Math.round((pipelineCalls / published24h) * 10) / 10 : null,
},
savings7d: {
duplicatesSkipped: statSum('duplicates_skipped'),
groupedByHeadline: statSum('grouped_by_headline'),
matchCallsSkipped: statSum('match_calls_skipped'),
triageBatched: statSum('triage_batched'),
},
failures: failures.slice(0, 60),
importance7d,
+4 -1
View File
@@ -319,6 +319,8 @@ async function playgroundRun(ctx: Ctx, url: string, models: string[]): Promise<P
summaries.push({
model,
ok: w.languageOk,
tokensIn: w.tokensIn,
tokensOut: w.tokensOut,
title: w.title,
description: w.description,
category: w.category,
@@ -329,7 +331,7 @@ async function playgroundRun(ctx: Ctx, url: string, models: string[]): Promise<P
error: w.languageOk ? null : `Not standard Croatian: it ${w.languageProblem}`,
});
} catch (err) {
summaries.push({ model, ok: false, title: null, description: null, category: null, importance: null, formatRetries: 0, languageOk: false, ms: Date.now() - t0, error: errorMessage(err) });
summaries.push({ model, ok: false, tokensIn: null, tokensOut: null, title: null, description: null, category: null, importance: null, formatRetries: 0, languageOk: false, ms: Date.now() - t0, error: errorMessage(err) });
}
}
return { extraction, triage, summaries };
@@ -378,6 +380,7 @@ export function apiRouter(ctx: Ctx): express.Router {
r.get('/status', (_req, res: Response<StatusDTO>) => {
res.json({
llm: ctx.llm.state(),
llmError: ctx.llm.errorMessage(),
waiting: waitingCount(ctx),
version: ctx.config.version,
latestDigestId: latestPublishedDigest(ctx)?.id ?? null,
+33 -2
View File
@@ -113,6 +113,8 @@ export class LlmClient {
private failures = 0;
private nextProbe = 0;
private probing = false;
/** Set when the server rejects requests as misconfigured (401/403/404); pauses LLM work. */
private configError: { message: string; until: number } | null = null;
private timer: NodeJS.Timeout | null = null;
private readonly listeners: (() => void)[] = [];
@@ -140,12 +142,28 @@ export class LlmClient {
this.sem.limit = this.getSettings().concurrency;
this.modeCache.clear();
// New connection settings deserve a fresh chance.
const hadConfigError = this.configError !== null;
this.configError = null;
if (this.failures > 0) this.reset();
else if (hadConfigError) for (const l of this.listeners) l();
}
/** Why LLM work is paused because the server rejected requests, if it is. */
errorMessage(): string | null {
if (!this.configError) return null;
if (Date.now() >= this.configError.until) {
// Try again after a while (the server may have been fixed without a settings change).
this.configError = null;
for (const l of this.listeners) l();
return null;
}
return this.configError.message;
}
state(): LlmState {
const s = this.getSettings();
if (!s.baseUrl || !s.summaryModel) return 'unconfigured';
if (this.errorMessage()) return 'offline';
return this.failures >= BREAKER_THRESHOLD ? 'offline' : 'ok';
}
@@ -252,6 +270,7 @@ export class LlmClient {
const s = this.conn(opts.override);
const affectBreaker = !opts.override;
if (affectBreaker && this.failures >= BREAKER_THRESHOLD) throw new LlmUnavailableError('LLM is offline');
if (affectBreaker && this.errorMessage()) throw new LlmUnavailableError(this.errorMessage()!);
if (!opts.model) throw new LlmUnavailableError('No model configured');
const release = await this.sem.acquire();
const started = Date.now();
@@ -273,6 +292,7 @@ export class LlmClient {
max_tokens: opts.maxTokens ?? 2048,
stream: false,
};
if (s.reasoningEffort) body.reasoning_effort = s.reasoningEffort;
if (mode === 'json_schema') {
body.response_format = { type: 'json_schema', json_schema: { name: opts.schemaName, strict: true, schema: opts.schema } };
} else if (mode === 'json_object') {
@@ -282,9 +302,11 @@ export class LlmClient {
try {
data = await this.request('/chat/completions', body, s, s.timeoutSec * 1000);
} catch (err) {
if (err instanceof LlmRequestError && mode !== 'prompt') {
// Only a rejected request *shape* (400/422) is worth retrying in a simpler output mode;
// a bad key, unknown model or wrong URL would just fail three times.
if (err instanceof LlmRequestError && (err.status === 400 || err.status === 422) && mode !== 'prompt') {
firstError ??= err;
continue; // try a simpler structured-output mode
continue;
}
throw err;
}
@@ -298,6 +320,14 @@ export class LlmClient {
const e = err as Error;
this.record(opts.purpose, opts.model, false, Date.now() - started, null, null, e.message);
if (affectBreaker && err instanceof LlmUnavailableError) this.recordFailure(e);
if (affectBreaker && err instanceof LlmRequestError && [401, 403, 404].includes(err.status)) {
// Misconfiguration (bad key, unknown model, wrong URL): every request would fail the same
// way, so pause all LLM work instead of burning attempts on each article.
const message = `The LLM server rejected requests for "${opts.model}" (HTTP ${err.status}). Check Settings → LLM connection. ${err.message.slice(0, 200)}`;
if (!this.configError) log.warn('pausing LLM work: the server rejected the request', { model: opts.model, status: err.status });
this.configError = { message, until: Date.now() + 10 * 60_000 };
throw new LlmUnavailableError(message);
}
throw err;
} finally {
release();
@@ -341,6 +371,7 @@ export class LlmClient {
// One repair attempt with the model's own answer in context.
const repairBody: Record<string, unknown> = {
...(s.reasoningEffort ? { reasoning_effort: s.reasoningEffort } : {}),
model: opts.model,
messages: [
...messages,
+60
View File
@@ -203,6 +203,66 @@ export function validateTriage(v: unknown): TriageOutput | string {
};
}
// ───────────────────────────── triage, batched ─────────────────────────────
export const TRIAGE_BATCH_SCHEMA: JsonSchema = {
type: 'object',
properties: {
results: {
type: 'array',
items: {
type: 'object',
properties: {
n: { type: 'integer' },
decision: { type: 'string', enum: ['keep', 'reject'] },
rule_id: { type: ['integer', 'null'] },
reason: { type: 'string' },
confidence: { type: 'number' },
},
required: ['n', 'decision', 'rule_id', 'reason', 'confidence'],
additionalProperties: false,
},
},
},
required: ['results'],
additionalProperties: false,
};
/** Several articles from one source in one call: the rules and instructions are sent once. */
export function buildTriageBatchPrompt(
instructions: string,
negatives: TriageRuleRef[],
exceptions: TriageRuleRef[],
examples: TriageExample[],
articles: (ArticleForPrompt & { n: number })[],
): { system: string; user: string } {
const single = buildTriagePrompt(instructions, negatives, exceptions, examples, articles[0]!);
const system = single.system.replace(
/\n\nOutput format:[\s\S]*$/,
`\n\nYou get several numbered articles. Judge each one on its own.
Output format: reply with ONLY a JSON object:
{"results": [{"n": <article number>, "decision": "keep" or "reject", "rule_id": <the id in brackets of the rule that decided it, or null>, "reason": "<one short sentence in English>", "confidence": <number from 0 to 1>}]}
Include every article number exactly once.`,
);
const user = articles.map((a) => articleBlock(a, `article_${a.n}`)).join('\n\n');
return { system, user };
}
export function validateTriageBatch(v: unknown): (TriageOutput & { n: number })[] | string {
if (!v || typeof v !== 'object') return 'expected a JSON object';
const results = (v as Record<string, unknown>).results;
if (!Array.isArray(results)) return 'results must be an array';
const out: (TriageOutput & { n: number })[] = [];
for (const r of results) {
if (!r || typeof r !== 'object') continue;
const n = Number((r as Record<string, unknown>).n);
const t = validateTriage(r);
if (Number.isInteger(n) && typeof t !== 'string') out.push({ ...t, n });
}
return out;
}
// ───────────────────────────── dry run (batched) ─────────────────────────────
export const DRY_RUN_SCHEMA: JsonSchema = {
+70 -4
View File
@@ -1,4 +1,5 @@
import type { Ctx } from '../context.ts';
import { bumpStat } from '../context.ts';
import { HOUR, isoMinus } from '../lib/time.ts';
import { truncate } from '../lib/text.ts';
import { log } from '../lib/log.ts';
@@ -157,7 +158,47 @@ function sourceName(ctx: Ctx, id: number): string {
return ctx.db.prepare(`SELECT name FROM source WHERE id = ?`).get<{ name: string }>(id)?.name ?? 'Unknown';
}
const MAX_LLM_CANDIDATES = 60;
/** Recent stories scanned for a possible match, and the most sent to the model. */
const SCAN_STORIES = 200;
const MAX_LLM_CANDIDATES = 12;
/** Capitalised words that start sentences or headlines in Croatian and English and say nothing about the event. */
const COMMON = new Set(
(
'the a an this that these those there here in on at for from with after before when how why what who new his her its their our your ' +
'we you they it he she said says says breaking live update watch video photo photos exclusive report reports ' +
'u na za iz od do po kako sto zasto gdje tko koji koja koje novi nova novo ovo ova ovaj taj ta to nakon prema ' +
'danas jucer sutra foto video uzivo pogledajte evo ovo ovako sve svi vijesti hrvatska hrvatski hrvatske ' +
// Dates are shared by unrelated stories of the same day.
'monday tuesday wednesday thursday friday saturday sunday january february march april may june july august ' +
'september october november december ponedjeljak utorak srijeda srijedu cetvrtak petak subota subotu nedjelja nedjelju ' +
'mr mrs ms dr'
).split(/\s+/),
);
function fold(s: string): string {
return s.toLowerCase().normalize('NFKD').replace(/\p{M}+/gu, '').replace(/đ/g, 'd');
}
/**
* Cheap "could these be the same event?" signal that works across Croatian and English: names
* (capitalised words, cut to 4 letters so "Trump"/"Trumpov" and "Bahrain"/"Bahrein" agree) and
* numbers. A story that shares none of them with the new article is not worth asking the model about.
*/
export function eventStems(text: string): Set<string> {
const out = new Set<string>();
for (const m of text.matchAll(/(?<![\p{L}\p{N}])(\p{Lu}[\p{L}\p{M}'’-]{2,})/gu)) {
const w = fold(m[1]!).replace(/['’-].*$/, '');
if (w.length < 3 || COMMON.has(w)) continue;
out.add(w.slice(0, 4));
}
for (const m of text.matchAll(/(?<![\p{L}\p{N}])\d[\d.,]*\d(?![\p{L}\p{N}])/gu)) {
const n = m[0].replace(/[.,]/g, '');
if (/^(19|20)\d\d$/.test(n)) continue; // years match everything
out.add(`#${n}`);
}
return out;
}
/** Grouping without embeddings: the triage model picks the matching recent story, if any. */
async function clusterByLlm(ctx: Ctx, a: ArticleRow): Promise<void> {
@@ -170,13 +211,38 @@ async function clusterByLlm(ctx: Ctx, a: ArticleRow): Promise<void> {
WHERE s.first_reported_at >= ? AND s.hidden = 0 AND s.state != 'failed'
ORDER BY s.last_reported_at DESC LIMIT ?`,
)
.all<{ id: number; title: string | null; lead_title: string | null }>(since, MAX_LLM_CANDIDATES);
if (!stories.length) {
.all<{ id: number; title: string | null; lead_title: string | null }>(since, SCAN_STORIES);
// Free pre-filter: only stories that share a name or number with the new article are candidates.
const mine = eventStems(`${a.title}\n${articleText(a).slice(0, 600)}`);
const memberTitles = new Map<number, string[]>();
if (stories.length) {
const ids = stories.map((x) => x.id);
for (const r of ctx.db
.prepare(`SELECT story_id, title FROM article WHERE story_id IN (${ids.map(() => '?').join(',')})`)
.all<{ story_id: number; title: string }>(...ids)) {
const list = memberTitles.get(r.story_id) ?? [];
list.push(r.title);
memberTitles.set(r.story_id, list);
}
}
const scored = stories
.map((x) => {
const theirs = eventStems([x.title ?? '', ...(memberTitles.get(x.id) ?? [])].join('\n'));
let shared = 0;
for (const st of mine) if (theirs.has(st)) shared++;
return { story: x, shared };
})
.filter((x) => x.shared > 0)
.sort((p, q) => q.shared - p.shared)
.slice(0, MAX_LLM_CANDIDATES);
if (!scored.length) {
if (stories.length) bumpStat(ctx.db, 'match_calls_skipped');
createStoryForArticle(ctx, a);
ctx.wake();
return;
}
const candidates = stories.map((x) => ({ id: x.id, title: x.title ?? x.lead_title ?? '', originalTitle: x.lead_title }));
const candidates = scored.map(({ story: x }) => ({ id: x.id, title: x.title ?? x.lead_title ?? '', originalTitle: x.lead_title }));
const prompt = buildStoryMatchPrompt(
promptText(s, 'sameEvent'),
{ sourceName: sourceName(ctx, a.source_id), title: a.title, publishedAt: a.published_at, categories: parseJsonArray(a.categories), text: truncate(articleText(a), 800) },
+61 -9
View File
@@ -7,6 +7,7 @@ import { normalizeLang, normalizeTitle, sha1 } from '../lib/text.ts';
import { HOUR, MINUTE, isoMinus, isoPlus, nowIso } from '../lib/time.ts';
import { log, errorMessage } from '../lib/log.ts';
import { setMeta } from '../db/schema.ts';
import { attachArticle } from './stories.ts';
export interface SourceRow {
id: number;
@@ -80,6 +81,19 @@ export async function fetchSource(ctx: Ctx, sourceId: number): Promise<{ newItem
}
}
interface Twin {
id: number;
source_id: number;
state: string;
story_id: number | null;
}
/**
* Stores new feed items. Everything here is free (no AI): an item is dropped when its URL was
* ever seen before, when its source already delivered the same GUID or headline, and an
* identical headline from another outlet (wire copy) joins the existing story, or is filtered
* the same way, without triage, grouping or a summary.
*/
function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType<typeof parseFeed>['items'], feedLang: string | null): number {
const s = ctx.settings.get().pipeline;
const now = new Date();
@@ -87,31 +101,60 @@ function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType<typeof parseFeed
const maxAgeCutoff = isoMinus(s.maxAgeHours * HOUR, now);
const firstAddCutoff = isoMinus(s.firstAddHours * HOUR, now);
const titleWindow = isoMinus(48 * HOUR, now);
const seen = ctx.db.prepare(`SELECT 1 FROM seen WHERE url_hash = ?`);
const markSeen = ctx.db.prepare(`INSERT OR IGNORE INTO seen (url_hash, first_seen) VALUES (?, ?)`);
const byUrl = ctx.db.prepare(`SELECT 1 FROM article WHERE url_hash = ?`);
const byGuid = ctx.db.prepare(`SELECT 1 FROM article WHERE source_id = ? AND guid = ?`);
const byTitle = ctx.db.prepare(`SELECT 1 FROM article WHERE title_hash = ? AND fetched_at >= ?`);
const byTitle = ctx.db.prepare(
`SELECT id, source_id, state, story_id FROM article WHERE title_hash = ? AND fetched_at >= ? ORDER BY id LIMIT 1`,
);
const insert = ctx.db.prepare(
`INSERT INTO article (source_id, guid, url, url_hash, title_hash, title, excerpt, categories, text, partial, author,
image_url, language, published_at, fetched_at, state, state_changed_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
);
const copyVerdict = ctx.db.prepare(
`INSERT OR REPLACE INTO verdict (article_id, decision, decided_by, rule_id, rule_text, reason, confidence, model, prompt_version, overridden, decided_at)
SELECT ?, decision, decided_by, rule_id, rule_text, 'Same headline as an article already filtered. ' || reason, confidence, model, prompt_version, 0, ?
FROM verdict WHERE article_id = ?`,
);
let fresh = 0;
let duplicates = 0;
const attachLater: { storyId: number; article: { id: number; published_at: string | null; fetched_at: string } }[] = [];
ctx.db.transaction(() => {
for (const item of items) {
const url = item.link ? canonicalizeUrl(item.link) : null;
if (!url) continue;
const urlHash = sha1(url);
if (byUrl.get(urlHash)) continue;
if (item.guid && byGuid.get(src.id, item.guid)) continue;
if (seen.get(urlHash) || byUrl.get(urlHash)) continue;
markSeen.run(urlHash, fetchedAt);
if (item.guid && byGuid.get(src.id, item.guid)) {
duplicates++;
continue;
}
const normTitle = normalizeTitle(item.title);
const titleHash = sha1(normTitle);
if (normTitle.length > 12 && byTitle.get(titleHash, titleWindow)) continue;
// Feeds sometimes publish future dates; never trust a date later than now.
const published = item.publishedAt && item.publishedAt.getTime() <= now.getTime() ? item.publishedAt.toISOString() : fetchedAt;
const tooOld = published < maxAgeCutoff || (!src.initial_import_done && published < firstAddCutoff);
const fullText = src.full_text === 1;
const state = tooOld ? 'skipped' : fullText ? 'new' : 'extracted';
insert.run(
let state = tooOld ? 'skipped' : fullText ? 'new' : 'extracted';
let twin: Twin | undefined;
if (!tooOld && normTitle.length > 12) {
twin = byTitle.get<Twin>(titleHash, titleWindow);
if (twin) {
if (twin.source_id === src.id || (twin.state !== 'grouped' && twin.state !== 'rejected') || (twin.state === 'grouped' && !twin.story_id)) {
// Same outlet under a new URL, or the twin is still being processed: nothing to add.
duplicates++;
continue;
}
// Identical headline from another outlet: reuse the twin's outcome instead of the AI.
state = twin.state === 'rejected' ? 'rejected' : 'kept';
}
}
const id = insert.run(
src.id,
item.guid,
url,
@@ -120,8 +163,8 @@ function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType<typeof parseFeed
item.title.slice(0, 1000),
item.excerpt.slice(0, 20_000) || null,
JSON.stringify(item.categories),
fullText ? null : item.excerpt.slice(0, 50_000) || null,
fullText ? 0 : 1,
fullText && !twin ? null : item.excerpt.slice(0, 50_000) || null,
fullText && !twin ? 0 : 1,
item.author?.slice(0, 300) ?? null,
item.imageUrl,
feedLang,
@@ -129,10 +172,19 @@ function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType<typeof parseFeed
fetchedAt,
state,
fetchedAt,
);
).lastInsertRowid;
if (twin?.state === 'rejected') {
copyVerdict.run(id, fetchedAt, twin.id);
} else if (twin?.story_id) {
attachLater.push({ storyId: twin.story_id, article: { id, published_at: published, fetched_at: fetchedAt } });
}
if (!tooOld) fresh++;
}
});
// Outside the insert transaction: attaching recomputes the story and may queue a rewrite.
for (const a of attachLater) attachArticle(ctx, a.storyId, a.article, { rewrite: false });
if (duplicates) bumpStat(ctx.db, 'duplicates_skipped', duplicates);
if (attachLater.length) bumpStat(ctx.db, 'grouped_by_headline', attachLater.length);
return fresh;
}
+2
View File
@@ -27,6 +27,8 @@ export function runRetention(ctx: Ctx): void {
.prepare(`UPDATE article SET text = NULL WHERE text IS NOT NULL AND fetched_at < ? AND state NOT IN ('new','extracted','kept')`)
.run(textCutoff).changes;
ctx.db.prepare(`DELETE FROM llm_call WHERE at < ?`).run(isoMinus(8 * DAY));
// The seen-URL ledger outlives the articles, so a pruned item still in a feed isn't reprocessed.
ctx.db.prepare(`DELETE FROM seen WHERE first_seen < ?`).run(isoMinus(Math.max(180, r.publishedDays * 2) * DAY));
ctx.db.prepare(`DELETE FROM stat WHERE day < ?`).run(isoMinus(60 * DAY).slice(0, 10));
ctx.db.prepare(`DELETE FROM edition WHERE scheduled_for < ?`).run(isoMinus(60 * DAY));
ctx.db.prepare(`DELETE FROM digest WHERE created_at < ?`).run(publishedCutoff);
+11 -3
View File
@@ -257,8 +257,16 @@ export function createStoryForArticle(ctx: Ctx, a: Pick<ArticleRow, 'id' | 'publ
});
}
/** Adds an article to an existing story and triggers regeneration / feed re-evaluation as needed. */
export function attachArticle(ctx: Ctx, storyId: number, a: Pick<ArticleRow, 'id' | 'published_at' | 'fetched_at'>): void {
/**
* Adds an article to an existing story and triggers regeneration / feed re-evaluation as needed.
* `rewrite: false` for wire copy: the same text adds a source but nothing worth a new summary.
*/
export function attachArticle(
ctx: Ctx,
storyId: number,
a: Pick<ArticleRow, 'id' | 'published_at' | 'fetched_at'>,
opts: { rewrite?: boolean } = {},
): void {
const now = nowIso();
ctx.db.transaction(() => {
ctx.db.prepare(`UPDATE article SET story_id = ?, state = 'grouped', state_changed_at = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ?`).run(storyId, now, a.id);
@@ -268,7 +276,7 @@ export function attachArticle(ctx: Ctx, storyId: number, a: Pick<ArticleRow, 'id
const c = ctx.settings.get().clustering;
if (s.state === 'failed') {
ctx.db.prepare(`UPDATE story SET state = 'pending', attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ?`).run(storyId);
} else if (s.state === 'published' && s.regen_count < c.regenCap) {
} else if (s.state === 'published' && s.regen_count < c.regenCap && opts.rewrite !== false) {
const crossed = r.prevSourceCount < c.regenAtSources && r.sourceCount >= c.regenAtSources;
const written = s.version_written_at ? Date.parse(s.version_written_at) : Date.now();
const gap = c.regenGapHours > 0 && Date.parse(reportedAt(a)) - written > c.regenGapHours * HOUR;
+2 -1
View File
@@ -124,7 +124,8 @@ export async function summarizeStory(ctx: Ctx, storyId: number): Promise<void> {
const lead = all.find((a) => a.id === story.lead_article_id) ?? all[0]!;
const others = all.filter((a) => a.id !== lead.id).sort((x, y) => x.partial - y.partial);
const chosen = [lead, ...others].slice(0, 4);
const perArticle = chosen.length > 1 ? Math.max(2000, Math.floor(settings.llm.maxArticleChars / chosen.length)) : settings.llm.maxArticleChars;
// A rewrite from several sources shares one article-sized budget instead of multiplying it.
const perArticle = chosen.length > 1 ? Math.max(1500, Math.floor(settings.llm.maxArticleChars / chosen.length)) : settings.llm.maxArticleChars;
const written = await writeTitleAndDescription(ctx.llm, settings, chosen.map((a) => articleForPrompt(ctx, a, perArticle)));
if (written.formatRetries) bumpStat(ctx.db, 'format_retries', written.formatRetries);
if (!written.languageOk) {
+146 -54
View File
@@ -3,14 +3,19 @@ import type { DryRunHit, DryRunStatus, RuleDTO, RuleInput } from '../../../share
import type { Ctx } from '../context.ts';
import {
buildDryRunPrompt,
buildTriageBatchPrompt,
buildTriagePrompt,
DRY_RUN_SCHEMA,
promptText,
promptVersion,
TRIAGE_BATCH_SCHEMA,
TRIAGE_SCHEMA,
validateDryRun,
validateTriage,
validateTriageBatch,
} from '../llm/prompts.ts';
import type { TriageOutput, TriageRuleRef } from '../llm/prompts.ts';
import { bumpStat } from '../context.ts';
import { nowIso } from '../lib/time.ts';
import { truncate } from '../lib/text.ts';
import { errorMessage } from '../lib/log.ts';
@@ -120,13 +125,9 @@ function writeVerdict(ctx: Ctx, articleId: number, v: VerdictInput): void {
.run(articleId, v.decision, v.decidedBy, v.ruleId, v.ruleText, v.reason, v.confidence, v.model, v.promptVersion, nowIso());
}
/** Whether triage for this article needs the LLM (any enabled AI negative rule applies). */
/** Whether triage for this article needs the LLM (no keyword rule decides it and an AI negative applies). */
export function triageNeedsLlm(ctx: Ctx, a: Pick<ArticleRow, 'source_id' | 'title' | 'categories'>): boolean {
const rules = activeRules(ctx, a.source_id);
const hay = keywordHaystack(a);
if (rules.some((r) => r.kind === 'keyword' && r.polarity === 'exception' && keywordMatcher(r.text)(hay))) return false;
if (rules.some((r) => r.kind === 'keyword' && r.polarity === 'negative' && keywordMatcher(r.text)(hay))) return false;
return rules.some((r) => r.kind === 'ai' && r.polarity === 'negative');
return freeVerdict(activeRules(ctx, a.source_id), a) === null;
}
function sourceName(ctx: Ctx, sourceId: number): string {
@@ -148,44 +149,85 @@ export interface TriageVerdict extends VerdictInput {
ms: number | null;
}
/** Decides keep/reject without touching the database (used by triage and the playground). */
export async function evaluateTriage(ctx: Ctx, input: TriageInput): Promise<TriageVerdict> {
const rules =
input.sourceId === null
? listRules(ctx).filter((r) => r.enabled)
: activeRules(ctx, input.sourceId);
const hay = keywordHaystack(input);
const base = { model: null, promptVersion: null, confidence: null, ms: null };
/** Characters of article text the triage model sees: headline + lede is enough to judge the subject. */
const TRIAGE_TEXT_CHARS = 800;
/** Articles from one source judged in one call. */
export const TRIAGE_BATCH_SIZE = 8;
const NO_LLM = { model: null, promptVersion: null, confidence: null, ms: null };
/** Keyword exceptions, keyword negatives, and "no AI rule applies" — all free. Null = needs the LLM. */
function freeVerdict(rules: RuleDTO[], input: Pick<TriageInput, 'title' | 'categories'>): TriageVerdict | null {
const hay = keywordHaystack(input);
for (const r of rules.filter((x) => x.kind === 'keyword' && x.polarity === 'exception')) {
const hit = keywordMatcher(r.text)(hay);
if (hit) return { ...base, decision: 'keep', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Kept by exception: matched "${hit}"`, hitRuleId: r.id };
if (hit) return { ...NO_LLM, decision: 'keep', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Kept by exception: matched "${hit}"`, hitRuleId: r.id };
}
for (const r of rules.filter((x) => x.kind === 'keyword' && x.polarity === 'negative')) {
const hit = keywordMatcher(r.text)(hay);
if (hit) return { ...base, decision: 'reject', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Matched keyword "${hit}"`, hitRuleId: r.id };
if (hit) return { ...NO_LLM, decision: 'reject', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Matched keyword "${hit}"`, hitRuleId: r.id };
}
if (!rules.some((r) => r.kind === 'ai' && r.polarity === 'negative')) {
return { ...NO_LLM, decision: 'keep', decidedBy: 'none', ruleId: null, ruleText: null, reason: 'No rule applies', hitRuleId: null };
}
return null;
}
/** Turns the model's answer into a verdict: reject only with a valid rule at or above the threshold. */
function interpretTriage(
out: TriageOutput,
rules: RuleDTO[],
threshold: number,
common: { model: string; promptVersion: string; ms: number | null },
): TriageVerdict {
const negatives = rules.filter((r) => r.kind === 'ai' && r.polarity === 'negative');
const exceptions = rules.filter((r) => r.kind === 'ai' && r.polarity === 'exception');
if (negatives.length === 0) {
return { ...base, decision: 'keep', decidedBy: 'none', ruleId: null, ruleText: null, reason: 'No rule applies', hitRuleId: null };
const negRule = negatives.find((r) => r.id === out.ruleId);
const excRule = exceptions.find((r) => r.id === out.ruleId);
const base = { ...common, decidedBy: 'ai' as const, confidence: out.confidence };
if (out.decision === 'reject' && negRule && (out.confidence ?? 0) >= threshold) {
return { ...base, decision: 'reject', ruleId: negRule.id, ruleText: negRule.text, reason: out.reason, hitRuleId: negRule.id };
}
let reason = out.reason;
if (out.decision === 'reject') {
reason = negRule
? `Kept: confidence ${Math.round((out.confidence ?? 0) * 100)}% is below the ${Math.round(threshold * 100)}% threshold. ${out.reason}`
: `Kept: the model did not name a valid rule. ${out.reason}`;
}
return {
...base,
decision: 'keep',
ruleId: excRule?.id ?? negRule?.id ?? null,
ruleText: excRule?.text ?? negRule?.text ?? null,
reason,
hitRuleId: excRule?.id ?? null,
};
}
function ruleRefs(rules: RuleDTO[], polarity: 'negative' | 'exception'): TriageRuleRef[] {
return rules.filter((r) => r.kind === 'ai' && r.polarity === polarity).map((r) => ({ id: r.id, text: r.text }));
}
/** Decides keep/reject without touching the database (used by triage and the playground). */
export async function evaluateTriage(ctx: Ctx, input: TriageInput): Promise<TriageVerdict> {
const rules = input.sourceId === null ? listRules(ctx).filter((r) => r.enabled) : activeRules(ctx, input.sourceId);
const free = freeVerdict(rules, input);
if (free) return free;
const settings = ctx.settings.get();
const model = settings.llm.triageModel || settings.llm.summaryModel;
const instructions = promptText(settings, 'triage');
const prompt = buildTriagePrompt(
instructions,
negatives.map((r) => ({ id: r.id, text: r.text })),
exceptions.map((r) => ({ id: r.id, text: r.text })),
ruleRefs(rules, 'negative'),
ruleRefs(rules, 'exception'),
filterExamples(ctx.db, settings.pipeline.examplesInPrompt),
{
sourceName: input.sourceName,
title: input.title,
publishedAt: input.publishedAt,
categories: parseJsonArray<string>(input.categories),
text: truncate(input.text, 1500),
text: truncate(input.text, TRIAGE_TEXT_CHARS),
},
);
const res = await ctx.llm.chatJson({
@@ -198,46 +240,96 @@ export async function evaluateTriage(ctx: Ctx, input: TriageInput): Promise<Tria
validate: validateTriage,
maxTokens: 1024,
});
const out = res.value;
const pv = promptVersion(instructions);
const negRule = negatives.find((r) => r.id === out.ruleId);
const excRule = exceptions.find((r) => r.id === out.ruleId);
const threshold = settings.pipeline.rejectThreshold;
const common = { decidedBy: 'ai' as const, confidence: out.confidence, model, promptVersion: pv, ms: res.ms };
if (out.decision === 'reject' && negRule && (out.confidence ?? 0) >= threshold) {
return { ...common, decision: 'reject', ruleId: negRule.id, ruleText: negRule.text, reason: out.reason, hitRuleId: negRule.id };
}
let reason = out.reason;
if (out.decision === 'reject') {
reason = negRule
? `Kept: confidence ${Math.round((out.confidence ?? 0) * 100)}% is below the ${Math.round(threshold * 100)}% threshold. ${out.reason}`
: `Kept: the model did not name a valid rule. ${out.reason}`;
}
return {
...common,
decision: 'keep',
ruleId: excRule?.id ?? negRule?.id ?? null,
ruleText: excRule?.text ?? negRule?.text ?? null,
reason,
hitRuleId: excRule?.id ?? null,
};
return interpretTriage(res.value, rules, settings.pipeline.rejectThreshold, { model, promptVersion: promptVersion(instructions), ms: res.ms });
}
function inputOf(ctx: Ctx, a: ArticleRow): TriageInput {
return { sourceId: a.source_id, sourceName: sourceName(ctx, a.source_id), title: a.title, categories: a.categories, text: articleText(a), publishedAt: a.published_at };
}
function record(ctx: Ctx, articleId: number, v: TriageVerdict): void {
if (v.hitRuleId !== null) bumpRule(ctx, v.hitRuleId);
writeVerdict(ctx, articleId, v);
}
/** Decides keep/reject for one stored article and writes the verdict. */
export async function triageArticle(ctx: Ctx, a: ArticleRow): Promise<'keep' | 'reject'> {
const v = await evaluateTriage(ctx, {
sourceId: a.source_id,
sourceName: sourceName(ctx, a.source_id),
title: a.title,
categories: a.categories,
text: articleText(a),
publishedAt: a.published_at,
});
if (v.hitRuleId !== null) bumpRule(ctx, v.hitRuleId);
writeVerdict(ctx, a.id, v);
const v = await evaluateTriage(ctx, inputOf(ctx, a));
record(ctx, a.id, v);
return v.decision;
}
/**
* Triage for several articles from one source. Free decisions (keyword rules, no AI rule) are
* made first; the rest share one LLM call, so the rules and instructions are sent once.
*/
export async function triageBatch(ctx: Ctx, articles: ArticleRow[]): Promise<Map<number, 'keep' | 'reject'>> {
const out = new Map<number, 'keep' | 'reject'>();
if (!articles.length) return out;
const rules = activeRules(ctx, articles[0]!.source_id);
const needAi: ArticleRow[] = [];
for (const a of articles) {
const free = freeVerdict(rules, a);
if (free) {
record(ctx, a.id, free);
out.set(a.id, free.decision);
} else {
needAi.push(a);
}
}
if (needAi.length === 0) return out;
if (needAi.length === 1) {
out.set(needAi[0]!.id, await triageArticle(ctx, needAi[0]!));
return out;
}
const settings = ctx.settings.get();
const model = settings.llm.triageModel || settings.llm.summaryModel;
const instructions = promptText(settings, 'triage');
const name = sourceName(ctx, needAi[0]!.source_id);
const prompt = buildTriageBatchPrompt(
instructions,
ruleRefs(rules, 'negative'),
ruleRefs(rules, 'exception'),
filterExamples(ctx.db, settings.pipeline.examplesInPrompt),
needAi.map((a, i) => ({
n: i + 1,
sourceName: name,
title: a.title,
publishedAt: a.published_at,
categories: parseJsonArray<string>(a.categories),
text: truncate(articleText(a), TRIAGE_TEXT_CHARS),
})),
);
const res = await ctx.llm.chatJson({
purpose: 'triage',
model,
system: prompt.system,
user: prompt.user,
schemaName: 'triage_batch',
schema: TRIAGE_BATCH_SCHEMA,
validate: validateTriageBatch,
maxTokens: 512 + 160 * needAi.length,
});
const byN = new Map(res.value.map((r) => [r.n, r]));
const common = { model, promptVersion: promptVersion(instructions), ms: res.ms };
const missing: ArticleRow[] = [];
needAi.forEach((a, i) => {
const r = byN.get(i + 1);
if (!r) {
missing.push(a);
return;
}
const v = interpretTriage(r, rules, settings.pipeline.rejectThreshold, common);
record(ctx, a.id, v);
out.set(a.id, v.decision);
});
bumpStat(ctx.db, 'triage_batched', needAi.length - missing.length - 1);
// The model skipped some: judge those one by one rather than guessing.
for (const a of missing) out.set(a.id, await triageArticle(ctx, a));
return out;
}
// ───────────────────────────── rule CRUD ─────────────────────────────
function normalizeRuleInput(input: Partial<RuleInput>): Partial<RuleInput> {
+48 -12
View File
@@ -4,7 +4,7 @@ import { log, errorMessage } from '../lib/log.ts';
import { LlmUnavailableError } from '../llm/client.ts';
import { normalizeLang } from '../lib/text.ts';
import { extractArticle } from './extract.ts';
import { triageArticle, triageNeedsLlm } from './triage.ts';
import { TRIAGE_BATCH_SIZE, triageBatch, triageNeedsLlm } from './triage.ts';
import { clusterArticle, clusteringActive } from './cluster.ts';
import { PermanentError, summarizeStory } from './summarize.ts';
import { nextPendingDigest, processDigest } from './digest.ts';
@@ -14,7 +14,15 @@ const EXTRACT_SLOTS = 3;
const MAX_ATTEMPTS = 3;
const BACKOFF = [1 * MINUTE, 5 * MINUTE, 15 * MINUTE];
type Task = { key: string; lane: 'extract' | 'llm'; label: string; run: () => Promise<void>; onError: (err: unknown) => void };
type Task = {
key: string;
/** Extra items this task owns (batched triage), claimed alongside `key`. */
extraKeys?: string[];
lane: 'extract' | 'llm';
label: string;
run: () => Promise<void>;
onError: (err: unknown) => void;
};
/**
* Background processing. State lives in the database (article.state / story.state), so a
@@ -78,7 +86,8 @@ export class Worker {
}
private launch(t: Task): void {
this.claimed.add(t.key);
const keys = [t.key, ...(t.extraKeys ?? [])];
for (const k of keys) this.claimed.add(k);
this.active[t.lane]++;
void t
.run()
@@ -90,7 +99,7 @@ export class Worker {
}
})
.finally(() => {
this.claimed.delete(t.key);
for (const k of keys) this.claimed.delete(k);
this.active[t.lane]--;
this.wake();
});
@@ -112,6 +121,10 @@ export class Worker {
return (a.published_at ?? a.fetched_at) < isoMinus(maxAge * HOUR);
}
private sourceStillExtracting(sourceId: number): boolean {
return !!this.ctx.db.prepare(`SELECT 1 FROM article WHERE source_id = ? AND state = 'new' LIMIT 1`).get(sourceId);
}
private skip(a: ArticleRow): void {
this.ctx.db.prepare(`UPDATE article SET state = 'skipped', state_changed_at = ? WHERE id = ?`).run(nowIso(), a.id);
}
@@ -166,15 +179,37 @@ export class Worker {
}
}
for (const a of this.dueArticles('extracted', 50)) {
const extracted = this.dueArticles('extracted', 100);
for (const a of extracted) {
const key = `a${a.id}`;
if (this.claimed.has(key)) continue;
if (this.tooOld(a)) {
this.skip(a);
continue;
}
if (!llmUp && triageNeedsLlm(ctx, a)) continue;
return { key, lane: 'llm', label: 'triage', run: () => this.runTriage(a), onError: (err) => this.articleFailed(a, err) };
const needsLlm = triageNeedsLlm(ctx, a);
if (!llmUp && needsLlm) continue;
// Give a fetch's other articles a moment to finish extracting so they can join this call.
if (needsLlm && (a.state_changed_at > isoMinus(60_000)) && this.sourceStillExtracting(a.source_id)) continue;
// Articles from the same source share the rules, so they can share one triage call.
const batch = [a];
if (needsLlm) {
for (const b of extracted) {
if (batch.length >= TRIAGE_BATCH_SIZE) break;
if (b.id === a.id || b.source_id !== a.source_id || this.claimed.has(`a${b.id}`) || this.tooOld(b)) continue;
if (triageNeedsLlm(ctx, b)) batch.push(b);
}
}
return {
key,
extraKeys: batch.slice(1).map((b) => `a${b.id}`),
lane: 'llm',
label: 'triage',
run: () => this.runTriage(batch),
onError: (err) => {
for (const b of batch) this.articleFailed(b, err);
},
};
}
return null;
}
@@ -193,11 +228,12 @@ export class Worker {
.run(ex.text || null, ex.partial ? 1 : 0, ex.imageUrl, ex.author, language, nowIso(), ex.error, a.id);
}
private async runTriage(a: ArticleRow): Promise<void> {
const decision = await triageArticle(this.ctx, a);
this.ctx.db
.prepare(`UPDATE article SET state = ?, state_changed_at = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ?`)
.run(decision === 'reject' ? 'rejected' : 'kept', nowIso(), a.id);
private async runTriage(batch: ArticleRow[]): Promise<void> {
const decisions = await triageBatch(this.ctx, batch);
const update = this.ctx.db.prepare(
`UPDATE article SET state = ?, state_changed_at = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ? AND state = 'extracted'`,
);
for (const [id, decision] of decisions) update.run(decision === 'reject' ? 'rejected' : 'kept', nowIso(), id);
}
// ───────────────────────────── failure handling ─────────────────────────────
+183
View File
@@ -0,0 +1,183 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import http from 'node:http';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import type { AddressInfo } from 'node:net';
import { loadConfig } from './config.ts';
import { startRssFeed } from './server.ts';
import type { RunningApp } from './server.ts';
import { startMockLlm } from './testing/mock-llm.ts';
import type { MockLlm } from './testing/mock-llm.ts';
import { eventStems } from './pipeline/cluster.ts';
import type { ActivityDTO, StatusDTO, StoryListResponse } from '../../shared/types.ts';
/** Everything that keeps AI calls down: duplicate ledger, free grouping, pre-filter, batching, pausing on bad config. */
const BODY = (s: string) => `${s} `.repeat(8);
const ARTICLES: Record<string, [string, string]> = {
'/a/quake': ['Earthquake hits Turkey, hundreds injured', BODY('A strong earthquake hit eastern Turkey on Sunday, injuring hundreds of people in Malatya.')],
'/a/bank': ['Central bank keeps interest rates unchanged', BODY('The European Central Bank kept rates unchanged on Thursday in Frankfurt.')],
'/a/celeb': ['Celebrity couple announces divorce after reality show', BODY('The celebrity couple from a reality show announced a divorce; showbiz fans reacted.')],
'/a/fish': ['Scientists discover new species of deep sea fish', BODY('Marine biologists found a new fish species in the Pacific Ocean near Hawaii.')],
// Same headline as /a/quake from another outlet (wire copy), different URL.
'/b/quake': ['Earthquake hits Turkey, hundreds injured', BODY('A strong earthquake hit eastern Turkey on Sunday, injuring hundreds of people in Malatya.')],
'/b/match': ['Dinamo wins derby against Hajduk in Zagreb', BODY('Dinamo beat Hajduk 3-1 in Zagreb on Saturday evening.')],
};
interface Harness {
app: RunningApp;
mock: MockLlm;
feeds: Record<string, string[]>;
base: string;
call<T>(method: string, p: string, body?: unknown): Promise<T>;
idle(): Promise<void>;
close(): Promise<void>;
}
async function harness(feeds: Record<string, string[]>): Promise<Harness> {
let base = '';
const fixture = http.createServer((req, res) => {
const u = (req.url ?? '').replace(/\?.*$/, '');
const feed = feeds[u];
if (feed) {
return res.writeHead(200, { 'content-type': 'application/rss+xml' }).end(
`<?xml version="1.0"?><rss version="2.0"><channel><title>F</title>${feed
.map((k, i) => `<item><title>${ARTICLES[k]![0]}</title><link>${base}${k}</link><guid>${k}</guid><pubDate>${new Date(Date.now() - (i + 1) * 60_000).toUTCString()}</pubDate></item>`)
.join('')}</channel></rss>`,
);
}
const a = ARTICLES[u];
if (a) return res.writeHead(200, { 'content-type': 'text/html' }).end(`<html lang="en"><head><title>${a[0]}</title></head><body><article><h1>${a[0]}</h1><p>${a[1]}</p><p>${a[1]}</p></article></body></html>`);
res.writeHead(404).end();
});
await new Promise<void>((r) => fixture.listen(0, '127.0.0.1', () => r()));
base = `http://127.0.0.1:${(fixture.address() as AddressInfo).port}`;
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'rssfeed-sav-'));
const mock = await startMockLlm(0, { embeddings: false });
const config = { ...loadConfig({ RSSFEED_DATA_DIR: dir, TZ: 'Europe/Zagreb', RSSFEED_ALLOW_PRIVATE_URLS: '1', RSSFEED_WEB_DIST: path.join(dir, 'none') }), port: 0, host: '127.0.0.1' };
const app = await startRssFeed(config, { scheduler: false });
const call = async <T>(method: string, p: string, body?: unknown): Promise<T> => {
const res = await fetch(`http://127.0.0.1:${app.port}${p}`, { method, headers: { 'content-type': 'application/json' }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) });
const text = await res.text();
assert.ok(res.ok, `${method} ${p} → ${res.status}: ${text}`);
return (text ? JSON.parse(text) : null) as T;
};
const idle = async () => {
// Sources added via the API are fetched in the background: wait until every one has been fetched.
for (let i = 0; i < 200; i++) {
const sources = await call<{ lastFetchAt: string | null }[]>('GET', '/api/sources');
if (sources.every((s) => s.lastFetchAt)) break;
await new Promise((r) => setTimeout(r, 100));
}
for (let i = 0; i < 200; i++) {
const q = (await call<ActivityDTO>('GET', '/api/activity')).queue;
if (q.articlesNew + q.articlesExtracted + q.articlesKept + q.storiesPending + q.storiesUpdating === 0 && !app.worker.busy) return;
await new Promise((r) => setTimeout(r, 150));
}
throw new Error('pipeline did not go idle');
};
return {
app,
mock,
feeds,
base,
call,
idle,
async close() {
await app.close();
await mock.close();
await new Promise<void>((r) => fixture.close(() => r()));
fs.rmSync(dir, { recursive: true, force: true });
},
};
}
const chatCalls = (m: MockLlm, schema?: string) => m.calls.filter((c) => c.path.endsWith('/chat/completions') && (!schema || c.schema === schema)).length;
test('eventStems: names cut to 4 letters, numbers, years and sentence starters ignored', () => {
const hr = eventStems('Trumpov plan za golf teren u Washingtonu propao, sud blokirao zatvaranje 15 rupa');
const en = eventStems('Judge blocks Trump plan to close Washington golf course with 15 holes in 2026');
assert.ok(hr.has('trum') && en.has('trum'));
assert.ok(hr.has('wash') && en.has('wash'));
assert.ok(hr.has('#15') && en.has('#15'));
assert.ok(!en.has('#2026'));
assert.ok(!eventStems('The new report says').size);
});
test('AI savings: batched triage, free wire-copy grouping, match pre-filter, seen ledger', { timeout: 90_000 }, async () => {
const h = await harness({ '/one.xml': ['/a/quake', '/a/bank', '/a/celeb', '/a/fish'], '/two.xml': [] });
try {
await h.call('PUT', '/api/settings', { llm: { baseUrl: h.mock.url, triageModel: 'mock-small', summaryModel: 'mock-large', embeddingModel: '' } });
await h.call('POST', '/api/rules', { kind: 'ai', polarity: 'negative', text: 'Celebrity gossip and showbiz news' });
const one = await h.call<{ id: number }>('POST', '/api/sources', { feedUrl: `${h.base}/one.xml`, name: 'One' });
await h.idle();
// 4 articles from one source → one batched triage call, not four.
assert.equal(chatCalls(h.mock, 'triage_batch'), 1, JSON.stringify(h.mock.calls.map((c) => c.schema)));
assert.equal(chatCalls(h.mock, 'triage'), 0);
// Unrelated articles share no names/numbers with earlier stories → no story-match calls at all.
assert.equal(chatCalls(h.mock, 'story_match'), 0);
const first = await h.call<StoryListResponse>('GET', '/api/stories?view=all');
assert.equal(first.total, 3, 'celebrity article filtered, three stories');
// Wire copy: identical headline from a second outlet joins the story without any AI call.
h.feeds['/two.xml'] = ['/b/quake'];
const before = chatCalls(h.mock);
const two = await h.call<{ id: number }>('POST', '/api/sources', { feedUrl: `${h.base}/two.xml`, name: 'Two' });
await h.idle();
const quake = (await h.call<StoryListResponse>('GET', '/api/stories?view=all')).items.find((s) => s.sourceCount === 2);
assert.ok(quake, 'wire copy joined the earthquake story');
assert.equal(chatCalls(h.mock), before, 'no AI calls for wire copy');
// A related article (shares "Zagreb"? no: shares nothing) vs one that shares names gets a match call.
h.feeds['/two.xml'] = ['/b/quake', '/b/match'];
await h.call('POST', `/api/sources/${two.id}/fetch`);
await h.idle();
assert.equal(chatCalls(h.mock, 'story_match'), 0, 'Dinamo/Hajduk shares nothing with earlier stories');
// Seen ledger: an article pruned from the database is not processed again if the feed still lists it.
h.app.ctx.db.prepare(`DELETE FROM article WHERE url LIKE '%/a/celeb'`).run();
const callsBefore = chatCalls(h.mock);
await h.call('POST', `/api/sources/${one.id}/fetch`);
await h.idle();
assert.equal(chatCalls(h.mock), callsBefore, 'pruned article was not re-triaged');
assert.equal(h.app.ctx.db.prepare(`SELECT COUNT(*) AS n FROM article WHERE url LIKE '%/a/celeb'`).get<{ n: number }>()!.n, 0);
const act = await h.call<ActivityDTO>('GET', '/api/activity');
assert.equal(act.savings7d.groupedByHeadline, 1);
assert.ok(act.savings7d.triageBatched >= 2);
assert.ok(act.savings7d.matchCallsSkipped >= 3);
assert.ok(act.llm.byPurpose.some((p) => p.purpose === 'triage'));
} finally {
await h.close();
}
});
test('a misconfigured model pauses LLM work instead of failing every article', { timeout: 60_000 }, async () => {
const h = await harness({ '/one.xml': ['/a/quake', '/a/bank'] });
try {
await h.call('PUT', '/api/settings', { llm: { baseUrl: h.mock.url, triageModel: 'mock-small', summaryModel: 'no-such-model', embeddingModel: '' } });
await h.call('POST', '/api/sources', { feedUrl: `${h.base}/one.xml`, name: 'One' });
for (let i = 0; i < 100; i++) {
if ((await h.call<StatusDTO>('GET', '/api/status')).llmError) break;
await new Promise((r) => setTimeout(r, 150));
}
const st = await h.call<StatusDTO>('GET', '/api/status');
assert.equal(st.llm, 'offline');
assert.match(st.llmError ?? '', /no-such-model/);
await new Promise((r) => setTimeout(r, 1500));
// One rejected request, not one per article per output format.
assert.equal(h.mock.calls.filter((c) => c.model === 'no-such-model').length, 1);
const act = await h.call<ActivityDTO>('GET', '/api/activity');
assert.equal(act.failures.length, 0, 'articles wait instead of failing');
// Fixing the setting resumes work.
await h.call('PUT', '/api/settings', { llm: { summaryModel: 'mock-large' } });
await h.idle();
assert.equal((await h.call<StoryListResponse>('GET', '/api/stories?view=all')).total, 2);
} finally {
await h.close();
}
});
+5 -2
View File
@@ -1,4 +1,4 @@
import { CATEGORIES, PROMPT_KEYS } from '../../shared/types.ts';
import { CATEGORIES, PROMPT_KEYS, REASONING_EFFORTS } from '../../shared/types.ts';
import type { Category, CategoryControl, DeepPartial, FeedControls, Settings } from '../../shared/types.ts';
import type { Config } from './config.ts';
import type { Database } from './db/database.ts';
@@ -40,8 +40,10 @@ export function defaultSettings(): Settings {
embeddingModel: '',
concurrency: 1,
timeoutSec: 120,
maxArticleChars: 12_000,
// Two sentences need the lede and the key paragraphs, not the whole article.
maxArticleChars: 6_000,
temperature: 0.2,
reasoningEffort: '',
},
pipeline: {
defaultPollMinutes: 15,
@@ -115,6 +117,7 @@ export function sanitizeSettings(input: unknown): Settings {
timeoutSec: clampInt(llm.timeoutSec, 10, 900, d.llm.timeoutSec),
maxArticleChars: clampInt(llm.maxArticleChars, 1000, 200_000, d.llm.maxArticleChars),
temperature: clamp(llm.temperature, 0, 2, d.llm.temperature),
reasoningEffort: oneOf(llm.reasoningEffort, REASONING_EFFORTS, ''),
},
pipeline: {
defaultPollMinutes: clampInt(s.pipeline.defaultPollMinutes, 1, 1440, d.pipeline.defaultPollMinutes),
+19 -2
View File
@@ -22,6 +22,7 @@ export interface MockLlm {
close(): Promise<void>;
}
const KNOWN_MODELS = ['mock-small', 'mock-large', 'mock-embed'];
const WORD = /[\p{L}\p{N}]+/gu;
const STOP = new Set(['the', 'a', 'an', 'of', 'in', 'on', 'to', 'and', 'for', 'is', 'at', 'by', 'with', 'from', 'je', 'i', 'u', 'na', 'za', 'se', 'od', 'da']);
@@ -66,6 +67,17 @@ function answer(schema: string | null, system: string, user: string): unknown {
? { decision: 'reject', rule_id: ruleId, reason: 'The article is celebrity gossip.', confidence: 0.93 }
: { decision: 'keep', rule_id: null, reason: 'No rule matches the article.', confidence: 0.9 };
}
case 'triage_batch': {
const ruleId = Number(/\[(\d+)\]/.exec(system.split('Negative rules')[1] ?? '')?.[1] ?? 0) || null;
return {
results: [...user.matchAll(/<article_(\d+)>([\s\S]*?)<\/article_\d+>/g)].map((b) => {
const celeb = /celebrit|showbiz|slavn|influencer|reality/i.test(b[2] ?? '');
return celeb
? { n: Number(b[1]), decision: 'reject', rule_id: ruleId, reason: 'The article is celebrity gossip.', confidence: 0.93 }
: { n: Number(b[1]), decision: 'keep', rule_id: null, reason: 'No rule matches the article.', confidence: 0.9 };
}),
};
}
case 'same_event': {
const [a, b] = user.split('<article_b>');
const sim = jaccard(headline(a ?? ''), headline(b ?? ''));
@@ -130,7 +142,7 @@ export async function startMockLlm(port = 0, opts: MockOptions = {}): Promise<Mo
if (down) return send(503, { error: 'mock outage' });
const path = (req.url ?? '').replace(/\?.*$/, '');
if (req.method === 'GET' && path.endsWith('/models')) {
return send(200, { object: 'list', data: ['mock-small', 'mock-large', 'mock-embed'].map((id) => ({ id, object: 'model' })) });
return send(200, { object: 'list', data: KNOWN_MODELS.map((id) => ({ id, object: 'model' })) });
}
let body: Record<string, unknown> = {};
try {
@@ -153,9 +165,14 @@ export async function startMockLlm(port = 0, opts: MockOptions = {}): Promise<Mo
const msgs = (body.messages ?? []) as { role: string; content: string }[];
const system = msgs.find((m) => m.role === 'system')?.content ?? '';
const user = msgs.find((m) => m.role === 'user')?.content ?? '';
if (!KNOWN_MODELS.includes(String(body.model))) {
calls.push({ path, schema: null, model: String(body.model) });
return send(404, { error: { message: `model '${String(body.model)}' not found` } });
}
let schema = rf?.json_schema?.name ?? null;
if (!schema) {
if (/"decision"/.test(system)) schema = 'triage';
if (/"results"/.test(system) && /"decision"/.test(system)) schema = 'triage_batch';
else if (/"decision"/.test(system)) schema = 'triage';
else if (/same_event/.test(system)) schema = 'same_event';
else if (/title_hr/.test(system)) schema = 'story_summary';
else if (/"opening"/.test(system)) schema = 'digest_opening';
+18
View File
@@ -357,6 +357,9 @@ export interface DigestDTO {
// ───────────────────────────── Settings ─────────────────────────────
export const REASONING_EFFORTS = ['', 'minimal', 'low', 'medium', 'high'] as const;
export type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
export interface LlmSettings {
baseUrl: string;
apiKey: string;
@@ -367,6 +370,8 @@ export interface LlmSettings {
timeoutSec: number;
maxArticleChars: number;
temperature: number;
/** Sent as `reasoning_effort`; '' = don't send (server default). Lower = fewer hidden thinking tokens. */
reasoningEffort: ReasoningEffort;
}
export interface PipelineSettings {
@@ -535,6 +540,9 @@ export interface LlmTestResult {
export interface PlaygroundSummary {
model: string;
ok: boolean;
/** Tokens billed for this summary (including any retry and hidden reasoning). */
tokensIn: number | null;
tokensOut: number | null;
title: string | null;
description: string | null;
category: Category | null;
@@ -601,7 +609,15 @@ export interface ActivityDTO {
p95LatencyMs: number | null;
tokensIn24h: number;
tokensOut24h: number;
/** Why LLM work is paused, when the server rejected requests (bad key, unknown model). */
error: string | null;
/** Last 24 h, per pipeline step (triage, story-match, summarize, …). */
byPurpose: { purpose: string; calls: number; errors: number; tokensIn: number; tokensOut: number }[];
/** LLM calls in the last 24 h per story published in the last 24 h. */
callsPerStory: number | null;
};
/** Work saved without AI in the last 7 days. */
savings7d: { duplicatesSkipped: number; groupedByHeadline: number; matchCallsSkipped: number; triageBatched: number };
failures: ActivityFailure[];
importance7d: { score: number; count: number }[];
grouping7d: { articles: number; inMultiSourceStories: number; share: number };
@@ -621,6 +637,8 @@ export interface ActivityDTO {
*/
export interface StatusDTO {
llm: LlmState;
/** Set when LLM work is paused because the server rejected requests (bad key, unknown model). */
llmError: string | null;
/** Articles and stories waiting on the LLM. */
waiting: number;
version: string;
+13
View File
@@ -31,6 +31,19 @@ function StatusBanner() {
);
}
if (!status) return null;
if (status.llm === 'offline' && status.llmError) {
return (
<div className="banner banner-danger" role="status">
<div className="banner__inner">
<strong>LLM paused</strong>
<span>
{status.llmError} {status.waiting > 0 ? `${plural(status.waiting, 'item')} waiting; nothing is lost.` : ''}
</span>
<Link to="/settings/llm">Fix it</Link>
</div>
</div>
);
}
if (status.llm === 'offline') {
return (
<div className="banner banner-warn" role="status">
+65 -8
View File
@@ -32,6 +32,20 @@ function LlmStateBadge({ state }: { state: ActivityDTO['llm']['state'] }) {
return <Badge tone="warn">Not configured</Badge>;
}
const PURPOSE_LABELS: Record<string, string> = {
triage: 'Triage (negatives)',
'story-match': 'Grouping (triage model)',
'same-event': 'Grouping check',
cluster: 'Grouping (embeddings)',
summarize: 'Title and description',
regenerate: 'Rewrite from several sources',
digest: 'Digest opening',
'dry-run': 'Rule dry runs',
playground: 'Playground',
test: 'Connection tests',
embed: 'Embeddings',
};
function ActivityBody({ data, onChanged }: { data: ActivityDTO; onChanged: () => void }) {
const [busy, setBusy] = useState<string | null>(null);
const q = data.queue;
@@ -93,18 +107,61 @@ function ActivityBody({ data, onChanged }: { data: ActivityDTO; onChanged: () =>
Connect a server in <Link to="/settings/llm">Settings → LLM connection</Link>.
</p>
)}
{l.state === 'offline' && (
<p className="small">
{plural(l.consecutiveFailures, 'failure')} in a row. {l.nextProbeAt ? `Next check ${relTime(l.nextProbeAt)}.` : ''} Nothing is lost while
it's offline.
</p>
)}
{l.state === 'offline' &&
(l.error ? (
<p className="small danger">
{l.error} Work is paused and nothing is lost; it resumes when you change the LLM settings, or by itself within 10 minutes.
</p>
) : (
<p className="small">
{plural(l.consecutiveFailures, 'failure')} in a row. {l.nextProbeAt ? `Next check ${relTime(l.nextProbeAt)}.` : ''} Nothing is lost while
it's offline.
</p>
))}
<div className="stats">
<Stat label="Calls, 24 h" value={fmtNumber(l.calls24h)} hint={l.errors24h ? `${l.errors24h} errors` : 'no errors'} />
<Stat label="Calls per story" value={l.callsPerStory === null ? '—' : String(l.callsPerStory)} hint="pipeline calls per published story" />
<Stat label="Average latency" value={fmtMs(l.avgLatencyMs)} hint={`p95 ${fmtMs(l.p95LatencyMs)}`} />
<Stat label="In flight" value={fmtNumber(l.inFlight)} />
<Stat label="Tokens in, 24 h" value={fmtNumber(l.tokensIn24h)} />
<Stat label="Tokens out, 24 h" value={fmtNumber(l.tokensOut24h)} />
<Stat label="Tokens out, 24 h" value={fmtNumber(l.tokensOut24h)} hint="includes hidden reasoning" />
</div>
{l.byPurpose.length > 0 && (
<div className="table-wrap">
<table className="table">
<thead>
<tr>
<th>Step, last 24 h</th>
<th>Calls</th>
<th>Errors</th>
<th>Tokens in</th>
<th>Tokens out</th>
<th>Out per call</th>
</tr>
</thead>
<tbody>
{l.byPurpose.map((p) => (
<tr key={p.purpose}>
<td>{PURPOSE_LABELS[p.purpose] ?? p.purpose}</td>
<td>{fmtNumber(p.calls)}</td>
<td className={p.errors ? 'danger' : 'faint'}>{fmtNumber(p.errors)}</td>
<td>{fmtNumber(p.tokensIn)}</td>
<td>{fmtNumber(p.tokensOut)}</td>
<td className="faint">{p.calls - p.errors > 0 ? fmtNumber(Math.round(p.tokensOut / Math.max(1, p.calls - p.errors))) : '—'}</td>
</tr>
))}
</tbody>
</table>
</div>
)}
</section>
<section className="panel">
<h2 className="panel__title">Saved without AI, last 7 days</h2>
<div className="stats">
<Stat label="Duplicates dropped" value={fmtNumber(data.savings7d.duplicatesSkipped)} hint="same URL, item or headline" />
<Stat label="Wire copy grouped" value={fmtNumber(data.savings7d.groupedByHeadline)} hint="identical headline, joined for free" />
<Stat label="Grouping calls skipped" value={fmtNumber(data.savings7d.matchCallsSkipped)} hint="no shared names with recent stories" />
<Stat label="Triage calls saved" value={fmtNumber(data.savings7d.triageBatched)} hint="articles judged together" />
</div>
</section>
+5
View File
@@ -166,6 +166,11 @@ function SummaryCard({ s }: { s: PlaygroundSummary }) {
<Badge tone={s.languageOk ? 'ok' : 'danger'}>{s.languageOk ? 'Croatian OK' : 'Language check failed'}</Badge>
{s.formatRetries > 0 && <span> · {s.formatRetries} format retries</span>}
</p>
{(s.tokensIn !== null || s.tokensOut !== null) && (
<p className="faint small">
{s.tokensIn ?? '—'} tokens in · {s.tokensOut ?? '—'} tokens out (including hidden reasoning)
</p>
)}
</div>
);
}
+14 -2
View File
@@ -1,6 +1,6 @@
import { useState, type ReactNode } from 'react';
import { api } from '../../api.ts';
import type { LlmSettings, LlmTestResult } from '../../contract.ts';
import type { LlmSettings, LlmTestResult, ReasoningEffort } from '../../contract.ts';
import { useDraft } from '../../hooks.ts';
import { Badge, ErrorNote, Field, NumberField, Spinner, TextField } from '../../components.tsx';
import { errorMessage, fmtMs } from '../../util.ts';
@@ -122,7 +122,7 @@ export function LlmSection({ data, onSaved }: SectionProps) {
<NumberField label="Timeout" suffix="s" min={10} max={900} value={draft.timeoutSec} onChange={(v) => set('timeoutSec', Math.round(v))} />
<NumberField
label="Max article characters"
hint="Longer text is truncated to fit the context window."
hint="Text sent for the title and description. 6000 covers the lede and key paragraphs; more mostly costs tokens."
min={1000}
max={200000}
step={500}
@@ -130,6 +130,18 @@ export function LlmSection({ data, onSaved }: SectionProps) {
onChange={(v) => set('maxArticleChars', Math.round(v))}
/>
<NumberField label="Temperature" hint="Low keeps summaries factual." min={0} max={2} step={0.05} value={draft.temperature} onChange={(v) => set('temperature', v)} />
<Field
label="Reasoning effort"
hint="Sent as reasoning_effort. Models that think before answering bill those hidden tokens as output; Low is plenty for two sentences. Compare token counts in the Playground."
>
<select value={draft.reasoningEffort} onChange={(e) => set('reasoningEffort', e.target.value as ReasoningEffort)}>
<option value="">Server default (not sent)</option>
<option value="minimal">Minimal</option>
<option value="low">Low</option>
<option value="medium">Medium</option>
<option value="high">High</option>
</select>
</Field>
</div>
</fieldset>