Cut AI usage: free dedupe and grouping, batched triage, fewer tokens
Release / build (push) Successful in 1m10s
Release / build (push) Successful in 1m10s
Before the AI (free): - Seen-URL ledger: every article URL is remembered for 180+ days, so an item still listed in a feed after its article was pruned (rejected ones go after 14 days) is never fetched or judged again. - Wire copy: an identical headline from another outlet within 48 h joins the existing story (or is filtered like its twin) with no triage, grouping or summary call, and does not trigger a rewrite. Triage: - Up to 8 articles from one source are judged in one call, so rules, instructions and examples are sent once; the worker waits up to a minute for a fetch's other articles to finish extracting. Articles the model skips are judged singly. - 800 characters of article text instead of 1500. Grouping without embeddings: - Free pre-filter on shared names (4-letter stems, so Trump/Trumpov and Bahrain/Bahrein agree) and numbers; no call when no recent story shares one, and at most 12 candidates instead of 60. Title and description: - Max article characters default 6000 (was 12000); a multi-source rewrite shares that budget. - New "Reasoning effort" setting, sent as reasoning_effort. Measured on production: Haiku 5.5 spends ~590 hidden reasoning tokens per summary. No wasted retries: - 401/403/404 from the chat endpoint (bad key, unknown model, wrong URL) pauses LLM work with a banner instead of failing every article; it resumes on a settings change or after 10 minutes. - Simpler structured-output modes are tried only on 400/422, so a misconfiguration costs one request, not three. Visibility: - Activity: calls and tokens per pipeline step, calls per published story, and work saved without AI. Playground: tokens per model. Tests for each saving and for the pause on misconfiguration. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4789767103
commit
29036127a9
20 files changed
+786
-100
No files matched your search
@@ -203,6 +203,14 @@ const MIGRATIONS: string[] = [
|
||||
PRIMARY KEY (day, key)
|
||||
);
|
||||
`,
|
||||
/* 2 — long-lived ledger of every article URL ever seen, so an item that is still in a feed
|
||||
after its article was pruned (e.g. rejected ones after 14 days) is never processed again */ `
|
||||
CREATE TABLE seen (
|
||||
url_hash TEXT PRIMARY KEY,
|
||||
first_seen TEXT NOT NULL
|
||||
);
|
||||
INSERT OR IGNORE INTO seen (url_hash, first_seen) SELECT url_hash, fetched_at FROM article;
|
||||
`,
|
||||
];
|
||||
|
||||
export function migrate(db: Database): number {
|
||||
|
||||
@@ -19,6 +19,16 @@ export function buildActivity(ctx: Ctx): ActivityDTO {
|
||||
const calls = db.prepare(`SELECT ok, ms, tokens_in, tokens_out FROM llm_call WHERE at >= ?`).all<{ ok: number; ms: number; tokens_in: number | null; tokens_out: number | null }>(since24h);
|
||||
const okMs = calls.filter((c) => c.ok).map((c) => c.ms).sort((a, b) => a - b);
|
||||
const breaker = ctx.llm.breakerInfo();
|
||||
const byPurpose = db
|
||||
.prepare(
|
||||
`SELECT purpose, COUNT(*) AS calls, SUM(CASE WHEN ok = 0 THEN 1 ELSE 0 END) AS errors,
|
||||
COALESCE(SUM(tokens_in), 0) AS tokens_in, COALESCE(SUM(tokens_out), 0) AS tokens_out
|
||||
FROM llm_call WHERE at >= ? GROUP BY purpose ORDER BY calls DESC`,
|
||||
)
|
||||
.all<{ purpose: string; calls: number; errors: number; tokens_in: number; tokens_out: number }>(since24h)
|
||||
.map((r) => ({ purpose: r.purpose, calls: r.calls, errors: r.errors, tokensIn: r.tokens_in, tokensOut: r.tokens_out }));
|
||||
const pipelineCalls = byPurpose.filter((p) => !['test', 'playground', 'dry-run'].includes(p.purpose)).reduce((s, p) => s + p.calls, 0);
|
||||
const published24h = count(`SELECT COUNT(*) AS n FROM story WHERE first_published_at >= ?`, since24h);
|
||||
|
||||
const failures: ActivityFailure[] = [
|
||||
...db
|
||||
@@ -87,6 +97,15 @@ export function buildActivity(ctx: Ctx): ActivityDTO {
|
||||
p95LatencyMs: okMs.length ? okMs[Math.min(okMs.length - 1, Math.floor(okMs.length * 0.95))]! : null,
|
||||
tokensIn24h: calls.reduce((s, c) => s + (c.tokens_in ?? 0), 0),
|
||||
tokensOut24h: calls.reduce((s, c) => s + (c.tokens_out ?? 0), 0),
|
||||
error: ctx.llm.errorMessage(),
|
||||
byPurpose,
|
||||
callsPerStory: published24h ? Math.round((pipelineCalls / published24h) * 10) / 10 : null,
|
||||
},
|
||||
savings7d: {
|
||||
duplicatesSkipped: statSum('duplicates_skipped'),
|
||||
groupedByHeadline: statSum('grouped_by_headline'),
|
||||
matchCallsSkipped: statSum('match_calls_skipped'),
|
||||
triageBatched: statSum('triage_batched'),
|
||||
},
|
||||
failures: failures.slice(0, 60),
|
||||
importance7d,
|
||||
|
||||
@@ -319,6 +319,8 @@ async function playgroundRun(ctx: Ctx, url: string, models: string[]): Promise<P
|
||||
summaries.push({
|
||||
model,
|
||||
ok: w.languageOk,
|
||||
tokensIn: w.tokensIn,
|
||||
tokensOut: w.tokensOut,
|
||||
title: w.title,
|
||||
description: w.description,
|
||||
category: w.category,
|
||||
@@ -329,7 +331,7 @@ async function playgroundRun(ctx: Ctx, url: string, models: string[]): Promise<P
|
||||
error: w.languageOk ? null : `Not standard Croatian: it ${w.languageProblem}`,
|
||||
});
|
||||
} catch (err) {
|
||||
summaries.push({ model, ok: false, title: null, description: null, category: null, importance: null, formatRetries: 0, languageOk: false, ms: Date.now() - t0, error: errorMessage(err) });
|
||||
summaries.push({ model, ok: false, tokensIn: null, tokensOut: null, title: null, description: null, category: null, importance: null, formatRetries: 0, languageOk: false, ms: Date.now() - t0, error: errorMessage(err) });
|
||||
}
|
||||
}
|
||||
return { extraction, triage, summaries };
|
||||
@@ -378,6 +380,7 @@ export function apiRouter(ctx: Ctx): express.Router {
|
||||
r.get('/status', (_req, res: Response<StatusDTO>) => {
|
||||
res.json({
|
||||
llm: ctx.llm.state(),
|
||||
llmError: ctx.llm.errorMessage(),
|
||||
waiting: waitingCount(ctx),
|
||||
version: ctx.config.version,
|
||||
latestDigestId: latestPublishedDigest(ctx)?.id ?? null,
|
||||
|
||||
@@ -113,6 +113,8 @@ export class LlmClient {
|
||||
private failures = 0;
|
||||
private nextProbe = 0;
|
||||
private probing = false;
|
||||
/** Set when the server rejects requests as misconfigured (401/403/404); pauses LLM work. */
|
||||
private configError: { message: string; until: number } | null = null;
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
private readonly listeners: (() => void)[] = [];
|
||||
|
||||
@@ -140,12 +142,28 @@ export class LlmClient {
|
||||
this.sem.limit = this.getSettings().concurrency;
|
||||
this.modeCache.clear();
|
||||
// New connection settings deserve a fresh chance.
|
||||
const hadConfigError = this.configError !== null;
|
||||
this.configError = null;
|
||||
if (this.failures > 0) this.reset();
|
||||
else if (hadConfigError) for (const l of this.listeners) l();
|
||||
}
|
||||
|
||||
/** Why LLM work is paused because the server rejected requests, if it is. */
|
||||
errorMessage(): string | null {
|
||||
if (!this.configError) return null;
|
||||
if (Date.now() >= this.configError.until) {
|
||||
// Try again after a while (the server may have been fixed without a settings change).
|
||||
this.configError = null;
|
||||
for (const l of this.listeners) l();
|
||||
return null;
|
||||
}
|
||||
return this.configError.message;
|
||||
}
|
||||
|
||||
state(): LlmState {
|
||||
const s = this.getSettings();
|
||||
if (!s.baseUrl || !s.summaryModel) return 'unconfigured';
|
||||
if (this.errorMessage()) return 'offline';
|
||||
return this.failures >= BREAKER_THRESHOLD ? 'offline' : 'ok';
|
||||
}
|
||||
|
||||
@@ -252,6 +270,7 @@ export class LlmClient {
|
||||
const s = this.conn(opts.override);
|
||||
const affectBreaker = !opts.override;
|
||||
if (affectBreaker && this.failures >= BREAKER_THRESHOLD) throw new LlmUnavailableError('LLM is offline');
|
||||
if (affectBreaker && this.errorMessage()) throw new LlmUnavailableError(this.errorMessage()!);
|
||||
if (!opts.model) throw new LlmUnavailableError('No model configured');
|
||||
const release = await this.sem.acquire();
|
||||
const started = Date.now();
|
||||
@@ -273,6 +292,7 @@ export class LlmClient {
|
||||
max_tokens: opts.maxTokens ?? 2048,
|
||||
stream: false,
|
||||
};
|
||||
if (s.reasoningEffort) body.reasoning_effort = s.reasoningEffort;
|
||||
if (mode === 'json_schema') {
|
||||
body.response_format = { type: 'json_schema', json_schema: { name: opts.schemaName, strict: true, schema: opts.schema } };
|
||||
} else if (mode === 'json_object') {
|
||||
@@ -282,9 +302,11 @@ export class LlmClient {
|
||||
try {
|
||||
data = await this.request('/chat/completions', body, s, s.timeoutSec * 1000);
|
||||
} catch (err) {
|
||||
if (err instanceof LlmRequestError && mode !== 'prompt') {
|
||||
// Only a rejected request *shape* (400/422) is worth retrying in a simpler output mode;
|
||||
// a bad key, unknown model or wrong URL would just fail three times.
|
||||
if (err instanceof LlmRequestError && (err.status === 400 || err.status === 422) && mode !== 'prompt') {
|
||||
firstError ??= err;
|
||||
continue; // try a simpler structured-output mode
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
@@ -298,6 +320,14 @@ export class LlmClient {
|
||||
const e = err as Error;
|
||||
this.record(opts.purpose, opts.model, false, Date.now() - started, null, null, e.message);
|
||||
if (affectBreaker && err instanceof LlmUnavailableError) this.recordFailure(e);
|
||||
if (affectBreaker && err instanceof LlmRequestError && [401, 403, 404].includes(err.status)) {
|
||||
// Misconfiguration (bad key, unknown model, wrong URL): every request would fail the same
|
||||
// way, so pause all LLM work instead of burning attempts on each article.
|
||||
const message = `The LLM server rejected requests for "${opts.model}" (HTTP ${err.status}). Check Settings → LLM connection. ${err.message.slice(0, 200)}`;
|
||||
if (!this.configError) log.warn('pausing LLM work: the server rejected the request', { model: opts.model, status: err.status });
|
||||
this.configError = { message, until: Date.now() + 10 * 60_000 };
|
||||
throw new LlmUnavailableError(message);
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
release();
|
||||
@@ -341,6 +371,7 @@ export class LlmClient {
|
||||
|
||||
// One repair attempt with the model's own answer in context.
|
||||
const repairBody: Record<string, unknown> = {
|
||||
...(s.reasoningEffort ? { reasoning_effort: s.reasoningEffort } : {}),
|
||||
model: opts.model,
|
||||
messages: [
|
||||
...messages,
|
||||
|
||||
@@ -203,6 +203,66 @@ export function validateTriage(v: unknown): TriageOutput | string {
|
||||
};
|
||||
}
|
||||
|
||||
// ───────────────────────────── triage, batched ─────────────────────────────
|
||||
|
||||
export const TRIAGE_BATCH_SCHEMA: JsonSchema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
results: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
n: { type: 'integer' },
|
||||
decision: { type: 'string', enum: ['keep', 'reject'] },
|
||||
rule_id: { type: ['integer', 'null'] },
|
||||
reason: { type: 'string' },
|
||||
confidence: { type: 'number' },
|
||||
},
|
||||
required: ['n', 'decision', 'rule_id', 'reason', 'confidence'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ['results'],
|
||||
additionalProperties: false,
|
||||
};
|
||||
|
||||
/** Several articles from one source in one call: the rules and instructions are sent once. */
|
||||
export function buildTriageBatchPrompt(
|
||||
instructions: string,
|
||||
negatives: TriageRuleRef[],
|
||||
exceptions: TriageRuleRef[],
|
||||
examples: TriageExample[],
|
||||
articles: (ArticleForPrompt & { n: number })[],
|
||||
): { system: string; user: string } {
|
||||
const single = buildTriagePrompt(instructions, negatives, exceptions, examples, articles[0]!);
|
||||
const system = single.system.replace(
|
||||
/\n\nOutput format:[\s\S]*$/,
|
||||
`\n\nYou get several numbered articles. Judge each one on its own.
|
||||
|
||||
Output format: reply with ONLY a JSON object:
|
||||
{"results": [{"n": <article number>, "decision": "keep" or "reject", "rule_id": <the id in brackets of the rule that decided it, or null>, "reason": "<one short sentence in English>", "confidence": <number from 0 to 1>}]}
|
||||
Include every article number exactly once.`,
|
||||
);
|
||||
const user = articles.map((a) => articleBlock(a, `article_${a.n}`)).join('\n\n');
|
||||
return { system, user };
|
||||
}
|
||||
|
||||
export function validateTriageBatch(v: unknown): (TriageOutput & { n: number })[] | string {
|
||||
if (!v || typeof v !== 'object') return 'expected a JSON object';
|
||||
const results = (v as Record<string, unknown>).results;
|
||||
if (!Array.isArray(results)) return 'results must be an array';
|
||||
const out: (TriageOutput & { n: number })[] = [];
|
||||
for (const r of results) {
|
||||
if (!r || typeof r !== 'object') continue;
|
||||
const n = Number((r as Record<string, unknown>).n);
|
||||
const t = validateTriage(r);
|
||||
if (Number.isInteger(n) && typeof t !== 'string') out.push({ ...t, n });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ───────────────────────────── dry run (batched) ─────────────────────────────
|
||||
|
||||
export const DRY_RUN_SCHEMA: JsonSchema = {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { Ctx } from '../context.ts';
|
||||
import { bumpStat } from '../context.ts';
|
||||
import { HOUR, isoMinus } from '../lib/time.ts';
|
||||
import { truncate } from '../lib/text.ts';
|
||||
import { log } from '../lib/log.ts';
|
||||
@@ -157,7 +158,47 @@ function sourceName(ctx: Ctx, id: number): string {
|
||||
return ctx.db.prepare(`SELECT name FROM source WHERE id = ?`).get<{ name: string }>(id)?.name ?? 'Unknown';
|
||||
}
|
||||
|
||||
const MAX_LLM_CANDIDATES = 60;
|
||||
/** Recent stories scanned for a possible match, and the most sent to the model. */
|
||||
const SCAN_STORIES = 200;
|
||||
const MAX_LLM_CANDIDATES = 12;
|
||||
|
||||
/** Capitalised words that start sentences or headlines in Croatian and English and say nothing about the event. */
|
||||
const COMMON = new Set(
|
||||
(
|
||||
'the a an this that these those there here in on at for from with after before when how why what who new his her its their our your ' +
|
||||
'we you they it he she said says says breaking live update watch video photo photos exclusive report reports ' +
|
||||
'u na za iz od do po kako sto zasto gdje tko koji koja koje novi nova novo ovo ova ovaj taj ta to nakon prema ' +
|
||||
'danas jucer sutra foto video uzivo pogledajte evo ovo ovako sve svi vijesti hrvatska hrvatski hrvatske ' +
|
||||
// Dates are shared by unrelated stories of the same day.
|
||||
'monday tuesday wednesday thursday friday saturday sunday january february march april may june july august ' +
|
||||
'september october november december ponedjeljak utorak srijeda srijedu cetvrtak petak subota subotu nedjelja nedjelju ' +
|
||||
'mr mrs ms dr'
|
||||
).split(/\s+/),
|
||||
);
|
||||
|
||||
function fold(s: string): string {
|
||||
return s.toLowerCase().normalize('NFKD').replace(/\p{M}+/gu, '').replace(/đ/g, 'd');
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap "could these be the same event?" signal that works across Croatian and English: names
|
||||
* (capitalised words, cut to 4 letters so "Trump"/"Trumpov" and "Bahrain"/"Bahrein" agree) and
|
||||
* numbers. A story that shares none of them with the new article is not worth asking the model about.
|
||||
*/
|
||||
export function eventStems(text: string): Set<string> {
|
||||
const out = new Set<string>();
|
||||
for (const m of text.matchAll(/(?<![\p{L}\p{N}])(\p{Lu}[\p{L}\p{M}'’-]{2,})/gu)) {
|
||||
const w = fold(m[1]!).replace(/['’-].*$/, '');
|
||||
if (w.length < 3 || COMMON.has(w)) continue;
|
||||
out.add(w.slice(0, 4));
|
||||
}
|
||||
for (const m of text.matchAll(/(?<![\p{L}\p{N}])\d[\d.,]*\d(?![\p{L}\p{N}])/gu)) {
|
||||
const n = m[0].replace(/[.,]/g, '');
|
||||
if (/^(19|20)\d\d$/.test(n)) continue; // years match everything
|
||||
out.add(`#${n}`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Grouping without embeddings: the triage model picks the matching recent story, if any. */
|
||||
async function clusterByLlm(ctx: Ctx, a: ArticleRow): Promise<void> {
|
||||
@@ -170,13 +211,38 @@ async function clusterByLlm(ctx: Ctx, a: ArticleRow): Promise<void> {
|
||||
WHERE s.first_reported_at >= ? AND s.hidden = 0 AND s.state != 'failed'
|
||||
ORDER BY s.last_reported_at DESC LIMIT ?`,
|
||||
)
|
||||
.all<{ id: number; title: string | null; lead_title: string | null }>(since, MAX_LLM_CANDIDATES);
|
||||
if (!stories.length) {
|
||||
.all<{ id: number; title: string | null; lead_title: string | null }>(since, SCAN_STORIES);
|
||||
|
||||
// Free pre-filter: only stories that share a name or number with the new article are candidates.
|
||||
const mine = eventStems(`${a.title}\n${articleText(a).slice(0, 600)}`);
|
||||
const memberTitles = new Map<number, string[]>();
|
||||
if (stories.length) {
|
||||
const ids = stories.map((x) => x.id);
|
||||
for (const r of ctx.db
|
||||
.prepare(`SELECT story_id, title FROM article WHERE story_id IN (${ids.map(() => '?').join(',')})`)
|
||||
.all<{ story_id: number; title: string }>(...ids)) {
|
||||
const list = memberTitles.get(r.story_id) ?? [];
|
||||
list.push(r.title);
|
||||
memberTitles.set(r.story_id, list);
|
||||
}
|
||||
}
|
||||
const scored = stories
|
||||
.map((x) => {
|
||||
const theirs = eventStems([x.title ?? '', ...(memberTitles.get(x.id) ?? [])].join('\n'));
|
||||
let shared = 0;
|
||||
for (const st of mine) if (theirs.has(st)) shared++;
|
||||
return { story: x, shared };
|
||||
})
|
||||
.filter((x) => x.shared > 0)
|
||||
.sort((p, q) => q.shared - p.shared)
|
||||
.slice(0, MAX_LLM_CANDIDATES);
|
||||
if (!scored.length) {
|
||||
if (stories.length) bumpStat(ctx.db, 'match_calls_skipped');
|
||||
createStoryForArticle(ctx, a);
|
||||
ctx.wake();
|
||||
return;
|
||||
}
|
||||
const candidates = stories.map((x) => ({ id: x.id, title: x.title ?? x.lead_title ?? '', originalTitle: x.lead_title }));
|
||||
const candidates = scored.map(({ story: x }) => ({ id: x.id, title: x.title ?? x.lead_title ?? '', originalTitle: x.lead_title }));
|
||||
const prompt = buildStoryMatchPrompt(
|
||||
promptText(s, 'sameEvent'),
|
||||
{ sourceName: sourceName(ctx, a.source_id), title: a.title, publishedAt: a.published_at, categories: parseJsonArray(a.categories), text: truncate(articleText(a), 800) },
|
||||
|
||||
@@ -7,6 +7,7 @@ import { normalizeLang, normalizeTitle, sha1 } from '../lib/text.ts';
|
||||
import { HOUR, MINUTE, isoMinus, isoPlus, nowIso } from '../lib/time.ts';
|
||||
import { log, errorMessage } from '../lib/log.ts';
|
||||
import { setMeta } from '../db/schema.ts';
|
||||
import { attachArticle } from './stories.ts';
|
||||
|
||||
export interface SourceRow {
|
||||
id: number;
|
||||
@@ -80,6 +81,19 @@ export async function fetchSource(ctx: Ctx, sourceId: number): Promise<{ newItem
|
||||
}
|
||||
}
|
||||
|
||||
interface Twin {
|
||||
id: number;
|
||||
source_id: number;
|
||||
state: string;
|
||||
story_id: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores new feed items. Everything here is free (no AI): an item is dropped when its URL was
|
||||
* ever seen before, when its source already delivered the same GUID or headline, and an
|
||||
* identical headline from another outlet (wire copy) joins the existing story, or is filtered
|
||||
* the same way, without triage, grouping or a summary.
|
||||
*/
|
||||
function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType<typeof parseFeed>['items'], feedLang: string | null): number {
|
||||
const s = ctx.settings.get().pipeline;
|
||||
const now = new Date();
|
||||
@@ -87,31 +101,60 @@ function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType<typeof parseFeed
|
||||
const maxAgeCutoff = isoMinus(s.maxAgeHours * HOUR, now);
|
||||
const firstAddCutoff = isoMinus(s.firstAddHours * HOUR, now);
|
||||
const titleWindow = isoMinus(48 * HOUR, now);
|
||||
const seen = ctx.db.prepare(`SELECT 1 FROM seen WHERE url_hash = ?`);
|
||||
const markSeen = ctx.db.prepare(`INSERT OR IGNORE INTO seen (url_hash, first_seen) VALUES (?, ?)`);
|
||||
const byUrl = ctx.db.prepare(`SELECT 1 FROM article WHERE url_hash = ?`);
|
||||
const byGuid = ctx.db.prepare(`SELECT 1 FROM article WHERE source_id = ? AND guid = ?`);
|
||||
const byTitle = ctx.db.prepare(`SELECT 1 FROM article WHERE title_hash = ? AND fetched_at >= ?`);
|
||||
const byTitle = ctx.db.prepare(
|
||||
`SELECT id, source_id, state, story_id FROM article WHERE title_hash = ? AND fetched_at >= ? ORDER BY id LIMIT 1`,
|
||||
);
|
||||
const insert = ctx.db.prepare(
|
||||
`INSERT INTO article (source_id, guid, url, url_hash, title_hash, title, excerpt, categories, text, partial, author,
|
||||
image_url, language, published_at, fetched_at, state, state_changed_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
);
|
||||
const copyVerdict = ctx.db.prepare(
|
||||
`INSERT OR REPLACE INTO verdict (article_id, decision, decided_by, rule_id, rule_text, reason, confidence, model, prompt_version, overridden, decided_at)
|
||||
SELECT ?, decision, decided_by, rule_id, rule_text, 'Same headline as an article already filtered. ' || reason, confidence, model, prompt_version, 0, ?
|
||||
FROM verdict WHERE article_id = ?`,
|
||||
);
|
||||
let fresh = 0;
|
||||
let duplicates = 0;
|
||||
const attachLater: { storyId: number; article: { id: number; published_at: string | null; fetched_at: string } }[] = [];
|
||||
|
||||
ctx.db.transaction(() => {
|
||||
for (const item of items) {
|
||||
const url = item.link ? canonicalizeUrl(item.link) : null;
|
||||
if (!url) continue;
|
||||
const urlHash = sha1(url);
|
||||
if (byUrl.get(urlHash)) continue;
|
||||
if (item.guid && byGuid.get(src.id, item.guid)) continue;
|
||||
if (seen.get(urlHash) || byUrl.get(urlHash)) continue;
|
||||
markSeen.run(urlHash, fetchedAt);
|
||||
if (item.guid && byGuid.get(src.id, item.guid)) {
|
||||
duplicates++;
|
||||
continue;
|
||||
}
|
||||
const normTitle = normalizeTitle(item.title);
|
||||
const titleHash = sha1(normTitle);
|
||||
if (normTitle.length > 12 && byTitle.get(titleHash, titleWindow)) continue;
|
||||
// Feeds sometimes publish future dates; never trust a date later than now.
|
||||
const published = item.publishedAt && item.publishedAt.getTime() <= now.getTime() ? item.publishedAt.toISOString() : fetchedAt;
|
||||
const tooOld = published < maxAgeCutoff || (!src.initial_import_done && published < firstAddCutoff);
|
||||
const fullText = src.full_text === 1;
|
||||
const state = tooOld ? 'skipped' : fullText ? 'new' : 'extracted';
|
||||
insert.run(
|
||||
|
||||
let state = tooOld ? 'skipped' : fullText ? 'new' : 'extracted';
|
||||
let twin: Twin | undefined;
|
||||
if (!tooOld && normTitle.length > 12) {
|
||||
twin = byTitle.get<Twin>(titleHash, titleWindow);
|
||||
if (twin) {
|
||||
if (twin.source_id === src.id || (twin.state !== 'grouped' && twin.state !== 'rejected') || (twin.state === 'grouped' && !twin.story_id)) {
|
||||
// Same outlet under a new URL, or the twin is still being processed: nothing to add.
|
||||
duplicates++;
|
||||
continue;
|
||||
}
|
||||
// Identical headline from another outlet: reuse the twin's outcome instead of the AI.
|
||||
state = twin.state === 'rejected' ? 'rejected' : 'kept';
|
||||
}
|
||||
}
|
||||
const id = insert.run(
|
||||
src.id,
|
||||
item.guid,
|
||||
url,
|
||||
@@ -120,8 +163,8 @@ function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType<typeof parseFeed
|
||||
item.title.slice(0, 1000),
|
||||
item.excerpt.slice(0, 20_000) || null,
|
||||
JSON.stringify(item.categories),
|
||||
fullText ? null : item.excerpt.slice(0, 50_000) || null,
|
||||
fullText ? 0 : 1,
|
||||
fullText && !twin ? null : item.excerpt.slice(0, 50_000) || null,
|
||||
fullText && !twin ? 0 : 1,
|
||||
item.author?.slice(0, 300) ?? null,
|
||||
item.imageUrl,
|
||||
feedLang,
|
||||
@@ -129,10 +172,19 @@ function storeItems(ctx: Ctx, src: SourceRow, items: ReturnType<typeof parseFeed
|
||||
fetchedAt,
|
||||
state,
|
||||
fetchedAt,
|
||||
);
|
||||
).lastInsertRowid;
|
||||
if (twin?.state === 'rejected') {
|
||||
copyVerdict.run(id, fetchedAt, twin.id);
|
||||
} else if (twin?.story_id) {
|
||||
attachLater.push({ storyId: twin.story_id, article: { id, published_at: published, fetched_at: fetchedAt } });
|
||||
}
|
||||
if (!tooOld) fresh++;
|
||||
}
|
||||
});
|
||||
// Outside the insert transaction: attaching recomputes the story and may queue a rewrite.
|
||||
for (const a of attachLater) attachArticle(ctx, a.storyId, a.article, { rewrite: false });
|
||||
if (duplicates) bumpStat(ctx.db, 'duplicates_skipped', duplicates);
|
||||
if (attachLater.length) bumpStat(ctx.db, 'grouped_by_headline', attachLater.length);
|
||||
return fresh;
|
||||
}
|
||||
|
||||
|
||||
@@ -27,6 +27,8 @@ export function runRetention(ctx: Ctx): void {
|
||||
.prepare(`UPDATE article SET text = NULL WHERE text IS NOT NULL AND fetched_at < ? AND state NOT IN ('new','extracted','kept')`)
|
||||
.run(textCutoff).changes;
|
||||
ctx.db.prepare(`DELETE FROM llm_call WHERE at < ?`).run(isoMinus(8 * DAY));
|
||||
// The seen-URL ledger outlives the articles, so a pruned item still in a feed isn't reprocessed.
|
||||
ctx.db.prepare(`DELETE FROM seen WHERE first_seen < ?`).run(isoMinus(Math.max(180, r.publishedDays * 2) * DAY));
|
||||
ctx.db.prepare(`DELETE FROM stat WHERE day < ?`).run(isoMinus(60 * DAY).slice(0, 10));
|
||||
ctx.db.prepare(`DELETE FROM edition WHERE scheduled_for < ?`).run(isoMinus(60 * DAY));
|
||||
ctx.db.prepare(`DELETE FROM digest WHERE created_at < ?`).run(publishedCutoff);
|
||||
|
||||
@@ -257,8 +257,16 @@ export function createStoryForArticle(ctx: Ctx, a: Pick<ArticleRow, 'id' | 'publ
|
||||
});
|
||||
}
|
||||
|
||||
/** Adds an article to an existing story and triggers regeneration / feed re-evaluation as needed. */
|
||||
export function attachArticle(ctx: Ctx, storyId: number, a: Pick<ArticleRow, 'id' | 'published_at' | 'fetched_at'>): void {
|
||||
/**
|
||||
* Adds an article to an existing story and triggers regeneration / feed re-evaluation as needed.
|
||||
* `rewrite: false` for wire copy: the same text adds a source but nothing worth a new summary.
|
||||
*/
|
||||
export function attachArticle(
|
||||
ctx: Ctx,
|
||||
storyId: number,
|
||||
a: Pick<ArticleRow, 'id' | 'published_at' | 'fetched_at'>,
|
||||
opts: { rewrite?: boolean } = {},
|
||||
): void {
|
||||
const now = nowIso();
|
||||
ctx.db.transaction(() => {
|
||||
ctx.db.prepare(`UPDATE article SET story_id = ?, state = 'grouped', state_changed_at = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ?`).run(storyId, now, a.id);
|
||||
@@ -268,7 +276,7 @@ export function attachArticle(ctx: Ctx, storyId: number, a: Pick<ArticleRow, 'id
|
||||
const c = ctx.settings.get().clustering;
|
||||
if (s.state === 'failed') {
|
||||
ctx.db.prepare(`UPDATE story SET state = 'pending', attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ?`).run(storyId);
|
||||
} else if (s.state === 'published' && s.regen_count < c.regenCap) {
|
||||
} else if (s.state === 'published' && s.regen_count < c.regenCap && opts.rewrite !== false) {
|
||||
const crossed = r.prevSourceCount < c.regenAtSources && r.sourceCount >= c.regenAtSources;
|
||||
const written = s.version_written_at ? Date.parse(s.version_written_at) : Date.now();
|
||||
const gap = c.regenGapHours > 0 && Date.parse(reportedAt(a)) - written > c.regenGapHours * HOUR;
|
||||
|
||||
@@ -124,7 +124,8 @@ export async function summarizeStory(ctx: Ctx, storyId: number): Promise<void> {
|
||||
const lead = all.find((a) => a.id === story.lead_article_id) ?? all[0]!;
|
||||
const others = all.filter((a) => a.id !== lead.id).sort((x, y) => x.partial - y.partial);
|
||||
const chosen = [lead, ...others].slice(0, 4);
|
||||
const perArticle = chosen.length > 1 ? Math.max(2000, Math.floor(settings.llm.maxArticleChars / chosen.length)) : settings.llm.maxArticleChars;
|
||||
// A rewrite from several sources shares one article-sized budget instead of multiplying it.
|
||||
const perArticle = chosen.length > 1 ? Math.max(1500, Math.floor(settings.llm.maxArticleChars / chosen.length)) : settings.llm.maxArticleChars;
|
||||
const written = await writeTitleAndDescription(ctx.llm, settings, chosen.map((a) => articleForPrompt(ctx, a, perArticle)));
|
||||
if (written.formatRetries) bumpStat(ctx.db, 'format_retries', written.formatRetries);
|
||||
if (!written.languageOk) {
|
||||
|
||||
+146
-54
@@ -3,14 +3,19 @@ import type { DryRunHit, DryRunStatus, RuleDTO, RuleInput } from '../../../share
|
||||
import type { Ctx } from '../context.ts';
|
||||
import {
|
||||
buildDryRunPrompt,
|
||||
buildTriageBatchPrompt,
|
||||
buildTriagePrompt,
|
||||
DRY_RUN_SCHEMA,
|
||||
promptText,
|
||||
promptVersion,
|
||||
TRIAGE_BATCH_SCHEMA,
|
||||
TRIAGE_SCHEMA,
|
||||
validateDryRun,
|
||||
validateTriage,
|
||||
validateTriageBatch,
|
||||
} from '../llm/prompts.ts';
|
||||
import type { TriageOutput, TriageRuleRef } from '../llm/prompts.ts';
|
||||
import { bumpStat } from '../context.ts';
|
||||
import { nowIso } from '../lib/time.ts';
|
||||
import { truncate } from '../lib/text.ts';
|
||||
import { errorMessage } from '../lib/log.ts';
|
||||
@@ -120,13 +125,9 @@ function writeVerdict(ctx: Ctx, articleId: number, v: VerdictInput): void {
|
||||
.run(articleId, v.decision, v.decidedBy, v.ruleId, v.ruleText, v.reason, v.confidence, v.model, v.promptVersion, nowIso());
|
||||
}
|
||||
|
||||
/** Whether triage for this article needs the LLM (any enabled AI negative rule applies). */
|
||||
/** Whether triage for this article needs the LLM (no keyword rule decides it and an AI negative applies). */
|
||||
export function triageNeedsLlm(ctx: Ctx, a: Pick<ArticleRow, 'source_id' | 'title' | 'categories'>): boolean {
|
||||
const rules = activeRules(ctx, a.source_id);
|
||||
const hay = keywordHaystack(a);
|
||||
if (rules.some((r) => r.kind === 'keyword' && r.polarity === 'exception' && keywordMatcher(r.text)(hay))) return false;
|
||||
if (rules.some((r) => r.kind === 'keyword' && r.polarity === 'negative' && keywordMatcher(r.text)(hay))) return false;
|
||||
return rules.some((r) => r.kind === 'ai' && r.polarity === 'negative');
|
||||
return freeVerdict(activeRules(ctx, a.source_id), a) === null;
|
||||
}
|
||||
|
||||
function sourceName(ctx: Ctx, sourceId: number): string {
|
||||
@@ -148,44 +149,85 @@ export interface TriageVerdict extends VerdictInput {
|
||||
ms: number | null;
|
||||
}
|
||||
|
||||
/** Decides keep/reject without touching the database (used by triage and the playground). */
|
||||
export async function evaluateTriage(ctx: Ctx, input: TriageInput): Promise<TriageVerdict> {
|
||||
const rules =
|
||||
input.sourceId === null
|
||||
? listRules(ctx).filter((r) => r.enabled)
|
||||
: activeRules(ctx, input.sourceId);
|
||||
const hay = keywordHaystack(input);
|
||||
const base = { model: null, promptVersion: null, confidence: null, ms: null };
|
||||
/** Characters of article text the triage model sees: headline + lede is enough to judge the subject. */
|
||||
const TRIAGE_TEXT_CHARS = 800;
|
||||
/** Articles from one source judged in one call. */
|
||||
export const TRIAGE_BATCH_SIZE = 8;
|
||||
|
||||
const NO_LLM = { model: null, promptVersion: null, confidence: null, ms: null };
|
||||
|
||||
/** Keyword exceptions, keyword negatives, and "no AI rule applies" — all free. Null = needs the LLM. */
|
||||
function freeVerdict(rules: RuleDTO[], input: Pick<TriageInput, 'title' | 'categories'>): TriageVerdict | null {
|
||||
const hay = keywordHaystack(input);
|
||||
for (const r of rules.filter((x) => x.kind === 'keyword' && x.polarity === 'exception')) {
|
||||
const hit = keywordMatcher(r.text)(hay);
|
||||
if (hit) return { ...base, decision: 'keep', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Kept by exception: matched "${hit}"`, hitRuleId: r.id };
|
||||
if (hit) return { ...NO_LLM, decision: 'keep', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Kept by exception: matched "${hit}"`, hitRuleId: r.id };
|
||||
}
|
||||
for (const r of rules.filter((x) => x.kind === 'keyword' && x.polarity === 'negative')) {
|
||||
const hit = keywordMatcher(r.text)(hay);
|
||||
if (hit) return { ...base, decision: 'reject', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Matched keyword "${hit}"`, hitRuleId: r.id };
|
||||
if (hit) return { ...NO_LLM, decision: 'reject', decidedBy: 'keyword', ruleId: r.id, ruleText: r.text, reason: `Matched keyword "${hit}"`, hitRuleId: r.id };
|
||||
}
|
||||
if (!rules.some((r) => r.kind === 'ai' && r.polarity === 'negative')) {
|
||||
return { ...NO_LLM, decision: 'keep', decidedBy: 'none', ruleId: null, ruleText: null, reason: 'No rule applies', hitRuleId: null };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Turns the model's answer into a verdict: reject only with a valid rule at or above the threshold. */
|
||||
function interpretTriage(
|
||||
out: TriageOutput,
|
||||
rules: RuleDTO[],
|
||||
threshold: number,
|
||||
common: { model: string; promptVersion: string; ms: number | null },
|
||||
): TriageVerdict {
|
||||
const negatives = rules.filter((r) => r.kind === 'ai' && r.polarity === 'negative');
|
||||
const exceptions = rules.filter((r) => r.kind === 'ai' && r.polarity === 'exception');
|
||||
if (negatives.length === 0) {
|
||||
return { ...base, decision: 'keep', decidedBy: 'none', ruleId: null, ruleText: null, reason: 'No rule applies', hitRuleId: null };
|
||||
const negRule = negatives.find((r) => r.id === out.ruleId);
|
||||
const excRule = exceptions.find((r) => r.id === out.ruleId);
|
||||
const base = { ...common, decidedBy: 'ai' as const, confidence: out.confidence };
|
||||
if (out.decision === 'reject' && negRule && (out.confidence ?? 0) >= threshold) {
|
||||
return { ...base, decision: 'reject', ruleId: negRule.id, ruleText: negRule.text, reason: out.reason, hitRuleId: negRule.id };
|
||||
}
|
||||
let reason = out.reason;
|
||||
if (out.decision === 'reject') {
|
||||
reason = negRule
|
||||
? `Kept: confidence ${Math.round((out.confidence ?? 0) * 100)}% is below the ${Math.round(threshold * 100)}% threshold. ${out.reason}`
|
||||
: `Kept: the model did not name a valid rule. ${out.reason}`;
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
decision: 'keep',
|
||||
ruleId: excRule?.id ?? negRule?.id ?? null,
|
||||
ruleText: excRule?.text ?? negRule?.text ?? null,
|
||||
reason,
|
||||
hitRuleId: excRule?.id ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function ruleRefs(rules: RuleDTO[], polarity: 'negative' | 'exception'): TriageRuleRef[] {
|
||||
return rules.filter((r) => r.kind === 'ai' && r.polarity === polarity).map((r) => ({ id: r.id, text: r.text }));
|
||||
}
|
||||
|
||||
/** Decides keep/reject without touching the database (used by triage and the playground). */
|
||||
export async function evaluateTriage(ctx: Ctx, input: TriageInput): Promise<TriageVerdict> {
|
||||
const rules = input.sourceId === null ? listRules(ctx).filter((r) => r.enabled) : activeRules(ctx, input.sourceId);
|
||||
const free = freeVerdict(rules, input);
|
||||
if (free) return free;
|
||||
|
||||
const settings = ctx.settings.get();
|
||||
const model = settings.llm.triageModel || settings.llm.summaryModel;
|
||||
const instructions = promptText(settings, 'triage');
|
||||
const prompt = buildTriagePrompt(
|
||||
instructions,
|
||||
negatives.map((r) => ({ id: r.id, text: r.text })),
|
||||
exceptions.map((r) => ({ id: r.id, text: r.text })),
|
||||
ruleRefs(rules, 'negative'),
|
||||
ruleRefs(rules, 'exception'),
|
||||
filterExamples(ctx.db, settings.pipeline.examplesInPrompt),
|
||||
{
|
||||
sourceName: input.sourceName,
|
||||
title: input.title,
|
||||
publishedAt: input.publishedAt,
|
||||
categories: parseJsonArray<string>(input.categories),
|
||||
text: truncate(input.text, 1500),
|
||||
text: truncate(input.text, TRIAGE_TEXT_CHARS),
|
||||
},
|
||||
);
|
||||
const res = await ctx.llm.chatJson({
|
||||
@@ -198,46 +240,96 @@ export async function evaluateTriage(ctx: Ctx, input: TriageInput): Promise<Tria
|
||||
validate: validateTriage,
|
||||
maxTokens: 1024,
|
||||
});
|
||||
const out = res.value;
|
||||
const pv = promptVersion(instructions);
|
||||
const negRule = negatives.find((r) => r.id === out.ruleId);
|
||||
const excRule = exceptions.find((r) => r.id === out.ruleId);
|
||||
const threshold = settings.pipeline.rejectThreshold;
|
||||
const common = { decidedBy: 'ai' as const, confidence: out.confidence, model, promptVersion: pv, ms: res.ms };
|
||||
if (out.decision === 'reject' && negRule && (out.confidence ?? 0) >= threshold) {
|
||||
return { ...common, decision: 'reject', ruleId: negRule.id, ruleText: negRule.text, reason: out.reason, hitRuleId: negRule.id };
|
||||
}
|
||||
let reason = out.reason;
|
||||
if (out.decision === 'reject') {
|
||||
reason = negRule
|
||||
? `Kept: confidence ${Math.round((out.confidence ?? 0) * 100)}% is below the ${Math.round(threshold * 100)}% threshold. ${out.reason}`
|
||||
: `Kept: the model did not name a valid rule. ${out.reason}`;
|
||||
}
|
||||
return {
|
||||
...common,
|
||||
decision: 'keep',
|
||||
ruleId: excRule?.id ?? negRule?.id ?? null,
|
||||
ruleText: excRule?.text ?? negRule?.text ?? null,
|
||||
reason,
|
||||
hitRuleId: excRule?.id ?? null,
|
||||
};
|
||||
return interpretTriage(res.value, rules, settings.pipeline.rejectThreshold, { model, promptVersion: promptVersion(instructions), ms: res.ms });
|
||||
}
|
||||
|
||||
function inputOf(ctx: Ctx, a: ArticleRow): TriageInput {
|
||||
return { sourceId: a.source_id, sourceName: sourceName(ctx, a.source_id), title: a.title, categories: a.categories, text: articleText(a), publishedAt: a.published_at };
|
||||
}
|
||||
|
||||
function record(ctx: Ctx, articleId: number, v: TriageVerdict): void {
|
||||
if (v.hitRuleId !== null) bumpRule(ctx, v.hitRuleId);
|
||||
writeVerdict(ctx, articleId, v);
|
||||
}
|
||||
|
||||
/** Decides keep/reject for one stored article and writes the verdict. */
|
||||
export async function triageArticle(ctx: Ctx, a: ArticleRow): Promise<'keep' | 'reject'> {
|
||||
const v = await evaluateTriage(ctx, {
|
||||
sourceId: a.source_id,
|
||||
sourceName: sourceName(ctx, a.source_id),
|
||||
title: a.title,
|
||||
categories: a.categories,
|
||||
text: articleText(a),
|
||||
publishedAt: a.published_at,
|
||||
});
|
||||
if (v.hitRuleId !== null) bumpRule(ctx, v.hitRuleId);
|
||||
writeVerdict(ctx, a.id, v);
|
||||
const v = await evaluateTriage(ctx, inputOf(ctx, a));
|
||||
record(ctx, a.id, v);
|
||||
return v.decision;
|
||||
}
|
||||
|
||||
/**
|
||||
* Triage for several articles from one source. Free decisions (keyword rules, no AI rule) are
|
||||
* made first; the rest share one LLM call, so the rules and instructions are sent once.
|
||||
*/
|
||||
export async function triageBatch(ctx: Ctx, articles: ArticleRow[]): Promise<Map<number, 'keep' | 'reject'>> {
|
||||
const out = new Map<number, 'keep' | 'reject'>();
|
||||
if (!articles.length) return out;
|
||||
const rules = activeRules(ctx, articles[0]!.source_id);
|
||||
const needAi: ArticleRow[] = [];
|
||||
for (const a of articles) {
|
||||
const free = freeVerdict(rules, a);
|
||||
if (free) {
|
||||
record(ctx, a.id, free);
|
||||
out.set(a.id, free.decision);
|
||||
} else {
|
||||
needAi.push(a);
|
||||
}
|
||||
}
|
||||
if (needAi.length === 0) return out;
|
||||
if (needAi.length === 1) {
|
||||
out.set(needAi[0]!.id, await triageArticle(ctx, needAi[0]!));
|
||||
return out;
|
||||
}
|
||||
|
||||
const settings = ctx.settings.get();
|
||||
const model = settings.llm.triageModel || settings.llm.summaryModel;
|
||||
const instructions = promptText(settings, 'triage');
|
||||
const name = sourceName(ctx, needAi[0]!.source_id);
|
||||
const prompt = buildTriageBatchPrompt(
|
||||
instructions,
|
||||
ruleRefs(rules, 'negative'),
|
||||
ruleRefs(rules, 'exception'),
|
||||
filterExamples(ctx.db, settings.pipeline.examplesInPrompt),
|
||||
needAi.map((a, i) => ({
|
||||
n: i + 1,
|
||||
sourceName: name,
|
||||
title: a.title,
|
||||
publishedAt: a.published_at,
|
||||
categories: parseJsonArray<string>(a.categories),
|
||||
text: truncate(articleText(a), TRIAGE_TEXT_CHARS),
|
||||
})),
|
||||
);
|
||||
const res = await ctx.llm.chatJson({
|
||||
purpose: 'triage',
|
||||
model,
|
||||
system: prompt.system,
|
||||
user: prompt.user,
|
||||
schemaName: 'triage_batch',
|
||||
schema: TRIAGE_BATCH_SCHEMA,
|
||||
validate: validateTriageBatch,
|
||||
maxTokens: 512 + 160 * needAi.length,
|
||||
});
|
||||
const byN = new Map(res.value.map((r) => [r.n, r]));
|
||||
const common = { model, promptVersion: promptVersion(instructions), ms: res.ms };
|
||||
const missing: ArticleRow[] = [];
|
||||
needAi.forEach((a, i) => {
|
||||
const r = byN.get(i + 1);
|
||||
if (!r) {
|
||||
missing.push(a);
|
||||
return;
|
||||
}
|
||||
const v = interpretTriage(r, rules, settings.pipeline.rejectThreshold, common);
|
||||
record(ctx, a.id, v);
|
||||
out.set(a.id, v.decision);
|
||||
});
|
||||
bumpStat(ctx.db, 'triage_batched', needAi.length - missing.length - 1);
|
||||
// The model skipped some: judge those one by one rather than guessing.
|
||||
for (const a of missing) out.set(a.id, await triageArticle(ctx, a));
|
||||
return out;
|
||||
}
|
||||
|
||||
// ───────────────────────────── rule CRUD ─────────────────────────────
|
||||
|
||||
function normalizeRuleInput(input: Partial<RuleInput>): Partial<RuleInput> {
|
||||
|
||||
@@ -4,7 +4,7 @@ import { log, errorMessage } from '../lib/log.ts';
|
||||
import { LlmUnavailableError } from '../llm/client.ts';
|
||||
import { normalizeLang } from '../lib/text.ts';
|
||||
import { extractArticle } from './extract.ts';
|
||||
import { triageArticle, triageNeedsLlm } from './triage.ts';
|
||||
import { TRIAGE_BATCH_SIZE, triageBatch, triageNeedsLlm } from './triage.ts';
|
||||
import { clusterArticle, clusteringActive } from './cluster.ts';
|
||||
import { PermanentError, summarizeStory } from './summarize.ts';
|
||||
import { nextPendingDigest, processDigest } from './digest.ts';
|
||||
@@ -14,7 +14,15 @@ const EXTRACT_SLOTS = 3;
|
||||
const MAX_ATTEMPTS = 3;
|
||||
const BACKOFF = [1 * MINUTE, 5 * MINUTE, 15 * MINUTE];
|
||||
|
||||
type Task = { key: string; lane: 'extract' | 'llm'; label: string; run: () => Promise<void>; onError: (err: unknown) => void };
|
||||
type Task = {
|
||||
key: string;
|
||||
/** Extra items this task owns (batched triage), claimed alongside `key`. */
|
||||
extraKeys?: string[];
|
||||
lane: 'extract' | 'llm';
|
||||
label: string;
|
||||
run: () => Promise<void>;
|
||||
onError: (err: unknown) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* Background processing. State lives in the database (article.state / story.state), so a
|
||||
@@ -78,7 +86,8 @@ export class Worker {
|
||||
}
|
||||
|
||||
private launch(t: Task): void {
|
||||
this.claimed.add(t.key);
|
||||
const keys = [t.key, ...(t.extraKeys ?? [])];
|
||||
for (const k of keys) this.claimed.add(k);
|
||||
this.active[t.lane]++;
|
||||
void t
|
||||
.run()
|
||||
@@ -90,7 +99,7 @@ export class Worker {
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
this.claimed.delete(t.key);
|
||||
for (const k of keys) this.claimed.delete(k);
|
||||
this.active[t.lane]--;
|
||||
this.wake();
|
||||
});
|
||||
@@ -112,6 +121,10 @@ export class Worker {
|
||||
return (a.published_at ?? a.fetched_at) < isoMinus(maxAge * HOUR);
|
||||
}
|
||||
|
||||
private sourceStillExtracting(sourceId: number): boolean {
|
||||
return !!this.ctx.db.prepare(`SELECT 1 FROM article WHERE source_id = ? AND state = 'new' LIMIT 1`).get(sourceId);
|
||||
}
|
||||
|
||||
private skip(a: ArticleRow): void {
|
||||
this.ctx.db.prepare(`UPDATE article SET state = 'skipped', state_changed_at = ? WHERE id = ?`).run(nowIso(), a.id);
|
||||
}
|
||||
@@ -166,15 +179,37 @@ export class Worker {
|
||||
}
|
||||
}
|
||||
|
||||
for (const a of this.dueArticles('extracted', 50)) {
|
||||
const extracted = this.dueArticles('extracted', 100);
|
||||
for (const a of extracted) {
|
||||
const key = `a${a.id}`;
|
||||
if (this.claimed.has(key)) continue;
|
||||
if (this.tooOld(a)) {
|
||||
this.skip(a);
|
||||
continue;
|
||||
}
|
||||
if (!llmUp && triageNeedsLlm(ctx, a)) continue;
|
||||
return { key, lane: 'llm', label: 'triage', run: () => this.runTriage(a), onError: (err) => this.articleFailed(a, err) };
|
||||
const needsLlm = triageNeedsLlm(ctx, a);
|
||||
if (!llmUp && needsLlm) continue;
|
||||
// Give a fetch's other articles a moment to finish extracting so they can join this call.
|
||||
if (needsLlm && (a.state_changed_at > isoMinus(60_000)) && this.sourceStillExtracting(a.source_id)) continue;
|
||||
// Articles from the same source share the rules, so they can share one triage call.
|
||||
const batch = [a];
|
||||
if (needsLlm) {
|
||||
for (const b of extracted) {
|
||||
if (batch.length >= TRIAGE_BATCH_SIZE) break;
|
||||
if (b.id === a.id || b.source_id !== a.source_id || this.claimed.has(`a${b.id}`) || this.tooOld(b)) continue;
|
||||
if (triageNeedsLlm(ctx, b)) batch.push(b);
|
||||
}
|
||||
}
|
||||
return {
|
||||
key,
|
||||
extraKeys: batch.slice(1).map((b) => `a${b.id}`),
|
||||
lane: 'llm',
|
||||
label: 'triage',
|
||||
run: () => this.runTriage(batch),
|
||||
onError: (err) => {
|
||||
for (const b of batch) this.articleFailed(b, err);
|
||||
},
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -193,11 +228,12 @@ export class Worker {
|
||||
.run(ex.text || null, ex.partial ? 1 : 0, ex.imageUrl, ex.author, language, nowIso(), ex.error, a.id);
|
||||
}
|
||||
|
||||
private async runTriage(a: ArticleRow): Promise<void> {
|
||||
const decision = await triageArticle(this.ctx, a);
|
||||
this.ctx.db
|
||||
.prepare(`UPDATE article SET state = ?, state_changed_at = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ?`)
|
||||
.run(decision === 'reject' ? 'rejected' : 'kept', nowIso(), a.id);
|
||||
private async runTriage(batch: ArticleRow[]): Promise<void> {
|
||||
const decisions = await triageBatch(this.ctx, batch);
|
||||
const update = this.ctx.db.prepare(
|
||||
`UPDATE article SET state = ?, state_changed_at = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ? AND state = 'extracted'`,
|
||||
);
|
||||
for (const [id, decision] of decisions) update.run(decision === 'reject' ? 'rejected' : 'kept', nowIso(), id);
|
||||
}
|
||||
|
||||
// ───────────────────────────── failure handling ─────────────────────────────
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import http from 'node:http';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import type { AddressInfo } from 'node:net';
|
||||
import { loadConfig } from './config.ts';
|
||||
import { startRssFeed } from './server.ts';
|
||||
import type { RunningApp } from './server.ts';
|
||||
import { startMockLlm } from './testing/mock-llm.ts';
|
||||
import type { MockLlm } from './testing/mock-llm.ts';
|
||||
import { eventStems } from './pipeline/cluster.ts';
|
||||
import type { ActivityDTO, StatusDTO, StoryListResponse } from '../../shared/types.ts';
|
||||
|
||||
/** Everything that keeps AI calls down: duplicate ledger, free grouping, pre-filter, batching, pausing on bad config. */
|
||||
|
||||
const BODY = (s: string) => `${s} `.repeat(8);
|
||||
const ARTICLES: Record<string, [string, string]> = {
|
||||
'/a/quake': ['Earthquake hits Turkey, hundreds injured', BODY('A strong earthquake hit eastern Turkey on Sunday, injuring hundreds of people in Malatya.')],
|
||||
'/a/bank': ['Central bank keeps interest rates unchanged', BODY('The European Central Bank kept rates unchanged on Thursday in Frankfurt.')],
|
||||
'/a/celeb': ['Celebrity couple announces divorce after reality show', BODY('The celebrity couple from a reality show announced a divorce; showbiz fans reacted.')],
|
||||
'/a/fish': ['Scientists discover new species of deep sea fish', BODY('Marine biologists found a new fish species in the Pacific Ocean near Hawaii.')],
|
||||
// Same headline as /a/quake from another outlet (wire copy), different URL.
|
||||
'/b/quake': ['Earthquake hits Turkey, hundreds injured', BODY('A strong earthquake hit eastern Turkey on Sunday, injuring hundreds of people in Malatya.')],
|
||||
'/b/match': ['Dinamo wins derby against Hajduk in Zagreb', BODY('Dinamo beat Hajduk 3-1 in Zagreb on Saturday evening.')],
|
||||
};
|
||||
|
||||
interface Harness {
|
||||
app: RunningApp;
|
||||
mock: MockLlm;
|
||||
feeds: Record<string, string[]>;
|
||||
base: string;
|
||||
call<T>(method: string, p: string, body?: unknown): Promise<T>;
|
||||
idle(): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
async function harness(feeds: Record<string, string[]>): Promise<Harness> {
|
||||
let base = '';
|
||||
const fixture = http.createServer((req, res) => {
|
||||
const u = (req.url ?? '').replace(/\?.*$/, '');
|
||||
const feed = feeds[u];
|
||||
if (feed) {
|
||||
return res.writeHead(200, { 'content-type': 'application/rss+xml' }).end(
|
||||
`<?xml version="1.0"?><rss version="2.0"><channel><title>F</title>${feed
|
||||
.map((k, i) => `<item><title>${ARTICLES[k]![0]}</title><link>${base}${k}</link><guid>${k}</guid><pubDate>${new Date(Date.now() - (i + 1) * 60_000).toUTCString()}</pubDate></item>`)
|
||||
.join('')}</channel></rss>`,
|
||||
);
|
||||
}
|
||||
const a = ARTICLES[u];
|
||||
if (a) return res.writeHead(200, { 'content-type': 'text/html' }).end(`<html lang="en"><head><title>${a[0]}</title></head><body><article><h1>${a[0]}</h1><p>${a[1]}</p><p>${a[1]}</p></article></body></html>`);
|
||||
res.writeHead(404).end();
|
||||
});
|
||||
await new Promise<void>((r) => fixture.listen(0, '127.0.0.1', () => r()));
|
||||
base = `http://127.0.0.1:${(fixture.address() as AddressInfo).port}`;
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'rssfeed-sav-'));
|
||||
const mock = await startMockLlm(0, { embeddings: false });
|
||||
const config = { ...loadConfig({ RSSFEED_DATA_DIR: dir, TZ: 'Europe/Zagreb', RSSFEED_ALLOW_PRIVATE_URLS: '1', RSSFEED_WEB_DIST: path.join(dir, 'none') }), port: 0, host: '127.0.0.1' };
|
||||
const app = await startRssFeed(config, { scheduler: false });
|
||||
const call = async <T>(method: string, p: string, body?: unknown): Promise<T> => {
|
||||
const res = await fetch(`http://127.0.0.1:${app.port}${p}`, { method, headers: { 'content-type': 'application/json' }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) });
|
||||
const text = await res.text();
|
||||
assert.ok(res.ok, `${method} ${p} → ${res.status}: ${text}`);
|
||||
return (text ? JSON.parse(text) : null) as T;
|
||||
};
|
||||
const idle = async () => {
|
||||
// Sources added via the API are fetched in the background: wait until every one has been fetched.
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const sources = await call<{ lastFetchAt: string | null }[]>('GET', '/api/sources');
|
||||
if (sources.every((s) => s.lastFetchAt)) break;
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const q = (await call<ActivityDTO>('GET', '/api/activity')).queue;
|
||||
if (q.articlesNew + q.articlesExtracted + q.articlesKept + q.storiesPending + q.storiesUpdating === 0 && !app.worker.busy) return;
|
||||
await new Promise((r) => setTimeout(r, 150));
|
||||
}
|
||||
throw new Error('pipeline did not go idle');
|
||||
};
|
||||
return {
|
||||
app,
|
||||
mock,
|
||||
feeds,
|
||||
base,
|
||||
call,
|
||||
idle,
|
||||
async close() {
|
||||
await app.close();
|
||||
await mock.close();
|
||||
await new Promise<void>((r) => fixture.close(() => r()));
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const chatCalls = (m: MockLlm, schema?: string) => m.calls.filter((c) => c.path.endsWith('/chat/completions') && (!schema || c.schema === schema)).length;
|
||||
|
||||
test('eventStems: names cut to 4 letters, numbers, years and sentence starters ignored', () => {
|
||||
const hr = eventStems('Trumpov plan za golf teren u Washingtonu propao, sud blokirao zatvaranje 15 rupa');
|
||||
const en = eventStems('Judge blocks Trump plan to close Washington golf course with 15 holes in 2026');
|
||||
assert.ok(hr.has('trum') && en.has('trum'));
|
||||
assert.ok(hr.has('wash') && en.has('wash'));
|
||||
assert.ok(hr.has('#15') && en.has('#15'));
|
||||
assert.ok(!en.has('#2026'));
|
||||
assert.ok(!eventStems('The new report says').size);
|
||||
});
|
||||
|
||||
test('AI savings: batched triage, free wire-copy grouping, match pre-filter, seen ledger', { timeout: 90_000 }, async () => {
|
||||
const h = await harness({ '/one.xml': ['/a/quake', '/a/bank', '/a/celeb', '/a/fish'], '/two.xml': [] });
|
||||
try {
|
||||
await h.call('PUT', '/api/settings', { llm: { baseUrl: h.mock.url, triageModel: 'mock-small', summaryModel: 'mock-large', embeddingModel: '' } });
|
||||
await h.call('POST', '/api/rules', { kind: 'ai', polarity: 'negative', text: 'Celebrity gossip and showbiz news' });
|
||||
const one = await h.call<{ id: number }>('POST', '/api/sources', { feedUrl: `${h.base}/one.xml`, name: 'One' });
|
||||
await h.idle();
|
||||
|
||||
// 4 articles from one source → one batched triage call, not four.
|
||||
assert.equal(chatCalls(h.mock, 'triage_batch'), 1, JSON.stringify(h.mock.calls.map((c) => c.schema)));
|
||||
assert.equal(chatCalls(h.mock, 'triage'), 0);
|
||||
// Unrelated articles share no names/numbers with earlier stories → no story-match calls at all.
|
||||
assert.equal(chatCalls(h.mock, 'story_match'), 0);
|
||||
const first = await h.call<StoryListResponse>('GET', '/api/stories?view=all');
|
||||
assert.equal(first.total, 3, 'celebrity article filtered, three stories');
|
||||
|
||||
// Wire copy: identical headline from a second outlet joins the story without any AI call.
|
||||
h.feeds['/two.xml'] = ['/b/quake'];
|
||||
const before = chatCalls(h.mock);
|
||||
const two = await h.call<{ id: number }>('POST', '/api/sources', { feedUrl: `${h.base}/two.xml`, name: 'Two' });
|
||||
await h.idle();
|
||||
const quake = (await h.call<StoryListResponse>('GET', '/api/stories?view=all')).items.find((s) => s.sourceCount === 2);
|
||||
assert.ok(quake, 'wire copy joined the earthquake story');
|
||||
assert.equal(chatCalls(h.mock), before, 'no AI calls for wire copy');
|
||||
|
||||
// A related article (shares "Zagreb"? no: shares nothing) vs one that shares names gets a match call.
|
||||
h.feeds['/two.xml'] = ['/b/quake', '/b/match'];
|
||||
await h.call('POST', `/api/sources/${two.id}/fetch`);
|
||||
await h.idle();
|
||||
assert.equal(chatCalls(h.mock, 'story_match'), 0, 'Dinamo/Hajduk shares nothing with earlier stories');
|
||||
|
||||
// Seen ledger: an article pruned from the database is not processed again if the feed still lists it.
|
||||
h.app.ctx.db.prepare(`DELETE FROM article WHERE url LIKE '%/a/celeb'`).run();
|
||||
const callsBefore = chatCalls(h.mock);
|
||||
await h.call('POST', `/api/sources/${one.id}/fetch`);
|
||||
await h.idle();
|
||||
assert.equal(chatCalls(h.mock), callsBefore, 'pruned article was not re-triaged');
|
||||
assert.equal(h.app.ctx.db.prepare(`SELECT COUNT(*) AS n FROM article WHERE url LIKE '%/a/celeb'`).get<{ n: number }>()!.n, 0);
|
||||
|
||||
const act = await h.call<ActivityDTO>('GET', '/api/activity');
|
||||
assert.equal(act.savings7d.groupedByHeadline, 1);
|
||||
assert.ok(act.savings7d.triageBatched >= 2);
|
||||
assert.ok(act.savings7d.matchCallsSkipped >= 3);
|
||||
assert.ok(act.llm.byPurpose.some((p) => p.purpose === 'triage'));
|
||||
} finally {
|
||||
await h.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('a misconfigured model pauses LLM work instead of failing every article', { timeout: 60_000 }, async () => {
|
||||
const h = await harness({ '/one.xml': ['/a/quake', '/a/bank'] });
|
||||
try {
|
||||
await h.call('PUT', '/api/settings', { llm: { baseUrl: h.mock.url, triageModel: 'mock-small', summaryModel: 'no-such-model', embeddingModel: '' } });
|
||||
await h.call('POST', '/api/sources', { feedUrl: `${h.base}/one.xml`, name: 'One' });
|
||||
for (let i = 0; i < 100; i++) {
|
||||
if ((await h.call<StatusDTO>('GET', '/api/status')).llmError) break;
|
||||
await new Promise((r) => setTimeout(r, 150));
|
||||
}
|
||||
const st = await h.call<StatusDTO>('GET', '/api/status');
|
||||
assert.equal(st.llm, 'offline');
|
||||
assert.match(st.llmError ?? '', /no-such-model/);
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
// One rejected request, not one per article per output format.
|
||||
assert.equal(h.mock.calls.filter((c) => c.model === 'no-such-model').length, 1);
|
||||
const act = await h.call<ActivityDTO>('GET', '/api/activity');
|
||||
assert.equal(act.failures.length, 0, 'articles wait instead of failing');
|
||||
|
||||
// Fixing the setting resumes work.
|
||||
await h.call('PUT', '/api/settings', { llm: { summaryModel: 'mock-large' } });
|
||||
await h.idle();
|
||||
assert.equal((await h.call<StoryListResponse>('GET', '/api/stories?view=all')).total, 2);
|
||||
} finally {
|
||||
await h.close();
|
||||
}
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { CATEGORIES, PROMPT_KEYS } from '../../shared/types.ts';
|
||||
import { CATEGORIES, PROMPT_KEYS, REASONING_EFFORTS } from '../../shared/types.ts';
|
||||
import type { Category, CategoryControl, DeepPartial, FeedControls, Settings } from '../../shared/types.ts';
|
||||
import type { Config } from './config.ts';
|
||||
import type { Database } from './db/database.ts';
|
||||
@@ -40,8 +40,10 @@ export function defaultSettings(): Settings {
|
||||
embeddingModel: '',
|
||||
concurrency: 1,
|
||||
timeoutSec: 120,
|
||||
maxArticleChars: 12_000,
|
||||
// Two sentences need the lede and the key paragraphs, not the whole article.
|
||||
maxArticleChars: 6_000,
|
||||
temperature: 0.2,
|
||||
reasoningEffort: '',
|
||||
},
|
||||
pipeline: {
|
||||
defaultPollMinutes: 15,
|
||||
@@ -115,6 +117,7 @@ export function sanitizeSettings(input: unknown): Settings {
|
||||
timeoutSec: clampInt(llm.timeoutSec, 10, 900, d.llm.timeoutSec),
|
||||
maxArticleChars: clampInt(llm.maxArticleChars, 1000, 200_000, d.llm.maxArticleChars),
|
||||
temperature: clamp(llm.temperature, 0, 2, d.llm.temperature),
|
||||
reasoningEffort: oneOf(llm.reasoningEffort, REASONING_EFFORTS, ''),
|
||||
},
|
||||
pipeline: {
|
||||
defaultPollMinutes: clampInt(s.pipeline.defaultPollMinutes, 1, 1440, d.pipeline.defaultPollMinutes),
|
||||
|
||||
@@ -22,6 +22,7 @@ export interface MockLlm {
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
const KNOWN_MODELS = ['mock-small', 'mock-large', 'mock-embed'];
|
||||
const WORD = /[\p{L}\p{N}]+/gu;
|
||||
const STOP = new Set(['the', 'a', 'an', 'of', 'in', 'on', 'to', 'and', 'for', 'is', 'at', 'by', 'with', 'from', 'je', 'i', 'u', 'na', 'za', 'se', 'od', 'da']);
|
||||
|
||||
@@ -66,6 +67,17 @@ function answer(schema: string | null, system: string, user: string): unknown {
|
||||
? { decision: 'reject', rule_id: ruleId, reason: 'The article is celebrity gossip.', confidence: 0.93 }
|
||||
: { decision: 'keep', rule_id: null, reason: 'No rule matches the article.', confidence: 0.9 };
|
||||
}
|
||||
case 'triage_batch': {
|
||||
const ruleId = Number(/\[(\d+)\]/.exec(system.split('Negative rules')[1] ?? '')?.[1] ?? 0) || null;
|
||||
return {
|
||||
results: [...user.matchAll(/<article_(\d+)>([\s\S]*?)<\/article_\d+>/g)].map((b) => {
|
||||
const celeb = /celebrit|showbiz|slavn|influencer|reality/i.test(b[2] ?? '');
|
||||
return celeb
|
||||
? { n: Number(b[1]), decision: 'reject', rule_id: ruleId, reason: 'The article is celebrity gossip.', confidence: 0.93 }
|
||||
: { n: Number(b[1]), decision: 'keep', rule_id: null, reason: 'No rule matches the article.', confidence: 0.9 };
|
||||
}),
|
||||
};
|
||||
}
|
||||
case 'same_event': {
|
||||
const [a, b] = user.split('<article_b>');
|
||||
const sim = jaccard(headline(a ?? ''), headline(b ?? ''));
|
||||
@@ -130,7 +142,7 @@ export async function startMockLlm(port = 0, opts: MockOptions = {}): Promise<Mo
|
||||
if (down) return send(503, { error: 'mock outage' });
|
||||
const path = (req.url ?? '').replace(/\?.*$/, '');
|
||||
if (req.method === 'GET' && path.endsWith('/models')) {
|
||||
return send(200, { object: 'list', data: ['mock-small', 'mock-large', 'mock-embed'].map((id) => ({ id, object: 'model' })) });
|
||||
return send(200, { object: 'list', data: KNOWN_MODELS.map((id) => ({ id, object: 'model' })) });
|
||||
}
|
||||
let body: Record<string, unknown> = {};
|
||||
try {
|
||||
@@ -153,9 +165,14 @@ export async function startMockLlm(port = 0, opts: MockOptions = {}): Promise<Mo
|
||||
const msgs = (body.messages ?? []) as { role: string; content: string }[];
|
||||
const system = msgs.find((m) => m.role === 'system')?.content ?? '';
|
||||
const user = msgs.find((m) => m.role === 'user')?.content ?? '';
|
||||
if (!KNOWN_MODELS.includes(String(body.model))) {
|
||||
calls.push({ path, schema: null, model: String(body.model) });
|
||||
return send(404, { error: { message: `model '${String(body.model)}' not found` } });
|
||||
}
|
||||
let schema = rf?.json_schema?.name ?? null;
|
||||
if (!schema) {
|
||||
if (/"decision"/.test(system)) schema = 'triage';
|
||||
if (/"results"/.test(system) && /"decision"/.test(system)) schema = 'triage_batch';
|
||||
else if (/"decision"/.test(system)) schema = 'triage';
|
||||
else if (/same_event/.test(system)) schema = 'same_event';
|
||||
else if (/title_hr/.test(system)) schema = 'story_summary';
|
||||
else if (/"opening"/.test(system)) schema = 'digest_opening';
|
||||
|
||||
@@ -357,6 +357,9 @@ export interface DigestDTO {
|
||||
|
||||
// ───────────────────────────── Settings ─────────────────────────────
|
||||
|
||||
export const REASONING_EFFORTS = ['', 'minimal', 'low', 'medium', 'high'] as const;
|
||||
export type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
|
||||
|
||||
export interface LlmSettings {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
@@ -367,6 +370,8 @@ export interface LlmSettings {
|
||||
timeoutSec: number;
|
||||
maxArticleChars: number;
|
||||
temperature: number;
|
||||
/** Sent as `reasoning_effort`; '' = don't send (server default). Lower = fewer hidden thinking tokens. */
|
||||
reasoningEffort: ReasoningEffort;
|
||||
}
|
||||
|
||||
export interface PipelineSettings {
|
||||
@@ -535,6 +540,9 @@ export interface LlmTestResult {
|
||||
export interface PlaygroundSummary {
|
||||
model: string;
|
||||
ok: boolean;
|
||||
/** Tokens billed for this summary (including any retry and hidden reasoning). */
|
||||
tokensIn: number | null;
|
||||
tokensOut: number | null;
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
category: Category | null;
|
||||
@@ -601,7 +609,15 @@ export interface ActivityDTO {
|
||||
p95LatencyMs: number | null;
|
||||
tokensIn24h: number;
|
||||
tokensOut24h: number;
|
||||
/** Why LLM work is paused, when the server rejected requests (bad key, unknown model). */
|
||||
error: string | null;
|
||||
/** Last 24 h, per pipeline step (triage, story-match, summarize, …). */
|
||||
byPurpose: { purpose: string; calls: number; errors: number; tokensIn: number; tokensOut: number }[];
|
||||
/** LLM calls in the last 24 h per story published in the last 24 h. */
|
||||
callsPerStory: number | null;
|
||||
};
|
||||
/** Work saved without AI in the last 7 days. */
|
||||
savings7d: { duplicatesSkipped: number; groupedByHeadline: number; matchCallsSkipped: number; triageBatched: number };
|
||||
failures: ActivityFailure[];
|
||||
importance7d: { score: number; count: number }[];
|
||||
grouping7d: { articles: number; inMultiSourceStories: number; share: number };
|
||||
@@ -621,6 +637,8 @@ export interface ActivityDTO {
|
||||
*/
|
||||
export interface StatusDTO {
|
||||
llm: LlmState;
|
||||
/** Set when LLM work is paused because the server rejected requests (bad key, unknown model). */
|
||||
llmError: string | null;
|
||||
/** Articles and stories waiting on the LLM. */
|
||||
waiting: number;
|
||||
version: string;
|
||||
|
||||
@@ -31,6 +31,19 @@ function StatusBanner() {
|
||||
);
|
||||
}
|
||||
if (!status) return null;
|
||||
if (status.llm === 'offline' && status.llmError) {
|
||||
return (
|
||||
<div className="banner banner-danger" role="status">
|
||||
<div className="banner__inner">
|
||||
<strong>LLM paused</strong>
|
||||
<span>
|
||||
{status.llmError} {status.waiting > 0 ? `${plural(status.waiting, 'item')} waiting; nothing is lost.` : ''}
|
||||
</span>
|
||||
<Link to="/settings/llm">Fix it</Link>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (status.llm === 'offline') {
|
||||
return (
|
||||
<div className="banner banner-warn" role="status">
|
||||
|
||||
@@ -32,6 +32,20 @@ function LlmStateBadge({ state }: { state: ActivityDTO['llm']['state'] }) {
|
||||
return <Badge tone="warn">Not configured</Badge>;
|
||||
}
|
||||
|
||||
const PURPOSE_LABELS: Record<string, string> = {
|
||||
triage: 'Triage (negatives)',
|
||||
'story-match': 'Grouping (triage model)',
|
||||
'same-event': 'Grouping check',
|
||||
cluster: 'Grouping (embeddings)',
|
||||
summarize: 'Title and description',
|
||||
regenerate: 'Rewrite from several sources',
|
||||
digest: 'Digest opening',
|
||||
'dry-run': 'Rule dry runs',
|
||||
playground: 'Playground',
|
||||
test: 'Connection tests',
|
||||
embed: 'Embeddings',
|
||||
};
|
||||
|
||||
function ActivityBody({ data, onChanged }: { data: ActivityDTO; onChanged: () => void }) {
|
||||
const [busy, setBusy] = useState<string | null>(null);
|
||||
const q = data.queue;
|
||||
@@ -93,18 +107,61 @@ function ActivityBody({ data, onChanged }: { data: ActivityDTO; onChanged: () =>
|
||||
Connect a server in <Link to="/settings/llm">Settings → LLM connection</Link>.
|
||||
</p>
|
||||
)}
|
||||
{l.state === 'offline' && (
|
||||
<p className="small">
|
||||
{plural(l.consecutiveFailures, 'failure')} in a row. {l.nextProbeAt ? `Next check ${relTime(l.nextProbeAt)}.` : ''} Nothing is lost while
|
||||
it's offline.
|
||||
</p>
|
||||
)}
|
||||
{l.state === 'offline' &&
|
||||
(l.error ? (
|
||||
<p className="small danger">
|
||||
{l.error} Work is paused and nothing is lost; it resumes when you change the LLM settings, or by itself within 10 minutes.
|
||||
</p>
|
||||
) : (
|
||||
<p className="small">
|
||||
{plural(l.consecutiveFailures, 'failure')} in a row. {l.nextProbeAt ? `Next check ${relTime(l.nextProbeAt)}.` : ''} Nothing is lost while
|
||||
it's offline.
|
||||
</p>
|
||||
))}
|
||||
<div className="stats">
|
||||
<Stat label="Calls, 24 h" value={fmtNumber(l.calls24h)} hint={l.errors24h ? `${l.errors24h} errors` : 'no errors'} />
|
||||
<Stat label="Calls per story" value={l.callsPerStory === null ? '—' : String(l.callsPerStory)} hint="pipeline calls per published story" />
|
||||
<Stat label="Average latency" value={fmtMs(l.avgLatencyMs)} hint={`p95 ${fmtMs(l.p95LatencyMs)}`} />
|
||||
<Stat label="In flight" value={fmtNumber(l.inFlight)} />
|
||||
<Stat label="Tokens in, 24 h" value={fmtNumber(l.tokensIn24h)} />
|
||||
<Stat label="Tokens out, 24 h" value={fmtNumber(l.tokensOut24h)} />
|
||||
<Stat label="Tokens out, 24 h" value={fmtNumber(l.tokensOut24h)} hint="includes hidden reasoning" />
|
||||
</div>
|
||||
{l.byPurpose.length > 0 && (
|
||||
<div className="table-wrap">
|
||||
<table className="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Step, last 24 h</th>
|
||||
<th>Calls</th>
|
||||
<th>Errors</th>
|
||||
<th>Tokens in</th>
|
||||
<th>Tokens out</th>
|
||||
<th>Out per call</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{l.byPurpose.map((p) => (
|
||||
<tr key={p.purpose}>
|
||||
<td>{PURPOSE_LABELS[p.purpose] ?? p.purpose}</td>
|
||||
<td>{fmtNumber(p.calls)}</td>
|
||||
<td className={p.errors ? 'danger' : 'faint'}>{fmtNumber(p.errors)}</td>
|
||||
<td>{fmtNumber(p.tokensIn)}</td>
|
||||
<td>{fmtNumber(p.tokensOut)}</td>
|
||||
<td className="faint">{p.calls - p.errors > 0 ? fmtNumber(Math.round(p.tokensOut / Math.max(1, p.calls - p.errors))) : '—'}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<section className="panel">
|
||||
<h2 className="panel__title">Saved without AI, last 7 days</h2>
|
||||
<div className="stats">
|
||||
<Stat label="Duplicates dropped" value={fmtNumber(data.savings7d.duplicatesSkipped)} hint="same URL, item or headline" />
|
||||
<Stat label="Wire copy grouped" value={fmtNumber(data.savings7d.groupedByHeadline)} hint="identical headline, joined for free" />
|
||||
<Stat label="Grouping calls skipped" value={fmtNumber(data.savings7d.matchCallsSkipped)} hint="no shared names with recent stories" />
|
||||
<Stat label="Triage calls saved" value={fmtNumber(data.savings7d.triageBatched)} hint="articles judged together" />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -166,6 +166,11 @@ function SummaryCard({ s }: { s: PlaygroundSummary }) {
|
||||
<Badge tone={s.languageOk ? 'ok' : 'danger'}>{s.languageOk ? 'Croatian OK' : 'Language check failed'}</Badge>
|
||||
{s.formatRetries > 0 && <span> · {s.formatRetries} format retries</span>}
|
||||
</p>
|
||||
{(s.tokensIn !== null || s.tokensOut !== null) && (
|
||||
<p className="faint small">
|
||||
{s.tokensIn ?? '—'} tokens in · {s.tokensOut ?? '—'} tokens out (including hidden reasoning)
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useState, type ReactNode } from 'react';
|
||||
import { api } from '../../api.ts';
|
||||
import type { LlmSettings, LlmTestResult } from '../../contract.ts';
|
||||
import type { LlmSettings, LlmTestResult, ReasoningEffort } from '../../contract.ts';
|
||||
import { useDraft } from '../../hooks.ts';
|
||||
import { Badge, ErrorNote, Field, NumberField, Spinner, TextField } from '../../components.tsx';
|
||||
import { errorMessage, fmtMs } from '../../util.ts';
|
||||
@@ -122,7 +122,7 @@ export function LlmSection({ data, onSaved }: SectionProps) {
|
||||
<NumberField label="Timeout" suffix="s" min={10} max={900} value={draft.timeoutSec} onChange={(v) => set('timeoutSec', Math.round(v))} />
|
||||
<NumberField
|
||||
label="Max article characters"
|
||||
hint="Longer text is truncated to fit the context window."
|
||||
hint="Text sent for the title and description. 6000 covers the lede and key paragraphs; more mostly costs tokens."
|
||||
min={1000}
|
||||
max={200000}
|
||||
step={500}
|
||||
@@ -130,6 +130,18 @@ export function LlmSection({ data, onSaved }: SectionProps) {
|
||||
onChange={(v) => set('maxArticleChars', Math.round(v))}
|
||||
/>
|
||||
<NumberField label="Temperature" hint="Low keeps summaries factual." min={0} max={2} step={0.05} value={draft.temperature} onChange={(v) => set('temperature', v)} />
|
||||
<Field
|
||||
label="Reasoning effort"
|
||||
hint="Sent as reasoning_effort. Models that think before answering bill those hidden tokens as output; Low is plenty for two sentences. Compare token counts in the Playground."
|
||||
>
|
||||
<select value={draft.reasoningEffort} onChange={(e) => set('reasoningEffort', e.target.value as ReasoningEffort)}>
|
||||
<option value="">Server default (not sent)</option>
|
||||
<option value="minimal">Minimal</option>
|
||||
<option value="low">Low</option>
|
||||
<option value="medium">Medium</option>
|
||||
<option value="high">High</option>
|
||||
</select>
|
||||
</Field>
|
||||
</div>
|
||||
</fieldset>
|
||||
|
||||
|
||||
Reference in new issue
Block a user