Files
RSSFeed/server/src/http/activity.ts
T
TMCandClaude Opus 5.5 29036127a9
Release / build (push) Successful in 1m10s
Cut AI usage: free dedupe and grouping, batched triage, fewer tokens
Before the AI (free):
- Seen-URL ledger: every article URL is remembered for 180+ days, so an
  item still listed in a feed after its article was pruned (rejected ones
  go after 14 days) is never fetched or judged again.
- Wire copy: an identical headline from another outlet within 48 h joins
  the existing story (or is filtered like its twin) with no triage,
  grouping or summary call, and does not trigger a rewrite.

Triage:
- Up to 8 articles from one source are judged in one call, so rules,
  instructions and examples are sent once; the worker waits up to a
  minute for a fetch's other articles to finish extracting. Articles the
  model skips are judged singly.
- 800 characters of article text instead of 1500.

Grouping without embeddings:
- Free pre-filter on shared names (4-letter stems, so Trump/Trumpov and
  Bahrain/Bahrein agree) and numbers; no call when no recent story shares
  one, and at most 12 candidates instead of 60.

Title and description:
- Max article characters default 6000 (was 12000); a multi-source rewrite
  shares that budget.
- New "Reasoning effort" setting, sent as reasoning_effort. Measured on
  production: Haiku 5.5 spends ~590 hidden reasoning tokens per summary.

No wasted retries:
- 401/403/404 from the chat endpoint (bad key, unknown model, wrong URL)
  pauses LLM work with a banner instead of failing every article; it
  resumes on a settings change or after 10 minutes.
- Simpler structured-output modes are tried only on 400/422, so a
  misconfiguration costs one request, not three.

Visibility:
- Activity: calls and tokens per pipeline step, calls per published
  story, and work saved without AI. Playground: tokens per model.

Tests for each saving and for the pause on misconfiguration.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 12:10:00 +01:00

173 lines
9.3 KiB
TypeScript

import type { ActivityDTO, ActivityFailure, FeedPreviewDay, FeedReason } from '../../../shared/types.ts';
import type { Ctx } from '../context.ts';
import { getMeta } from '../db/schema.ts';
import { DAY, isoMinus, localDay, localToUtc } from '../lib/time.ts';
function lastDays(n: number): string[] {
const days = [localDay()];
for (let i = 1; i < n; i++) days.unshift(localDay(new Date(localToUtc(days[0]!, '12:00').getTime() - DAY)));
return days;
}
export function buildActivity(ctx: Ctx): ActivityDTO {
const db = ctx.db;
const count = (sql: string, ...p: (string | number)[]) => db.prepare(sql).get<{ n: number }>(...p)?.n ?? 0;
const since24h = isoMinus(DAY);
const since7d = isoMinus(7 * DAY);
const days = lastDays(7);
const calls = db.prepare(`SELECT ok, ms, tokens_in, tokens_out FROM llm_call WHERE at >= ?`).all<{ ok: number; ms: number; tokens_in: number | null; tokens_out: number | null }>(since24h);
const okMs = calls.filter((c) => c.ok).map((c) => c.ms).sort((a, b) => a - b);
const breaker = ctx.llm.breakerInfo();
const byPurpose = db
.prepare(
`SELECT purpose, COUNT(*) AS calls, SUM(CASE WHEN ok = 0 THEN 1 ELSE 0 END) AS errors,
COALESCE(SUM(tokens_in), 0) AS tokens_in, COALESCE(SUM(tokens_out), 0) AS tokens_out
FROM llm_call WHERE at >= ? GROUP BY purpose ORDER BY calls DESC`,
)
.all<{ purpose: string; calls: number; errors: number; tokens_in: number; tokens_out: number }>(since24h)
.map((r) => ({ purpose: r.purpose, calls: r.calls, errors: r.errors, tokensIn: r.tokens_in, tokensOut: r.tokens_out }));
const pipelineCalls = byPurpose.filter((p) => !['test', 'playground', 'dry-run'].includes(p.purpose)).reduce((s, p) => s + p.calls, 0);
const published24h = count(`SELECT COUNT(*) AS n FROM story WHERE first_published_at >= ?`, since24h);
const failures: ActivityFailure[] = [
...db
.prepare(`SELECT id, title, last_error, attempts, state_changed_at FROM article WHERE state = 'failed' ORDER BY state_changed_at DESC LIMIT 50`)
.all<{ id: number; title: string; last_error: string | null; attempts: number; state_changed_at: string }>()
.map((a) => ({ kind: 'article' as const, id: a.id, title: a.title, error: a.last_error ?? 'Unknown error', attempts: a.attempts, at: a.state_changed_at })),
...db
.prepare(
`SELECT s.id, COALESCE(s.title, a.title, 'Untitled story') AS title, s.last_error, s.attempts, s.created_at
FROM story s LEFT JOIN article a ON a.id = s.lead_article_id
WHERE s.state = 'failed' OR (s.last_error IS NOT NULL AND s.attempts >= 3) ORDER BY s.id DESC LIMIT 50`,
)
.all<{ id: number; title: string; last_error: string | null; attempts: number; created_at: string }>()
.map((s) => ({ kind: 'story' as const, id: s.id, title: s.title, error: s.last_error ?? 'Unknown error', attempts: s.attempts, at: s.created_at })),
...db
.prepare(`SELECT id, title, last_error, attempts, created_at FROM digest WHERE last_error IS NOT NULL AND created_at >= ? ORDER BY id DESC LIMIT 10`)
.all<{ id: number; title: string; last_error: string; attempts: number; created_at: string }>(since7d)
.map((d) => ({ kind: 'digest' as const, id: d.id, title: d.title, error: d.last_error, attempts: d.attempts, at: d.created_at })),
].sort((a, b) => (a.at < b.at ? 1 : -1));
const imp = db
.prepare(`SELECT importance AS score, COUNT(*) AS n FROM story WHERE first_published_at >= ? AND importance IS NOT NULL GROUP BY importance`)
.all<{ score: number; n: number }>(since7d);
const importance7d = [1, 2, 3, 4, 5].map((score) => ({ score, count: imp.find((r) => r.score === score)?.n ?? 0 }));
const grouped = count(`SELECT COUNT(*) AS n FROM article WHERE state = 'grouped' AND state_changed_at >= ?`, since7d);
const inMulti = count(
`SELECT COUNT(*) AS n FROM article a JOIN story s ON s.id = a.story_id WHERE a.state = 'grouped' AND a.state_changed_at >= ? AND s.source_count >= 2`,
since7d,
);
const feedRows = db
.prepare(`SELECT feed_day, feed_status, feed_reason, COUNT(*) AS n FROM story WHERE feed_day >= ? AND feed_status IN ('in_feed','left_out') GROUP BY feed_day, feed_status, feed_reason`)
.all<{ feed_day: string; feed_status: string; feed_reason: FeedReason | null; n: number }>(days[0]!);
const feed7d: FeedPreviewDay[] = days.map((d) => {
const rows = feedRows.filter((r) => r.feed_day === d);
const byReason: Partial<Record<FeedReason, number>> = {};
for (const r of rows) if (r.feed_reason) byReason[r.feed_reason] = (byReason[r.feed_reason] ?? 0) + r.n;
return {
date: d,
inFeed: rows.filter((r) => r.feed_status === 'in_feed').reduce((s, r) => s + r.n, 0),
leftOut: rows.filter((r) => r.feed_status === 'left_out').reduce((s, r) => s + r.n, 0),
byReason,
};
});
const statSum = (key: string) => count(`SELECT COALESCE(SUM(n), 0) AS n FROM stat WHERE key = ? AND day >= ?`, key, days[0]!);
return {
queue: {
articlesNew: count(`SELECT COUNT(*) AS n FROM article WHERE state = 'new'`),
articlesExtracted: count(`SELECT COUNT(*) AS n FROM article WHERE state = 'extracted'`),
articlesKept: count(`SELECT COUNT(*) AS n FROM article WHERE state = 'kept'`),
storiesPending: count(`SELECT COUNT(*) AS n FROM story WHERE state = 'pending'`),
storiesUpdating: count(`SELECT COUNT(*) AS n FROM story WHERE state = 'updating'`),
digestsPending: count(`SELECT COUNT(*) AS n FROM digest WHERE state = 'pending'`),
},
llm: {
state: ctx.llm.state(),
consecutiveFailures: breaker.consecutiveFailures,
nextProbeAt: breaker.nextProbeAt,
inFlight: breaker.inFlight,
calls24h: calls.length,
errors24h: calls.filter((c) => !c.ok).length,
avgLatencyMs: okMs.length ? Math.round(okMs.reduce((s, v) => s + v, 0) / okMs.length) : null,
p95LatencyMs: okMs.length ? okMs[Math.min(okMs.length - 1, Math.floor(okMs.length * 0.95))]! : null,
tokensIn24h: calls.reduce((s, c) => s + (c.tokens_in ?? 0), 0),
tokensOut24h: calls.reduce((s, c) => s + (c.tokens_out ?? 0), 0),
error: ctx.llm.errorMessage(),
byPurpose,
callsPerStory: published24h ? Math.round((pipelineCalls / published24h) * 10) / 10 : null,
},
savings7d: {
duplicatesSkipped: statSum('duplicates_skipped'),
groupedByHeadline: statSum('grouped_by_headline'),
matchCallsSkipped: statSum('match_calls_skipped'),
triageBatched: statSum('triage_batched'),
},
failures: failures.slice(0, 60),
importance7d,
grouping7d: { articles: grouped, inMultiSourceStories: inMulti, share: grouped ? Math.round((inMulti / grouped) * 1000) / 1000 : 0 },
feed7d,
formatRetries7d: statSum('format_retries'),
languageFailures7d: statSum('language_failures'),
lastRuns: {
fetch: getMeta(db, 'last_fetch'),
edition: getMeta(db, 'last_edition'),
digest: getMeta(db, 'last_digest'),
cleanup: getMeta(db, 'last_cleanup'),
},
sources: {
total: count(`SELECT COUNT(*) AS n FROM source`),
enabled: count(`SELECT COUNT(*) AS n FROM source WHERE enabled = 1`),
failing: count(`SELECT COUNT(*) AS n FROM source WHERE enabled = 1 AND failure_count > 0`),
},
};
}
export function waitingCount(ctx: Ctx): number {
const a = ctx.db.prepare(`SELECT COUNT(*) AS n FROM article WHERE state IN ('new','extracted','kept')`).get<{ n: number }>()?.n ?? 0;
const s = ctx.db.prepare(`SELECT COUNT(*) AS n FROM story WHERE state IN ('pending','updating')`).get<{ n: number }>()?.n ?? 0;
return a + s;
}
/** Puts a failed item back into the pipeline. */
export function retryItem(ctx: Ctx, kind: string, id: number): boolean {
if (kind === 'article') {
const a = ctx.db
.prepare(`SELECT a.state, a.story_id, a.text, v.decision FROM article a LEFT JOIN verdict v ON v.article_id = a.id WHERE a.id = ?`)
.get<{ state: string; story_id: number | null; text: string | null; decision: string | null }>(id);
if (!a || a.state !== 'failed') return false;
const state = a.decision === 'keep' ? 'kept' : 'extracted';
ctx.db
.prepare(`UPDATE article SET state = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL, state_changed_at = ? WHERE id = ?`)
.run(state, new Date().toISOString(), id);
} else if (kind === 'story') {
const s = ctx.db.prepare(`SELECT current_version FROM story WHERE id = ?`).get<{ current_version: number | null }>(id);
if (!s) return false;
ctx.db
.prepare(`UPDATE story SET state = ?, attempts = 0, next_attempt_at = NULL, last_error = NULL WHERE id = ?`)
.run(s.current_version === null ? 'pending' : 'updating', id);
} else if (kind === 'digest') {
const changed = ctx.db
.prepare(`UPDATE digest SET state = 'pending', attempts = 0, next_attempt_at = NULL, last_error = NULL, created_at = ? WHERE id = ?`)
.run(new Date().toISOString(), id).changes;
if (!changed) return false;
} else {
return false;
}
ctx.wake();
return true;
}
export function retryAllFailed(ctx: Ctx): number {
let n = 0;
for (const a of ctx.db.prepare(`SELECT id FROM article WHERE state = 'failed'`).all<{ id: number }>()) if (retryItem(ctx, 'article', a.id)) n++;
for (const s of ctx.db.prepare(`SELECT id FROM story WHERE state = 'failed' OR (last_error IS NOT NULL AND attempts >= 3)`).all<{ id: number }>()) {
if (retryItem(ctx, 'story', s.id)) n++;
}
return n;
}