Files
RSSFeed/server/src/db/schema.ts
T
TMCandClaude Opus 5.5 29036127a9
Release / build (push) Successful in 1m10s
Cut AI usage: free dedupe and grouping, batched triage, fewer tokens
Before the AI (free):
- Seen-URL ledger: every article URL is remembered for 180+ days, so an
  item still listed in a feed after its article was pruned (rejected ones
  go after 14 days) is never fetched or judged again.
- Wire copy: an identical headline from another outlet within 48 h joins
  the existing story (or is filtered like its twin) with no triage,
  grouping or summary call, and does not trigger a rewrite.

Triage:
- Up to 8 articles from one source are judged in one call, so rules,
  instructions and examples are sent once; the worker waits up to a
  minute for a fetch's other articles to finish extracting. Articles the
  model skips are judged singly.
- 800 characters of article text instead of 1500.

Grouping without embeddings:
- Free pre-filter on shared names (4-letter stems, so Trump/Trumpov and
  Bahrain/Bahrein agree) and numbers; no call when no recent story shares
  one, and at most 12 candidates instead of 60.

Title and description:
- Max article characters default 6000 (was 12000); a multi-source rewrite
  shares that budget.
- New "Reasoning effort" setting, sent as reasoning_effort. Measured on
  production: Haiku 5.5 spends ~590 hidden reasoning tokens per summary.

No wasted retries:
- 401/403/404 from the chat endpoint (bad key, unknown model, wrong URL)
  pauses LLM work with a banner instead of failing every article; it
  resumes on a settings change or after 10 minutes.
- Simpler structured-output modes are tried only on 400/422, so a
  misconfiguration costs one request, not three.

Visibility:
- Activity: calls and tokens per pipeline step, calls per published
  story, and work saved without AI. Playground: tokens per model.

Tests for each saving and for the pause on misconfiguration.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 12:10:00 +01:00

241 lines
8.1 KiB
TypeScript

import type { Database } from './database.ts';
/**
* Forward-only migrations. Each entry runs once inside a transaction; the applied
* version is stored in meta.schema_version.
*/
const MIGRATIONS: string[] = [
/* 1 — initial schema */ `
CREATE TABLE setting (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE source (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
feed_url TEXT NOT NULL UNIQUE,
site_url TEXT,
enabled INTEGER NOT NULL DEFAULT 1,
poll_minutes INTEGER NOT NULL DEFAULT 15,
full_text INTEGER NOT NULL DEFAULT 1,
language_hint TEXT,
etag TEXT,
last_modified TEXT,
last_fetch_at TEXT,
last_success_at TEXT,
last_error TEXT,
failure_count INTEGER NOT NULL DEFAULT 0,
next_fetch_at TEXT,
initial_import_done INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL
);
CREATE TABLE rule (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('ai','keyword')),
polarity TEXT NOT NULL CHECK (polarity IN ('negative','exception')),
text TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
source_ids TEXT NOT NULL DEFAULT '[]',
hit_count INTEGER NOT NULL DEFAULT 0,
last_hit_at TEXT,
created_at TEXT NOT NULL
);
CREATE TABLE edition (
id INTEGER PRIMARY KEY,
day TEXT NOT NULL,
label TEXT NOT NULL,
scheduled_for TEXT NOT NULL,
released_at TEXT,
size INTEGER NOT NULL,
UNIQUE (day, label)
);
CREATE TABLE story (
id INTEGER PRIMARY KEY,
guid TEXT NOT NULL UNIQUE,
lead_article_id INTEGER,
current_version INTEGER,
title TEXT,
description TEXT,
category TEXT,
model_importance INTEGER,
coverage_boost INTEGER NOT NULL DEFAULT 0,
importance_override INTEGER,
importance INTEGER,
source_count INTEGER NOT NULL DEFAULT 1,
first_reported_at TEXT NOT NULL,
last_reported_at TEXT NOT NULL,
version_written_at TEXT,
first_published_at TEXT,
state TEXT NOT NULL,
regen_count INTEGER NOT NULL DEFAULT 0,
feed_status TEXT NOT NULL DEFAULT 'none',
feed_reason TEXT,
feed_day TEXT,
feed_decided_at TEXT,
entered_feed_at TEXT,
edition_id INTEGER REFERENCES edition(id) ON DELETE SET NULL,
read INTEGER NOT NULL DEFAULT 0,
bookmarked INTEGER NOT NULL DEFAULT 0,
hidden INTEGER NOT NULL DEFAULT 0,
attempts INTEGER NOT NULL DEFAULT 0,
next_attempt_at TEXT,
last_error TEXT,
created_at TEXT NOT NULL
);
CREATE INDEX story_state ON story(state, last_reported_at);
CREATE INDEX story_feed ON story(feed_status, entered_feed_at);
CREATE INDEX story_first_reported ON story(first_reported_at);
CREATE INDEX story_feed_day ON story(feed_day);
CREATE TABLE article (
id INTEGER PRIMARY KEY,
source_id INTEGER NOT NULL REFERENCES source(id) ON DELETE CASCADE,
guid TEXT,
url TEXT NOT NULL,
url_hash TEXT NOT NULL,
title_hash TEXT NOT NULL,
title TEXT NOT NULL,
excerpt TEXT,
categories TEXT NOT NULL DEFAULT '[]',
text TEXT,
partial INTEGER NOT NULL DEFAULT 0,
author TEXT,
image_url TEXT,
language TEXT,
published_at TEXT,
fetched_at TEXT NOT NULL,
state TEXT NOT NULL,
state_changed_at TEXT NOT NULL,
story_id INTEGER REFERENCES story(id) ON DELETE SET NULL,
embedding BLOB,
attempts INTEGER NOT NULL DEFAULT 0,
next_attempt_at TEXT,
last_error TEXT
);
CREATE UNIQUE INDEX article_url_hash ON article(url_hash);
CREATE INDEX article_guid ON article(source_id, guid);
CREATE INDEX article_title_hash ON article(title_hash, fetched_at);
CREATE INDEX article_state ON article(state, published_at);
CREATE INDEX article_story ON article(story_id);
CREATE INDEX article_fetched ON article(fetched_at);
CREATE TABLE verdict (
article_id INTEGER PRIMARY KEY REFERENCES article(id) ON DELETE CASCADE,
decision TEXT NOT NULL CHECK (decision IN ('keep','reject')),
decided_by TEXT NOT NULL,
rule_id INTEGER REFERENCES rule(id) ON DELETE SET NULL,
rule_text TEXT,
reason TEXT NOT NULL,
confidence REAL,
model TEXT,
prompt_version TEXT,
overridden INTEGER NOT NULL DEFAULT 0,
decided_at TEXT NOT NULL
);
CREATE INDEX verdict_decision ON verdict(decision, decided_at);
CREATE TABLE story_version (
id INTEGER PRIMARY KEY,
story_id INTEGER NOT NULL REFERENCES story(id) ON DELETE CASCADE,
version INTEGER NOT NULL,
title TEXT NOT NULL,
description TEXT NOT NULL,
category TEXT NOT NULL,
importance INTEGER NOT NULL,
model TEXT NOT NULL,
prompt_version TEXT NOT NULL,
format_retries INTEGER NOT NULL DEFAULT 0,
tokens_in INTEGER,
tokens_out INTEGER,
duration_ms INTEGER NOT NULL,
created_at TEXT NOT NULL,
UNIQUE (story_id, version)
);
CREATE TABLE digest (
id INTEGER PRIMARY KEY,
day TEXT NOT NULL UNIQUE,
title TEXT NOT NULL,
opening TEXT,
kind TEXT,
state TEXT NOT NULL,
story_ids TEXT NOT NULL DEFAULT '[]',
window_start TEXT,
window_end TEXT,
model TEXT,
attempts INTEGER NOT NULL DEFAULT 0,
next_attempt_at TEXT,
last_error TEXT,
created_at TEXT NOT NULL,
published_at TEXT
);
CREATE TABLE example (
id INTEGER PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('filter','clustering')),
expected TEXT NOT NULL,
label TEXT NOT NULL,
payload TEXT NOT NULL,
created_at TEXT NOT NULL
);
CREATE TABLE llm_call (
id INTEGER PRIMARY KEY,
at TEXT NOT NULL,
purpose TEXT NOT NULL,
model TEXT NOT NULL,
ok INTEGER NOT NULL,
ms INTEGER NOT NULL,
tokens_in INTEGER,
tokens_out INTEGER,
error TEXT
);
CREATE INDEX llm_call_at ON llm_call(at);
CREATE TABLE stat (
day TEXT NOT NULL,
key TEXT NOT NULL,
n INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (day, key)
);
`,
/* 2 — long-lived ledger of every article URL ever seen, so an item that is still in a feed
after its article was pruned (e.g. rejected ones after 14 days) is never processed again */ `
CREATE TABLE seen (
url_hash TEXT PRIMARY KEY,
first_seen TEXT NOT NULL
);
INSERT OR IGNORE INTO seen (url_hash, first_seen) SELECT url_hash, fetched_at FROM article;
`,
];
export function migrate(db: Database): number {
db.exec(`CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get<{ value: string }>();
let version = row ? Number(row.value) : 0;
for (let i = version; i < MIGRATIONS.length; i++) {
const sql = MIGRATIONS[i]!;
db.transaction(() => {
db.exec(sql);
db.prepare(`INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)`).run(String(i + 1));
});
version = i + 1;
}
return version;
}
export function getMeta(db: Database, key: string): string | null {
return db.prepare(`SELECT value FROM meta WHERE key = ?`).get<{ value: string }>(key)?.value ?? null;
}
export function setMeta(db: Database, key: string, value: string): void {
db.prepare(`INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(
key,
value,
);
}