Release / build (push) Successful in 1m10s
Before the AI (free): - Seen-URL ledger: every article URL is remembered for 180+ days, so an item still listed in a feed after its article was pruned (rejected ones go after 14 days) is never fetched or judged again. - Wire copy: an identical headline from another outlet within 48 h joins the existing story (or is filtered like its twin) with no triage, grouping or summary call, and does not trigger a rewrite. Triage: - Up to 8 articles from one source are judged in one call, so rules, instructions and examples are sent once; the worker waits up to a minute for a fetch's other articles to finish extracting. Articles the model skips are judged singly. - 800 characters of article text instead of 1500. Grouping without embeddings: - Free pre-filter on shared names (4-letter stems, so Trump/Trumpov and Bahrain/Bahrein agree) and numbers; no call when no recent story shares one, and at most 12 candidates instead of 60. Title and description: - Max article characters default 6000 (was 12000); a multi-source rewrite shares that budget. - New "Reasoning effort" setting, sent as reasoning_effort. Measured on production: Haiku 5.5 spends ~590 hidden reasoning tokens per summary. No wasted retries: - 401/403/404 from the chat endpoint (bad key, unknown model, wrong URL) pauses LLM work with a banner instead of failing every article; it resumes on a settings change or after 10 minutes. - Simpler structured-output modes are tried only on 400/422, so a misconfiguration costs one request, not three. Visibility: - Activity: calls and tokens per pipeline step, calls per published story, and work saved without AI. Playground: tokens per model. Tests for each saving and for the pause on misconfiguration. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
241 lines
8.1 KiB
TypeScript
241 lines
8.1 KiB
TypeScript
import type { Database } from './database.ts';
|
|
|
|
/**
|
|
* Forward-only migrations. Each entry runs once inside a transaction; the applied
|
|
* version is stored in meta.schema_version.
|
|
*/
|
|
const MIGRATIONS: string[] = [
|
|
/* 1 — initial schema */ `
|
|
CREATE TABLE setting (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE source (
|
|
id INTEGER PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
feed_url TEXT NOT NULL UNIQUE,
|
|
site_url TEXT,
|
|
enabled INTEGER NOT NULL DEFAULT 1,
|
|
poll_minutes INTEGER NOT NULL DEFAULT 15,
|
|
full_text INTEGER NOT NULL DEFAULT 1,
|
|
language_hint TEXT,
|
|
etag TEXT,
|
|
last_modified TEXT,
|
|
last_fetch_at TEXT,
|
|
last_success_at TEXT,
|
|
last_error TEXT,
|
|
failure_count INTEGER NOT NULL DEFAULT 0,
|
|
next_fetch_at TEXT,
|
|
initial_import_done INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE rule (
|
|
id INTEGER PRIMARY KEY,
|
|
kind TEXT NOT NULL CHECK (kind IN ('ai','keyword')),
|
|
polarity TEXT NOT NULL CHECK (polarity IN ('negative','exception')),
|
|
text TEXT NOT NULL,
|
|
enabled INTEGER NOT NULL DEFAULT 1,
|
|
source_ids TEXT NOT NULL DEFAULT '[]',
|
|
hit_count INTEGER NOT NULL DEFAULT 0,
|
|
last_hit_at TEXT,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE edition (
|
|
id INTEGER PRIMARY KEY,
|
|
day TEXT NOT NULL,
|
|
label TEXT NOT NULL,
|
|
scheduled_for TEXT NOT NULL,
|
|
released_at TEXT,
|
|
size INTEGER NOT NULL,
|
|
UNIQUE (day, label)
|
|
);
|
|
|
|
CREATE TABLE story (
|
|
id INTEGER PRIMARY KEY,
|
|
guid TEXT NOT NULL UNIQUE,
|
|
lead_article_id INTEGER,
|
|
current_version INTEGER,
|
|
title TEXT,
|
|
description TEXT,
|
|
category TEXT,
|
|
model_importance INTEGER,
|
|
coverage_boost INTEGER NOT NULL DEFAULT 0,
|
|
importance_override INTEGER,
|
|
importance INTEGER,
|
|
source_count INTEGER NOT NULL DEFAULT 1,
|
|
first_reported_at TEXT NOT NULL,
|
|
last_reported_at TEXT NOT NULL,
|
|
version_written_at TEXT,
|
|
first_published_at TEXT,
|
|
state TEXT NOT NULL,
|
|
regen_count INTEGER NOT NULL DEFAULT 0,
|
|
feed_status TEXT NOT NULL DEFAULT 'none',
|
|
feed_reason TEXT,
|
|
feed_day TEXT,
|
|
feed_decided_at TEXT,
|
|
entered_feed_at TEXT,
|
|
edition_id INTEGER REFERENCES edition(id) ON DELETE SET NULL,
|
|
read INTEGER NOT NULL DEFAULT 0,
|
|
bookmarked INTEGER NOT NULL DEFAULT 0,
|
|
hidden INTEGER NOT NULL DEFAULT 0,
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
next_attempt_at TEXT,
|
|
last_error TEXT,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX story_state ON story(state, last_reported_at);
|
|
CREATE INDEX story_feed ON story(feed_status, entered_feed_at);
|
|
CREATE INDEX story_first_reported ON story(first_reported_at);
|
|
CREATE INDEX story_feed_day ON story(feed_day);
|
|
|
|
CREATE TABLE article (
|
|
id INTEGER PRIMARY KEY,
|
|
source_id INTEGER NOT NULL REFERENCES source(id) ON DELETE CASCADE,
|
|
guid TEXT,
|
|
url TEXT NOT NULL,
|
|
url_hash TEXT NOT NULL,
|
|
title_hash TEXT NOT NULL,
|
|
title TEXT NOT NULL,
|
|
excerpt TEXT,
|
|
categories TEXT NOT NULL DEFAULT '[]',
|
|
text TEXT,
|
|
partial INTEGER NOT NULL DEFAULT 0,
|
|
author TEXT,
|
|
image_url TEXT,
|
|
language TEXT,
|
|
published_at TEXT,
|
|
fetched_at TEXT NOT NULL,
|
|
state TEXT NOT NULL,
|
|
state_changed_at TEXT NOT NULL,
|
|
story_id INTEGER REFERENCES story(id) ON DELETE SET NULL,
|
|
embedding BLOB,
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
next_attempt_at TEXT,
|
|
last_error TEXT
|
|
);
|
|
CREATE UNIQUE INDEX article_url_hash ON article(url_hash);
|
|
CREATE INDEX article_guid ON article(source_id, guid);
|
|
CREATE INDEX article_title_hash ON article(title_hash, fetched_at);
|
|
CREATE INDEX article_state ON article(state, published_at);
|
|
CREATE INDEX article_story ON article(story_id);
|
|
CREATE INDEX article_fetched ON article(fetched_at);
|
|
|
|
CREATE TABLE verdict (
|
|
article_id INTEGER PRIMARY KEY REFERENCES article(id) ON DELETE CASCADE,
|
|
decision TEXT NOT NULL CHECK (decision IN ('keep','reject')),
|
|
decided_by TEXT NOT NULL,
|
|
rule_id INTEGER REFERENCES rule(id) ON DELETE SET NULL,
|
|
rule_text TEXT,
|
|
reason TEXT NOT NULL,
|
|
confidence REAL,
|
|
model TEXT,
|
|
prompt_version TEXT,
|
|
overridden INTEGER NOT NULL DEFAULT 0,
|
|
decided_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX verdict_decision ON verdict(decision, decided_at);
|
|
|
|
CREATE TABLE story_version (
|
|
id INTEGER PRIMARY KEY,
|
|
story_id INTEGER NOT NULL REFERENCES story(id) ON DELETE CASCADE,
|
|
version INTEGER NOT NULL,
|
|
title TEXT NOT NULL,
|
|
description TEXT NOT NULL,
|
|
category TEXT NOT NULL,
|
|
importance INTEGER NOT NULL,
|
|
model TEXT NOT NULL,
|
|
prompt_version TEXT NOT NULL,
|
|
format_retries INTEGER NOT NULL DEFAULT 0,
|
|
tokens_in INTEGER,
|
|
tokens_out INTEGER,
|
|
duration_ms INTEGER NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
UNIQUE (story_id, version)
|
|
);
|
|
|
|
CREATE TABLE digest (
|
|
id INTEGER PRIMARY KEY,
|
|
day TEXT NOT NULL UNIQUE,
|
|
title TEXT NOT NULL,
|
|
opening TEXT,
|
|
kind TEXT,
|
|
state TEXT NOT NULL,
|
|
story_ids TEXT NOT NULL DEFAULT '[]',
|
|
window_start TEXT,
|
|
window_end TEXT,
|
|
model TEXT,
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
next_attempt_at TEXT,
|
|
last_error TEXT,
|
|
created_at TEXT NOT NULL,
|
|
published_at TEXT
|
|
);
|
|
|
|
CREATE TABLE example (
|
|
id INTEGER PRIMARY KEY,
|
|
kind TEXT NOT NULL CHECK (kind IN ('filter','clustering')),
|
|
expected TEXT NOT NULL,
|
|
label TEXT NOT NULL,
|
|
payload TEXT NOT NULL,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE llm_call (
|
|
id INTEGER PRIMARY KEY,
|
|
at TEXT NOT NULL,
|
|
purpose TEXT NOT NULL,
|
|
model TEXT NOT NULL,
|
|
ok INTEGER NOT NULL,
|
|
ms INTEGER NOT NULL,
|
|
tokens_in INTEGER,
|
|
tokens_out INTEGER,
|
|
error TEXT
|
|
);
|
|
CREATE INDEX llm_call_at ON llm_call(at);
|
|
|
|
CREATE TABLE stat (
|
|
day TEXT NOT NULL,
|
|
key TEXT NOT NULL,
|
|
n INTEGER NOT NULL DEFAULT 0,
|
|
PRIMARY KEY (day, key)
|
|
);
|
|
`,
|
|
/* 2 — long-lived ledger of every article URL ever seen, so an item that is still in a feed
|
|
after its article was pruned (e.g. rejected ones after 14 days) is never processed again */ `
|
|
CREATE TABLE seen (
|
|
url_hash TEXT PRIMARY KEY,
|
|
first_seen TEXT NOT NULL
|
|
);
|
|
INSERT OR IGNORE INTO seen (url_hash, first_seen) SELECT url_hash, fetched_at FROM article;
|
|
`,
|
|
];
|
|
|
|
export function migrate(db: Database): number {
|
|
db.exec(`CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
|
|
const row = db.prepare(`SELECT value FROM meta WHERE key = 'schema_version'`).get<{ value: string }>();
|
|
let version = row ? Number(row.value) : 0;
|
|
for (let i = version; i < MIGRATIONS.length; i++) {
|
|
const sql = MIGRATIONS[i]!;
|
|
db.transaction(() => {
|
|
db.exec(sql);
|
|
db.prepare(`INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)`).run(String(i + 1));
|
|
});
|
|
version = i + 1;
|
|
}
|
|
return version;
|
|
}
|
|
|
|
export function getMeta(db: Database, key: string): string | null {
|
|
return db.prepare(`SELECT value FROM meta WHERE key = ?`).get<{ value: string }>(key)?.value ?? null;
|
|
}
|
|
|
|
export function setMeta(db: Database, key: string, value: string): void {
|
|
db.prepare(`INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(
|
|
key,
|
|
value,
|
|
);
|
|
}
|