Files
RSSFeed/shared/types.ts
T
TMCandClaude Opus 5.5 1cc99df9bb
Release / build (push) Successful in 1m25s
Build RSSFeed: self-hosted Croatian news feed with a local LLM
Merges RSS/Atom/JSON sources into one feed. A local OpenAI-compatible
LLM filters articles against plain-language negatives, groups reports
of the same event into stories, and writes a one-sentence Croatian
title and a description of at most two sentences for each story.
Original articles are only ever linked as published.

- Pipeline: fetch, dedupe, Readability extraction, keyword + AI triage
  with exceptions and learned corrections, embedding clustering with an
  LLM same-event check, summary with format check and Croatian
  language guard, regeneration for developing stories
- Neutral importance scoring with coverage boost and manual override
- Feed controls: Live or Editions, daily/category/source limits,
  breaking bypass, 7-day preview; daily digest with offline fallback
- Circuit breaker that pauses and resumes the queue on LLM outages
- Express 5 + node:sqlite server, React 19 + Vite web UI (dark, PWA)
- Feeds under /feeds/ for a single Pangolin rule; optional feed token
- Dockerfile runs typecheck and the full test suite before building;
  Gitea workflow builds, scans and pushes the image

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 10:58:32 +01:00

640 lines
20 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* API contract between the server and the web UI.
*
* All endpoints are JSON under /api unless noted. Timestamps are ISO-8601 strings (UTC).
* Errors come back as HTTP 4xx/5xx with body { error: string }.
*
* This file is imported at runtime by the server (for CATEGORIES and defaults) and by the
* web build, so it must stay plain, erasable TypeScript: no enums, no namespaces.
*/
// ───────────────────────────── Constants ─────────────────────────────
export const CATEGORIES = [
'Politika',
'Svijet',
'Gospodarstvo',
'Tehnologija',
'Znanost',
'Sport',
'Kultura',
'Zdravlje',
'Crna kronika',
'Ostalo',
] as const;
export type Category = (typeof CATEGORIES)[number];
/** The digest's own category. Never assigned to a story. */
export const DIGEST_CATEGORY = 'Pregled';
export const PROMPT_KEYS = ['triage', 'sameEvent', 'summarize', 'regenerate', 'digest'] as const;
export type PromptKey = (typeof PROMPT_KEYS)[number];
// ───────────────────────────── States ─────────────────────────────
export type ArticleState = 'new' | 'extracted' | 'kept' | 'grouped' | 'rejected' | 'skipped' | 'failed';
export type StoryState = 'pending' | 'published' | 'updating' | 'failed';
/** none = not published yet; queued = waiting for the next edition. */
export type FeedStatus = 'none' | 'in_feed' | 'queued' | 'left_out';
/**
* Why a story is (or is not) in the feed.
* in_feed: 'auto' | 'breaking' | 'manual'
* left_out: everything else
*/
export type FeedReason =
| 'auto'
| 'breaking'
| 'manual'
| 'daily_limit'
| 'edition_full'
| 'below_min_importance'
| 'category_excluded'
| 'category_limit'
| 'source_limit'
| 'marked_unwanted';
export const FEED_REASON_LABELS: Record<FeedReason, string> = {
auto: 'In feed',
breaking: 'Breaking',
manual: 'Added by hand',
daily_limit: 'Daily limit reached',
edition_full: 'Edition full',
below_min_importance: 'Below minimum importance',
category_excluded: 'Category excluded',
category_limit: 'Category limit reached',
source_limit: 'Source limit reached',
marked_unwanted: 'Marked "should have been filtered"',
};
// ───────────────────────────── Sources ─────────────────────────────
export interface SourceDTO {
id: number;
name: string;
feedUrl: string;
siteUrl: string | null;
enabled: boolean;
pollMinutes: number;
fullText: boolean;
/** ISO 639-1 code or null for auto-detect. */
languageHint: string | null;
lastFetchAt: string | null;
lastSuccessAt: string | null;
lastError: string | null;
failureCount: number;
nextFetchAt: string | null;
/** Articles fetched in the last 24 h, and how many of them were filtered. */
stats24h: { articles: number; filtered: number };
createdAt: string;
}
export interface SourceInput {
feedUrl: string;
name?: string;
siteUrl?: string | null;
enabled?: boolean;
pollMinutes?: number;
fullText?: boolean;
languageHint?: string | null;
}
export interface FeedCandidate {
url: string;
title: string | null;
format: 'rss' | 'atom' | 'rdf' | 'json';
}
/**
* GET /api/sources -> SourceDTO[]
* POST /api/sources/discover {url} -> { candidates: FeedCandidate[] } (url may be a site or a feed)
* POST /api/sources SourceInput -> SourceDTO
* PATCH /api/sources/:id Partial<SourceInput> -> SourceDTO
* DELETE /api/sources/:id -> { ok: true }
* POST /api/sources/:id/fetch -> { ok: true; newItems: number } (fetch now)
* GET /api/sources/opml -> OPML file download (text/x-opml)
* POST /api/sources/opml {opml} -> { added: number; skipped: number }
*/
export interface DiscoverResponse {
candidates: FeedCandidate[];
}
// ───────────────────────────── Rules ─────────────────────────────
export type RuleKind = 'ai' | 'keyword';
/** negative = filter out; exception = always keep (overrides negatives). */
export type RulePolarity = 'negative' | 'exception';
export interface RuleDTO {
id: number;
kind: RuleKind;
polarity: RulePolarity;
/** AI: plain-language description. Keyword: comma/newline separated phrases, or /regex/flags. */
text: string;
enabled: boolean;
/** Empty = applies to all sources. */
sourceIds: number[];
hitCount: number;
lastHitAt: string | null;
createdAt: string;
}
export interface RuleInput {
kind: RuleKind;
polarity: RulePolarity;
text: string;
enabled?: boolean;
sourceIds?: number[];
}
/**
* GET /api/rules -> RuleDTO[]
* POST /api/rules RuleInput -> RuleDTO
* PATCH /api/rules/:id Partial<RuleInput> -> RuleDTO
* DELETE /api/rules/:id -> { ok: true }
* POST /api/rules/dry-run { rule: RuleInput } -> DryRunStatus (starts a job; keyword rules finish instantly)
* GET /api/rules/dry-run/:jobId -> DryRunStatus (poll every ~1 s while status = running)
*/
export interface DryRunHit {
articleId: number;
title: string;
sourceName: string;
url: string;
reason: string;
confidence: number | null;
/** True when the article is currently published (so this rule would remove something you saw). */
currentlyPublished: boolean;
}
export interface DryRunStatus {
jobId: string;
status: 'running' | 'done' | 'error';
checked: number;
total: number;
hits: DryRunHit[];
error: string | null;
}
// ───────────────────────────── Examples (learned corrections) ─────────────────────────────
export interface ExampleDTO {
id: number;
kind: 'filter' | 'clustering';
/** filter: reject | keep. clustering: same | different. */
expected: 'reject' | 'keep' | 'same' | 'different';
/** Short human-readable summary, e.g. the article headline or "A ↔ B". */
label: string;
createdAt: string;
}
/**
* GET /api/examples -> ExampleDTO[]
* DELETE /api/examples/:id -> { ok: true }
*/
// ───────────────────────────── Stories ─────────────────────────────
export interface StorySourceDTO {
articleId: number;
sourceId: number;
sourceName: string;
/** The article's own headline, unchanged. */
originalTitle: string;
url: string;
publishedAt: string | null;
language: string | null;
partial: boolean;
isLead: boolean;
}
export interface StoryListItem {
id: number;
guid: string;
/** Croatian one-sentence title; null until the first version is written. */
title: string | null;
/** Croatian description, at most two sentences. */
description: string | null;
category: Category | null;
/** Effective importance (override, or model score + coverage boost). Null until scored. */
importance: number | null;
modelImportance: number | null;
importanceOverride: number | null;
coverageBoost: number;
sourceCount: number;
leadSourceName: string;
leadUrl: string;
leadOriginalTitle: string;
imageUrl: string | null;
/** True when the lead article could only be read from the feed excerpt. */
partial: boolean;
firstReportedAt: string;
lastReportedAt: string;
/** When the current title/description version was written. */
updatedAt: string | null;
state: StoryState;
feedStatus: FeedStatus;
feedReason: FeedReason | null;
enteredFeedAt: string | null;
editionId: number | null;
read: boolean;
bookmarked: boolean;
}
export interface StoryVersionDTO {
version: number;
title: string;
description: string;
category: Category;
importance: number;
model: string;
promptVersion: string;
formatRetries: number;
tokensIn: number | null;
tokensOut: number | null;
durationMs: number;
createdAt: string;
}
export interface StoryDetail extends StoryListItem {
sources: StorySourceDTO[];
versions: StoryVersionDTO[];
regenerationCount: number;
lastError: string | null;
}
export type StoryView = 'feed' | 'all';
export type StorySort = 'time' | 'importance';
/**
* GET /api/stories?view=feed|all&q=&category=&sourceId=&minImportance=&sort=time|importance
* &unread=1&bookmarked=1&offset=0&limit=50 -> StoryListResponse
* view=feed: stories in the RSS feed (or queued for an edition).
* view=all: every published story, including the ones feed controls left out.
* GET /api/stories/:id -> StoryDetail
* PATCH /api/stories/:id { read?, bookmarked?, importanceOverride?: number | null } -> StoryListItem
* POST /api/stories/mark-read { ids: number[] } -> { ok: true }
* POST /api/stories/:id/regenerate -> { ok: true }
* POST /api/stories/:id/add-to-feed -> StoryListItem
* POST /api/stories/:id/split { articleId } -> { newStoryId: number }
* POST /api/stories/:id/merge { otherStoryId } -> { storyId: number } (other is merged into :id)
* POST /api/stories/:id/unwanted -> { ok: true }
* "Should have been filtered": stores a filter example and hides the story.
*/
export interface StoryListResponse {
items: StoryListItem[];
total: number;
offset: number;
limit: number;
}
// ───────────────────────────── Editions ─────────────────────────────
export interface EditionDTO {
id: number;
/** Local time label, e.g. "07:00". */
label: string;
scheduledFor: string;
releasedAt: string | null;
size: number;
storyCount: number;
}
/** GET /api/editions?limit=20 -> EditionDTO[] (newest first) */
// ───────────────────────────── Filtered ─────────────────────────────
export interface FilteredArticleDTO {
id: number;
title: string;
url: string;
sourceId: number;
sourceName: string;
publishedAt: string | null;
fetchedAt: string;
decidedBy: 'keyword' | 'ai';
ruleId: number | null;
ruleText: string | null;
/** English, one sentence. */
reason: string;
confidence: number | null;
decidedAt: string;
}
/**
* GET /api/filtered?ruleId=&q=&offset=0&limit=50 -> { items: FilteredArticleDTO[]; total: number }
* POST /api/articles/:id/rescue -> { ok: true }
*/
export interface FilteredListResponse {
items: FilteredArticleDTO[];
total: number;
}
// ───────────────────────────── Digest ─────────────────────────────
export interface DigestDTO {
id: number;
/** Local date the digest was published for, YYYY-MM-DD. */
date: string;
/** "Jutarnji pregled — 15. studenoga 2026." */
title: string;
/** Two-sentence opening; null for the fallback version or while pending. */
opening: string | null;
kind: 'llm' | 'fallback' | null;
state: 'pending' | 'published' | 'skipped';
/** Stories in importance order, exactly as published. */
stories: StoryListItem[];
createdAt: string;
publishedAt: string | null;
}
/**
* GET /api/digests?limit=14 -> DigestDTO[] (newest first)
* GET /api/digests/latest -> DigestDTO | null (latest published)
* GET /api/digests/:id -> DigestDTO
* POST /api/digests/run -> DigestDTO (build one now, for testing)
*/
// ───────────────────────────── Settings ─────────────────────────────
export interface LlmSettings {
baseUrl: string;
apiKey: string;
triageModel: string;
summaryModel: string;
embeddingModel: string;
concurrency: number;
timeoutSec: number;
maxArticleChars: number;
temperature: number;
}
export interface PipelineSettings {
defaultPollMinutes: number;
/** Items older than this when fetched or processed are skipped. */
maxAgeHours: number;
/** A newly added source imports only items from this many recent hours. */
firstAddHours: number;
/** Reject only when triage confidence is at or above this (0–1). */
rejectThreshold: number;
/** Most recent corrections included in prompts. */
examplesInPrompt: number;
}
export interface ClusteringSettings {
enabled: boolean;
windowHours: number;
mergeThreshold: number;
reviewThreshold: number;
regenAtSources: number;
regenGapHours: number;
regenCap: number;
preferCroatianLead: boolean;
}
export interface ImportanceSettings {
/** Sources needed for the coverage boost. */
boostSources: number;
/** Points added, capped at 5. 0 disables the boost. */
boost: number;
}
export interface DigestSettings {
enabled: boolean;
/** Local time, HH:MM. */
time: string;
size: number;
minImportance: number;
}
export interface CategoryControl {
include: boolean;
/** Daily maximum; 0 = no maximum. */
max: number;
}
export interface FeedControls {
mode: 'live' | 'editions';
/** Live mode; 0 = unlimited. */
storiesPerDay: number;
/** Editions mode: local HH:MM release times. */
editionTimes: string[];
editionSize: number;
minImportance: number;
/** 1–5; 0 disables the breaking bypass. */
breakingLevel: number;
/** Stories per day led by one source; 0 = off. */
perSourceLimit: number;
categories: Record<Category, CategoryControl>;
order: 'newest' | 'importance';
updatedStories: 'keep' | 'bump';
includeDigest: boolean;
itemSources: boolean;
itemImage: boolean;
feedLength: number;
timeWindowHours: number;
}
export interface RetentionSettings {
publishedDays: number;
rejectedDays: number;
textDays: number;
}
export interface GlossaryEntry {
term: string;
/** Empty string = never translate this term. */
translation: string;
}
export interface AccessSettings {
/** Secret path segment for feeds; null = off. Ignored when RSSFEED_FEED_TOKEN is set. */
feedToken: string | null;
/** e.g. https://rss.example.com; null = derive from forwarded headers. Ignored when RSSFEED_PUBLIC_URL is set. */
publicUrl: string | null;
}
export interface Settings {
llm: LlmSettings;
pipeline: PipelineSettings;
clustering: ClusteringSettings;
importance: ImportanceSettings;
digest: DigestSettings;
feed: FeedControls;
retention: RetentionSettings;
/** null = use the default prompt. */
prompts: Record<PromptKey, string | null>;
glossary: GlossaryEntry[];
access: AccessSettings;
}
export type DeepPartial<T> = {
[K in keyof T]?: T[K] extends (infer U)[] ? U[] : T[K] extends object ? DeepPartial<T[K]> : T[K];
};
/**
* GET /api/settings -> SettingsResponse
* PUT /api/settings DeepPartial<Settings> -> SettingsResponse (arrays and prompt strings replace; objects merge)
* POST /api/settings/feed-preview { feed: FeedControls } -> FeedPreview
*/
export interface SettingsResponse {
settings: Settings;
defaultPrompts: Record<PromptKey, string>;
/** Which values are pinned by environment variables (read-only in the UI). */
env: { llmBaseUrl: boolean; llmApiKey: boolean; feedToken: boolean; publicUrl: boolean };
/** Absolute URLs to subscribe to, built from the public URL / request host. */
feedUrls: { rss: string; json: string; digest: string };
timezone: string;
}
export interface FeedPreviewDay {
date: string;
inFeed: number;
leftOut: number;
byReason: Partial<Record<FeedReason, number>>;
}
export interface FeedPreview {
days: FeedPreviewDay[];
averagePerDay: number;
leftOut: { storyId: number; title: string; importance: number | null; category: Category | null; reason: FeedReason; date: string }[];
}
// ───────────────────────────── LLM ─────────────────────────────
export type StructuredMode = 'json_schema' | 'json_object' | 'prompt';
/**
* POST /api/llm/models { baseUrl?, apiKey? } -> { models: string[] } (defaults to saved settings)
* POST /api/llm/test { llm?: Partial<LlmSettings> } -> LlmTestResult
*/
export interface LlmTestStep {
ok: boolean;
ms: number;
error: string | null;
}
export interface LlmTestResult {
reachable: boolean;
models: string[];
latencyMs: number | null;
structuredMode: StructuredMode | null;
triage: LlmTestStep & { decision: string | null };
summary: LlmTestStep & { title: string | null; description: string | null };
embedding: LlmTestStep & { dimensions: number | null };
error: string | null;
}
// ───────────────────────────── Playground ─────────────────────────────
/**
* POST /api/playground/run { url, models?: string[] } -> PlaygroundResult (models: up to 2 summary models; default = settings)
* POST /api/playground/similarity { a: url, b: url } -> SimilarityResult
* Both can take a minute; show a spinner.
*/
export interface PlaygroundSummary {
model: string;
ok: boolean;
title: string | null;
description: string | null;
category: Category | null;
importance: number | null;
formatRetries: number;
languageOk: boolean;
ms: number;
error: string | null;
}
export interface PlaygroundResult {
extraction: {
title: string;
text: string;
chars: number;
partial: boolean;
imageUrl: string | null;
language: string | null;
};
triage:
| { skipped: true; note: string }
| { skipped: false; decision: 'keep' | 'reject'; ruleText: string | null; reason: string; confidence: number | null; ms: number };
summaries: PlaygroundSummary[];
}
export interface SimilarityResult {
similarity: number;
mergeThreshold: number;
reviewThreshold: number;
verdict: 'merge' | 'review' | 'new';
titles: { a: string; b: string };
}
// ───────────────────────────── Activity & status ─────────────────────────────
export type LlmState = 'ok' | 'offline' | 'unconfigured';
export interface ActivityFailure {
kind: 'article' | 'story' | 'digest';
id: number;
title: string;
error: string;
attempts: number;
at: string;
}
export interface ActivityDTO {
queue: {
articlesNew: number;
articlesExtracted: number;
articlesKept: number;
storiesPending: number;
storiesUpdating: number;
digestsPending: number;
};
llm: {
state: LlmState;
consecutiveFailures: number;
nextProbeAt: string | null;
inFlight: number;
calls24h: number;
errors24h: number;
avgLatencyMs: number | null;
p95LatencyMs: number | null;
tokensIn24h: number;
tokensOut24h: number;
};
failures: ActivityFailure[];
importance7d: { score: number; count: number }[];
grouping7d: { articles: number; inMultiSourceStories: number; share: number };
feed7d: FeedPreviewDay[];
formatRetries7d: number;
languageFailures7d: number;
lastRuns: { fetch: string | null; edition: string | null; digest: string | null; cleanup: string | null };
sources: { total: number; enabled: number; failing: number };
}
/**
* GET /api/activity -> ActivityDTO
* POST /api/activity/retry { kind, id } -> { ok: true }
* POST /api/activity/retry-failed -> { retried: number }
* GET /api/status -> StatusDTO (cheap; poll every ~20 s for the offline banner)
* GET /api/health -> HealthDTO
*/
export interface StatusDTO {
llm: LlmState;
/** Articles and stories waiting on the LLM. */
waiting: number;
version: string;
latestDigestId: number | null;
feedMode: 'live' | 'editions';
}
export interface HealthDTO {
ok: boolean;
version: string;
db: 'ok' | 'error';
queueDepth: number;
llm: LlmState;
lastEditionAt: string | null;
lastDigestAt: string | null;
}