Files
RSSFeed/server/src/settings.ts
T
TMCandClaude Opus 5.5 1cc99df9bb
Release / build (push) Successful in 1m25s
Build RSSFeed: self-hosted Croatian news feed with a local LLM
Merges RSS/Atom/JSON sources into one feed. A local OpenAI-compatible
LLM filters articles against plain-language negatives, groups reports
of the same event into stories, and writes a one-sentence Croatian
title and a description of at most two sentences for each story.
Original articles are only ever linked as published.

- Pipeline: fetch, dedupe, Readability extraction, keyword + AI triage
  with exceptions and learned corrections, embedding clustering with an
  LLM same-event check, summary with format check and Croatian
  language guard, regeneration for developing stories
- Neutral importance scoring with coverage boost and manual override
- Feed controls: Live or Editions, daily/category/source limits,
  breaking bypass, 7-day preview; daily digest with offline fallback
- Circuit breaker that pauses and resumes the queue on LLM outages
- Express 5 + node:sqlite server, React 19 + Vite web UI (dark, PWA)
- Feeds under /feeds/ for a single Pangolin rule; optional feed token
- Dockerfile runs typecheck and the full test suite before building;
  Gitea workflow builds, scans and pushes the image

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 10:58:32 +01:00

265 lines
10 KiB
TypeScript

import { CATEGORIES, PROMPT_KEYS } from '../../shared/types.ts';
import type { Category, CategoryControl, DeepPartial, FeedControls, Settings } from '../../shared/types.ts';
import type { Config } from './config.ts';
import type { Database } from './db/database.ts';
import { isValidHM } from './lib/time.ts';
function defaultCategories(): Record<Category, CategoryControl> {
const out = {} as Record<Category, CategoryControl>;
for (const c of CATEGORIES) out[c] = { include: true, max: 0 };
return out;
}
export function defaultFeedControls(): FeedControls {
return {
mode: 'live',
storiesPerDay: 50,
editionTimes: ['07:00', '12:00', '18:00'],
editionSize: 15,
minImportance: 1,
breakingLevel: 5,
perSourceLimit: 0,
categories: defaultCategories(),
order: 'newest',
updatedStories: 'keep',
includeDigest: true,
itemSources: true,
itemImage: true,
feedLength: 100,
timeWindowHours: 48,
};
}
export function defaultSettings(): Settings {
return {
llm: {
baseUrl: '',
apiKey: '',
triageModel: '',
summaryModel: '',
embeddingModel: '',
concurrency: 1,
timeoutSec: 120,
maxArticleChars: 12_000,
temperature: 0.2,
},
pipeline: {
defaultPollMinutes: 15,
maxAgeHours: 72,
firstAddHours: 24,
rejectThreshold: 0.7,
examplesInPrompt: 10,
},
clustering: {
enabled: true,
windowHours: 48,
mergeThreshold: 0.86,
reviewThreshold: 0.72,
regenAtSources: 3,
regenGapHours: 6,
regenCap: 3,
preferCroatianLead: true,
},
importance: { boostSources: 3, boost: 1 },
digest: { enabled: true, time: '06:30', size: 10, minImportance: 3 },
feed: defaultFeedControls(),
retention: { publishedDays: 90, rejectedDays: 14, textDays: 30 },
prompts: { triage: null, sameEvent: null, summarize: null, regenerate: null, digest: null },
glossary: [],
access: { feedToken: null, publicUrl: null },
};
}
function isPlainObject(v: unknown): v is Record<string, unknown> {
return typeof v === 'object' && v !== null && !Array.isArray(v);
}
/** Objects merge recursively; arrays, strings, numbers and null replace. */
export function deepMerge<T>(base: T, patch: unknown): T {
if (!isPlainObject(patch) || !isPlainObject(base)) return (patch === undefined ? base : (patch as T));
const out: Record<string, unknown> = { ...(base as Record<string, unknown>) };
for (const [k, v] of Object.entries(patch)) {
if (v === undefined) continue;
const cur = out[k];
out[k] = isPlainObject(cur) && isPlainObject(v) ? deepMerge(cur, v) : v;
}
return out as T;
}
const clamp = (v: unknown, min: number, max: number, fallback: number): number => {
const n = typeof v === 'number' ? v : Number(v);
if (!Number.isFinite(n)) return fallback;
return Math.min(max, Math.max(min, n));
};
const clampInt = (v: unknown, min: number, max: number, fallback: number): number =>
Math.round(clamp(v, min, max, fallback));
const bool = (v: unknown, fallback: boolean): boolean => (typeof v === 'boolean' ? v : fallback);
const str = (v: unknown, fallback: string): string => (typeof v === 'string' ? v.trim() : fallback);
function oneOf<T extends string>(v: unknown, options: readonly T[], fallback: T): T {
return options.includes(v as T) ? (v as T) : fallback;
}
/** Coerces any stored or submitted settings object into a valid Settings. */
export function sanitizeSettings(input: unknown): Settings {
const d = defaultSettings();
const s = deepMerge(d, input);
const llm = s.llm;
const out: Settings = {
llm: {
baseUrl: str(llm.baseUrl, '').replace(/\/+$/, ''),
apiKey: str(llm.apiKey, ''),
triageModel: str(llm.triageModel, ''),
summaryModel: str(llm.summaryModel, ''),
embeddingModel: str(llm.embeddingModel, ''),
concurrency: clampInt(llm.concurrency, 1, 16, d.llm.concurrency),
timeoutSec: clampInt(llm.timeoutSec, 10, 900, d.llm.timeoutSec),
maxArticleChars: clampInt(llm.maxArticleChars, 1000, 200_000, d.llm.maxArticleChars),
temperature: clamp(llm.temperature, 0, 2, d.llm.temperature),
},
pipeline: {
defaultPollMinutes: clampInt(s.pipeline.defaultPollMinutes, 1, 1440, d.pipeline.defaultPollMinutes),
maxAgeHours: clampInt(s.pipeline.maxAgeHours, 1, 24 * 30, d.pipeline.maxAgeHours),
firstAddHours: clampInt(s.pipeline.firstAddHours, 1, 24 * 30, d.pipeline.firstAddHours),
rejectThreshold: clamp(s.pipeline.rejectThreshold, 0, 1, d.pipeline.rejectThreshold),
examplesInPrompt: clampInt(s.pipeline.examplesInPrompt, 0, 50, d.pipeline.examplesInPrompt),
},
clustering: {
enabled: bool(s.clustering.enabled, true),
windowHours: clampInt(s.clustering.windowHours, 1, 24 * 14, d.clustering.windowHours),
mergeThreshold: clamp(s.clustering.mergeThreshold, 0, 1, d.clustering.mergeThreshold),
reviewThreshold: clamp(s.clustering.reviewThreshold, 0, 1, d.clustering.reviewThreshold),
regenAtSources: clampInt(s.clustering.regenAtSources, 2, 50, d.clustering.regenAtSources),
regenGapHours: clamp(s.clustering.regenGapHours, 0, 24 * 7, d.clustering.regenGapHours),
regenCap: clampInt(s.clustering.regenCap, 0, 20, d.clustering.regenCap),
preferCroatianLead: bool(s.clustering.preferCroatianLead, true),
},
importance: {
boostSources: clampInt(s.importance.boostSources, 2, 50, d.importance.boostSources),
boost: clampInt(s.importance.boost, 0, 4, d.importance.boost),
},
digest: {
enabled: bool(s.digest.enabled, true),
time: isValidHM(str(s.digest.time, '')) ? str(s.digest.time, '') : d.digest.time,
size: clampInt(s.digest.size, 1, 50, d.digest.size),
minImportance: clampInt(s.digest.minImportance, 1, 5, d.digest.minImportance),
},
feed: sanitizeFeedControls(s.feed),
retention: {
publishedDays: clampInt(s.retention.publishedDays, 1, 3650, d.retention.publishedDays),
rejectedDays: clampInt(s.retention.rejectedDays, 1, 3650, d.retention.rejectedDays),
textDays: clampInt(s.retention.textDays, 1, 3650, d.retention.textDays),
},
prompts: { ...d.prompts },
glossary: [],
access: {
feedToken: null,
publicUrl: null,
},
};
if (out.clustering.reviewThreshold > out.clustering.mergeThreshold) {
out.clustering.reviewThreshold = out.clustering.mergeThreshold;
}
for (const key of PROMPT_KEYS) {
const v = (s.prompts as Record<string, unknown>)[key];
out.prompts[key] = typeof v === 'string' && v.trim() ? v : null;
}
if (Array.isArray(s.glossary)) {
for (const e of s.glossary as unknown[]) {
if (!isPlainObject(e)) continue;
const term = str(e.term, '');
if (!term) continue;
out.glossary.push({ term, translation: str(e.translation, '') });
}
}
const token = str(s.access.feedToken, '');
out.access.feedToken = /^[A-Za-z0-9_-]{8,128}$/.test(token) ? token : null;
const pub = str(s.access.publicUrl, '').replace(/\/+$/, '');
out.access.publicUrl = /^https?:\/\/[^\s/]+/.test(pub) ? pub : null;
return out;
}
export function sanitizeFeedControls(input: unknown): FeedControls {
const d = defaultFeedControls();
const f = deepMerge(d, input);
const times = Array.isArray(f.editionTimes)
? [...new Set((f.editionTimes as unknown[]).map((t) => str(t, '')).filter(isValidHM))].sort()
: d.editionTimes;
const categories = defaultCategories();
for (const c of CATEGORIES) {
const v = (f.categories as Record<string, unknown> | undefined)?.[c];
if (isPlainObject(v)) categories[c] = { include: bool(v.include, true), max: clampInt(v.max, 0, 1000, 0) };
}
return {
mode: oneOf(f.mode, ['live', 'editions'] as const, d.mode),
storiesPerDay: clampInt(f.storiesPerDay, 0, 10_000, d.storiesPerDay),
editionTimes: times.length ? times : d.editionTimes,
editionSize: clampInt(f.editionSize, 1, 1000, d.editionSize),
minImportance: clampInt(f.minImportance, 1, 5, d.minImportance),
breakingLevel: clampInt(f.breakingLevel, 0, 5, d.breakingLevel),
perSourceLimit: clampInt(f.perSourceLimit, 0, 1000, d.perSourceLimit),
categories,
order: oneOf(f.order, ['newest', 'importance'] as const, d.order),
updatedStories: oneOf(f.updatedStories, ['keep', 'bump'] as const, d.updatedStories),
includeDigest: bool(f.includeDigest, d.includeDigest),
itemSources: bool(f.itemSources, d.itemSources),
itemImage: bool(f.itemImage, d.itemImage),
feedLength: clampInt(f.feedLength, 1, 1000, d.feedLength),
timeWindowHours: clampInt(f.timeWindowHours, 1, 24 * 60, d.timeWindowHours),
};
}
export class SettingsStore {
private readonly db: Database;
private readonly config: Config;
private cache: Settings | null = null;
private listeners: ((s: Settings) => void)[] = [];
constructor(db: Database, config: Config) {
this.db = db;
this.config = config;
}
/** Stored settings, with env-provided initial LLM values filled in. */
get(): Settings {
if (!this.cache) {
const row = this.db.prepare(`SELECT value FROM setting WHERE key = 'settings'`).get<{ value: string }>();
let stored: unknown = {};
if (row) {
try {
stored = JSON.parse(row.value);
} catch {
stored = {};
}
}
const s = sanitizeSettings(stored);
if (!s.llm.baseUrl && this.config.llmBaseUrlEnv) s.llm.baseUrl = this.config.llmBaseUrlEnv.replace(/\/+$/, '');
if (!s.llm.apiKey && this.config.llmApiKeyEnv) s.llm.apiKey = this.config.llmApiKeyEnv;
this.cache = s;
}
return this.cache;
}
update(patch: DeepPartial<Settings>): Settings {
const merged = sanitizeSettings(deepMerge(this.get(), patch));
this.db
.prepare(`INSERT INTO setting (key, value) VALUES ('settings', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`)
.run(JSON.stringify(merged));
this.cache = null;
const fresh = this.get();
for (const l of this.listeners) l(fresh);
return fresh;
}
onChange(fn: (s: Settings) => void): void {
this.listeners.push(fn);
}
feedToken(): string | null {
return this.config.feedTokenEnv ?? this.get().access.feedToken;
}
publicUrl(): string | null {
return this.config.publicUrlEnv ?? this.get().access.publicUrl;
}
}