Release / build (push) Successful in 1m25s
Merges RSS/Atom/JSON sources into one feed. A local OpenAI-compatible LLM filters articles against plain-language negatives, groups reports of the same event into stories, and writes a one-sentence Croatian title and a description of at most two sentences for each story. Original articles are only ever linked as published. - Pipeline: fetch, dedupe, Readability extraction, keyword + AI triage with exceptions and learned corrections, embedding clustering with an LLM same-event check, summary with format check and Croatian language guard, regeneration for developing stories - Neutral importance scoring with coverage boost and manual override - Feed controls: Live or Editions, daily/category/source limits, breaking bypass, 7-day preview; daily digest with offline fallback - Circuit breaker that pauses and resumes the queue on LLM outages - Express 5 + node:sqlite server, React 19 + Vite web UI (dark, PWA) - Feeds under /feeds/ for a single Pangolin rule; optional feed token - Dockerfile runs typecheck and the full test suite before building; Gitea workflow builds, scans and pushes the image Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
265 lines
10 KiB
TypeScript
265 lines
10 KiB
TypeScript
import { CATEGORIES, PROMPT_KEYS } from '../../shared/types.ts';
|
|
import type { Category, CategoryControl, DeepPartial, FeedControls, Settings } from '../../shared/types.ts';
|
|
import type { Config } from './config.ts';
|
|
import type { Database } from './db/database.ts';
|
|
import { isValidHM } from './lib/time.ts';
|
|
|
|
function defaultCategories(): Record<Category, CategoryControl> {
|
|
const out = {} as Record<Category, CategoryControl>;
|
|
for (const c of CATEGORIES) out[c] = { include: true, max: 0 };
|
|
return out;
|
|
}
|
|
|
|
export function defaultFeedControls(): FeedControls {
|
|
return {
|
|
mode: 'live',
|
|
storiesPerDay: 50,
|
|
editionTimes: ['07:00', '12:00', '18:00'],
|
|
editionSize: 15,
|
|
minImportance: 1,
|
|
breakingLevel: 5,
|
|
perSourceLimit: 0,
|
|
categories: defaultCategories(),
|
|
order: 'newest',
|
|
updatedStories: 'keep',
|
|
includeDigest: true,
|
|
itemSources: true,
|
|
itemImage: true,
|
|
feedLength: 100,
|
|
timeWindowHours: 48,
|
|
};
|
|
}
|
|
|
|
export function defaultSettings(): Settings {
|
|
return {
|
|
llm: {
|
|
baseUrl: '',
|
|
apiKey: '',
|
|
triageModel: '',
|
|
summaryModel: '',
|
|
embeddingModel: '',
|
|
concurrency: 1,
|
|
timeoutSec: 120,
|
|
maxArticleChars: 12_000,
|
|
temperature: 0.2,
|
|
},
|
|
pipeline: {
|
|
defaultPollMinutes: 15,
|
|
maxAgeHours: 72,
|
|
firstAddHours: 24,
|
|
rejectThreshold: 0.7,
|
|
examplesInPrompt: 10,
|
|
},
|
|
clustering: {
|
|
enabled: true,
|
|
windowHours: 48,
|
|
mergeThreshold: 0.86,
|
|
reviewThreshold: 0.72,
|
|
regenAtSources: 3,
|
|
regenGapHours: 6,
|
|
regenCap: 3,
|
|
preferCroatianLead: true,
|
|
},
|
|
importance: { boostSources: 3, boost: 1 },
|
|
digest: { enabled: true, time: '06:30', size: 10, minImportance: 3 },
|
|
feed: defaultFeedControls(),
|
|
retention: { publishedDays: 90, rejectedDays: 14, textDays: 30 },
|
|
prompts: { triage: null, sameEvent: null, summarize: null, regenerate: null, digest: null },
|
|
glossary: [],
|
|
access: { feedToken: null, publicUrl: null },
|
|
};
|
|
}
|
|
|
|
function isPlainObject(v: unknown): v is Record<string, unknown> {
|
|
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
}
|
|
|
|
/** Objects merge recursively; arrays, strings, numbers and null replace. */
|
|
export function deepMerge<T>(base: T, patch: unknown): T {
|
|
if (!isPlainObject(patch) || !isPlainObject(base)) return (patch === undefined ? base : (patch as T));
|
|
const out: Record<string, unknown> = { ...(base as Record<string, unknown>) };
|
|
for (const [k, v] of Object.entries(patch)) {
|
|
if (v === undefined) continue;
|
|
const cur = out[k];
|
|
out[k] = isPlainObject(cur) && isPlainObject(v) ? deepMerge(cur, v) : v;
|
|
}
|
|
return out as T;
|
|
}
|
|
|
|
const clamp = (v: unknown, min: number, max: number, fallback: number): number => {
|
|
const n = typeof v === 'number' ? v : Number(v);
|
|
if (!Number.isFinite(n)) return fallback;
|
|
return Math.min(max, Math.max(min, n));
|
|
};
|
|
const clampInt = (v: unknown, min: number, max: number, fallback: number): number =>
|
|
Math.round(clamp(v, min, max, fallback));
|
|
const bool = (v: unknown, fallback: boolean): boolean => (typeof v === 'boolean' ? v : fallback);
|
|
const str = (v: unknown, fallback: string): string => (typeof v === 'string' ? v.trim() : fallback);
|
|
function oneOf<T extends string>(v: unknown, options: readonly T[], fallback: T): T {
|
|
return options.includes(v as T) ? (v as T) : fallback;
|
|
}
|
|
|
|
/** Coerces any stored or submitted settings object into a valid Settings. */
|
|
export function sanitizeSettings(input: unknown): Settings {
|
|
const d = defaultSettings();
|
|
const s = deepMerge(d, input);
|
|
const llm = s.llm;
|
|
const out: Settings = {
|
|
llm: {
|
|
baseUrl: str(llm.baseUrl, '').replace(/\/+$/, ''),
|
|
apiKey: str(llm.apiKey, ''),
|
|
triageModel: str(llm.triageModel, ''),
|
|
summaryModel: str(llm.summaryModel, ''),
|
|
embeddingModel: str(llm.embeddingModel, ''),
|
|
concurrency: clampInt(llm.concurrency, 1, 16, d.llm.concurrency),
|
|
timeoutSec: clampInt(llm.timeoutSec, 10, 900, d.llm.timeoutSec),
|
|
maxArticleChars: clampInt(llm.maxArticleChars, 1000, 200_000, d.llm.maxArticleChars),
|
|
temperature: clamp(llm.temperature, 0, 2, d.llm.temperature),
|
|
},
|
|
pipeline: {
|
|
defaultPollMinutes: clampInt(s.pipeline.defaultPollMinutes, 1, 1440, d.pipeline.defaultPollMinutes),
|
|
maxAgeHours: clampInt(s.pipeline.maxAgeHours, 1, 24 * 30, d.pipeline.maxAgeHours),
|
|
firstAddHours: clampInt(s.pipeline.firstAddHours, 1, 24 * 30, d.pipeline.firstAddHours),
|
|
rejectThreshold: clamp(s.pipeline.rejectThreshold, 0, 1, d.pipeline.rejectThreshold),
|
|
examplesInPrompt: clampInt(s.pipeline.examplesInPrompt, 0, 50, d.pipeline.examplesInPrompt),
|
|
},
|
|
clustering: {
|
|
enabled: bool(s.clustering.enabled, true),
|
|
windowHours: clampInt(s.clustering.windowHours, 1, 24 * 14, d.clustering.windowHours),
|
|
mergeThreshold: clamp(s.clustering.mergeThreshold, 0, 1, d.clustering.mergeThreshold),
|
|
reviewThreshold: clamp(s.clustering.reviewThreshold, 0, 1, d.clustering.reviewThreshold),
|
|
regenAtSources: clampInt(s.clustering.regenAtSources, 2, 50, d.clustering.regenAtSources),
|
|
regenGapHours: clamp(s.clustering.regenGapHours, 0, 24 * 7, d.clustering.regenGapHours),
|
|
regenCap: clampInt(s.clustering.regenCap, 0, 20, d.clustering.regenCap),
|
|
preferCroatianLead: bool(s.clustering.preferCroatianLead, true),
|
|
},
|
|
importance: {
|
|
boostSources: clampInt(s.importance.boostSources, 2, 50, d.importance.boostSources),
|
|
boost: clampInt(s.importance.boost, 0, 4, d.importance.boost),
|
|
},
|
|
digest: {
|
|
enabled: bool(s.digest.enabled, true),
|
|
time: isValidHM(str(s.digest.time, '')) ? str(s.digest.time, '') : d.digest.time,
|
|
size: clampInt(s.digest.size, 1, 50, d.digest.size),
|
|
minImportance: clampInt(s.digest.minImportance, 1, 5, d.digest.minImportance),
|
|
},
|
|
feed: sanitizeFeedControls(s.feed),
|
|
retention: {
|
|
publishedDays: clampInt(s.retention.publishedDays, 1, 3650, d.retention.publishedDays),
|
|
rejectedDays: clampInt(s.retention.rejectedDays, 1, 3650, d.retention.rejectedDays),
|
|
textDays: clampInt(s.retention.textDays, 1, 3650, d.retention.textDays),
|
|
},
|
|
prompts: { ...d.prompts },
|
|
glossary: [],
|
|
access: {
|
|
feedToken: null,
|
|
publicUrl: null,
|
|
},
|
|
};
|
|
if (out.clustering.reviewThreshold > out.clustering.mergeThreshold) {
|
|
out.clustering.reviewThreshold = out.clustering.mergeThreshold;
|
|
}
|
|
for (const key of PROMPT_KEYS) {
|
|
const v = (s.prompts as Record<string, unknown>)[key];
|
|
out.prompts[key] = typeof v === 'string' && v.trim() ? v : null;
|
|
}
|
|
if (Array.isArray(s.glossary)) {
|
|
for (const e of s.glossary as unknown[]) {
|
|
if (!isPlainObject(e)) continue;
|
|
const term = str(e.term, '');
|
|
if (!term) continue;
|
|
out.glossary.push({ term, translation: str(e.translation, '') });
|
|
}
|
|
}
|
|
const token = str(s.access.feedToken, '');
|
|
out.access.feedToken = /^[A-Za-z0-9_-]{8,128}$/.test(token) ? token : null;
|
|
const pub = str(s.access.publicUrl, '').replace(/\/+$/, '');
|
|
out.access.publicUrl = /^https?:\/\/[^\s/]+/.test(pub) ? pub : null;
|
|
return out;
|
|
}
|
|
|
|
export function sanitizeFeedControls(input: unknown): FeedControls {
|
|
const d = defaultFeedControls();
|
|
const f = deepMerge(d, input);
|
|
const times = Array.isArray(f.editionTimes)
|
|
? [...new Set((f.editionTimes as unknown[]).map((t) => str(t, '')).filter(isValidHM))].sort()
|
|
: d.editionTimes;
|
|
const categories = defaultCategories();
|
|
for (const c of CATEGORIES) {
|
|
const v = (f.categories as Record<string, unknown> | undefined)?.[c];
|
|
if (isPlainObject(v)) categories[c] = { include: bool(v.include, true), max: clampInt(v.max, 0, 1000, 0) };
|
|
}
|
|
return {
|
|
mode: oneOf(f.mode, ['live', 'editions'] as const, d.mode),
|
|
storiesPerDay: clampInt(f.storiesPerDay, 0, 10_000, d.storiesPerDay),
|
|
editionTimes: times.length ? times : d.editionTimes,
|
|
editionSize: clampInt(f.editionSize, 1, 1000, d.editionSize),
|
|
minImportance: clampInt(f.minImportance, 1, 5, d.minImportance),
|
|
breakingLevel: clampInt(f.breakingLevel, 0, 5, d.breakingLevel),
|
|
perSourceLimit: clampInt(f.perSourceLimit, 0, 1000, d.perSourceLimit),
|
|
categories,
|
|
order: oneOf(f.order, ['newest', 'importance'] as const, d.order),
|
|
updatedStories: oneOf(f.updatedStories, ['keep', 'bump'] as const, d.updatedStories),
|
|
includeDigest: bool(f.includeDigest, d.includeDigest),
|
|
itemSources: bool(f.itemSources, d.itemSources),
|
|
itemImage: bool(f.itemImage, d.itemImage),
|
|
feedLength: clampInt(f.feedLength, 1, 1000, d.feedLength),
|
|
timeWindowHours: clampInt(f.timeWindowHours, 1, 24 * 60, d.timeWindowHours),
|
|
};
|
|
}
|
|
|
|
export class SettingsStore {
|
|
private readonly db: Database;
|
|
private readonly config: Config;
|
|
private cache: Settings | null = null;
|
|
private listeners: ((s: Settings) => void)[] = [];
|
|
|
|
constructor(db: Database, config: Config) {
|
|
this.db = db;
|
|
this.config = config;
|
|
}
|
|
|
|
/** Stored settings, with env-provided initial LLM values filled in. */
|
|
get(): Settings {
|
|
if (!this.cache) {
|
|
const row = this.db.prepare(`SELECT value FROM setting WHERE key = 'settings'`).get<{ value: string }>();
|
|
let stored: unknown = {};
|
|
if (row) {
|
|
try {
|
|
stored = JSON.parse(row.value);
|
|
} catch {
|
|
stored = {};
|
|
}
|
|
}
|
|
const s = sanitizeSettings(stored);
|
|
if (!s.llm.baseUrl && this.config.llmBaseUrlEnv) s.llm.baseUrl = this.config.llmBaseUrlEnv.replace(/\/+$/, '');
|
|
if (!s.llm.apiKey && this.config.llmApiKeyEnv) s.llm.apiKey = this.config.llmApiKeyEnv;
|
|
this.cache = s;
|
|
}
|
|
return this.cache;
|
|
}
|
|
|
|
update(patch: DeepPartial<Settings>): Settings {
|
|
const merged = sanitizeSettings(deepMerge(this.get(), patch));
|
|
this.db
|
|
.prepare(`INSERT INTO setting (key, value) VALUES ('settings', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`)
|
|
.run(JSON.stringify(merged));
|
|
this.cache = null;
|
|
const fresh = this.get();
|
|
for (const l of this.listeners) l(fresh);
|
|
return fresh;
|
|
}
|
|
|
|
onChange(fn: (s: Settings) => void): void {
|
|
this.listeners.push(fn);
|
|
}
|
|
|
|
feedToken(): string | null {
|
|
return this.config.feedTokenEnv ?? this.get().access.feedToken;
|
|
}
|
|
|
|
publicUrl(): string | null {
|
|
return this.config.publicUrlEnv ?? this.get().access.publicUrl;
|
|
}
|
|
}
|