Build RSSFeed: self-hosted Croatian news feed with a local LLM
Release / build (push) Successful in 1m25s

Merges RSS/Atom/JSON sources into one feed. A local OpenAI-compatible
LLM filters articles against plain-language negatives, groups reports
of the same event into stories, and writes a one-sentence Croatian
title and a description of at most two sentences for each story.
Original articles are only ever linked as published.

- Pipeline: fetch, dedupe, Readability extraction, keyword + AI triage
  with exceptions and learned corrections, embedding clustering with an
  LLM same-event check, summary with format check and Croatian
  language guard, regeneration for developing stories
- Neutral importance scoring with coverage boost and manual override
- Feed controls: Live or Editions, daily/category/source limits,
  breaking bypass, 7-day preview; daily digest with offline fallback
- Circuit breaker that pauses and resumes the queue on LLM outages
- Express 5 + node:sqlite server, React 19 + Vite web UI (dark, PWA)
- Feeds under /feeds/ for a single Pangolin rule; optional feed token
- Dockerfile runs typecheck and the full test suite before building;
  Gitea workflow builds, scans and pushes the image

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
TMCandClaude Opus 5.5 committed 2026-11-15 10:58:32 +01:00
commit 1cc99df9bb
94 files changed
+19181

No files matched your search

+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env node
/**
* One-command dev environment: backend (Express + API on :34500) and the Vite dev server
* (:5173, proxying /api and /feeds to the backend), with prefixed output and clean shutdown.
* npm run dev → open http://localhost:5173
* npm run dev -- --mock → also starts the mock LLM on :11435 (no real LLM needed)
*/
import { spawn } from 'node:child_process';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const dataDir = process.env.RSSFEED_DATA_DIR ?? path.join(root, '.data');
const withMock = process.argv.includes('--mock');
const procs = [];
let shuttingDown = false;
function start(name, color, command, extraEnv = {}) {
const child = spawn(command, { cwd: root, shell: true, env: { ...process.env, ...extraEnv } });
const prefix = `\x1b[${color}m[${name}]\x1b[0m `;
const pipe = (stream, out) => {
let buf = '';
stream.on('data', (d) => {
buf += d.toString();
let i;
while ((i = buf.indexOf('\n')) >= 0) {
out.write(prefix + buf.slice(0, i) + '\n');
buf = buf.slice(i + 1);
}
});
stream.on('end', () => {
if (buf.length > 0) out.write(prefix + buf + '\n');
});
};
pipe(child.stdout, process.stdout);
pipe(child.stderr, process.stderr);
child.on('exit', (code) => {
process.stdout.write(prefix + `exited (code ${code})\n`);
shutdown();
});
procs.push(child);
}
function shutdown() {
if (shuttingDown) return;
shuttingDown = true;
for (const p of procs) {
try {
if (process.platform === 'win32') spawn('taskkill', ['/pid', String(p.pid), '/T', '/F']);
else p.kill('SIGTERM');
} catch {
/* already gone */
}
}
setTimeout(() => process.exit(0), 500);
}
process.on('SIGINT', shutdown);
process.on('SIGTERM', shutdown);
console.log('RSSFeed dev — backend :34500 + web :5173' + (withMock ? ' + mock LLM :11435' : ''));
console.log(` data dir: ${dataDir}`);
console.log(' open → \x1b[36mhttp://localhost:5173\x1b[0m\n');
const serverEnv = { RSSFEED_DATA_DIR: dataDir, RSSFEED_ALLOW_PRIVATE_URLS: process.env.RSSFEED_ALLOW_PRIVATE_URLS ?? '1' };
if (withMock) {
serverEnv.RSSFEED_LLM_BASE_URL = 'http://127.0.0.1:11435/v1';
start('mock-llm', '33', 'node server/src/testing/mock-llm.ts 11435');
}
start('server', '36', 'npm run dev --workspace server', serverEnv);
start('web', '35', 'npm run dev --workspace web');
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env node
/**
* Generates the PWA PNG icons in web/public with no dependencies (zlib + a CRC32 PNG
* encoder). Draws the same RSS-style glyph as web/public/icon.svg, anti-aliased by
* 4×4 supersampling. Run once after changing the design: node scripts/make-icons.mjs
*/
import { deflateSync } from 'node:zlib';
import { writeFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const outDir = path.join(root, 'web', 'public');
const BG = [0x14, 0x1a, 0x22];
const FG = [0x6c, 0xb4, 0xff];
const CRC_TABLE = (() => {
const table = new Uint32Array(256);
for (let n = 0; n < 256; n++) {
let c = n;
for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
table[n] = c >>> 0;
}
return table;
})();
function crc32(buf) {
let c = 0xffffffff;
for (const b of buf) c = CRC_TABLE[(c ^ b) & 0xff] ^ (c >>> 8);
return (c ^ 0xffffffff) >>> 0;
}
function chunk(type, data) {
const len = Buffer.alloc(4);
len.writeUInt32BE(data.length);
const typed = Buffer.concat([Buffer.from(type, 'ascii'), data]);
const crc = Buffer.alloc(4);
crc.writeUInt32BE(crc32(typed));
return Buffer.concat([len, typed, crc]);
}
function encodePng(size, rgba) {
const ihdr = Buffer.alloc(13);
ihdr.writeUInt32BE(size, 0);
ihdr.writeUInt32BE(size, 4);
ihdr[8] = 8; // bit depth
ihdr[9] = 6; // RGBA
ihdr[10] = 0;
ihdr[11] = 0;
ihdr[12] = 0;
const stride = size * 4;
const raw = Buffer.alloc((stride + 1) * size);
for (let y = 0; y < size; y++) {
raw[y * (stride + 1)] = 0; // filter: none
rgba.copy(raw, y * (stride + 1) + 1, y * stride, (y + 1) * stride);
}
return Buffer.concat([
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
chunk('IHDR', ihdr),
chunk('IDAT', deflateSync(raw, { level: 9 })),
chunk('IEND', Buffer.alloc(0)),
]);
}
function inRoundedSquare(x, y, r) {
const cx = Math.min(Math.max(x, r), 1 - r);
const cy = Math.min(Math.max(y, r), 1 - r);
return (x - cx) ** 2 + (y - cy) ** 2 <= r * r;
}
// Glyph in unit coordinates, matching icon.svg (viewBox 100): dot at (31,69) r 7.5,
// arcs of radius 24.5 and 42.5 with stroke 9 and round caps.
const C = { x: 0.31, y: 0.69 };
const DOT = 0.075;
const HALF = 0.045;
const ARCS = [0.245, 0.425];
function inGlyph(x, y, scale) {
const u = 0.5 + (x - 0.5) / scale;
const v = 0.5 + (y - 0.5) / scale;
const dx = u - C.x;
const dy = C.y - v;
const d = Math.hypot(dx, dy);
if (d <= DOT) return true;
for (const r of ARCS) {
if (dx >= 0 && dy >= 0 && Math.abs(d - r) <= HALF) return true;
// round caps at both ends of the quarter arc
if (Math.hypot(u - (C.x + r), v - C.y) <= HALF) return true;
if (Math.hypot(u - C.x, v - (C.y - r)) <= HALF) return true;
}
return false;
}
function render(size, { rounded, glyphScale }) {
const buf = Buffer.alloc(size * size * 4);
const ss = 4;
for (let y = 0; y < size; y++) {
for (let x = 0; x < size; x++) {
let bg = 0;
let fg = 0;
for (let sy = 0; sy < ss; sy++) {
for (let sx = 0; sx < ss; sx++) {
const u = (x + (sx + 0.5) / ss) / size;
const v = (y + (sy + 0.5) / ss) / size;
if (rounded && !inRoundedSquare(u, v, 0.22)) continue;
bg++;
if (inGlyph(u, v, glyphScale)) fg++;
}
}
const alpha = bg / (ss * ss);
const mix = bg ? fg / bg : 0;
const i = (y * size + x) * 4;
for (let c = 0; c < 3; c++) buf[i + c] = Math.round(BG[c] + (FG[c] - BG[c]) * mix);
buf[i + 3] = Math.round(alpha * 255);
}
}
return encodePng(size, buf);
}
const outputs = [
['icon-192.png', 192, { rounded: true, glyphScale: 1 }],
['icon-512.png', 512, { rounded: true, glyphScale: 1 }],
// Maskable: full-bleed background, glyph kept inside the 80% safe zone.
['icon-maskable-512.png', 512, { rounded: false, glyphScale: 0.78 }],
];
for (const [name, size, opts] of outputs) {
writeFileSync(path.join(outDir, name), render(size, opts));
console.log(`wrote web/public/${name}`);
}