Build RSSFeed: self-hosted Croatian news feed with a local LLM
Release / build (push) Successful in 1m25s
Release / build (push) Successful in 1m25s
Merges RSS/Atom/JSON sources into one feed. A local OpenAI-compatible LLM filters articles against plain-language negatives, groups reports of the same event into stories, and writes a one-sentence Croatian title and a description of at most two sentences for each story. Original articles are only ever linked as published. - Pipeline: fetch, dedupe, Readability extraction, keyword + AI triage with exceptions and learned corrections, embedding clustering with an LLM same-event check, summary with format check and Croatian language guard, regeneration for developing stories - Neutral importance scoring with coverage boost and manual override - Feed controls: Live or Editions, daily/category/source limits, breaking bypass, 7-day preview; daily digest with offline fallback - Circuit breaker that pauses and resumes the queue on LLM outages - Express 5 + node:sqlite server, React 19 + Vite web UI (dark, PWA) - Feeds under /feeds/ for a single Pangolin rule; optional feed token - Dockerfile runs typecheck and the full test suite before building; Gitea workflow builds, scans and pushes the image Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
1cc99df9bb
94 files changed
+19181
No files matched your search
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* One-command dev environment: backend (Express + API on :34500) and the Vite dev server
|
||||
* (:5173, proxying /api and /feeds to the backend), with prefixed output and clean shutdown.
|
||||
* npm run dev → open http://localhost:5173
|
||||
* npm run dev -- --mock → also starts the mock LLM on :11435 (no real LLM needed)
|
||||
*/
|
||||
import { spawn } from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const dataDir = process.env.RSSFEED_DATA_DIR ?? path.join(root, '.data');
|
||||
const withMock = process.argv.includes('--mock');
|
||||
|
||||
const procs = [];
|
||||
let shuttingDown = false;
|
||||
|
||||
function start(name, color, command, extraEnv = {}) {
|
||||
const child = spawn(command, { cwd: root, shell: true, env: { ...process.env, ...extraEnv } });
|
||||
const prefix = `\x1b[${color}m[${name}]\x1b[0m `;
|
||||
const pipe = (stream, out) => {
|
||||
let buf = '';
|
||||
stream.on('data', (d) => {
|
||||
buf += d.toString();
|
||||
let i;
|
||||
while ((i = buf.indexOf('\n')) >= 0) {
|
||||
out.write(prefix + buf.slice(0, i) + '\n');
|
||||
buf = buf.slice(i + 1);
|
||||
}
|
||||
});
|
||||
stream.on('end', () => {
|
||||
if (buf.length > 0) out.write(prefix + buf + '\n');
|
||||
});
|
||||
};
|
||||
pipe(child.stdout, process.stdout);
|
||||
pipe(child.stderr, process.stderr);
|
||||
child.on('exit', (code) => {
|
||||
process.stdout.write(prefix + `exited (code ${code})\n`);
|
||||
shutdown();
|
||||
});
|
||||
procs.push(child);
|
||||
}
|
||||
|
||||
function shutdown() {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
for (const p of procs) {
|
||||
try {
|
||||
if (process.platform === 'win32') spawn('taskkill', ['/pid', String(p.pid), '/T', '/F']);
|
||||
else p.kill('SIGTERM');
|
||||
} catch {
|
||||
/* already gone */
|
||||
}
|
||||
}
|
||||
setTimeout(() => process.exit(0), 500);
|
||||
}
|
||||
|
||||
process.on('SIGINT', shutdown);
|
||||
process.on('SIGTERM', shutdown);
|
||||
|
||||
console.log('RSSFeed dev — backend :34500 + web :5173' + (withMock ? ' + mock LLM :11435' : ''));
|
||||
console.log(` data dir: ${dataDir}`);
|
||||
console.log(' open → \x1b[36mhttp://localhost:5173\x1b[0m\n');
|
||||
|
||||
const serverEnv = { RSSFEED_DATA_DIR: dataDir, RSSFEED_ALLOW_PRIVATE_URLS: process.env.RSSFEED_ALLOW_PRIVATE_URLS ?? '1' };
|
||||
if (withMock) {
|
||||
serverEnv.RSSFEED_LLM_BASE_URL = 'http://127.0.0.1:11435/v1';
|
||||
start('mock-llm', '33', 'node server/src/testing/mock-llm.ts 11435');
|
||||
}
|
||||
start('server', '36', 'npm run dev --workspace server', serverEnv);
|
||||
start('web', '35', 'npm run dev --workspace web');
|
||||
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Generates the PWA PNG icons in web/public with no dependencies (zlib + a CRC32 PNG
|
||||
* encoder). Draws the same RSS-style glyph as web/public/icon.svg, anti-aliased by
|
||||
* 4×4 supersampling. Run once after changing the design: node scripts/make-icons.mjs
|
||||
*/
|
||||
import { deflateSync } from 'node:zlib';
|
||||
import { writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const outDir = path.join(root, 'web', 'public');
|
||||
|
||||
const BG = [0x14, 0x1a, 0x22];
|
||||
const FG = [0x6c, 0xb4, 0xff];
|
||||
|
||||
const CRC_TABLE = (() => {
|
||||
const table = new Uint32Array(256);
|
||||
for (let n = 0; n < 256; n++) {
|
||||
let c = n;
|
||||
for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
||||
table[n] = c >>> 0;
|
||||
}
|
||||
return table;
|
||||
})();
|
||||
|
||||
function crc32(buf) {
|
||||
let c = 0xffffffff;
|
||||
for (const b of buf) c = CRC_TABLE[(c ^ b) & 0xff] ^ (c >>> 8);
|
||||
return (c ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function chunk(type, data) {
|
||||
const len = Buffer.alloc(4);
|
||||
len.writeUInt32BE(data.length);
|
||||
const typed = Buffer.concat([Buffer.from(type, 'ascii'), data]);
|
||||
const crc = Buffer.alloc(4);
|
||||
crc.writeUInt32BE(crc32(typed));
|
||||
return Buffer.concat([len, typed, crc]);
|
||||
}
|
||||
|
||||
function encodePng(size, rgba) {
|
||||
const ihdr = Buffer.alloc(13);
|
||||
ihdr.writeUInt32BE(size, 0);
|
||||
ihdr.writeUInt32BE(size, 4);
|
||||
ihdr[8] = 8; // bit depth
|
||||
ihdr[9] = 6; // RGBA
|
||||
ihdr[10] = 0;
|
||||
ihdr[11] = 0;
|
||||
ihdr[12] = 0;
|
||||
const stride = size * 4;
|
||||
const raw = Buffer.alloc((stride + 1) * size);
|
||||
for (let y = 0; y < size; y++) {
|
||||
raw[y * (stride + 1)] = 0; // filter: none
|
||||
rgba.copy(raw, y * (stride + 1) + 1, y * stride, (y + 1) * stride);
|
||||
}
|
||||
return Buffer.concat([
|
||||
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
chunk('IHDR', ihdr),
|
||||
chunk('IDAT', deflateSync(raw, { level: 9 })),
|
||||
chunk('IEND', Buffer.alloc(0)),
|
||||
]);
|
||||
}
|
||||
|
||||
function inRoundedSquare(x, y, r) {
|
||||
const cx = Math.min(Math.max(x, r), 1 - r);
|
||||
const cy = Math.min(Math.max(y, r), 1 - r);
|
||||
return (x - cx) ** 2 + (y - cy) ** 2 <= r * r;
|
||||
}
|
||||
|
||||
// Glyph in unit coordinates, matching icon.svg (viewBox 100): dot at (31,69) r 7.5,
|
||||
// arcs of radius 24.5 and 42.5 with stroke 9 and round caps.
|
||||
const C = { x: 0.31, y: 0.69 };
|
||||
const DOT = 0.075;
|
||||
const HALF = 0.045;
|
||||
const ARCS = [0.245, 0.425];
|
||||
|
||||
function inGlyph(x, y, scale) {
|
||||
const u = 0.5 + (x - 0.5) / scale;
|
||||
const v = 0.5 + (y - 0.5) / scale;
|
||||
const dx = u - C.x;
|
||||
const dy = C.y - v;
|
||||
const d = Math.hypot(dx, dy);
|
||||
if (d <= DOT) return true;
|
||||
for (const r of ARCS) {
|
||||
if (dx >= 0 && dy >= 0 && Math.abs(d - r) <= HALF) return true;
|
||||
// round caps at both ends of the quarter arc
|
||||
if (Math.hypot(u - (C.x + r), v - C.y) <= HALF) return true;
|
||||
if (Math.hypot(u - C.x, v - (C.y - r)) <= HALF) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function render(size, { rounded, glyphScale }) {
|
||||
const buf = Buffer.alloc(size * size * 4);
|
||||
const ss = 4;
|
||||
for (let y = 0; y < size; y++) {
|
||||
for (let x = 0; x < size; x++) {
|
||||
let bg = 0;
|
||||
let fg = 0;
|
||||
for (let sy = 0; sy < ss; sy++) {
|
||||
for (let sx = 0; sx < ss; sx++) {
|
||||
const u = (x + (sx + 0.5) / ss) / size;
|
||||
const v = (y + (sy + 0.5) / ss) / size;
|
||||
if (rounded && !inRoundedSquare(u, v, 0.22)) continue;
|
||||
bg++;
|
||||
if (inGlyph(u, v, glyphScale)) fg++;
|
||||
}
|
||||
}
|
||||
const alpha = bg / (ss * ss);
|
||||
const mix = bg ? fg / bg : 0;
|
||||
const i = (y * size + x) * 4;
|
||||
for (let c = 0; c < 3; c++) buf[i + c] = Math.round(BG[c] + (FG[c] - BG[c]) * mix);
|
||||
buf[i + 3] = Math.round(alpha * 255);
|
||||
}
|
||||
}
|
||||
return encodePng(size, buf);
|
||||
}
|
||||
|
||||
const outputs = [
|
||||
['icon-192.png', 192, { rounded: true, glyphScale: 1 }],
|
||||
['icon-512.png', 512, { rounded: true, glyphScale: 1 }],
|
||||
// Maskable: full-bleed background, glyph kept inside the 80% safe zone.
|
||||
['icon-maskable-512.png', 512, { rounded: false, glyphScale: 0.78 }],
|
||||
];
|
||||
|
||||
for (const [name, size, opts] of outputs) {
|
||||
writeFileSync(path.join(outDir, name), render(size, opts));
|
||||
console.log(`wrote web/public/${name}`);
|
||||
}
|
||||
Reference in new issue
Block a user