Settings > Fetching & filtering > Quiet hours: a local window (e.g. 23:00-05:00, may run past midnight) with no scheduled fetching and no background processing, so no extraction and no AI calls. New items wait in the feeds; when the window ends the scheduler and worker pick everything up on their next tick. - Editions, digest creation and cleanup still run on time (no AI); a digest due inside the window gets its opening when it ends. The no-opening fallback after 6 hours now only applies while the LLM is actually unavailable, so a long quiet window doesn't trigger it. - Things started by hand still run: Fetch now, adding a source, the Playground, the connection test. - /api/status reports quiet hours; a banner shows until when, and how many items are waiting. The settings panel shows the window length and warns when the digest time falls inside it. - Off by default. Tests for the window (past midnight, DST) and for nothing being processed until it ends. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5dfcb9c6e2
commit
a57d88aff9
14 files changed
+201
-3
No files matched your search
@@ -161,6 +161,7 @@ Two rules hold at every stage: **no article is dropped silently**, and **no stor
|
||||
- Deduplication catches the same article seen twice. Recognizing different articles about the same event is clustering (§5.5).
|
||||
- On first add, a source imports only items from the last 24 hours, so a new source does not flood the queue with its whole backlog. A source added on a quiet day still gets its 10 newest items from within the maximum age, so it never looks empty.
|
||||
- Removing a source also forgets its article URLs, so adding the same feed again imports its current items.
|
||||
- Quiet hours (optional, e.g. 23:00–05:00 local time): no scheduled fetching and no background processing, so no AI calls. New items wait in the feeds and are fetched and processed when the window ends. Editions, digest creation and cleanup still run on time (they cost no AI), and a digest due in the window gets its opening when it ends. Things started by hand (Fetch now, Playground, connection test) still run. A banner shows when quiet hours are on.
|
||||
- Items older than a configurable maximum age (default 72 hours) at fetch time are stored but skipped by the pipeline.
|
||||
|
||||
### 5.3 Full-text extraction
|
||||
|
||||
@@ -8,6 +8,7 @@ The original articles are never edited or republished: every item links to the a
|
||||
- **Stories, not duplicates:** the same event from five outlets becomes one item listing every source, across languages.
|
||||
- **Importance 1–5** (neutral scale), **feed controls** (daily limit, editions, minimum importance, per-source and per-category caps, per-source importance, with a 7-day preview), and a **daily digest** ("Jutarnji pregled").
|
||||
- **Standard Croatian, checked:** a format check enforces the sentence limits and a language guard blocks Serbian/Bosnian forms and untranslated text.
|
||||
- **Cheap to run:** duplicates and wire copy are caught without AI, triage is batched, and optional **quiet hours** (e.g. 23:00–05:00) pause fetching and AI work overnight.
|
||||
|
||||
The full product spec is in [PRD.md](PRD.md).
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ import { DEFAULT_PROMPTS, TRIAGE_SCHEMA, buildTriagePrompt, validateTriage } fro
|
||||
import { getMeta } from '../db/schema.ts';
|
||||
import { fetchSource } from '../pipeline/fetcher.ts';
|
||||
import type { SourceRow } from '../pipeline/fetcher.ts';
|
||||
import { quietEndsAt } from '../pipeline/quiet.ts';
|
||||
import { extractArticle } from '../pipeline/extract.ts';
|
||||
import { createRule, deleteRule, evaluateTriage, getDryRun, listRules, rescueArticle, startDryRun, updateRule } from '../pipeline/triage.ts';
|
||||
import { listExamples } from '../pipeline/examples.ts';
|
||||
@@ -385,7 +386,10 @@ export function apiRouter(ctx: Ctx): express.Router {
|
||||
});
|
||||
|
||||
r.get('/status', (_req, res: Response<StatusDTO>) => {
|
||||
const q = ctx.settings.get().pipeline.quietHours;
|
||||
const endsAt = quietEndsAt(q);
|
||||
res.json({
|
||||
quiet: endsAt ? { until: q.end, endsAt: endsAt.toISOString() } : null,
|
||||
llm: ctx.llm.state(),
|
||||
llmError: ctx.llm.errorMessage(),
|
||||
waiting: waitingCount(ctx),
|
||||
|
||||
@@ -150,6 +150,8 @@ function publish(ctx: Ctx, id: number, opening: string | null, kind: 'llm' | 'fa
|
||||
|
||||
/** Publishes pending digests without an opening once the LLM has been unreachable for 6 hours. */
|
||||
export function publishOverdueFallbacks(ctx: Ctx, now: Date = new Date()): void {
|
||||
// With the LLM up, a waiting digest is written soon: it was only held by quiet hours or a busy queue.
|
||||
if (ctx.llm.available()) return;
|
||||
const cutoff = new Date(now.getTime() - 6 * HOUR).toISOString();
|
||||
const overdue = ctx.db.prepare(`SELECT id FROM digest WHERE state = 'pending' AND created_at <= ?`).all<{ id: number }>(cutoff);
|
||||
for (const d of overdue) {
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { setTimezone } from '../lib/time.ts';
|
||||
import { isQuiet, quietEndsAt } from './quiet.ts';
|
||||
|
||||
const night = { enabled: true, start: '23:00', end: '05:00' };
|
||||
|
||||
test('quiet hours: a window past midnight, its end, and off states', () => {
|
||||
setTimezone('Europe/Zagreb'); // UTC+1 in November
|
||||
const at = (iso: string) => new Date(iso);
|
||||
assert.equal(isQuiet(night, at('2026-11-15T21:59:00Z')), false); // 22:59
|
||||
assert.equal(isQuiet(night, at('2026-11-15T22:00:00Z')), true); // 23:00
|
||||
assert.equal(isQuiet(night, at('2026-11-16T03:59:00Z')), true); // 04:59
|
||||
assert.equal(isQuiet(night, at('2026-11-16T04:00:00Z')), false); // 05:00
|
||||
// Before midnight it ends tomorrow; after midnight, today.
|
||||
assert.equal(quietEndsAt(night, at('2026-11-15T22:30:00Z'))?.toISOString(), '2026-11-16T04:00:00.000Z');
|
||||
assert.equal(quietEndsAt(night, at('2026-11-16T01:00:00Z'))?.toISOString(), '2026-11-16T04:00:00.000Z');
|
||||
assert.equal(quietEndsAt(night, at('2026-11-15T12:00:00Z')), null);
|
||||
// A window inside one day.
|
||||
const early = { enabled: true, start: '01:00', end: '06:00' };
|
||||
assert.equal(isQuiet(early, at('2026-11-15T23:30:00Z')), false); // 00:30
|
||||
assert.equal(isQuiet(early, at('2026-11-16T02:00:00Z')), true); // 03:00
|
||||
// Off, or an empty window.
|
||||
assert.equal(isQuiet({ ...night, enabled: false }, at('2026-11-16T01:00:00Z')), false);
|
||||
assert.equal(isQuiet({ enabled: true, start: '02:00', end: '02:00' }, at('2026-11-16T01:00:00Z')), false);
|
||||
// Across the spring DST change (02:00 → 03:00 on 29 March 2026), the end is still 05:00 local.
|
||||
assert.equal(quietEndsAt(night, at('2026-03-29T00:30:00Z'))?.toISOString(), '2026-03-29T03:00:00.000Z');
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
import type { QuietHours } from '../../../shared/types.ts';
|
||||
import { DAY, localDay, localHM, localToUtc } from '../lib/time.ts';
|
||||
|
||||
/**
|
||||
* Quiet hours: no scheduled fetching and no background processing. Work simply waits in the
|
||||
* database and the worker and scheduler pick it up on their next tick after the window ends.
|
||||
*/
|
||||
export function isQuiet(q: QuietHours, now: Date = new Date()): boolean {
|
||||
if (!q.enabled || q.start === q.end) return false;
|
||||
const hm = localHM(now);
|
||||
// 23:00–05:00 runs past midnight; 01:00–06:00 does not.
|
||||
return q.start < q.end ? hm >= q.start && hm < q.end : hm >= q.start || hm < q.end;
|
||||
}
|
||||
|
||||
/** When the current quiet window ends, or null outside it. */
|
||||
export function quietEndsAt(q: QuietHours, now: Date = new Date()): Date | null {
|
||||
if (!isQuiet(q, now)) return null;
|
||||
const today = localDay(now);
|
||||
const day = localHM(now) < q.end ? today : localDay(new Date(localToUtc(today, '12:00').getTime() + DAY));
|
||||
return localToUtc(day, q.end);
|
||||
}
|
||||
@@ -6,12 +6,14 @@ import { fetchDueSources } from './fetcher.ts';
|
||||
import { releaseDueEdition, sweepQueuedInLiveMode } from './feedcontrol.ts';
|
||||
import { maybeCreateDigest, publishOverdueFallbacks } from './digest.ts';
|
||||
import { runRetention } from './maintenance.ts';
|
||||
import { quietEndsAt } from './quiet.ts';
|
||||
|
||||
/** Ticks every 30 s: polls due sources, releases editions, builds the digest, runs cleanup. */
|
||||
/** Ticks every 30 s: polls due sources (outside quiet hours), releases editions, builds the digest, runs cleanup. */
|
||||
export class Scheduler {
|
||||
private readonly ctx: Ctx;
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
private fetching = false;
|
||||
private quiet = false;
|
||||
|
||||
constructor(ctx: Ctx) {
|
||||
this.ctx = ctx;
|
||||
@@ -48,6 +50,15 @@ export class Scheduler {
|
||||
log.error(`scheduler step failed: ${name}`, { error: err });
|
||||
}
|
||||
}
|
||||
// Editions, digest creation and cleanup cost no AI and keep running; fetching waits.
|
||||
const q = this.ctx.settings.get().pipeline.quietHours;
|
||||
const endsAt = quietEndsAt(q, now);
|
||||
if (Boolean(endsAt) !== this.quiet) {
|
||||
this.quiet = Boolean(endsAt);
|
||||
if (endsAt) log.info('quiet hours: fetching and processing paused', { until: q.end });
|
||||
else log.info('quiet hours over: fetching and processing resume');
|
||||
}
|
||||
if (this.quiet) return;
|
||||
if (!this.fetching) {
|
||||
this.fetching = true;
|
||||
try {
|
||||
|
||||
@@ -8,6 +8,7 @@ import { TRIAGE_BATCH_SIZE, triageBatch, triageNeedsLlm } from './triage.ts';
|
||||
import { clusterArticle, clusteringActive } from './cluster.ts';
|
||||
import { PermanentError, summarizeStory } from './summarize.ts';
|
||||
import { nextPendingDigest, processDigest } from './digest.ts';
|
||||
import { isQuiet } from './quiet.ts';
|
||||
import type { ArticleRow } from './rows.ts';
|
||||
|
||||
const EXTRACT_SLOTS = 3;
|
||||
@@ -65,6 +66,8 @@ export class Worker {
|
||||
|
||||
pump(): void {
|
||||
if (this.stopped || this.pumping) return;
|
||||
// Quiet hours: start nothing new (tasks in flight finish). The 5 s timer resumes work after.
|
||||
if (isQuiet(this.ctx.settings.get().pipeline.quietHours)) return;
|
||||
this.pumping = true;
|
||||
try {
|
||||
while (this.active.extract < EXTRACT_SLOTS) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import type { RunningApp } from './server.ts';
|
||||
import { startMockLlm } from './testing/mock-llm.ts';
|
||||
import type { MockLlm } from './testing/mock-llm.ts';
|
||||
import { eventStems } from './pipeline/cluster.ts';
|
||||
import { HOUR, localHM } from './lib/time.ts';
|
||||
import type { ActivityDTO, StatusDTO, StoryListResponse } from '../../shared/types.ts';
|
||||
|
||||
/** Everything that keeps AI calls down: duplicate ledger, free grouping, pre-filter, batching, pausing on bad config. */
|
||||
@@ -181,3 +182,31 @@ test('a misconfigured model pauses LLM work instead of failing every article', {
|
||||
await h.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('quiet hours: nothing is extracted or sent to the AI until they end, then it catches up', { timeout: 60_000 }, async () => {
|
||||
const h = await harness({ '/one.xml': ['/a/quake', '/a/bank'] });
|
||||
try {
|
||||
const now = Date.now();
|
||||
const quietHours = { enabled: true, start: localHM(new Date(now - HOUR)), end: localHM(new Date(now + HOUR)) };
|
||||
await h.call('PUT', '/api/settings', { llm: { baseUrl: h.mock.url, triageModel: 'mock-small', summaryModel: 'mock-large', embeddingModel: '' }, pipeline: { quietHours } });
|
||||
const st = await h.call<StatusDTO>('GET', '/api/status');
|
||||
assert.equal(st.quiet?.until, quietHours.end);
|
||||
|
||||
// Adding a source by hand still fetches it; the articles then wait.
|
||||
await h.call('POST', '/api/sources', { feedUrl: `${h.base}/one.xml`, name: 'One' });
|
||||
for (let i = 0; i < 100; i++) {
|
||||
if ((await h.call<{ lastFetchAt: string | null }[]>('GET', '/api/sources'))[0]?.lastFetchAt) break;
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
assert.equal((await h.call<ActivityDTO>('GET', '/api/activity')).queue.articlesNew, 2, 'not extracted during quiet hours');
|
||||
assert.equal(chatCalls(h.mock), 0, 'no AI calls during quiet hours');
|
||||
|
||||
await h.call('PUT', '/api/settings', { pipeline: { quietHours: { enabled: false } } });
|
||||
assert.equal((await h.call<StatusDTO>('GET', '/api/status')).quiet, null);
|
||||
await h.idle();
|
||||
assert.equal((await h.call<StoryListResponse>('GET', '/api/stories?view=all')).total, 2);
|
||||
} finally {
|
||||
await h.close();
|
||||
}
|
||||
});
|
||||
+12
-1
@@ -1,5 +1,5 @@
|
||||
import { CATEGORIES, PROMPT_KEYS, REASONING_EFFORTS } from '../../shared/types.ts';
|
||||
import type { Category, CategoryControl, DeepPartial, FeedControls, Settings, SourceControl } from '../../shared/types.ts';
|
||||
import type { Category, CategoryControl, DeepPartial, FeedControls, QuietHours, Settings, SourceControl } from '../../shared/types.ts';
|
||||
import type { Config } from './config.ts';
|
||||
import type { Database } from './db/database.ts';
|
||||
import { isValidHM } from './lib/time.ts';
|
||||
@@ -52,6 +52,7 @@ export function defaultSettings(): Settings {
|
||||
firstAddHours: 24,
|
||||
rejectThreshold: 0.7,
|
||||
examplesInPrompt: 10,
|
||||
quietHours: { enabled: false, start: '23:00', end: '05:00' },
|
||||
},
|
||||
clustering: {
|
||||
enabled: true,
|
||||
@@ -102,6 +103,15 @@ function oneOf<T extends string>(v: unknown, options: readonly T[], fallback: T)
|
||||
return options.includes(v as T) ? (v as T) : fallback;
|
||||
}
|
||||
|
||||
function sanitizeQuietHours(v: unknown, d: QuietHours): QuietHours {
|
||||
const q = isPlainObject(v) ? v : {};
|
||||
const hm = (x: unknown, fallback: string) => {
|
||||
const t = str(x, '');
|
||||
return isValidHM(t) ? t : fallback;
|
||||
};
|
||||
return { enabled: bool(q.enabled, d.enabled), start: hm(q.start, d.start), end: hm(q.end, d.end) };
|
||||
}
|
||||
|
||||
/** Coerces any stored or submitted settings object into a valid Settings. */
|
||||
export function sanitizeSettings(input: unknown): Settings {
|
||||
const d = defaultSettings();
|
||||
@@ -126,6 +136,7 @@ export function sanitizeSettings(input: unknown): Settings {
|
||||
firstAddHours: clampInt(s.pipeline.firstAddHours, 1, 24 * 30, d.pipeline.firstAddHours),
|
||||
rejectThreshold: clamp(s.pipeline.rejectThreshold, 0, 1, d.pipeline.rejectThreshold),
|
||||
examplesInPrompt: clampInt(s.pipeline.examplesInPrompt, 0, 50, d.pipeline.examplesInPrompt),
|
||||
quietHours: sanitizeQuietHours(s.pipeline.quietHours, d.pipeline.quietHours),
|
||||
},
|
||||
clustering: {
|
||||
enabled: bool(s.clustering.enabled, true),
|
||||
|
||||
@@ -384,6 +384,18 @@ export interface PipelineSettings {
|
||||
rejectThreshold: number;
|
||||
/** Most recent corrections included in prompts. */
|
||||
examplesInPrompt: number;
|
||||
quietHours: QuietHours;
|
||||
}
|
||||
|
||||
/**
|
||||
* A nightly window with no scheduled fetching and no background processing (extraction or AI).
|
||||
* Everything waits and catches up when it ends; things you start by hand still run.
|
||||
*/
|
||||
export interface QuietHours {
|
||||
enabled: boolean;
|
||||
/** Local HH:MM. When end is earlier than start, the window runs past midnight. */
|
||||
start: string;
|
||||
end: string;
|
||||
}
|
||||
|
||||
export interface ClusteringSettings {
|
||||
@@ -666,6 +678,8 @@ export interface StatusDTO {
|
||||
version: string;
|
||||
latestDigestId: number | null;
|
||||
feedMode: 'live' | 'editions';
|
||||
/** Set during quiet hours: the local end time (HH:MM) and the same moment as ISO. */
|
||||
quiet: { until: string; endsAt: string } | null;
|
||||
}
|
||||
|
||||
export interface HealthDTO {
|
||||
|
||||
@@ -75,6 +75,19 @@ function StatusBanner() {
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (status.quiet) {
|
||||
return (
|
||||
<div className="banner banner-info" role="status">
|
||||
<div className="banner__inner">
|
||||
<strong>Quiet hours</strong>
|
||||
<span>
|
||||
No fetching or processing until {status.quiet.until}.{status.waiting > 0 ? ` ${plural(status.waiting, 'item')} waiting.` : ''}
|
||||
</span>
|
||||
<Link to="/settings/pipeline">Change</Link>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import type {
|
||||
GlossaryEntry,
|
||||
ImportanceSettings,
|
||||
PipelineSettings,
|
||||
QuietHours,
|
||||
RetentionSettings,
|
||||
} from '../../contract.ts';
|
||||
import { useDraft } from '../../hooks.ts';
|
||||
@@ -207,6 +208,16 @@ export function PipelineSection({ data, onSaved }: SectionProps) {
|
||||
const { draft, setDraft, dirty, reset } = useDraft<PipelineSettings>(data.settings.pipeline);
|
||||
const { saving, save } = useSaver(onSaved);
|
||||
const set = <K extends keyof PipelineSettings>(k: K, v: PipelineSettings[K]) => setDraft((d) => ({ ...d, [k]: v }));
|
||||
const q = draft.quietHours;
|
||||
const setQuiet = (patch: Partial<QuietHours>) => setDraft((d) => ({ ...d, quietHours: { ...d.quietHours, ...patch } }));
|
||||
const digest = data.settings.digest;
|
||||
const quietProblem = !q.enabled
|
||||
? null
|
||||
: !HM.test(q.start) || !HM.test(q.end)
|
||||
? 'Quiet hours need a start and an end time.'
|
||||
: q.start === q.end
|
||||
? 'Quiet hours start and end at the same time.'
|
||||
: null;
|
||||
|
||||
return (
|
||||
<div className="settings-section">
|
||||
@@ -219,6 +230,29 @@ export function PipelineSection({ data, onSaved }: SectionProps) {
|
||||
<NumberField label="New source imports" suffix="hours" hint="A new source imports items this recent, plus older ones within the maximum age until it has 10, so a source added on a quiet day isn't empty." min={1} max={168} value={draft.firstAddHours} onChange={(v) => set('firstAddHours', Math.round(v))} />
|
||||
</div>
|
||||
</fieldset>
|
||||
<fieldset className="panel">
|
||||
<legend className="panel__title">Quiet hours</legend>
|
||||
<Checkbox
|
||||
label="Pause every night"
|
||||
hint="No fetching and no AI work in this window. New items wait in the feeds and are fetched and processed when it ends. Fetch now, the Playground and the connection test still work."
|
||||
checked={q.enabled}
|
||||
onChange={(v) => setQuiet({ enabled: v })}
|
||||
/>
|
||||
<div className="form-grid">
|
||||
<TextField label="From" type="time" value={q.start} onChange={(v) => setQuiet({ start: v })} />
|
||||
<TextField label="Until" type="time" value={q.end} onChange={(v) => setQuiet({ end: v })} hint={`Local time (${data.timezone}).`} />
|
||||
</div>
|
||||
{q.enabled && !quietProblem && (
|
||||
<p className="faint small">
|
||||
Paused {q.start}–{q.end} every day, {fmtHours(quietMinutes(q.start, q.end))}.
|
||||
</p>
|
||||
)}
|
||||
{q.enabled && !quietProblem && digest.enabled && inWindow(digest.time, q.start, q.end) && (
|
||||
<p className="note note-info small">
|
||||
The daily digest ({digest.time}) falls inside quiet hours, so it's written when they end at {q.end}.
|
||||
</p>
|
||||
)}
|
||||
</fieldset>
|
||||
<fieldset className="panel">
|
||||
<legend className="panel__title">Filtering</legend>
|
||||
<label className="field">
|
||||
@@ -244,11 +278,29 @@ export function PipelineSection({ data, onSaved }: SectionProps) {
|
||||
onChange={(v) => set('examplesInPrompt', Math.round(v))}
|
||||
/>
|
||||
</fieldset>
|
||||
<SaveBar dirty={dirty} saving={saving} onDiscard={reset} onSave={() => void save({ pipeline: draft })} />
|
||||
<SaveBar dirty={dirty} saving={saving} problem={quietProblem} onDiscard={reset} onSave={() => void save({ pipeline: draft })} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const HM = /^([01]\d|2[0-3]):[0-5]\d$/;
|
||||
const toMinutes = (hm: string) => Number(hm.slice(0, 2)) * 60 + Number(hm.slice(3, 5));
|
||||
|
||||
/** Length of a nightly window; one that ends earlier than it starts runs past midnight. */
|
||||
function quietMinutes(start: string, end: string): number {
|
||||
return (toMinutes(end) - toMinutes(start) + 1440) % 1440;
|
||||
}
|
||||
|
||||
function inWindow(hm: string, start: string, end: string): boolean {
|
||||
return start < end ? hm >= start && hm < end : hm >= start || hm < end;
|
||||
}
|
||||
|
||||
function fmtHours(minutes: number): string {
|
||||
const h = Math.floor(minutes / 60);
|
||||
const m = minutes % 60;
|
||||
return [h ? `${h} hour${h === 1 ? '' : 's'}` : '', m ? `${m} minutes` : ''].filter(Boolean).join(' ');
|
||||
}
|
||||
|
||||
// ───────────────────────────── Retention ─────────────────────────────
|
||||
|
||||
export function RetentionSection({ data, onSaved }: SectionProps) {
|
||||
|
||||
@@ -261,6 +261,14 @@ img {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.banner-info {
|
||||
background: var(--accent-soft);
|
||||
}
|
||||
|
||||
.banner-info strong {
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.page-header {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
|
||||
Reference in new issue
Block a user