From 72980e0dd6ac71e73d7b745ea1c8918dec8f3c0f Mon Sep 17 00:00:00 2001 From: mariosemes Date: Thu, 26 Mar 2026 23:12:44 +0100 Subject: [PATCH] Flag broken stores on search page with red X and auto-deselect - Add last_scrape_ok and last_scrape_at columns (migration 004) - Update scrape status after every search, test, and streaming search - Search page: broken stores show red X checkbox, strikethrough name, "failing" label, and are auto-deselected on page load - Untested stores show "untested" label - Users can still manually select broken stores if they want to try Co-Authored-By: Claude Opus 4.6 (1M context) --- src/client/src/routes/+page.svelte | 33 ++++++++++++++----- .../migrations/004_add_last_scrape_status.sql | 2 ++ src/server/models/store.ts | 8 +++++ src/server/routes/test.ts | 4 ++- src/server/scraper/engine.ts | 6 +++- 5 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 src/server/db/migrations/004_add_last_scrape_status.sql diff --git a/src/client/src/routes/+page.svelte b/src/client/src/routes/+page.svelte index 2aa3b33..dec6c13 100644 --- a/src/client/src/routes/+page.svelte +++ b/src/client/src/routes/+page.svelte @@ -12,9 +12,9 @@ onMount(async () => { try { [categories, stores] = await Promise.all([getCategories(), getStores()]); - // Start with all enabled stores selected - const enabledIds = stores.filter(s => s.enabled).map(s => s.id); - selectedStoreIds = new Set(enabledIds); + // Start with all enabled & healthy stores selected (exclude broken ones) + const healthyIds = stores.filter(s => s.enabled && s.last_scrape_ok !== 0).map(s => s.id); + selectedStoreIds = new Set(healthyIds); } catch { /* server may not be ready yet */ } }); @@ -156,21 +156,38 @@
{#each group.stores as store} + {@const isBroken = store.last_scrape_ok === 0} {/each}
diff --git a/src/server/db/migrations/004_add_last_scrape_status.sql b/src/server/db/migrations/004_add_last_scrape_status.sql new file mode 100644 index 0000000..6c23398 --- /dev/null +++ b/src/server/db/migrations/004_add_last_scrape_status.sql @@ -0,0 +1,2 @@ +ALTER TABLE stores ADD COLUMN last_scrape_ok INTEGER DEFAULT NULL; +ALTER TABLE stores ADD COLUMN last_scrape_at TEXT DEFAULT NULL; diff --git a/src/server/models/store.ts b/src/server/models/store.ts index e6889e7..2b32637 100644 --- a/src/server/models/store.ts +++ b/src/server/models/store.ts @@ -9,6 +9,8 @@ export interface Store { enabled: number; render_js: number; test_query: string; + last_scrape_ok: number | null; + last_scrape_at: string | null; sel_container: string; sel_name: string; sel_price: string; @@ -163,6 +165,12 @@ export function updateStore(id: number, input: Partial): Store return getStoreById(id); } +export function updateScrapeStatus(id: number, success: boolean): void { + const db = getDatabase(); + db.run("UPDATE stores SET last_scrape_ok = ?, last_scrape_at = datetime('now') WHERE id = ?", [success ? 1 : 0, id]); + saveDatabase(); +} + export function toggleStoreEnabled(id: number): Store | undefined { const db = getDatabase(); db.run("UPDATE stores SET enabled = CASE WHEN enabled = 1 THEN 0 ELSE 1 END, updated_at = datetime('now') WHERE id = ?", [id]); diff --git a/src/server/routes/test.ts b/src/server/routes/test.ts index e12a3ca..61b1f88 100644 --- a/src/server/routes/test.ts +++ b/src/server/routes/test.ts @@ -1,5 +1,5 @@ import type { FastifyPluginAsync } from 'fastify'; -import { getStoreById } from '../models/store.js'; +import { getStoreById, updateScrapeStatus } from '../models/store.js'; import { logScrape, getLogsByStore, getStoreHealth } from '../models/scrape-log.js'; import { scrapeStore } from '../scraper/http-scraper.js'; import { scrapeStoreWithBrowser } from '../scraper/browser-scraper.js'; @@ -37,6 +37,7 @@ export const testRoutes: FastifyPluginAsync = async (app) => { ); logScrape(store.id, request.body.query, true, products.length, duration); + updateScrapeStatus(store.id, true); return { success: true, @@ -55,6 +56,7 @@ export const testRoutes: FastifyPluginAsync = async (app) => { const duration = Date.now() - startTime; const errorMessage = err instanceof Error ? err.message : String(err); logScrape(store.id, request.body.query, false, 0, duration, errorMessage); + updateScrapeStatus(store.id, false); return { success: false, diff --git a/src/server/scraper/engine.ts b/src/server/scraper/engine.ts index 08a3bfc..08e93b7 100644 --- a/src/server/scraper/engine.ts +++ b/src/server/scraper/engine.ts @@ -1,6 +1,6 @@ import pLimit from 'p-limit'; import type { Store } from '../models/store.js'; -import { getEnabledStores, getStoresByCategory, getStoresByGroup, getStoresByIds } from '../models/store.js'; +import { getEnabledStores, getStoresByCategory, getStoresByGroup, getStoresByIds, updateScrapeStatus } from '../models/store.js'; import { logScrape } from '../models/scrape-log.js'; import { scrapeStore } from './http-scraper.js'; import { scrapeStoreWithBrowser } from './browser-scraper.js'; @@ -90,11 +90,13 @@ export async function search(options: SearchOptions): Promise { ); logScrape(store.id, query, true, products.length, duration); + updateScrapeStatus(store.id, true); return products; } catch (err) { const duration = Date.now() - storeStart; const errorMessage = err instanceof Error ? err.message : String(err); logScrape(store.id, query, false, 0, duration, errorMessage); + updateScrapeStatus(store.id, false); errors.push({ storeId: store.id, storeName: store.name, error: errorMessage }); return []; } @@ -188,6 +190,7 @@ export async function searchStreaming( ); logScrape(store.id, query, true, products.length, duration); + updateScrapeStatus(store.id, true); totalResults += products.length; onProgress({ @@ -202,6 +205,7 @@ export async function searchStreaming( const duration = Date.now() - storeStart; const errorMessage = err instanceof Error ? err.message : String(err); logScrape(store.id, query, false, 0, duration, errorMessage); + updateScrapeStatus(store.id, false); errors.push({ storeId: store.id, storeName: store.name, error: errorMessage }); onProgress({