diff --git a/src/client/src/routes/+page.svelte b/src/client/src/routes/+page.svelte
index 2aa3b33..dec6c13 100644
--- a/src/client/src/routes/+page.svelte
+++ b/src/client/src/routes/+page.svelte
@@ -12,9 +12,9 @@
onMount(async () => {
try {
[categories, stores] = await Promise.all([getCategories(), getStores()]);
- // Start with all enabled stores selected
- const enabledIds = stores.filter(s => s.enabled).map(s => s.id);
- selectedStoreIds = new Set(enabledIds);
+ // Start with all enabled & healthy stores selected (exclude broken ones)
+ const healthyIds = stores.filter(s => s.enabled && s.last_scrape_ok !== 0).map(s => s.id);
+ selectedStoreIds = new Set(healthyIds);
} catch { /* server may not be ready yet */ }
});
@@ -156,21 +156,38 @@
{#each group.stores as store}
+ {@const isBroken = store.last_scrape_ok === 0}
{/each}
diff --git a/src/server/db/migrations/004_add_last_scrape_status.sql b/src/server/db/migrations/004_add_last_scrape_status.sql
new file mode 100644
index 0000000..6c23398
--- /dev/null
+++ b/src/server/db/migrations/004_add_last_scrape_status.sql
@@ -0,0 +1,2 @@
+ALTER TABLE stores ADD COLUMN last_scrape_ok INTEGER DEFAULT NULL;
+ALTER TABLE stores ADD COLUMN last_scrape_at TEXT DEFAULT NULL;
diff --git a/src/server/models/store.ts b/src/server/models/store.ts
index e6889e7..2b32637 100644
--- a/src/server/models/store.ts
+++ b/src/server/models/store.ts
@@ -9,6 +9,8 @@ export interface Store {
enabled: number;
render_js: number;
test_query: string;
+ last_scrape_ok: number | null;
+ last_scrape_at: string | null;
sel_container: string;
sel_name: string;
sel_price: string;
@@ -163,6 +165,12 @@ export function updateStore(id: number, input: Partial): Store
return getStoreById(id);
}
+export function updateScrapeStatus(id: number, success: boolean): void {
+ const db = getDatabase();
+ db.run("UPDATE stores SET last_scrape_ok = ?, last_scrape_at = datetime('now') WHERE id = ?", [success ? 1 : 0, id]);
+ saveDatabase();
+}
+
export function toggleStoreEnabled(id: number): Store | undefined {
const db = getDatabase();
db.run("UPDATE stores SET enabled = CASE WHEN enabled = 1 THEN 0 ELSE 1 END, updated_at = datetime('now') WHERE id = ?", [id]);
diff --git a/src/server/routes/test.ts b/src/server/routes/test.ts
index e12a3ca..61b1f88 100644
--- a/src/server/routes/test.ts
+++ b/src/server/routes/test.ts
@@ -1,5 +1,5 @@
import type { FastifyPluginAsync } from 'fastify';
-import { getStoreById } from '../models/store.js';
+import { getStoreById, updateScrapeStatus } from '../models/store.js';
import { logScrape, getLogsByStore, getStoreHealth } from '../models/scrape-log.js';
import { scrapeStore } from '../scraper/http-scraper.js';
import { scrapeStoreWithBrowser } from '../scraper/browser-scraper.js';
@@ -37,6 +37,7 @@ export const testRoutes: FastifyPluginAsync = async (app) => {
);
logScrape(store.id, request.body.query, true, products.length, duration);
+ updateScrapeStatus(store.id, true);
return {
success: true,
@@ -55,6 +56,7 @@ export const testRoutes: FastifyPluginAsync = async (app) => {
const duration = Date.now() - startTime;
const errorMessage = err instanceof Error ? err.message : String(err);
logScrape(store.id, request.body.query, false, 0, duration, errorMessage);
+ updateScrapeStatus(store.id, false);
return {
success: false,
diff --git a/src/server/scraper/engine.ts b/src/server/scraper/engine.ts
index 08a3bfc..08e93b7 100644
--- a/src/server/scraper/engine.ts
+++ b/src/server/scraper/engine.ts
@@ -1,6 +1,6 @@
import pLimit from 'p-limit';
import type { Store } from '../models/store.js';
-import { getEnabledStores, getStoresByCategory, getStoresByGroup, getStoresByIds } from '../models/store.js';
+import { getEnabledStores, getStoresByCategory, getStoresByGroup, getStoresByIds, updateScrapeStatus } from '../models/store.js';
import { logScrape } from '../models/scrape-log.js';
import { scrapeStore } from './http-scraper.js';
import { scrapeStoreWithBrowser } from './browser-scraper.js';
@@ -90,11 +90,13 @@ export async function search(options: SearchOptions): Promise {
);
logScrape(store.id, query, true, products.length, duration);
+ updateScrapeStatus(store.id, true);
return products;
} catch (err) {
const duration = Date.now() - storeStart;
const errorMessage = err instanceof Error ? err.message : String(err);
logScrape(store.id, query, false, 0, duration, errorMessage);
+ updateScrapeStatus(store.id, false);
errors.push({ storeId: store.id, storeName: store.name, error: errorMessage });
return [];
}
@@ -188,6 +190,7 @@ export async function searchStreaming(
);
logScrape(store.id, query, true, products.length, duration);
+ updateScrapeStatus(store.id, true);
totalResults += products.length;
onProgress({
@@ -202,6 +205,7 @@ export async function searchStreaming(
const duration = Date.now() - storeStart;
const errorMessage = err instanceof Error ? err.message : String(err);
logScrape(store.id, query, false, 0, duration, errorMessage);
+ updateScrapeStatus(store.id, false);
errors.push({ storeId: store.id, storeName: store.name, error: errorMessage });
onProgress({