msemes/PriceHunter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Flag broken stores on search page with red X and auto-deselect

Browse Source 
- Add last_scrape_ok and last_scrape_at columns (migration 004)
- Update scrape status after every search, test, and streaming search
- Search page: broken stores show red X checkbox, strikethrough name,
  "failing" label, and are auto-deselected on page load
- Untested stores show "untested" label
- Users can still manually select broken stores if they want to try

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
mariosemes
2026-03-26 23:12:44 +01:00
parent cb71421d8d
commit 72980e0dd6
5 changed files with 43 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/server/scraper/engine.ts
View File

@@ -1,6 +1,6 @@
import pLimit from 'p-limit';
import type { Store } from '../models/store.js';
import { getEnabledStores, getStoresByCategory, getStoresByGroup, getStoresByIds } from '../models/store.js';
import { getEnabledStores, getStoresByCategory, getStoresByGroup, getStoresByIds, updateScrapeStatus } from '../models/store.js';
import { logScrape } from '../models/scrape-log.js';
import { scrapeStore } from './http-scraper.js';
import { scrapeStoreWithBrowser } from './browser-scraper.js';
@@ -90,11 +90,13 @@ export async function search(options: SearchOptions): Promise<SearchResult> {
);
logScrape(store.id, query, true, products.length, duration);
updateScrapeStatus(store.id, true);
return products;
} catch (err) {
const duration = Date.now() - storeStart;
const errorMessage = err instanceof Error ? err.message : String(err);
logScrape(store.id, query, false, 0, duration, errorMessage);
updateScrapeStatus(store.id, false);
errors.push({ storeId: store.id, storeName: store.name, error: errorMessage });
return [];
}
@@ -188,6 +190,7 @@ export async function searchStreaming(
);
logScrape(store.id, query, true, products.length, duration);
updateScrapeStatus(store.id, true);
totalResults += products.length;
onProgress({
@@ -202,6 +205,7 @@ export async function searchStreaming(
const duration = Date.now() - storeStart;
const errorMessage = err instanceof Error ? err.message : String(err);
logScrape(store.id, query, false, 0, duration, errorMessage);
updateScrapeStatus(store.id, false);
errors.push({ storeId: store.id, storeName: store.name, error: errorMessage });
onProgress({