msemes/PriceHunter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add real-time per-store search progress via SSE streaming

Browse Source 
Backend: new /api/search/stream SSE endpoint that emits events
as each store completes: start (store list), store_complete
(results + duration), store_error, and done (final meta).

Frontend: results page now shows live progress per store with
spinning indicators while searching, checkmarks when done, and
X marks on errors. Each store chip shows product count and
response time. Results stream into the table as stores complete
instead of waiting for all stores to finish.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
mariosemes
2026-03-26 22:15:50 +01:00
parent fe56c3b17e
commit 37425812e0
4 changed files with 322 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/client/src/lib/api.ts
View File

@@ -94,6 +94,34 @@ export function searchProducts(query: string, params?: { stores?: string; catego
return api<any>(`/api/search?${searchParams}`);
}
export function searchProductsStream(
query: string,
params: { stores?: string; category?: string; group?: string } | undefined,
onEvent: (event: any) => void,
): () => void {
const searchParams = new URLSearchParams({ q: query });
if (params?.stores) searchParams.set('stores', params.stores);
if (params?.category) searchParams.set('category', params.category);
if (params?.group) searchParams.set('group', params.group);
const evtSource = new EventSource(`/api/search/stream?${searchParams}`);
evtSource.onmessage = (e) => {
try {
const event = JSON.parse(e.data);
onEvent(event);
if (event.type === 'done') evtSource.close();
} catch { /* ignore parse errors */ }
};
evtSource.onerror = () => {
evtSource.close();
};
// Return cleanup function
return () => evtSource.close();
}
export function testStore(id: number, query: string) {
return api<any>(`/api/stores/${id}/test`, { method: 'POST', body: JSON.stringify({ query }) });
}

210
src/client/src/routes/results/+page.svelte
View File

@@ -1,7 +1,7 @@
<script>
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import { searchProducts } from '$lib/api';
import { searchProductsStream } from '$lib/api';
import { onMount } from 'svelte';
let query = $state('');
@@ -13,38 +13,73 @@
let filterText = $state('');
let excludedStores = $state(new Set());
// Per-store progress tracking
let storeProgress = $state([]);
let searchDone = $state(false);
let cleanupStream = null;
onMount(async () => {
const params = $page.url.searchParams;
query = params.get('q') || '';
if (!query) { goto('/'); return; }
await doSearch();
doSearch();
return () => { if (cleanupStream) cleanupStream(); };
});
async function doSearch() {
function doSearch() {
if (!query.trim()) return;
loading = true;
try {
searchDone = false;
results = [];
meta = null;
storeProgress = [];
const params = $page.url.searchParams;
const data = await searchProducts(query, {
cleanupStream = searchProductsStream(query, {
category: params.get('category') || undefined,
group: params.get('group') || undefined,
stores: params.get('stores') || undefined,
});
results = data.results;
meta = data.meta;
} catch (err) {
console.error('Search failed:', err);
results = [];
} finally {
}, (event) => {
switch (event.type) {
case 'start':
storeProgress = event.stores.map((s) => ({
id: s.id,
name: s.name,
renderJs: s.renderJs,
status: 'searching',
resultCount: 0,
duration: 0,
error: null,
}));
loading = false;
}
}
break;
function handleSearch(e) {
e.preventDefault();
if (!query.trim()) return;
goto(`/results?q=${encodeURIComponent(query.trim())}`);
doSearch();
case 'store_complete':
results = [...results, ...event.results];
storeProgress = storeProgress.map((s) =>
s.id === event.storeId
? { ...s, status: 'done', resultCount: event.resultCount, duration: event.duration }
: s
);
break;
case 'store_error':
storeProgress = storeProgress.map((s) =>
s.id === event.storeId
? { ...s, status: 'error', error: event.error, duration: event.duration }
: s
);
break;
case 'done':
meta = event.meta;
searchDone = true;
break;
}
});
}
function handleSort(column) {
@@ -56,7 +91,6 @@
}
}
// Unique store names for the filter dropdown
let storeNames = $derived(() => {
const names = [...new Set(results.map(r => r.storeName))];
return names.sort();
@@ -64,55 +98,44 @@
let filteredAndSorted = $derived(() => {
let items = [...results];
// Text filter
if (filterText) {
const lower = filterText.toLowerCase();
items = items.filter(r => r.name.toLowerCase().includes(lower));
}
// Store filter
if (excludedStores.size > 0) {
items = items.filter(r => !excludedStores.has(r.storeName));
}
// Sort
items.sort((a, b) => {
let cmp = 0;
switch (sortBy) {
case 'price':
cmp = (a.price ?? Infinity) - (b.price ?? Infinity);
break;
case 'name':
cmp = a.name.localeCompare(b.name);
break;
case 'store':
cmp = a.storeName.localeCompare(b.storeName);
break;
case 'price': cmp = (a.price ?? Infinity) - (b.price ?? Infinity); break;
case 'name': cmp = a.name.localeCompare(b.name); break;
case 'store': cmp = a.storeName.localeCompare(b.storeName); break;
}
return sortAsc ? cmp : -cmp;
});
return items;
});
function toggleStore(name) {
const next = new Set(excludedStores);
if (next.has(name)) next.delete(name); else next.add(name);
excludedStores = next;
}
function formatPrice(price, currency) {
if (price === null || price === undefined) return 'N/A';
try { return new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(price); }
catch { return `${currency} ${price.toFixed(2)}`; }
}
function toggleStore(name) {
const next = new Set(excludedStores);
if (next.has(name)) next.delete(name);
else next.add(name);
excludedStores = next;
}
function sortIcon(column) {
if (sortBy !== column) return '';
return sortAsc ? '\u2191' : '\u2193';
}
let completedCount = $derived(() => storeProgress.filter(s => s.status !== 'searching').length);
let totalStores = $derived(() => storeProgress.length);
</script>
<div class="h-full flex flex-col max-w-[1400px] mx-auto w-full">
@@ -131,19 +154,69 @@
<a href="/" class="btn-secondary text-xs py-1">New Search</a>
</div>
<!-- Errors -->
{#if meta?.errors?.length > 0}
<div class="px-6 pt-3 space-y-1">
{#each meta.errors as err}
<div class="text-xs text-red-400 bg-red-500/10 border border-red-500/20 px-3 py-1.5 rounded">
{err.storeName}: {err.error}
<!-- Store Progress -->
{#if storeProgress.length > 0}
<div class="flex-shrink-0 border-b border-surface-border px-6 py-3">
<div class="flex items-center gap-2 mb-2">
<span class="text-xs font-medium text-text-secondary">
{#if searchDone}
All stores completed
{:else}
Searching stores ({completedCount()}/{totalStores()})
{/if}
</span>
{#if !searchDone}
<div class="flex-1 h-1 bg-surface-hover rounded-full overflow-hidden max-w-xs">
<div class="h-full bg-accent rounded-full transition-all duration-300"
style="width: {totalStores() > 0 ? (completedCount() / totalStores() * 100) : 0}%"></div>
</div>
{/if}
</div>
<div class="flex flex-wrap gap-2">
{#each storeProgress as store}
<div class="flex items-center gap-2 text-xs px-3 py-1.5 rounded-lg border transition-all duration-300
{store.status === 'searching'
? 'bg-surface-raised border-surface-border text-text-secondary'
: store.status === 'done'
? 'bg-emerald-500/10 border-emerald-500/20 text-emerald-400'
: 'bg-red-500/10 border-red-500/20 text-red-400'}">
<!-- Status indicator -->
{#if store.status === 'searching'}
<svg class="w-3.5 h-3.5 animate-spin text-accent-text" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
{:else if store.status === 'done'}
<svg class="w-3.5 h-3.5" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2.5">
<path stroke-linecap="round" stroke-linejoin="round" d="M4.5 12.75l6 6 9-13.5" />
</svg>
{:else}
<svg class="w-3.5 h-3.5" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M6 18L18 6M6 6l12 12" />
</svg>
{/if}
<span class="font-medium">{store.name}</span>
{#if store.status === 'searching'}
<span class="text-text-tertiary">
{store.renderJs ? 'rendering...' : 'fetching...'}
</span>
{:else if store.status === 'done'}
<span>{store.resultCount} products</span>
<span class="text-emerald-600">{store.duration}ms</span>
{:else}
<span class="truncate max-w-[150px]" title={store.error}>{store.error}</span>
{/if}
</div>
{/each}
</div>
</div>
{/if}
<!-- Filters row -->
{#if !loading && results.length > 0}
{#if results.length > 0}
<div class="flex-shrink-0 border-b border-surface-border px-6 py-3 space-y-2.5">
<div class="flex items-center gap-3">
<div class="relative flex-1">
@@ -189,12 +262,14 @@
<!-- Content -->
<div class="flex-1 overflow-auto">
{#if loading}
<div class="p-6 space-y-2">
{#each Array(10) as _}
<div class="bg-surface-raised h-10 rounded animate-pulse"></div>
{/each}
<div class="flex flex-col items-center justify-center py-20 text-text-tertiary">
<svg class="w-8 h-8 animate-spin text-accent mb-3" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
<p class="text-sm">Connecting to stores...</p>
</div>
{:else if results.length === 0}
{:else if searchDone && results.length === 0}
<div class="flex flex-col items-center justify-center py-20 text-text-tertiary">
<svg class="w-10 h-10 mb-3 text-text-tertiary/50" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="1">
<path stroke-linecap="round" stroke-linejoin="round" d="M21 21l-5.197-5.197m0 0A7.5 7.5 0 105.196 5.196a7.5 7.5 0 0010.607 10.607z" />
@@ -202,7 +277,7 @@
<p class="text-sm">No results found for "{query}"</p>
<p class="text-xs mt-1">Try a different search term or check your store configurations.</p>
</div>
{:else}
{:else if results.length > 0}
<table class="w-full">
<thead class="sticky top-0 z-10 bg-surface">
<tr class="border-b border-surface-border">
@@ -231,9 +306,8 @@
</tr>
</thead>
<tbody>
{#each filteredAndSorted() as product, i}
{#each filteredAndSorted() as product}
<tr class="border-b border-surface-border hover:bg-surface-hover/50 transition-colors duration-100 group">
<!-- Thumbnail -->
<td class="px-6 py-2">
{#if product.image}
<div class="w-8 h-8 rounded bg-surface flex items-center justify-center overflow-hidden">
@@ -247,29 +321,15 @@
</div>
{/if}
</td>
<!-- Product name -->
<td class="px-4 py-2">
<span class="text-sm text-text-primary group-hover:text-white transition-colors line-clamp-1">
{product.name}
</span>
<span class="text-sm text-text-primary group-hover:text-white transition-colors line-clamp-1">{product.name}</span>
</td>
<!-- Price -->
<td class="px-4 py-2 text-right">
<span class="text-sm font-semibold text-accent-text whitespace-nowrap">
{formatPrice(product.price, product.currency)}
</span>
<span class="text-sm font-semibold text-accent-text whitespace-nowrap">{formatPrice(product.price, product.currency)}</span>
</td>
<!-- Store -->
<td class="px-4 py-2">
<span class="text-2xs text-text-tertiary bg-surface px-1.5 py-0.5 rounded">
{product.storeName}
</span>
<span class="text-2xs text-text-tertiary bg-surface px-1.5 py-0.5 rounded">{product.storeName}</span>
</td>
<!-- Link -->
<td class="px-6 py-2 text-right">
<a href={product.url} target="_blank" rel="noopener noreferrer"
class="text-2xs text-accent-text hover:text-accent transition-colors opacity-0 group-hover:opacity-100">

52
src/server/routes/search.ts
View File

@@ -1,7 +1,8 @@
import type { FastifyPluginAsync } from 'fastify';
import { search } from '../scraper/engine.js';
import { search, searchStreaming } from '../scraper/engine.js';
export const searchRoutes: FastifyPluginAsync = async (app) => {
// Original endpoint (kept for compatibility)
app.get<{
Querystring: {
q: string;
@@ -36,4 +37,53 @@ export const searchRoutes: FastifyPluginAsync = async (app) => {
groupId: group ? Number(group) : undefined,
});
});
// SSE streaming endpoint
app.get<{
Querystring: {
q: string;
stores?: string;
category?: string;
group?: string;
};
}>('/search/stream', {
schema: {
querystring: {
type: 'object',
required: ['q'],
properties: {
q: { type: 'string', minLength: 1 },
stores: { type: 'string' },
category: { type: 'string' },
group: { type: 'string' },
},
},
},
}, async (request, reply) => {
const { q, stores, category, group } = request.query;
const storeIds = stores
? stores.split(',').map(Number).filter((n) => !isNaN(n))
: undefined;
reply.raw.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
});
await searchStreaming(
{
query: q,
storeIds,
categoryId: category ? Number(category) : undefined,
groupId: group ? Number(group) : undefined,
},
(event) => {
reply.raw.write(`data: ${JSON.stringify(event)}\n\n`);
},
);
reply.raw.end();
});
};

101
src/server/scraper/engine.ts
View File

@@ -17,6 +17,18 @@ export interface SearchOptions {
groupId?: number;
}
export interface StoreProgress {
type: 'start' | 'store_complete' | 'store_error' | 'done';
storeId?: number;
storeName?: string;
results?: Product[];
resultCount?: number;
duration?: number;
error?: string;
stores?: Array<{ id: number; name: string; renderJs: boolean }>;
meta?: SearchResult['meta'];
}
export interface SearchResult {
results: Product[];
meta: {
@@ -122,3 +134,92 @@ export async function search(options: SearchOptions): Promise<SearchResult> {
},
};
}
function resolveStores(options: SearchOptions): Store[] {
if (options.storeIds?.length) return getStoresByIds(options.storeIds);
if (options.groupId) return getStoresByGroup(options.groupId);
if (options.categoryId) return getStoresByCategory(options.categoryId);
return getEnabledStores();
}
export async function searchStreaming(
options: SearchOptions,
onProgress: (event: StoreProgress) => void,
): Promise<void> {
const startTime = Date.now();
const { query } = options;
const stores = resolveStores(options);
if (stores.length === 0) {
onProgress({ type: 'done', meta: { query, duration: 0, storeCount: 0, totalResults: 0, errors: [] } });
return;
}
// Send start event with store list
onProgress({
type: 'start',
stores: stores.map((s) => ({ id: s.id, name: s.name, renderJs: !!s.render_js })),
});
const limit = pLimit(MAX_CONCURRENCY);
const errors: SearchResult['meta']['errors'] = [];
let totalResults = 0;
const scrapePromises = stores.map((store) =>
limit(async () => {
const searchUrl = store.search_url.replace('{query}', encodeURIComponent(query));
const storeStart = Date.now();
const rateLimiter = getLimiter(store.id, 1, Math.floor(store.rate_window / store.rate_limit));
try {
const scrapeFn = store.render_js
? () => scrapeStoreWithBrowser(store, searchUrl)
: () => scrapeStore(store, searchUrl);
const result = await rateLimiter.schedule(scrapeFn);
const duration = Date.now() - storeStart;
const products = result.items.map((item) =>
normalizeResult(item, store.id, store.name, store.base_url, store.currency)
);
logScrape(store.id, query, true, products.length, duration);
totalResults += products.length;
onProgress({
type: 'store_complete',
storeId: store.id,
storeName: store.name,
results: products,
resultCount: products.length,
duration,
});
} catch (err) {
const duration = Date.now() - storeStart;
const errorMessage = err instanceof Error ? err.message : String(err);
logScrape(store.id, query, false, 0, duration, errorMessage);
errors.push({ storeId: store.id, storeName: store.name, error: errorMessage });
onProgress({
type: 'store_error',
storeId: store.id,
storeName: store.name,
error: errorMessage,
duration,
});
}
})
);
await Promise.all(scrapePromises);
onProgress({
type: 'done',
meta: {
query,
duration: Date.now() - startTime,
storeCount: stores.length,
totalResults,
errors,
},
});
}