Add real-time per-store search progress via SSE streaming

Backend: new /api/search/stream SSE endpoint that emits events as each store completes: start (store list), store_complete (results + duration), store_error, and done (final meta). Frontend: results page now shows live progress per store with spinning indicators while searching, checkmarks when done, and X marks on errors. Each store chip shows product count and response time. Results stream into the table as stores complete instead of waiting for all stores to finish. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-26 22:15:50 +01:00
parent fe56c3b17e
commit 37425812e0
4 changed files with 322 additions and 83 deletions
--- a/src/client/src/lib/api.ts
+++ b/src/client/src/lib/api.ts
@@ -94,6 +94,34 @@ export function searchProducts(query: string, params?: { stores?: string; catego
  return api<any>(`/api/search?${searchParams}`);
 }

+export function searchProductsStream(
+  query: string,
+  params: { stores?: string; category?: string; group?: string } | undefined,
+  onEvent: (event: any) => void,
+): () => void {
+  const searchParams = new URLSearchParams({ q: query });
+  if (params?.stores) searchParams.set('stores', params.stores);
+  if (params?.category) searchParams.set('category', params.category);
+  if (params?.group) searchParams.set('group', params.group);
+
+  const evtSource = new EventSource(`/api/search/stream?${searchParams}`);
+
+  evtSource.onmessage = (e) => {
+    try {
+      const event = JSON.parse(e.data);
+      onEvent(event);
+      if (event.type === 'done') evtSource.close();
+    } catch { /* ignore parse errors */ }
+  };
+
+  evtSource.onerror = () => {
+    evtSource.close();
+  };
+
+  // Return cleanup function
+  return () => evtSource.close();
+}
+
 export function testStore(id: number, query: string) {
  return api<any>(`/api/stores/${id}/test`, { method: 'POST', body: JSON.stringify({ query }) });
 }
--- a/src/client/src/routes/results/+page.svelte
+++ b/src/client/src/routes/results/+page.svelte
@@ -1,7 +1,7 @@
 <script>
  import { page } from '$app/stores';
  import { goto } from '$app/navigation';
-  import { searchProducts } from '$lib/api';
+  import { searchProductsStream } from '$lib/api';
  import { onMount } from 'svelte';

  let query = $state('');
@@ -13,38 +13,73 @@
  let filterText = $state('');
  let excludedStores = $state(new Set());

+  // Per-store progress tracking
+  let storeProgress = $state([]);
+  let searchDone = $state(false);
+
+  let cleanupStream = null;
+
  onMount(async () => {
    const params = $page.url.searchParams;
    query = params.get('q') || '';
    if (!query) { goto('/'); return; }
-    await doSearch();
+    doSearch();
+
+    return () => { if (cleanupStream) cleanupStream(); };
  });

-  async function doSearch() {
+  function doSearch() {
    if (!query.trim()) return;
    loading = true;
-    try {
-      const params = $page.url.searchParams;
-      const data = await searchProducts(query, {
-        category: params.get('category') || undefined,
-        group: params.get('group') || undefined,
-        stores: params.get('stores') || undefined,
-      });
-      results = data.results;
-      meta = data.meta;
-    } catch (err) {
-      console.error('Search failed:', err);
-      results = [];
-    } finally {
-      loading = false;
-    }
-  }
+    searchDone = false;
+    results = [];
+    meta = null;
+    storeProgress = [];

-  function handleSearch(e) {
-    e.preventDefault();
-    if (!query.trim()) return;
-    goto(`/results?q=${encodeURIComponent(query.trim())}`);
-    doSearch();
+    const params = $page.url.searchParams;
+
+    cleanupStream = searchProductsStream(query, {
+      category: params.get('category') || undefined,
+      group: params.get('group') || undefined,
+      stores: params.get('stores') || undefined,
+    }, (event) => {
+      switch (event.type) {
+        case 'start':
+          storeProgress = event.stores.map((s) => ({
+            id: s.id,
+            name: s.name,
+            renderJs: s.renderJs,
+            status: 'searching',
+            resultCount: 0,
+            duration: 0,
+            error: null,
+          }));
+          loading = false;
+          break;
+
+        case 'store_complete':
+          results = [...results, ...event.results];
+          storeProgress = storeProgress.map((s) =>
+            s.id === event.storeId
+              ? { ...s, status: 'done', resultCount: event.resultCount, duration: event.duration }
+              : s
+          );
+          break;
+
+        case 'store_error':
+          storeProgress = storeProgress.map((s) =>
+            s.id === event.storeId
+              ? { ...s, status: 'error', error: event.error, duration: event.duration }
+              : s
+          );
+          break;
+
+        case 'done':
+          meta = event.meta;
+          searchDone = true;
+          break;
+      }
+    });
  }

  function handleSort(column) {
@@ -56,7 +91,6 @@
    }
  }

-  // Unique store names for the filter dropdown
  let storeNames = $derived(() => {
    const names = [...new Set(results.map(r => r.storeName))];
    return names.sort();
@@ -64,55 +98,44 @@

  let filteredAndSorted = $derived(() => {
    let items = [...results];
-
-    // Text filter
    if (filterText) {
      const lower = filterText.toLowerCase();
      items = items.filter(r => r.name.toLowerCase().includes(lower));
    }
-
-    // Store filter
    if (excludedStores.size > 0) {
      items = items.filter(r => !excludedStores.has(r.storeName));
    }
-
-    // Sort
    items.sort((a, b) => {
      let cmp = 0;
      switch (sortBy) {
-        case 'price':
-          cmp = (a.price ?? Infinity) - (b.price ?? Infinity);
-          break;
-        case 'name':
-          cmp = a.name.localeCompare(b.name);
-          break;
-        case 'store':
-          cmp = a.storeName.localeCompare(b.storeName);
-          break;
+        case 'price': cmp = (a.price ?? Infinity) - (b.price ?? Infinity); break;
+        case 'name': cmp = a.name.localeCompare(b.name); break;
+        case 'store': cmp = a.storeName.localeCompare(b.storeName); break;
      }
      return sortAsc ? cmp : -cmp;
    });
-
    return items;
  });

+  function toggleStore(name) {
+    const next = new Set(excludedStores);
+    if (next.has(name)) next.delete(name); else next.add(name);
+    excludedStores = next;
+  }
+
  function formatPrice(price, currency) {
    if (price === null || price === undefined) return 'N/A';
    try { return new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(price); }
    catch { return `${currency} ${price.toFixed(2)}`; }
  }

-  function toggleStore(name) {
-    const next = new Set(excludedStores);
-    if (next.has(name)) next.delete(name);
-    else next.add(name);
-    excludedStores = next;
-  }
-
  function sortIcon(column) {
    if (sortBy !== column) return '';
    return sortAsc ? '\u2191' : '\u2193';
  }
+
+  let completedCount = $derived(() => storeProgress.filter(s => s.status !== 'searching').length);
+  let totalStores = $derived(() => storeProgress.length);
 </script>

 <div class="h-full flex flex-col max-w-[1400px] mx-auto w-full">
@@ -131,19 +154,69 @@
    <a href="/" class="btn-secondary text-xs py-1">New Search</a>
  </div>

-  <!-- Errors -->
-  {#if meta?.errors?.length > 0}
-    <div class="px-6 pt-3 space-y-1">
-      {#each meta.errors as err}
-        <div class="text-xs text-red-400 bg-red-500/10 border border-red-500/20 px-3 py-1.5 rounded">
-          {err.storeName}: {err.error}
-        </div>
-      {/each}
+  <!-- Store Progress -->
+  {#if storeProgress.length > 0}
+    <div class="flex-shrink-0 border-b border-surface-border px-6 py-3">
+      <div class="flex items-center gap-2 mb-2">
+        <span class="text-xs font-medium text-text-secondary">
+          {#if searchDone}
+            All stores completed
+          {:else}
+            Searching stores ({completedCount()}/{totalStores()})
+          {/if}
+        </span>
+        {#if !searchDone}
+          <div class="flex-1 h-1 bg-surface-hover rounded-full overflow-hidden max-w-xs">
+            <div class="h-full bg-accent rounded-full transition-all duration-300"
+              style="width: {totalStores() > 0 ? (completedCount() / totalStores() * 100) : 0}%"></div>
+          </div>
+        {/if}
+      </div>
+      <div class="flex flex-wrap gap-2">
+        {#each storeProgress as store}
+          <div class="flex items-center gap-2 text-xs px-3 py-1.5 rounded-lg border transition-all duration-300
+            {store.status === 'searching'
+              ? 'bg-surface-raised border-surface-border text-text-secondary'
+              : store.status === 'done'
+                ? 'bg-emerald-500/10 border-emerald-500/20 text-emerald-400'
+                : 'bg-red-500/10 border-red-500/20 text-red-400'}">
+
+            <!-- Status indicator -->
+            {#if store.status === 'searching'}
+              <svg class="w-3.5 h-3.5 animate-spin text-accent-text" fill="none" viewBox="0 0 24 24">
+                <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
+                <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
+              </svg>
+            {:else if store.status === 'done'}
+              <svg class="w-3.5 h-3.5" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2.5">
+                <path stroke-linecap="round" stroke-linejoin="round" d="M4.5 12.75l6 6 9-13.5" />
+              </svg>
+            {:else}
+              <svg class="w-3.5 h-3.5" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
+                <path stroke-linecap="round" stroke-linejoin="round" d="M6 18L18 6M6 6l12 12" />
+              </svg>
+            {/if}
+
+            <span class="font-medium">{store.name}</span>
+
+            {#if store.status === 'searching'}
+              <span class="text-text-tertiary">
+                {store.renderJs ? 'rendering...' : 'fetching...'}
+              </span>
+            {:else if store.status === 'done'}
+              <span>{store.resultCount} products</span>
+              <span class="text-emerald-600">{store.duration}ms</span>
+            {:else}
+              <span class="truncate max-w-[150px]" title={store.error}>{store.error}</span>
+            {/if}
+          </div>
+        {/each}
+      </div>
    </div>
  {/if}

  <!-- Filters row -->
-  {#if !loading && results.length > 0}
+  {#if results.length > 0}
    <div class="flex-shrink-0 border-b border-surface-border px-6 py-3 space-y-2.5">
      <div class="flex items-center gap-3">
        <div class="relative flex-1">
@@ -189,12 +262,14 @@
  <!-- Content -->
  <div class="flex-1 overflow-auto">
    {#if loading}
-      <div class="p-6 space-y-2">
-        {#each Array(10) as _}
-          <div class="bg-surface-raised h-10 rounded animate-pulse"></div>
-        {/each}
+      <div class="flex flex-col items-center justify-center py-20 text-text-tertiary">
+        <svg class="w-8 h-8 animate-spin text-accent mb-3" fill="none" viewBox="0 0 24 24">
+          <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
+          <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
+        </svg>
+        <p class="text-sm">Connecting to stores...</p>
      </div>
-    {:else if results.length === 0}
+    {:else if searchDone && results.length === 0}
      <div class="flex flex-col items-center justify-center py-20 text-text-tertiary">
        <svg class="w-10 h-10 mb-3 text-text-tertiary/50" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="1">
          <path stroke-linecap="round" stroke-linejoin="round" d="M21 21l-5.197-5.197m0 0A7.5 7.5 0 105.196 5.196a7.5 7.5 0 0010.607 10.607z" />
@@ -202,7 +277,7 @@
        <p class="text-sm">No results found for "{query}"</p>
        <p class="text-xs mt-1">Try a different search term or check your store configurations.</p>
      </div>
-    {:else}
+    {:else if results.length > 0}
      <table class="w-full">
        <thead class="sticky top-0 z-10 bg-surface">
          <tr class="border-b border-surface-border">
@@ -231,9 +306,8 @@
          </tr>
        </thead>
        <tbody>
-          {#each filteredAndSorted() as product, i}
+          {#each filteredAndSorted() as product}
            <tr class="border-b border-surface-border hover:bg-surface-hover/50 transition-colors duration-100 group">
-              <!-- Thumbnail -->
              <td class="px-6 py-2">
                {#if product.image}
                  <div class="w-8 h-8 rounded bg-surface flex items-center justify-center overflow-hidden">
@@ -247,29 +321,15 @@
                  </div>
                {/if}
              </td>
-
-              <!-- Product name -->
              <td class="px-4 py-2">
-                <span class="text-sm text-text-primary group-hover:text-white transition-colors line-clamp-1">
-                  {product.name}
-                </span>
+                <span class="text-sm text-text-primary group-hover:text-white transition-colors line-clamp-1">{product.name}</span>
              </td>
-
-              <!-- Price -->
              <td class="px-4 py-2 text-right">
-                <span class="text-sm font-semibold text-accent-text whitespace-nowrap">
-                  {formatPrice(product.price, product.currency)}
-                </span>
+                <span class="text-sm font-semibold text-accent-text whitespace-nowrap">{formatPrice(product.price, product.currency)}</span>
              </td>
-
-              <!-- Store -->
              <td class="px-4 py-2">
-                <span class="text-2xs text-text-tertiary bg-surface px-1.5 py-0.5 rounded">
-                  {product.storeName}
-                </span>
+                <span class="text-2xs text-text-tertiary bg-surface px-1.5 py-0.5 rounded">{product.storeName}</span>
              </td>
-
-              <!-- Link -->
              <td class="px-6 py-2 text-right">
                <a href={product.url} target="_blank" rel="noopener noreferrer"
                  class="text-2xs text-accent-text hover:text-accent transition-colors opacity-0 group-hover:opacity-100">
--- a/src/server/routes/search.ts
+++ b/src/server/routes/search.ts
@@ -1,7 +1,8 @@
 import type { FastifyPluginAsync } from 'fastify';
-import { search } from '../scraper/engine.js';
+import { search, searchStreaming } from '../scraper/engine.js';

 export const searchRoutes: FastifyPluginAsync = async (app) => {
+  // Original endpoint (kept for compatibility)
  app.get<{
    Querystring: {
      q: string;
@@ -36,4 +37,53 @@ export const searchRoutes: FastifyPluginAsync = async (app) => {
      groupId: group ? Number(group) : undefined,
    });
  });
+
+  // SSE streaming endpoint
+  app.get<{
+    Querystring: {
+      q: string;
+      stores?: string;
+      category?: string;
+      group?: string;
+    };
+  }>('/search/stream', {
+    schema: {
+      querystring: {
+        type: 'object',
+        required: ['q'],
+        properties: {
+          q: { type: 'string', minLength: 1 },
+          stores: { type: 'string' },
+          category: { type: 'string' },
+          group: { type: 'string' },
+        },
+      },
+    },
+  }, async (request, reply) => {
+    const { q, stores, category, group } = request.query;
+
+    const storeIds = stores
+      ? stores.split(',').map(Number).filter((n) => !isNaN(n))
+      : undefined;
+
+    reply.raw.writeHead(200, {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      'Connection': 'keep-alive',
+    });
+
+    await searchStreaming(
+      {
+        query: q,
+        storeIds,
+        categoryId: category ? Number(category) : undefined,
+        groupId: group ? Number(group) : undefined,
+      },
+      (event) => {
+        reply.raw.write(`data: ${JSON.stringify(event)}\n\n`);
+      },
+    );
+
+    reply.raw.end();
+  });
 };
--- a/src/server/scraper/engine.ts
+++ b/src/server/scraper/engine.ts
@@ -17,6 +17,18 @@ export interface SearchOptions {
  groupId?: number;
 }

+export interface StoreProgress {
+  type: 'start' | 'store_complete' | 'store_error' | 'done';
+  storeId?: number;
+  storeName?: string;
+  results?: Product[];
+  resultCount?: number;
+  duration?: number;
+  error?: string;
+  stores?: Array<{ id: number; name: string; renderJs: boolean }>;
+  meta?: SearchResult['meta'];
+}
+
 export interface SearchResult {
  results: Product[];
  meta: {
@@ -122,3 +134,92 @@ export async function search(options: SearchOptions): Promise<SearchResult> {
    },
  };
 }
+
+function resolveStores(options: SearchOptions): Store[] {
+  if (options.storeIds?.length) return getStoresByIds(options.storeIds);
+  if (options.groupId) return getStoresByGroup(options.groupId);
+  if (options.categoryId) return getStoresByCategory(options.categoryId);
+  return getEnabledStores();
+}
+
+export async function searchStreaming(
+  options: SearchOptions,
+  onProgress: (event: StoreProgress) => void,
+): Promise<void> {
+  const startTime = Date.now();
+  const { query } = options;
+  const stores = resolveStores(options);
+
+  if (stores.length === 0) {
+    onProgress({ type: 'done', meta: { query, duration: 0, storeCount: 0, totalResults: 0, errors: [] } });
+    return;
+  }
+
+  // Send start event with store list
+  onProgress({
+    type: 'start',
+    stores: stores.map((s) => ({ id: s.id, name: s.name, renderJs: !!s.render_js })),
+  });
+
+  const limit = pLimit(MAX_CONCURRENCY);
+  const errors: SearchResult['meta']['errors'] = [];
+  let totalResults = 0;
+
+  const scrapePromises = stores.map((store) =>
+    limit(async () => {
+      const searchUrl = store.search_url.replace('{query}', encodeURIComponent(query));
+      const storeStart = Date.now();
+      const rateLimiter = getLimiter(store.id, 1, Math.floor(store.rate_window / store.rate_limit));
+
+      try {
+        const scrapeFn = store.render_js
+          ? () => scrapeStoreWithBrowser(store, searchUrl)
+          : () => scrapeStore(store, searchUrl);
+        const result = await rateLimiter.schedule(scrapeFn);
+        const duration = Date.now() - storeStart;
+
+        const products = result.items.map((item) =>
+          normalizeResult(item, store.id, store.name, store.base_url, store.currency)
+        );
+
+        logScrape(store.id, query, true, products.length, duration);
+        totalResults += products.length;
+
+        onProgress({
+          type: 'store_complete',
+          storeId: store.id,
+          storeName: store.name,
+          results: products,
+          resultCount: products.length,
+          duration,
+        });
+      } catch (err) {
+        const duration = Date.now() - storeStart;
+        const errorMessage = err instanceof Error ? err.message : String(err);
+        logScrape(store.id, query, false, 0, duration, errorMessage);
+        errors.push({ storeId: store.id, storeName: store.name, error: errorMessage });
+
+        onProgress({
+          type: 'store_error',
+          storeId: store.id,
+          storeName: store.name,
+          error: errorMessage,
+          duration,
+        });
+      }
+    })
+  );
+
+  await Promise.all(scrapePromises);
+
+  onProgress({
+    type: 'done',
+    meta: {
+      query,
+      duration: Date.now() - startTime,
+      storeCount: stores.length,
+      totalResults,
+      errors,
+    },
+  });
+}