diff --git a/src/client/src/routes/admin/+page.svelte b/src/client/src/routes/admin/+page.svelte index f8df5d4..f7db9e1 100644 --- a/src/client/src/routes/admin/+page.svelte +++ b/src/client/src/routes/admin/+page.svelte @@ -8,8 +8,6 @@ // Test all state let testRunning = $state(false); - let testQuery = $state(''); - let showTestPrompt = $state(false); let testResults = $state(new Map()); onMount(async () => { @@ -52,15 +50,7 @@ setTimeout(() => syncMessage = '', 5000); } - function startTestAll() { - testQuery = ''; - testResults = new Map(); - showTestPrompt = true; - } - async function runTestAll() { - if (!testQuery.trim()) return; - showTestPrompt = false; testRunning = true; testResults = new Map(); @@ -68,19 +58,21 @@ // Mark all as pending for (const store of enabledStores) { - testResults.set(store.id, { status: 'pending' }); + testResults.set(store.id, { status: 'pending', query: store.test_query || 'test' }); } testResults = new Map(testResults); - // Test each store sequentially + // Test each store sequentially using its own test_query for (const store of enabledStores) { - testResults.set(store.id, { status: 'testing' }); + const query = store.test_query || 'test'; + testResults.set(store.id, { status: 'testing', query }); testResults = new Map(testResults); try { - const result = await testStore(store.id, testQuery.trim()); + const result = await testStore(store.id, query); testResults.set(store.id, { status: result.success ? 'success' : 'error', + query, resultCount: result.parsedProducts?.length || 0, duration: result.duration, error: result.error, @@ -88,6 +80,7 @@ } catch (err) { testResults.set(store.id, { status: 'error', + query, resultCount: 0, duration: 0, error: err.message || 'Request failed', @@ -109,7 +102,7 @@

Stores

-
{/if} - - {#if showTestPrompt} -
-

Enter a search term to test all enabled stores:

-
{ e.preventDefault(); runTestAll(); }} class="flex gap-2"> - - - -
-
- {/if} -
{#if loading} @@ -218,7 +193,8 @@ Testing... {:else if test.status === 'success'} - + diff --git a/src/client/src/routes/admin/stores/[id]/+page.svelte b/src/client/src/routes/admin/stores/[id]/+page.svelte index d20e445..bf08967 100644 --- a/src/client/src/routes/admin/stores/[id]/+page.svelte +++ b/src/client/src/routes/admin/stores/[id]/+page.svelte @@ -12,7 +12,7 @@ let form = $state({ name: '', base_url: '', search_url: '', sel_container: '', sel_name: '', sel_price: '', sel_link: '', sel_image: '', - category_id: '', currency: 'EUR', rate_limit: 2, render_js: false, + category_id: '', currency: 'EUR', rate_limit: 2, render_js: false, test_query: '', user_agent: '', proxy_url: '', headers_json: '', }); @@ -24,7 +24,7 @@ sel_container: store.sel_container, sel_name: store.sel_name, sel_price: store.sel_price, sel_link: store.sel_link, sel_image: store.sel_image || '', category_id: store.category_id?.toString() || '', currency: store.currency, - rate_limit: store.rate_limit, render_js: !!store.render_js, + rate_limit: store.rate_limit, render_js: !!store.render_js, test_query: store.test_query || '', user_agent: store.user_agent || '', headers_json: store.headers_json || '', }; loading = false; @@ -93,6 +93,11 @@
+
+ + +

Search term used when testing this store

+
diff --git a/src/client/src/routes/admin/stores/new/+page.svelte b/src/client/src/routes/admin/stores/new/+page.svelte index ceff8e8..eba5d94 100644 --- a/src/client/src/routes/admin/stores/new/+page.svelte +++ b/src/client/src/routes/admin/stores/new/+page.svelte @@ -10,7 +10,7 @@ let form = $state({ name: '', base_url: '', search_url: '', sel_container: '', sel_name: '', sel_price: '', sel_link: '', sel_image: '', - category_id: '', currency: 'EUR', rate_limit: 2, render_js: false, + category_id: '', currency: 'EUR', rate_limit: 2, render_js: false, test_query: '', user_agent: '', proxy_url: '', headers_json: '', }); @@ -76,6 +76,11 @@ +
+ + +

Search term used when testing this store

+
diff --git a/src/server/db/migrations/003_add_test_query.sql b/src/server/db/migrations/003_add_test_query.sql new file mode 100644 index 0000000..70e968b --- /dev/null +++ b/src/server/db/migrations/003_add_test_query.sql @@ -0,0 +1 @@ +ALTER TABLE stores ADD COLUMN test_query TEXT DEFAULT 'test'; diff --git a/src/server/models/store.ts b/src/server/models/store.ts index e424674..e6889e7 100644 --- a/src/server/models/store.ts +++ b/src/server/models/store.ts @@ -8,6 +8,7 @@ export interface Store { search_url: string; enabled: number; render_js: number; + test_query: string; sel_container: string; sel_name: string; sel_price: string; @@ -40,6 +41,7 @@ export interface CreateStoreInput { sel_link: string; sel_image?: string; render_js?: boolean; + test_query?: string; rate_limit?: number; rate_window?: number; proxy_url?: string; @@ -120,12 +122,12 @@ export function createStore(input: CreateStoreInput): Store { db.run(` INSERT INTO stores (name, slug, base_url, search_url, sel_container, sel_name, sel_price, sel_link, sel_image, - render_js, rate_limit, rate_window, proxy_url, user_agent, headers_json, currency, category_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + render_js, test_query, rate_limit, rate_window, proxy_url, user_agent, headers_json, currency, category_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, [ input.name, slug, input.base_url, input.search_url, input.sel_container, input.sel_name, input.sel_price, input.sel_link, input.sel_image || null, - input.render_js ? 1 : 0, + input.render_js ? 1 : 0, input.test_query || 'test', input.rate_limit ?? 2, input.rate_window ?? 1000, input.proxy_url || null, input.user_agent || null, input.headers_json || null, input.currency || 'EUR', input.category_id || null, diff --git a/src/server/routes/stores.ts b/src/server/routes/stores.ts index 68165c9..90d6749 100644 --- a/src/server/routes/stores.ts +++ b/src/server/routes/stores.ts @@ -36,6 +36,7 @@ export const storeRoutes: FastifyPluginAsync = async (app) => { sel_link: { type: 'string', minLength: 1 }, sel_image: { type: 'string' }, render_js: { type: 'boolean' }, + test_query: { type: 'string' }, rate_limit: { type: 'number' }, rate_window: { type: 'number' }, proxy_url: { type: 'string' }, diff --git a/src/server/services/store-sync.ts b/src/server/services/store-sync.ts index 401435f..715966d 100644 --- a/src/server/services/store-sync.ts +++ b/src/server/services/store-sync.ts @@ -9,6 +9,7 @@ export interface StoreFileConfig { search_url: string; enabled?: boolean; render_js?: boolean; + test_query?: string; category?: string; currency?: string; selectors: { @@ -99,7 +100,7 @@ export function syncFromFiles(storesDir: string): { created: number; updated: nu if (existing) { db.run(` UPDATE stores SET - name = ?, base_url = ?, search_url = ?, enabled = ?, render_js = ?, + name = ?, base_url = ?, search_url = ?, enabled = ?, render_js = ?, test_query = ?, sel_container = ?, sel_name = ?, sel_price = ?, sel_link = ?, sel_image = ?, rate_limit = ?, rate_window = ?, proxy_url = ?, user_agent = ?, headers_json = ?, currency = ?, category_id = ?, updated_at = datetime('now') @@ -107,7 +108,7 @@ export function syncFromFiles(storesDir: string): { created: number; updated: nu `, [ config.name, config.base_url, config.search_url, config.enabled === false ? 0 : 1, - config.render_js ? 1 : 0, + config.render_js ? 1 : 0, config.test_query || 'test', config.selectors.container, config.selectors.name, config.selectors.price, config.selectors.link, config.selectors.image || null, @@ -119,15 +120,15 @@ export function syncFromFiles(storesDir: string): { created: number; updated: nu updated++; } else { db.run(` - INSERT INTO stores (name, slug, base_url, search_url, enabled, render_js, + INSERT INTO stores (name, slug, base_url, search_url, enabled, render_js, test_query, sel_container, sel_name, sel_price, sel_link, sel_image, rate_limit, rate_window, proxy_url, user_agent, headers_json, currency, category_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, [ config.name, slug, config.base_url, config.search_url, config.enabled === false ? 0 : 1, - config.render_js ? 1 : 0, + config.render_js ? 1 : 0, config.test_query || 'test', config.selectors.container, config.selectors.name, config.selectors.price, config.selectors.link, config.selectors.image || null, @@ -165,6 +166,7 @@ function storeToConfig(store: any, categoryName?: string): StoreFileConfig { if (store.sel_image) config.selectors.image = store.sel_image; if (store.enabled === 0) config.enabled = false; if (store.render_js) config.render_js = true; + if (store.test_query && store.test_query !== 'test') config.test_query = store.test_query; if (categoryName) config.category = categoryName; if (store.currency && store.currency !== 'EUR') config.currency = store.currency; if (store.rate_limit && store.rate_limit !== 2) config.rate_limit = store.rate_limit; diff --git a/stores/hgspot.yaml b/stores/hgspot.yaml index be0cc5f..7e5825e 100644 --- a/stores/hgspot.yaml +++ b/stores/hgspot.yaml @@ -4,6 +4,7 @@ search_url: https://www.hgspot.hr/pretraga?q={query}&page=0 category: Electronics currency: EUR render_js: true +test_query: logitech rate_limit: 1 selectors: diff --git a/stores/links-hr.yaml b/stores/links-hr.yaml index a414aab..41dd94f 100644 --- a/stores/links-hr.yaml +++ b/stores/links-hr.yaml @@ -3,6 +3,7 @@ base_url: https://www.links.hr search_url: https://www.links.hr/hr/search?q={query} category: Electronics currency: EUR +test_query: logitech selectors: container: ".card.mobile-card"