Initial commit: Price Hunter — self-hosted price comparison engine
Complete application scaffolding with: - Backend: Node.js + Fastify + sql.js (SQLite) - Frontend: SvelteKit + Tailwind CSS - Scraper engine with parallel fan-out, rate limiting, cheerio-based parsing - Store management with CSS selector config and per-store test pages - Docker setup for single-command deployment Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
7
.dockerignore
Normal file
7
.dockerignore
Normal file
@@ -0,0 +1,7 @@
|
||||
node_modules
|
||||
dist
|
||||
data/*.db
|
||||
.git
|
||||
.env
|
||||
*.md
|
||||
tests
|
||||
2
.env.example
Normal file
2
.env.example
Normal file
@@ -0,0 +1,2 @@
|
||||
PORT=3000
|
||||
DATABASE_PATH=./data/pricehunter.db
|
||||
8
.gitignore
vendored
Normal file
8
.gitignore
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
node_modules/
|
||||
dist/
|
||||
data/*.db
|
||||
.env
|
||||
.svelte-kit/
|
||||
src/client/build/
|
||||
src/client/.svelte-kit/
|
||||
*.tsbuildinfo
|
||||
36
Dockerfile
Normal file
36
Dockerfile
Normal file
@@ -0,0 +1,36 @@
|
||||
# Stage 1: Build client
|
||||
FROM node:20-alpine AS client-build
|
||||
WORKDIR /app/src/client
|
||||
COPY src/client/package*.json ./
|
||||
RUN npm ci
|
||||
COPY src/client/ ./
|
||||
RUN npm run build
|
||||
|
||||
# Stage 2: Build server
|
||||
FROM node:20-alpine AS server-build
|
||||
WORKDIR /app
|
||||
COPY package*.json ./
|
||||
RUN npm ci
|
||||
COPY tsconfig.json ./
|
||||
COPY src/server/ ./src/server/
|
||||
RUN npx tsc
|
||||
|
||||
# Stage 3: Production runtime
|
||||
FROM node:20-alpine
|
||||
WORKDIR /app
|
||||
|
||||
COPY package*.json ./
|
||||
RUN npm ci --omit=dev
|
||||
|
||||
COPY --from=server-build /app/dist ./dist
|
||||
COPY --from=client-build /app/src/client/build ./dist/client
|
||||
COPY src/server/db/migrations ./dist/server/db/migrations
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=3000
|
||||
ENV DATABASE_PATH=/app/data/pricehunter.db
|
||||
|
||||
EXPOSE 3000
|
||||
VOLUME /app/data
|
||||
|
||||
CMD ["node", "dist/server/index.js"]
|
||||
0
data/.gitkeep
Normal file
0
data/.gitkeep
Normal file
12
docker-compose.yml
Normal file
12
docker-compose.yml
Normal file
@@ -0,0 +1,12 @@
|
||||
services:
|
||||
pricehunter:
|
||||
build: .
|
||||
ports:
|
||||
- "${PORT:-3000}:3000"
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
- DATABASE_PATH=/app/data/pricehunter.db
|
||||
- PORT=3000
|
||||
restart: unless-stopped
|
||||
3851
package-lock.json
generated
Normal file
3851
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
35
package.json
Normal file
35
package.json
Normal file
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"name": "pricehunter",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev:server": "tsx watch src/server/index.ts",
|
||||
"dev:client": "cd src/client && npm run dev",
|
||||
"dev": "concurrently \"npm run dev:server\" \"npm run dev:client\"",
|
||||
"build:client": "cd src/client && npm run build",
|
||||
"build:server": "tsc",
|
||||
"build": "npm run build:client && npm run build:server",
|
||||
"start": "node dist/server/index.js",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
},
|
||||
"dependencies": {
|
||||
"@fastify/cors": "^10.0.1",
|
||||
"@fastify/static": "^8.0.3",
|
||||
"bottleneck": "^2.19.5",
|
||||
"cheerio": "^1.0.0",
|
||||
"dotenv": "^16.4.7",
|
||||
"fastify": "^5.2.1",
|
||||
"p-limit": "^6.2.0",
|
||||
"sql.js": "^1.11.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.0",
|
||||
"concurrently": "^9.1.2",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"tsx": "^4.19.2",
|
||||
"typescript": "^5.7.2",
|
||||
"vitest": "^2.1.8"
|
||||
}
|
||||
}
|
||||
2502
src/client/package-lock.json
generated
Normal file
2502
src/client/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
23
src/client/package.json
Normal file
23
src/client/package.json
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "pricehunter-client",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite dev",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@sveltejs/adapter-static": "^3.0.8",
|
||||
"@sveltejs/kit": "^2.15.0",
|
||||
"@sveltejs/vite-plugin-svelte": "^5.0.0",
|
||||
"svelte": "^5.16.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"autoprefixer": "^10.4.20",
|
||||
"postcss": "^8.4.49",
|
||||
"tailwindcss": "^3.4.17",
|
||||
"vite": "^6.0.6"
|
||||
}
|
||||
}
|
||||
6
src/client/postcss.config.js
Normal file
6
src/client/postcss.config.js
Normal file
@@ -0,0 +1,6 @@
|
||||
export default {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
};
|
||||
13
src/client/src/app.css
Normal file
13
src/client/src/app.css
Normal file
@@ -0,0 +1,13 @@
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
|
||||
:root {
|
||||
--color-primary: #2563eb;
|
||||
--color-primary-hover: #1d4ed8;
|
||||
}
|
||||
|
||||
body {
|
||||
@apply bg-gray-50 text-gray-900 dark:bg-gray-950 dark:text-gray-100;
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
}
|
||||
13
src/client/src/app.html
Normal file
13
src/client/src/app.html
Normal file
@@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<link rel="icon" href="%sveltekit.assets%/favicon.png" />
|
||||
<title>Price Hunter</title>
|
||||
%sveltekit.head%
|
||||
</head>
|
||||
<body data-sveltekit-prerender="true">
|
||||
<div style="display: contents">%sveltekit.body%</div>
|
||||
</body>
|
||||
</html>
|
||||
91
src/client/src/lib/api.ts
Normal file
91
src/client/src/lib/api.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
const BASE = '';
|
||||
|
||||
class ApiError extends Error {
|
||||
constructor(public status: number, public body: string) {
|
||||
super(`API error ${status}: ${body}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function api<T>(path: string, options?: RequestInit): Promise<T> {
|
||||
const res = await fetch(`${BASE}${path}`, {
|
||||
headers: { 'Content-Type': 'application/json', ...options?.headers },
|
||||
...options,
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new ApiError(res.status, await res.text());
|
||||
}
|
||||
if (res.status === 204) return undefined as T;
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export function getStores() {
|
||||
return api<any[]>('/api/stores');
|
||||
}
|
||||
|
||||
export function getStore(id: number) {
|
||||
return api<any>(`/api/stores/${id}`);
|
||||
}
|
||||
|
||||
export function createStore(data: any) {
|
||||
return api<any>('/api/stores', { method: 'POST', body: JSON.stringify(data) });
|
||||
}
|
||||
|
||||
export function updateStore(id: number, data: any) {
|
||||
return api<any>(`/api/stores/${id}`, { method: 'PUT', body: JSON.stringify(data) });
|
||||
}
|
||||
|
||||
export function toggleStore(id: number) {
|
||||
return api<any>(`/api/stores/${id}/toggle`, { method: 'PATCH' });
|
||||
}
|
||||
|
||||
export function deleteStore(id: number) {
|
||||
return api<void>(`/api/stores/${id}`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
export function getCategories() {
|
||||
return api<any[]>('/api/categories');
|
||||
}
|
||||
|
||||
export function createCategory(name: string, color?: string) {
|
||||
return api<any>('/api/categories', { method: 'POST', body: JSON.stringify({ name, color }) });
|
||||
}
|
||||
|
||||
export function updateCategory(id: number, data: any) {
|
||||
return api<any>(`/api/categories/${id}`, { method: 'PUT', body: JSON.stringify(data) });
|
||||
}
|
||||
|
||||
export function deleteCategory(id: number) {
|
||||
return api<void>(`/api/categories/${id}`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
export function getGroups() {
|
||||
return api<any[]>('/api/groups');
|
||||
}
|
||||
|
||||
export function createGroup(name: string, description?: string) {
|
||||
return api<any>('/api/groups', { method: 'POST', body: JSON.stringify({ name, description }) });
|
||||
}
|
||||
|
||||
export function updateGroupApi(id: number, data: any) {
|
||||
return api<any>(`/api/groups/${id}`, { method: 'PUT', body: JSON.stringify(data) });
|
||||
}
|
||||
|
||||
export function setGroupMembersApi(id: number, storeIds: number[]) {
|
||||
return api<any>(`/api/groups/${id}/members`, { method: 'PUT', body: JSON.stringify({ store_ids: storeIds }) });
|
||||
}
|
||||
|
||||
export function deleteGroup(id: number) {
|
||||
return api<void>(`/api/groups/${id}`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
export function searchProducts(query: string, params?: { stores?: string; category?: string; group?: string }) {
|
||||
const searchParams = new URLSearchParams({ q: query });
|
||||
if (params?.stores) searchParams.set('stores', params.stores);
|
||||
if (params?.category) searchParams.set('category', params.category);
|
||||
if (params?.group) searchParams.set('group', params.group);
|
||||
return api<any>(`/api/search?${searchParams}`);
|
||||
}
|
||||
|
||||
export function testStore(id: number, query: string) {
|
||||
return api<any>(`/api/stores/${id}/test`, { method: 'POST', body: JSON.stringify({ query }) });
|
||||
}
|
||||
22
src/client/src/routes/+layout.svelte
Normal file
22
src/client/src/routes/+layout.svelte
Normal file
@@ -0,0 +1,22 @@
|
||||
<script>
|
||||
import '../app.css';
|
||||
let { children } = $props();
|
||||
</script>
|
||||
|
||||
<div class="min-h-screen flex flex-col">
|
||||
<nav class="bg-white dark:bg-gray-900 border-b border-gray-200 dark:border-gray-800 px-4 py-3">
|
||||
<div class="max-w-7xl mx-auto flex items-center justify-between">
|
||||
<a href="/" class="text-xl font-bold text-blue-600 dark:text-blue-400 hover:text-blue-700">
|
||||
Price Hunter
|
||||
</a>
|
||||
<div class="flex gap-4 text-sm">
|
||||
<a href="/" class="text-gray-600 dark:text-gray-300 hover:text-gray-900 dark:hover:text-white">Search</a>
|
||||
<a href="/admin" class="text-gray-600 dark:text-gray-300 hover:text-gray-900 dark:hover:text-white">Stores</a>
|
||||
<a href="/admin/categories" class="text-gray-600 dark:text-gray-300 hover:text-gray-900 dark:hover:text-white">Categories</a>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
<main class="flex-1">
|
||||
{@render children()}
|
||||
</main>
|
||||
</div>
|
||||
81
src/client/src/routes/+page.svelte
Normal file
81
src/client/src/routes/+page.svelte
Normal file
@@ -0,0 +1,81 @@
|
||||
<script>
|
||||
import { goto } from '$app/navigation';
|
||||
import { getCategories, getGroups } from '$lib/api';
|
||||
import { onMount } from 'svelte';
|
||||
|
||||
let query = $state('');
|
||||
let categories = $state([]);
|
||||
let groups = $state([]);
|
||||
let selectedCategory = $state('');
|
||||
let selectedGroup = $state('');
|
||||
|
||||
onMount(async () => {
|
||||
[categories, groups] = await Promise.all([getCategories(), getGroups()]);
|
||||
});
|
||||
|
||||
function handleSearch(e) {
|
||||
e.preventDefault();
|
||||
if (!query.trim()) return;
|
||||
|
||||
const params = new URLSearchParams({ q: query.trim() });
|
||||
if (selectedCategory) params.set('category', selectedCategory);
|
||||
if (selectedGroup) params.set('group', selectedGroup);
|
||||
|
||||
goto(`/results?${params}`);
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="flex flex-col items-center justify-center min-h-[80vh] px-4">
|
||||
<h1 class="text-5xl font-bold text-gray-900 dark:text-white mb-2">Price Hunter</h1>
|
||||
<p class="text-gray-500 dark:text-gray-400 mb-8">Search across all your stores at once</p>
|
||||
|
||||
<form onsubmit={handleSearch} class="w-full max-w-2xl">
|
||||
<div class="relative">
|
||||
<input
|
||||
type="text"
|
||||
bind:value={query}
|
||||
placeholder="Search for a product..."
|
||||
class="w-full px-6 py-4 text-lg border-2 border-gray-200 dark:border-gray-700 rounded-full
|
||||
bg-white dark:bg-gray-900 text-gray-900 dark:text-white
|
||||
focus:border-blue-500 focus:outline-none shadow-sm hover:shadow-md transition-shadow"
|
||||
/>
|
||||
<button
|
||||
type="submit"
|
||||
class="absolute right-2 top-1/2 -translate-y-1/2 bg-blue-600 hover:bg-blue-700 text-white
|
||||
px-6 py-2.5 rounded-full font-medium transition-colors"
|
||||
>
|
||||
Search
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{#if categories.length > 0 || groups.length > 0}
|
||||
<div class="flex flex-wrap gap-2 mt-4 justify-center">
|
||||
{#if categories.length > 0}
|
||||
<select
|
||||
bind:value={selectedCategory}
|
||||
class="px-3 py-1.5 text-sm border border-gray-200 dark:border-gray-700 rounded-lg
|
||||
bg-white dark:bg-gray-900 text-gray-700 dark:text-gray-300"
|
||||
>
|
||||
<option value="">All categories</option>
|
||||
{#each categories as cat}
|
||||
<option value={cat.id}>{cat.name}</option>
|
||||
{/each}
|
||||
</select>
|
||||
{/if}
|
||||
|
||||
{#if groups.length > 0}
|
||||
<select
|
||||
bind:value={selectedGroup}
|
||||
class="px-3 py-1.5 text-sm border border-gray-200 dark:border-gray-700 rounded-lg
|
||||
bg-white dark:bg-gray-900 text-gray-700 dark:text-gray-300"
|
||||
>
|
||||
<option value="">All groups</option>
|
||||
{#each groups as group}
|
||||
<option value={group.id}>{group.name}</option>
|
||||
{/each}
|
||||
</select>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
</form>
|
||||
</div>
|
||||
101
src/client/src/routes/admin/+page.svelte
Normal file
101
src/client/src/routes/admin/+page.svelte
Normal file
@@ -0,0 +1,101 @@
|
||||
<script>
|
||||
import { getStores, toggleStore, deleteStore } from '$lib/api';
|
||||
import { onMount } from 'svelte';
|
||||
|
||||
let stores = $state([]);
|
||||
let loading = $state(true);
|
||||
|
||||
onMount(async () => {
|
||||
stores = await getStores();
|
||||
loading = false;
|
||||
});
|
||||
|
||||
async function handleToggle(id) {
|
||||
const updated = await toggleStore(id);
|
||||
stores = stores.map((s) => (s.id === id ? { ...s, ...updated } : s));
|
||||
}
|
||||
|
||||
async function handleDelete(id, name) {
|
||||
if (!confirm(`Delete store "${name}"? This cannot be undone.`)) return;
|
||||
await deleteStore(id);
|
||||
stores = stores.filter((s) => s.id !== id);
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="max-w-7xl mx-auto px-4 py-6">
|
||||
<div class="flex items-center justify-between mb-6">
|
||||
<h1 class="text-2xl font-bold text-gray-900 dark:text-white">Store Management</h1>
|
||||
<a
|
||||
href="/admin/stores/new"
|
||||
class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-lg text-sm font-medium transition-colors"
|
||||
>
|
||||
Add Store
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{#if loading}
|
||||
<div class="animate-pulse space-y-3">
|
||||
{#each Array(5) as _}
|
||||
<div class="bg-gray-200 dark:bg-gray-800 h-16 rounded-lg"></div>
|
||||
{/each}
|
||||
</div>
|
||||
{:else if stores.length === 0}
|
||||
<div class="text-center py-20 text-gray-500 dark:text-gray-400">
|
||||
<p class="text-lg mb-2">No stores configured yet</p>
|
||||
<a href="/admin/stores/new" class="text-blue-600 hover:underline">Add your first store</a>
|
||||
</div>
|
||||
{:else}
|
||||
<div class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 overflow-hidden">
|
||||
<table class="w-full">
|
||||
<thead>
|
||||
<tr class="bg-gray-50 dark:bg-gray-800 text-left text-sm text-gray-600 dark:text-gray-400">
|
||||
<th class="px-4 py-3 font-medium">Name</th>
|
||||
<th class="px-4 py-3 font-medium">URL</th>
|
||||
<th class="px-4 py-3 font-medium">Category</th>
|
||||
<th class="px-4 py-3 font-medium text-center">Enabled</th>
|
||||
<th class="px-4 py-3 font-medium text-right">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each stores as store}
|
||||
<tr class="border-t border-gray-100 dark:border-gray-800 hover:bg-gray-50 dark:hover:bg-gray-800/50">
|
||||
<td class="px-4 py-3">
|
||||
<span class="font-medium text-gray-900 dark:text-white">{store.name}</span>
|
||||
</td>
|
||||
<td class="px-4 py-3 text-sm text-gray-500 dark:text-gray-400 truncate max-w-xs">
|
||||
{store.base_url}
|
||||
</td>
|
||||
<td class="px-4 py-3">
|
||||
{#if store.category_name}
|
||||
<span
|
||||
class="text-xs px-2 py-0.5 rounded-full text-white"
|
||||
style="background-color: {store.category_color}"
|
||||
>
|
||||
{store.category_name}
|
||||
</span>
|
||||
{:else}
|
||||
<span class="text-xs text-gray-400">—</span>
|
||||
{/if}
|
||||
</td>
|
||||
<td class="px-4 py-3 text-center">
|
||||
<button
|
||||
onclick={() => handleToggle(store.id)}
|
||||
class="w-10 h-5 rounded-full transition-colors {store.enabled ? 'bg-green-500' : 'bg-gray-300 dark:bg-gray-600'} relative"
|
||||
>
|
||||
<span class="absolute top-0.5 {store.enabled ? 'right-0.5' : 'left-0.5'} w-4 h-4 bg-white rounded-full shadow transition-all"></span>
|
||||
</button>
|
||||
</td>
|
||||
<td class="px-4 py-3 text-right">
|
||||
<div class="flex gap-2 justify-end">
|
||||
<a href="/admin/stores/{store.id}" class="text-sm text-blue-600 hover:underline">Edit</a>
|
||||
<a href="/admin/stores/{store.id}/test" class="text-sm text-green-600 hover:underline">Test</a>
|
||||
<button onclick={() => handleDelete(store.id, store.name)} class="text-sm text-red-600 hover:underline">Delete</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
155
src/client/src/routes/admin/categories/+page.svelte
Normal file
155
src/client/src/routes/admin/categories/+page.svelte
Normal file
@@ -0,0 +1,155 @@
|
||||
<script>
|
||||
import {
|
||||
getCategories, createCategory, updateCategory, deleteCategory,
|
||||
getGroups, createGroup, updateGroupApi, deleteGroup, setGroupMembersApi, getStores,
|
||||
} from '$lib/api';
|
||||
import { onMount } from 'svelte';
|
||||
|
||||
let categories = $state([]);
|
||||
let groups = $state([]);
|
||||
let stores = $state([]);
|
||||
let loading = $state(true);
|
||||
|
||||
let newCatName = $state('');
|
||||
let newCatColor = $state('#6B7280');
|
||||
let newGroupName = $state('');
|
||||
let editingCat = $state(null);
|
||||
let editingGroup = $state(null);
|
||||
|
||||
onMount(async () => {
|
||||
[categories, groups, stores] = await Promise.all([getCategories(), getGroups(), getStores()]);
|
||||
loading = false;
|
||||
});
|
||||
|
||||
async function handleAddCategory() {
|
||||
if (!newCatName.trim()) return;
|
||||
const cat = await createCategory(newCatName.trim(), newCatColor);
|
||||
categories = [...categories, cat];
|
||||
newCatName = '';
|
||||
newCatColor = '#6B7280';
|
||||
}
|
||||
|
||||
async function handleUpdateCategory(id) {
|
||||
if (!editingCat) return;
|
||||
await updateCategory(id, { name: editingCat.name, color: editingCat.color });
|
||||
categories = categories.map((c) => (c.id === id ? { ...c, ...editingCat } : c));
|
||||
editingCat = null;
|
||||
}
|
||||
|
||||
async function handleDeleteCategory(id, name) {
|
||||
if (!confirm(`Delete category "${name}"?`)) return;
|
||||
await deleteCategory(id);
|
||||
categories = categories.filter((c) => c.id !== id);
|
||||
}
|
||||
|
||||
async function handleAddGroup() {
|
||||
if (!newGroupName.trim()) return;
|
||||
const group = await createGroup(newGroupName.trim());
|
||||
groups = [...groups, group];
|
||||
newGroupName = '';
|
||||
}
|
||||
|
||||
async function handleDeleteGroup(id, name) {
|
||||
if (!confirm(`Delete group "${name}"?`)) return;
|
||||
await deleteGroup(id);
|
||||
groups = groups.filter((g) => g.id !== id);
|
||||
}
|
||||
|
||||
async function handleToggleGroupMember(groupId, storeId) {
|
||||
const group = groups.find((g) => g.id === groupId);
|
||||
if (!group) return;
|
||||
|
||||
const ids = group.store_ids.includes(storeId)
|
||||
? group.store_ids.filter((id) => id !== storeId)
|
||||
: [...group.store_ids, storeId];
|
||||
|
||||
await setGroupMembersApi(groupId, ids);
|
||||
groups = groups.map((g) => (g.id === groupId ? { ...g, store_ids: ids } : g));
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="max-w-5xl mx-auto px-4 py-6">
|
||||
{#if loading}
|
||||
<div class="animate-pulse space-y-4">
|
||||
<div class="bg-gray-200 dark:bg-gray-800 h-40 rounded-lg"></div>
|
||||
<div class="bg-gray-200 dark:bg-gray-800 h-40 rounded-lg"></div>
|
||||
</div>
|
||||
{:else}
|
||||
<!-- Categories -->
|
||||
<section class="mb-10">
|
||||
<h1 class="text-2xl font-bold text-gray-900 dark:text-white mb-4">Categories</h1>
|
||||
|
||||
<div class="flex gap-2 mb-4">
|
||||
<input type="text" bind:value={newCatName} placeholder="Category name"
|
||||
class="flex-1 px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
<input type="color" bind:value={newCatColor}
|
||||
class="w-10 h-10 rounded border border-gray-300 dark:border-gray-600 cursor-pointer" />
|
||||
<button onclick={handleAddCategory}
|
||||
class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-lg text-sm font-medium">Add</button>
|
||||
</div>
|
||||
|
||||
{#if categories.length === 0}
|
||||
<p class="text-gray-500 dark:text-gray-400">No categories yet.</p>
|
||||
{:else}
|
||||
<div class="space-y-2">
|
||||
{#each categories as cat}
|
||||
<div class="flex items-center gap-3 bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 px-4 py-3">
|
||||
{#if editingCat?.id === cat.id}
|
||||
<input type="color" bind:value={editingCat.color} class="w-8 h-8 rounded cursor-pointer" />
|
||||
<input type="text" bind:value={editingCat.name}
|
||||
class="flex-1 px-2 py-1 border border-gray-300 dark:border-gray-600 rounded bg-white dark:bg-gray-800 text-gray-900 dark:text-white" />
|
||||
<button onclick={() => handleUpdateCategory(cat.id)} class="text-sm text-blue-600 hover:underline">Save</button>
|
||||
<button onclick={() => editingCat = null} class="text-sm text-gray-500 hover:underline">Cancel</button>
|
||||
{:else}
|
||||
<span class="w-4 h-4 rounded-full" style="background-color: {cat.color}"></span>
|
||||
<span class="flex-1 text-gray-900 dark:text-white">{cat.name}</span>
|
||||
<button onclick={() => editingCat = { id: cat.id, name: cat.name, color: cat.color }} class="text-sm text-blue-600 hover:underline">Edit</button>
|
||||
<button onclick={() => handleDeleteCategory(cat.id, cat.name)} class="text-sm text-red-600 hover:underline">Delete</button>
|
||||
{/if}
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<!-- Groups -->
|
||||
<section>
|
||||
<h1 class="text-2xl font-bold text-gray-900 dark:text-white mb-4">Custom Groups</h1>
|
||||
|
||||
<div class="flex gap-2 mb-4">
|
||||
<input type="text" bind:value={newGroupName} placeholder="Group name"
|
||||
class="flex-1 px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
<button onclick={handleAddGroup}
|
||||
class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-lg text-sm font-medium">Add</button>
|
||||
</div>
|
||||
|
||||
{#if groups.length === 0}
|
||||
<p class="text-gray-500 dark:text-gray-400">No groups yet.</p>
|
||||
{:else}
|
||||
<div class="space-y-4">
|
||||
{#each groups as group}
|
||||
<div class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-4">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<h3 class="font-medium text-gray-900 dark:text-white">{group.name}</h3>
|
||||
<button onclick={() => handleDeleteGroup(group.id, group.name)} class="text-sm text-red-600 hover:underline">Delete</button>
|
||||
</div>
|
||||
<div class="flex flex-wrap gap-2">
|
||||
{#each stores as store}
|
||||
<button
|
||||
onclick={() => handleToggleGroupMember(group.id, store.id)}
|
||||
class="text-xs px-3 py-1 rounded-full border transition-colors
|
||||
{group.store_ids.includes(store.id)
|
||||
? 'bg-blue-100 dark:bg-blue-900/30 border-blue-300 dark:border-blue-700 text-blue-700 dark:text-blue-300'
|
||||
: 'bg-gray-50 dark:bg-gray-800 border-gray-200 dark:border-gray-700 text-gray-500 dark:text-gray-400 hover:border-gray-300'}"
|
||||
>
|
||||
{store.name}
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</section>
|
||||
{/if}
|
||||
</div>
|
||||
200
src/client/src/routes/admin/stores/[id]/+page.svelte
Normal file
200
src/client/src/routes/admin/stores/[id]/+page.svelte
Normal file
@@ -0,0 +1,200 @@
|
||||
<script>
|
||||
import { page } from '$app/stores';
|
||||
import { goto } from '$app/navigation';
|
||||
import { getStore, updateStore, getCategories } from '$lib/api';
|
||||
import { onMount } from 'svelte';
|
||||
|
||||
let categories = $state([]);
|
||||
let error = $state('');
|
||||
let saving = $state(false);
|
||||
let loading = $state(true);
|
||||
|
||||
let form = $state({
|
||||
name: '',
|
||||
base_url: '',
|
||||
search_url: '',
|
||||
sel_container: '',
|
||||
sel_name: '',
|
||||
sel_price: '',
|
||||
sel_link: '',
|
||||
sel_image: '',
|
||||
category_id: '',
|
||||
currency: 'EUR',
|
||||
rate_limit: 2,
|
||||
user_agent: '',
|
||||
proxy_url: '',
|
||||
headers_json: '',
|
||||
});
|
||||
|
||||
onMount(async () => {
|
||||
const id = $page.params.id;
|
||||
const [store, cats] = await Promise.all([getStore(Number(id)), getCategories()]);
|
||||
categories = cats;
|
||||
|
||||
form = {
|
||||
name: store.name,
|
||||
base_url: store.base_url,
|
||||
search_url: store.search_url,
|
||||
sel_container: store.sel_container,
|
||||
sel_name: store.sel_name,
|
||||
sel_price: store.sel_price,
|
||||
sel_link: store.sel_link,
|
||||
sel_image: store.sel_image || '',
|
||||
category_id: store.category_id?.toString() || '',
|
||||
currency: store.currency,
|
||||
rate_limit: store.rate_limit,
|
||||
user_agent: store.user_agent || '',
|
||||
proxy_url: store.proxy_url || '',
|
||||
headers_json: store.headers_json || '',
|
||||
};
|
||||
loading = false;
|
||||
});
|
||||
|
||||
async function handleSubmit(e) {
|
||||
e.preventDefault();
|
||||
error = '';
|
||||
saving = true;
|
||||
|
||||
try {
|
||||
const data = { ...form };
|
||||
if (data.category_id) data.category_id = Number(data.category_id);
|
||||
else delete data.category_id;
|
||||
|
||||
await updateStore(Number($page.params.id), data);
|
||||
goto('/admin');
|
||||
} catch (err) {
|
||||
error = err.message || 'Failed to update store';
|
||||
} finally {
|
||||
saving = false;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="max-w-3xl mx-auto px-4 py-6">
|
||||
<div class="flex items-center justify-between mb-6">
|
||||
<h1 class="text-2xl font-bold text-gray-900 dark:text-white">Edit Store</h1>
|
||||
<a href="/admin/stores/{$page.params.id}/test"
|
||||
class="bg-green-600 hover:bg-green-700 text-white px-4 py-2 rounded-lg text-sm font-medium">
|
||||
Test Store
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{#if loading}
|
||||
<div class="animate-pulse space-y-4">
|
||||
<div class="bg-gray-200 dark:bg-gray-800 h-40 rounded-lg"></div>
|
||||
<div class="bg-gray-200 dark:bg-gray-800 h-60 rounded-lg"></div>
|
||||
</div>
|
||||
{:else}
|
||||
{#if error}
|
||||
<div class="bg-red-50 dark:bg-red-900/20 text-red-600 dark:text-red-400 px-4 py-3 rounded-lg mb-4">{error}</div>
|
||||
{/if}
|
||||
|
||||
<form onsubmit={handleSubmit} class="space-y-6">
|
||||
<section class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-6">
|
||||
<h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-4">Basic Information</h2>
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Store Name *</label>
|
||||
<input type="text" bind:value={form.name} required
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Category</label>
|
||||
<select bind:value={form.category_id}
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white">
|
||||
<option value="">No category</option>
|
||||
{#each categories as cat}
|
||||
<option value={cat.id}>{cat.name}</option>
|
||||
{/each}
|
||||
</select>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Base URL *</label>
|
||||
<input type="url" bind:value={form.base_url} required
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Search URL *</label>
|
||||
<input type="text" bind:value={form.search_url} required
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
<p class="text-xs text-gray-500 mt-1">Use {'{query}'} as placeholder</p>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Currency</label>
|
||||
<input type="text" bind:value={form.currency}
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-6">
|
||||
<h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-4">CSS Selectors</h2>
|
||||
<div class="space-y-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Product Container *</label>
|
||||
<input type="text" bind:value={form.sel_container} required
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Product Name *</label>
|
||||
<input type="text" bind:value={form.sel_name} required
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Price *</label>
|
||||
<input type="text" bind:value={form.sel_price} required
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Product Link *</label>
|
||||
<input type="text" bind:value={form.sel_link} required
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Image</label>
|
||||
<input type="text" bind:value={form.sel_image}
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-6">
|
||||
<h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-4">Advanced Settings</h2>
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Rate Limit</label>
|
||||
<input type="number" bind:value={form.rate_limit} min="1" max="10"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">User Agent</label>
|
||||
<input type="text" bind:value={form.user_agent}
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Proxy URL</label>
|
||||
<input type="text" bind:value={form.proxy_url}
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Extra Headers (JSON)</label>
|
||||
<input type="text" bind:value={form.headers_json}
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="flex gap-3">
|
||||
<button type="submit" disabled={saving}
|
||||
class="bg-blue-600 hover:bg-blue-700 disabled:bg-blue-400 text-white px-6 py-2.5 rounded-lg font-medium">
|
||||
{saving ? 'Saving...' : 'Save Changes'}
|
||||
</button>
|
||||
<a href="/admin" class="px-6 py-2.5 rounded-lg border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-800">
|
||||
Cancel
|
||||
</a>
|
||||
</div>
|
||||
</form>
|
||||
{/if}
|
||||
</div>
|
||||
200
src/client/src/routes/admin/stores/[id]/test/+page.svelte
Normal file
200
src/client/src/routes/admin/stores/[id]/test/+page.svelte
Normal file
@@ -0,0 +1,200 @@
|
||||
<script>
|
||||
import { page } from '$app/stores';
|
||||
import { getStore, testStore } from '$lib/api';
|
||||
import { onMount } from 'svelte';
|
||||
|
||||
let store = $state(null);
|
||||
let query = $state('');
|
||||
let result = $state(null);
|
||||
let testing = $state(false);
|
||||
let loading = $state(true);
|
||||
|
||||
onMount(async () => {
|
||||
store = await getStore(Number($page.params.id));
|
||||
loading = false;
|
||||
});
|
||||
|
||||
async function handleTest(e) {
|
||||
e.preventDefault();
|
||||
if (!query.trim()) return;
|
||||
testing = true;
|
||||
result = null;
|
||||
|
||||
try {
|
||||
result = await testStore(Number($page.params.id), query.trim());
|
||||
} catch (err) {
|
||||
result = { success: false, error: err.message };
|
||||
} finally {
|
||||
testing = false;
|
||||
}
|
||||
}
|
||||
|
||||
function formatPrice(price, currency) {
|
||||
if (price === null || price === undefined) return 'N/A';
|
||||
try {
|
||||
return new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(price);
|
||||
} catch {
|
||||
return `${currency} ${price.toFixed(2)}`;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="max-w-7xl mx-auto px-4 py-6">
|
||||
{#if loading}
|
||||
<div class="animate-pulse">
|
||||
<div class="bg-gray-200 dark:bg-gray-800 h-8 w-48 rounded mb-4"></div>
|
||||
<div class="bg-gray-200 dark:bg-gray-800 h-12 rounded mb-4"></div>
|
||||
</div>
|
||||
{:else if store}
|
||||
<div class="flex items-center gap-4 mb-6">
|
||||
<a href="/admin/stores/{store.id}" class="text-gray-400 hover:text-gray-600">←</a>
|
||||
<div>
|
||||
<h1 class="text-2xl font-bold text-gray-900 dark:text-white">Test: {store.name}</h1>
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">{store.base_url}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Health Stats -->
|
||||
{#if store.health}
|
||||
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mb-6">
|
||||
<div class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-4">
|
||||
<div class="text-2xl font-bold text-gray-900 dark:text-white">{store.health.total}</div>
|
||||
<div class="text-xs text-gray-500">Total Scrapes</div>
|
||||
</div>
|
||||
<div class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-4">
|
||||
<div class="text-2xl font-bold text-green-600">{store.health.successful}</div>
|
||||
<div class="text-xs text-gray-500">Successful</div>
|
||||
</div>
|
||||
<div class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-4">
|
||||
<div class="text-2xl font-bold text-red-600">{store.health.failed}</div>
|
||||
<div class="text-xs text-gray-500">Failed</div>
|
||||
</div>
|
||||
<div class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-4">
|
||||
<div class="text-2xl font-bold text-gray-900 dark:text-white">{store.health.avg_duration_ms}ms</div>
|
||||
<div class="text-xs text-gray-500">Avg Duration</div>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Test Form -->
|
||||
<form onsubmit={handleTest} class="mb-6">
|
||||
<div class="flex gap-2">
|
||||
<input
|
||||
type="text"
|
||||
bind:value={query}
|
||||
placeholder="Enter a test search query..."
|
||||
class="flex-1 px-4 py-2.5 border border-gray-300 dark:border-gray-600 rounded-lg
|
||||
bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none"
|
||||
/>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={testing}
|
||||
class="bg-green-600 hover:bg-green-700 disabled:bg-green-400 text-white px-6 py-2.5 rounded-lg font-medium"
|
||||
>
|
||||
{testing ? 'Testing...' : 'Run Test'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<!-- Test Results -->
|
||||
{#if result}
|
||||
<div class="space-y-6">
|
||||
<!-- Status Banner -->
|
||||
<div class="p-4 rounded-lg {result.success ? 'bg-green-50 dark:bg-green-900/20 text-green-700 dark:text-green-400' : 'bg-red-50 dark:bg-red-900/20 text-red-700 dark:text-red-400'}">
|
||||
<div class="font-medium">
|
||||
{result.success ? `Found ${result.itemsFound} products` : 'Test failed'}
|
||||
</div>
|
||||
{#if result.searchUrl}
|
||||
<div class="text-sm mt-1 break-all opacity-80">URL: {result.searchUrl}</div>
|
||||
{/if}
|
||||
{#if result.duration}
|
||||
<div class="text-sm mt-1 opacity-80">Duration: {result.duration}ms</div>
|
||||
{/if}
|
||||
{#if result.error}
|
||||
<div class="text-sm mt-1">{result.error}</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if result.success}
|
||||
<!-- Parsed Results -->
|
||||
<div>
|
||||
<h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-3">Parsed Products ({result.parsedProducts?.length || 0})</h2>
|
||||
{#if result.parsedProducts?.length > 0}
|
||||
<div class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 overflow-hidden">
|
||||
<table class="w-full text-sm">
|
||||
<thead>
|
||||
<tr class="bg-gray-50 dark:bg-gray-800 text-left">
|
||||
<th class="px-4 py-2 font-medium text-gray-600 dark:text-gray-400">Name</th>
|
||||
<th class="px-4 py-2 font-medium text-gray-600 dark:text-gray-400">Price</th>
|
||||
<th class="px-4 py-2 font-medium text-gray-600 dark:text-gray-400">Raw Price</th>
|
||||
<th class="px-4 py-2 font-medium text-gray-600 dark:text-gray-400">Link</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each result.parsedProducts as product}
|
||||
<tr class="border-t border-gray-100 dark:border-gray-800">
|
||||
<td class="px-4 py-2 text-gray-900 dark:text-white max-w-xs truncate">{product.name}</td>
|
||||
<td class="px-4 py-2 font-medium text-blue-600 dark:text-blue-400">{formatPrice(product.price, product.currency)}</td>
|
||||
<td class="px-4 py-2 text-gray-500 font-mono text-xs">{product.priceText}</td>
|
||||
<td class="px-4 py-2">
|
||||
<a href={product.url} target="_blank" rel="noopener" class="text-blue-600 hover:underline truncate block max-w-xs">
|
||||
{product.url}
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{:else}
|
||||
<p class="text-gray-500 dark:text-gray-400">No products parsed. Check your CSS selectors.</p>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- Raw HTML Preview -->
|
||||
<details class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800">
|
||||
<summary class="px-4 py-3 cursor-pointer font-medium text-gray-900 dark:text-white hover:bg-gray-50 dark:hover:bg-gray-800">
|
||||
Raw HTML Preview ({result.rawHtmlLength?.toLocaleString()} bytes)
|
||||
</summary>
|
||||
<div class="px-4 py-3 border-t border-gray-200 dark:border-gray-800">
|
||||
<pre class="text-xs text-gray-600 dark:text-gray-400 overflow-x-auto whitespace-pre-wrap max-h-96 overflow-y-auto">{result.rawHtmlPreview}</pre>
|
||||
</div>
|
||||
</details>
|
||||
{/if}
|
||||
|
||||
<!-- Recent Logs -->
|
||||
{#if result.recentLogs?.length > 0}
|
||||
<div>
|
||||
<h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-3">Recent Scrape Logs</h2>
|
||||
<div class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 overflow-hidden">
|
||||
<table class="w-full text-sm">
|
||||
<thead>
|
||||
<tr class="bg-gray-50 dark:bg-gray-800 text-left">
|
||||
<th class="px-4 py-2 font-medium text-gray-600 dark:text-gray-400">Status</th>
|
||||
<th class="px-4 py-2 font-medium text-gray-600 dark:text-gray-400">Query</th>
|
||||
<th class="px-4 py-2 font-medium text-gray-600 dark:text-gray-400">Results</th>
|
||||
<th class="px-4 py-2 font-medium text-gray-600 dark:text-gray-400">Duration</th>
|
||||
<th class="px-4 py-2 font-medium text-gray-600 dark:text-gray-400">Time</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each result.recentLogs as log}
|
||||
<tr class="border-t border-gray-100 dark:border-gray-800">
|
||||
<td class="px-4 py-2">
|
||||
<span class="inline-block w-2 h-2 rounded-full {log.success ? 'bg-green-500' : 'bg-red-500'}"></span>
|
||||
</td>
|
||||
<td class="px-4 py-2 text-gray-700 dark:text-gray-300">{log.query}</td>
|
||||
<td class="px-4 py-2 text-gray-700 dark:text-gray-300">{log.result_count}</td>
|
||||
<td class="px-4 py-2 text-gray-500">{log.duration_ms}ms</td>
|
||||
<td class="px-4 py-2 text-gray-500 text-xs">{log.scraped_at}</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
{/if}
|
||||
</div>
|
||||
174
src/client/src/routes/admin/stores/new/+page.svelte
Normal file
174
src/client/src/routes/admin/stores/new/+page.svelte
Normal file
@@ -0,0 +1,174 @@
|
||||
<script>
|
||||
import { goto } from '$app/navigation';
|
||||
import { createStore, getCategories } from '$lib/api';
|
||||
import { onMount } from 'svelte';
|
||||
|
||||
let categories = $state([]);
|
||||
let error = $state('');
|
||||
let saving = $state(false);
|
||||
|
||||
let form = $state({
|
||||
name: '',
|
||||
base_url: '',
|
||||
search_url: '',
|
||||
sel_container: '',
|
||||
sel_name: '',
|
||||
sel_price: '',
|
||||
sel_link: '',
|
||||
sel_image: '',
|
||||
category_id: '',
|
||||
currency: 'EUR',
|
||||
rate_limit: 2,
|
||||
user_agent: '',
|
||||
proxy_url: '',
|
||||
headers_json: '',
|
||||
});
|
||||
|
||||
onMount(async () => {
|
||||
categories = await getCategories();
|
||||
});
|
||||
|
||||
async function handleSubmit(e) {
|
||||
e.preventDefault();
|
||||
error = '';
|
||||
saving = true;
|
||||
|
||||
try {
|
||||
const data = { ...form };
|
||||
if (data.category_id) data.category_id = Number(data.category_id);
|
||||
else delete data.category_id;
|
||||
if (!data.sel_image) delete data.sel_image;
|
||||
if (!data.user_agent) delete data.user_agent;
|
||||
if (!data.proxy_url) delete data.proxy_url;
|
||||
if (!data.headers_json) delete data.headers_json;
|
||||
|
||||
const store = await createStore(data);
|
||||
goto(`/admin/stores/${store.id}/test`);
|
||||
} catch (err) {
|
||||
error = err.message || 'Failed to create store';
|
||||
} finally {
|
||||
saving = false;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="max-w-3xl mx-auto px-4 py-6">
|
||||
<h1 class="text-2xl font-bold text-gray-900 dark:text-white mb-6">Add New Store</h1>
|
||||
|
||||
{#if error}
|
||||
<div class="bg-red-50 dark:bg-red-900/20 text-red-600 dark:text-red-400 px-4 py-3 rounded-lg mb-4">{error}</div>
|
||||
{/if}
|
||||
|
||||
<form onsubmit={handleSubmit} class="space-y-6">
|
||||
<!-- Basic Info -->
|
||||
<section class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-6">
|
||||
<h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-4">Basic Information</h2>
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Store Name *</label>
|
||||
<input type="text" bind:value={form.name} required
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Category</label>
|
||||
<select bind:value={form.category_id}
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white">
|
||||
<option value="">No category</option>
|
||||
{#each categories as cat}
|
||||
<option value={cat.id}>{cat.name}</option>
|
||||
{/each}
|
||||
</select>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Base URL *</label>
|
||||
<input type="url" bind:value={form.base_url} required placeholder="https://store.example.com"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Search URL *</label>
|
||||
<input type="text" bind:value={form.search_url} required placeholder={'https://store.example.com/search?q={query}'}
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
<p class="text-xs text-gray-500 mt-1">Use {'{query}'} as placeholder for the search term</p>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Currency</label>
|
||||
<input type="text" bind:value={form.currency} placeholder="EUR"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- CSS Selectors -->
|
||||
<section class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-6">
|
||||
<h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-4">CSS Selectors</h2>
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400 mb-4">Define how to extract product data from the store's search results page.</p>
|
||||
<div class="space-y-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Product Container *</label>
|
||||
<input type="text" bind:value={form.sel_container} required placeholder=".product-card"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
<p class="text-xs text-gray-500 mt-1">Selector for each product listing item</p>
|
||||
</div>
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Product Name *</label>
|
||||
<input type="text" bind:value={form.sel_name} required placeholder=".product-title"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Price *</label>
|
||||
<input type="text" bind:value={form.sel_price} required placeholder=".product-price"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Product Link *</label>
|
||||
<input type="text" bind:value={form.sel_link} required placeholder="a"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Image (optional)</label>
|
||||
<input type="text" bind:value={form.sel_image} placeholder="img"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Advanced -->
|
||||
<section class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-6">
|
||||
<h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-4">Advanced Settings</h2>
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Rate Limit (req/sec)</label>
|
||||
<input type="number" bind:value={form.rate_limit} min="1" max="10"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">User Agent</label>
|
||||
<input type="text" bind:value={form.user_agent} placeholder="Default browser UA"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Proxy URL</label>
|
||||
<input type="text" bind:value={form.proxy_url} placeholder="http://user:pass@host:port"
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">Extra Headers (JSON)</label>
|
||||
<input type="text" bind:value={form.headers_json} placeholder={'{"X-Custom": "value"}'}
|
||||
class="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 text-gray-900 dark:text-white font-mono text-sm focus:border-blue-500 focus:outline-none" />
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="flex gap-3">
|
||||
<button type="submit" disabled={saving}
|
||||
class="bg-blue-600 hover:bg-blue-700 disabled:bg-blue-400 text-white px-6 py-2.5 rounded-lg font-medium transition-colors">
|
||||
{saving ? 'Creating...' : 'Create Store'}
|
||||
</button>
|
||||
<a href="/admin" class="px-6 py-2.5 rounded-lg border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-800">
|
||||
Cancel
|
||||
</a>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
186
src/client/src/routes/results/+page.svelte
Normal file
186
src/client/src/routes/results/+page.svelte
Normal file
@@ -0,0 +1,186 @@
|
||||
<script>
|
||||
import { page } from '$app/stores';
|
||||
import { goto } from '$app/navigation';
|
||||
import { searchProducts } from '$lib/api';
|
||||
import { onMount } from 'svelte';
|
||||
|
||||
let query = $state('');
|
||||
let results = $state([]);
|
||||
let meta = $state(null);
|
||||
let loading = $state(true);
|
||||
let sortBy = $state('price');
|
||||
|
||||
onMount(async () => {
|
||||
const params = $page.url.searchParams;
|
||||
query = params.get('q') || '';
|
||||
|
||||
if (!query) {
|
||||
goto('/');
|
||||
return;
|
||||
}
|
||||
|
||||
await doSearch();
|
||||
});
|
||||
|
||||
async function doSearch() {
|
||||
if (!query.trim()) return;
|
||||
loading = true;
|
||||
|
||||
try {
|
||||
const params = $page.url.searchParams;
|
||||
const data = await searchProducts(query, {
|
||||
category: params.get('category') || undefined,
|
||||
group: params.get('group') || undefined,
|
||||
stores: params.get('stores') || undefined,
|
||||
});
|
||||
results = data.results;
|
||||
meta = data.meta;
|
||||
} catch (err) {
|
||||
console.error('Search failed:', err);
|
||||
results = [];
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
|
||||
function handleSearch(e) {
|
||||
e.preventDefault();
|
||||
if (!query.trim()) return;
|
||||
goto(`/results?q=${encodeURIComponent(query.trim())}`);
|
||||
doSearch();
|
||||
}
|
||||
|
||||
let sortedResults = $derived(() => {
|
||||
const sorted = [...results];
|
||||
switch (sortBy) {
|
||||
case 'price':
|
||||
sorted.sort((a, b) => (a.price ?? Infinity) - (b.price ?? Infinity));
|
||||
break;
|
||||
case 'price-desc':
|
||||
sorted.sort((a, b) => (b.price ?? -Infinity) - (a.price ?? -Infinity));
|
||||
break;
|
||||
case 'name':
|
||||
sorted.sort((a, b) => a.name.localeCompare(b.name));
|
||||
break;
|
||||
case 'store':
|
||||
sorted.sort((a, b) => a.storeName.localeCompare(b.storeName));
|
||||
break;
|
||||
}
|
||||
return sorted;
|
||||
});
|
||||
|
||||
function formatPrice(price, currency) {
|
||||
if (price === null || price === undefined) return 'N/A';
|
||||
try {
|
||||
return new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(price);
|
||||
} catch {
|
||||
return `${currency} ${price.toFixed(2)}`;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="max-w-7xl mx-auto px-4 py-6">
|
||||
<!-- Search bar -->
|
||||
<form onsubmit={handleSearch} class="mb-6">
|
||||
<div class="relative max-w-2xl">
|
||||
<input
|
||||
type="text"
|
||||
bind:value={query}
|
||||
placeholder="Search for a product..."
|
||||
class="w-full px-4 py-2.5 border border-gray-200 dark:border-gray-700 rounded-full
|
||||
bg-white dark:bg-gray-900 text-gray-900 dark:text-white focus:border-blue-500 focus:outline-none"
|
||||
/>
|
||||
<button type="submit" class="absolute right-2 top-1/2 -translate-y-1/2 bg-blue-600 hover:bg-blue-700
|
||||
text-white px-4 py-1.5 rounded-full text-sm">Search</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<!-- Meta info & sort -->
|
||||
{#if meta}
|
||||
<div class="flex items-center justify-between mb-4">
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">
|
||||
{meta.totalResults} results from {meta.storeCount} stores in {meta.duration}ms
|
||||
</p>
|
||||
<select
|
||||
bind:value={sortBy}
|
||||
class="text-sm border border-gray-200 dark:border-gray-700 rounded-lg px-3 py-1.5
|
||||
bg-white dark:bg-gray-900 text-gray-700 dark:text-gray-300"
|
||||
>
|
||||
<option value="price">Price: Low to High</option>
|
||||
<option value="price-desc">Price: High to Low</option>
|
||||
<option value="name">Name</option>
|
||||
<option value="store">Store</option>
|
||||
</select>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Errors -->
|
||||
{#if meta?.errors?.length > 0}
|
||||
<div class="mb-4 space-y-1">
|
||||
{#each meta.errors as err}
|
||||
<div class="text-sm text-red-600 dark:text-red-400 bg-red-50 dark:bg-red-900/20 px-3 py-1.5 rounded">
|
||||
{err.storeName}: {err.error}
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Loading -->
|
||||
{#if loading}
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4">
|
||||
{#each Array(8) as _}
|
||||
<div class="animate-pulse bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800 p-4">
|
||||
<div class="bg-gray-200 dark:bg-gray-700 h-40 rounded mb-3"></div>
|
||||
<div class="bg-gray-200 dark:bg-gray-700 h-4 rounded w-3/4 mb-2"></div>
|
||||
<div class="bg-gray-200 dark:bg-gray-700 h-6 rounded w-1/3"></div>
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{:else if sortedResults().length === 0}
|
||||
<div class="text-center py-20 text-gray-500 dark:text-gray-400">
|
||||
<p class="text-lg">No results found for "{query}"</p>
|
||||
<p class="text-sm mt-2">Try a different search term or check your store configurations.</p>
|
||||
</div>
|
||||
{:else}
|
||||
<!-- Results grid -->
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4">
|
||||
{#each sortedResults() as product}
|
||||
<a
|
||||
href={product.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="bg-white dark:bg-gray-900 rounded-lg border border-gray-200 dark:border-gray-800
|
||||
hover:shadow-lg transition-shadow overflow-hidden group"
|
||||
>
|
||||
{#if product.image}
|
||||
<div class="h-40 bg-gray-100 dark:bg-gray-800 flex items-center justify-center overflow-hidden">
|
||||
<img
|
||||
src={product.image}
|
||||
alt={product.name}
|
||||
class="max-h-full max-w-full object-contain p-2 group-hover:scale-105 transition-transform"
|
||||
/>
|
||||
</div>
|
||||
{:else}
|
||||
<div class="h-40 bg-gray-100 dark:bg-gray-800 flex items-center justify-center">
|
||||
<span class="text-gray-400 text-4xl">?</span>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<div class="p-4">
|
||||
<h3 class="text-sm font-medium text-gray-900 dark:text-white line-clamp-2 mb-2">
|
||||
{product.name}
|
||||
</h3>
|
||||
<div class="flex items-center justify-between">
|
||||
<span class="text-lg font-bold text-blue-600 dark:text-blue-400">
|
||||
{formatPrice(product.price, product.currency)}
|
||||
</span>
|
||||
<span class="text-xs text-gray-500 dark:text-gray-400 bg-gray-100 dark:bg-gray-800 px-2 py-0.5 rounded">
|
||||
{product.storeName}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</a>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
16
src/client/svelte.config.js
Normal file
16
src/client/svelte.config.js
Normal file
@@ -0,0 +1,16 @@
|
||||
import adapter from '@sveltejs/adapter-static';
|
||||
|
||||
/** @type {import('@sveltejs/kit').Config} */
|
||||
const config = {
|
||||
kit: {
|
||||
adapter: adapter({
|
||||
pages: 'build',
|
||||
assets: 'build',
|
||||
fallback: 'index.html',
|
||||
precompress: false,
|
||||
strict: false,
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
export default config;
|
||||
9
src/client/tailwind.config.js
Normal file
9
src/client/tailwind.config.js
Normal file
@@ -0,0 +1,9 @@
|
||||
/** @type {import('tailwindcss').Config} */
|
||||
export default {
|
||||
content: ['./src/**/*.{html,js,svelte,ts}'],
|
||||
darkMode: 'media',
|
||||
theme: {
|
||||
extend: {},
|
||||
},
|
||||
plugins: [],
|
||||
};
|
||||
14
src/client/vite.config.ts
Normal file
14
src/client/vite.config.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { sveltekit } from '@sveltejs/kit/vite';
|
||||
import { defineConfig } from 'vite';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [sveltekit()],
|
||||
server: {
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:3000',
|
||||
changeOrigin: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
8
src/server/config.ts
Normal file
8
src/server/config.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
import 'dotenv/config';
|
||||
|
||||
export const config = {
|
||||
port: parseInt(process.env.PORT || '3000', 10),
|
||||
host: process.env.HOST || '0.0.0.0',
|
||||
databasePath: process.env.DATABASE_PATH || './data/pricehunter.db',
|
||||
isProduction: process.env.NODE_ENV === 'production',
|
||||
};
|
||||
60
src/server/db/connection.ts
Normal file
60
src/server/db/connection.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import initSqlJs, { type Database } from 'sql.js';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { config } from '../config.js';
|
||||
|
||||
let db: Database;
|
||||
|
||||
export async function initDatabase(): Promise<Database> {
|
||||
if (db) return db;
|
||||
|
||||
const SQL = await initSqlJs();
|
||||
|
||||
const dbDir = path.dirname(config.databasePath);
|
||||
if (!fs.existsSync(dbDir)) {
|
||||
fs.mkdirSync(dbDir, { recursive: true });
|
||||
}
|
||||
|
||||
if (fs.existsSync(config.databasePath)) {
|
||||
const buffer = fs.readFileSync(config.databasePath);
|
||||
db = new SQL.Database(buffer);
|
||||
} else {
|
||||
db = new SQL.Database();
|
||||
}
|
||||
|
||||
db.run('PRAGMA journal_mode = WAL');
|
||||
db.run('PRAGMA foreign_keys = ON');
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
export function getDatabase(): Database {
|
||||
if (!db) {
|
||||
throw new Error('Database not initialized. Call initDatabase() first.');
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
export function saveDatabase(): void {
|
||||
if (!db) return;
|
||||
const data = db.export();
|
||||
const buffer = Buffer.from(data);
|
||||
fs.writeFileSync(config.databasePath, buffer);
|
||||
}
|
||||
|
||||
// Auto-save periodically
|
||||
let saveInterval: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
export function startAutoSave(intervalMs = 5000): void {
|
||||
if (saveInterval) return;
|
||||
saveInterval = setInterval(() => {
|
||||
try { saveDatabase(); } catch { /* ignore */ }
|
||||
}, intervalMs);
|
||||
}
|
||||
|
||||
export function stopAutoSave(): void {
|
||||
if (saveInterval) {
|
||||
clearInterval(saveInterval);
|
||||
saveInterval = null;
|
||||
}
|
||||
}
|
||||
51
src/server/db/migrate.ts
Normal file
51
src/server/db/migrate.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { getDatabase, saveDatabase } from './connection.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export function runMigrations(): void {
|
||||
const db = getDatabase();
|
||||
|
||||
db.run(`
|
||||
CREATE TABLE IF NOT EXISTS _migrations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
applied_at TEXT DEFAULT (datetime('now'))
|
||||
)
|
||||
`);
|
||||
|
||||
const migrationsDir = path.join(__dirname, 'migrations');
|
||||
|
||||
// In production (compiled), migrations may be alongside the compiled JS
|
||||
const altMigrationsDir = path.join(__dirname, '..', '..', '..', 'src', 'server', 'db', 'migrations');
|
||||
const dir = fs.existsSync(migrationsDir) ? migrationsDir : (fs.existsSync(altMigrationsDir) ? altMigrationsDir : null);
|
||||
|
||||
if (!dir) {
|
||||
console.warn('No migrations directory found');
|
||||
return;
|
||||
}
|
||||
|
||||
const files = fs.readdirSync(dir)
|
||||
.filter((f) => f.endsWith('.sql'))
|
||||
.sort();
|
||||
|
||||
const appliedStmt = db.prepare('SELECT name FROM _migrations');
|
||||
const applied = new Set<string>();
|
||||
while (appliedStmt.step()) {
|
||||
applied.add(appliedStmt.getAsObject().name as string);
|
||||
}
|
||||
appliedStmt.free();
|
||||
|
||||
for (const file of files) {
|
||||
if (applied.has(file)) continue;
|
||||
|
||||
const sql = fs.readFileSync(path.join(dir, file), 'utf-8');
|
||||
db.run(sql);
|
||||
db.run('INSERT INTO _migrations (name) VALUES (?)', [file]);
|
||||
console.log(`Migration applied: ${file}`);
|
||||
}
|
||||
|
||||
saveDatabase();
|
||||
}
|
||||
57
src/server/db/migrations/001_initial.sql
Normal file
57
src/server/db/migrations/001_initial.sql
Normal file
@@ -0,0 +1,57 @@
|
||||
CREATE TABLE categories (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
color TEXT DEFAULT '#6B7280',
|
||||
sort_order INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE stores (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
base_url TEXT NOT NULL,
|
||||
search_url TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
sel_container TEXT NOT NULL,
|
||||
sel_name TEXT NOT NULL,
|
||||
sel_price TEXT NOT NULL,
|
||||
sel_link TEXT NOT NULL,
|
||||
sel_image TEXT,
|
||||
rate_limit INTEGER DEFAULT 2,
|
||||
rate_window INTEGER DEFAULT 1000,
|
||||
proxy_url TEXT,
|
||||
user_agent TEXT,
|
||||
headers_json TEXT,
|
||||
currency TEXT DEFAULT 'EUR',
|
||||
category_id INTEGER REFERENCES categories(id) ON DELETE SET NULL,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE store_groups (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE store_group_members (
|
||||
group_id INTEGER REFERENCES store_groups(id) ON DELETE CASCADE,
|
||||
store_id INTEGER REFERENCES stores(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (group_id, store_id)
|
||||
);
|
||||
|
||||
CREATE TABLE scrape_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
store_id INTEGER REFERENCES stores(id) ON DELETE CASCADE,
|
||||
query TEXT NOT NULL,
|
||||
success INTEGER NOT NULL,
|
||||
result_count INTEGER DEFAULT 0,
|
||||
duration_ms INTEGER,
|
||||
error_message TEXT,
|
||||
scraped_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX idx_scrape_logs_store_id ON scrape_logs(store_id);
|
||||
CREATE INDEX idx_scrape_logs_scraped_at ON scrape_logs(scraped_at);
|
||||
CREATE INDEX idx_stores_enabled ON stores(enabled);
|
||||
CREATE INDEX idx_stores_category_id ON stores(category_id);
|
||||
66
src/server/index.ts
Normal file
66
src/server/index.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import Fastify from 'fastify';
|
||||
import cors from '@fastify/cors';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { config } from './config.js';
|
||||
import { initDatabase, startAutoSave, saveDatabase } from './db/connection.js';
|
||||
import { runMigrations } from './db/migrate.js';
|
||||
import { storeRoutes } from './routes/stores.js';
|
||||
import { categoryRoutes } from './routes/categories.js';
|
||||
import { searchRoutes } from './routes/search.js';
|
||||
import { testRoutes } from './routes/test.js';
|
||||
import { healthRoutes } from './routes/health.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const app = Fastify({
|
||||
logger: {
|
||||
level: config.isProduction ? 'info' : 'debug',
|
||||
transport: config.isProduction ? undefined : { target: 'pino-pretty' },
|
||||
},
|
||||
});
|
||||
|
||||
await app.register(cors, { origin: true });
|
||||
|
||||
// API routes
|
||||
await app.register(storeRoutes, { prefix: '/api' });
|
||||
await app.register(categoryRoutes, { prefix: '/api' });
|
||||
await app.register(searchRoutes, { prefix: '/api' });
|
||||
await app.register(testRoutes, { prefix: '/api' });
|
||||
await app.register(healthRoutes, { prefix: '/api' });
|
||||
|
||||
// Serve static frontend in production
|
||||
if (config.isProduction) {
|
||||
const clientPath = path.join(__dirname, '..', 'client');
|
||||
await app.register(fastifyStatic, {
|
||||
root: clientPath,
|
||||
wildcard: false,
|
||||
});
|
||||
|
||||
// SPA fallback: serve index.html for all non-API routes
|
||||
app.setNotFoundHandler((request, reply) => {
|
||||
if (request.url.startsWith('/api')) {
|
||||
reply.code(404).send({ error: 'Not found' });
|
||||
} else {
|
||||
reply.sendFile('index.html');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Initialize database and run migrations
|
||||
await initDatabase();
|
||||
runMigrations();
|
||||
startAutoSave();
|
||||
|
||||
// Save database on shutdown
|
||||
process.on('SIGINT', () => { saveDatabase(); process.exit(0); });
|
||||
process.on('SIGTERM', () => { saveDatabase(); process.exit(0); });
|
||||
|
||||
try {
|
||||
await app.listen({ port: config.port, host: config.host });
|
||||
app.log.info(`Price Hunter running at http://localhost:${config.port}`);
|
||||
} catch (err) {
|
||||
app.log.error(err);
|
||||
process.exit(1);
|
||||
}
|
||||
139
src/server/models/category.ts
Normal file
139
src/server/models/category.ts
Normal file
@@ -0,0 +1,139 @@
|
||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||
|
||||
export interface Category {
|
||||
id: number;
|
||||
name: string;
|
||||
color: string;
|
||||
sort_order: number;
|
||||
}
|
||||
|
||||
export interface StoreGroup {
|
||||
id: number;
|
||||
name: string;
|
||||
description: string | null;
|
||||
}
|
||||
|
||||
export interface StoreGroupWithMembers extends StoreGroup {
|
||||
store_ids: number[];
|
||||
}
|
||||
|
||||
function queryAll(sql: string, params: any[] = []): any[] {
|
||||
const db = getDatabase();
|
||||
const stmt = db.prepare(sql);
|
||||
if (params.length) stmt.bind(params);
|
||||
const rows: any[] = [];
|
||||
while (stmt.step()) {
|
||||
rows.push(stmt.getAsObject());
|
||||
}
|
||||
stmt.free();
|
||||
return rows;
|
||||
}
|
||||
|
||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
||||
const rows = queryAll(sql, params);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
// Categories
|
||||
|
||||
export function getAllCategories(): Category[] {
|
||||
return queryAll('SELECT * FROM categories ORDER BY sort_order, name');
|
||||
}
|
||||
|
||||
export function getCategoryById(id: number): Category | undefined {
|
||||
return queryOne('SELECT * FROM categories WHERE id = ?', [id]);
|
||||
}
|
||||
|
||||
export function createCategory(name: string, color?: string): Category {
|
||||
const db = getDatabase();
|
||||
const maxOrder = queryOne('SELECT MAX(sort_order) as max_order FROM categories');
|
||||
const sortOrder = (maxOrder?.max_order ?? -1) + 1;
|
||||
db.run('INSERT INTO categories (name, color, sort_order) VALUES (?, ?, ?)', [name, color || '#6B7280', sortOrder]);
|
||||
const lastId = queryOne('SELECT last_insert_rowid() as id')?.id;
|
||||
saveDatabase();
|
||||
return getCategoryById(lastId) as Category;
|
||||
}
|
||||
|
||||
export function updateCategory(id: number, data: { name?: string; color?: string; sort_order?: number }): Category | undefined {
|
||||
const db = getDatabase();
|
||||
const fields: string[] = [];
|
||||
const values: any[] = [];
|
||||
|
||||
if (data.name !== undefined) { fields.push('name = ?'); values.push(data.name); }
|
||||
if (data.color !== undefined) { fields.push('color = ?'); values.push(data.color); }
|
||||
if (data.sort_order !== undefined) { fields.push('sort_order = ?'); values.push(data.sort_order); }
|
||||
|
||||
if (fields.length === 0) return getCategoryById(id);
|
||||
|
||||
values.push(id);
|
||||
db.run(`UPDATE categories SET ${fields.join(', ')} WHERE id = ?`, values);
|
||||
saveDatabase();
|
||||
return getCategoryById(id);
|
||||
}
|
||||
|
||||
export function deleteCategory(id: number): boolean {
|
||||
const db = getDatabase();
|
||||
db.run('DELETE FROM categories WHERE id = ?', [id]);
|
||||
const changes = db.getRowsModified();
|
||||
if (changes > 0) saveDatabase();
|
||||
return changes > 0;
|
||||
}
|
||||
|
||||
// Groups
|
||||
|
||||
export function getAllGroups(): StoreGroupWithMembers[] {
|
||||
const groups = queryAll('SELECT * FROM store_groups ORDER BY name');
|
||||
return groups.map((group) => {
|
||||
const members = queryAll('SELECT store_id FROM store_group_members WHERE group_id = ?', [group.id]);
|
||||
return { ...group, store_ids: members.map((m: any) => m.store_id) };
|
||||
});
|
||||
}
|
||||
|
||||
export function getGroupById(id: number): StoreGroupWithMembers | undefined {
|
||||
const group = queryOne('SELECT * FROM store_groups WHERE id = ?', [id]);
|
||||
if (!group) return undefined;
|
||||
|
||||
const members = queryAll('SELECT store_id FROM store_group_members WHERE group_id = ?', [id]);
|
||||
return { ...group, store_ids: members.map((m: any) => m.store_id) };
|
||||
}
|
||||
|
||||
export function createGroup(name: string, description?: string): StoreGroupWithMembers {
|
||||
const db = getDatabase();
|
||||
db.run('INSERT INTO store_groups (name, description) VALUES (?, ?)', [name, description || null]);
|
||||
const lastId = queryOne('SELECT last_insert_rowid() as id')?.id;
|
||||
saveDatabase();
|
||||
return getGroupById(lastId) as StoreGroupWithMembers;
|
||||
}
|
||||
|
||||
export function updateGroup(id: number, data: { name?: string; description?: string }): StoreGroupWithMembers | undefined {
|
||||
const db = getDatabase();
|
||||
const fields: string[] = [];
|
||||
const values: any[] = [];
|
||||
|
||||
if (data.name !== undefined) { fields.push('name = ?'); values.push(data.name); }
|
||||
if (data.description !== undefined) { fields.push('description = ?'); values.push(data.description); }
|
||||
|
||||
if (fields.length > 0) {
|
||||
values.push(id);
|
||||
db.run(`UPDATE store_groups SET ${fields.join(', ')} WHERE id = ?`, values);
|
||||
saveDatabase();
|
||||
}
|
||||
return getGroupById(id);
|
||||
}
|
||||
|
||||
export function deleteGroup(id: number): boolean {
|
||||
const db = getDatabase();
|
||||
db.run('DELETE FROM store_groups WHERE id = ?', [id]);
|
||||
const changes = db.getRowsModified();
|
||||
if (changes > 0) saveDatabase();
|
||||
return changes > 0;
|
||||
}
|
||||
|
||||
export function setGroupMembers(groupId: number, storeIds: number[]): void {
|
||||
const db = getDatabase();
|
||||
db.run('DELETE FROM store_group_members WHERE group_id = ?', [groupId]);
|
||||
for (const storeId of storeIds) {
|
||||
db.run('INSERT INTO store_group_members (group_id, store_id) VALUES (?, ?)', [groupId, storeId]);
|
||||
}
|
||||
saveDatabase();
|
||||
}
|
||||
85
src/server/models/scrape-log.ts
Normal file
85
src/server/models/scrape-log.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||
|
||||
export interface ScrapeLog {
|
||||
id: number;
|
||||
store_id: number;
|
||||
query: string;
|
||||
success: number;
|
||||
result_count: number;
|
||||
duration_ms: number;
|
||||
error_message: string | null;
|
||||
scraped_at: string;
|
||||
}
|
||||
|
||||
function queryAll(sql: string, params: any[] = []): any[] {
|
||||
const db = getDatabase();
|
||||
const stmt = db.prepare(sql);
|
||||
if (params.length) stmt.bind(params);
|
||||
const rows: any[] = [];
|
||||
while (stmt.step()) {
|
||||
rows.push(stmt.getAsObject());
|
||||
}
|
||||
stmt.free();
|
||||
return rows;
|
||||
}
|
||||
|
||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
||||
const rows = queryAll(sql, params);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
export function logScrape(
|
||||
storeId: number,
|
||||
query: string,
|
||||
success: boolean,
|
||||
resultCount: number,
|
||||
durationMs: number,
|
||||
errorMessage?: string
|
||||
): ScrapeLog {
|
||||
const db = getDatabase();
|
||||
db.run(`
|
||||
INSERT INTO scrape_logs (store_id, query, success, result_count, duration_ms, error_message)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`, [storeId, query, success ? 1 : 0, resultCount, durationMs, errorMessage || null]);
|
||||
|
||||
const lastId = queryOne('SELECT last_insert_rowid() as id')?.id;
|
||||
saveDatabase();
|
||||
return queryOne('SELECT * FROM scrape_logs WHERE id = ?', [lastId]) as ScrapeLog;
|
||||
}
|
||||
|
||||
export function getLogsByStore(storeId: number, limit = 20): ScrapeLog[] {
|
||||
return queryAll('SELECT * FROM scrape_logs WHERE store_id = ? ORDER BY scraped_at DESC LIMIT ?', [storeId, limit]);
|
||||
}
|
||||
|
||||
export function getStoreHealth(storeId: number): {
|
||||
total: number;
|
||||
successful: number;
|
||||
failed: number;
|
||||
avg_duration_ms: number;
|
||||
last_success: string | null;
|
||||
last_error: string | null;
|
||||
} {
|
||||
const stats = queryOne(`
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful,
|
||||
SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failed,
|
||||
AVG(duration_ms) as avg_duration_ms,
|
||||
MAX(CASE WHEN success = 1 THEN scraped_at END) as last_success
|
||||
FROM scrape_logs WHERE store_id = ?
|
||||
`, [storeId]);
|
||||
|
||||
const lastError = queryOne(
|
||||
'SELECT error_message FROM scrape_logs WHERE store_id = ? AND success = 0 ORDER BY scraped_at DESC LIMIT 1',
|
||||
[storeId]
|
||||
);
|
||||
|
||||
return {
|
||||
total: stats?.total || 0,
|
||||
successful: stats?.successful || 0,
|
||||
failed: stats?.failed || 0,
|
||||
avg_duration_ms: Math.round(stats?.avg_duration_ms || 0),
|
||||
last_success: stats?.last_success || null,
|
||||
last_error: lastError?.error_message || null,
|
||||
};
|
||||
}
|
||||
174
src/server/models/store.ts
Normal file
174
src/server/models/store.ts
Normal file
@@ -0,0 +1,174 @@
|
||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||
|
||||
export interface Store {
|
||||
id: number;
|
||||
name: string;
|
||||
slug: string;
|
||||
base_url: string;
|
||||
search_url: string;
|
||||
enabled: number;
|
||||
sel_container: string;
|
||||
sel_name: string;
|
||||
sel_price: string;
|
||||
sel_link: string;
|
||||
sel_image: string | null;
|
||||
rate_limit: number;
|
||||
rate_window: number;
|
||||
proxy_url: string | null;
|
||||
user_agent: string | null;
|
||||
headers_json: string | null;
|
||||
currency: string;
|
||||
category_id: number | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface StoreWithCategory extends Store {
|
||||
category_name: string | null;
|
||||
category_color: string | null;
|
||||
}
|
||||
|
||||
export interface CreateStoreInput {
|
||||
name: string;
|
||||
slug?: string;
|
||||
base_url: string;
|
||||
search_url: string;
|
||||
sel_container: string;
|
||||
sel_name: string;
|
||||
sel_price: string;
|
||||
sel_link: string;
|
||||
sel_image?: string;
|
||||
rate_limit?: number;
|
||||
rate_window?: number;
|
||||
proxy_url?: string;
|
||||
user_agent?: string;
|
||||
headers_json?: string;
|
||||
currency?: string;
|
||||
category_id?: number;
|
||||
}
|
||||
|
||||
function queryAll(sql: string, params: any[] = []): any[] {
|
||||
const db = getDatabase();
|
||||
const stmt = db.prepare(sql);
|
||||
if (params.length) stmt.bind(params);
|
||||
const rows: any[] = [];
|
||||
while (stmt.step()) {
|
||||
rows.push(stmt.getAsObject());
|
||||
}
|
||||
stmt.free();
|
||||
return rows;
|
||||
}
|
||||
|
||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
||||
const rows = queryAll(sql, params);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
function slugify(text: string): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '');
|
||||
}
|
||||
|
||||
export function getAllStores(): StoreWithCategory[] {
|
||||
return queryAll(`
|
||||
SELECT s.*, c.name as category_name, c.color as category_color
|
||||
FROM stores s
|
||||
LEFT JOIN categories c ON s.category_id = c.id
|
||||
ORDER BY s.name
|
||||
`);
|
||||
}
|
||||
|
||||
export function getStoreById(id: number): StoreWithCategory | undefined {
|
||||
return queryOne(`
|
||||
SELECT s.*, c.name as category_name, c.color as category_color
|
||||
FROM stores s
|
||||
LEFT JOIN categories c ON s.category_id = c.id
|
||||
WHERE s.id = ?
|
||||
`, [id]);
|
||||
}
|
||||
|
||||
export function getEnabledStores(): Store[] {
|
||||
return queryAll('SELECT * FROM stores WHERE enabled = 1 ORDER BY name');
|
||||
}
|
||||
|
||||
export function getStoresByCategory(categoryId: number): Store[] {
|
||||
return queryAll('SELECT * FROM stores WHERE enabled = 1 AND category_id = ? ORDER BY name', [categoryId]);
|
||||
}
|
||||
|
||||
export function getStoresByGroup(groupId: number): Store[] {
|
||||
return queryAll(`
|
||||
SELECT s.* FROM stores s
|
||||
JOIN store_group_members sgm ON s.id = sgm.store_id
|
||||
WHERE s.enabled = 1 AND sgm.group_id = ?
|
||||
ORDER BY s.name
|
||||
`, [groupId]);
|
||||
}
|
||||
|
||||
export function getStoresByIds(ids: number[]): Store[] {
|
||||
if (ids.length === 0) return [];
|
||||
const placeholders = ids.map(() => '?').join(',');
|
||||
return queryAll(`SELECT * FROM stores WHERE enabled = 1 AND id IN (${placeholders}) ORDER BY name`, ids);
|
||||
}
|
||||
|
||||
export function createStore(input: CreateStoreInput): Store {
|
||||
const db = getDatabase();
|
||||
const slug = input.slug || slugify(input.name);
|
||||
|
||||
db.run(`
|
||||
INSERT INTO stores (name, slug, base_url, search_url, sel_container, sel_name, sel_price, sel_link, sel_image,
|
||||
rate_limit, rate_window, proxy_url, user_agent, headers_json, currency, category_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, [
|
||||
input.name, slug, input.base_url, input.search_url,
|
||||
input.sel_container, input.sel_name, input.sel_price, input.sel_link, input.sel_image || null,
|
||||
input.rate_limit ?? 2, input.rate_window ?? 1000,
|
||||
input.proxy_url || null, input.user_agent || null, input.headers_json || null,
|
||||
input.currency || 'EUR', input.category_id || null,
|
||||
]);
|
||||
|
||||
const lastId = queryOne('SELECT last_insert_rowid() as id')?.id;
|
||||
saveDatabase();
|
||||
return getStoreById(lastId) as Store;
|
||||
}
|
||||
|
||||
export function updateStore(id: number, input: Partial<CreateStoreInput>): Store | undefined {
|
||||
const existing = getStoreById(id);
|
||||
if (!existing) return undefined;
|
||||
|
||||
const db = getDatabase();
|
||||
const fields: string[] = [];
|
||||
const values: any[] = [];
|
||||
|
||||
for (const [key, value] of Object.entries(input)) {
|
||||
if (value !== undefined) {
|
||||
fields.push(`${key} = ?`);
|
||||
values.push(value);
|
||||
}
|
||||
}
|
||||
|
||||
if (fields.length === 0) return existing;
|
||||
|
||||
fields.push("updated_at = datetime('now')");
|
||||
values.push(id);
|
||||
|
||||
db.run(`UPDATE stores SET ${fields.join(', ')} WHERE id = ?`, values);
|
||||
saveDatabase();
|
||||
return getStoreById(id);
|
||||
}
|
||||
|
||||
export function toggleStoreEnabled(id: number): Store | undefined {
|
||||
const db = getDatabase();
|
||||
db.run("UPDATE stores SET enabled = CASE WHEN enabled = 1 THEN 0 ELSE 1 END, updated_at = datetime('now') WHERE id = ?", [id]);
|
||||
saveDatabase();
|
||||
return getStoreById(id);
|
||||
}
|
||||
|
||||
export function deleteStore(id: number): boolean {
|
||||
const db = getDatabase();
|
||||
db.run('DELETE FROM stores WHERE id = ?', [id]);
|
||||
const changes = db.getRowsModified();
|
||||
if (changes > 0) saveDatabase();
|
||||
return changes > 0;
|
||||
}
|
||||
98
src/server/routes/categories.ts
Normal file
98
src/server/routes/categories.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
import type { FastifyPluginAsync } from 'fastify';
|
||||
import {
|
||||
getAllCategories, createCategory, updateCategory, deleteCategory,
|
||||
getAllGroups, createGroup, updateGroup, deleteGroup, setGroupMembers,
|
||||
} from '../models/category.js';
|
||||
|
||||
export const categoryRoutes: FastifyPluginAsync = async (app) => {
|
||||
// Categories
|
||||
app.get('/categories', async () => getAllCategories());
|
||||
|
||||
app.post<{ Body: { name: string; color?: string } }>('/categories', {
|
||||
schema: {
|
||||
body: {
|
||||
type: 'object',
|
||||
required: ['name'],
|
||||
properties: {
|
||||
name: { type: 'string', minLength: 1 },
|
||||
color: { type: 'string' },
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
try {
|
||||
const category = createCategory(request.body.name, request.body.color);
|
||||
return reply.code(201).send(category);
|
||||
} catch (err: any) {
|
||||
if (err.message?.includes('UNIQUE constraint failed')) {
|
||||
return reply.code(409).send({ error: 'Category already exists' });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
app.put<{ Params: { id: string }; Body: { name?: string; color?: string; sort_order?: number } }>('/categories/:id', async (request, reply) => {
|
||||
const result = updateCategory(Number(request.params.id), request.body);
|
||||
if (!result) return reply.code(404).send({ error: 'Category not found' });
|
||||
return result;
|
||||
});
|
||||
|
||||
app.delete<{ Params: { id: string } }>('/categories/:id', async (request, reply) => {
|
||||
const deleted = deleteCategory(Number(request.params.id));
|
||||
if (!deleted) return reply.code(404).send({ error: 'Category not found' });
|
||||
return reply.code(204).send();
|
||||
});
|
||||
|
||||
// Groups
|
||||
app.get('/groups', async () => getAllGroups());
|
||||
|
||||
app.post<{ Body: { name: string; description?: string } }>('/groups', {
|
||||
schema: {
|
||||
body: {
|
||||
type: 'object',
|
||||
required: ['name'],
|
||||
properties: {
|
||||
name: { type: 'string', minLength: 1 },
|
||||
description: { type: 'string' },
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
try {
|
||||
const group = createGroup(request.body.name, request.body.description);
|
||||
return reply.code(201).send(group);
|
||||
} catch (err: any) {
|
||||
if (err.message?.includes('UNIQUE constraint failed')) {
|
||||
return reply.code(409).send({ error: 'Group already exists' });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
app.put<{ Params: { id: string }; Body: { name?: string; description?: string } }>('/groups/:id', async (request, reply) => {
|
||||
const result = updateGroup(Number(request.params.id), request.body);
|
||||
if (!result) return reply.code(404).send({ error: 'Group not found' });
|
||||
return result;
|
||||
});
|
||||
|
||||
app.put<{ Params: { id: string }; Body: { store_ids: number[] } }>('/groups/:id/members', {
|
||||
schema: {
|
||||
body: {
|
||||
type: 'object',
|
||||
required: ['store_ids'],
|
||||
properties: {
|
||||
store_ids: { type: 'array', items: { type: 'number' } },
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
setGroupMembers(Number(request.params.id), request.body.store_ids);
|
||||
return { success: true };
|
||||
});
|
||||
|
||||
app.delete<{ Params: { id: string } }>('/groups/:id', async (request, reply) => {
|
||||
const deleted = deleteGroup(Number(request.params.id));
|
||||
if (!deleted) return reply.code(404).send({ error: 'Group not found' });
|
||||
return reply.code(204).send();
|
||||
});
|
||||
};
|
||||
37
src/server/routes/health.ts
Normal file
37
src/server/routes/health.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import type { FastifyPluginAsync } from 'fastify';
|
||||
import fs from 'node:fs';
|
||||
import { config } from '../config.js';
|
||||
import { getDatabase } from '../db/connection.js';
|
||||
|
||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
||||
const db = getDatabase();
|
||||
const stmt = db.prepare(sql);
|
||||
if (params.length) stmt.bind(params);
|
||||
let result: any;
|
||||
if (stmt.step()) {
|
||||
result = stmt.getAsObject();
|
||||
}
|
||||
stmt.free();
|
||||
return result;
|
||||
}
|
||||
|
||||
export const healthRoutes: FastifyPluginAsync = async (app) => {
|
||||
app.get('/health', async () => {
|
||||
const storeCount = queryOne('SELECT COUNT(*) as count FROM stores')?.count ?? 0;
|
||||
const enabledCount = queryOne('SELECT COUNT(*) as count FROM stores WHERE enabled = 1')?.count ?? 0;
|
||||
|
||||
let dbSizeBytes = 0;
|
||||
try {
|
||||
const stats = fs.statSync(config.databasePath);
|
||||
dbSizeBytes = stats.size;
|
||||
} catch {
|
||||
// DB file may not exist yet
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'ok',
|
||||
stores: { total: storeCount, enabled: enabledCount },
|
||||
database: { sizeBytes: dbSizeBytes, sizeMB: Math.round(dbSizeBytes / 1024 / 1024 * 100) / 100 },
|
||||
};
|
||||
});
|
||||
};
|
||||
39
src/server/routes/search.ts
Normal file
39
src/server/routes/search.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import type { FastifyPluginAsync } from 'fastify';
|
||||
import { search } from '../scraper/engine.js';
|
||||
|
||||
export const searchRoutes: FastifyPluginAsync = async (app) => {
|
||||
app.get<{
|
||||
Querystring: {
|
||||
q: string;
|
||||
stores?: string;
|
||||
category?: string;
|
||||
group?: string;
|
||||
};
|
||||
}>('/search', {
|
||||
schema: {
|
||||
querystring: {
|
||||
type: 'object',
|
||||
required: ['q'],
|
||||
properties: {
|
||||
q: { type: 'string', minLength: 1 },
|
||||
stores: { type: 'string' },
|
||||
category: { type: 'string' },
|
||||
group: { type: 'string' },
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (request) => {
|
||||
const { q, stores, category, group } = request.query;
|
||||
|
||||
const storeIds = stores
|
||||
? stores.split(',').map(Number).filter((n) => !isNaN(n))
|
||||
: undefined;
|
||||
|
||||
return search({
|
||||
query: q,
|
||||
storeIds,
|
||||
categoryId: category ? Number(category) : undefined,
|
||||
groupId: group ? Number(group) : undefined,
|
||||
});
|
||||
});
|
||||
};
|
||||
73
src/server/routes/stores.ts
Normal file
73
src/server/routes/stores.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
import type { FastifyPluginAsync } from 'fastify';
|
||||
import { getAllStores, getStoreById, createStore, updateStore, toggleStoreEnabled, deleteStore } from '../models/store.js';
|
||||
import { getLogsByStore, getStoreHealth } from '../models/scrape-log.js';
|
||||
|
||||
export const storeRoutes: FastifyPluginAsync = async (app) => {
|
||||
app.get('/stores', async () => {
|
||||
return getAllStores();
|
||||
});
|
||||
|
||||
app.get<{ Params: { id: string } }>('/stores/:id', async (request, reply) => {
|
||||
const store = getStoreById(Number(request.params.id));
|
||||
if (!store) return reply.code(404).send({ error: 'Store not found' });
|
||||
|
||||
const health = getStoreHealth(store.id);
|
||||
const recentLogs = getLogsByStore(store.id, 10);
|
||||
return { ...store, health, recentLogs };
|
||||
});
|
||||
|
||||
app.post<{ Body: any }>('/stores', {
|
||||
schema: {
|
||||
body: {
|
||||
type: 'object',
|
||||
required: ['name', 'base_url', 'search_url', 'sel_container', 'sel_name', 'sel_price', 'sel_link'],
|
||||
properties: {
|
||||
name: { type: 'string', minLength: 1 },
|
||||
slug: { type: 'string' },
|
||||
base_url: { type: 'string', minLength: 1 },
|
||||
search_url: { type: 'string', minLength: 1 },
|
||||
sel_container: { type: 'string', minLength: 1 },
|
||||
sel_name: { type: 'string', minLength: 1 },
|
||||
sel_price: { type: 'string', minLength: 1 },
|
||||
sel_link: { type: 'string', minLength: 1 },
|
||||
sel_image: { type: 'string' },
|
||||
rate_limit: { type: 'number' },
|
||||
rate_window: { type: 'number' },
|
||||
proxy_url: { type: 'string' },
|
||||
user_agent: { type: 'string' },
|
||||
headers_json: { type: 'string' },
|
||||
currency: { type: 'string' },
|
||||
category_id: { type: 'number' },
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
try {
|
||||
const store = createStore(request.body);
|
||||
return reply.code(201).send(store);
|
||||
} catch (err: any) {
|
||||
if (err.message?.includes('UNIQUE constraint failed')) {
|
||||
return reply.code(409).send({ error: 'A store with this slug already exists' });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
app.put<{ Params: { id: string }; Body: any }>('/stores/:id', async (request, reply) => {
|
||||
const store = updateStore(Number(request.params.id), request.body);
|
||||
if (!store) return reply.code(404).send({ error: 'Store not found' });
|
||||
return store;
|
||||
});
|
||||
|
||||
app.patch<{ Params: { id: string } }>('/stores/:id/toggle', async (request, reply) => {
|
||||
const store = toggleStoreEnabled(Number(request.params.id));
|
||||
if (!store) return reply.code(404).send({ error: 'Store not found' });
|
||||
return store;
|
||||
});
|
||||
|
||||
app.delete<{ Params: { id: string } }>('/stores/:id', async (request, reply) => {
|
||||
const deleted = deleteStore(Number(request.params.id));
|
||||
if (!deleted) return reply.code(404).send({ error: 'Store not found' });
|
||||
return reply.code(204).send();
|
||||
});
|
||||
};
|
||||
66
src/server/routes/test.ts
Normal file
66
src/server/routes/test.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import type { FastifyPluginAsync } from 'fastify';
|
||||
import { getStoreById } from '../models/store.js';
|
||||
import { logScrape, getLogsByStore, getStoreHealth } from '../models/scrape-log.js';
|
||||
import { scrapeStore } from '../scraper/http-scraper.js';
|
||||
import { normalizeResult } from '../scraper/result-parser.js';
|
||||
|
||||
export const testRoutes: FastifyPluginAsync = async (app) => {
|
||||
app.post<{
|
||||
Params: { id: string };
|
||||
Body: { query: string };
|
||||
}>('/stores/:id/test', {
|
||||
schema: {
|
||||
body: {
|
||||
type: 'object',
|
||||
required: ['query'],
|
||||
properties: {
|
||||
query: { type: 'string', minLength: 1 },
|
||||
},
|
||||
},
|
||||
},
|
||||
}, async (request, reply) => {
|
||||
const store = getStoreById(Number(request.params.id));
|
||||
if (!store) return reply.code(404).send({ error: 'Store not found' });
|
||||
|
||||
const searchUrl = store.search_url.replace('{query}', encodeURIComponent(request.body.query));
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const result = await scrapeStore(store, searchUrl);
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
const products = result.items.map((item) =>
|
||||
normalizeResult(item, store.id, store.name, store.base_url, store.currency)
|
||||
);
|
||||
|
||||
logScrape(store.id, request.body.query, true, products.length, duration);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
searchUrl,
|
||||
statusCode: result.statusCode,
|
||||
duration,
|
||||
rawHtmlLength: result.html.length,
|
||||
rawHtmlPreview: result.html.substring(0, 5000),
|
||||
itemsFound: result.items.length,
|
||||
rawItems: result.items,
|
||||
parsedProducts: products,
|
||||
health: getStoreHealth(store.id),
|
||||
recentLogs: getLogsByStore(store.id, 10),
|
||||
};
|
||||
} catch (err) {
|
||||
const duration = Date.now() - startTime;
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
logScrape(store.id, request.body.query, false, 0, duration, errorMessage);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
searchUrl,
|
||||
duration,
|
||||
error: errorMessage,
|
||||
health: getStoreHealth(store.id),
|
||||
recentLogs: getLogsByStore(store.id, 10),
|
||||
};
|
||||
}
|
||||
});
|
||||
};
|
||||
120
src/server/scraper/engine.ts
Normal file
120
src/server/scraper/engine.ts
Normal file
@@ -0,0 +1,120 @@
|
||||
import pLimit from 'p-limit';
|
||||
import type { Store } from '../models/store.js';
|
||||
import { getEnabledStores, getStoresByCategory, getStoresByGroup, getStoresByIds } from '../models/store.js';
|
||||
import { logScrape } from '../models/scrape-log.js';
|
||||
import { scrapeStore } from './http-scraper.js';
|
||||
import { normalizeResult, type Product } from './result-parser.js';
|
||||
import { getLimiter } from './rate-limiter.js';
|
||||
|
||||
const MAX_CONCURRENCY = 5;
|
||||
const SEARCH_TIMEOUT = 60_000;
|
||||
|
||||
export interface SearchOptions {
|
||||
query: string;
|
||||
storeIds?: number[];
|
||||
categoryId?: number;
|
||||
groupId?: number;
|
||||
}
|
||||
|
||||
export interface SearchResult {
|
||||
results: Product[];
|
||||
meta: {
|
||||
query: string;
|
||||
duration: number;
|
||||
storeCount: number;
|
||||
totalResults: number;
|
||||
errors: Array<{ storeId: number; storeName: string; error: string }>;
|
||||
};
|
||||
}
|
||||
|
||||
export async function search(options: SearchOptions): Promise<SearchResult> {
|
||||
const startTime = Date.now();
|
||||
const { query } = options;
|
||||
|
||||
// Determine which stores to scrape
|
||||
let stores: Store[];
|
||||
if (options.storeIds?.length) {
|
||||
stores = getStoresByIds(options.storeIds);
|
||||
} else if (options.groupId) {
|
||||
stores = getStoresByGroup(options.groupId);
|
||||
} else if (options.categoryId) {
|
||||
stores = getStoresByCategory(options.categoryId);
|
||||
} else {
|
||||
stores = getEnabledStores();
|
||||
}
|
||||
|
||||
if (stores.length === 0) {
|
||||
return {
|
||||
results: [],
|
||||
meta: { query, duration: Date.now() - startTime, storeCount: 0, totalResults: 0, errors: [] },
|
||||
};
|
||||
}
|
||||
|
||||
const limit = pLimit(MAX_CONCURRENCY);
|
||||
const errors: SearchResult['meta']['errors'] = [];
|
||||
const allProducts: Product[] = [];
|
||||
|
||||
// Create an overall timeout
|
||||
const timeoutPromise = new Promise<never>((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Search timeout')), SEARCH_TIMEOUT)
|
||||
);
|
||||
|
||||
const scrapePromises = stores.map((store) =>
|
||||
limit(async () => {
|
||||
const searchUrl = store.search_url.replace('{query}', encodeURIComponent(query));
|
||||
const storeStart = Date.now();
|
||||
const rateLimiter = getLimiter(store.id, 1, Math.floor(store.rate_window / store.rate_limit));
|
||||
|
||||
try {
|
||||
const result = await rateLimiter.schedule(() => scrapeStore(store, searchUrl));
|
||||
const duration = Date.now() - storeStart;
|
||||
|
||||
const products = result.items.map((item) =>
|
||||
normalizeResult(item, store.id, store.name, store.base_url, store.currency)
|
||||
);
|
||||
|
||||
logScrape(store.id, query, true, products.length, duration);
|
||||
return products;
|
||||
} catch (err) {
|
||||
const duration = Date.now() - storeStart;
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
logScrape(store.id, query, false, 0, duration, errorMessage);
|
||||
errors.push({ storeId: store.id, storeName: store.name, error: errorMessage });
|
||||
return [];
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
try {
|
||||
const results = await Promise.race([
|
||||
Promise.all(scrapePromises),
|
||||
timeoutPromise,
|
||||
]) as Product[][];
|
||||
|
||||
for (const products of results) {
|
||||
allProducts.push(...products);
|
||||
}
|
||||
} catch (err) {
|
||||
// Timeout — collect whatever we have
|
||||
errors.push({ storeId: 0, storeName: 'System', error: 'Search timed out' });
|
||||
}
|
||||
|
||||
// Sort by price ascending, nulls last
|
||||
allProducts.sort((a, b) => {
|
||||
if (a.price === null && b.price === null) return 0;
|
||||
if (a.price === null) return 1;
|
||||
if (b.price === null) return -1;
|
||||
return a.price - b.price;
|
||||
});
|
||||
|
||||
return {
|
||||
results: allProducts,
|
||||
meta: {
|
||||
query,
|
||||
duration: Date.now() - startTime,
|
||||
storeCount: stores.length,
|
||||
totalResults: allProducts.length,
|
||||
errors,
|
||||
},
|
||||
};
|
||||
}
|
||||
64
src/server/scraper/http-scraper.ts
Normal file
64
src/server/scraper/http-scraper.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
import * as cheerio from 'cheerio';
|
||||
import type { Store } from '../models/store.js';
|
||||
import type { ScrapedItem } from './result-parser.js';
|
||||
|
||||
const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
const DEFAULT_TIMEOUT = 10_000;
|
||||
|
||||
export interface ScrapeResult {
|
||||
items: ScrapedItem[];
|
||||
html: string;
|
||||
statusCode: number;
|
||||
}
|
||||
|
||||
export async function scrapeStore(store: Store, searchUrl: string): Promise<ScrapeResult> {
|
||||
const headers: Record<string, string> = {
|
||||
'User-Agent': store.user_agent || DEFAULT_USER_AGENT,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
};
|
||||
|
||||
if (store.headers_json) {
|
||||
try {
|
||||
const extra = JSON.parse(store.headers_json);
|
||||
Object.assign(headers, extra);
|
||||
} catch {
|
||||
// Ignore invalid headers JSON
|
||||
}
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), DEFAULT_TIMEOUT);
|
||||
|
||||
try {
|
||||
const response = await fetch(searchUrl, {
|
||||
headers,
|
||||
signal: controller.signal,
|
||||
redirect: 'follow',
|
||||
});
|
||||
|
||||
const html = await response.text();
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const items: ScrapedItem[] = [];
|
||||
const containers = $(store.sel_container);
|
||||
|
||||
containers.each((_, el) => {
|
||||
const $el = $(el);
|
||||
const name = $el.find(store.sel_name).first().text().trim();
|
||||
const priceText = $el.find(store.sel_price).first().text().trim();
|
||||
const link = $el.find(store.sel_link).first().attr('href') || '';
|
||||
const image = store.sel_image
|
||||
? $el.find(store.sel_image).first().attr('src') || $el.find(store.sel_image).first().attr('data-src') || null
|
||||
: null;
|
||||
|
||||
if (name && priceText) {
|
||||
items.push({ name, priceText, link, image });
|
||||
}
|
||||
});
|
||||
|
||||
return { items, html, statusCode: response.status };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
22
src/server/scraper/rate-limiter.ts
Normal file
22
src/server/scraper/rate-limiter.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import Bottleneck from 'bottleneck';
|
||||
|
||||
const limiters = new Map<number, Bottleneck>();
|
||||
|
||||
export function getLimiter(storeId: number, maxConcurrent: number, minTime: number): Bottleneck {
|
||||
let limiter = limiters.get(storeId);
|
||||
if (!limiter) {
|
||||
limiter = new Bottleneck({
|
||||
maxConcurrent,
|
||||
minTime,
|
||||
});
|
||||
limiters.set(storeId, limiter);
|
||||
}
|
||||
return limiter;
|
||||
}
|
||||
|
||||
export function clearLimiters(): void {
|
||||
for (const limiter of limiters.values()) {
|
||||
limiter.disconnect();
|
||||
}
|
||||
limiters.clear();
|
||||
}
|
||||
83
src/server/scraper/result-parser.ts
Normal file
83
src/server/scraper/result-parser.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
export interface ScrapedItem {
|
||||
name: string;
|
||||
priceText: string;
|
||||
link: string;
|
||||
image: string | null;
|
||||
}
|
||||
|
||||
export interface Product {
|
||||
name: string;
|
||||
price: number | null;
|
||||
priceText: string;
|
||||
currency: string;
|
||||
url: string;
|
||||
image: string | null;
|
||||
storeName: string;
|
||||
storeId: number;
|
||||
}
|
||||
|
||||
export function parsePrice(text: string): number | null {
|
||||
if (!text) return null;
|
||||
|
||||
const cleaned = text.trim().toLowerCase();
|
||||
if (cleaned === 'free' || cleaned === 'gratis') return 0;
|
||||
|
||||
// Handle range prices like "$12 - $15" — take the lower bound
|
||||
const rangeParts = cleaned.split(/\s*[-–]\s*/);
|
||||
const priceStr = rangeParts[0];
|
||||
|
||||
// Remove currency symbols and whitespace
|
||||
let normalized = priceStr.replace(/[^\d.,]/g, '').trim();
|
||||
|
||||
if (!normalized) return null;
|
||||
|
||||
// Determine decimal separator:
|
||||
// "1.299,00" or "1 299,00" → comma is decimal
|
||||
// "1,299.00" → period is decimal
|
||||
// "12,99" → comma is decimal (no thousands)
|
||||
// "12.99" → period is decimal
|
||||
|
||||
const lastComma = normalized.lastIndexOf(',');
|
||||
const lastPeriod = normalized.lastIndexOf('.');
|
||||
|
||||
if (lastComma > lastPeriod) {
|
||||
// Comma is the decimal separator (European style)
|
||||
normalized = normalized.replace(/\./g, '').replace(',', '.');
|
||||
} else if (lastPeriod > lastComma) {
|
||||
// Period is the decimal separator (US style)
|
||||
normalized = normalized.replace(/,/g, '');
|
||||
} else {
|
||||
// Only one type or neither
|
||||
normalized = normalized.replace(/,/g, '.');
|
||||
}
|
||||
|
||||
const parsed = parseFloat(normalized);
|
||||
return isNaN(parsed) ? null : Math.round(parsed * 100) / 100;
|
||||
}
|
||||
|
||||
export function normalizeUrl(href: string, baseUrl: string): string {
|
||||
if (!href) return baseUrl;
|
||||
|
||||
try {
|
||||
// Already absolute
|
||||
if (href.startsWith('http://') || href.startsWith('https://')) {
|
||||
return href;
|
||||
}
|
||||
return new URL(href, baseUrl).href;
|
||||
} catch {
|
||||
return baseUrl;
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeResult(raw: ScrapedItem, storeId: number, storeName: string, baseUrl: string, currency: string): Product {
|
||||
return {
|
||||
name: raw.name.trim(),
|
||||
price: parsePrice(raw.priceText),
|
||||
priceText: raw.priceText.trim(),
|
||||
currency,
|
||||
url: normalizeUrl(raw.link, baseUrl),
|
||||
image: raw.image ? normalizeUrl(raw.image, baseUrl) : null,
|
||||
storeName,
|
||||
storeId,
|
||||
};
|
||||
}
|
||||
34
tests/fixtures/sample-store.html
vendored
Normal file
34
tests/fixtures/sample-store.html
vendored
Normal file
@@ -0,0 +1,34 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div class="search-results">
|
||||
<div class="product-card">
|
||||
<a href="/products/sony-wh1000xm5" class="product-link">
|
||||
<img src="/images/xm5.jpg" alt="Sony WH-1000XM5" class="product-image" />
|
||||
<h3 class="product-title">Sony WH-1000XM5 Wireless Headphones</h3>
|
||||
<span class="product-price">€299,99</span>
|
||||
</a>
|
||||
</div>
|
||||
<div class="product-card">
|
||||
<a href="/products/sony-wh1000xm4" class="product-link">
|
||||
<img src="/images/xm4.jpg" alt="Sony WH-1000XM4" class="product-image" />
|
||||
<h3 class="product-title">Sony WH-1000XM4 Wireless Headphones</h3>
|
||||
<span class="product-price">€219,00</span>
|
||||
</a>
|
||||
</div>
|
||||
<div class="product-card">
|
||||
<a href="/products/airpods-max" class="product-link">
|
||||
<img src="/images/airpods.jpg" alt="Apple AirPods Max" class="product-image" />
|
||||
<h3 class="product-title">Apple AirPods Max</h3>
|
||||
<span class="product-price">€579,00</span>
|
||||
</a>
|
||||
</div>
|
||||
<div class="product-card">
|
||||
<a href="/products/bose-qc45" class="product-link">
|
||||
<h3 class="product-title">Bose QuietComfort 45</h3>
|
||||
<span class="product-price">$249.95</span>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
70
tests/server/scraper/http-scraper.test.ts
Normal file
70
tests/server/scraper/http-scraper.test.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { scrapeStore } from '../../../src/server/scraper/http-scraper.js';
|
||||
import type { Store } from '../../../src/server/models/store.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const sampleHtml = fs.readFileSync(path.join(__dirname, '../../fixtures/sample-store.html'), 'utf-8');
|
||||
|
||||
const mockStore: Store = {
|
||||
id: 1,
|
||||
name: 'Test Store',
|
||||
slug: 'test-store',
|
||||
base_url: 'https://teststore.com',
|
||||
search_url: 'https://teststore.com/search?q={query}',
|
||||
enabled: 1,
|
||||
sel_container: '.product-card',
|
||||
sel_name: '.product-title',
|
||||
sel_price: '.product-price',
|
||||
sel_link: 'a',
|
||||
sel_image: '.product-image',
|
||||
rate_limit: 2,
|
||||
rate_window: 1000,
|
||||
proxy_url: null,
|
||||
user_agent: null,
|
||||
headers_json: null,
|
||||
currency: 'EUR',
|
||||
category_id: null,
|
||||
created_at: '',
|
||||
updated_at: '',
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('scrapeStore', () => {
|
||||
it('extracts products from HTML using CSS selectors', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
||||
text: () => Promise.resolve(sampleHtml),
|
||||
status: 200,
|
||||
}));
|
||||
|
||||
const result = await scrapeStore(mockStore, 'https://teststore.com/search?q=headphones');
|
||||
|
||||
expect(result.statusCode).toBe(200);
|
||||
expect(result.items).toHaveLength(4);
|
||||
expect(result.items[0]).toEqual({
|
||||
name: 'Sony WH-1000XM5 Wireless Headphones',
|
||||
priceText: '€299,99',
|
||||
link: '/products/sony-wh1000xm5',
|
||||
image: '/images/xm5.jpg',
|
||||
});
|
||||
|
||||
// Item without image
|
||||
expect(result.items[3].name).toBe('Bose QuietComfort 45');
|
||||
expect(result.items[3].image).toBe(null);
|
||||
});
|
||||
|
||||
it('returns empty items when no containers match', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
||||
text: () => Promise.resolve('<html><body>No products here</body></html>'),
|
||||
status: 200,
|
||||
}));
|
||||
|
||||
const result = await scrapeStore(mockStore, 'https://teststore.com/search?q=nothing');
|
||||
expect(result.items).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
51
tests/server/scraper/result-parser.test.ts
Normal file
51
tests/server/scraper/result-parser.test.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { parsePrice, normalizeUrl } from '../../../src/server/scraper/result-parser.js';
|
||||
|
||||
describe('parsePrice', () => {
|
||||
it('parses US format prices', () => {
|
||||
expect(parsePrice('$12.99')).toBe(12.99);
|
||||
expect(parsePrice('$1,299.00')).toBe(1299);
|
||||
expect(parsePrice('$0.50')).toBe(0.5);
|
||||
});
|
||||
|
||||
it('parses European format prices', () => {
|
||||
expect(parsePrice('€12,99')).toBe(12.99);
|
||||
expect(parsePrice('1.299,00 EUR')).toBe(1299);
|
||||
expect(parsePrice('€299,99')).toBe(299.99);
|
||||
});
|
||||
|
||||
it('handles free', () => {
|
||||
expect(parsePrice('free')).toBe(0);
|
||||
expect(parsePrice('FREE')).toBe(0);
|
||||
expect(parsePrice('gratis')).toBe(0);
|
||||
});
|
||||
|
||||
it('handles price ranges', () => {
|
||||
expect(parsePrice('$12 - $15')).toBe(12);
|
||||
expect(parsePrice('€10,00 – €20,00')).toBe(10);
|
||||
});
|
||||
|
||||
it('handles plain numbers', () => {
|
||||
expect(parsePrice('42')).toBe(42);
|
||||
expect(parsePrice('12.99')).toBe(12.99);
|
||||
});
|
||||
|
||||
it('returns null for invalid input', () => {
|
||||
expect(parsePrice('')).toBe(null);
|
||||
expect(parsePrice('abc')).toBe(null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeUrl', () => {
|
||||
it('returns absolute URLs as-is', () => {
|
||||
expect(normalizeUrl('https://example.com/product', 'https://base.com')).toBe('https://example.com/product');
|
||||
});
|
||||
|
||||
it('resolves relative URLs against base', () => {
|
||||
expect(normalizeUrl('/products/123', 'https://store.com')).toBe('https://store.com/products/123');
|
||||
});
|
||||
|
||||
it('returns base URL for empty href', () => {
|
||||
expect(normalizeUrl('', 'https://store.com')).toBe('https://store.com');
|
||||
});
|
||||
});
|
||||
20
tsconfig.json
Normal file
20
tsconfig.json
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"moduleResolution": "bundler",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true
|
||||
},
|
||||
"include": ["src/server/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist", "src/client"]
|
||||
}
|
||||
7
vitest.config.ts
Normal file
7
vitest.config.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user