diff --git a/.env.example b/.env.example index f5e93c7..3cadc53 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,3 @@ PORT=3000 DATABASE_PATH=./data/pricehunter.db +STORES_DIR=./stores diff --git a/docker-compose.yml b/docker-compose.yml index 9cc00f0..c4f2a84 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,10 @@ services: - "${PORT:-3000}:3000" volumes: - ./data:/app/data + - ./stores:/app/stores environment: - NODE_ENV=production - DATABASE_PATH=/app/data/pricehunter.db + - STORES_DIR=/app/stores - PORT=3000 restart: unless-stopped diff --git a/package-lock.json b/package-lock.json index 0deef33..173b651 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,8 @@ "dotenv": "^16.4.7", "fastify": "^5.2.1", "p-limit": "^6.2.0", - "sql.js": "^1.11.0" + "sql.js": "^1.11.0", + "yaml": "^2.8.3" }, "devDependencies": { "@types/node": "^22.10.0", @@ -3806,6 +3807,21 @@ "node": ">=10" } }, + "node_modules/yaml": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", diff --git a/package.json b/package.json index 3b35cfd..8c726c4 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "dotenv": "^16.4.7", "fastify": "^5.2.1", "p-limit": "^6.2.0", - "sql.js": "^1.11.0" + "sql.js": "^1.11.0", + "yaml": "^2.8.3" }, "devDependencies": { "@types/node": "^22.10.0", diff --git a/src/client/src/lib/api.ts b/src/client/src/lib/api.ts index f549212..9be5a14 100644 --- a/src/client/src/lib/api.ts +++ b/src/client/src/lib/api.ts @@ -42,6 +42,14 @@ export function deleteStore(id: number) { return api(`/api/stores/${id}`, { method: 'DELETE' }); } +export function exportStores() { + return api<{ exported: number; directory: string }>('/api/stores/export', { method: 'POST' }); +} + +export function syncStores() { + return api<{ created: number; updated: number; errors: string[] }>('/api/stores/sync', { method: 'POST' }); +} + export function getCategories() { return api('/api/categories'); } diff --git a/src/client/src/routes/admin/+page.svelte b/src/client/src/routes/admin/+page.svelte index 12d5274..5debb78 100644 --- a/src/client/src/routes/admin/+page.svelte +++ b/src/client/src/routes/admin/+page.svelte @@ -1,9 +1,10 @@

Store Management

- - Add Store - +
+ + + + Add Store + +
+ {#if syncMessage} +
+ {syncMessage} +
+ {/if} + {#if loading}
{#each Array(5) as _} diff --git a/src/server/config.ts b/src/server/config.ts index cbbfbec..459cddf 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -4,5 +4,6 @@ export const config = { port: parseInt(process.env.PORT || '3000', 10), host: process.env.HOST || '0.0.0.0', databasePath: process.env.DATABASE_PATH || './data/pricehunter.db', + storesDir: process.env.STORES_DIR || './stores', isProduction: process.env.NODE_ENV === 'production', }; diff --git a/src/server/index.ts b/src/server/index.ts index eff93f7..0bebeb2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url'; import { config } from './config.js'; import { initDatabase, startAutoSave, saveDatabase } from './db/connection.js'; import { runMigrations } from './db/migrate.js'; +import { syncFromFiles } from './services/store-sync.js'; import { storeRoutes } from './routes/stores.js'; import { categoryRoutes } from './routes/categories.js'; import { searchRoutes } from './routes/search.js'; @@ -48,9 +49,18 @@ if (config.isProduction) { }); } -// Initialize database and run migrations +// Initialize database, run migrations, sync store configs from YAML files await initDatabase(); runMigrations(); + +const sync = syncFromFiles(config.storesDir); +if (sync.created > 0 || sync.updated > 0) { + app.log.info(`Store sync: ${sync.created} created, ${sync.updated} updated`); +} +if (sync.errors.length > 0) { + for (const err of sync.errors) app.log.warn(`Store sync error: ${err}`); +} + startAutoSave(); // Save database on shutdown diff --git a/src/server/routes/stores.ts b/src/server/routes/stores.ts index 6144f5b..1979f8f 100644 --- a/src/server/routes/stores.ts +++ b/src/server/routes/stores.ts @@ -1,6 +1,10 @@ +import fs from 'node:fs'; +import path from 'node:path'; import type { FastifyPluginAsync } from 'fastify'; import { getAllStores, getStoreById, createStore, updateStore, toggleStoreEnabled, deleteStore } from '../models/store.js'; import { getLogsByStore, getStoreHealth } from '../models/scrape-log.js'; +import { writeStoreBackToFile, exportAllStoresToFiles, syncFromFiles } from '../services/store-sync.js'; +import { config } from '../config.js'; export const storeRoutes: FastifyPluginAsync = async (app) => { app.get('/stores', async () => { @@ -44,6 +48,7 @@ export const storeRoutes: FastifyPluginAsync = async (app) => { }, async (request, reply) => { try { const store = createStore(request.body); + writeStoreBackToFile(store.id, config.storesDir); return reply.code(201).send(store); } catch (err: any) { if (err.message?.includes('UNIQUE constraint failed')) { @@ -56,18 +61,41 @@ export const storeRoutes: FastifyPluginAsync = async (app) => { app.put<{ Params: { id: string }; Body: any }>('/stores/:id', async (request, reply) => { const store = updateStore(Number(request.params.id), request.body); if (!store) return reply.code(404).send({ error: 'Store not found' }); + writeStoreBackToFile(store.id, config.storesDir); return store; }); app.patch<{ Params: { id: string } }>('/stores/:id/toggle', async (request, reply) => { const store = toggleStoreEnabled(Number(request.params.id)); if (!store) return reply.code(404).send({ error: 'Store not found' }); + writeStoreBackToFile(store.id, config.storesDir); return store; }); app.delete<{ Params: { id: string } }>('/stores/:id', async (request, reply) => { + // Get store slug before deleting so we can remove the YAML file + const store = getStoreById(Number(request.params.id)); const deleted = deleteStore(Number(request.params.id)); if (!deleted) return reply.code(404).send({ error: 'Store not found' }); + + // Remove the YAML file if it exists + if (store) { + const filePath = path.join(config.storesDir, `${store.slug}.yaml`); + if (fs.existsSync(filePath)) fs.unlinkSync(filePath); + } + return reply.code(204).send(); }); + + // Export all stores to YAML files + app.post('/stores/export', async () => { + const count = exportAllStoresToFiles(config.storesDir); + return { exported: count, directory: config.storesDir }; + }); + + // Re-sync from YAML files (reload configs) + app.post('/stores/sync', async () => { + const result = syncFromFiles(config.storesDir); + return result; + }); }; diff --git a/src/server/services/store-sync.ts b/src/server/services/store-sync.ts new file mode 100644 index 0000000..16103d3 --- /dev/null +++ b/src/server/services/store-sync.ts @@ -0,0 +1,225 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import YAML from 'yaml'; +import { getDatabase, saveDatabase } from '../db/connection.js'; + +export interface StoreFileConfig { + name: string; + base_url: string; + search_url: string; + enabled?: boolean; + category?: string; + currency?: string; + selectors: { + container: string; + name: string; + price: string; + link: string; + image?: string; + }; + rate_limit?: number; + rate_window?: number; + proxy_url?: string; + user_agent?: string; + headers?: Record; +} + +function queryAll(sql: string, params: any[] = []): any[] { + const db = getDatabase(); + const stmt = db.prepare(sql); + if (params.length) stmt.bind(params); + const rows: any[] = []; + while (stmt.step()) { + rows.push(stmt.getAsObject()); + } + stmt.free(); + return rows; +} + +function queryOne(sql: string, params: any[] = []): any | undefined { + const rows = queryAll(sql, params); + return rows[0]; +} + +function slugify(text: string): string { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); +} + +function ensureCategoryExists(categoryName: string): number { + const db = getDatabase(); + const existing = queryOne('SELECT id FROM categories WHERE name = ?', [categoryName]); + if (existing) return existing.id; + + db.run('INSERT INTO categories (name) VALUES (?)', [categoryName]); + return queryOne('SELECT last_insert_rowid() as id')?.id; +} + +/** + * Load all YAML store configs from a directory into the database. + * Uses the filename (without extension) as the slug for matching. + * If a store with the same slug exists, it gets updated. Otherwise, it's created. + */ +export function syncFromFiles(storesDir: string): { created: number; updated: number; errors: string[] } { + if (!fs.existsSync(storesDir)) { + fs.mkdirSync(storesDir, { recursive: true }); + return { created: 0, updated: 0, errors: [] }; + } + + const files = fs.readdirSync(storesDir).filter((f) => f.endsWith('.yaml') || f.endsWith('.yml')); + const db = getDatabase(); + let created = 0; + let updated = 0; + const errors: string[] = []; + + for (const file of files) { + try { + const content = fs.readFileSync(path.join(storesDir, file), 'utf-8'); + const config = YAML.parse(content) as StoreFileConfig; + const slug = path.basename(file, path.extname(file)); + + if (!config.name || !config.base_url || !config.search_url || !config.selectors) { + errors.push(`${file}: missing required fields (name, base_url, search_url, selectors)`); + continue; + } + + if (!config.selectors.container || !config.selectors.name || !config.selectors.price || !config.selectors.link) { + errors.push(`${file}: missing required selectors (container, name, price, link)`); + continue; + } + + const categoryId = config.category ? ensureCategoryExists(config.category) : null; + const headersJson = config.headers ? JSON.stringify(config.headers) : null; + + const existing = queryOne('SELECT id FROM stores WHERE slug = ?', [slug]); + + if (existing) { + db.run(` + UPDATE stores SET + name = ?, base_url = ?, search_url = ?, enabled = ?, + sel_container = ?, sel_name = ?, sel_price = ?, sel_link = ?, sel_image = ?, + rate_limit = ?, rate_window = ?, proxy_url = ?, user_agent = ?, headers_json = ?, + currency = ?, category_id = ?, updated_at = datetime('now') + WHERE slug = ? + `, [ + config.name, config.base_url, config.search_url, + config.enabled === false ? 0 : 1, + config.selectors.container, config.selectors.name, + config.selectors.price, config.selectors.link, + config.selectors.image || null, + config.rate_limit ?? 2, config.rate_window ?? 1000, + config.proxy_url || null, config.user_agent || null, headersJson, + config.currency || 'EUR', categoryId, + slug, + ]); + updated++; + } else { + db.run(` + INSERT INTO stores (name, slug, base_url, search_url, enabled, + sel_container, sel_name, sel_price, sel_link, sel_image, + rate_limit, rate_window, proxy_url, user_agent, headers_json, + currency, category_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, [ + config.name, slug, config.base_url, config.search_url, + config.enabled === false ? 0 : 1, + config.selectors.container, config.selectors.name, + config.selectors.price, config.selectors.link, + config.selectors.image || null, + config.rate_limit ?? 2, config.rate_window ?? 1000, + config.proxy_url || null, config.user_agent || null, headersJson, + config.currency || 'EUR', categoryId, + ]); + created++; + } + } catch (err) { + errors.push(`${file}: ${err instanceof Error ? err.message : String(err)}`); + } + } + + if (created > 0 || updated > 0) saveDatabase(); + return { created, updated, errors }; +} + +/** + * Export a single store from the database to a YAML config object. + */ +function storeToConfig(store: any, categoryName?: string): StoreFileConfig { + const config: StoreFileConfig = { + name: store.name, + base_url: store.base_url, + search_url: store.search_url, + selectors: { + container: store.sel_container, + name: store.sel_name, + price: store.sel_price, + link: store.sel_link, + }, + }; + + if (store.sel_image) config.selectors.image = store.sel_image; + if (store.enabled === 0) config.enabled = false; + if (categoryName) config.category = categoryName; + if (store.currency && store.currency !== 'EUR') config.currency = store.currency; + if (store.rate_limit && store.rate_limit !== 2) config.rate_limit = store.rate_limit; + if (store.rate_window && store.rate_window !== 1000) config.rate_window = store.rate_window; + if (store.proxy_url) config.proxy_url = store.proxy_url; + if (store.user_agent) config.user_agent = store.user_agent; + if (store.headers_json) { + try { config.headers = JSON.parse(store.headers_json); } catch { /* ignore */ } + } + + return config; +} + +/** + * Export a single store to a YAML file. + */ +export function exportStoreToFile(storeId: number, storesDir: string): string | null { + const store = queryOne(` + SELECT s.*, c.name as category_name + FROM stores s LEFT JOIN categories c ON s.category_id = c.id + WHERE s.id = ? + `, [storeId]); + + if (!store) return null; + + const config = storeToConfig(store, store.category_name); + const yaml = YAML.stringify(config, { lineWidth: 120 }); + const filePath = path.join(storesDir, `${store.slug}.yaml`); + + if (!fs.existsSync(storesDir)) fs.mkdirSync(storesDir, { recursive: true }); + fs.writeFileSync(filePath, yaml); + + return filePath; +} + +/** + * Export all stores to YAML files. + */ +export function exportAllStoresToFiles(storesDir: string): number { + const stores = queryAll(` + SELECT s.*, c.name as category_name + FROM stores s LEFT JOIN categories c ON s.category_id = c.id + ORDER BY s.name + `); + + if (!fs.existsSync(storesDir)) fs.mkdirSync(storesDir, { recursive: true }); + + for (const store of stores) { + const config = storeToConfig(store, store.category_name); + const yaml = YAML.stringify(config, { lineWidth: 120 }); + fs.writeFileSync(path.join(storesDir, `${store.slug}.yaml`), yaml); + } + + return stores.length; +} + +/** + * Write a single store back to its YAML file after a DB update. + */ +export function writeStoreBackToFile(storeId: number, storesDir: string): void { + exportStoreToFile(storeId, storesDir); +} diff --git a/stores/_example-store.yaml b/stores/_example-store.yaml new file mode 100644 index 0000000..a03249e --- /dev/null +++ b/stores/_example-store.yaml @@ -0,0 +1,34 @@ +# Example store configuration for Price Hunter +# Copy this file, rename it (the filename becomes the store slug), and fill in your values. +# +# Required fields: name, base_url, search_url, selectors (container, name, price, link) +# The search_url must contain {query} where the search term will be inserted. +# +# Tips for finding CSS selectors: +# 1. Open the store's search results page in your browser +# 2. Right-click a product and choose "Inspect" +# 3. Identify the repeating container element for each product card +# 4. Find child elements for name, price, link, and image within that container +# 5. Use the Test page in the admin UI to verify your selectors work + +name: Example Store +base_url: https://www.example-store.com +search_url: https://www.example-store.com/search?q={query} +category: Electronics +currency: EUR +enabled: false + +selectors: + container: ".product-card" + name: ".product-title" + price: ".product-price" + link: "a.product-link" + image: "img.product-image" + +# Optional settings: +# rate_limit: 2 # Max requests per second (default: 2) +# rate_window: 1000 # Rate limit window in ms (default: 1000) +# user_agent: "Custom UA" # Override the default browser user agent +# proxy_url: "http://user:pass@proxy:8080" # Route requests through a proxy +# headers: # Extra HTTP headers +# X-Custom-Header: "value"