msemes/PriceHunter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add YAML-based store configs with bidirectional sync

Browse Source 
Stores can now be defined as YAML files in the stores/ directory.
On startup, YAML files are synced into the database. Changes made
via the admin UI are written back to YAML files automatically.

- Add store-sync service (load from files, export to files, write-back)
- Add /api/stores/sync and /api/stores/export endpoints
- Add Sync/Export buttons to admin UI
- Mount stores/ volume in Docker
- Include example store config template

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
mariosemes
2026-03-26 21:06:29 +01:00
parent 8ce5ba62dc
commit 26467a6368
11 changed files with 374 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.env.example
View File

@@ -1,2 +1,3 @@
PORT=3000
DATABASE_PATH=./data/pricehunter.db
STORES_DIR=./stores

2
docker-compose.yml
View File

@@ -5,8 +5,10 @@ services:
- "${PORT:-3000}:3000"
volumes:
- ./data:/app/data
- ./stores:/app/stores
environment:
- NODE_ENV=production
- DATABASE_PATH=/app/data/pricehunter.db
- STORES_DIR=/app/stores
- PORT=3000
restart: unless-stopped

18
package-lock.json generated
View File

@@ -15,7 +15,8 @@
"dotenv": "^16.4.7",
"fastify": "^5.2.1",
"p-limit": "^6.2.0",
"sql.js": "^1.11.0"
"sql.js": "^1.11.0",
"yaml": "^2.8.3"
},
"devDependencies": {
"@types/node": "^22.10.0",
@@ -3806,6 +3807,21 @@
"node": ">=10"
}
},
"node_modules/yaml": {
"version": "2.8.3",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz",
"integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==",
"license": "ISC",
"bin": {
"yaml": "bin.mjs"
},
"engines": {
"node": ">= 14.6"
},
"funding": {
"url": "https://github.com/sponsors/eemeli"
}
},
"node_modules/yargs": {
"version": "17.7.2",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",

3
package.json
View File

@@ -22,7 +22,8 @@
"dotenv": "^16.4.7",
"fastify": "^5.2.1",
"p-limit": "^6.2.0",
"sql.js": "^1.11.0"
"sql.js": "^1.11.0",
"yaml": "^2.8.3"
},
"devDependencies": {
"@types/node": "^22.10.0",

8
src/client/src/lib/api.ts
View File

@@ -42,6 +42,14 @@ export function deleteStore(id: number) {
return api<void>(`/api/stores/${id}`, { method: 'DELETE' });
}
export function exportStores() {
return api<{ exported: number; directory: string }>('/api/stores/export', { method: 'POST' });
}
export function syncStores() {
return api<{ created: number; updated: number; errors: string[] }>('/api/stores/sync', { method: 'POST' });
}
export function getCategories() {
return api<any[]>('/api/categories');
}

52
src/client/src/routes/admin/+page.svelte
View File

@@ -1,9 +1,10 @@
<script>
import { getStores, toggleStore, deleteStore } from '$lib/api';
import { getStores, toggleStore, deleteStore, exportStores, syncStores } from '$lib/api';
import { onMount } from 'svelte';
let stores = $state([]);
let loading = $state(true);
let syncMessage = $state('');
onMount(async () => {
stores = await getStores();
@@ -20,19 +21,56 @@
await deleteStore(id);
stores = stores.filter((s) => s.id !== id);
}
async function handleSync() {
syncMessage = '';
const result = await syncStores();
stores = await getStores();
const parts = [];
if (result.created) parts.push(`${result.created} created`);
if (result.updated) parts.push(`${result.updated} updated`);
if (result.errors.length) parts.push(`${result.errors.length} errors`);
syncMessage = parts.length ? `Sync: ${parts.join(', ')}` : 'No changes found';
if (result.errors.length) syncMessage += ' — ' + result.errors.join('; ');
setTimeout(() => syncMessage = '', 5000);
}
async function handleExport() {
const result = await exportStores();
syncMessage = `Exported ${result.exported} stores to ${result.directory}/`;
setTimeout(() => syncMessage = '', 5000);
}
</script>
<div class="max-w-7xl mx-auto px-4 py-6">
<div class="flex items-center justify-between mb-6">
<h1 class="text-2xl font-bold text-gray-900 dark:text-white">Store Management</h1>
<a
href="/admin/stores/new"
class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-lg text-sm font-medium transition-colors"
>
Add Store
</a>
<div class="flex gap-2">
<button onclick={handleSync}
class="border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-800 px-4 py-2 rounded-lg text-sm font-medium transition-colors"
title="Reload store configs from YAML files in the stores/ directory">
Sync from Files
</button>
<button onclick={handleExport}
class="border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-800 px-4 py-2 rounded-lg text-sm font-medium transition-colors"
title="Export all store configs to YAML files">
Export to Files
</button>
<a
href="/admin/stores/new"
class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-lg text-sm font-medium transition-colors"
>
Add Store
</a>
</div>
</div>
{#if syncMessage}
<div class="mb-4 px-4 py-2.5 rounded-lg bg-blue-50 dark:bg-blue-900/20 text-blue-700 dark:text-blue-300 text-sm">
{syncMessage}
</div>
{/if}
{#if loading}
<div class="animate-pulse space-y-3">
{#each Array(5) as _}

1
src/server/config.ts
View File

@@ -4,5 +4,6 @@ export const config = {
port: parseInt(process.env.PORT || '3000', 10),
host: process.env.HOST || '0.0.0.0',
databasePath: process.env.DATABASE_PATH || './data/pricehunter.db',
storesDir: process.env.STORES_DIR || './stores',
isProduction: process.env.NODE_ENV === 'production',
};

12
src/server/index.ts
View File

@@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url';
import { config } from './config.js';
import { initDatabase, startAutoSave, saveDatabase } from './db/connection.js';
import { runMigrations } from './db/migrate.js';
import { syncFromFiles } from './services/store-sync.js';
import { storeRoutes } from './routes/stores.js';
import { categoryRoutes } from './routes/categories.js';
import { searchRoutes } from './routes/search.js';
@@ -48,9 +49,18 @@ if (config.isProduction) {
});
}
// Initialize database and run migrations
// Initialize database, run migrations, sync store configs from YAML files
await initDatabase();
runMigrations();
const sync = syncFromFiles(config.storesDir);
if (sync.created > 0 || sync.updated > 0) {
app.log.info(`Store sync: ${sync.created} created, ${sync.updated} updated`);
}
if (sync.errors.length > 0) {
for (const err of sync.errors) app.log.warn(`Store sync error: ${err}`);
}
startAutoSave();
// Save database on shutdown

28
src/server/routes/stores.ts
View File

@@ -1,6 +1,10 @@
import fs from 'node:fs';
import path from 'node:path';
import type { FastifyPluginAsync } from 'fastify';
import { getAllStores, getStoreById, createStore, updateStore, toggleStoreEnabled, deleteStore } from '../models/store.js';
import { getLogsByStore, getStoreHealth } from '../models/scrape-log.js';
import { writeStoreBackToFile, exportAllStoresToFiles, syncFromFiles } from '../services/store-sync.js';
import { config } from '../config.js';
export const storeRoutes: FastifyPluginAsync = async (app) => {
app.get('/stores', async () => {
@@ -44,6 +48,7 @@ export const storeRoutes: FastifyPluginAsync = async (app) => {
}, async (request, reply) => {
try {
const store = createStore(request.body);
writeStoreBackToFile(store.id, config.storesDir);
return reply.code(201).send(store);
} catch (err: any) {
if (err.message?.includes('UNIQUE constraint failed')) {
@@ -56,18 +61,41 @@ export const storeRoutes: FastifyPluginAsync = async (app) => {
app.put<{ Params: { id: string }; Body: any }>('/stores/:id', async (request, reply) => {
const store = updateStore(Number(request.params.id), request.body);
if (!store) return reply.code(404).send({ error: 'Store not found' });
writeStoreBackToFile(store.id, config.storesDir);
return store;
});
app.patch<{ Params: { id: string } }>('/stores/:id/toggle', async (request, reply) => {
const store = toggleStoreEnabled(Number(request.params.id));
if (!store) return reply.code(404).send({ error: 'Store not found' });
writeStoreBackToFile(store.id, config.storesDir);
return store;
});
app.delete<{ Params: { id: string } }>('/stores/:id', async (request, reply) => {
// Get store slug before deleting so we can remove the YAML file
const store = getStoreById(Number(request.params.id));
const deleted = deleteStore(Number(request.params.id));
if (!deleted) return reply.code(404).send({ error: 'Store not found' });
// Remove the YAML file if it exists
if (store) {
const filePath = path.join(config.storesDir, `${store.slug}.yaml`);
if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
}
return reply.code(204).send();
});
// Export all stores to YAML files
app.post('/stores/export', async () => {
const count = exportAllStoresToFiles(config.storesDir);
return { exported: count, directory: config.storesDir };
});
// Re-sync from YAML files (reload configs)
app.post('/stores/sync', async () => {
const result = syncFromFiles(config.storesDir);
return result;
});
};

225
src/server/services/store-sync.ts Normal file
View File

@@ -0,0 +1,225 @@
import fs from 'node:fs';
import path from 'node:path';
import YAML from 'yaml';
import { getDatabase, saveDatabase } from '../db/connection.js';
export interface StoreFileConfig {
name: string;
base_url: string;
search_url: string;
enabled?: boolean;
category?: string;
currency?: string;
selectors: {
container: string;
name: string;
price: string;
link: string;
image?: string;
};
rate_limit?: number;
rate_window?: number;
proxy_url?: string;
user_agent?: string;
headers?: Record<string, string>;
}
function queryAll(sql: string, params: any[] = []): any[] {
const db = getDatabase();
const stmt = db.prepare(sql);
if (params.length) stmt.bind(params);
const rows: any[] = [];
while (stmt.step()) {
rows.push(stmt.getAsObject());
}
stmt.free();
return rows;
}
function queryOne(sql: string, params: any[] = []): any | undefined {
const rows = queryAll(sql, params);
return rows[0];
}
function slugify(text: string): string {
return text
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-|-$/g, '');
}
function ensureCategoryExists(categoryName: string): number {
const db = getDatabase();
const existing = queryOne('SELECT id FROM categories WHERE name = ?', [categoryName]);
if (existing) return existing.id;
db.run('INSERT INTO categories (name) VALUES (?)', [categoryName]);
return queryOne('SELECT last_insert_rowid() as id')?.id;
}
/**
* Load all YAML store configs from a directory into the database.
* Uses the filename (without extension) as the slug for matching.
* If a store with the same slug exists, it gets updated. Otherwise, it's created.
*/
export function syncFromFiles(storesDir: string): { created: number; updated: number; errors: string[] } {
if (!fs.existsSync(storesDir)) {
fs.mkdirSync(storesDir, { recursive: true });
return { created: 0, updated: 0, errors: [] };
}
const files = fs.readdirSync(storesDir).filter((f) => f.endsWith('.yaml') || f.endsWith('.yml'));
const db = getDatabase();
let created = 0;
let updated = 0;
const errors: string[] = [];
for (const file of files) {
try {
const content = fs.readFileSync(path.join(storesDir, file), 'utf-8');
const config = YAML.parse(content) as StoreFileConfig;
const slug = path.basename(file, path.extname(file));
if (!config.name || !config.base_url || !config.search_url || !config.selectors) {
errors.push(`${file}: missing required fields (name, base_url, search_url, selectors)`);
continue;
}
if (!config.selectors.container || !config.selectors.name || !config.selectors.price || !config.selectors.link) {
errors.push(`${file}: missing required selectors (container, name, price, link)`);
continue;
}
const categoryId = config.category ? ensureCategoryExists(config.category) : null;
const headersJson = config.headers ? JSON.stringify(config.headers) : null;
const existing = queryOne('SELECT id FROM stores WHERE slug = ?', [slug]);
if (existing) {
db.run(`
UPDATE stores SET
name = ?, base_url = ?, search_url = ?, enabled = ?,
sel_container = ?, sel_name = ?, sel_price = ?, sel_link = ?, sel_image = ?,
rate_limit = ?, rate_window = ?, proxy_url = ?, user_agent = ?, headers_json = ?,
currency = ?, category_id = ?, updated_at = datetime('now')
WHERE slug = ?
`, [
config.name, config.base_url, config.search_url,
config.enabled === false ? 0 : 1,
config.selectors.container, config.selectors.name,
config.selectors.price, config.selectors.link,
config.selectors.image || null,
config.rate_limit ?? 2, config.rate_window ?? 1000,
config.proxy_url || null, config.user_agent || null, headersJson,
config.currency || 'EUR', categoryId,
slug,
]);
updated++;
} else {
db.run(`
INSERT INTO stores (name, slug, base_url, search_url, enabled,
sel_container, sel_name, sel_price, sel_link, sel_image,
rate_limit, rate_window, proxy_url, user_agent, headers_json,
currency, category_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, [
config.name, slug, config.base_url, config.search_url,
config.enabled === false ? 0 : 1,
config.selectors.container, config.selectors.name,
config.selectors.price, config.selectors.link,
config.selectors.image || null,
config.rate_limit ?? 2, config.rate_window ?? 1000,
config.proxy_url || null, config.user_agent || null, headersJson,
config.currency || 'EUR', categoryId,
]);
created++;
}
} catch (err) {
errors.push(`${file}: ${err instanceof Error ? err.message : String(err)}`);
}
}
if (created > 0 || updated > 0) saveDatabase();
return { created, updated, errors };
}
/**
* Export a single store from the database to a YAML config object.
*/
function storeToConfig(store: any, categoryName?: string): StoreFileConfig {
const config: StoreFileConfig = {
name: store.name,
base_url: store.base_url,
search_url: store.search_url,
selectors: {
container: store.sel_container,
name: store.sel_name,
price: store.sel_price,
link: store.sel_link,
},
};
if (store.sel_image) config.selectors.image = store.sel_image;
if (store.enabled === 0) config.enabled = false;
if (categoryName) config.category = categoryName;
if (store.currency && store.currency !== 'EUR') config.currency = store.currency;
if (store.rate_limit && store.rate_limit !== 2) config.rate_limit = store.rate_limit;
if (store.rate_window && store.rate_window !== 1000) config.rate_window = store.rate_window;
if (store.proxy_url) config.proxy_url = store.proxy_url;
if (store.user_agent) config.user_agent = store.user_agent;
if (store.headers_json) {
try { config.headers = JSON.parse(store.headers_json); } catch { /* ignore */ }
}
return config;
}
/**
* Export a single store to a YAML file.
*/
export function exportStoreToFile(storeId: number, storesDir: string): string | null {
const store = queryOne(`
SELECT s.*, c.name as category_name
FROM stores s LEFT JOIN categories c ON s.category_id = c.id
WHERE s.id = ?
`, [storeId]);
if (!store) return null;
const config = storeToConfig(store, store.category_name);
const yaml = YAML.stringify(config, { lineWidth: 120 });
const filePath = path.join(storesDir, `${store.slug}.yaml`);
if (!fs.existsSync(storesDir)) fs.mkdirSync(storesDir, { recursive: true });
fs.writeFileSync(filePath, yaml);
return filePath;
}
/**
* Export all stores to YAML files.
*/
export function exportAllStoresToFiles(storesDir: string): number {
const stores = queryAll(`
SELECT s.*, c.name as category_name
FROM stores s LEFT JOIN categories c ON s.category_id = c.id
ORDER BY s.name
`);
if (!fs.existsSync(storesDir)) fs.mkdirSync(storesDir, { recursive: true });
for (const store of stores) {
const config = storeToConfig(store, store.category_name);
const yaml = YAML.stringify(config, { lineWidth: 120 });
fs.writeFileSync(path.join(storesDir, `${store.slug}.yaml`), yaml);
}
return stores.length;
}
/**
* Write a single store back to its YAML file after a DB update.
*/
export function writeStoreBackToFile(storeId: number, storesDir: string): void {
exportStoreToFile(storeId, storesDir);
}

34
stores/_example-store.yaml Normal file
View File

@@ -0,0 +1,34 @@
# Example store configuration for Price Hunter
# Copy this file, rename it (the filename becomes the store slug), and fill in your values.
#
# Required fields: name, base_url, search_url, selectors (container, name, price, link)
# The search_url must contain {query} where the search term will be inserted.
#
# Tips for finding CSS selectors:
# 1. Open the store's search results page in your browser
# 2. Right-click a product and choose "Inspect"
# 3. Identify the repeating container element for each product card
# 4. Find child elements for name, price, link, and image within that container
# 5. Use the Test page in the admin UI to verify your selectors work
name: Example Store
base_url: https://www.example-store.com
search_url: https://www.example-store.com/search?q={query}
category: Electronics
currency: EUR
enabled: false
selectors:
container: ".product-card"
name: ".product-title"
price: ".product-price"
link: "a.product-link"
image: "img.product-image"
# Optional settings:
# rate_limit: 2 # Max requests per second (default: 2)
# rate_window: 1000 # Rate limit window in ms (default: 1000)
# user_agent: "Custom UA" # Override the default browser user agent
# proxy_url: "http://user:pass@proxy:8080" # Route requests through a proxy
# headers: # Extra HTTP headers
# X-Custom-Header: "value"