Add YAML-based store configs with bidirectional sync

Stores can now be defined as YAML files in the stores/ directory.
On startup, YAML files are synced into the database. Changes made
via the admin UI are written back to YAML files automatically.

- Add store-sync service (load from files, export to files, write-back)
- Add /api/stores/sync and /api/stores/export endpoints
- Add Sync/Export buttons to admin UI
- Mount stores/ volume in Docker
- Include example store config template

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.env.example

@@ -1,2 +1,3 @@
 PORT=3000
 DATABASE_PATH=./data/pricehunter.db
+STORES_DIR=./stores

docker-compose.yml

@@ -5,8 +5,10 @@ services:
       - "${PORT:-3000}:3000"
     volumes:
       - ./data:/app/data
+      - ./stores:/app/stores
     environment:
       - NODE_ENV=production
       - DATABASE_PATH=/app/data/pricehunter.db
+      - STORES_DIR=/app/stores
       - PORT=3000
     restart: unless-stopped

package-lock.json

@@ -15,7 +15,8 @@
         "dotenv": "^16.4.7",
         "fastify": "^5.2.1",
         "p-limit": "^6.2.0",
-        "sql.js": "^1.11.0"
+        "sql.js": "^1.11.0",
+        "yaml": "^2.8.3"
       },
       "devDependencies": {
         "@types/node": "^22.10.0",
@@ -3806,6 +3807,21 @@
         "node": ">=10"
       }
     },
+    "node_modules/yaml": {
+      "version": "2.8.3",
+      "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz",
+      "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==",
+      "license": "ISC",
+      "bin": {
+        "yaml": "bin.mjs"
+      },
+      "engines": {
+        "node": ">= 14.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/eemeli"
+      }
+    },
     "node_modules/yargs": {
       "version": "17.7.2",
       "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",

package.json

@@ -22,7 +22,8 @@
     "dotenv": "^16.4.7",
     "fastify": "^5.2.1",
     "p-limit": "^6.2.0",
-    "sql.js": "^1.11.0"
+    "sql.js": "^1.11.0",
+    "yaml": "^2.8.3"
   },
   "devDependencies": {
     "@types/node": "^22.10.0",

src/client/src/lib/api.ts

@@ -42,6 +42,14 @@ export function deleteStore(id: number) {
   return api<void>(`/api/stores/${id}`, { method: 'DELETE' });
 }
 
+export function exportStores() {
+  return api<{ exported: number; directory: string }>('/api/stores/export', { method: 'POST' });
+}
+
+export function syncStores() {
+  return api<{ created: number; updated: number; errors: string[] }>('/api/stores/sync', { method: 'POST' });
+}
+
 export function getCategories() {
   return api<any[]>('/api/categories');
 }

src/client/src/routes/admin/+page.svelte

@@ -1,9 +1,10 @@
 <script>
-  import { getStores, toggleStore, deleteStore } from '$lib/api';
+  import { getStores, toggleStore, deleteStore, exportStores, syncStores } from '$lib/api';
   import { onMount } from 'svelte';
 
   let stores = $state([]);
   let loading = $state(true);
+  let syncMessage = $state('');
 
   onMount(async () => {
     stores = await getStores();
@@ -20,19 +21,56 @@
     await deleteStore(id);
     stores = stores.filter((s) => s.id !== id);
   }
+
+  async function handleSync() {
+    syncMessage = '';
+    const result = await syncStores();
+    stores = await getStores();
+    const parts = [];
+    if (result.created) parts.push(`${result.created} created`);
+    if (result.updated) parts.push(`${result.updated} updated`);
+    if (result.errors.length) parts.push(`${result.errors.length} errors`);
+    syncMessage = parts.length ? `Sync: ${parts.join(', ')}` : 'No changes found';
+    if (result.errors.length) syncMessage += ' — ' + result.errors.join('; ');
+    setTimeout(() => syncMessage = '', 5000);
+  }
+
+  async function handleExport() {
+    const result = await exportStores();
+    syncMessage = `Exported ${result.exported} stores to ${result.directory}/`;
+    setTimeout(() => syncMessage = '', 5000);
+  }
 </script>
 
 <div class="max-w-7xl mx-auto px-4 py-6">
   <div class="flex items-center justify-between mb-6">
     <h1 class="text-2xl font-bold text-gray-900 dark:text-white">Store Management</h1>
-    <a
+    <div class="flex gap-2">
-      href="/admin/stores/new"
+      <button onclick={handleSync}
-      class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-lg text-sm font-medium transition-colors"
+        class="border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-800 px-4 py-2 rounded-lg text-sm font-medium transition-colors"
-    >
+        title="Reload store configs from YAML files in the stores/ directory">
-      Add Store
+        Sync from Files
-    </a>
+      </button>
+      <button onclick={handleExport}
+        class="border border-gray-300 dark:border-gray-600 text-gray-700 dark:text-gray-300 hover:bg-gray-50 dark:hover:bg-gray-800 px-4 py-2 rounded-lg text-sm font-medium transition-colors"
+        title="Export all store configs to YAML files">
+        Export to Files
+      </button>
+      <a
+        href="/admin/stores/new"
+        class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-lg text-sm font-medium transition-colors"
+      >
+        Add Store
+      </a>
+    </div>
   </div>
 
+  {#if syncMessage}
+    <div class="mb-4 px-4 py-2.5 rounded-lg bg-blue-50 dark:bg-blue-900/20 text-blue-700 dark:text-blue-300 text-sm">
+      {syncMessage}
+    </div>
+  {/if}
+
   {#if loading}
     <div class="animate-pulse space-y-3">
       {#each Array(5) as _}

src/server/config.ts

@@ -4,5 +4,6 @@ export const config = {
   port: parseInt(process.env.PORT || '3000', 10),
   host: process.env.HOST || '0.0.0.0',
   databasePath: process.env.DATABASE_PATH || './data/pricehunter.db',
+  storesDir: process.env.STORES_DIR || './stores',
   isProduction: process.env.NODE_ENV === 'production',
 };

src/server/index.ts

@@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url';
 import { config } from './config.js';
 import { initDatabase, startAutoSave, saveDatabase } from './db/connection.js';
 import { runMigrations } from './db/migrate.js';
+import { syncFromFiles } from './services/store-sync.js';
 import { storeRoutes } from './routes/stores.js';
 import { categoryRoutes } from './routes/categories.js';
 import { searchRoutes } from './routes/search.js';
@@ -48,9 +49,18 @@ if (config.isProduction) {
   });
 }
 
-// Initialize database and run migrations
+// Initialize database, run migrations, sync store configs from YAML files
 await initDatabase();
 runMigrations();
+
+const sync = syncFromFiles(config.storesDir);
+if (sync.created > 0 || sync.updated > 0) {
+  app.log.info(`Store sync: ${sync.created} created, ${sync.updated} updated`);
+}
+if (sync.errors.length > 0) {
+  for (const err of sync.errors) app.log.warn(`Store sync error: ${err}`);
+}
+
 startAutoSave();
 
 // Save database on shutdown

src/server/routes/stores.ts

@@ -1,6 +1,10 @@
+import fs from 'node:fs';
+import path from 'node:path';
 import type { FastifyPluginAsync } from 'fastify';
 import { getAllStores, getStoreById, createStore, updateStore, toggleStoreEnabled, deleteStore } from '../models/store.js';
 import { getLogsByStore, getStoreHealth } from '../models/scrape-log.js';
+import { writeStoreBackToFile, exportAllStoresToFiles, syncFromFiles } from '../services/store-sync.js';
+import { config } from '../config.js';
 
 export const storeRoutes: FastifyPluginAsync = async (app) => {
   app.get('/stores', async () => {
@@ -44,6 +48,7 @@ export const storeRoutes: FastifyPluginAsync = async (app) => {
   }, async (request, reply) => {
     try {
       const store = createStore(request.body);
+      writeStoreBackToFile(store.id, config.storesDir);
       return reply.code(201).send(store);
     } catch (err: any) {
       if (err.message?.includes('UNIQUE constraint failed')) {
@@ -56,18 +61,41 @@ export const storeRoutes: FastifyPluginAsync = async (app) => {
   app.put<{ Params: { id: string }; Body: any }>('/stores/:id', async (request, reply) => {
     const store = updateStore(Number(request.params.id), request.body);
     if (!store) return reply.code(404).send({ error: 'Store not found' });
+    writeStoreBackToFile(store.id, config.storesDir);
     return store;
   });
 
   app.patch<{ Params: { id: string } }>('/stores/:id/toggle', async (request, reply) => {
     const store = toggleStoreEnabled(Number(request.params.id));
     if (!store) return reply.code(404).send({ error: 'Store not found' });
+    writeStoreBackToFile(store.id, config.storesDir);
     return store;
   });
 
   app.delete<{ Params: { id: string } }>('/stores/:id', async (request, reply) => {
+    // Get store slug before deleting so we can remove the YAML file
+    const store = getStoreById(Number(request.params.id));
     const deleted = deleteStore(Number(request.params.id));
     if (!deleted) return reply.code(404).send({ error: 'Store not found' });
+
+    // Remove the YAML file if it exists
+    if (store) {
+      const filePath = path.join(config.storesDir, `${store.slug}.yaml`);
+      if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
+    }
+
     return reply.code(204).send();
   });
+
+  // Export all stores to YAML files
+  app.post('/stores/export', async () => {
+    const count = exportAllStoresToFiles(config.storesDir);
+    return { exported: count, directory: config.storesDir };
+  });
+
+  // Re-sync from YAML files (reload configs)
+  app.post('/stores/sync', async () => {
+    const result = syncFromFiles(config.storesDir);
+    return result;
+  });
 };

src/server/services/store-sync.ts

@@ -0,0 +1,225 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import YAML from 'yaml';
+import { getDatabase, saveDatabase } from '../db/connection.js';
+
+export interface StoreFileConfig {
+  name: string;
+  base_url: string;
+  search_url: string;
+  enabled?: boolean;
+  category?: string;
+  currency?: string;
+  selectors: {
+    container: string;
+    name: string;
+    price: string;
+    link: string;
+    image?: string;
+  };
+  rate_limit?: number;
+  rate_window?: number;
+  proxy_url?: string;
+  user_agent?: string;
+  headers?: Record<string, string>;
+}
+
+function queryAll(sql: string, params: any[] = []): any[] {
+  const db = getDatabase();
+  const stmt = db.prepare(sql);
+  if (params.length) stmt.bind(params);
+  const rows: any[] = [];
+  while (stmt.step()) {
+    rows.push(stmt.getAsObject());
+  }
+  stmt.free();
+  return rows;
+}
+
+function queryOne(sql: string, params: any[] = []): any | undefined {
+  const rows = queryAll(sql, params);
+  return rows[0];
+}
+
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '');
+}
+
+function ensureCategoryExists(categoryName: string): number {
+  const db = getDatabase();
+  const existing = queryOne('SELECT id FROM categories WHERE name = ?', [categoryName]);
+  if (existing) return existing.id;
+
+  db.run('INSERT INTO categories (name) VALUES (?)', [categoryName]);
+  return queryOne('SELECT last_insert_rowid() as id')?.id;
+}
+
+/**
+ * Load all YAML store configs from a directory into the database.
+ * Uses the filename (without extension) as the slug for matching.
+ * If a store with the same slug exists, it gets updated. Otherwise, it's created.
+ */
+export function syncFromFiles(storesDir: string): { created: number; updated: number; errors: string[] } {
+  if (!fs.existsSync(storesDir)) {
+    fs.mkdirSync(storesDir, { recursive: true });
+    return { created: 0, updated: 0, errors: [] };
+  }
+
+  const files = fs.readdirSync(storesDir).filter((f) => f.endsWith('.yaml') || f.endsWith('.yml'));
+  const db = getDatabase();
+  let created = 0;
+  let updated = 0;
+  const errors: string[] = [];
+
+  for (const file of files) {
+    try {
+      const content = fs.readFileSync(path.join(storesDir, file), 'utf-8');
+      const config = YAML.parse(content) as StoreFileConfig;
+      const slug = path.basename(file, path.extname(file));
+
+      if (!config.name || !config.base_url || !config.search_url || !config.selectors) {
+        errors.push(`${file}: missing required fields (name, base_url, search_url, selectors)`);
+        continue;
+      }
+
+      if (!config.selectors.container || !config.selectors.name || !config.selectors.price || !config.selectors.link) {
+        errors.push(`${file}: missing required selectors (container, name, price, link)`);
+        continue;
+      }
+
+      const categoryId = config.category ? ensureCategoryExists(config.category) : null;
+      const headersJson = config.headers ? JSON.stringify(config.headers) : null;
+
+      const existing = queryOne('SELECT id FROM stores WHERE slug = ?', [slug]);
+
+      if (existing) {
+        db.run(`
+          UPDATE stores SET
+            name = ?, base_url = ?, search_url = ?, enabled = ?,
+            sel_container = ?, sel_name = ?, sel_price = ?, sel_link = ?, sel_image = ?,
+            rate_limit = ?, rate_window = ?, proxy_url = ?, user_agent = ?, headers_json = ?,
+            currency = ?, category_id = ?, updated_at = datetime('now')
+          WHERE slug = ?
+        `, [
+          config.name, config.base_url, config.search_url,
+          config.enabled === false ? 0 : 1,
+          config.selectors.container, config.selectors.name,
+          config.selectors.price, config.selectors.link,
+          config.selectors.image || null,
+          config.rate_limit ?? 2, config.rate_window ?? 1000,
+          config.proxy_url || null, config.user_agent || null, headersJson,
+          config.currency || 'EUR', categoryId,
+          slug,
+        ]);
+        updated++;
+      } else {
+        db.run(`
+          INSERT INTO stores (name, slug, base_url, search_url, enabled,
+            sel_container, sel_name, sel_price, sel_link, sel_image,
+            rate_limit, rate_window, proxy_url, user_agent, headers_json,
+            currency, category_id)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `, [
+          config.name, slug, config.base_url, config.search_url,
+          config.enabled === false ? 0 : 1,
+          config.selectors.container, config.selectors.name,
+          config.selectors.price, config.selectors.link,
+          config.selectors.image || null,
+          config.rate_limit ?? 2, config.rate_window ?? 1000,
+          config.proxy_url || null, config.user_agent || null, headersJson,
+          config.currency || 'EUR', categoryId,
+        ]);
+        created++;
+      }
+    } catch (err) {
+      errors.push(`${file}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  if (created > 0 || updated > 0) saveDatabase();
+  return { created, updated, errors };
+}
+
+/**
+ * Export a single store from the database to a YAML config object.
+ */
+function storeToConfig(store: any, categoryName?: string): StoreFileConfig {
+  const config: StoreFileConfig = {
+    name: store.name,
+    base_url: store.base_url,
+    search_url: store.search_url,
+    selectors: {
+      container: store.sel_container,
+      name: store.sel_name,
+      price: store.sel_price,
+      link: store.sel_link,
+    },
+  };
+
+  if (store.sel_image) config.selectors.image = store.sel_image;
+  if (store.enabled === 0) config.enabled = false;
+  if (categoryName) config.category = categoryName;
+  if (store.currency && store.currency !== 'EUR') config.currency = store.currency;
+  if (store.rate_limit && store.rate_limit !== 2) config.rate_limit = store.rate_limit;
+  if (store.rate_window && store.rate_window !== 1000) config.rate_window = store.rate_window;
+  if (store.proxy_url) config.proxy_url = store.proxy_url;
+  if (store.user_agent) config.user_agent = store.user_agent;
+  if (store.headers_json) {
+    try { config.headers = JSON.parse(store.headers_json); } catch { /* ignore */ }
+  }
+
+  return config;
+}
+
+/**
+ * Export a single store to a YAML file.
+ */
+export function exportStoreToFile(storeId: number, storesDir: string): string | null {
+  const store = queryOne(`
+    SELECT s.*, c.name as category_name
+    FROM stores s LEFT JOIN categories c ON s.category_id = c.id
+    WHERE s.id = ?
+  `, [storeId]);
+
+  if (!store) return null;
+
+  const config = storeToConfig(store, store.category_name);
+  const yaml = YAML.stringify(config, { lineWidth: 120 });
+  const filePath = path.join(storesDir, `${store.slug}.yaml`);
+
+  if (!fs.existsSync(storesDir)) fs.mkdirSync(storesDir, { recursive: true });
+  fs.writeFileSync(filePath, yaml);
+
+  return filePath;
+}
+
+/**
+ * Export all stores to YAML files.
+ */
+export function exportAllStoresToFiles(storesDir: string): number {
+  const stores = queryAll(`
+    SELECT s.*, c.name as category_name
+    FROM stores s LEFT JOIN categories c ON s.category_id = c.id
+    ORDER BY s.name
+  `);
+
+  if (!fs.existsSync(storesDir)) fs.mkdirSync(storesDir, { recursive: true });
+
+  for (const store of stores) {
+    const config = storeToConfig(store, store.category_name);
+    const yaml = YAML.stringify(config, { lineWidth: 120 });
+    fs.writeFileSync(path.join(storesDir, `${store.slug}.yaml`), yaml);
+  }
+
+  return stores.length;
+}
+
+/**
+ * Write a single store back to its YAML file after a DB update.
+ */
+export function writeStoreBackToFile(storeId: number, storesDir: string): void {
+  exportStoreToFile(storeId, storesDir);
+}

stores/_example-store.yaml

@@ -0,0 +1,34 @@
+# Example store configuration for Price Hunter
+# Copy this file, rename it (the filename becomes the store slug), and fill in your values.
+#
+# Required fields: name, base_url, search_url, selectors (container, name, price, link)
+# The search_url must contain {query} where the search term will be inserted.
+#
+# Tips for finding CSS selectors:
+#   1. Open the store's search results page in your browser
+#   2. Right-click a product and choose "Inspect"
+#   3. Identify the repeating container element for each product card
+#   4. Find child elements for name, price, link, and image within that container
+#   5. Use the Test page in the admin UI to verify your selectors work
+
+name: Example Store
+base_url: https://www.example-store.com
+search_url: https://www.example-store.com/search?q={query}
+category: Electronics
+currency: EUR
+enabled: false
+
+selectors:
+  container: ".product-card"
+  name: ".product-title"
+  price: ".product-price"
+  link: "a.product-link"
+  image: "img.product-image"
+
+# Optional settings:
+# rate_limit: 2          # Max requests per second (default: 2)
+# rate_window: 1000      # Rate limit window in ms (default: 1000)
+# user_agent: "Custom UA" # Override the default browser user agent
+# proxy_url: "http://user:pass@proxy:8080"  # Route requests through a proxy
+# headers:               # Extra HTTP headers
+#   X-Custom-Header: "value"