Add YAML-based store configs with bidirectional sync
Stores can now be defined as YAML files in the stores/ directory. On startup, YAML files are synced into the database. Changes made via the admin UI are written back to YAML files automatically. - Add store-sync service (load from files, export to files, write-back) - Add /api/stores/sync and /api/stores/export endpoints - Add Sync/Export buttons to admin UI - Mount stores/ volume in Docker - Include example store config template Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
mariosemes
225
src/server/services/store-sync.ts
Normal file
225
src/server/services/store-sync.ts
Normal file
@@ -0,0 +1,225 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import YAML from 'yaml';
|
||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||
|
||||
export interface StoreFileConfig {
|
||||
name: string;
|
||||
base_url: string;
|
||||
search_url: string;
|
||||
enabled?: boolean;
|
||||
category?: string;
|
||||
currency?: string;
|
||||
selectors: {
|
||||
container: string;
|
||||
name: string;
|
||||
price: string;
|
||||
link: string;
|
||||
image?: string;
|
||||
};
|
||||
rate_limit?: number;
|
||||
rate_window?: number;
|
||||
proxy_url?: string;
|
||||
user_agent?: string;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
function queryAll(sql: string, params: any[] = []): any[] {
|
||||
const db = getDatabase();
|
||||
const stmt = db.prepare(sql);
|
||||
if (params.length) stmt.bind(params);
|
||||
const rows: any[] = [];
|
||||
while (stmt.step()) {
|
||||
rows.push(stmt.getAsObject());
|
||||
}
|
||||
stmt.free();
|
||||
return rows;
|
||||
}
|
||||
|
||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
||||
const rows = queryAll(sql, params);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
function slugify(text: string): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '');
|
||||
}
|
||||
|
||||
function ensureCategoryExists(categoryName: string): number {
|
||||
const db = getDatabase();
|
||||
const existing = queryOne('SELECT id FROM categories WHERE name = ?', [categoryName]);
|
||||
if (existing) return existing.id;
|
||||
|
||||
db.run('INSERT INTO categories (name) VALUES (?)', [categoryName]);
|
||||
return queryOne('SELECT last_insert_rowid() as id')?.id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all YAML store configs from a directory into the database.
|
||||
* Uses the filename (without extension) as the slug for matching.
|
||||
* If a store with the same slug exists, it gets updated. Otherwise, it's created.
|
||||
*/
|
||||
export function syncFromFiles(storesDir: string): { created: number; updated: number; errors: string[] } {
|
||||
if (!fs.existsSync(storesDir)) {
|
||||
fs.mkdirSync(storesDir, { recursive: true });
|
||||
return { created: 0, updated: 0, errors: [] };
|
||||
}
|
||||
|
||||
const files = fs.readdirSync(storesDir).filter((f) => f.endsWith('.yaml') || f.endsWith('.yml'));
|
||||
const db = getDatabase();
|
||||
let created = 0;
|
||||
let updated = 0;
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
const content = fs.readFileSync(path.join(storesDir, file), 'utf-8');
|
||||
const config = YAML.parse(content) as StoreFileConfig;
|
||||
const slug = path.basename(file, path.extname(file));
|
||||
|
||||
if (!config.name || !config.base_url || !config.search_url || !config.selectors) {
|
||||
errors.push(`${file}: missing required fields (name, base_url, search_url, selectors)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!config.selectors.container || !config.selectors.name || !config.selectors.price || !config.selectors.link) {
|
||||
errors.push(`${file}: missing required selectors (container, name, price, link)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const categoryId = config.category ? ensureCategoryExists(config.category) : null;
|
||||
const headersJson = config.headers ? JSON.stringify(config.headers) : null;
|
||||
|
||||
const existing = queryOne('SELECT id FROM stores WHERE slug = ?', [slug]);
|
||||
|
||||
if (existing) {
|
||||
db.run(`
|
||||
UPDATE stores SET
|
||||
name = ?, base_url = ?, search_url = ?, enabled = ?,
|
||||
sel_container = ?, sel_name = ?, sel_price = ?, sel_link = ?, sel_image = ?,
|
||||
rate_limit = ?, rate_window = ?, proxy_url = ?, user_agent = ?, headers_json = ?,
|
||||
currency = ?, category_id = ?, updated_at = datetime('now')
|
||||
WHERE slug = ?
|
||||
`, [
|
||||
config.name, config.base_url, config.search_url,
|
||||
config.enabled === false ? 0 : 1,
|
||||
config.selectors.container, config.selectors.name,
|
||||
config.selectors.price, config.selectors.link,
|
||||
config.selectors.image || null,
|
||||
config.rate_limit ?? 2, config.rate_window ?? 1000,
|
||||
config.proxy_url || null, config.user_agent || null, headersJson,
|
||||
config.currency || 'EUR', categoryId,
|
||||
slug,
|
||||
]);
|
||||
updated++;
|
||||
} else {
|
||||
db.run(`
|
||||
INSERT INTO stores (name, slug, base_url, search_url, enabled,
|
||||
sel_container, sel_name, sel_price, sel_link, sel_image,
|
||||
rate_limit, rate_window, proxy_url, user_agent, headers_json,
|
||||
currency, category_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, [
|
||||
config.name, slug, config.base_url, config.search_url,
|
||||
config.enabled === false ? 0 : 1,
|
||||
config.selectors.container, config.selectors.name,
|
||||
config.selectors.price, config.selectors.link,
|
||||
config.selectors.image || null,
|
||||
config.rate_limit ?? 2, config.rate_window ?? 1000,
|
||||
config.proxy_url || null, config.user_agent || null, headersJson,
|
||||
config.currency || 'EUR', categoryId,
|
||||
]);
|
||||
created++;
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push(`${file}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (created > 0 || updated > 0) saveDatabase();
|
||||
return { created, updated, errors };
|
||||
}
|
||||
|
||||
/**
|
||||
* Export a single store from the database to a YAML config object.
|
||||
*/
|
||||
function storeToConfig(store: any, categoryName?: string): StoreFileConfig {
|
||||
const config: StoreFileConfig = {
|
||||
name: store.name,
|
||||
base_url: store.base_url,
|
||||
search_url: store.search_url,
|
||||
selectors: {
|
||||
container: store.sel_container,
|
||||
name: store.sel_name,
|
||||
price: store.sel_price,
|
||||
link: store.sel_link,
|
||||
},
|
||||
};
|
||||
|
||||
if (store.sel_image) config.selectors.image = store.sel_image;
|
||||
if (store.enabled === 0) config.enabled = false;
|
||||
if (categoryName) config.category = categoryName;
|
||||
if (store.currency && store.currency !== 'EUR') config.currency = store.currency;
|
||||
if (store.rate_limit && store.rate_limit !== 2) config.rate_limit = store.rate_limit;
|
||||
if (store.rate_window && store.rate_window !== 1000) config.rate_window = store.rate_window;
|
||||
if (store.proxy_url) config.proxy_url = store.proxy_url;
|
||||
if (store.user_agent) config.user_agent = store.user_agent;
|
||||
if (store.headers_json) {
|
||||
try { config.headers = JSON.parse(store.headers_json); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Export a single store to a YAML file.
|
||||
*/
|
||||
export function exportStoreToFile(storeId: number, storesDir: string): string | null {
|
||||
const store = queryOne(`
|
||||
SELECT s.*, c.name as category_name
|
||||
FROM stores s LEFT JOIN categories c ON s.category_id = c.id
|
||||
WHERE s.id = ?
|
||||
`, [storeId]);
|
||||
|
||||
if (!store) return null;
|
||||
|
||||
const config = storeToConfig(store, store.category_name);
|
||||
const yaml = YAML.stringify(config, { lineWidth: 120 });
|
||||
const filePath = path.join(storesDir, `${store.slug}.yaml`);
|
||||
|
||||
if (!fs.existsSync(storesDir)) fs.mkdirSync(storesDir, { recursive: true });
|
||||
fs.writeFileSync(filePath, yaml);
|
||||
|
||||
return filePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Export all stores to YAML files.
|
||||
*/
|
||||
export function exportAllStoresToFiles(storesDir: string): number {
|
||||
const stores = queryAll(`
|
||||
SELECT s.*, c.name as category_name
|
||||
FROM stores s LEFT JOIN categories c ON s.category_id = c.id
|
||||
ORDER BY s.name
|
||||
`);
|
||||
|
||||
if (!fs.existsSync(storesDir)) fs.mkdirSync(storesDir, { recursive: true });
|
||||
|
||||
for (const store of stores) {
|
||||
const config = storeToConfig(store, store.category_name);
|
||||
const yaml = YAML.stringify(config, { lineWidth: 120 });
|
||||
fs.writeFileSync(path.join(storesDir, `${store.slug}.yaml`), yaml);
|
||||
}
|
||||
|
||||
return stores.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a single store back to its YAML file after a DB update.
|
||||
*/
|
||||
export function writeStoreBackToFile(storeId: number, storesDir: string): void {
|
||||
exportStoreToFile(storeId, storesDir);
|
||||
}
|
||||
Reference in New Issue
Block a user