import fs from 'node:fs'; import path from 'node:path'; import YAML from 'yaml'; import { getDatabase, saveDatabase } from '../db/connection.js'; import { queryAll, queryOne } from '../db/query.js'; export interface StoreFileConfig { name: string; base_url: string; search_url: string; enabled?: boolean; render_js?: boolean; test_query?: string; category?: string; currency?: string; selectors: { container: string; name: string; price: string; link: string; image?: string; }; rate_limit?: number; rate_window?: number; proxy_url?: string; user_agent?: string; headers?: Record; } function slugify(text: string): string { return text .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-|-$/g, ''); } function ensureCategoryExists(categoryName: string): number { const db = getDatabase(); const existing = queryOne('SELECT id FROM categories WHERE name = ?', [categoryName]); if (existing) return existing.id; db.run('INSERT INTO categories (name) VALUES (?)', [categoryName]); return queryOne('SELECT last_insert_rowid() as id')?.id; } /** * Load all YAML store configs from a directory into the database. * Uses the filename (without extension) as the slug for matching. * If a store with the same slug exists, it gets updated. Otherwise, it's created. */ export function syncFromFiles(storesDir: string): { created: number; updated: number; errors: string[] } { if (!fs.existsSync(storesDir)) { fs.mkdirSync(storesDir, { recursive: true }); return { created: 0, updated: 0, errors: [] }; } const files = fs.readdirSync(storesDir).filter((f) => f.endsWith('.yaml') || f.endsWith('.yml')); const db = getDatabase(); let created = 0; let updated = 0; const errors: string[] = []; for (const file of files) { try { const content = fs.readFileSync(path.join(storesDir, file), 'utf-8'); const config = YAML.parse(content) as StoreFileConfig; const slug = path.basename(file, path.extname(file)); if (!config.name || !config.base_url || !config.search_url || !config.selectors) { errors.push(`${file}: missing required fields (name, base_url, search_url, selectors)`); continue; } if (!config.selectors.container || !config.selectors.name || !config.selectors.price || !config.selectors.link) { errors.push(`${file}: missing required selectors (container, name, price, link)`); continue; } const categoryId = config.category ? ensureCategoryExists(config.category) : null; const headersJson = config.headers ? JSON.stringify(config.headers) : null; const existing = queryOne('SELECT id FROM stores WHERE slug = ?', [slug]); if (existing) { db.run(` UPDATE stores SET name = ?, base_url = ?, search_url = ?, enabled = ?, render_js = ?, test_query = ?, sel_container = ?, sel_name = ?, sel_price = ?, sel_link = ?, sel_image = ?, rate_limit = ?, rate_window = ?, proxy_url = ?, user_agent = ?, headers_json = ?, currency = ?, category_id = ?, updated_at = datetime('now') WHERE slug = ? `, [ config.name, config.base_url, config.search_url, config.enabled === false ? 0 : 1, config.render_js ? 1 : 0, config.test_query || 'test', config.selectors.container, config.selectors.name, config.selectors.price, config.selectors.link, config.selectors.image || null, config.rate_limit ?? 2, config.rate_window ?? 1000, config.proxy_url || null, config.user_agent || null, headersJson, config.currency || 'EUR', categoryId, slug, ]); updated++; } else { db.run(` INSERT INTO stores (name, slug, base_url, search_url, enabled, render_js, test_query, sel_container, sel_name, sel_price, sel_link, sel_image, rate_limit, rate_window, proxy_url, user_agent, headers_json, currency, category_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, [ config.name, slug, config.base_url, config.search_url, config.enabled === false ? 0 : 1, config.render_js ? 1 : 0, config.test_query || 'test', config.selectors.container, config.selectors.name, config.selectors.price, config.selectors.link, config.selectors.image || null, config.rate_limit ?? 2, config.rate_window ?? 1000, config.proxy_url || null, config.user_agent || null, headersJson, config.currency || 'EUR', categoryId, ]); created++; } } catch (err) { errors.push(`${file}: ${err instanceof Error ? err.message : String(err)}`); } } if (created > 0 || updated > 0) saveDatabase(); return { created, updated, errors }; } /** * Export a single store from the database to a YAML config object. */ function storeToConfig(store: any, categoryName?: string): StoreFileConfig { const config: StoreFileConfig = { name: store.name, base_url: store.base_url, search_url: store.search_url, selectors: { container: store.sel_container, name: store.sel_name, price: store.sel_price, link: store.sel_link, }, }; if (store.sel_image) config.selectors.image = store.sel_image; if (store.enabled === 0) config.enabled = false; if (store.render_js) config.render_js = true; if (store.test_query && store.test_query !== 'test') config.test_query = store.test_query; if (categoryName) config.category = categoryName; if (store.currency && store.currency !== 'EUR') config.currency = store.currency; if (store.rate_limit && store.rate_limit !== 2) config.rate_limit = store.rate_limit; if (store.rate_window && store.rate_window !== 1000) config.rate_window = store.rate_window; if (store.proxy_url) config.proxy_url = store.proxy_url; if (store.user_agent) config.user_agent = store.user_agent; if (store.headers_json) { try { config.headers = JSON.parse(store.headers_json); } catch { /* ignore */ } } return config; } /** * Export a single store to a YAML file. */ export function exportStoreToFile(storeId: number, storesDir: string): string | null { const store = queryOne(` SELECT s.*, c.name as category_name FROM stores s LEFT JOIN categories c ON s.category_id = c.id WHERE s.id = ? `, [storeId]); if (!store) return null; const config = storeToConfig(store, store.category_name); const yaml = YAML.stringify(config, { lineWidth: 120 }); const filePath = path.join(storesDir, `${store.slug}.yaml`); if (!fs.existsSync(storesDir)) fs.mkdirSync(storesDir, { recursive: true }); fs.writeFileSync(filePath, yaml); return filePath; } /** * Export all stores to YAML files. */ export function exportAllStoresToFiles(storesDir: string): number { const stores = queryAll(` SELECT s.*, c.name as category_name FROM stores s LEFT JOIN categories c ON s.category_id = c.id ORDER BY s.name `); if (!fs.existsSync(storesDir)) fs.mkdirSync(storesDir, { recursive: true }); for (const store of stores) { const config = storeToConfig(store, store.category_name); const yaml = YAML.stringify(config, { lineWidth: 120 }); fs.writeFileSync(path.join(storesDir, `${store.slug}.yaml`), yaml); } return stores.length; } /** * Write a single store back to its YAML file after a DB update. */ export function writeStoreBackToFile(storeId: number, storesDir: string): void { exportStoreToFile(storeId, storesDir); }