Initial commit: Price Hunter — self-hosted price comparison engine
Complete application scaffolding with: - Backend: Node.js + Fastify + sql.js (SQLite) - Frontend: SvelteKit + Tailwind CSS - Scraper engine with parallel fan-out, rate limiting, cheerio-based parsing - Store management with CSS selector config and per-store test pages - Docker setup for single-command deployment Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
mariosemes
139
src/server/models/category.ts
Normal file
139
src/server/models/category.ts
Normal file
@@ -0,0 +1,139 @@
|
||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||
|
||||
export interface Category {
|
||||
id: number;
|
||||
name: string;
|
||||
color: string;
|
||||
sort_order: number;
|
||||
}
|
||||
|
||||
export interface StoreGroup {
|
||||
id: number;
|
||||
name: string;
|
||||
description: string | null;
|
||||
}
|
||||
|
||||
export interface StoreGroupWithMembers extends StoreGroup {
|
||||
store_ids: number[];
|
||||
}
|
||||
|
||||
function queryAll(sql: string, params: any[] = []): any[] {
|
||||
const db = getDatabase();
|
||||
const stmt = db.prepare(sql);
|
||||
if (params.length) stmt.bind(params);
|
||||
const rows: any[] = [];
|
||||
while (stmt.step()) {
|
||||
rows.push(stmt.getAsObject());
|
||||
}
|
||||
stmt.free();
|
||||
return rows;
|
||||
}
|
||||
|
||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
||||
const rows = queryAll(sql, params);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
// Categories
|
||||
|
||||
export function getAllCategories(): Category[] {
|
||||
return queryAll('SELECT * FROM categories ORDER BY sort_order, name');
|
||||
}
|
||||
|
||||
export function getCategoryById(id: number): Category | undefined {
|
||||
return queryOne('SELECT * FROM categories WHERE id = ?', [id]);
|
||||
}
|
||||
|
||||
export function createCategory(name: string, color?: string): Category {
|
||||
const db = getDatabase();
|
||||
const maxOrder = queryOne('SELECT MAX(sort_order) as max_order FROM categories');
|
||||
const sortOrder = (maxOrder?.max_order ?? -1) + 1;
|
||||
db.run('INSERT INTO categories (name, color, sort_order) VALUES (?, ?, ?)', [name, color || '#6B7280', sortOrder]);
|
||||
const lastId = queryOne('SELECT last_insert_rowid() as id')?.id;
|
||||
saveDatabase();
|
||||
return getCategoryById(lastId) as Category;
|
||||
}
|
||||
|
||||
export function updateCategory(id: number, data: { name?: string; color?: string; sort_order?: number }): Category | undefined {
|
||||
const db = getDatabase();
|
||||
const fields: string[] = [];
|
||||
const values: any[] = [];
|
||||
|
||||
if (data.name !== undefined) { fields.push('name = ?'); values.push(data.name); }
|
||||
if (data.color !== undefined) { fields.push('color = ?'); values.push(data.color); }
|
||||
if (data.sort_order !== undefined) { fields.push('sort_order = ?'); values.push(data.sort_order); }
|
||||
|
||||
if (fields.length === 0) return getCategoryById(id);
|
||||
|
||||
values.push(id);
|
||||
db.run(`UPDATE categories SET ${fields.join(', ')} WHERE id = ?`, values);
|
||||
saveDatabase();
|
||||
return getCategoryById(id);
|
||||
}
|
||||
|
||||
export function deleteCategory(id: number): boolean {
|
||||
const db = getDatabase();
|
||||
db.run('DELETE FROM categories WHERE id = ?', [id]);
|
||||
const changes = db.getRowsModified();
|
||||
if (changes > 0) saveDatabase();
|
||||
return changes > 0;
|
||||
}
|
||||
|
||||
// Groups
|
||||
|
||||
export function getAllGroups(): StoreGroupWithMembers[] {
|
||||
const groups = queryAll('SELECT * FROM store_groups ORDER BY name');
|
||||
return groups.map((group) => {
|
||||
const members = queryAll('SELECT store_id FROM store_group_members WHERE group_id = ?', [group.id]);
|
||||
return { ...group, store_ids: members.map((m: any) => m.store_id) };
|
||||
});
|
||||
}
|
||||
|
||||
export function getGroupById(id: number): StoreGroupWithMembers | undefined {
|
||||
const group = queryOne('SELECT * FROM store_groups WHERE id = ?', [id]);
|
||||
if (!group) return undefined;
|
||||
|
||||
const members = queryAll('SELECT store_id FROM store_group_members WHERE group_id = ?', [id]);
|
||||
return { ...group, store_ids: members.map((m: any) => m.store_id) };
|
||||
}
|
||||
|
||||
export function createGroup(name: string, description?: string): StoreGroupWithMembers {
|
||||
const db = getDatabase();
|
||||
db.run('INSERT INTO store_groups (name, description) VALUES (?, ?)', [name, description || null]);
|
||||
const lastId = queryOne('SELECT last_insert_rowid() as id')?.id;
|
||||
saveDatabase();
|
||||
return getGroupById(lastId) as StoreGroupWithMembers;
|
||||
}
|
||||
|
||||
export function updateGroup(id: number, data: { name?: string; description?: string }): StoreGroupWithMembers | undefined {
|
||||
const db = getDatabase();
|
||||
const fields: string[] = [];
|
||||
const values: any[] = [];
|
||||
|
||||
if (data.name !== undefined) { fields.push('name = ?'); values.push(data.name); }
|
||||
if (data.description !== undefined) { fields.push('description = ?'); values.push(data.description); }
|
||||
|
||||
if (fields.length > 0) {
|
||||
values.push(id);
|
||||
db.run(`UPDATE store_groups SET ${fields.join(', ')} WHERE id = ?`, values);
|
||||
saveDatabase();
|
||||
}
|
||||
return getGroupById(id);
|
||||
}
|
||||
|
||||
export function deleteGroup(id: number): boolean {
|
||||
const db = getDatabase();
|
||||
db.run('DELETE FROM store_groups WHERE id = ?', [id]);
|
||||
const changes = db.getRowsModified();
|
||||
if (changes > 0) saveDatabase();
|
||||
return changes > 0;
|
||||
}
|
||||
|
||||
export function setGroupMembers(groupId: number, storeIds: number[]): void {
|
||||
const db = getDatabase();
|
||||
db.run('DELETE FROM store_group_members WHERE group_id = ?', [groupId]);
|
||||
for (const storeId of storeIds) {
|
||||
db.run('INSERT INTO store_group_members (group_id, store_id) VALUES (?, ?)', [groupId, storeId]);
|
||||
}
|
||||
saveDatabase();
|
||||
}
|
||||
85
src/server/models/scrape-log.ts
Normal file
85
src/server/models/scrape-log.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||
|
||||
export interface ScrapeLog {
|
||||
id: number;
|
||||
store_id: number;
|
||||
query: string;
|
||||
success: number;
|
||||
result_count: number;
|
||||
duration_ms: number;
|
||||
error_message: string | null;
|
||||
scraped_at: string;
|
||||
}
|
||||
|
||||
function queryAll(sql: string, params: any[] = []): any[] {
|
||||
const db = getDatabase();
|
||||
const stmt = db.prepare(sql);
|
||||
if (params.length) stmt.bind(params);
|
||||
const rows: any[] = [];
|
||||
while (stmt.step()) {
|
||||
rows.push(stmt.getAsObject());
|
||||
}
|
||||
stmt.free();
|
||||
return rows;
|
||||
}
|
||||
|
||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
||||
const rows = queryAll(sql, params);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
export function logScrape(
|
||||
storeId: number,
|
||||
query: string,
|
||||
success: boolean,
|
||||
resultCount: number,
|
||||
durationMs: number,
|
||||
errorMessage?: string
|
||||
): ScrapeLog {
|
||||
const db = getDatabase();
|
||||
db.run(`
|
||||
INSERT INTO scrape_logs (store_id, query, success, result_count, duration_ms, error_message)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`, [storeId, query, success ? 1 : 0, resultCount, durationMs, errorMessage || null]);
|
||||
|
||||
const lastId = queryOne('SELECT last_insert_rowid() as id')?.id;
|
||||
saveDatabase();
|
||||
return queryOne('SELECT * FROM scrape_logs WHERE id = ?', [lastId]) as ScrapeLog;
|
||||
}
|
||||
|
||||
export function getLogsByStore(storeId: number, limit = 20): ScrapeLog[] {
|
||||
return queryAll('SELECT * FROM scrape_logs WHERE store_id = ? ORDER BY scraped_at DESC LIMIT ?', [storeId, limit]);
|
||||
}
|
||||
|
||||
export function getStoreHealth(storeId: number): {
|
||||
total: number;
|
||||
successful: number;
|
||||
failed: number;
|
||||
avg_duration_ms: number;
|
||||
last_success: string | null;
|
||||
last_error: string | null;
|
||||
} {
|
||||
const stats = queryOne(`
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful,
|
||||
SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failed,
|
||||
AVG(duration_ms) as avg_duration_ms,
|
||||
MAX(CASE WHEN success = 1 THEN scraped_at END) as last_success
|
||||
FROM scrape_logs WHERE store_id = ?
|
||||
`, [storeId]);
|
||||
|
||||
const lastError = queryOne(
|
||||
'SELECT error_message FROM scrape_logs WHERE store_id = ? AND success = 0 ORDER BY scraped_at DESC LIMIT 1',
|
||||
[storeId]
|
||||
);
|
||||
|
||||
return {
|
||||
total: stats?.total || 0,
|
||||
successful: stats?.successful || 0,
|
||||
failed: stats?.failed || 0,
|
||||
avg_duration_ms: Math.round(stats?.avg_duration_ms || 0),
|
||||
last_success: stats?.last_success || null,
|
||||
last_error: lastError?.error_message || null,
|
||||
};
|
||||
}
|
||||
174
src/server/models/store.ts
Normal file
174
src/server/models/store.ts
Normal file
@@ -0,0 +1,174 @@
|
||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||
|
||||
export interface Store {
|
||||
id: number;
|
||||
name: string;
|
||||
slug: string;
|
||||
base_url: string;
|
||||
search_url: string;
|
||||
enabled: number;
|
||||
sel_container: string;
|
||||
sel_name: string;
|
||||
sel_price: string;
|
||||
sel_link: string;
|
||||
sel_image: string | null;
|
||||
rate_limit: number;
|
||||
rate_window: number;
|
||||
proxy_url: string | null;
|
||||
user_agent: string | null;
|
||||
headers_json: string | null;
|
||||
currency: string;
|
||||
category_id: number | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface StoreWithCategory extends Store {
|
||||
category_name: string | null;
|
||||
category_color: string | null;
|
||||
}
|
||||
|
||||
export interface CreateStoreInput {
|
||||
name: string;
|
||||
slug?: string;
|
||||
base_url: string;
|
||||
search_url: string;
|
||||
sel_container: string;
|
||||
sel_name: string;
|
||||
sel_price: string;
|
||||
sel_link: string;
|
||||
sel_image?: string;
|
||||
rate_limit?: number;
|
||||
rate_window?: number;
|
||||
proxy_url?: string;
|
||||
user_agent?: string;
|
||||
headers_json?: string;
|
||||
currency?: string;
|
||||
category_id?: number;
|
||||
}
|
||||
|
||||
function queryAll(sql: string, params: any[] = []): any[] {
|
||||
const db = getDatabase();
|
||||
const stmt = db.prepare(sql);
|
||||
if (params.length) stmt.bind(params);
|
||||
const rows: any[] = [];
|
||||
while (stmt.step()) {
|
||||
rows.push(stmt.getAsObject());
|
||||
}
|
||||
stmt.free();
|
||||
return rows;
|
||||
}
|
||||
|
||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
||||
const rows = queryAll(sql, params);
|
||||
return rows[0];
|
||||
}
|
||||
|
||||
function slugify(text: string): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '');
|
||||
}
|
||||
|
||||
export function getAllStores(): StoreWithCategory[] {
|
||||
return queryAll(`
|
||||
SELECT s.*, c.name as category_name, c.color as category_color
|
||||
FROM stores s
|
||||
LEFT JOIN categories c ON s.category_id = c.id
|
||||
ORDER BY s.name
|
||||
`);
|
||||
}
|
||||
|
||||
export function getStoreById(id: number): StoreWithCategory | undefined {
|
||||
return queryOne(`
|
||||
SELECT s.*, c.name as category_name, c.color as category_color
|
||||
FROM stores s
|
||||
LEFT JOIN categories c ON s.category_id = c.id
|
||||
WHERE s.id = ?
|
||||
`, [id]);
|
||||
}
|
||||
|
||||
export function getEnabledStores(): Store[] {
|
||||
return queryAll('SELECT * FROM stores WHERE enabled = 1 ORDER BY name');
|
||||
}
|
||||
|
||||
export function getStoresByCategory(categoryId: number): Store[] {
|
||||
return queryAll('SELECT * FROM stores WHERE enabled = 1 AND category_id = ? ORDER BY name', [categoryId]);
|
||||
}
|
||||
|
||||
export function getStoresByGroup(groupId: number): Store[] {
|
||||
return queryAll(`
|
||||
SELECT s.* FROM stores s
|
||||
JOIN store_group_members sgm ON s.id = sgm.store_id
|
||||
WHERE s.enabled = 1 AND sgm.group_id = ?
|
||||
ORDER BY s.name
|
||||
`, [groupId]);
|
||||
}
|
||||
|
||||
export function getStoresByIds(ids: number[]): Store[] {
|
||||
if (ids.length === 0) return [];
|
||||
const placeholders = ids.map(() => '?').join(',');
|
||||
return queryAll(`SELECT * FROM stores WHERE enabled = 1 AND id IN (${placeholders}) ORDER BY name`, ids);
|
||||
}
|
||||
|
||||
export function createStore(input: CreateStoreInput): Store {
|
||||
const db = getDatabase();
|
||||
const slug = input.slug || slugify(input.name);
|
||||
|
||||
db.run(`
|
||||
INSERT INTO stores (name, slug, base_url, search_url, sel_container, sel_name, sel_price, sel_link, sel_image,
|
||||
rate_limit, rate_window, proxy_url, user_agent, headers_json, currency, category_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, [
|
||||
input.name, slug, input.base_url, input.search_url,
|
||||
input.sel_container, input.sel_name, input.sel_price, input.sel_link, input.sel_image || null,
|
||||
input.rate_limit ?? 2, input.rate_window ?? 1000,
|
||||
input.proxy_url || null, input.user_agent || null, input.headers_json || null,
|
||||
input.currency || 'EUR', input.category_id || null,
|
||||
]);
|
||||
|
||||
const lastId = queryOne('SELECT last_insert_rowid() as id')?.id;
|
||||
saveDatabase();
|
||||
return getStoreById(lastId) as Store;
|
||||
}
|
||||
|
||||
export function updateStore(id: number, input: Partial<CreateStoreInput>): Store | undefined {
|
||||
const existing = getStoreById(id);
|
||||
if (!existing) return undefined;
|
||||
|
||||
const db = getDatabase();
|
||||
const fields: string[] = [];
|
||||
const values: any[] = [];
|
||||
|
||||
for (const [key, value] of Object.entries(input)) {
|
||||
if (value !== undefined) {
|
||||
fields.push(`${key} = ?`);
|
||||
values.push(value);
|
||||
}
|
||||
}
|
||||
|
||||
if (fields.length === 0) return existing;
|
||||
|
||||
fields.push("updated_at = datetime('now')");
|
||||
values.push(id);
|
||||
|
||||
db.run(`UPDATE stores SET ${fields.join(', ')} WHERE id = ?`, values);
|
||||
saveDatabase();
|
||||
return getStoreById(id);
|
||||
}
|
||||
|
||||
export function toggleStoreEnabled(id: number): Store | undefined {
|
||||
const db = getDatabase();
|
||||
db.run("UPDATE stores SET enabled = CASE WHEN enabled = 1 THEN 0 ELSE 1 END, updated_at = datetime('now') WHERE id = ?", [id]);
|
||||
saveDatabase();
|
||||
return getStoreById(id);
|
||||
}
|
||||
|
||||
export function deleteStore(id: number): boolean {
|
||||
const db = getDatabase();
|
||||
db.run('DELETE FROM stores WHERE id = ?', [id]);
|
||||
const changes = db.getRowsModified();
|
||||
if (changes > 0) saveDatabase();
|
||||
return changes > 0;
|
||||
}
|
||||
Reference in New Issue
Block a user