Security and code quality audit fixes
Security: - Fix SQL injection in updateStore — whitelist allowed field names - Restrict CORS to same-origin in production - Cap results at 200 per store to prevent memory issues Code quality: - Extract shared queryAll/queryOne to src/server/db/query.ts - Remove duplicated DB helpers from 5 files - Handle render_js boolean-to-integer conversion in updateStore UX: - Validate headers_json as valid JSON before saving (both forms) - Show error message if JSON is invalid Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -37,6 +37,7 @@
|
|||||||
try {
|
try {
|
||||||
const data = { ...form };
|
const data = { ...form };
|
||||||
if (data.category_id) data.category_id = Number(data.category_id); else delete data.category_id;
|
if (data.category_id) data.category_id = Number(data.category_id); else delete data.category_id;
|
||||||
|
if (data.headers_json) { try { JSON.parse(data.headers_json); } catch { error = 'Extra Headers must be valid JSON'; saving = false; return; } }
|
||||||
await updateStore(Number($page.params.id), data);
|
await updateStore(Number($page.params.id), data);
|
||||||
goto('/admin');
|
goto('/admin');
|
||||||
} catch (err) { error = err.message || 'Failed to update store'; }
|
} catch (err) { error = err.message || 'Failed to update store'; }
|
||||||
|
|||||||
@@ -27,6 +27,8 @@
|
|||||||
if (!data.user_agent) delete data.user_agent;
|
if (!data.user_agent) delete data.user_agent;
|
||||||
if (!data.proxy_url) delete data.proxy_url;
|
if (!data.proxy_url) delete data.proxy_url;
|
||||||
if (!data.headers_json) delete data.headers_json;
|
if (!data.headers_json) delete data.headers_json;
|
||||||
|
else { try { JSON.parse(data.headers_json); } catch { error = 'Extra Headers must be valid JSON'; saving = false; return; } }
|
||||||
|
if (!data.test_query) delete data.test_query;
|
||||||
const store = await createStore(data);
|
const store = await createStore(data);
|
||||||
goto(`/admin/stores/${store.id}/test`);
|
goto(`/admin/stores/${store.id}/test`);
|
||||||
} catch (err) { error = err.message || 'Failed to create store'; }
|
} catch (err) { error = err.message || 'Failed to create store'; }
|
||||||
|
|||||||
18
src/server/db/query.ts
Normal file
18
src/server/db/query.ts
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
import { getDatabase } from './connection.js';
|
||||||
|
|
||||||
|
export function queryAll(sql: string, params: any[] = []): any[] {
|
||||||
|
const db = getDatabase();
|
||||||
|
const stmt = db.prepare(sql);
|
||||||
|
if (params.length) stmt.bind(params);
|
||||||
|
const rows: any[] = [];
|
||||||
|
while (stmt.step()) {
|
||||||
|
rows.push(stmt.getAsObject());
|
||||||
|
}
|
||||||
|
stmt.free();
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function queryOne(sql: string, params: any[] = []): any | undefined {
|
||||||
|
const rows = queryAll(sql, params);
|
||||||
|
return rows[0];
|
||||||
|
}
|
||||||
@@ -24,7 +24,9 @@ const app = Fastify({
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
await app.register(cors, { origin: true });
|
await app.register(cors, {
|
||||||
|
origin: config.isProduction ? false : true,
|
||||||
|
});
|
||||||
|
|
||||||
// API routes
|
// API routes
|
||||||
await app.register(storeRoutes, { prefix: '/api' });
|
await app.register(storeRoutes, { prefix: '/api' });
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||||
|
import { queryAll, queryOne } from '../db/query.js';
|
||||||
|
|
||||||
export interface Category {
|
export interface Category {
|
||||||
id: number;
|
id: number;
|
||||||
@@ -17,23 +18,6 @@ export interface StoreGroupWithMembers extends StoreGroup {
|
|||||||
store_ids: number[];
|
store_ids: number[];
|
||||||
}
|
}
|
||||||
|
|
||||||
function queryAll(sql: string, params: any[] = []): any[] {
|
|
||||||
const db = getDatabase();
|
|
||||||
const stmt = db.prepare(sql);
|
|
||||||
if (params.length) stmt.bind(params);
|
|
||||||
const rows: any[] = [];
|
|
||||||
while (stmt.step()) {
|
|
||||||
rows.push(stmt.getAsObject());
|
|
||||||
}
|
|
||||||
stmt.free();
|
|
||||||
return rows;
|
|
||||||
}
|
|
||||||
|
|
||||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
|
||||||
const rows = queryAll(sql, params);
|
|
||||||
return rows[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
// Categories
|
// Categories
|
||||||
|
|
||||||
export function getAllCategories(): Category[] {
|
export function getAllCategories(): Category[] {
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||||
|
import { queryAll, queryOne } from '../db/query.js';
|
||||||
|
|
||||||
export interface ScrapeLog {
|
export interface ScrapeLog {
|
||||||
id: number;
|
id: number;
|
||||||
@@ -11,23 +12,6 @@ export interface ScrapeLog {
|
|||||||
scraped_at: string;
|
scraped_at: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
function queryAll(sql: string, params: any[] = []): any[] {
|
|
||||||
const db = getDatabase();
|
|
||||||
const stmt = db.prepare(sql);
|
|
||||||
if (params.length) stmt.bind(params);
|
|
||||||
const rows: any[] = [];
|
|
||||||
while (stmt.step()) {
|
|
||||||
rows.push(stmt.getAsObject());
|
|
||||||
}
|
|
||||||
stmt.free();
|
|
||||||
return rows;
|
|
||||||
}
|
|
||||||
|
|
||||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
|
||||||
const rows = queryAll(sql, params);
|
|
||||||
return rows[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
export function logScrape(
|
export function logScrape(
|
||||||
storeId: number,
|
storeId: number,
|
||||||
query: string,
|
query: string,
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||||
|
import { queryAll, queryOne } from '../db/query.js';
|
||||||
|
|
||||||
export interface Store {
|
export interface Store {
|
||||||
id: number;
|
id: number;
|
||||||
@@ -53,23 +54,6 @@ export interface CreateStoreInput {
|
|||||||
category_id?: number;
|
category_id?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
function queryAll(sql: string, params: any[] = []): any[] {
|
|
||||||
const db = getDatabase();
|
|
||||||
const stmt = db.prepare(sql);
|
|
||||||
if (params.length) stmt.bind(params);
|
|
||||||
const rows: any[] = [];
|
|
||||||
while (stmt.step()) {
|
|
||||||
rows.push(stmt.getAsObject());
|
|
||||||
}
|
|
||||||
stmt.free();
|
|
||||||
return rows;
|
|
||||||
}
|
|
||||||
|
|
||||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
|
||||||
const rows = queryAll(sql, params);
|
|
||||||
return rows[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
function slugify(text: string): string {
|
function slugify(text: string): string {
|
||||||
return text
|
return text
|
||||||
.toLowerCase()
|
.toLowerCase()
|
||||||
@@ -148,10 +132,17 @@ export function updateStore(id: number, input: Partial<CreateStoreInput>): Store
|
|||||||
const fields: string[] = [];
|
const fields: string[] = [];
|
||||||
const values: any[] = [];
|
const values: any[] = [];
|
||||||
|
|
||||||
|
const allowedFields = new Set([
|
||||||
|
'name', 'slug', 'base_url', 'search_url',
|
||||||
|
'sel_container', 'sel_name', 'sel_price', 'sel_link', 'sel_image',
|
||||||
|
'render_js', 'test_query', 'rate_limit', 'rate_window',
|
||||||
|
'proxy_url', 'user_agent', 'headers_json', 'currency', 'category_id',
|
||||||
|
]);
|
||||||
|
|
||||||
for (const [key, value] of Object.entries(input)) {
|
for (const [key, value] of Object.entries(input)) {
|
||||||
if (value !== undefined) {
|
if (value !== undefined && allowedFields.has(key)) {
|
||||||
fields.push(`${key} = ?`);
|
fields.push(`${key} = ?`);
|
||||||
values.push(value);
|
values.push(key === 'render_js' ? (value ? 1 : 0) : value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,19 +1,7 @@
|
|||||||
import type { FastifyPluginAsync } from 'fastify';
|
import type { FastifyPluginAsync } from 'fastify';
|
||||||
import fs from 'node:fs';
|
import fs from 'node:fs';
|
||||||
import { config } from '../config.js';
|
import { config } from '../config.js';
|
||||||
import { getDatabase } from '../db/connection.js';
|
import { queryOne } from '../db/query.js';
|
||||||
|
|
||||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
|
||||||
const db = getDatabase();
|
|
||||||
const stmt = db.prepare(sql);
|
|
||||||
if (params.length) stmt.bind(params);
|
|
||||||
let result: any;
|
|
||||||
if (stmt.step()) {
|
|
||||||
result = stmt.getAsObject();
|
|
||||||
}
|
|
||||||
stmt.free();
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const healthRoutes: FastifyPluginAsync = async (app) => {
|
export const healthRoutes: FastifyPluginAsync = async (app) => {
|
||||||
app.get('/health', async () => {
|
app.get('/health', async () => {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import { getLimiter } from './rate-limiter.js';
|
|||||||
|
|
||||||
const MAX_CONCURRENCY = 5;
|
const MAX_CONCURRENCY = 5;
|
||||||
const SEARCH_TIMEOUT = 60_000;
|
const SEARCH_TIMEOUT = 60_000;
|
||||||
|
const MAX_RESULTS_PER_STORE = 200;
|
||||||
|
|
||||||
export interface SearchOptions {
|
export interface SearchOptions {
|
||||||
query: string;
|
query: string;
|
||||||
@@ -85,7 +86,7 @@ export async function search(options: SearchOptions): Promise<SearchResult> {
|
|||||||
const result = await rateLimiter.schedule(scrapeFn);
|
const result = await rateLimiter.schedule(scrapeFn);
|
||||||
const duration = Date.now() - storeStart;
|
const duration = Date.now() - storeStart;
|
||||||
|
|
||||||
const products = result.items.map((item) =>
|
const products = result.items.slice(0, MAX_RESULTS_PER_STORE).map((item) =>
|
||||||
normalizeResult(item, store.id, store.name, store.base_url, store.currency)
|
normalizeResult(item, store.id, store.name, store.base_url, store.currency)
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -185,7 +186,7 @@ export async function searchStreaming(
|
|||||||
const result = await rateLimiter.schedule(scrapeFn);
|
const result = await rateLimiter.schedule(scrapeFn);
|
||||||
const duration = Date.now() - storeStart;
|
const duration = Date.now() - storeStart;
|
||||||
|
|
||||||
const products = result.items.map((item) =>
|
const products = result.items.slice(0, MAX_RESULTS_PER_STORE).map((item) =>
|
||||||
normalizeResult(item, store.id, store.name, store.base_url, store.currency)
|
normalizeResult(item, store.id, store.name, store.base_url, store.currency)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import fs from 'node:fs';
|
|||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import YAML from 'yaml';
|
import YAML from 'yaml';
|
||||||
import { getDatabase, saveDatabase } from '../db/connection.js';
|
import { getDatabase, saveDatabase } from '../db/connection.js';
|
||||||
|
import { queryAll, queryOne } from '../db/query.js';
|
||||||
|
|
||||||
export interface StoreFileConfig {
|
export interface StoreFileConfig {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -26,22 +27,6 @@ export interface StoreFileConfig {
|
|||||||
headers?: Record<string, string>;
|
headers?: Record<string, string>;
|
||||||
}
|
}
|
||||||
|
|
||||||
function queryAll(sql: string, params: any[] = []): any[] {
|
|
||||||
const db = getDatabase();
|
|
||||||
const stmt = db.prepare(sql);
|
|
||||||
if (params.length) stmt.bind(params);
|
|
||||||
const rows: any[] = [];
|
|
||||||
while (stmt.step()) {
|
|
||||||
rows.push(stmt.getAsObject());
|
|
||||||
}
|
|
||||||
stmt.free();
|
|
||||||
return rows;
|
|
||||||
}
|
|
||||||
|
|
||||||
function queryOne(sql: string, params: any[] = []): any | undefined {
|
|
||||||
const rows = queryAll(sql, params);
|
|
||||||
return rows[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
function slugify(text: string): string {
|
function slugify(text: string): string {
|
||||||
return text
|
return text
|
||||||
|
|||||||
Reference in New Issue
Block a user