Initial implementation of Website Change Detection Monitor MVP
Features implemented: - Backend API with Express + TypeScript - User authentication (register/login with JWT) - Monitor CRUD operations with plan-based limits - Automated change detection engine - Email alert system - Frontend with Next.js + TypeScript - Dashboard with monitor management - Login/register pages - Monitor history viewer - PostgreSQL database schema - Docker setup for local development Technical stack: - Backend: Express, TypeScript, PostgreSQL, Redis (ready) - Frontend: Next.js 14, React Query, Tailwind CSS - Database: PostgreSQL with migrations - Services: Page fetching, diff detection, email alerts Documentation: - README with full setup instructions - SETUP guide for quick start - PROJECT_STATUS with current capabilities - Complete technical specifications Ready for local testing and feature expansion. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Timo
284
backend/src/db/index.ts
Normal file
284
backend/src/db/index.ts
Normal file
@@ -0,0 +1,284 @@
|
||||
import { Pool, QueryResult } from 'pg';
|
||||
import { User, Monitor, Snapshot, Alert } from '../types';
|
||||
|
||||
const pool = new Pool({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
max: 20,
|
||||
idleTimeoutMillis: 30000,
|
||||
connectionTimeoutMillis: 2000,
|
||||
});
|
||||
|
||||
pool.on('error', (err) => {
|
||||
console.error('Unexpected database error:', err);
|
||||
process.exit(-1);
|
||||
});
|
||||
|
||||
export const query = async <T = any>(
|
||||
text: string,
|
||||
params?: any[]
|
||||
): Promise<QueryResult<T>> => {
|
||||
const start = Date.now();
|
||||
const result = await pool.query<T>(text, params);
|
||||
const duration = Date.now() - start;
|
||||
|
||||
if (duration > 1000) {
|
||||
console.warn(`Slow query (${duration}ms):`, text);
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
export const getClient = () => pool.connect();
|
||||
|
||||
// User queries
|
||||
export const db = {
|
||||
users: {
|
||||
async create(email: string, passwordHash: string): Promise<User> {
|
||||
const result = await query<User>(
|
||||
'INSERT INTO users (email, password_hash) VALUES ($1, $2) RETURNING *',
|
||||
[email, passwordHash]
|
||||
);
|
||||
return result.rows[0];
|
||||
},
|
||||
|
||||
async findById(id: string): Promise<User | null> {
|
||||
const result = await query<User>(
|
||||
'SELECT * FROM users WHERE id = $1',
|
||||
[id]
|
||||
);
|
||||
return result.rows[0] || null;
|
||||
},
|
||||
|
||||
async findByEmail(email: string): Promise<User | null> {
|
||||
const result = await query<User>(
|
||||
'SELECT * FROM users WHERE email = $1',
|
||||
[email]
|
||||
);
|
||||
return result.rows[0] || null;
|
||||
},
|
||||
|
||||
async update(id: string, updates: Partial<User>): Promise<User | null> {
|
||||
const fields = Object.keys(updates);
|
||||
const values = Object.values(updates);
|
||||
const setClause = fields.map((field, i) => `${field} = $${i + 2}`).join(', ');
|
||||
|
||||
const result = await query<User>(
|
||||
`UPDATE users SET ${setClause} WHERE id = $1 RETURNING *`,
|
||||
[id, ...values]
|
||||
);
|
||||
return result.rows[0] || null;
|
||||
},
|
||||
|
||||
async updateLastLogin(id: string): Promise<void> {
|
||||
await query('UPDATE users SET last_login_at = NOW() WHERE id = $1', [id]);
|
||||
},
|
||||
},
|
||||
|
||||
monitors: {
|
||||
async create(data: Omit<Monitor, 'id' | 'createdAt' | 'updatedAt' | 'consecutiveErrors'>): Promise<Monitor> {
|
||||
const result = await query<Monitor>(
|
||||
`INSERT INTO monitors (
|
||||
user_id, url, name, frequency, status, element_selector,
|
||||
ignore_rules, keyword_rules
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *`,
|
||||
[
|
||||
data.userId,
|
||||
data.url,
|
||||
data.name,
|
||||
data.frequency,
|
||||
data.status,
|
||||
data.elementSelector || null,
|
||||
data.ignoreRules ? JSON.stringify(data.ignoreRules) : null,
|
||||
data.keywordRules ? JSON.stringify(data.keywordRules) : null,
|
||||
]
|
||||
);
|
||||
return result.rows[0];
|
||||
},
|
||||
|
||||
async findById(id: string): Promise<Monitor | null> {
|
||||
const result = await query<Monitor>(
|
||||
'SELECT * FROM monitors WHERE id = $1',
|
||||
[id]
|
||||
);
|
||||
return result.rows[0] || null;
|
||||
},
|
||||
|
||||
async findByUserId(userId: string): Promise<Monitor[]> {
|
||||
const result = await query<Monitor>(
|
||||
'SELECT * FROM monitors WHERE user_id = $1 ORDER BY created_at DESC',
|
||||
[userId]
|
||||
);
|
||||
return result.rows;
|
||||
},
|
||||
|
||||
async countByUserId(userId: string): Promise<number> {
|
||||
const result = await query<{ count: string }>(
|
||||
'SELECT COUNT(*) as count FROM monitors WHERE user_id = $1',
|
||||
[userId]
|
||||
);
|
||||
return parseInt(result.rows[0].count);
|
||||
},
|
||||
|
||||
async findActiveMonitors(): Promise<Monitor[]> {
|
||||
const result = await query<Monitor>(
|
||||
'SELECT * FROM monitors WHERE status = $1',
|
||||
['active']
|
||||
);
|
||||
return result.rows;
|
||||
},
|
||||
|
||||
async update(id: string, updates: Partial<Monitor>): Promise<Monitor | null> {
|
||||
const fields: string[] = [];
|
||||
const values: any[] = [];
|
||||
let paramCount = 2;
|
||||
|
||||
Object.entries(updates).forEach(([key, value]) => {
|
||||
if (value !== undefined) {
|
||||
const snakeKey = key.replace(/([A-Z])/g, '_$1').toLowerCase();
|
||||
if (key === 'ignoreRules' || key === 'keywordRules') {
|
||||
fields.push(`${snakeKey} = $${paramCount}`);
|
||||
values.push(JSON.stringify(value));
|
||||
} else {
|
||||
fields.push(`${snakeKey} = $${paramCount}`);
|
||||
values.push(value);
|
||||
}
|
||||
paramCount++;
|
||||
}
|
||||
});
|
||||
|
||||
if (fields.length === 0) return null;
|
||||
|
||||
const result = await query<Monitor>(
|
||||
`UPDATE monitors SET ${fields.join(', ')} WHERE id = $1 RETURNING *`,
|
||||
[id, ...values]
|
||||
);
|
||||
return result.rows[0] || null;
|
||||
},
|
||||
|
||||
async delete(id: string): Promise<boolean> {
|
||||
const result = await query('DELETE FROM monitors WHERE id = $1', [id]);
|
||||
return (result.rowCount ?? 0) > 0;
|
||||
},
|
||||
|
||||
async updateLastChecked(id: string, changed: boolean): Promise<void> {
|
||||
if (changed) {
|
||||
await query(
|
||||
'UPDATE monitors SET last_checked_at = NOW(), last_changed_at = NOW(), consecutive_errors = 0 WHERE id = $1',
|
||||
[id]
|
||||
);
|
||||
} else {
|
||||
await query(
|
||||
'UPDATE monitors SET last_checked_at = NOW(), consecutive_errors = 0 WHERE id = $1',
|
||||
[id]
|
||||
);
|
||||
}
|
||||
},
|
||||
|
||||
async incrementErrors(id: string): Promise<void> {
|
||||
await query(
|
||||
'UPDATE monitors SET last_checked_at = NOW(), consecutive_errors = consecutive_errors + 1 WHERE id = $1',
|
||||
[id]
|
||||
);
|
||||
},
|
||||
},
|
||||
|
||||
snapshots: {
|
||||
async create(data: Omit<Snapshot, 'id' | 'createdAt'>): Promise<Snapshot> {
|
||||
const result = await query<Snapshot>(
|
||||
`INSERT INTO snapshots (
|
||||
monitor_id, html_content, text_content, content_hash, screenshot_url,
|
||||
http_status, response_time, changed, change_percentage, error_message
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *`,
|
||||
[
|
||||
data.monitorId,
|
||||
data.htmlContent,
|
||||
data.textContent,
|
||||
data.contentHash,
|
||||
data.screenshotUrl || null,
|
||||
data.httpStatus,
|
||||
data.responseTime,
|
||||
data.changed,
|
||||
data.changePercentage || null,
|
||||
data.errorMessage || null,
|
||||
]
|
||||
);
|
||||
return result.rows[0];
|
||||
},
|
||||
|
||||
async findByMonitorId(monitorId: string, limit = 50): Promise<Snapshot[]> {
|
||||
const result = await query<Snapshot>(
|
||||
'SELECT * FROM snapshots WHERE monitor_id = $1 ORDER BY created_at DESC LIMIT $2',
|
||||
[monitorId, limit]
|
||||
);
|
||||
return result.rows;
|
||||
},
|
||||
|
||||
async findLatestByMonitorId(monitorId: string): Promise<Snapshot | null> {
|
||||
const result = await query<Snapshot>(
|
||||
'SELECT * FROM snapshots WHERE monitor_id = $1 ORDER BY created_at DESC LIMIT 1',
|
||||
[monitorId]
|
||||
);
|
||||
return result.rows[0] || null;
|
||||
},
|
||||
|
||||
async findById(id: string): Promise<Snapshot | null> {
|
||||
const result = await query<Snapshot>(
|
||||
'SELECT * FROM snapshots WHERE id = $1',
|
||||
[id]
|
||||
);
|
||||
return result.rows[0] || null;
|
||||
},
|
||||
|
||||
async deleteOldSnapshots(monitorId: string, keepCount: number): Promise<void> {
|
||||
await query(
|
||||
`DELETE FROM snapshots
|
||||
WHERE monitor_id = $1
|
||||
AND id NOT IN (
|
||||
SELECT id FROM snapshots
|
||||
WHERE monitor_id = $1
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $2
|
||||
)`,
|
||||
[monitorId, keepCount]
|
||||
);
|
||||
},
|
||||
},
|
||||
|
||||
alerts: {
|
||||
async create(data: Omit<Alert, 'id' | 'createdAt' | 'deliveredAt' | 'readAt'>): Promise<Alert> {
|
||||
const result = await query<Alert>(
|
||||
`INSERT INTO alerts (
|
||||
monitor_id, snapshot_id, user_id, type, title, summary, channels
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING *`,
|
||||
[
|
||||
data.monitorId,
|
||||
data.snapshotId,
|
||||
data.userId,
|
||||
data.type,
|
||||
data.title,
|
||||
data.summary || null,
|
||||
JSON.stringify(data.channels),
|
||||
]
|
||||
);
|
||||
return result.rows[0];
|
||||
},
|
||||
|
||||
async findByUserId(userId: string, limit = 50): Promise<Alert[]> {
|
||||
const result = await query<Alert>(
|
||||
'SELECT * FROM alerts WHERE user_id = $1 ORDER BY created_at DESC LIMIT $2',
|
||||
[userId, limit]
|
||||
);
|
||||
return result.rows;
|
||||
},
|
||||
|
||||
async markAsDelivered(id: string): Promise<void> {
|
||||
await query('UPDATE alerts SET delivered_at = NOW() WHERE id = $1', [id]);
|
||||
},
|
||||
|
||||
async markAsRead(id: string): Promise<void> {
|
||||
await query('UPDATE alerts SET read_at = NOW() WHERE id = $1', [id]);
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export default db;
|
||||
37
backend/src/db/migrate.ts
Normal file
37
backend/src/db/migrate.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { Pool } from 'pg';
|
||||
import dotenv from 'dotenv';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const pool = new Pool({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
});
|
||||
|
||||
async function runMigration() {
|
||||
console.log('🔄 Running database migrations...');
|
||||
|
||||
try {
|
||||
const client = await pool.connect();
|
||||
|
||||
try {
|
||||
const schemaPath = path.join(__dirname, 'schema.sql');
|
||||
const schemaSql = fs.readFileSync(schemaPath, 'utf-8');
|
||||
|
||||
console.log('📝 Executing schema...');
|
||||
await client.query(schemaSql);
|
||||
|
||||
console.log('✅ Migrations completed successfully!');
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('❌ Migration failed:', error);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
runMigration();
|
||||
93
backend/src/db/schema.sql
Normal file
93
backend/src/db/schema.sql
Normal file
@@ -0,0 +1,93 @@
|
||||
-- Database schema for Website Change Detection Monitor
|
||||
|
||||
-- Users table
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
email VARCHAR(255) UNIQUE NOT NULL,
|
||||
password_hash VARCHAR(255) NOT NULL,
|
||||
plan VARCHAR(20) DEFAULT 'free' CHECK (plan IN ('free', 'pro', 'business', 'enterprise')),
|
||||
stripe_customer_id VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
last_login_at TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_users_email ON users(email);
|
||||
CREATE INDEX idx_users_plan ON users(plan);
|
||||
|
||||
-- Monitors table
|
||||
CREATE TABLE IF NOT EXISTS monitors (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
url TEXT NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
frequency INTEGER NOT NULL DEFAULT 60 CHECK (frequency > 0),
|
||||
status VARCHAR(20) DEFAULT 'active' CHECK (status IN ('active', 'paused', 'error')),
|
||||
element_selector TEXT,
|
||||
ignore_rules JSONB,
|
||||
keyword_rules JSONB,
|
||||
last_checked_at TIMESTAMP,
|
||||
last_changed_at TIMESTAMP,
|
||||
consecutive_errors INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_monitors_user_id ON monitors(user_id);
|
||||
CREATE INDEX idx_monitors_status ON monitors(status);
|
||||
CREATE INDEX idx_monitors_last_checked_at ON monitors(last_checked_at);
|
||||
|
||||
-- Snapshots table
|
||||
CREATE TABLE IF NOT EXISTS snapshots (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
monitor_id UUID NOT NULL REFERENCES monitors(id) ON DELETE CASCADE,
|
||||
html_content TEXT,
|
||||
text_content TEXT,
|
||||
content_hash VARCHAR(64) NOT NULL,
|
||||
screenshot_url TEXT,
|
||||
http_status INTEGER NOT NULL,
|
||||
response_time INTEGER,
|
||||
changed BOOLEAN DEFAULT false,
|
||||
change_percentage DECIMAL(5,2),
|
||||
error_message TEXT,
|
||||
created_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_snapshots_monitor_id ON snapshots(monitor_id);
|
||||
CREATE INDEX idx_snapshots_created_at ON snapshots(created_at);
|
||||
CREATE INDEX idx_snapshots_changed ON snapshots(changed);
|
||||
|
||||
-- Alerts table
|
||||
CREATE TABLE IF NOT EXISTS alerts (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
monitor_id UUID NOT NULL REFERENCES monitors(id) ON DELETE CASCADE,
|
||||
snapshot_id UUID NOT NULL REFERENCES snapshots(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
type VARCHAR(20) NOT NULL CHECK (type IN ('change', 'error', 'keyword')),
|
||||
title VARCHAR(255) NOT NULL,
|
||||
summary TEXT,
|
||||
channels JSONB NOT NULL DEFAULT '["email"]',
|
||||
delivered_at TIMESTAMP,
|
||||
read_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_alerts_user_id ON alerts(user_id);
|
||||
CREATE INDEX idx_alerts_monitor_id ON alerts(monitor_id);
|
||||
CREATE INDEX idx_alerts_created_at ON alerts(created_at);
|
||||
CREATE INDEX idx_alerts_read_at ON alerts(read_at);
|
||||
|
||||
-- Update timestamps trigger
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
CREATE TRIGGER update_users_updated_at BEFORE UPDATE ON users
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
CREATE TRIGGER update_monitors_updated_at BEFORE UPDATE ON monitors
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
Reference in New Issue
Block a user