tknuth/website-monitor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Production ready

Browse Source
This commit is contained in:
Timo Knuth
2026-02-09 22:31:22 +01:00
parent fd6e7c44e1
commit 7814548e11
82 changed files with 3390 additions and 2026 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/.env.example
View File

@@ -19,9 +19,11 @@ SMTP_PORT=587
SMTP_USER=apikey
SMTP_PASS=your-sendgrid-api-key
# App
APP_URL=http://localhost:3000
API_URL=http://localhost:3002
# App
APP_URL=http://localhost:3000
API_URL=http://localhost:3002
LANDING_ONLY_MODE=false
ADMIN_PASSWORD=change-me-for-admin-waitlist
# Rate Limiting
MAX_MONITORS_FREE=5

14
backend/.eslintrc.json Normal file
View File

@@ -0,0 +1,14 @@
{
"root": true,
"env": {
"node": true,
"es2022": true
},
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": "latest",
"sourceType": "module"
},
"plugins": ["@typescript-eslint"],
"ignorePatterns": ["dist", "node_modules"]
}

0
backend/nul Normal file
View File

8
backend/src/config.ts
View File

@@ -47,10 +47,10 @@ export function getMaxMonitors(plan: UserPlan): number {
/**
* Check if a plan has a specific feature
*/
export function hasFeature(plan: UserPlan, feature: string): boolean {
const planConfig = PLAN_LIMITS[plan] || PLAN_LIMITS.free;
return planConfig.features.includes(feature as any);
}
export function hasFeature(plan: UserPlan, feature: string): boolean {
const planConfig = PLAN_LIMITS[plan] || PLAN_LIMITS.free;
return (planConfig.features as readonly string[]).includes(feature);
}
/**
* Webhook retry configuration

24
backend/src/db/index.ts
View File

@@ -53,11 +53,13 @@ export const query = async <T extends QueryResultRow = any>(
return result;
};
export const getClient = () => pool.connect();
// User queries
export const db = {
users: {
export const getClient = () => pool.connect();
// User queries
export const db = {
query,
users: {
async create(email: string, passwordHash: string): Promise<User> {
const result = await query(
'INSERT INTO users (email, password_hash) VALUES ($1, $2) RETURNING *',
@@ -455,12 +457,12 @@ export const db = {
return toCamelCase<any>(result.rows[0]);
},
async findLatestByMonitorId(monitorId: string, limit = 50): Promise<any[]> {
// Gets the latest check per keyword for this monitor
// Using DISTINCT ON is efficient in Postgres
const result = await query(
`SELECT DISTINCT ON (keyword) *
FROM monitor_rankings
async findLatestByMonitorId(monitorId: string): Promise<any[]> {
// Gets the latest check per keyword for this monitor
// Using DISTINCT ON is efficient in Postgres
const result = await query(
`SELECT DISTINCT ON (keyword) *
FROM monitor_rankings
WHERE monitor_id = $1
ORDER BY keyword, created_at DESC`,
[monitorId]

2
backend/src/db/migrations/tmpclaude-0f94-cwd
View File

@@ -1 +1 @@
/c/Users/timo/Documents/Websites/website-monitor/backend/src/db/migrations
/c/Users/timo/Documents/Websites/website-monitor/backend/src/db/migrations

57
backend/src/index.ts
View File

@@ -8,8 +8,28 @@ import { authMiddleware } from './middleware/auth';
import { apiLimiter, authLimiter } from './middleware/rateLimiter';
import { startWorker, shutdownScheduler, getSchedulerStats } from './services/scheduler';
const app = express();
const PORT = process.env.PORT || 3002;
const app = express();
const PORT = process.env.PORT || 3002;
const isLandingOnlyMode = process.env.LANDING_ONLY_MODE === 'true';
const isAllowedInLandingOnlyMode = (req: express.Request): boolean => {
if ((req.method === 'GET' || req.method === 'HEAD') && req.path === '/health') {
return true;
}
if (
(req.path === '/api/tools/meta-preview' || req.path === '/api/tools/meta-preview/') &&
(req.method === 'POST' || req.method === 'OPTIONS')
) {
return true;
}
if (req.path === '/api/waitlist' || req.path.startsWith('/api/waitlist/')) {
return true;
}
return false;
};
// Middleware
app.use(cors({
@@ -24,13 +44,27 @@ app.use(express.urlencoded({ extended: true }));
app.use('/api/', apiLimiter);
// Request logging
app.use((req, _res, next) => {
console.log(`${new Date().toISOString()} ${req.method} ${req.path}`);
next();
});
// Health check
app.get('/health', async (_req, res) => {
app.use((req, _res, next) => {
console.log(`${new Date().toISOString()} ${req.method} ${req.path}`);
next();
});
if (isLandingOnlyMode) {
app.use((req, res, next) => {
if (isAllowedInLandingOnlyMode(req)) {
return next();
}
return res.status(403).json({
error: 'landing_only_mode',
message: 'This endpoint is disabled while landing-only mode is active.',
path: req.path,
});
});
}
// Health check
app.get('/health', async (_req, res) => {
const schedulerStats = await getSchedulerStats();
res.json({
status: 'ok',
@@ -62,8 +96,9 @@ app.use((req, res) => {
});
// Error handler
app.use((err: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
console.error('Unhandled error:', err);
app.use((err: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
void _next;
console.error('Unhandled error:', err);
res.status(500).json({
error: 'server_error',

132
backend/src/routes/tools.ts
View File

@@ -2,78 +2,82 @@ import { Router } from 'express';
import axios from 'axios';
import * as cheerio from 'cheerio';
import { z } from 'zod';
const router = Router();
const previewSchema = z.object({
url: z.string().min(1)
});
router.post('/meta-preview', async (req, res) => {
try {
let { url } = previewSchema.parse(req.body);
// Add protocol if missing
if (!url.startsWith('http://') && !url.startsWith('https://')) {
url = `https://${url}`;
}
const response = await axios.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Cache-Control': 'max-age=0'
import { Agent as HttpAgent } from 'http';
import { Agent as HttpsAgent } from 'https';
const router = Router();
const previewSchema = z.object({
url: z.string().min(1)
});
router.post('/meta-preview', async (req, res) => {
try {
let { url } = previewSchema.parse(req.body);
// Add protocol if missing
if (!url.startsWith('http://') && !url.startsWith('https://')) {
url = `https://${url}`;
}
const response = await axios.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Cache-Control': 'max-age=0'
},
timeout: 30000,
httpAgent: new (require('http').Agent)({ family: 4, keepAlive: true }),
httpsAgent: new (require('https').Agent)({ family: 4, rejectUnauthorized: false, keepAlive: true }),
httpAgent: new HttpAgent({ family: 4, keepAlive: true }),
httpsAgent: new HttpsAgent({ family: 4, rejectUnauthorized: false, keepAlive: true }),
validateStatus: (status) => status < 500
});
const html = response.data;
const $ = cheerio.load(html);
const title = $('title').text() || $('meta[property="og:title"]').attr('content') || '';
const description = $('meta[name="description"]').attr('content') || $('meta[property="og:description"]').attr('content') || '';
// Attempt to find favicon
let favicon = '';
const linkIcon = $('link[rel="icon"], link[rel="shortcut icon"], link[rel="apple-touch-icon"]').attr('href');
if (linkIcon) {
if (linkIcon.startsWith('http')) {
favicon = linkIcon;
} else if (linkIcon.startsWith('//')) {
favicon = `https:${linkIcon}`;
} else {
const urlObj = new URL(url);
favicon = `${urlObj.protocol}//${urlObj.host}${linkIcon.startsWith('/') ? '' : '/'}${linkIcon}`;
}
} else {
const urlObj = new URL(url);
favicon = `${urlObj.protocol}//${urlObj.host}/favicon.ico`;
}
res.json({
title: title.trim(),
description: description.trim(),
favicon,
url: url
});
const html = response.data;
const $ = cheerio.load(html);
const title = $('title').text() || $('meta[property="og:title"]').attr('content') || '';
const description = $('meta[name="description"]').attr('content') || $('meta[property="og:description"]').attr('content') || '';
// Attempt to find favicon
let favicon = '';
const linkIcon = $('link[rel="icon"], link[rel="shortcut icon"], link[rel="apple-touch-icon"]').attr('href');
if (linkIcon) {
if (linkIcon.startsWith('http')) {
favicon = linkIcon;
} else if (linkIcon.startsWith('//')) {
favicon = `https:${linkIcon}`;
} else {
const urlObj = new URL(url);
favicon = `${urlObj.protocol}//${urlObj.host}${linkIcon.startsWith('/') ? '' : '/'}${linkIcon}`;
}
} else {
const urlObj = new URL(url);
favicon = `${urlObj.protocol}//${urlObj.host}/favicon.ico`;
}
res.json({
title: title.trim(),
description: description.trim(),
favicon,
url: url
});
} catch (error) {
console.error('Meta preview error:', error);
if (error instanceof z.ZodError) {
return res.status(400).json({ error: 'Invalid URL provided' });
res.status(400).json({ error: 'Invalid URL provided' });
return;
}
res.status(500).json({ error: 'Failed to fetch page metadata' });
return;
}
});
export const toolsRouter = router;
export const toolsRouter = router;

219
backend/src/routes/waitlist.ts
View File

@@ -1,131 +1,134 @@
import { Router } from 'express';
import { query } from '../db';
import { z } from 'zod';
const router = Router();
// Validation schema
const waitlistSchema = z.object({
email: z.string().email('Invalid email address'),
source: z.string().optional().default('landing_page'),
referrer: z.string().optional(),
});
// POST /api/waitlist - Add email to waitlist
router.post('/', async (req, res) => {
try {
const data = waitlistSchema.parse(req.body);
// Check if email already exists
const existing = await query(
'SELECT id FROM waitlist_leads WHERE email = $1',
[data.email.toLowerCase()]
);
import { Router } from 'express';
import { query } from '../db';
import { z } from 'zod';
const router = Router();
// Validation schema
const waitlistSchema = z.object({
email: z.string().email('Invalid email address'),
source: z.string().optional().default('landing_page'),
referrer: z.string().optional(),
});
// POST /api/waitlist - Add email to waitlist
router.post('/', async (req, res) => {
try {
const data = waitlistSchema.parse(req.body);
// Check if email already exists
const existing = await query(
'SELECT id FROM waitlist_leads WHERE email = $1',
[data.email.toLowerCase()]
);
if (existing.rows.length > 0) {
// Already on waitlist - return success anyway (don't reveal they're already signed up)
const countResult = await query('SELECT COUNT(*) FROM waitlist_leads');
const position = parseInt(countResult.rows[0].count, 10);
return res.json({
res.json({
success: true,
message: 'You\'re on the list!',
position,
alreadySignedUp: true,
});
return;
}
// Insert new lead
await query(
'INSERT INTO waitlist_leads (email, source, referrer) VALUES ($1, $2, $3)',
[data.email.toLowerCase(), data.source, data.referrer || null]
);
// Get current position (total count)
const countResult = await query('SELECT COUNT(*) FROM waitlist_leads');
const position = parseInt(countResult.rows[0].count, 10);
console.log(`✅ Waitlist signup: ${data.email} (Position #${position})`);
res.json({
success: true,
message: 'You\'re on the list!',
position,
});
// Insert new lead
await query(
'INSERT INTO waitlist_leads (email, source, referrer) VALUES ($1, $2, $3)',
[data.email.toLowerCase(), data.source, data.referrer || null]
);
// Get current position (total count)
const countResult = await query('SELECT COUNT(*) FROM waitlist_leads');
const position = parseInt(countResult.rows[0].count, 10);
console.log(`✅ Waitlist signup: ${data.email} (Position #${position})`);
res.json({
success: true,
message: 'You\'re on the list!',
position,
});
} catch (error) {
if (error instanceof z.ZodError) {
return res.status(400).json({
res.status(400).json({
success: false,
error: 'validation_error',
message: error.errors[0].message,
});
return;
}
console.error('Waitlist signup error:', error);
res.status(500).json({
success: false,
error: 'server_error',
message: 'Failed to join waitlist. Please try again.',
});
}
});
// GET /api/waitlist/count - Get current waitlist count (public)
router.get('/count', async (_req, res) => {
try {
const result = await query('SELECT COUNT(*) FROM waitlist_leads');
const count = parseInt(result.rows[0].count, 10);
// Add a base number to make it look more impressive at launch
const displayCount = count + 430; // Starting with "430+ waiting"
res.json({
success: true,
count: displayCount,
});
} catch (error) {
console.error('Waitlist count error:', error);
res.status(500).json({
success: false,
count: 430, // Fallback to base number
});
}
});
// GET /api/waitlist/admin - Get waitlist leads (Admin only)
router.get('/admin', async (req, res) => {
try {
const adminPassword = process.env.ADMIN_PASSWORD;
const providedPassword = req.headers['x-admin-password'];
console.error('Waitlist signup error:', error);
res.status(500).json({
success: false,
error: 'server_error',
message: 'Failed to join waitlist. Please try again.',
});
}
});
// GET /api/waitlist/count - Get current waitlist count (public)
router.get('/count', async (_req, res) => {
try {
const result = await query('SELECT COUNT(*) FROM waitlist_leads');
const count = parseInt(result.rows[0].count, 10);
// Add a base number to make it look more impressive at launch
const displayCount = count + 430; // Starting with "430+ waiting"
res.json({
success: true,
count: displayCount,
});
} catch (error) {
console.error('Waitlist count error:', error);
res.status(500).json({
success: false,
count: 430, // Fallback to base number
});
}
});
// GET /api/waitlist/admin - Get waitlist leads (Admin only)
router.get('/admin', async (req, res) => {
try {
const adminPassword = process.env.ADMIN_PASSWORD;
const providedPassword = req.headers['x-admin-password'];
if (!adminPassword || providedPassword !== adminPassword) {
return res.status(401).json({
res.status(401).json({
success: false,
message: 'Unauthorized',
});
return;
}
// Get stats
const countResult = await query('SELECT COUNT(*) FROM waitlist_leads');
const total = parseInt(countResult.rows[0].count, 10);
// Get leads
const leadsResult = await query(
'SELECT * FROM waitlist_leads ORDER BY created_at DESC LIMIT 100'
);
res.json({
success: true,
total,
leads: leadsResult.rows,
});
} catch (error) {
console.error('Waitlist admin error:', error);
res.status(500).json({
success: false,
message: 'Server error',
});
}
});
export default router;
// Get stats
const countResult = await query('SELECT COUNT(*) FROM waitlist_leads');
const total = parseInt(countResult.rows[0].count, 10);
// Get leads
const leadsResult = await query(
'SELECT * FROM waitlist_leads ORDER BY created_at DESC LIMIT 100'
);
res.json({
success: true,
total,
leads: leadsResult.rows,
});
} catch (error) {
console.error('Waitlist admin error:', error);
res.status(500).json({
success: false,
message: 'Server error',
});
}
});
export default router;

29
backend/src/services/digest.ts
View File

@@ -1,16 +1,17 @@
import { Queue, Worker } from 'bullmq';
import Redis from 'ioredis';
import nodemailer from 'nodemailer';
import db from '../db';
import { ConnectionOptions, Queue, Worker } from 'bullmq';
import Redis from 'ioredis';
import nodemailer from 'nodemailer';
import db from '../db';
// Redis connection (reuse from main scheduler)
const redisConnection = new Redis(process.env.REDIS_URL || 'redis://localhost:6380', {
maxRetriesPerRequest: null,
});
const redisConnection = new Redis(process.env.REDIS_URL || 'redis://localhost:6380', {
maxRetriesPerRequest: null,
});
const queueConnection = redisConnection as unknown as ConnectionOptions;
// Digest queue
export const digestQueue = new Queue('change-digests', {
connection: redisConnection,
export const digestQueue = new Queue('change-digests', {
connection: queueConnection,
defaultJobOptions: {
removeOnComplete: 10,
removeOnFail: 10,
@@ -263,11 +264,11 @@ export function startDigestWorker(): Worker {
const { interval } = job.data;
await processDigests(interval);
},
{
connection: redisConnection,
concurrency: 1,
}
);
{
connection: queueConnection,
concurrency: 1,
}
);
worker.on('completed', (job) => {
console.log(`[Digest] Job ${job.id} completed`);

25
backend/src/services/monitor.ts
View File

@@ -1,5 +1,5 @@
import db from '../db';
import { Monitor, Snapshot } from '../types';
import { Snapshot } from '../types';
import { fetchPage } from './fetcher';
import {
applyIgnoreRules,
@@ -11,6 +11,20 @@ import { calculateChangeImportance } from './importance';
import { sendChangeAlert, sendErrorAlert, sendKeywordAlert } from './alerter';
import { generateSimpleSummary, generateAISummary } from './summarizer';
import { processSeoChecks } from './seo';
import Redis from 'ioredis';
const redis = new Redis(process.env.REDIS_URL || 'redis://localhost:6380', {
maxRetriesPerRequest: null,
});
async function acquireLock(key: string, ttlMs = 120000): Promise<boolean> {
const result = await redis.set(key, '1', 'PX', ttlMs, 'NX');
return result === 'OK';
}
async function releaseLock(key: string): Promise<void> {
await redis.del(key);
}
export interface CheckResult {
snapshot: Snapshot;
@@ -24,6 +38,13 @@ export async function checkMonitor(
): Promise<{ snapshot?: Snapshot; alertSent: boolean } | void> {
console.log(`[Monitor] Starting check: ${monitorId} | Type: ${checkType} | ForceSEO: ${forceSeo}`);
const lockKey = `lock:monitor-check:${monitorId}`;
const acquired = await acquireLock(lockKey);
if (!acquired) {
console.log(`[Monitor] Skipping ${monitorId} - another check is already running`);
return;
}
try {
const monitor = await db.monitors.findById(monitorId);
@@ -247,6 +268,8 @@ export async function checkMonitor(
} catch (error) {
console.error(`[Monitor] Error checking monitor ${monitorId}:`, error);
await db.monitors.incrementErrors(monitorId);
} finally {
await releaseLock(lockKey);
}
}

31
backend/src/services/scheduler.ts
View File

@@ -1,16 +1,17 @@
import { Queue, Worker, QueueEvents } from 'bullmq';
import Redis from 'ioredis';
import { checkMonitor } from './monitor';
import { Monitor } from '../db';
import { ConnectionOptions, Queue, Worker, QueueEvents } from 'bullmq';
import Redis from 'ioredis';
import { checkMonitor } from './monitor';
import { Monitor } from '../types';
// Redis connection
const redisConnection = new Redis(process.env.REDIS_URL || 'redis://localhost:6380', {
maxRetriesPerRequest: null,
});
const redisConnection = new Redis(process.env.REDIS_URL || 'redis://localhost:6380', {
maxRetriesPerRequest: null,
});
const queueConnection = redisConnection as unknown as ConnectionOptions;
// Monitor check queue
export const monitorQueue = new Queue('monitor-checks', {
connection: redisConnection,
export const monitorQueue = new Queue('monitor-checks', {
connection: queueConnection,
defaultJobOptions: {
removeOnComplete: 100, // Keep last 100 completed jobs
removeOnFail: 50, // Keep last 50 failed jobs
@@ -23,7 +24,7 @@ export const monitorQueue = new Queue('monitor-checks', {
});
// Queue events for monitoring
const queueEvents = new QueueEvents('monitor-checks', { connection: redisConnection });
const queueEvents = new QueueEvents('monitor-checks', { connection: queueConnection });
queueEvents.on('completed', ({ jobId }) => {
console.log(`[Scheduler] Job ${jobId} completed`);
@@ -130,11 +131,11 @@ export function startWorker(): Worker {
throw error; // Re-throw to mark job as failed
}
},
{
connection: redisConnection,
concurrency: 5, // Process up to 5 monitors concurrently
}
);
{
connection: queueConnection,
concurrency: 5, // Process up to 5 monitors concurrently
}
);
worker.on('completed', (job) => {
console.log(`[Worker] Job ${job.id} completed`);

6
backend/src/types/serpapi.d.ts vendored Normal file
View File

@@ -0,0 +1,6 @@
declare module 'serpapi' {
export function getJson(
params: Record<string, unknown>,
callback: (json: any) => void
): void;
}

2
backend/tmpclaude-005a-cwd
View File

@@ -1 +1 @@
/c/Users/timo/Documents/Websites/website-monitor/backend
/c/Users/timo/Documents/Websites/website-monitor/backend

2
backend/tmpclaude-08a7-cwd
View File

@@ -1 +1 @@
/c/Users/timo/Documents/Websites/website-monitor/backend
/c/Users/timo/Documents/Websites/website-monitor/backend

2
backend/tmpclaude-2dec-cwd
View File

@@ -1 +1 @@
/c/Users/timo/Documents/Websites/website-monitor/backend
/c/Users/timo/Documents/Websites/website-monitor/backend

2
backend/tmpclaude-3919-cwd
View File

@@ -1 +1 @@
/c/Users/timo/Documents/Websites/website-monitor/backend
/c/Users/timo/Documents/Websites/website-monitor/backend

2
backend/tmpclaude-7d8e-cwd
View File

@@ -1 +1 @@
/c/Users/timo/Documents/Websites/website-monitor/backend
/c/Users/timo/Documents/Websites/website-monitor/backend

2
backend/tmpclaude-b7cf-cwd
View File

@@ -1 +1 @@
/c/Users/timo/Documents/Websites/website-monitor/backend
/c/Users/timo/Documents/Websites/website-monitor/backend

2
backend/tmpclaude-cd46-cwd
View File

@@ -1 +1 @@
/c/Users/timo/Documents/Websites/website-monitor/backend
/c/Users/timo/Documents/Websites/website-monitor/backend

2
backend/tmpclaude-d344-cwd
View File

@@ -1 +1 @@
/c/Users/timo/Documents/Websites/website-monitor/backend
/c/Users/timo/Documents/Websites/website-monitor/backend

2
backend/tmpclaude-e961-cwd
View File

@@ -1 +1 @@
/c/Users/timo/Documents/Websites/website-monitor/backend
/c/Users/timo/Documents/Websites/website-monitor/backend