tknuth/website-monitor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

gitea +

Browse Source
This commit is contained in:
knuthtimo-lab
2026-01-21 08:21:19 +01:00
parent 4733e1a1cc
commit fd6e7c44e1
46 changed files with 3165 additions and 456 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
backend/src/db/fix_schema.ts Normal file
View File

@@ -0,0 +1,50 @@
import { Pool } from 'pg';
import dotenv from 'dotenv';
import path from 'path';
// Load env vars from .env file in backend root
dotenv.config({ path: path.join(__dirname, '../../.env') });
const pool = new Pool({
connectionString: process.env.DATABASE_URL,
});
async function fixSchema() {
console.log('🔧 Fixing schema...');
const client = await pool.connect();
try {
// Add seo_keywords column
console.log('Adding seo_keywords column...');
await client.query(`
ALTER TABLE monitors ADD COLUMN IF NOT EXISTS seo_keywords JSONB;
`);
// Create monitor_rankings table
console.log('Creating monitor_rankings table...');
await client.query(`
CREATE TABLE IF NOT EXISTS monitor_rankings (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
monitor_id UUID NOT NULL REFERENCES monitors(id) ON DELETE CASCADE,
keyword VARCHAR(255) NOT NULL,
rank INTEGER,
url_found TEXT,
created_at TIMESTAMP DEFAULT NOW()
);
`);
// Create indexes for monitor_rankings
await client.query(`CREATE INDEX IF NOT EXISTS idx_monitor_rankings_monitor_id ON monitor_rankings(monitor_id);`);
await client.query(`CREATE INDEX IF NOT EXISTS idx_monitor_rankings_keyword ON monitor_rankings(keyword);`);
await client.query(`CREATE INDEX IF NOT EXISTS idx_monitor_rankings_created_at ON monitor_rankings(created_at);`);
console.log('✅ Schema fixed successfully!');
} catch (err) {
console.error('❌ Schema fix failed:', err);
} finally {
client.release();
await pool.end();
}
}
fixSchema();

54
backend/src/db/index.ts
View File

@@ -13,7 +13,7 @@ function toCamelCase<T>(obj: any): T {
let value = obj[key];
// Parse JSON fields that are stored as strings in the database
if ((key === 'ignore_rules' || key === 'keyword_rules') && typeof value === 'string') {
if ((key === 'ignore_rules' || key === 'keyword_rules' || key === 'seo_keywords') && typeof value === 'string') {
try {
value = JSON.parse(value);
} catch (e) {
@@ -164,8 +164,8 @@ export const db = {
const result = await query(
`INSERT INTO monitors (
user_id, url, name, frequency, status, element_selector,
ignore_rules, keyword_rules
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *`,
ignore_rules, keyword_rules, seo_keywords, seo_interval
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *`,
[
data.userId,
data.url,
@@ -175,6 +175,8 @@ export const db = {
data.elementSelector || null,
data.ignoreRules ? JSON.stringify(data.ignoreRules) : null,
data.keywordRules ? JSON.stringify(data.keywordRules) : null,
data.seoKeywords ? JSON.stringify(data.seoKeywords) : null,
data.seoInterval || 'off',
]
);
return toCamelCase<Monitor>(result.rows[0]);
@@ -220,9 +222,12 @@ export const db = {
Object.entries(updates).forEach(([key, value]) => {
if (value !== undefined) {
const snakeKey = key.replace(/([A-Z])/g, '_$1').toLowerCase();
if (key === 'ignoreRules' || key === 'keywordRules') {
if (key === 'ignoreRules' || key === 'keywordRules' || key === 'seoKeywords') {
fields.push(`${snakeKey} = $${paramCount}`);
values.push(JSON.stringify(value));
} else if (key === 'seoInterval') {
fields.push(`seo_interval = $${paramCount}`);
values.push(value);
} else {
fields.push(`${snakeKey} = $${paramCount}`);
values.push(value);
@@ -433,6 +438,47 @@ export const db = {
return result.rows.map(row => toCamelCase<any>(row));
},
},
rankings: {
async create(data: {
monitorId: string;
keyword: string;
rank: number | null;
urlFound: string | null;
}): Promise<any> {
const result = await query(
`INSERT INTO monitor_rankings (
monitor_id, keyword, rank, url_found
) VALUES ($1, $2, $3, $4) RETURNING *`,
[data.monitorId, data.keyword, data.rank, data.urlFound]
);
return toCamelCase<any>(result.rows[0]);
},
async findLatestByMonitorId(monitorId: string, limit = 50): Promise<any[]> {
// Gets the latest check per keyword for this monitor
// Using DISTINCT ON is efficient in Postgres
const result = await query(
`SELECT DISTINCT ON (keyword) *
FROM monitor_rankings
WHERE monitor_id = $1
ORDER BY keyword, created_at DESC`,
[monitorId]
);
return result.rows.map(row => toCamelCase<any>(row));
},
async findHistoryByMonitorId(monitorId: string, limit = 100): Promise<any[]> {
const result = await query(
`SELECT * FROM monitor_rankings
WHERE monitor_id = $1
ORDER BY created_at DESC
LIMIT $2`,
[monitorId, limit]
);
return result.rows.map(row => toCamelCase<any>(row));
}
},
};
export default db;

2
backend/src/db/migrations/006_add_seo_scheduling.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE monitors ADD COLUMN IF NOT EXISTS seo_interval VARCHAR(20) DEFAULT 'off';
ALTER TABLE monitors ADD COLUMN IF NOT EXISTS last_seo_check_at TIMESTAMP;

34
backend/src/db/run_migration_006.ts Normal file
View File

@@ -0,0 +1,34 @@
import { Pool } from 'pg';
import dotenv from 'dotenv';
import fs from 'fs';
import path from 'path';
dotenv.config();
const pool = new Pool({
connectionString: process.env.DATABASE_URL,
});
async function runManualMigration() {
console.log('🔄 Running manual migration 006...');
try {
const client = await pool.connect();
try {
const sqlPath = path.join(__dirname, 'migrations/006_add_seo_scheduling.sql');
const sql = fs.readFileSync(sqlPath, 'utf-8');
console.log('📝 Executing SQL:', sql);
await client.query(sql);
console.log('✅ Migration 006 applied successfully!');
} finally {
client.release();
}
} catch (error) {
console.error('❌ Migration 006 failed:', error);
process.exit(1);
} finally {
await pool.end();
}
}
runManualMigration();

42
backend/src/db/schema.sql
View File

@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS users (
updated_at TIMESTAMP DEFAULT NOW()
);
CREATE INDEX idx_users_email ON users(email);
CREATE INDEX idx_users_plan ON users(plan);
CREATE INDEX IF NOT EXISTS idx_users_email ON users(email);
CREATE INDEX IF NOT EXISTS idx_users_plan ON users(plan);
CREATE INDEX IF NOT EXISTS idx_users_webhook_enabled ON users(webhook_enabled) WHERE webhook_enabled = true;
CREATE INDEX IF NOT EXISTS idx_users_slack_enabled ON users(slack_enabled) WHERE slack_enabled = true;
@@ -42,9 +42,9 @@ CREATE TABLE IF NOT EXISTS monitors (
updated_at TIMESTAMP DEFAULT NOW()
);
CREATE INDEX idx_monitors_user_id ON monitors(user_id);
CREATE INDEX idx_monitors_status ON monitors(status);
CREATE INDEX idx_monitors_last_checked_at ON monitors(last_checked_at);
CREATE INDEX IF NOT EXISTS idx_monitors_user_id ON monitors(user_id);
CREATE INDEX IF NOT EXISTS idx_monitors_status ON monitors(status);
CREATE INDEX IF NOT EXISTS idx_monitors_last_checked_at ON monitors(last_checked_at);
-- Snapshots table
CREATE TABLE IF NOT EXISTS snapshots (
@@ -62,9 +62,9 @@ CREATE TABLE IF NOT EXISTS snapshots (
created_at TIMESTAMP DEFAULT NOW()
);
CREATE INDEX idx_snapshots_monitor_id ON snapshots(monitor_id);
CREATE INDEX idx_snapshots_created_at ON snapshots(created_at);
CREATE INDEX idx_snapshots_changed ON snapshots(changed);
CREATE INDEX IF NOT EXISTS idx_snapshots_monitor_id ON snapshots(monitor_id);
CREATE INDEX IF NOT EXISTS idx_snapshots_created_at ON snapshots(created_at);
CREATE INDEX IF NOT EXISTS idx_snapshots_changed ON snapshots(changed);
-- Alerts table
CREATE TABLE IF NOT EXISTS alerts (
@@ -81,10 +81,10 @@ CREATE TABLE IF NOT EXISTS alerts (
created_at TIMESTAMP DEFAULT NOW()
);
CREATE INDEX idx_alerts_user_id ON alerts(user_id);
CREATE INDEX idx_alerts_monitor_id ON alerts(monitor_id);
CREATE INDEX idx_alerts_created_at ON alerts(created_at);
CREATE INDEX idx_alerts_read_at ON alerts(read_at);
CREATE INDEX IF NOT EXISTS idx_alerts_user_id ON alerts(user_id);
CREATE INDEX IF NOT EXISTS idx_alerts_monitor_id ON alerts(monitor_id);
CREATE INDEX IF NOT EXISTS idx_alerts_created_at ON alerts(created_at);
CREATE INDEX IF NOT EXISTS idx_alerts_read_at ON alerts(read_at);
-- Update timestamps trigger
CREATE OR REPLACE FUNCTION update_updated_at_column()
@@ -132,4 +132,20 @@ CREATE TABLE IF NOT EXISTS waitlist_leads (
);
CREATE INDEX IF NOT EXISTS idx_waitlist_leads_email ON waitlist_leads(email);
CREATE INDEX IF NOT EXISTS idx_waitlist_leads_created_at ON waitlist_leads(created_at);
-- SEO Rankings table
CREATE TABLE IF NOT EXISTS monitor_rankings (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
monitor_id UUID NOT NULL REFERENCES monitors(id) ON DELETE CASCADE,
keyword VARCHAR(255) NOT NULL,
rank INTEGER, -- Null if not found in top 100
url_found TEXT, -- The specific URL that ranked
created_at TIMESTAMP DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_monitor_rankings_monitor_id ON monitor_rankings(monitor_id);
CREATE INDEX IF NOT EXISTS idx_monitor_rankings_keyword ON monitor_rankings(keyword);
CREATE INDEX IF NOT EXISTS idx_monitor_rankings_created_at ON monitor_rankings(created_at);
-- Add seo_keywords to monitors if it doesn't exist
ALTER TABLE monitors ADD COLUMN IF NOT EXISTS seo_keywords JSONB;

52
backend/src/routes/monitors.ts
View File

@@ -31,6 +31,8 @@ const createMonitorSchema = z.object({
})
)
.optional(),
seoKeywords: z.array(z.string()).optional(),
seoInterval: z.enum(['daily', '2d', 'weekly', 'monthly', 'off']).optional(),
});
const updateMonitorSchema = z.object({
@@ -56,6 +58,8 @@ const updateMonitorSchema = z.object({
})
)
.optional(),
seoKeywords: z.array(z.string()).optional(),
seoInterval: z.enum(['daily', '2d', 'weekly', 'monthly', 'off']).optional(),
});
// Get plan limits
@@ -92,17 +96,21 @@ router.get('/', async (req: AuthRequest, res: Response): Promise<void> => {
const monitors = await db.monitors.findByUserId(req.user.userId);
// Attach recent snapshots to each monitor for sparklines
const monitorsWithSnapshots = await Promise.all(monitors.map(async (monitor) => {
// Attach recent snapshots and latest rankings to each monitor
const monitorsWithData = await Promise.all(monitors.map(async (monitor) => {
// Get last 20 snapshots for sparkline
const recentSnapshots = await db.snapshots.findByMonitorId(monitor.id, 20);
// Get latest rankings
const latestRankings = await db.rankings.findLatestByMonitorId(monitor.id);
return {
...monitor,
recentSnapshots
recentSnapshots,
latestRankings
};
}));
res.json({ monitors: monitorsWithSnapshots });
res.json({ monitors: monitorsWithData });
} catch (error) {
console.error('List monitors error:', error);
res.status(500).json({ error: 'server_error', message: 'Failed to list monitors' });
@@ -180,6 +188,7 @@ router.post('/', async (req: AuthRequest, res: Response): Promise<void> => {
elementSelector: input.elementSelector,
ignoreRules: input.ignoreRules,
keywordRules: input.keywordRules,
seoKeywords: input.seoKeywords,
});
// Schedule recurring checks
@@ -354,7 +363,10 @@ router.post('/:id/check', checkLimiter, async (req: AuthRequest, res: Response):
// Await the check so user gets immediate feedback
try {
await checkMonitor(monitor.id);
const { type = 'all' } = req.body;
const checkType = ['all', 'content', 'seo'].includes(type) ? type : 'all';
await checkMonitor(monitor.id, checkType === 'seo' || checkType === 'all', checkType as any);
// Get the latest snapshot to return to the user
const latestSnapshot = await db.snapshots.findLatestByMonitorId(monitor.id);
@@ -453,6 +465,36 @@ router.get(
}
);
// Get monitor ranking history
router.get('/:id/rankings', async (req: AuthRequest, res: Response): Promise<void> => {
try {
if (!req.user) {
res.status(401).json({ error: 'unauthorized', message: 'Not authenticated' });
return;
}
const monitor = await db.monitors.findById(req.params.id);
if (!monitor) {
res.status(404).json({ error: 'not_found', message: 'Monitor not found' });
return;
}
if (monitor.userId !== req.user.userId) {
res.status(403).json({ error: 'forbidden', message: 'Access denied' });
return;
}
const history = await db.rankings.findHistoryByMonitorId(req.params.id, 100);
const latest = await db.rankings.findLatestByMonitorId(req.params.id);
res.json({ history, latest });
} catch (error) {
console.error('Get rankings error:', error);
res.status(500).json({ error: 'server_error', message: 'Failed to get rankings' });
}
});
// Export monitor audit trail (JSON or CSV)
router.get('/:id/export', async (req: AuthRequest, res: Response): Promise<void> => {
try {

128
backend/src/routes/test.ts
View File

@@ -5,40 +5,120 @@ const router = Router();
router.get('/dynamic', (_req, res) => {
const now = new Date();
const timeString = now.toLocaleTimeString();
const randomValue = Math.floor(Math.random() * 1000);
// Toggle status based on seconds (even/odd) to guarantee change
const isNormal = now.getSeconds() % 2 === 0;
const statusMessage = isNormal
? "System Status: NORMAL - Everything is running smoothly."
: "System Status: WARNING - High load detected on server node!";
const statusColor = isNormal ? "green" : "red";
const minute = now.getMinutes();
const second = now.getSeconds();
const tenSecondBlock = Math.floor(second / 10);
// Dynamic Pricing Logic - changes every 10 seconds
const basicPrice = 9 + (tenSecondBlock % 5);
const proPrice = 29 + (tenSecondBlock % 10);
const enterprisePrice = 99 + (tenSecondBlock % 20);
// Dynamic Features
const features = [
"Unlimited Projects",
`Up to ${10 + (tenSecondBlock % 5)} team members`,
"Advanced Analytics",
tenSecondBlock % 2 === 0 ? "Priority Support" : "24/7 Live Chat Support",
second % 2 === 0 ? "Real-time Monitoring" : "Custom Webhooks"
];
// Dynamic Blog Posts
const blogPosts = [
{
id: 1,
title: tenSecondBlock % 3 === 0 ? "Scaling your SaaS in 2026" : "Growth Strategies for Modern Apps",
author: "Jane Doe",
date: "Jan 15, 2026"
},
{
id: 2,
title: "UI/UX Best Practices",
author: second % 20 > 10 ? "John Smith" : "Alex Rivera",
date: `Jan ${10 + (tenSecondBlock % 10)}, 2026`
}
];
const html = `
<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Dynamic Test Page</title>
<meta charset="UTF-8">
<title>CloudScale SaaS - Infrastructure for Growth</title>
<style>
body { font-family: sans-serif; padding: 20px; }
.time { font-size: 2em; color: #0066cc; }
.status { font-size: 1.5em; color: ${statusColor}; font-weight: bold; padding: 20px; border: 2px solid ${statusColor}; margin: 20px 0; }
body { font-family: 'Inter', -apple-system, sans-serif; line-height: 1.6; color: #333; max-width: 1000px; margin: 0 auto; padding: 40px; }
header { text-align: center; margin-bottom: 50px; }
.grid { display: grid; grid-template-columns: repeat(3, 1fr); gap: 20px; margin: 40px 0; }
.card { border: 1px solid #eee; border-radius: 12px; padding: 25px; text-align: center; transition: transform 0.2s; box-shadow: 0 4px 6px -1px rgba(0,0,0,0.1); }
.card.featured { border: 2px solid #3b82f6; background-color: #eff6ff; transform: scale(1.05); }
.price { font-size: 3rem; font-weight: 800; margin: 20px 0; color: #111; }
.price span { font-size: 1rem; color: #666; font-weight: normal; }
ul { list-style: none; padding: 0; margin: 30px 0; text-align: left; }
li { margin-bottom: 12px; display: flex; align-items: center; }
li::before { content: "✓"; color: #10b981; font-weight: bold; margin-right: 10px; }
.badge { background: #dcfce7; color: #166534; padding: 4px 12px; border-radius: 99px; font-size: 0.8rem; font-weight: 600; }
.blog-section { margin-top: 60px; border-top: 1px solid #eee; padding-top: 40px; }
.blog-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 30px; }
.post { background: #fafafa; padding: 20px; border-radius: 8px; }
.meta { color: #888; font-size: 0.9rem; margin-bottom: 10px; }
.footer { margin-top: 80px; text-align: center; color: #999; font-size: 0.9rem; }
</style>
</head>
<body>
<h1>Website Monitor Test</h1>
<div class="status">${statusMessage}</div>
<div class="content">
<p>Current Time: <span class="time">${timeString}</span></p>
<p>Random Value: <span class="random">${randomValue}</span></p>
<p>This page content flips every second to simulate a real website change.</p>
<div style="background: #f0f9ff; padding: 15px; margin-top: 20px; border-left: 4px solid #0066cc;">
<h3>New Feature Update</h3>
<p>We have deployed a new importance scoring update!</p>
<header>
<h1>CloudScale <span class="badge">v2.4 Updated</span></h1>
<p>Reliable infrastructure that scales with your business needs.</p>
<p style="color: #6366f1; font-weight: 500;">Current Update: ${now.toLocaleTimeString()}</p>
</header>
<div class="pricing-section">
<h2 style="text-align: center">Simple, Transparent Pricing</h2>
<div class="grid">
<div class="card">
<h3>Basic</h3>
<div class="price">$${basicPrice}<span>/mo</span></div>
<ul>
<li>5 Projects</li>
<li>Basic Analytics</li>
<li>Community Support</li>
</ul>
</div>
<div class="card featured">
<h3>Pro</h3>
<div class="price">$${proPrice}<span>/mo</span></div>
<ul>
${features.map(f => `<li>${f}</li>`).join('')}
</ul>
</div>
<div class="card">
<h3>Enterprise</h3>
<div class="price">$${enterprisePrice}<span>/mo</span></div>
<ul>
<li>Everything in Pro</li>
<li>Custom SLAs</li>
<li>Dedicated Account Manager</li>
<li>White-label Branding</li>
</ul>
</div>
</div>
</div>
<section class="blog-section">
<h2>From Our Blog</h2>
<div class="blog-grid">
${blogPosts.map(p => `
<div class="post">
<div class="meta">By ${p.author} • ${p.date}</div>
<h3>${p.title}</h3>
<p>Discover how the latest trends in technology are shaping the future of digital products...</p>
</div>
`).join('')}
</div>
</section>
<div class="footer">
<p>&copy; 2026 CloudScale Platform. Serving ${1000 + (minute * 10)} active customers worldwide.</p>
</div>
</body>
</html>
`;

17
backend/src/routes/tools.ts
View File

@@ -20,10 +20,21 @@ router.post('/meta-preview', async (req, res) => {
const response = await axios.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; WebsiteMonitorBot/1.0; +https://websitemonitor.com)'
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Cache-Control': 'max-age=0'
},
timeout: 5000,
validateStatus: (status) => status < 500 // Resolve even if 404/403 to avoid crashing flow immediately
timeout: 30000,
httpAgent: new (require('http').Agent)({ family: 4, keepAlive: true }),
httpsAgent: new (require('https').Agent)({ family: 4, rejectUnauthorized: false, keepAlive: true }),
validateStatus: (status) => status < 500
});
const html = response.data;

54
backend/src/routes/waitlist.ts
View File

@@ -1,5 +1,5 @@
import { Router } from 'express';
import { pool } from '../db';
import { query } from '../db';
import { z } from 'zod';
const router = Router();
@@ -17,16 +17,16 @@ router.post('/', async (req, res) => {
const data = waitlistSchema.parse(req.body);
// Check if email already exists
const existing = await pool.query(
const existing = await query(
'SELECT id FROM waitlist_leads WHERE email = $1',
[data.email.toLowerCase()]
);
if (existing.rows.length > 0) {
// Already on waitlist - return success anyway (don't reveal they're already signed up)
const countResult = await pool.query('SELECT COUNT(*) FROM waitlist_leads');
const countResult = await query('SELECT COUNT(*) FROM waitlist_leads');
const position = parseInt(countResult.rows[0].count, 10);
return res.json({
success: true,
message: 'You\'re on the list!',
@@ -36,13 +36,13 @@ router.post('/', async (req, res) => {
}
// Insert new lead
await pool.query(
await query(
'INSERT INTO waitlist_leads (email, source, referrer) VALUES ($1, $2, $3)',
[data.email.toLowerCase(), data.source, data.referrer || null]
);
// Get current position (total count)
const countResult = await pool.query('SELECT COUNT(*) FROM waitlist_leads');
const countResult = await query('SELECT COUNT(*) FROM waitlist_leads');
const position = parseInt(countResult.rows[0].count, 10);
console.log(`✅ Waitlist signup: ${data.email} (Position #${position})`);
@@ -73,12 +73,12 @@ router.post('/', async (req, res) => {
// GET /api/waitlist/count - Get current waitlist count (public)
router.get('/count', async (_req, res) => {
try {
const result = await pool.query('SELECT COUNT(*) FROM waitlist_leads');
const result = await query('SELECT COUNT(*) FROM waitlist_leads');
const count = parseInt(result.rows[0].count, 10);
// Add a base number to make it look more impressive at launch
const displayCount = count + 430; // Starting with "430+ waiting"
res.json({
success: true,
count: displayCount,
@@ -92,4 +92,40 @@ router.get('/count', async (_req, res) => {
}
});
// GET /api/waitlist/admin - Get waitlist leads (Admin only)
router.get('/admin', async (req, res) => {
try {
const adminPassword = process.env.ADMIN_PASSWORD;
const providedPassword = req.headers['x-admin-password'];
if (!adminPassword || providedPassword !== adminPassword) {
return res.status(401).json({
success: false,
message: 'Unauthorized',
});
}
// Get stats
const countResult = await query('SELECT COUNT(*) FROM waitlist_leads');
const total = parseInt(countResult.rows[0].count, 10);
// Get leads
const leadsResult = await query(
'SELECT * FROM waitlist_leads ORDER BY created_at DESC LIMIT 100'
);
res.json({
success: true,
total,
leads: leadsResult.rows,
});
} catch (error) {
console.error('Waitlist admin error:', error);
res.status(500).json({
success: false,
message: 'Server error',
});
}
});
export default router;

306
backend/src/services/monitor.ts
View File

@@ -10,14 +10,19 @@ import {
import { calculateChangeImportance } from './importance';
import { sendChangeAlert, sendErrorAlert, sendKeywordAlert } from './alerter';
import { generateSimpleSummary, generateAISummary } from './summarizer';
import { processSeoChecks } from './seo';
export interface CheckResult {
snapshot: Snapshot;
alertSent: boolean;
}
export async function checkMonitor(monitorId: string): Promise<CheckResult | void> {
console.log(`[Monitor] Checking monitor ${monitorId}`);
export async function checkMonitor(
monitorId: string,
forceSeo = false,
checkType: 'all' | 'content' | 'seo' = 'all'
): Promise<{ snapshot?: Snapshot; alertSent: boolean } | void> {
console.log(`[Monitor] Starting check: ${monitorId} | Type: ${checkType} | ForceSEO: ${forceSeo}`);
try {
const monitor = await db.monitors.findById(monitorId);
@@ -28,184 +33,217 @@ export async function checkMonitor(monitorId: string): Promise<CheckResult | voi
}
if (monitor.status !== 'active' && monitor.status !== 'error') {
console.log(`[Monitor] Monitor ${monitorId} is not active or error, skipping (status: ${monitor.status})`);
console.log(`[Monitor] Monitor ${monitorId} skipping (status: ${monitor.status})`);
return;
}
// Fetch page with retries
let fetchResult = await fetchPage(monitor.url, monitor.elementSelector);
let snapshot: Snapshot | undefined;
let changed = false;
// Retry on failure (max 3 attempts)
if (fetchResult.error) {
console.log(`[Monitor] Fetch failed, retrying... (1/3)`);
await new Promise((resolve) => setTimeout(resolve, 2000));
fetchResult = await fetchPage(monitor.url, monitor.elementSelector);
// Content Check Part
if (checkType === 'all' || checkType === 'content') {
console.log(`[Monitor] Running CONTENT check for ${monitor.name} (${monitor.url})`);
// Fetch page with retries
let fetchResult = await fetchPage(monitor.url, monitor.elementSelector);
// Retry on failure (max 3 attempts)
if (fetchResult.error) {
console.log(`[Monitor] Fetch failed, retrying... (2/3)`);
console.log(`[Monitor] Fetch failed, retrying... (1/3)`);
await new Promise((resolve) => setTimeout(resolve, 2000));
fetchResult = await fetchPage(monitor.url, monitor.elementSelector);
}
}
// Check for HTTP error status
if (!fetchResult.error && fetchResult.status >= 400) {
fetchResult.error = `HTTP ${fetchResult.status}`;
}
// If still failing after retries
if (fetchResult.error) {
console.error(`[Monitor] Failed to fetch ${monitor.url}: ${fetchResult.error}`);
// Create error snapshot
const failedSnapshot = await db.snapshots.create({
monitorId: monitor.id,
htmlContent: '',
textContent: '',
contentHash: '',
httpStatus: fetchResult.status,
responseTime: fetchResult.responseTime,
changed: false,
errorMessage: fetchResult.error,
});
await db.monitors.incrementErrors(monitor.id);
// Send error alert if consecutive errors > 3
if (monitor.consecutiveErrors >= 2) {
const user = await db.users.findById(monitor.userId);
if (user) {
await sendErrorAlert(monitor, user, fetchResult.error);
if (fetchResult.error) {
console.log(`[Monitor] Fetch failed, retrying... (2/3)`);
await new Promise((resolve) => setTimeout(resolve, 2000));
fetchResult = await fetchPage(monitor.url, monitor.elementSelector);
}
}
// Check for HTTP error status
if (!fetchResult.error && fetchResult.status >= 400) {
fetchResult.error = `HTTP ${fetchResult.status}`;
}
return { snapshot: failedSnapshot, alertSent: false };
}
// If still failing after retries
if (fetchResult.error) {
console.error(`[Monitor] Failed to fetch ${monitor.url}: ${fetchResult.error}`);
// Apply noise filters
console.log(`[Monitor] Ignore rules for ${monitor.name}:`, JSON.stringify(monitor.ignoreRules));
let processedHtml = applyCommonNoiseFilters(fetchResult.html);
processedHtml = applyIgnoreRules(processedHtml, monitor.ignoreRules);
// Create error snapshot
const failedSnapshot = await db.snapshots.create({
monitorId: monitor.id,
htmlContent: '',
textContent: '',
contentHash: '',
httpStatus: fetchResult.status,
responseTime: fetchResult.responseTime,
changed: false,
errorMessage: fetchResult.error,
});
// Get previous snapshot
const previousSnapshot = await db.snapshots.findLatestByMonitorId(monitor.id);
await db.monitors.incrementErrors(monitor.id);
let changed = false;
let changePercentage = 0;
let diffResult: ReturnType<typeof compareDiffs> | undefined;
// Send error alert if consecutive errors > 3
if (monitor.consecutiveErrors >= 2) {
const user = await db.users.findById(monitor.userId);
if (user) {
await sendErrorAlert(monitor, user, fetchResult.error);
}
}
if (previousSnapshot) {
// Apply same filters to previous content for fair comparison
let previousHtml = applyCommonNoiseFilters(previousSnapshot.htmlContent);
previousHtml = applyIgnoreRules(previousHtml, monitor.ignoreRules);
return { snapshot: failedSnapshot, alertSent: false };
}
// Compare
diffResult = compareDiffs(previousHtml, processedHtml);
changed = diffResult.changed;
changePercentage = diffResult.changePercentage;
// Apply noise filters
let processedHtml = applyCommonNoiseFilters(fetchResult.html);
processedHtml = applyIgnoreRules(processedHtml, monitor.ignoreRules);
console.log(
`[Monitor] ${monitor.name}: Changed=${changed}, Change%=${changePercentage.toFixed(2)}, Additions=${diffResult.additions}, Deletions=${diffResult.deletions}`
);
// Get previous snapshot
const previousSnapshot = await db.snapshots.findLatestByMonitorId(monitor.id);
// Check keywords
if (monitor.keywordRules && monitor.keywordRules.length > 0) {
const keywordMatches = checkKeywords(
previousHtml,
processedHtml,
monitor.keywordRules
let changePercentage = 0;
let diffResult: ReturnType<typeof compareDiffs> | undefined;
if (previousSnapshot) {
// Apply same filters to previous content for fair comparison
let previousHtml = applyCommonNoiseFilters(previousSnapshot.htmlContent);
previousHtml = applyIgnoreRules(previousHtml, monitor.ignoreRules);
// Compare
diffResult = compareDiffs(previousHtml, processedHtml);
changed = diffResult.changed;
changePercentage = diffResult.changePercentage;
console.log(
`[Monitor] ${monitor.name}: Changed=${changed}, Change%=${changePercentage.toFixed(2)}, Additions=${diffResult.additions}, Deletions=${diffResult.deletions}`
);
if (keywordMatches.length > 0) {
console.log(`[Monitor] Keyword matches found:`, keywordMatches);
const user = await db.users.findById(monitor.userId);
// Check keywords
if (monitor.keywordRules && monitor.keywordRules.length > 0) {
const keywordMatches = checkKeywords(
previousHtml,
processedHtml,
monitor.keywordRules
);
if (user) {
for (const match of keywordMatches) {
await sendKeywordAlert(monitor, user, match);
if (keywordMatches.length > 0) {
console.log(`[Monitor] Keyword matches found:`, keywordMatches);
const user = await db.users.findById(monitor.userId);
if (user) {
for (const match of keywordMatches) {
await sendKeywordAlert(monitor, user, match);
}
}
}
}
} else {
// First check - consider it as "changed" to create baseline
changed = true;
console.log(`[Monitor] First check for ${monitor.name}, creating baseline`);
}
} else {
// First check - consider it as "changed" to create baseline
changed = true;
console.log(`[Monitor] First check for ${monitor.name}, creating baseline`);
}
// Generate human-readable summary (Hybrid approach)
let summary: string | undefined;
// Generate human-readable summary (Hybrid approach)
let summary: string | undefined;
if (changed && previousSnapshot && diffResult) {
// Hybrid logic: AI for changes (≥5%), simple for very small changes
if (changePercentage >= 5) {
console.log(`[Monitor] Change (${changePercentage}%), using AI summary`);
try {
summary = await generateAISummary(diffResult.diff, changePercentage);
} catch (error) {
console.error('[Monitor] AI summary failed, falling back to simple summary:', error);
if (changed && previousSnapshot && diffResult) {
// Hybrid logic: AI for changes (≥5%), simple for very small changes
if (changePercentage >= 5) {
console.log(`[Monitor] Change (${changePercentage}%), using AI summary`);
try {
summary = await generateAISummary(diffResult.diff, changePercentage, monitor.url);
} catch (error) {
console.error('[Monitor] AI summary failed, falling back to simple summary:', error);
summary = generateSimpleSummary(
diffResult.diff,
previousSnapshot.htmlContent,
fetchResult.html
);
}
} else {
console.log(`[Monitor] Small change (${changePercentage}%), using simple summary`);
summary = generateSimpleSummary(
diffResult.diff,
previousSnapshot.htmlContent,
fetchResult.html
);
}
} else {
console.log(`[Monitor] Small change (${changePercentage}%), using simple summary`);
summary = generateSimpleSummary(
diffResult.diff,
previousSnapshot.htmlContent,
fetchResult.html
);
}
}
// Create snapshot
const snapshot = await db.snapshots.create({
monitorId: monitor.id,
htmlContent: fetchResult.html,
textContent: fetchResult.text,
contentHash: fetchResult.hash,
httpStatus: fetchResult.status,
responseTime: fetchResult.responseTime,
changed,
changePercentage: changed ? changePercentage : undefined,
importanceScore: changed ? await calculateChangeImportance(monitor.id, changePercentage, 0, processedHtml) : 0,
summary,
});
// Create snapshot
console.log(`[Monitor] Creating snapshot in DB for ${monitor.name}`);
snapshot = await db.snapshots.create({
monitorId: monitor.id,
htmlContent: fetchResult.html,
textContent: fetchResult.text,
contentHash: fetchResult.hash,
httpStatus: fetchResult.status,
responseTime: fetchResult.responseTime,
changed,
changePercentage: changed ? changePercentage : undefined,
importanceScore: changed ? await calculateChangeImportance(monitor.id, changePercentage, 0, processedHtml) : 0,
summary,
});
// Update monitor
await db.monitors.updateLastChecked(monitor.id, changed);
// Update monitor
await db.monitors.updateLastChecked(monitor.id, changed);
// Send alert if changed and not first check
if (changed && previousSnapshot) {
try {
const user = await db.users.findById(monitor.userId);
if (user) {
await sendChangeAlert(monitor, user, snapshot, changePercentage);
// Send alert if changed and not first check
if (changed && previousSnapshot) {
try {
const user = await db.users.findById(monitor.userId);
if (user) {
await sendChangeAlert(monitor, user, snapshot, changePercentage);
}
} catch (alertError) {
console.error(`[Monitor] Failed to send alert for ${monitor.id}:`, alertError);
}
} catch (alertError) {
console.error(`[Monitor] Failed to send alert for ${monitor.id}:`, alertError);
// Continue execution - do not fail the check
}
}
// Clean up old snapshots based on user plan retention period
try {
const retentionUser = await db.users.findById(monitor.userId);
if (retentionUser) {
const { getRetentionDays } = await import('../config');
const retentionDays = getRetentionDays(retentionUser.plan);
await db.snapshots.deleteOldSnapshotsByAge(monitor.id, retentionDays);
// SEO Check Part
if ((checkType === 'all' || checkType === 'seo') && monitor.seoKeywords && monitor.seoKeywords.length > 0) {
let shouldRunSeo = false;
if (forceSeo) {
console.log(`[Monitor] SEO check triggered manually for ${monitor.name}`);
shouldRunSeo = true;
} else if (monitor.seoInterval && monitor.seoInterval !== 'off') {
if (!monitor.lastSeoCheckAt) {
shouldRunSeo = true;
} else {
const hoursSinceLast = (Date.now() - new Date(monitor.lastSeoCheckAt).getTime()) / (1000 * 60 * 60);
switch (monitor.seoInterval) {
case 'daily': shouldRunSeo = hoursSinceLast >= 24; break;
case '2d': shouldRunSeo = hoursSinceLast >= 48; break;
case 'weekly': shouldRunSeo = hoursSinceLast >= 168; break;
case 'monthly': shouldRunSeo = hoursSinceLast >= 720; break;
}
}
}
if (shouldRunSeo) {
console.log(`[Monitor] Running SEO check for ${monitor.name} (Schedule: ${monitor.seoInterval})`);
// Update last_seo_check_at immediately to prevent double scheduling if slow
await db.monitors.update(monitor.id, { lastSeoCheckAt: new Date() });
if (forceSeo) {
// Await SEO check if explicitly forced (manual trigger)
await processSeoChecks(monitor.id, monitor.url, monitor.seoKeywords);
} else {
// Run in background for scheduled checks
processSeoChecks(monitor.id, monitor.url, monitor.seoKeywords)
.catch(err => console.error(`[Monitor] SEO check failed for ${monitor.name}:`, err));
}
}
} catch (cleanupError) {
console.error(`[Monitor] Failed to cleanup snapshots for ${monitor.id}:`, cleanupError);
}
console.log(`[Monitor] Check completed for ${monitor.name}`);
return { snapshot, alertSent: changed && !!previousSnapshot };
console.log(`[Monitor] Check completed for ${monitor.name} (Snapshot created: ${!!snapshot})`);
return {
snapshot,
alertSent: changed
};
} catch (error) {
console.error(`[Monitor] Error checking monitor ${monitorId}:`, error);
await db.monitors.incrementErrors(monitorId);

98
backend/src/services/seo.ts Normal file
View File

@@ -0,0 +1,98 @@
import { getJson } from 'serpapi';
import db from '../db';
interface RankingResult {
rank: number | null;
foundUrl: string | null;
error?: string;
}
export async function checkRanking(keyword: string, targetUrl: string): Promise<RankingResult> {
const apiKey = process.env.SERPAPI_KEY;
if (!apiKey) {
console.error('SERPAPI_KEY is missing');
return { rank: null, foundUrl: null, error: 'SERPAPI_KEY missing' };
}
// Domain normalization for easier matching
// Removes protocol, www, and trailing slashes
const normalizeUrl = (url: string) => {
try {
const u = new URL(url.startsWith('http') ? url : `https://${url}`);
return u.hostname.replace('www.', '') + u.pathname.replace(/\/$/, '');
} catch (e) {
return url.replace('www.', '').replace(/\/$/, '');
}
};
const normalizedTarget = normalizeUrl(targetUrl);
return new Promise((resolve) => {
try {
getJson(
{
engine: 'google',
q: keyword,
api_key: apiKey,
num: 100, // Check top 100 results
},
(json: any) => {
if (json.error) {
console.error('SerpApi error:', json.error);
resolve({ rank: null, foundUrl: null, error: json.error });
return;
}
if (!json.organic_results) {
resolve({ rank: null, foundUrl: null });
return;
}
for (const result of json.organic_results) {
const resultUrl = result.link;
const normalizedResult = normalizeUrl(resultUrl);
// Check if result contains our target domain
if (normalizedResult.includes(normalizedTarget) || normalizedTarget.includes(normalizedResult)) {
resolve({
rank: result.position,
foundUrl: resultUrl,
});
return;
}
}
resolve({ rank: null, foundUrl: null });
}
);
} catch (error: any) {
console.error('SeoService error:', error);
resolve({ rank: null, foundUrl: null, error: error.message });
}
});
}
export async function processSeoChecks(monitorId: string, url: string, keywords: string[]) {
if (!keywords || keywords.length === 0) return;
console.log(`[SEO] Starting checks for monitor ${monitorId} (${keywords.length} keywords)`);
for (const keyword of keywords) {
const result = await checkRanking(keyword, url);
if (result.rank !== null || result.foundUrl !== null) {
console.log(`[SEO] Found rank ${result.rank} for "${keyword}"`);
} else {
console.log(`[SEO] Not found in top 100 for "${keyword}"`);
}
// Save to DB
await db.rankings.create({
monitorId,
keyword,
rank: result.rank,
urlFound: result.foundUrl
});
}
}

36
backend/src/services/summarizer.ts
View File

@@ -76,11 +76,11 @@ export function generateSimpleSummary(
// Add text preview if available
if (textPreviews.length > 0) {
const previewText = textPreviews.slice(0, 2).join(' → ');
const previewText = textPreviews.join(' | ');
if (summary) {
summary += `. Changed: "${previewText}"`;
summary += `. Details: ${previewText}`;
} else {
summary = `Text changed: "${previewText}"`;
summary = `Changes: ${previewText}`;
}
}
@@ -213,7 +213,8 @@ function countRemovedElements(htmlOld: string, htmlNew: string, tag: string): nu
*/
export async function generateAISummary(
diff: Change[],
changePercentage: number
changePercentage: number,
url?: string
): Promise<string> {
try {
// Check if API key is configured
@@ -229,17 +230,30 @@ export async function generateAISummary(
// Format diff for AI (reduce token count)
const formattedDiff = formatDiffForAI(diff);
const prompt = `Analyze this website change and create a concise summary for non-programmers.
Focus on IMPORTANT changes only. Medium detail level.
const prompt = `Analyze the website changes for: ${url || 'unknown'}
You are an expert content monitor. Your task is to provide a high-quality, professional summary of what changed on this page.
Change percentage: ${changePercentage.toFixed(2)}%
GOAL:
Categorize changes by page section and describe their impact on the user.
Diff:
CRITICAL INSTRUCTIONS:
1. Identify the SECTION: Look at the tags and context. Is it the "Pricing Table", "Feature List", "Hero Section", "Blog Feed", or "Footer"? Mention it clearly.
2. Be SPECIFIC: Instead of "Pricing updated", say "The Pro Plan monthly price increased from $29 to $34".
3. CONTEXTUALIZE: Group related changes together. For example, "Updated the 'CloudScale v2.4' header and refreshed the blog post titles in the feed section."
4. NO JARGON: Avoid terms like "HTML", "div", "CSS", "selectors". Talk to the user, not a developer.
5. TONE: Professional, concise, and helpful.
Change magnitude: ${changePercentage.toFixed(2)}%
DIFF DATA TO ANALYZE:
${formattedDiff}
Format: "Section name: What changed. Details if important."
Example: "Pricing section updated: 3 prices increased. 2 new product links in footer."
Keep it under 100 words. Be specific about what changed, not how.`;
FORMAT:
- Start with a single summary sentence.
- Use DOUBLE NEWLINES between different sections (e.g., between Pricing and Blog).
- Each bullet point MUST be on its own new line.
- Use bold headers for sections like **Pricing Table:** or **Header Update:**.
- Limit response to 150 words.`;
const completion = await client.chat.completions.create({
model: 'gpt-4o-mini', // Fastest, cheapest

5
backend/src/types/index.ts
View File

@@ -47,6 +47,9 @@ export interface Monitor {
elementSelector?: string;
ignoreRules?: IgnoreRule[];
keywordRules?: KeywordRule[];
seoKeywords?: string[];
seoInterval?: string;
lastSeoCheckAt?: Date;
lastCheckedAt?: Date;
lastChangedAt?: Date;
consecutiveErrors: number;
@@ -98,6 +101,7 @@ export interface CreateMonitorInput {
elementSelector?: string;
ignoreRules?: IgnoreRule[];
keywordRules?: KeywordRule[];
seoInterval?: string;
}
export interface UpdateMonitorInput {
@@ -107,4 +111,5 @@ export interface UpdateMonitorInput {
elementSelector?: string;
ignoreRules?: IgnoreRule[];
keywordRules?: KeywordRule[];
seoInterval?: string;
}