11 seo pages
This commit is contained in:
Timo
368
scripts/build.js
368
scripts/build.js
@@ -1,185 +1,185 @@
|
||||
const { spawnSync } = require('child_process');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const repoRoot = path.resolve(__dirname, '..');
|
||||
const prismaSchemaPath = path.join(repoRoot, 'prisma', 'schema.prisma');
|
||||
const generatedClientDir = path.join(repoRoot, 'node_modules', '.prisma', 'client');
|
||||
const generatedSchemaPath = path.join(generatedClientDir, 'schema.prisma');
|
||||
|
||||
function readFileIfExists(filePath) {
|
||||
try {
|
||||
return fs.readFileSync(filePath, 'utf8');
|
||||
} catch (error) {
|
||||
if (error && error.code === 'ENOENT') {
|
||||
return null;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeSchema(schema) {
|
||||
return schema.replace(/\s+/g, '');
|
||||
}
|
||||
|
||||
function schemasMatch() {
|
||||
const sourceSchema = readFileIfExists(prismaSchemaPath);
|
||||
const generatedSchema = readFileIfExists(generatedSchemaPath);
|
||||
|
||||
return Boolean(
|
||||
sourceSchema &&
|
||||
generatedSchema &&
|
||||
normalizeSchema(sourceSchema) === normalizeSchema(generatedSchema)
|
||||
);
|
||||
}
|
||||
|
||||
function run(command, args, options = {}) {
|
||||
const shouldUseShell =
|
||||
process.platform === 'win32' && command.toLowerCase().endsWith('.cmd');
|
||||
|
||||
const result = spawnSync(command, args, {
|
||||
cwd: repoRoot,
|
||||
encoding: 'utf8',
|
||||
stdio: 'pipe',
|
||||
shell: shouldUseShell,
|
||||
env: {
|
||||
...process.env,
|
||||
...options.env,
|
||||
},
|
||||
});
|
||||
|
||||
if (result.stdout) {
|
||||
process.stdout.write(result.stdout);
|
||||
}
|
||||
|
||||
if (result.stderr) {
|
||||
process.stderr.write(result.stderr);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function isWSL() {
|
||||
return (
|
||||
process.platform === 'linux' &&
|
||||
fs.existsSync('/proc/version') &&
|
||||
fs.readFileSync('/proc/version', 'utf8').toLowerCase().includes('microsoft')
|
||||
);
|
||||
}
|
||||
|
||||
function isWindowsPrismaRenameLock(output) {
|
||||
const text = [output.stdout, output.stderr]
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
|
||||
return (
|
||||
process.platform === 'win32' &&
|
||||
text.includes('EPERM: operation not permitted, rename') &&
|
||||
text.includes('query_engine-windows.dll.node')
|
||||
);
|
||||
}
|
||||
|
||||
function isPrismaCopyfileEio(output) {
|
||||
const text = [output.stdout, output.stderr]
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
|
||||
return (
|
||||
text.includes('EIO: i/o error, copyfile') &&
|
||||
(text.includes('libquery_engine-') || text.includes('query_engine-'))
|
||||
);
|
||||
}
|
||||
|
||||
function cleanupPrismaTempFiles() {
|
||||
if (!fs.existsSync(generatedClientDir)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const entry of fs.readdirSync(generatedClientDir)) {
|
||||
if (!entry.includes('.tmp')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
fs.rmSync(path.join(generatedClientDir, entry), { force: true });
|
||||
} catch (error) {
|
||||
console.warn(`Failed to remove stale Prisma temp file ${entry}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runPrismaGenerate() {
|
||||
const prismaBin =
|
||||
process.platform === 'win32'
|
||||
? path.join(repoRoot, 'node_modules', '.bin', 'prisma.cmd')
|
||||
: path.join(repoRoot, 'node_modules', '.bin', 'prisma');
|
||||
|
||||
if (isWSL()) {
|
||||
cleanupPrismaTempFiles();
|
||||
}
|
||||
|
||||
let result = run(prismaBin, ['generate']);
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
if ((result.status ?? 1) === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const retryablePrismaFsError =
|
||||
isWindowsPrismaRenameLock(result) || isPrismaCopyfileEio(result);
|
||||
|
||||
if (retryablePrismaFsError) {
|
||||
cleanupPrismaTempFiles();
|
||||
result = run(prismaBin, ['generate']);
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
if ((result.status ?? 1) === 0) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (!retryablePrismaFsError || !schemasMatch()) {
|
||||
return result.status ?? 1;
|
||||
}
|
||||
|
||||
console.warn(
|
||||
'\nPrisma generate hit a filesystem copy/rename issue, but the generated client already matches prisma/schema.prisma. Continuing with the existing client.\n'
|
||||
);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
function runNextBuild() {
|
||||
const nextBin =
|
||||
process.platform === 'win32'
|
||||
? path.join(repoRoot, 'node_modules', '.bin', 'next.cmd')
|
||||
: path.join(repoRoot, 'node_modules', '.bin', 'next');
|
||||
|
||||
const memoryLimit = isWSL() ? '8192' : '4096';
|
||||
|
||||
return run(nextBin, ['build'], {
|
||||
env: {
|
||||
NODE_OPTIONS: `--max-old-space-size=${memoryLimit}`,
|
||||
SKIP_ENV_VALIDATION: 'true',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const prismaExitCode = runPrismaGenerate();
|
||||
if (prismaExitCode !== 0) {
|
||||
process.exit(prismaExitCode);
|
||||
}
|
||||
|
||||
const nextResult = runNextBuild();
|
||||
if (nextResult.error) {
|
||||
throw nextResult.error;
|
||||
}
|
||||
|
||||
const { spawnSync } = require('child_process');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const repoRoot = path.resolve(__dirname, '..');
|
||||
const prismaSchemaPath = path.join(repoRoot, 'prisma', 'schema.prisma');
|
||||
const generatedClientDir = path.join(repoRoot, 'node_modules', '.prisma', 'client');
|
||||
const generatedSchemaPath = path.join(generatedClientDir, 'schema.prisma');
|
||||
|
||||
function readFileIfExists(filePath) {
|
||||
try {
|
||||
return fs.readFileSync(filePath, 'utf8');
|
||||
} catch (error) {
|
||||
if (error && error.code === 'ENOENT') {
|
||||
return null;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeSchema(schema) {
|
||||
return schema.replace(/\s+/g, '');
|
||||
}
|
||||
|
||||
function schemasMatch() {
|
||||
const sourceSchema = readFileIfExists(prismaSchemaPath);
|
||||
const generatedSchema = readFileIfExists(generatedSchemaPath);
|
||||
|
||||
return Boolean(
|
||||
sourceSchema &&
|
||||
generatedSchema &&
|
||||
normalizeSchema(sourceSchema) === normalizeSchema(generatedSchema)
|
||||
);
|
||||
}
|
||||
|
||||
function run(command, args, options = {}) {
|
||||
const shouldUseShell =
|
||||
process.platform === 'win32' && command.toLowerCase().endsWith('.cmd');
|
||||
|
||||
const result = spawnSync(command, args, {
|
||||
cwd: repoRoot,
|
||||
encoding: 'utf8',
|
||||
stdio: 'pipe',
|
||||
shell: shouldUseShell,
|
||||
env: {
|
||||
...process.env,
|
||||
...options.env,
|
||||
},
|
||||
});
|
||||
|
||||
if (result.stdout) {
|
||||
process.stdout.write(result.stdout);
|
||||
}
|
||||
|
||||
if (result.stderr) {
|
||||
process.stderr.write(result.stderr);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function isWSL() {
|
||||
return (
|
||||
process.platform === 'linux' &&
|
||||
fs.existsSync('/proc/version') &&
|
||||
fs.readFileSync('/proc/version', 'utf8').toLowerCase().includes('microsoft')
|
||||
);
|
||||
}
|
||||
|
||||
function isWindowsPrismaRenameLock(output) {
|
||||
const text = [output.stdout, output.stderr]
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
|
||||
return (
|
||||
process.platform === 'win32' &&
|
||||
text.includes('EPERM: operation not permitted, rename') &&
|
||||
text.includes('query_engine-windows.dll.node')
|
||||
);
|
||||
}
|
||||
|
||||
function isPrismaCopyfileEio(output) {
|
||||
const text = [output.stdout, output.stderr]
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
|
||||
return (
|
||||
text.includes('EIO: i/o error, copyfile') &&
|
||||
(text.includes('libquery_engine-') || text.includes('query_engine-'))
|
||||
);
|
||||
}
|
||||
|
||||
function cleanupPrismaTempFiles() {
|
||||
if (!fs.existsSync(generatedClientDir)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const entry of fs.readdirSync(generatedClientDir)) {
|
||||
if (!entry.includes('.tmp')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
fs.rmSync(path.join(generatedClientDir, entry), { force: true });
|
||||
} catch (error) {
|
||||
console.warn(`Failed to remove stale Prisma temp file ${entry}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runPrismaGenerate() {
|
||||
const prismaBin =
|
||||
process.platform === 'win32'
|
||||
? path.join(repoRoot, 'node_modules', '.bin', 'prisma.cmd')
|
||||
: path.join(repoRoot, 'node_modules', '.bin', 'prisma');
|
||||
|
||||
if (isWSL()) {
|
||||
cleanupPrismaTempFiles();
|
||||
}
|
||||
|
||||
let result = run(prismaBin, ['generate']);
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
if ((result.status ?? 1) === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const retryablePrismaFsError =
|
||||
isWindowsPrismaRenameLock(result) || isPrismaCopyfileEio(result);
|
||||
|
||||
if (retryablePrismaFsError) {
|
||||
cleanupPrismaTempFiles();
|
||||
result = run(prismaBin, ['generate']);
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
if ((result.status ?? 1) === 0) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (!retryablePrismaFsError || !schemasMatch()) {
|
||||
return result.status ?? 1;
|
||||
}
|
||||
|
||||
console.warn(
|
||||
'\nPrisma generate hit a filesystem copy/rename issue, but the generated client already matches prisma/schema.prisma. Continuing with the existing client.\n'
|
||||
);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
function runNextBuild() {
|
||||
const nextBin =
|
||||
process.platform === 'win32'
|
||||
? path.join(repoRoot, 'node_modules', '.bin', 'next.cmd')
|
||||
: path.join(repoRoot, 'node_modules', '.bin', 'next');
|
||||
|
||||
const memoryLimit = isWSL() ? '8192' : '4096';
|
||||
|
||||
return run(nextBin, ['build'], {
|
||||
env: {
|
||||
NODE_OPTIONS: `--max-old-space-size=${memoryLimit}`,
|
||||
SKIP_ENV_VALIDATION: 'true',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const prismaExitCode = runPrismaGenerate();
|
||||
if (prismaExitCode !== 0) {
|
||||
process.exit(prismaExitCode);
|
||||
}
|
||||
|
||||
const nextResult = runNextBuild();
|
||||
if (nextResult.error) {
|
||||
throw nextResult.error;
|
||||
}
|
||||
|
||||
process.exit(nextResult.status ?? 1);
|
||||
@@ -1,249 +1,249 @@
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const root = process.cwd();
|
||||
const scanDirs = ["src", "marketing", "articles", "blog-posts-improved"];
|
||||
const sourceExtensions = new Set([".ts", ".tsx", ".js", ".jsx", ".md", ".mdx"]);
|
||||
const publicDir = path.join(root, "public");
|
||||
const appDir = path.join(root, "src", "app");
|
||||
|
||||
const ignoredPrefixes = [
|
||||
"/api/",
|
||||
"/_next/",
|
||||
"/auth/",
|
||||
"/r/",
|
||||
"/qr/",
|
||||
"/scan/",
|
||||
];
|
||||
|
||||
const knownDynamicPrefixes = [
|
||||
"/blog/",
|
||||
"/learn/",
|
||||
"/authors/",
|
||||
"/qr-code-for/",
|
||||
"/use-cases/",
|
||||
];
|
||||
|
||||
const ctaPatterns = [
|
||||
/get started/i,
|
||||
/start free/i,
|
||||
/try free/i,
|
||||
/create.*qr/i,
|
||||
/generate.*qr/i,
|
||||
/sign up/i,
|
||||
/pricing/i,
|
||||
/upgrade/i,
|
||||
/create.*free/i,
|
||||
/start tracking/i,
|
||||
/create.*editable/i,
|
||||
];
|
||||
|
||||
const nonConversionPageParts = [
|
||||
"/contact/",
|
||||
"/cookie-policy/",
|
||||
"/privacy/",
|
||||
"/press/",
|
||||
"/authors/",
|
||||
"/blog/",
|
||||
"/newsletter/",
|
||||
];
|
||||
|
||||
const findings = [];
|
||||
const ctas = [];
|
||||
|
||||
function walk(dir, files = []) {
|
||||
if (!fs.existsSync(dir)) return files;
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.name === "node_modules" || entry.name === ".next" || entry.name === ".git") continue;
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(fullPath, files);
|
||||
} else {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
function toPosix(value) {
|
||||
return value.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function routeFromPageFile(file) {
|
||||
const rel = toPosix(path.relative(appDir, file));
|
||||
if (!rel.endsWith("/page.tsx") && !rel.endsWith("/page.ts") && !rel.endsWith("/route.ts")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parts = rel.split("/");
|
||||
parts.pop();
|
||||
const routeParts = parts.filter((part) => {
|
||||
if (!part) return false;
|
||||
if (part.startsWith("(") && part.endsWith(")")) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
if (routeParts.some((part) => part.startsWith("[") && part.endsWith("]"))) return null;
|
||||
return "/" + routeParts.join("/");
|
||||
}
|
||||
|
||||
function collectRoutes() {
|
||||
const routes = new Set(["/"]);
|
||||
for (const file of walk(appDir)) {
|
||||
const route = routeFromPageFile(file);
|
||||
if (route) routes.add(route === "/" ? "/" : route.replace(/\/$/, ""));
|
||||
}
|
||||
|
||||
for (const file of walk(publicDir)) {
|
||||
const rel = "/" + toPosix(path.relative(publicDir, file));
|
||||
routes.add(rel);
|
||||
}
|
||||
|
||||
return routes;
|
||||
}
|
||||
|
||||
function normalizeHref(rawHref) {
|
||||
if (!rawHref) return null;
|
||||
let href = rawHref.trim();
|
||||
if (!href || href.startsWith("#")) return null;
|
||||
if (/^(https?:|mailto:|tel:|sms:|javascript:|data:)/i.test(href)) return null;
|
||||
|
||||
if (!href.startsWith("/")) return null;
|
||||
href = href.split("#")[0].split("?")[0];
|
||||
if (href.length > 1) href = href.replace(/\/$/, "");
|
||||
return href || "/";
|
||||
}
|
||||
|
||||
function isAllowedDynamicHref(href) {
|
||||
if (ignoredPrefixes.some((prefix) => href.startsWith(prefix))) return true;
|
||||
if (href.includes("[") || href.includes("${") || href.includes("`")) return true;
|
||||
return knownDynamicPrefixes.some((prefix) => href.startsWith(prefix) && href !== prefix.replace(/\/$/, ""));
|
||||
}
|
||||
|
||||
function lineNumber(content, index) {
|
||||
return content.slice(0, index).split(/\r?\n/).length;
|
||||
}
|
||||
|
||||
function extractHrefMatches(content) {
|
||||
const matches = [];
|
||||
const patterns = [
|
||||
/href\s*=\s*["']([^"']+)["']/g,
|
||||
/href\s*=\s*{\s*["']([^"']+)["']\s*}/g,
|
||||
/router\.push\(\s*["']([^"']+)["']\s*\)/g,
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
let match;
|
||||
while ((match = pattern.exec(content)) !== null) {
|
||||
matches.push({ href: match[1], index: match.index });
|
||||
}
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
function extractAnchors(content) {
|
||||
const anchors = [];
|
||||
const linkPattern = /<Link\b[\s\S]*?href\s*=\s*(?:["']([^"']+)["']|{\s*["']([^"']+)["']\s*})[\s\S]*?>([\s\S]*?)<\/Link>/g;
|
||||
const anchorPattern = /<a\b[\s\S]*?href\s*=\s*(?:["']([^"']+)["']|{\s*["']([^"']+)["']\s*})[\s\S]*?>([\s\S]*?)<\/a>/g;
|
||||
|
||||
for (const pattern of [linkPattern, anchorPattern]) {
|
||||
let match;
|
||||
while ((match = pattern.exec(content)) !== null) {
|
||||
const href = match[1] || match[2];
|
||||
const text = match[3]
|
||||
.replace(/<[^>]*>/g, " ")
|
||||
.replace(/\{[^}]*\}/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
anchors.push({ href, text, index: match.index });
|
||||
}
|
||||
}
|
||||
return anchors;
|
||||
}
|
||||
|
||||
function sourceFiles() {
|
||||
const files = [];
|
||||
for (const dir of scanDirs) {
|
||||
for (const file of walk(path.join(root, dir))) {
|
||||
if (sourceExtensions.has(path.extname(file))) files.push(file);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
function check() {
|
||||
const routes = collectRoutes();
|
||||
|
||||
for (const file of sourceFiles()) {
|
||||
const content = fs.readFileSync(file, "utf8");
|
||||
const rel = toPosix(path.relative(root, file));
|
||||
|
||||
for (const item of extractHrefMatches(content)) {
|
||||
const href = normalizeHref(item.href);
|
||||
if (!href) continue;
|
||||
if (routes.has(href) || isAllowedDynamicHref(href)) continue;
|
||||
|
||||
findings.push({
|
||||
type: "broken-internal-link",
|
||||
file: rel,
|
||||
line: lineNumber(content, item.index),
|
||||
href,
|
||||
});
|
||||
}
|
||||
|
||||
for (const anchor of extractAnchors(content)) {
|
||||
const text = anchor.text || "";
|
||||
if (!ctaPatterns.some((pattern) => pattern.test(text))) continue;
|
||||
|
||||
const href = normalizeHref(anchor.href);
|
||||
const status = !href
|
||||
? "external-or-non-http"
|
||||
: routes.has(href) || isAllowedDynamicHref(href)
|
||||
? "ok"
|
||||
: "broken";
|
||||
|
||||
ctas.push({
|
||||
file: rel,
|
||||
line: lineNumber(content, anchor.index),
|
||||
text: text.slice(0, 100),
|
||||
href: anchor.href,
|
||||
status,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const brokenCtas = ctas.filter((cta) => cta.status === "broken");
|
||||
const weakFiles = sourceFiles().filter((file) => {
|
||||
const rel = toPosix(path.relative(root, file));
|
||||
if (!rel.includes("src/app/") || !rel.endsWith("/page.tsx")) return false;
|
||||
if (!rel.includes("(marketing)")) return false;
|
||||
if (rel.includes("[")) return false;
|
||||
if (nonConversionPageParts.some((part) => rel.includes(part))) return false;
|
||||
|
||||
const content = fs.readFileSync(file, "utf8");
|
||||
return !extractAnchors(content).some((anchor) =>
|
||||
ctaPatterns.some((pattern) => pattern.test(anchor.text || "")),
|
||||
);
|
||||
});
|
||||
|
||||
const report = {
|
||||
checkedAt: new Date().toISOString(),
|
||||
routeCount: routes.size,
|
||||
filesChecked: sourceFiles().length,
|
||||
brokenInternalLinks: findings,
|
||||
ctaSummary: {
|
||||
total: ctas.length,
|
||||
broken: brokenCtas.length,
|
||||
sample: ctas.slice(0, 50),
|
||||
},
|
||||
pagesWithoutObviousCta: weakFiles.map((file) => toPosix(path.relative(root, file))).slice(0, 100),
|
||||
};
|
||||
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
|
||||
if (findings.length > 0 || brokenCtas.length > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
check();
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const root = process.cwd();
|
||||
const scanDirs = ["src", "marketing", "articles", "blog-posts-improved"];
|
||||
const sourceExtensions = new Set([".ts", ".tsx", ".js", ".jsx", ".md", ".mdx"]);
|
||||
const publicDir = path.join(root, "public");
|
||||
const appDir = path.join(root, "src", "app");
|
||||
|
||||
const ignoredPrefixes = [
|
||||
"/api/",
|
||||
"/_next/",
|
||||
"/auth/",
|
||||
"/r/",
|
||||
"/qr/",
|
||||
"/scan/",
|
||||
];
|
||||
|
||||
const knownDynamicPrefixes = [
|
||||
"/blog/",
|
||||
"/learn/",
|
||||
"/authors/",
|
||||
"/qr-code-for/",
|
||||
"/use-cases/",
|
||||
];
|
||||
|
||||
const ctaPatterns = [
|
||||
/get started/i,
|
||||
/start free/i,
|
||||
/try free/i,
|
||||
/create.*qr/i,
|
||||
/generate.*qr/i,
|
||||
/sign up/i,
|
||||
/pricing/i,
|
||||
/upgrade/i,
|
||||
/create.*free/i,
|
||||
/start tracking/i,
|
||||
/create.*editable/i,
|
||||
];
|
||||
|
||||
const nonConversionPageParts = [
|
||||
"/contact/",
|
||||
"/cookie-policy/",
|
||||
"/privacy/",
|
||||
"/press/",
|
||||
"/authors/",
|
||||
"/blog/",
|
||||
"/newsletter/",
|
||||
];
|
||||
|
||||
const findings = [];
|
||||
const ctas = [];
|
||||
|
||||
function walk(dir, files = []) {
|
||||
if (!fs.existsSync(dir)) return files;
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.name === "node_modules" || entry.name === ".next" || entry.name === ".git") continue;
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(fullPath, files);
|
||||
} else {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
function toPosix(value) {
|
||||
return value.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function routeFromPageFile(file) {
|
||||
const rel = toPosix(path.relative(appDir, file));
|
||||
if (!rel.endsWith("/page.tsx") && !rel.endsWith("/page.ts") && !rel.endsWith("/route.ts")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parts = rel.split("/");
|
||||
parts.pop();
|
||||
const routeParts = parts.filter((part) => {
|
||||
if (!part) return false;
|
||||
if (part.startsWith("(") && part.endsWith(")")) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
if (routeParts.some((part) => part.startsWith("[") && part.endsWith("]"))) return null;
|
||||
return "/" + routeParts.join("/");
|
||||
}
|
||||
|
||||
function collectRoutes() {
|
||||
const routes = new Set(["/"]);
|
||||
for (const file of walk(appDir)) {
|
||||
const route = routeFromPageFile(file);
|
||||
if (route) routes.add(route === "/" ? "/" : route.replace(/\/$/, ""));
|
||||
}
|
||||
|
||||
for (const file of walk(publicDir)) {
|
||||
const rel = "/" + toPosix(path.relative(publicDir, file));
|
||||
routes.add(rel);
|
||||
}
|
||||
|
||||
return routes;
|
||||
}
|
||||
|
||||
function normalizeHref(rawHref) {
|
||||
if (!rawHref) return null;
|
||||
let href = rawHref.trim();
|
||||
if (!href || href.startsWith("#")) return null;
|
||||
if (/^(https?:|mailto:|tel:|sms:|javascript:|data:)/i.test(href)) return null;
|
||||
|
||||
if (!href.startsWith("/")) return null;
|
||||
href = href.split("#")[0].split("?")[0];
|
||||
if (href.length > 1) href = href.replace(/\/$/, "");
|
||||
return href || "/";
|
||||
}
|
||||
|
||||
function isAllowedDynamicHref(href) {
|
||||
if (ignoredPrefixes.some((prefix) => href.startsWith(prefix))) return true;
|
||||
if (href.includes("[") || href.includes("${") || href.includes("`")) return true;
|
||||
return knownDynamicPrefixes.some((prefix) => href.startsWith(prefix) && href !== prefix.replace(/\/$/, ""));
|
||||
}
|
||||
|
||||
function lineNumber(content, index) {
|
||||
return content.slice(0, index).split(/\r?\n/).length;
|
||||
}
|
||||
|
||||
function extractHrefMatches(content) {
|
||||
const matches = [];
|
||||
const patterns = [
|
||||
/href\s*=\s*["']([^"']+)["']/g,
|
||||
/href\s*=\s*{\s*["']([^"']+)["']\s*}/g,
|
||||
/router\.push\(\s*["']([^"']+)["']\s*\)/g,
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
let match;
|
||||
while ((match = pattern.exec(content)) !== null) {
|
||||
matches.push({ href: match[1], index: match.index });
|
||||
}
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
function extractAnchors(content) {
|
||||
const anchors = [];
|
||||
const linkPattern = /<Link\b[\s\S]*?href\s*=\s*(?:["']([^"']+)["']|{\s*["']([^"']+)["']\s*})[\s\S]*?>([\s\S]*?)<\/Link>/g;
|
||||
const anchorPattern = /<a\b[\s\S]*?href\s*=\s*(?:["']([^"']+)["']|{\s*["']([^"']+)["']\s*})[\s\S]*?>([\s\S]*?)<\/a>/g;
|
||||
|
||||
for (const pattern of [linkPattern, anchorPattern]) {
|
||||
let match;
|
||||
while ((match = pattern.exec(content)) !== null) {
|
||||
const href = match[1] || match[2];
|
||||
const text = match[3]
|
||||
.replace(/<[^>]*>/g, " ")
|
||||
.replace(/\{[^}]*\}/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
anchors.push({ href, text, index: match.index });
|
||||
}
|
||||
}
|
||||
return anchors;
|
||||
}
|
||||
|
||||
function sourceFiles() {
|
||||
const files = [];
|
||||
for (const dir of scanDirs) {
|
||||
for (const file of walk(path.join(root, dir))) {
|
||||
if (sourceExtensions.has(path.extname(file))) files.push(file);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
function check() {
|
||||
const routes = collectRoutes();
|
||||
|
||||
for (const file of sourceFiles()) {
|
||||
const content = fs.readFileSync(file, "utf8");
|
||||
const rel = toPosix(path.relative(root, file));
|
||||
|
||||
for (const item of extractHrefMatches(content)) {
|
||||
const href = normalizeHref(item.href);
|
||||
if (!href) continue;
|
||||
if (routes.has(href) || isAllowedDynamicHref(href)) continue;
|
||||
|
||||
findings.push({
|
||||
type: "broken-internal-link",
|
||||
file: rel,
|
||||
line: lineNumber(content, item.index),
|
||||
href,
|
||||
});
|
||||
}
|
||||
|
||||
for (const anchor of extractAnchors(content)) {
|
||||
const text = anchor.text || "";
|
||||
if (!ctaPatterns.some((pattern) => pattern.test(text))) continue;
|
||||
|
||||
const href = normalizeHref(anchor.href);
|
||||
const status = !href
|
||||
? "external-or-non-http"
|
||||
: routes.has(href) || isAllowedDynamicHref(href)
|
||||
? "ok"
|
||||
: "broken";
|
||||
|
||||
ctas.push({
|
||||
file: rel,
|
||||
line: lineNumber(content, anchor.index),
|
||||
text: text.slice(0, 100),
|
||||
href: anchor.href,
|
||||
status,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const brokenCtas = ctas.filter((cta) => cta.status === "broken");
|
||||
const weakFiles = sourceFiles().filter((file) => {
|
||||
const rel = toPosix(path.relative(root, file));
|
||||
if (!rel.includes("src/app/") || !rel.endsWith("/page.tsx")) return false;
|
||||
if (!rel.includes("(marketing)")) return false;
|
||||
if (rel.includes("[")) return false;
|
||||
if (nonConversionPageParts.some((part) => rel.includes(part))) return false;
|
||||
|
||||
const content = fs.readFileSync(file, "utf8");
|
||||
return !extractAnchors(content).some((anchor) =>
|
||||
ctaPatterns.some((pattern) => pattern.test(anchor.text || "")),
|
||||
);
|
||||
});
|
||||
|
||||
const report = {
|
||||
checkedAt: new Date().toISOString(),
|
||||
routeCount: routes.size,
|
||||
filesChecked: sourceFiles().length,
|
||||
brokenInternalLinks: findings,
|
||||
ctaSummary: {
|
||||
total: ctas.length,
|
||||
broken: brokenCtas.length,
|
||||
sample: ctas.slice(0, 50),
|
||||
},
|
||||
pagesWithoutObviousCta: weakFiles.map((file) => toPosix(path.relative(root, file))).slice(0, 100),
|
||||
};
|
||||
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
|
||||
if (findings.length > 0 || brokenCtas.length > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
check();
|
||||
|
||||
@@ -1,384 +1,384 @@
|
||||
import nodemailer from 'nodemailer';
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
import dayjs from 'dayjs';
|
||||
import utc from 'dayjs/plugin/utc';
|
||||
import timezone from 'dayjs/plugin/timezone';
|
||||
|
||||
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
|
||||
|
||||
dayjs.extend(utc);
|
||||
dayjs.extend(timezone);
|
||||
|
||||
const SMTP_HOST = process.env.SMTP_HOST;
|
||||
const SMTP_PORT = parseInt(process.env.SMTP_PORT || '465');
|
||||
const SMTP_USER = process.env.SMTP_USER;
|
||||
const SMTP_PASS = process.env.SMTP_PASS;
|
||||
|
||||
const TEST_EMAIL = 'timo@qrmaster.net';
|
||||
|
||||
interface OutreachEmail {
|
||||
id: number;
|
||||
tier: number;
|
||||
name: string;
|
||||
recipient: string;
|
||||
subject: string;
|
||||
body: string;
|
||||
language: 'DE' | 'EN';
|
||||
}
|
||||
|
||||
const EMAILS: OutreachEmail[] = [
|
||||
{
|
||||
id: 1,
|
||||
tier: 3,
|
||||
name: 'eEducation Austria',
|
||||
recipient: 'eeducation@bildung.gv.at',
|
||||
language: 'DE',
|
||||
subject: 'eTapas-Liste – Tool-Ergänzung',
|
||||
body: `Hallo,
|
||||
|
||||
in eurer eTapas-Liste fehlt bislang ein QR-Code-Generator. Lehrkräfte nutzen QR-Codes häufig, um Arbeitsblätter oder Hörbeispiele direkt zugänglich zu machen – Schüler scannen, fertig, kein Link-Tippen.
|
||||
|
||||
qrmaster.net ist werbefrei und für einfache Codes ohne Anmeldung nutzbar. Falls ein Ziel-Link später noch geändert werden muss, ohne neu zu drucken, bieten wir auch dynamische QR-Codes an (erfordert einen kostenlosen Account).
|
||||
|
||||
Entspricht das euren Kriterien für eine Aufnahme in die Liste?
|
||||
|
||||
Viele Grüße,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
tier: 1,
|
||||
name: 'Promethean World',
|
||||
recipient: 'info@prometheanworld.com',
|
||||
language: 'DE',
|
||||
subject: 'Ergänzung zu eurem Artikel über digitale Tools',
|
||||
body: `Hallo,
|
||||
|
||||
in eurem Beitrag zu digitalen Tools im Lehreralltag geht es viel ums Teilen von Inhalten – Whiteboards, Classroom-Workflows, digitale Materialien. Was oft fehlt: ein Weg, Links sekundenschnell auf Papier zugänglich zu machen.
|
||||
|
||||
QR-Codes lösen genau das. qrmaster.net erstellt sie kostenlos und für statische Codes ohne Anmeldung. Unsere dynamischen QR-Codes erlauben es sogar, den Ziel-Link später noch zu ändern, falls sich Materialien aktualisieren (erfordert einen einfachen Account-Login).
|
||||
|
||||
Wäre das eine sinnvolle Ergänzung für den Artikel?
|
||||
|
||||
Viele Grüße,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
tier: 2,
|
||||
name: 'The Startup Project',
|
||||
recipient: 'hello@startupproject.org',
|
||||
language: 'EN',
|
||||
subject: 'missing tool in your startup resources',
|
||||
body: `Hi,
|
||||
|
||||
Your startup resource list covers all the bases – product, funding, marketing. One gap I noticed: no QR code generator.
|
||||
|
||||
Founders use them more than they expect – product packaging, pitch decks, business cards. qrmaster.net is free and works without an account for static codes. For more flexibility, we also offer dynamic QR codes so you can update the destination later without reprinting (requires a free account).
|
||||
|
||||
Worth adding to the marketing tools section?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 4,
|
||||
tier: 1,
|
||||
name: 'Cadmium',
|
||||
recipient: 'jessie.reyes@gocadmium.com',
|
||||
language: 'EN',
|
||||
subject: 'one tool missing from your event management list',
|
||||
body: `Hi Jessie,
|
||||
|
||||
Your "10 best event management tools" covers the core stack well. One gap: no QR code generator.
|
||||
|
||||
Event planners use them for check-in flows, session schedules, and feedback forms. qrmaster.net handles bulk generation and includes scan analytics. While basic codes are account-free, our dynamic QR codes let planners update destinations post-printing (requires a free account).
|
||||
|
||||
Would it make sense to add alongside your existing recommendations?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 5,
|
||||
tier: 2,
|
||||
name: 'EssayGrader',
|
||||
recipient: 'support@essaygrader.ai',
|
||||
language: 'EN',
|
||||
subject: 'one gap in your 76-resource list',
|
||||
body: `Hi,
|
||||
|
||||
Your teacher resource list covers a lot of ground. One thing missing: a QR code generator.
|
||||
|
||||
Teachers use them to link printed handouts to digital content without students typing URLs. qrmaster.net is free and doesn't require an account for static codes. We also offer dynamic QR codes for those who need to update links later (requires a simple account setup).
|
||||
|
||||
Would it fit your inclusion criteria?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 6,
|
||||
tier: 3,
|
||||
name: 'Serkan Cagatay',
|
||||
recipient: 'info@serkancagatay.com',
|
||||
language: 'DE',
|
||||
subject: 'Tool-Tipp für deine Ressourcen-Liste',
|
||||
body: `Hallo Serkan,
|
||||
|
||||
auf deiner Seite mit didaktischen Ressourcen fehlt ein Weg, Audiodateien oder Notenlinks direkt im Unterricht zugänglich zu machen.
|
||||
|
||||
qrmaster.net macht genau das: QR-Code in Sekunden, ohne Account für einfache Codes. Falls du dynamische Codes brauchst, die man nachträglich umleiten kann, bieten wir das mit einem kostenlosen Account ebenfalls an.
|
||||
|
||||
Passt das in deine Sammlung?
|
||||
|
||||
Viele Grüße,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 7,
|
||||
tier: 1,
|
||||
name: 'U.S. Chamber',
|
||||
recipient: 'jmulvey@uschamber.com',
|
||||
language: 'EN',
|
||||
subject: 'free tool gap in your small business resources',
|
||||
body: `Hi Jeanette,
|
||||
|
||||
Your list of free resources for small businesses is excellent. One tool I'd expect to see: a reliable QR code generator.
|
||||
|
||||
Small business owners use them for menus, signage, and packaging. qrmaster.net is 100% free and account-free for static codes. Our dynamic QR codes allow owners to update destinations without reprinting, which helps avoid wasted materials (requires a free account).
|
||||
|
||||
Would it make sense to include it in the marketing tools section?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 8,
|
||||
tier: 1,
|
||||
name: 'Teaching Channel',
|
||||
recipient: 'support@teachingchannel.com',
|
||||
language: 'EN',
|
||||
subject: 'missing tool from your digital resources list',
|
||||
body: `Hi,
|
||||
|
||||
Your roundup of 10 digital resources covers a solid range. One tool that's consistently useful for bridging physical and digital: a QR code generator.
|
||||
|
||||
qrmaster.net works without an account for simple codes. We also provide dynamic QR codes for teachers who want to update the link destination later without reprinting materials (requires a free account).
|
||||
|
||||
Would it be a fit for your list?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 9,
|
||||
tier: 2,
|
||||
name: 'Teachers of Tomorrow',
|
||||
recipient: 'MediaRelations@TeachersofTomorrow.org',
|
||||
language: 'EN',
|
||||
subject: 'resource suggestion for new teachers',
|
||||
body: `Hi,
|
||||
|
||||
Your guide on teaching resources is a solid reference. One practical gap: no QR code generator.
|
||||
|
||||
QR codes help teachers set up learning stations and share links without a projector. qrmaster.net is free and account-free for static codes. For dynamic management—allowing links to be updated post-printing—we offer a dedicated dashboard (requires a free account).
|
||||
|
||||
Does it fit what you'd add to the guide?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 10,
|
||||
tier: 2,
|
||||
name: 'Montgomery College',
|
||||
recipient: 'CTL@montgomerycollege.edu',
|
||||
language: 'EN',
|
||||
subject: 'tool suggestion for your tech tools page',
|
||||
body: `Hi CTL Team,
|
||||
|
||||
Your "Tech Tools to Support Teaching and Learning" page is a useful reference. One tool missing: a straightforward QR code generator.
|
||||
|
||||
qrmaster.net is free and doesn't require an account for basic codes. We also support dynamic QR codes, which allow faculty to update link destinations mid-semester without reprinting handouts (requires a free account).
|
||||
|
||||
Would it be worth adding to your list?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 11,
|
||||
tier: 3,
|
||||
name: 'PHSG',
|
||||
recipient: 'info@phsg.ch',
|
||||
language: 'DE',
|
||||
subject: 'Ergänzung zu euren ICT-Ressourcen',
|
||||
body: `Hallo,
|
||||
|
||||
eure ICT-Ressourcen-Seite deckt Medien und Programmierung gut ab. Was noch fehlt: ein einfaches Werkzeug, um Links zu solchen Projekten direkt im Unterricht per QR-Code zu teilen.
|
||||
|
||||
qrmaster.net ist für einfache Codes ohne Anmeldung nutzbar. Mit unseren dynamischen QR-Codes lässt sich der Ziel-Link auch nachträglich noch ändern, falls sich ein Projekt verschiebt (erfordert einen kostenlosen Account).
|
||||
|
||||
Wäre das eine Ergänzung für eure Liste?
|
||||
|
||||
Herzliche Grüße,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 12,
|
||||
tier: 2,
|
||||
name: 'Super Monitoring',
|
||||
recipient: 'stuart@surges.co',
|
||||
language: 'EN',
|
||||
subject: 'tool your audience probably uses weekly',
|
||||
body: `Hi Stuart,
|
||||
|
||||
Your post on marketing tools is a great read. One category often overlooked: QR code generators.
|
||||
|
||||
Marketers use them for direct mail, packaging, and OOH. qrmaster.net provides bulk generation and scan analytics. Static codes are account-free, while our dynamic QR codes allow for post-printing updates (requires a free account).
|
||||
|
||||
Would it fit as a mention in the article?
|
||||
|
||||
Cheers,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 13,
|
||||
tier: 3,
|
||||
name: 'Kunstunterricht',
|
||||
recipient: 'simon@kunstunterricht-ideen.de',
|
||||
language: 'DE',
|
||||
subject: 'Tool-Tipp für deine Online-Ressourcen',
|
||||
body: `Hallo Simon,
|
||||
|
||||
auf deiner Ressourcen-Seite für Kunstunterricht fehlt ein praktisches Werkzeug: ein QR-Code-Generator. Damit können Lehrkräfte Links zu Tutorials oder digitalen Museen direkt auf Ausdrucke packen.
|
||||
|
||||
qrmaster.net ist für statische Codes ohne Anmeldung nutzbar. Dynamische QR-Codes erlauben nachträgliche Änderungen am Ziel-Link, falls sich die Quelle ändert (erfordert einen kostenlosen Account).
|
||||
|
||||
Wäre das etwas für deine Liste?
|
||||
|
||||
Kreative Grüße,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 14,
|
||||
tier: 1,
|
||||
name: 'TeachThought',
|
||||
recipient: 'terry@teachthought.com',
|
||||
language: 'EN',
|
||||
subject: 'gap in your 21 literacy resources',
|
||||
body: `Hi Terry,
|
||||
|
||||
Your 21 literacy resources list covers annotation and digital storytelling—but no easy way to bridge physical books with digital resources.
|
||||
|
||||
qrmaster.net is free and doesn't require an account for basic codes. We also offer dynamic QR codes, allowing teachers to update destinations post-printing as literacy plans evolve (requires a free account).
|
||||
|
||||
Would it fit alongside your existing literacy tools?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 15,
|
||||
tier: 1,
|
||||
name: 'Tripleseat',
|
||||
recipient: 'info@tripleseat.com',
|
||||
language: 'EN',
|
||||
subject: 'tool suggestion for your event planning resources',
|
||||
body: `Hi,
|
||||
|
||||
Your event planning resources page is a solid reference. One practical tool missing: a clean QR code generator.
|
||||
|
||||
Venue planners use them for digital menus and schedules. qrmaster.net supports bulk generation and scan analytics. While static codes are account-free, we also offer dynamic QR codes for updating destinations after printing (requires a free account).
|
||||
|
||||
Would it fit your resource page?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
}
|
||||
];
|
||||
|
||||
async function sendEmail(email: OutreachEmail, isTest: boolean) {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: SMTP_HOST,
|
||||
port: SMTP_PORT,
|
||||
secure: SMTP_PORT === 465,
|
||||
auth: {
|
||||
user: SMTP_USER,
|
||||
pass: SMTP_PASS,
|
||||
},
|
||||
});
|
||||
|
||||
const mailOptions = {
|
||||
from: `"Timo | qrmaster.net" <${SMTP_USER}>`,
|
||||
to: isTest ? TEST_EMAIL : email.recipient,
|
||||
subject: isTest ? `[TEST] ${email.subject}` : email.subject,
|
||||
text: email.body,
|
||||
};
|
||||
|
||||
try {
|
||||
const info = await transporter.sendMail(mailOptions);
|
||||
console.log(`[${email.id}] Email sent to ${mailOptions.to}: ${info.messageId}`);
|
||||
} catch (error) {
|
||||
console.error(`[${email.id}] Error sending email to ${mailOptions.to}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const args = process.argv.slice(2);
|
||||
const isTest = args.includes('--test');
|
||||
const isSchedule = args.includes('--schedule');
|
||||
const isDryRun = args.includes('--dry-run');
|
||||
|
||||
if (!isTest && !isSchedule && !isDryRun) {
|
||||
console.log('Usage: tsx outreach-resource-emails.ts [--test | --schedule | --dry-run]');
|
||||
return;
|
||||
}
|
||||
|
||||
if (isDryRun) {
|
||||
console.log('--- DRY RUN: Showing refined 15 emails ---');
|
||||
EMAILS.forEach(email => {
|
||||
console.log(`\n--- Target: ${email.name} (#${email.id}) ---`);
|
||||
console.log(`Subject: ${email.subject}`);
|
||||
console.log(`Body:\n${email.body}\n`);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (isTest) {
|
||||
console.log(`--- Sending 15 refined test emails to ${TEST_EMAIL} ---`);
|
||||
for (const email of EMAILS) {
|
||||
await sendEmail(email, true);
|
||||
}
|
||||
console.log('--- Test send completed ---');
|
||||
}
|
||||
|
||||
if (isSchedule) {
|
||||
const targetTime = dayjs().tz('Europe/Berlin').add(1, 'day').hour(15).minute(30).second(0).millisecond(0);
|
||||
const now = dayjs().tz('Europe/Berlin');
|
||||
|
||||
console.log(`Current Time (CEST): ${now.format('YYYY-MM-DD HH:mm:ss')}`);
|
||||
console.log(`Target Time (CEST): ${targetTime.format('YYYY-MM-DD HH:mm:ss')}`);
|
||||
|
||||
const waitMs = targetTime.diff(now);
|
||||
|
||||
if (waitMs <= 0) {
|
||||
console.error('Target time is in the past!');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Waiting ${Math.round(waitMs / 1000 / 60 / 60 * 10) / 10} hours...`);
|
||||
|
||||
setTimeout(async () => {
|
||||
console.log('--- Starting scheduled outreach ---');
|
||||
for (const email of EMAILS) {
|
||||
await sendEmail(email, false);
|
||||
}
|
||||
console.log('--- Scheduled outreach completed ---');
|
||||
}, waitMs);
|
||||
}
|
||||
}
|
||||
|
||||
run();
|
||||
import nodemailer from 'nodemailer';
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
import dayjs from 'dayjs';
|
||||
import utc from 'dayjs/plugin/utc';
|
||||
import timezone from 'dayjs/plugin/timezone';
|
||||
|
||||
dotenv.config({ path: path.resolve(process.cwd(), '.env') });
|
||||
|
||||
dayjs.extend(utc);
|
||||
dayjs.extend(timezone);
|
||||
|
||||
const SMTP_HOST = process.env.SMTP_HOST;
|
||||
const SMTP_PORT = parseInt(process.env.SMTP_PORT || '465');
|
||||
const SMTP_USER = process.env.SMTP_USER;
|
||||
const SMTP_PASS = process.env.SMTP_PASS;
|
||||
|
||||
const TEST_EMAIL = 'timo@qrmaster.net';
|
||||
|
||||
interface OutreachEmail {
|
||||
id: number;
|
||||
tier: number;
|
||||
name: string;
|
||||
recipient: string;
|
||||
subject: string;
|
||||
body: string;
|
||||
language: 'DE' | 'EN';
|
||||
}
|
||||
|
||||
const EMAILS: OutreachEmail[] = [
|
||||
{
|
||||
id: 1,
|
||||
tier: 3,
|
||||
name: 'eEducation Austria',
|
||||
recipient: 'eeducation@bildung.gv.at',
|
||||
language: 'DE',
|
||||
subject: 'eTapas-Liste – Tool-Ergänzung',
|
||||
body: `Hallo,
|
||||
|
||||
in eurer eTapas-Liste fehlt bislang ein QR-Code-Generator. Lehrkräfte nutzen QR-Codes häufig, um Arbeitsblätter oder Hörbeispiele direkt zugänglich zu machen – Schüler scannen, fertig, kein Link-Tippen.
|
||||
|
||||
qrmaster.net ist werbefrei und für einfache Codes ohne Anmeldung nutzbar. Falls ein Ziel-Link später noch geändert werden muss, ohne neu zu drucken, bieten wir auch dynamische QR-Codes an (erfordert einen kostenlosen Account).
|
||||
|
||||
Entspricht das euren Kriterien für eine Aufnahme in die Liste?
|
||||
|
||||
Viele Grüße,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
tier: 1,
|
||||
name: 'Promethean World',
|
||||
recipient: 'info@prometheanworld.com',
|
||||
language: 'DE',
|
||||
subject: 'Ergänzung zu eurem Artikel über digitale Tools',
|
||||
body: `Hallo,
|
||||
|
||||
in eurem Beitrag zu digitalen Tools im Lehreralltag geht es viel ums Teilen von Inhalten – Whiteboards, Classroom-Workflows, digitale Materialien. Was oft fehlt: ein Weg, Links sekundenschnell auf Papier zugänglich zu machen.
|
||||
|
||||
QR-Codes lösen genau das. qrmaster.net erstellt sie kostenlos und für statische Codes ohne Anmeldung. Unsere dynamischen QR-Codes erlauben es sogar, den Ziel-Link später noch zu ändern, falls sich Materialien aktualisieren (erfordert einen einfachen Account-Login).
|
||||
|
||||
Wäre das eine sinnvolle Ergänzung für den Artikel?
|
||||
|
||||
Viele Grüße,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
tier: 2,
|
||||
name: 'The Startup Project',
|
||||
recipient: 'hello@startupproject.org',
|
||||
language: 'EN',
|
||||
subject: 'missing tool in your startup resources',
|
||||
body: `Hi,
|
||||
|
||||
Your startup resource list covers all the bases – product, funding, marketing. One gap I noticed: no QR code generator.
|
||||
|
||||
Founders use them more than they expect – product packaging, pitch decks, business cards. qrmaster.net is free and works without an account for static codes. For more flexibility, we also offer dynamic QR codes so you can update the destination later without reprinting (requires a free account).
|
||||
|
||||
Worth adding to the marketing tools section?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 4,
|
||||
tier: 1,
|
||||
name: 'Cadmium',
|
||||
recipient: 'jessie.reyes@gocadmium.com',
|
||||
language: 'EN',
|
||||
subject: 'one tool missing from your event management list',
|
||||
body: `Hi Jessie,
|
||||
|
||||
Your "10 best event management tools" covers the core stack well. One gap: no QR code generator.
|
||||
|
||||
Event planners use them for check-in flows, session schedules, and feedback forms. qrmaster.net handles bulk generation and includes scan analytics. While basic codes are account-free, our dynamic QR codes let planners update destinations post-printing (requires a free account).
|
||||
|
||||
Would it make sense to add alongside your existing recommendations?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 5,
|
||||
tier: 2,
|
||||
name: 'EssayGrader',
|
||||
recipient: 'support@essaygrader.ai',
|
||||
language: 'EN',
|
||||
subject: 'one gap in your 76-resource list',
|
||||
body: `Hi,
|
||||
|
||||
Your teacher resource list covers a lot of ground. One thing missing: a QR code generator.
|
||||
|
||||
Teachers use them to link printed handouts to digital content without students typing URLs. qrmaster.net is free and doesn't require an account for static codes. We also offer dynamic QR codes for those who need to update links later (requires a simple account setup).
|
||||
|
||||
Would it fit your inclusion criteria?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 6,
|
||||
tier: 3,
|
||||
name: 'Serkan Cagatay',
|
||||
recipient: 'info@serkancagatay.com',
|
||||
language: 'DE',
|
||||
subject: 'Tool-Tipp für deine Ressourcen-Liste',
|
||||
body: `Hallo Serkan,
|
||||
|
||||
auf deiner Seite mit didaktischen Ressourcen fehlt ein Weg, Audiodateien oder Notenlinks direkt im Unterricht zugänglich zu machen.
|
||||
|
||||
qrmaster.net macht genau das: QR-Code in Sekunden, ohne Account für einfache Codes. Falls du dynamische Codes brauchst, die man nachträglich umleiten kann, bieten wir das mit einem kostenlosen Account ebenfalls an.
|
||||
|
||||
Passt das in deine Sammlung?
|
||||
|
||||
Viele Grüße,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 7,
|
||||
tier: 1,
|
||||
name: 'U.S. Chamber',
|
||||
recipient: 'jmulvey@uschamber.com',
|
||||
language: 'EN',
|
||||
subject: 'free tool gap in your small business resources',
|
||||
body: `Hi Jeanette,
|
||||
|
||||
Your list of free resources for small businesses is excellent. One tool I'd expect to see: a reliable QR code generator.
|
||||
|
||||
Small business owners use them for menus, signage, and packaging. qrmaster.net is 100% free and account-free for static codes. Our dynamic QR codes allow owners to update destinations without reprinting, which helps avoid wasted materials (requires a free account).
|
||||
|
||||
Would it make sense to include it in the marketing tools section?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 8,
|
||||
tier: 1,
|
||||
name: 'Teaching Channel',
|
||||
recipient: 'support@teachingchannel.com',
|
||||
language: 'EN',
|
||||
subject: 'missing tool from your digital resources list',
|
||||
body: `Hi,
|
||||
|
||||
Your roundup of 10 digital resources covers a solid range. One tool that's consistently useful for bridging physical and digital: a QR code generator.
|
||||
|
||||
qrmaster.net works without an account for simple codes. We also provide dynamic QR codes for teachers who want to update the link destination later without reprinting materials (requires a free account).
|
||||
|
||||
Would it be a fit for your list?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 9,
|
||||
tier: 2,
|
||||
name: 'Teachers of Tomorrow',
|
||||
recipient: 'MediaRelations@TeachersofTomorrow.org',
|
||||
language: 'EN',
|
||||
subject: 'resource suggestion for new teachers',
|
||||
body: `Hi,
|
||||
|
||||
Your guide on teaching resources is a solid reference. One practical gap: no QR code generator.
|
||||
|
||||
QR codes help teachers set up learning stations and share links without a projector. qrmaster.net is free and account-free for static codes. For dynamic management—allowing links to be updated post-printing—we offer a dedicated dashboard (requires a free account).
|
||||
|
||||
Does it fit what you'd add to the guide?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 10,
|
||||
tier: 2,
|
||||
name: 'Montgomery College',
|
||||
recipient: 'CTL@montgomerycollege.edu',
|
||||
language: 'EN',
|
||||
subject: 'tool suggestion for your tech tools page',
|
||||
body: `Hi CTL Team,
|
||||
|
||||
Your "Tech Tools to Support Teaching and Learning" page is a useful reference. One tool missing: a straightforward QR code generator.
|
||||
|
||||
qrmaster.net is free and doesn't require an account for basic codes. We also support dynamic QR codes, which allow faculty to update link destinations mid-semester without reprinting handouts (requires a free account).
|
||||
|
||||
Would it be worth adding to your list?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 11,
|
||||
tier: 3,
|
||||
name: 'PHSG',
|
||||
recipient: 'info@phsg.ch',
|
||||
language: 'DE',
|
||||
subject: 'Ergänzung zu euren ICT-Ressourcen',
|
||||
body: `Hallo,
|
||||
|
||||
eure ICT-Ressourcen-Seite deckt Medien und Programmierung gut ab. Was noch fehlt: ein einfaches Werkzeug, um Links zu solchen Projekten direkt im Unterricht per QR-Code zu teilen.
|
||||
|
||||
qrmaster.net ist für einfache Codes ohne Anmeldung nutzbar. Mit unseren dynamischen QR-Codes lässt sich der Ziel-Link auch nachträglich noch ändern, falls sich ein Projekt verschiebt (erfordert einen kostenlosen Account).
|
||||
|
||||
Wäre das eine Ergänzung für eure Liste?
|
||||
|
||||
Herzliche Grüße,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 12,
|
||||
tier: 2,
|
||||
name: 'Super Monitoring',
|
||||
recipient: 'stuart@surges.co',
|
||||
language: 'EN',
|
||||
subject: 'tool your audience probably uses weekly',
|
||||
body: `Hi Stuart,
|
||||
|
||||
Your post on marketing tools is a great read. One category often overlooked: QR code generators.
|
||||
|
||||
Marketers use them for direct mail, packaging, and OOH. qrmaster.net provides bulk generation and scan analytics. Static codes are account-free, while our dynamic QR codes allow for post-printing updates (requires a free account).
|
||||
|
||||
Would it fit as a mention in the article?
|
||||
|
||||
Cheers,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 13,
|
||||
tier: 3,
|
||||
name: 'Kunstunterricht',
|
||||
recipient: 'simon@kunstunterricht-ideen.de',
|
||||
language: 'DE',
|
||||
subject: 'Tool-Tipp für deine Online-Ressourcen',
|
||||
body: `Hallo Simon,
|
||||
|
||||
auf deiner Ressourcen-Seite für Kunstunterricht fehlt ein praktisches Werkzeug: ein QR-Code-Generator. Damit können Lehrkräfte Links zu Tutorials oder digitalen Museen direkt auf Ausdrucke packen.
|
||||
|
||||
qrmaster.net ist für statische Codes ohne Anmeldung nutzbar. Dynamische QR-Codes erlauben nachträgliche Änderungen am Ziel-Link, falls sich die Quelle ändert (erfordert einen kostenlosen Account).
|
||||
|
||||
Wäre das etwas für deine Liste?
|
||||
|
||||
Kreative Grüße,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 14,
|
||||
tier: 1,
|
||||
name: 'TeachThought',
|
||||
recipient: 'terry@teachthought.com',
|
||||
language: 'EN',
|
||||
subject: 'gap in your 21 literacy resources',
|
||||
body: `Hi Terry,
|
||||
|
||||
Your 21 literacy resources list covers annotation and digital storytelling—but no easy way to bridge physical books with digital resources.
|
||||
|
||||
qrmaster.net is free and doesn't require an account for basic codes. We also offer dynamic QR codes, allowing teachers to update destinations post-printing as literacy plans evolve (requires a free account).
|
||||
|
||||
Would it fit alongside your existing literacy tools?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
},
|
||||
{
|
||||
id: 15,
|
||||
tier: 1,
|
||||
name: 'Tripleseat',
|
||||
recipient: 'info@tripleseat.com',
|
||||
language: 'EN',
|
||||
subject: 'tool suggestion for your event planning resources',
|
||||
body: `Hi,
|
||||
|
||||
Your event planning resources page is a solid reference. One practical tool missing: a clean QR code generator.
|
||||
|
||||
Venue planners use them for digital menus and schedules. qrmaster.net supports bulk generation and scan analytics. While static codes are account-free, we also offer dynamic QR codes for updating destinations after printing (requires a free account).
|
||||
|
||||
Would it fit your resource page?
|
||||
|
||||
Best,
|
||||
Timo | Founder, qrmaster.net`
|
||||
}
|
||||
];
|
||||
|
||||
async function sendEmail(email: OutreachEmail, isTest: boolean) {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: SMTP_HOST,
|
||||
port: SMTP_PORT,
|
||||
secure: SMTP_PORT === 465,
|
||||
auth: {
|
||||
user: SMTP_USER,
|
||||
pass: SMTP_PASS,
|
||||
},
|
||||
});
|
||||
|
||||
const mailOptions = {
|
||||
from: `"Timo | qrmaster.net" <${SMTP_USER}>`,
|
||||
to: isTest ? TEST_EMAIL : email.recipient,
|
||||
subject: isTest ? `[TEST] ${email.subject}` : email.subject,
|
||||
text: email.body,
|
||||
};
|
||||
|
||||
try {
|
||||
const info = await transporter.sendMail(mailOptions);
|
||||
console.log(`[${email.id}] Email sent to ${mailOptions.to}: ${info.messageId}`);
|
||||
} catch (error) {
|
||||
console.error(`[${email.id}] Error sending email to ${mailOptions.to}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const args = process.argv.slice(2);
|
||||
const isTest = args.includes('--test');
|
||||
const isSchedule = args.includes('--schedule');
|
||||
const isDryRun = args.includes('--dry-run');
|
||||
|
||||
if (!isTest && !isSchedule && !isDryRun) {
|
||||
console.log('Usage: tsx outreach-resource-emails.ts [--test | --schedule | --dry-run]');
|
||||
return;
|
||||
}
|
||||
|
||||
if (isDryRun) {
|
||||
console.log('--- DRY RUN: Showing refined 15 emails ---');
|
||||
EMAILS.forEach(email => {
|
||||
console.log(`\n--- Target: ${email.name} (#${email.id}) ---`);
|
||||
console.log(`Subject: ${email.subject}`);
|
||||
console.log(`Body:\n${email.body}\n`);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (isTest) {
|
||||
console.log(`--- Sending 15 refined test emails to ${TEST_EMAIL} ---`);
|
||||
for (const email of EMAILS) {
|
||||
await sendEmail(email, true);
|
||||
}
|
||||
console.log('--- Test send completed ---');
|
||||
}
|
||||
|
||||
if (isSchedule) {
|
||||
const targetTime = dayjs().tz('Europe/Berlin').add(1, 'day').hour(15).minute(30).second(0).millisecond(0);
|
||||
const now = dayjs().tz('Europe/Berlin');
|
||||
|
||||
console.log(`Current Time (CEST): ${now.format('YYYY-MM-DD HH:mm:ss')}`);
|
||||
console.log(`Target Time (CEST): ${targetTime.format('YYYY-MM-DD HH:mm:ss')}`);
|
||||
|
||||
const waitMs = targetTime.diff(now);
|
||||
|
||||
if (waitMs <= 0) {
|
||||
console.error('Target time is in the past!');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Waiting ${Math.round(waitMs / 1000 / 60 / 60 * 10) / 10} hours...`);
|
||||
|
||||
setTimeout(async () => {
|
||||
console.log('--- Starting scheduled outreach ---');
|
||||
for (const email of EMAILS) {
|
||||
await sendEmail(email, false);
|
||||
}
|
||||
console.log('--- Scheduled outreach completed ---');
|
||||
}, waitMs);
|
||||
}
|
||||
}
|
||||
|
||||
run();
|
||||
|
||||
@@ -1,421 +1,421 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
const OUTPUT_DIR = path.resolve(process.cwd(), 'output', 'outreach');
|
||||
const TARGET_PER_NICHE = Number(process.env.LEADS_PER_NICHE || 200);
|
||||
const CONCURRENCY = Number(process.env.LEAD_FETCH_CONCURRENCY || 8);
|
||||
const OVERPASS_DELAY_MS = Number(process.env.OVERPASS_DELAY_MS || 20000);
|
||||
const OVERPASS_429_DELAY_MS = Number(process.env.OVERPASS_429_DELAY_MS || 90000);
|
||||
const OVERPASS_MAX_ATTEMPTS = Number(process.env.OVERPASS_MAX_ATTEMPTS || 6);
|
||||
const OVERPASS_URLS = [
|
||||
'https://overpass-api.de/api/interpreter',
|
||||
];
|
||||
|
||||
const metros = [
|
||||
['New York', 'NY', 40.7128, -74.006],
|
||||
['Los Angeles', 'CA', 34.0522, -118.2437],
|
||||
['Chicago', 'IL', 41.8781, -87.6298],
|
||||
['Houston', 'TX', 29.7604, -95.3698],
|
||||
['Phoenix', 'AZ', 33.4484, -112.074],
|
||||
['Philadelphia', 'PA', 39.9526, -75.1652],
|
||||
['San Antonio', 'TX', 29.4241, -98.4936],
|
||||
['San Diego', 'CA', 32.7157, -117.1611],
|
||||
['Dallas', 'TX', 32.7767, -96.797],
|
||||
['San Jose', 'CA', 37.3382, -121.8863],
|
||||
['Austin', 'TX', 30.2672, -97.7431],
|
||||
['Jacksonville', 'FL', 30.3322, -81.6557],
|
||||
['Fort Worth', 'TX', 32.7555, -97.3308],
|
||||
['Columbus', 'OH', 39.9612, -82.9988],
|
||||
['Charlotte', 'NC', 35.2271, -80.8431],
|
||||
['San Francisco', 'CA', 37.7749, -122.4194],
|
||||
['Seattle', 'WA', 47.6062, -122.3321],
|
||||
['Denver', 'CO', 39.7392, -104.9903],
|
||||
['Miami', 'FL', 25.7617, -80.1918],
|
||||
['Nashville', 'TN', 36.1627, -86.7816],
|
||||
];
|
||||
|
||||
const niches = [
|
||||
{
|
||||
id: 'photographers',
|
||||
label: 'Photographers',
|
||||
targetUseCase: 'portfolio, booking, print cards, event galleries',
|
||||
queries: [
|
||||
['craft', 'photographer'],
|
||||
['shop', 'photo_studio'],
|
||||
['shop', 'photo'],
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'restaurants',
|
||||
label: 'Restaurants',
|
||||
targetUseCase: 'menu QR codes, table tents, review QR codes, coupons',
|
||||
queries: [
|
||||
['amenity', 'restaurant'],
|
||||
['amenity', 'cafe'],
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'real_estate',
|
||||
label: 'Real Estate',
|
||||
targetUseCase: 'yard signs, flyers, open houses, property sheets',
|
||||
queries: [
|
||||
['office', 'estate_agent'],
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'events_venues',
|
||||
label: 'Events & Venues',
|
||||
targetUseCase: 'tickets, schedules, check-in, feedback and post-event links',
|
||||
queries: [
|
||||
['amenity', 'events_venue'],
|
||||
['amenity', 'theatre'],
|
||||
['amenity', 'conference_centre'],
|
||||
['tourism', 'attraction'],
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'wellness_beauty',
|
||||
label: 'Wellness & Beauty',
|
||||
targetUseCase: 'booking links, price lists, reviews, loyalty offers',
|
||||
queries: [
|
||||
['shop', 'beauty'],
|
||||
['shop', 'hairdresser'],
|
||||
['leisure', 'fitness_centre'],
|
||||
['amenity', 'spa'],
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function csvEscape(value) {
|
||||
const text = String(value ?? '');
|
||||
if (/[",\n\r]/.test(text)) {
|
||||
return `"${text.replaceAll('"', '""')}"`;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
function normalizeWebsite(raw) {
|
||||
if (!raw) return '';
|
||||
let value = String(raw).trim();
|
||||
if (!value) return '';
|
||||
if (value.startsWith('mailto:') || value.includes('@') && !value.includes('/')) return '';
|
||||
if (!/^https?:\/\//i.test(value)) value = `https://${value}`;
|
||||
try {
|
||||
const url = new URL(value);
|
||||
if (!url.hostname.includes('.')) return '';
|
||||
url.hash = '';
|
||||
return url.toString().replace(/\/$/, '');
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function getTag(tags, names) {
|
||||
for (const name of names) {
|
||||
if (tags?.[name]) return tags[name];
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function buildOverpassQuery(niche, metro, offset) {
|
||||
const [, , lat, lon] = metro;
|
||||
const radius = 25000 + offset * 10000;
|
||||
const clauses = niche.queries.flatMap(([key, value]) => [
|
||||
`nwr(around:${radius},${lat},${lon})["${key}"="${value}"]["website"];`,
|
||||
`nwr(around:${radius},${lat},${lon})["${key}"="${value}"]["contact:website"];`,
|
||||
`nwr(around:${radius},${lat},${lon})["${key}"="${value}"]["email"];`,
|
||||
`nwr(around:${radius},${lat},${lon})["${key}"="${value}"]["contact:email"];`,
|
||||
]).join('\n');
|
||||
|
||||
return `[out:json][timeout:45];
|
||||
(
|
||||
${clauses}
|
||||
);
|
||||
out tags center ${Math.min(TARGET_PER_NICHE * 2, 500)};`;
|
||||
}
|
||||
|
||||
async function fetchOverpass(query, attempt = 0) {
|
||||
const endpoint = OVERPASS_URLS[attempt % OVERPASS_URLS.length];
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 90000);
|
||||
try {
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8' },
|
||||
body: new URLSearchParams({ data: query }),
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
if (response.status === 429 && attempt < OVERPASS_MAX_ATTEMPTS) {
|
||||
const waitMs = OVERPASS_429_DELAY_MS + attempt * 30000;
|
||||
console.warn(`Overpass rate limited; waiting ${Math.round(waitMs / 1000)}s before retry ${attempt + 1}/${OVERPASS_MAX_ATTEMPTS}`);
|
||||
await sleep(waitMs);
|
||||
return fetchOverpass(query, attempt + 1);
|
||||
}
|
||||
if (attempt < OVERPASS_MAX_ATTEMPTS) {
|
||||
await sleep(5000 * (attempt + 1));
|
||||
return fetchOverpass(query, attempt + 1);
|
||||
}
|
||||
throw new Error(`Overpass ${response.status} ${response.statusText}`);
|
||||
}
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
if (attempt < OVERPASS_MAX_ATTEMPTS) {
|
||||
await sleep(5000 * (attempt + 1));
|
||||
return fetchOverpass(query, attempt + 1);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
function elementToLead(element, niche, metro) {
|
||||
const tags = element.tags || {};
|
||||
const website = normalizeWebsite(getTag(tags, ['contact:website', 'website', 'url']));
|
||||
const email = getTag(tags, ['contact:email', 'email']);
|
||||
const phone = getTag(tags, ['contact:phone', 'phone']);
|
||||
const street = [tags['addr:housenumber'], tags['addr:street']].filter(Boolean).join(' ');
|
||||
const city = tags['addr:city'] || metro[0];
|
||||
const state = tags['addr:state'] || metro[1];
|
||||
|
||||
return {
|
||||
niche: niche.id,
|
||||
niche_label: niche.label,
|
||||
company: tags.name || '',
|
||||
website,
|
||||
email,
|
||||
phone,
|
||||
city,
|
||||
state,
|
||||
country: 'US',
|
||||
street,
|
||||
source: 'OpenStreetMap Overpass',
|
||||
source_id: `${element.type}/${element.id}`,
|
||||
source_url: `https://www.openstreetmap.org/${element.type}/${element.id}`,
|
||||
personalization_signal: '',
|
||||
qr_use_case: niche.targetUseCase,
|
||||
lead_score: 0,
|
||||
email_source: email ? 'osm' : '',
|
||||
opt_out_required: 'yes',
|
||||
};
|
||||
}
|
||||
|
||||
function visibleTextEmails(text) {
|
||||
const normalized = text
|
||||
.replaceAll('[at]', '@')
|
||||
.replaceAll('(at)', '@')
|
||||
.replaceAll(' at ', '@')
|
||||
.replaceAll('[dot]', '.')
|
||||
.replaceAll('(dot)', '.')
|
||||
.replaceAll(' dot ', '.');
|
||||
const matches = normalized.match(/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g) || [];
|
||||
return [...new Set(matches.map((email) => email.toLowerCase()))]
|
||||
.filter((email) => !email.endsWith('.png') && !email.endsWith('.jpg') && !email.includes('example.com'))
|
||||
.filter((email) => !email.includes('wixpress.com') && !email.includes('sentry.io'));
|
||||
}
|
||||
|
||||
function extractContactLinks(html, baseUrl) {
|
||||
const links = [];
|
||||
const regex = /href=["']([^"']+)["']/gi;
|
||||
let match;
|
||||
while ((match = regex.exec(html))) {
|
||||
const href = match[1];
|
||||
if (/^(mailto:|tel:)/i.test(href)) continue;
|
||||
if (!/(contact|about|team|booking|book|wedding|private-events|catering|visit|location)/i.test(href)) continue;
|
||||
try {
|
||||
const url = new URL(href, baseUrl);
|
||||
if (url.hostname === new URL(baseUrl).hostname) {
|
||||
url.hash = '';
|
||||
links.push(url.toString());
|
||||
}
|
||||
} catch {
|
||||
// Ignore malformed links.
|
||||
}
|
||||
}
|
||||
return [...new Set(links)].slice(0, 3);
|
||||
}
|
||||
|
||||
async function fetchText(url) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 10000);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'user-agent': 'QR Master lead research bot (+https://qrmaster.net/contact)',
|
||||
accept: 'text/html,application/xhtml+xml',
|
||||
},
|
||||
signal: controller.signal,
|
||||
redirect: 'follow',
|
||||
});
|
||||
if (!response.ok) return '';
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
if (!contentType.includes('text/html')) return '';
|
||||
return await response.text();
|
||||
} catch {
|
||||
return '';
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function enrichLead(lead) {
|
||||
if (!lead.website || lead.email) {
|
||||
return scoreLead(lead);
|
||||
}
|
||||
|
||||
const homepage = await fetchText(lead.website);
|
||||
const emails = visibleTextEmails(homepage);
|
||||
const contactLinks = extractContactLinks(homepage, lead.website);
|
||||
|
||||
for (const link of contactLinks) {
|
||||
if (emails.length > 0) break;
|
||||
const html = await fetchText(link);
|
||||
emails.push(...visibleTextEmails(html));
|
||||
}
|
||||
|
||||
const uniqueEmails = [...new Set(emails)];
|
||||
if (uniqueEmails.length > 0) {
|
||||
lead.email = uniqueEmails[0];
|
||||
lead.email_source = 'website';
|
||||
}
|
||||
|
||||
return scoreLead(lead);
|
||||
}
|
||||
|
||||
function scoreLead(lead) {
|
||||
let score = 30;
|
||||
if (lead.website) score += 20;
|
||||
if (lead.email) score += 30;
|
||||
if (lead.phone) score += 5;
|
||||
if (!/(gmail|yahoo|hotmail|outlook|icloud)\.com$/i.test(lead.email || '')) score += lead.email ? 10 : 0;
|
||||
if (lead.niche === 'real_estate' || lead.niche === 'restaurants') score += 5;
|
||||
|
||||
const signalByNiche = {
|
||||
photographers: `${lead.company} can use dynamic QR codes on print cards, gallery cards, event handouts, and portfolio links.`,
|
||||
restaurants: `${lead.company} can use dynamic QR codes for menus, table tents, reviews, coupons, and seasonal specials.`,
|
||||
real_estate: `${lead.company} can use dynamic QR codes on yard signs, flyers, property sheets, and open house material.`,
|
||||
events_venues: `${lead.company} can use dynamic QR codes for schedules, ticketing, venue maps, check-in, and post-event feedback.`,
|
||||
wellness_beauty: `${lead.company} can use dynamic QR codes for booking pages, service menus, price lists, reviews, and loyalty offers.`,
|
||||
};
|
||||
|
||||
lead.lead_score = Math.min(score, 100);
|
||||
lead.personalization_signal = signalByNiche[lead.niche] || '';
|
||||
return lead;
|
||||
}
|
||||
|
||||
async function mapLimit(items, limit, mapper) {
|
||||
const results = [];
|
||||
let index = 0;
|
||||
async function worker() {
|
||||
while (index < items.length) {
|
||||
const current = index++;
|
||||
results[current] = await mapper(items[current], current);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
||||
return results;
|
||||
}
|
||||
|
||||
async function collectNiche(niche) {
|
||||
const leadsByKey = new Map();
|
||||
for (let pass = 0; pass < 2 && leadsByKey.size < TARGET_PER_NICHE * 2; pass++) {
|
||||
for (const metro of metros) {
|
||||
if (leadsByKey.size >= TARGET_PER_NICHE * 2) break;
|
||||
const query = buildOverpassQuery(niche, metro, pass);
|
||||
try {
|
||||
const data = await fetchOverpass(query);
|
||||
for (const element of data.elements || []) {
|
||||
const lead = elementToLead(element, niche, metro);
|
||||
if (!lead.company) continue;
|
||||
if (!lead.website && !lead.email) continue;
|
||||
const key = lead.website || `${lead.company}|${lead.city}|${lead.state}`.toLowerCase();
|
||||
if (!leadsByKey.has(key)) leadsByKey.set(key, lead);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`[${niche.id}] ${metro[0]} skipped: ${error.message}`);
|
||||
}
|
||||
await sleep(OVERPASS_DELAY_MS);
|
||||
}
|
||||
}
|
||||
|
||||
const rawLeads = [...leadsByKey.values()].slice(0, TARGET_PER_NICHE * 2);
|
||||
console.log(`[${niche.id}] collected ${rawLeads.length}; enriching...`);
|
||||
const enriched = await mapLimit(rawLeads, CONCURRENCY, enrichLead);
|
||||
return enriched
|
||||
.filter((lead) => lead.website || lead.email)
|
||||
.sort((a, b) => b.lead_score - a.lead_score)
|
||||
.slice(0, TARGET_PER_NICHE);
|
||||
}
|
||||
|
||||
function toCsv(leads) {
|
||||
const headers = [
|
||||
'niche',
|
||||
'niche_label',
|
||||
'company',
|
||||
'website',
|
||||
'email',
|
||||
'email_source',
|
||||
'phone',
|
||||
'city',
|
||||
'state',
|
||||
'country',
|
||||
'street',
|
||||
'lead_score',
|
||||
'qr_use_case',
|
||||
'personalization_signal',
|
||||
'source',
|
||||
'source_id',
|
||||
'source_url',
|
||||
'opt_out_required',
|
||||
];
|
||||
return [
|
||||
headers.join(','),
|
||||
...leads.map((lead) => headers.map((header) => csvEscape(lead[header])).join(',')),
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await fs.mkdir(OUTPUT_DIR, { recursive: true });
|
||||
const allLeads = [];
|
||||
for (const niche of niches) {
|
||||
const leads = await collectNiche(niche);
|
||||
allLeads.push(...leads);
|
||||
const dated = new Date().toISOString().slice(0, 10);
|
||||
await fs.writeFile(path.join(OUTPUT_DIR, `qrmaster-us-leads-${niche.id}-${dated}.csv`), toCsv(leads), 'utf8');
|
||||
await fs.writeFile(path.join(OUTPUT_DIR, `qrmaster-us-leads-${niche.id}-${dated}.json`), JSON.stringify(leads, null, 2), 'utf8');
|
||||
console.log(`[${niche.id}] kept ${leads.length}`);
|
||||
}
|
||||
|
||||
const byKey = new Map();
|
||||
for (const lead of allLeads) {
|
||||
const key = lead.email || lead.website || `${lead.company}|${lead.city}|${lead.state}`.toLowerCase();
|
||||
if (!byKey.has(key)) byKey.set(key, lead);
|
||||
}
|
||||
const deduped = [...byKey.values()].sort((a, b) => b.lead_score - a.lead_score);
|
||||
const dated = new Date().toISOString().slice(0, 10);
|
||||
const csvPath = path.join(OUTPUT_DIR, `qrmaster-us-leads-${dated}.csv`);
|
||||
const jsonPath = path.join(OUTPUT_DIR, `qrmaster-us-leads-${dated}.json`);
|
||||
await fs.writeFile(csvPath, toCsv(deduped), 'utf8');
|
||||
await fs.writeFile(jsonPath, JSON.stringify(deduped, null, 2), 'utf8');
|
||||
|
||||
const summary = niches.map((niche) => {
|
||||
const leads = deduped.filter((lead) => lead.niche === niche.id);
|
||||
const withEmail = leads.filter((lead) => lead.email).length;
|
||||
return `${niche.label}: ${leads.length} leads, ${withEmail} emails`;
|
||||
}).join('\n');
|
||||
|
||||
console.log(`\nWrote ${deduped.length} leads`);
|
||||
console.log(csvPath);
|
||||
console.log(jsonPath);
|
||||
console.log(summary);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
const OUTPUT_DIR = path.resolve(process.cwd(), 'output', 'outreach');
|
||||
const TARGET_PER_NICHE = Number(process.env.LEADS_PER_NICHE || 200);
|
||||
const CONCURRENCY = Number(process.env.LEAD_FETCH_CONCURRENCY || 8);
|
||||
const OVERPASS_DELAY_MS = Number(process.env.OVERPASS_DELAY_MS || 20000);
|
||||
const OVERPASS_429_DELAY_MS = Number(process.env.OVERPASS_429_DELAY_MS || 90000);
|
||||
const OVERPASS_MAX_ATTEMPTS = Number(process.env.OVERPASS_MAX_ATTEMPTS || 6);
|
||||
const OVERPASS_URLS = [
|
||||
'https://overpass-api.de/api/interpreter',
|
||||
];
|
||||
|
||||
const metros = [
|
||||
['New York', 'NY', 40.7128, -74.006],
|
||||
['Los Angeles', 'CA', 34.0522, -118.2437],
|
||||
['Chicago', 'IL', 41.8781, -87.6298],
|
||||
['Houston', 'TX', 29.7604, -95.3698],
|
||||
['Phoenix', 'AZ', 33.4484, -112.074],
|
||||
['Philadelphia', 'PA', 39.9526, -75.1652],
|
||||
['San Antonio', 'TX', 29.4241, -98.4936],
|
||||
['San Diego', 'CA', 32.7157, -117.1611],
|
||||
['Dallas', 'TX', 32.7767, -96.797],
|
||||
['San Jose', 'CA', 37.3382, -121.8863],
|
||||
['Austin', 'TX', 30.2672, -97.7431],
|
||||
['Jacksonville', 'FL', 30.3322, -81.6557],
|
||||
['Fort Worth', 'TX', 32.7555, -97.3308],
|
||||
['Columbus', 'OH', 39.9612, -82.9988],
|
||||
['Charlotte', 'NC', 35.2271, -80.8431],
|
||||
['San Francisco', 'CA', 37.7749, -122.4194],
|
||||
['Seattle', 'WA', 47.6062, -122.3321],
|
||||
['Denver', 'CO', 39.7392, -104.9903],
|
||||
['Miami', 'FL', 25.7617, -80.1918],
|
||||
['Nashville', 'TN', 36.1627, -86.7816],
|
||||
];
|
||||
|
||||
const niches = [
|
||||
{
|
||||
id: 'photographers',
|
||||
label: 'Photographers',
|
||||
targetUseCase: 'portfolio, booking, print cards, event galleries',
|
||||
queries: [
|
||||
['craft', 'photographer'],
|
||||
['shop', 'photo_studio'],
|
||||
['shop', 'photo'],
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'restaurants',
|
||||
label: 'Restaurants',
|
||||
targetUseCase: 'menu QR codes, table tents, review QR codes, coupons',
|
||||
queries: [
|
||||
['amenity', 'restaurant'],
|
||||
['amenity', 'cafe'],
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'real_estate',
|
||||
label: 'Real Estate',
|
||||
targetUseCase: 'yard signs, flyers, open houses, property sheets',
|
||||
queries: [
|
||||
['office', 'estate_agent'],
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'events_venues',
|
||||
label: 'Events & Venues',
|
||||
targetUseCase: 'tickets, schedules, check-in, feedback and post-event links',
|
||||
queries: [
|
||||
['amenity', 'events_venue'],
|
||||
['amenity', 'theatre'],
|
||||
['amenity', 'conference_centre'],
|
||||
['tourism', 'attraction'],
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'wellness_beauty',
|
||||
label: 'Wellness & Beauty',
|
||||
targetUseCase: 'booking links, price lists, reviews, loyalty offers',
|
||||
queries: [
|
||||
['shop', 'beauty'],
|
||||
['shop', 'hairdresser'],
|
||||
['leisure', 'fitness_centre'],
|
||||
['amenity', 'spa'],
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function csvEscape(value) {
|
||||
const text = String(value ?? '');
|
||||
if (/[",\n\r]/.test(text)) {
|
||||
return `"${text.replaceAll('"', '""')}"`;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
function normalizeWebsite(raw) {
|
||||
if (!raw) return '';
|
||||
let value = String(raw).trim();
|
||||
if (!value) return '';
|
||||
if (value.startsWith('mailto:') || value.includes('@') && !value.includes('/')) return '';
|
||||
if (!/^https?:\/\//i.test(value)) value = `https://${value}`;
|
||||
try {
|
||||
const url = new URL(value);
|
||||
if (!url.hostname.includes('.')) return '';
|
||||
url.hash = '';
|
||||
return url.toString().replace(/\/$/, '');
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function getTag(tags, names) {
|
||||
for (const name of names) {
|
||||
if (tags?.[name]) return tags[name];
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function buildOverpassQuery(niche, metro, offset) {
|
||||
const [, , lat, lon] = metro;
|
||||
const radius = 25000 + offset * 10000;
|
||||
const clauses = niche.queries.flatMap(([key, value]) => [
|
||||
`nwr(around:${radius},${lat},${lon})["${key}"="${value}"]["website"];`,
|
||||
`nwr(around:${radius},${lat},${lon})["${key}"="${value}"]["contact:website"];`,
|
||||
`nwr(around:${radius},${lat},${lon})["${key}"="${value}"]["email"];`,
|
||||
`nwr(around:${radius},${lat},${lon})["${key}"="${value}"]["contact:email"];`,
|
||||
]).join('\n');
|
||||
|
||||
return `[out:json][timeout:45];
|
||||
(
|
||||
${clauses}
|
||||
);
|
||||
out tags center ${Math.min(TARGET_PER_NICHE * 2, 500)};`;
|
||||
}
|
||||
|
||||
async function fetchOverpass(query, attempt = 0) {
|
||||
const endpoint = OVERPASS_URLS[attempt % OVERPASS_URLS.length];
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 90000);
|
||||
try {
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8' },
|
||||
body: new URLSearchParams({ data: query }),
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
if (response.status === 429 && attempt < OVERPASS_MAX_ATTEMPTS) {
|
||||
const waitMs = OVERPASS_429_DELAY_MS + attempt * 30000;
|
||||
console.warn(`Overpass rate limited; waiting ${Math.round(waitMs / 1000)}s before retry ${attempt + 1}/${OVERPASS_MAX_ATTEMPTS}`);
|
||||
await sleep(waitMs);
|
||||
return fetchOverpass(query, attempt + 1);
|
||||
}
|
||||
if (attempt < OVERPASS_MAX_ATTEMPTS) {
|
||||
await sleep(5000 * (attempt + 1));
|
||||
return fetchOverpass(query, attempt + 1);
|
||||
}
|
||||
throw new Error(`Overpass ${response.status} ${response.statusText}`);
|
||||
}
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
if (attempt < OVERPASS_MAX_ATTEMPTS) {
|
||||
await sleep(5000 * (attempt + 1));
|
||||
return fetchOverpass(query, attempt + 1);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
function elementToLead(element, niche, metro) {
|
||||
const tags = element.tags || {};
|
||||
const website = normalizeWebsite(getTag(tags, ['contact:website', 'website', 'url']));
|
||||
const email = getTag(tags, ['contact:email', 'email']);
|
||||
const phone = getTag(tags, ['contact:phone', 'phone']);
|
||||
const street = [tags['addr:housenumber'], tags['addr:street']].filter(Boolean).join(' ');
|
||||
const city = tags['addr:city'] || metro[0];
|
||||
const state = tags['addr:state'] || metro[1];
|
||||
|
||||
return {
|
||||
niche: niche.id,
|
||||
niche_label: niche.label,
|
||||
company: tags.name || '',
|
||||
website,
|
||||
email,
|
||||
phone,
|
||||
city,
|
||||
state,
|
||||
country: 'US',
|
||||
street,
|
||||
source: 'OpenStreetMap Overpass',
|
||||
source_id: `${element.type}/${element.id}`,
|
||||
source_url: `https://www.openstreetmap.org/${element.type}/${element.id}`,
|
||||
personalization_signal: '',
|
||||
qr_use_case: niche.targetUseCase,
|
||||
lead_score: 0,
|
||||
email_source: email ? 'osm' : '',
|
||||
opt_out_required: 'yes',
|
||||
};
|
||||
}
|
||||
|
||||
function visibleTextEmails(text) {
|
||||
const normalized = text
|
||||
.replaceAll('[at]', '@')
|
||||
.replaceAll('(at)', '@')
|
||||
.replaceAll(' at ', '@')
|
||||
.replaceAll('[dot]', '.')
|
||||
.replaceAll('(dot)', '.')
|
||||
.replaceAll(' dot ', '.');
|
||||
const matches = normalized.match(/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g) || [];
|
||||
return [...new Set(matches.map((email) => email.toLowerCase()))]
|
||||
.filter((email) => !email.endsWith('.png') && !email.endsWith('.jpg') && !email.includes('example.com'))
|
||||
.filter((email) => !email.includes('wixpress.com') && !email.includes('sentry.io'));
|
||||
}
|
||||
|
||||
function extractContactLinks(html, baseUrl) {
|
||||
const links = [];
|
||||
const regex = /href=["']([^"']+)["']/gi;
|
||||
let match;
|
||||
while ((match = regex.exec(html))) {
|
||||
const href = match[1];
|
||||
if (/^(mailto:|tel:)/i.test(href)) continue;
|
||||
if (!/(contact|about|team|booking|book|wedding|private-events|catering|visit|location)/i.test(href)) continue;
|
||||
try {
|
||||
const url = new URL(href, baseUrl);
|
||||
if (url.hostname === new URL(baseUrl).hostname) {
|
||||
url.hash = '';
|
||||
links.push(url.toString());
|
||||
}
|
||||
} catch {
|
||||
// Ignore malformed links.
|
||||
}
|
||||
}
|
||||
return [...new Set(links)].slice(0, 3);
|
||||
}
|
||||
|
||||
async function fetchText(url) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 10000);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'user-agent': 'QR Master lead research bot (+https://qrmaster.net/contact)',
|
||||
accept: 'text/html,application/xhtml+xml',
|
||||
},
|
||||
signal: controller.signal,
|
||||
redirect: 'follow',
|
||||
});
|
||||
if (!response.ok) return '';
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
if (!contentType.includes('text/html')) return '';
|
||||
return await response.text();
|
||||
} catch {
|
||||
return '';
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function enrichLead(lead) {
|
||||
if (!lead.website || lead.email) {
|
||||
return scoreLead(lead);
|
||||
}
|
||||
|
||||
const homepage = await fetchText(lead.website);
|
||||
const emails = visibleTextEmails(homepage);
|
||||
const contactLinks = extractContactLinks(homepage, lead.website);
|
||||
|
||||
for (const link of contactLinks) {
|
||||
if (emails.length > 0) break;
|
||||
const html = await fetchText(link);
|
||||
emails.push(...visibleTextEmails(html));
|
||||
}
|
||||
|
||||
const uniqueEmails = [...new Set(emails)];
|
||||
if (uniqueEmails.length > 0) {
|
||||
lead.email = uniqueEmails[0];
|
||||
lead.email_source = 'website';
|
||||
}
|
||||
|
||||
return scoreLead(lead);
|
||||
}
|
||||
|
||||
function scoreLead(lead) {
|
||||
let score = 30;
|
||||
if (lead.website) score += 20;
|
||||
if (lead.email) score += 30;
|
||||
if (lead.phone) score += 5;
|
||||
if (!/(gmail|yahoo|hotmail|outlook|icloud)\.com$/i.test(lead.email || '')) score += lead.email ? 10 : 0;
|
||||
if (lead.niche === 'real_estate' || lead.niche === 'restaurants') score += 5;
|
||||
|
||||
const signalByNiche = {
|
||||
photographers: `${lead.company} can use dynamic QR codes on print cards, gallery cards, event handouts, and portfolio links.`,
|
||||
restaurants: `${lead.company} can use dynamic QR codes for menus, table tents, reviews, coupons, and seasonal specials.`,
|
||||
real_estate: `${lead.company} can use dynamic QR codes on yard signs, flyers, property sheets, and open house material.`,
|
||||
events_venues: `${lead.company} can use dynamic QR codes for schedules, ticketing, venue maps, check-in, and post-event feedback.`,
|
||||
wellness_beauty: `${lead.company} can use dynamic QR codes for booking pages, service menus, price lists, reviews, and loyalty offers.`,
|
||||
};
|
||||
|
||||
lead.lead_score = Math.min(score, 100);
|
||||
lead.personalization_signal = signalByNiche[lead.niche] || '';
|
||||
return lead;
|
||||
}
|
||||
|
||||
async function mapLimit(items, limit, mapper) {
|
||||
const results = [];
|
||||
let index = 0;
|
||||
async function worker() {
|
||||
while (index < items.length) {
|
||||
const current = index++;
|
||||
results[current] = await mapper(items[current], current);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
||||
return results;
|
||||
}
|
||||
|
||||
async function collectNiche(niche) {
|
||||
const leadsByKey = new Map();
|
||||
for (let pass = 0; pass < 2 && leadsByKey.size < TARGET_PER_NICHE * 2; pass++) {
|
||||
for (const metro of metros) {
|
||||
if (leadsByKey.size >= TARGET_PER_NICHE * 2) break;
|
||||
const query = buildOverpassQuery(niche, metro, pass);
|
||||
try {
|
||||
const data = await fetchOverpass(query);
|
||||
for (const element of data.elements || []) {
|
||||
const lead = elementToLead(element, niche, metro);
|
||||
if (!lead.company) continue;
|
||||
if (!lead.website && !lead.email) continue;
|
||||
const key = lead.website || `${lead.company}|${lead.city}|${lead.state}`.toLowerCase();
|
||||
if (!leadsByKey.has(key)) leadsByKey.set(key, lead);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`[${niche.id}] ${metro[0]} skipped: ${error.message}`);
|
||||
}
|
||||
await sleep(OVERPASS_DELAY_MS);
|
||||
}
|
||||
}
|
||||
|
||||
const rawLeads = [...leadsByKey.values()].slice(0, TARGET_PER_NICHE * 2);
|
||||
console.log(`[${niche.id}] collected ${rawLeads.length}; enriching...`);
|
||||
const enriched = await mapLimit(rawLeads, CONCURRENCY, enrichLead);
|
||||
return enriched
|
||||
.filter((lead) => lead.website || lead.email)
|
||||
.sort((a, b) => b.lead_score - a.lead_score)
|
||||
.slice(0, TARGET_PER_NICHE);
|
||||
}
|
||||
|
||||
function toCsv(leads) {
|
||||
const headers = [
|
||||
'niche',
|
||||
'niche_label',
|
||||
'company',
|
||||
'website',
|
||||
'email',
|
||||
'email_source',
|
||||
'phone',
|
||||
'city',
|
||||
'state',
|
||||
'country',
|
||||
'street',
|
||||
'lead_score',
|
||||
'qr_use_case',
|
||||
'personalization_signal',
|
||||
'source',
|
||||
'source_id',
|
||||
'source_url',
|
||||
'opt_out_required',
|
||||
];
|
||||
return [
|
||||
headers.join(','),
|
||||
...leads.map((lead) => headers.map((header) => csvEscape(lead[header])).join(',')),
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await fs.mkdir(OUTPUT_DIR, { recursive: true });
|
||||
const allLeads = [];
|
||||
for (const niche of niches) {
|
||||
const leads = await collectNiche(niche);
|
||||
allLeads.push(...leads);
|
||||
const dated = new Date().toISOString().slice(0, 10);
|
||||
await fs.writeFile(path.join(OUTPUT_DIR, `qrmaster-us-leads-${niche.id}-${dated}.csv`), toCsv(leads), 'utf8');
|
||||
await fs.writeFile(path.join(OUTPUT_DIR, `qrmaster-us-leads-${niche.id}-${dated}.json`), JSON.stringify(leads, null, 2), 'utf8');
|
||||
console.log(`[${niche.id}] kept ${leads.length}`);
|
||||
}
|
||||
|
||||
const byKey = new Map();
|
||||
for (const lead of allLeads) {
|
||||
const key = lead.email || lead.website || `${lead.company}|${lead.city}|${lead.state}`.toLowerCase();
|
||||
if (!byKey.has(key)) byKey.set(key, lead);
|
||||
}
|
||||
const deduped = [...byKey.values()].sort((a, b) => b.lead_score - a.lead_score);
|
||||
const dated = new Date().toISOString().slice(0, 10);
|
||||
const csvPath = path.join(OUTPUT_DIR, `qrmaster-us-leads-${dated}.csv`);
|
||||
const jsonPath = path.join(OUTPUT_DIR, `qrmaster-us-leads-${dated}.json`);
|
||||
await fs.writeFile(csvPath, toCsv(deduped), 'utf8');
|
||||
await fs.writeFile(jsonPath, JSON.stringify(deduped, null, 2), 'utf8');
|
||||
|
||||
const summary = niches.map((niche) => {
|
||||
const leads = deduped.filter((lead) => lead.niche === niche.id);
|
||||
const withEmail = leads.filter((lead) => lead.email).length;
|
||||
return `${niche.label}: ${leads.length} leads, ${withEmail} emails`;
|
||||
}).join('\n');
|
||||
|
||||
console.log(`\nWrote ${deduped.length} leads`);
|
||||
console.log(csvPath);
|
||||
console.log(jsonPath);
|
||||
console.log(summary);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
||||
@@ -1,326 +1,326 @@
|
||||
import { promises as dns } from "node:dns";
|
||||
import { readdir, readFile, mkdir, writeFile, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
const root = process.cwd();
|
||||
const leadRoot = path.resolve(root, process.argv[2] || "Leads");
|
||||
const excludeFile = path.resolve(root, process.argv[3] || "Leads/lead_emails_1000_2026-05-25.csv");
|
||||
const outputDir = path.resolve(root, process.argv[4] || "Leads/validated");
|
||||
const dateStamp = new Date().toISOString().slice(0, 10);
|
||||
|
||||
const emailPattern = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi;
|
||||
const strictEmailPattern = /^[A-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?(?:\.[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?)+$/i;
|
||||
const allowedExtensions = new Set([".csv", ".txt", ".md", ".json"]);
|
||||
const generatedPrefixes = [
|
||||
"lead_email_validation_all_",
|
||||
"lead_email_validation_valid_remaining_",
|
||||
"lead_email_validation_unknown_remaining_",
|
||||
"lead_email_validation_invalid_",
|
||||
"lead_email_validation_summary_",
|
||||
];
|
||||
const blockedLeadDomains = new Set([
|
||||
"qrmaster.net",
|
||||
]);
|
||||
const empiricalHighConfidenceDomains = new Set([
|
||||
"gmail.com",
|
||||
"googlemail.com",
|
||||
"accor.com",
|
||||
"hotelbb.com",
|
||||
"losteria.de",
|
||||
"breizhcafe.com",
|
||||
]);
|
||||
const empiricalLowConfidenceDomains = new Set([
|
||||
"aon.at",
|
||||
"countryinn.com",
|
||||
"hilton.com",
|
||||
"hyatt.com",
|
||||
"motel-one.com",
|
||||
"novum-hotels.de",
|
||||
"riu.com",
|
||||
]);
|
||||
|
||||
function csvCell(value) {
|
||||
const text = String(value ?? "");
|
||||
return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
|
||||
}
|
||||
|
||||
function toCsv(rows, columns) {
|
||||
const lines = [columns.map(csvCell).join(",")];
|
||||
for (const row of rows) {
|
||||
lines.push(columns.map((column) => csvCell(row[column])).join(","));
|
||||
}
|
||||
return `${lines.join("\r\n")}\r\n`;
|
||||
}
|
||||
|
||||
async function collectInputFiles(inputPath) {
|
||||
const inputStat = await stat(inputPath);
|
||||
if (inputStat.isFile()) {
|
||||
return [inputPath];
|
||||
}
|
||||
if (!inputStat.isDirectory()) {
|
||||
throw new Error(`Input path is not a file or directory: ${inputPath}`);
|
||||
}
|
||||
return walkFiles(inputPath);
|
||||
}
|
||||
|
||||
async function walkFiles(dir) {
|
||||
const entries = await readdir(dir, { withFileTypes: true });
|
||||
const files = [];
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...await walkFiles(fullPath));
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) continue;
|
||||
if (!allowedExtensions.has(path.extname(entry.name).toLowerCase())) continue;
|
||||
if (generatedPrefixes.some((prefix) => entry.name.startsWith(prefix))) continue;
|
||||
files.push(fullPath);
|
||||
}
|
||||
return files.sort((a, b) => a.localeCompare(b));
|
||||
}
|
||||
|
||||
async function extractEmailsFromFile(filePath) {
|
||||
try {
|
||||
const content = await readFile(filePath, "utf8");
|
||||
return [...content.matchAll(emailPattern)].map((match) =>
|
||||
match[0].trim().replace(/\.+$/, "").toLowerCase(),
|
||||
);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function loadExcludedEmails(filePathsArg) {
|
||||
const excluded = new Set();
|
||||
const filePaths = String(filePathsArg || "")
|
||||
.split(";")
|
||||
.map((filePath) => filePath.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
for (const filePath of filePaths) {
|
||||
try {
|
||||
await stat(filePath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const emails = await extractEmailsFromFile(filePath);
|
||||
for (const email of emails) excluded.add(email);
|
||||
}
|
||||
|
||||
return excluded;
|
||||
}
|
||||
|
||||
function withTimeout(promise, ms) {
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error("dns_timeout")), ms);
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
async function checkDomain(domain) {
|
||||
try {
|
||||
const mxRecords = await withTimeout(dns.resolveMx(domain), 2500);
|
||||
if (mxRecords.length > 0) {
|
||||
return {
|
||||
dns_status: "mx",
|
||||
mx_hosts: mxRecords
|
||||
.sort((a, b) => a.priority - b.priority)
|
||||
.map((record) => record.exchange)
|
||||
.join(";"),
|
||||
reason: "domain_has_mx",
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Fall through to A lookup. Some domains can receive via address fallback.
|
||||
}
|
||||
|
||||
try {
|
||||
const aRecords = await withTimeout(dns.resolve4(domain), 2000);
|
||||
if (aRecords.length > 0) {
|
||||
return {
|
||||
dns_status: "a_only",
|
||||
mx_hosts: "",
|
||||
reason: "domain_has_a_record_but_no_mx",
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Classified below.
|
||||
}
|
||||
|
||||
return {
|
||||
dns_status: "no_dns",
|
||||
mx_hosts: "",
|
||||
reason: "no_mx_or_a_record",
|
||||
};
|
||||
}
|
||||
|
||||
async function mapLimit(items, limit, worker) {
|
||||
const results = new Map();
|
||||
let index = 0;
|
||||
|
||||
async function runWorker() {
|
||||
while (index < items.length) {
|
||||
const currentIndex = index++;
|
||||
const item = items[currentIndex];
|
||||
if ((currentIndex + 1) % 100 === 0) {
|
||||
console.log(`DNS checked ${currentIndex + 1} / ${items.length} domains...`);
|
||||
}
|
||||
results.set(item, await worker(item));
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, runWorker));
|
||||
return results;
|
||||
}
|
||||
|
||||
function getConfidence(status, domain) {
|
||||
if (status !== "valid") {
|
||||
return {
|
||||
confidence: "reject",
|
||||
confidence_reason: "not_dns_valid",
|
||||
};
|
||||
}
|
||||
|
||||
if (empiricalLowConfidenceDomains.has(domain)) {
|
||||
return {
|
||||
confidence: "low",
|
||||
confidence_reason: "empirical_low_smartlead_valid_rate",
|
||||
};
|
||||
}
|
||||
|
||||
if (empiricalHighConfidenceDomains.has(domain)) {
|
||||
return {
|
||||
confidence: "high",
|
||||
confidence_reason: "empirical_high_smartlead_valid_rate",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
confidence: "medium",
|
||||
confidence_reason: "dns_valid_unproven_domain",
|
||||
};
|
||||
}
|
||||
|
||||
await mkdir(outputDir, { recursive: true });
|
||||
|
||||
const excludeEmails = await loadExcludedEmails(excludeFile);
|
||||
const files = await collectInputFiles(leadRoot);
|
||||
const emailSources = new Map();
|
||||
|
||||
for (const file of files) {
|
||||
const emails = await extractEmailsFromFile(file);
|
||||
for (const email of emails) {
|
||||
if (!emailSources.has(email)) emailSources.set(email, []);
|
||||
const sources = emailSources.get(email);
|
||||
if (sources.length < 5) sources.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
const domains = [...new Set(
|
||||
[...emailSources.keys()]
|
||||
.filter((email) => strictEmailPattern.test(email))
|
||||
.map((email) => email.split("@")[1]),
|
||||
)].sort((a, b) => a.localeCompare(b));
|
||||
|
||||
console.log(`Files scanned: ${files.length}`);
|
||||
console.log(`Unique emails found: ${emailSources.size}`);
|
||||
console.log(`Domains to check: ${domains.length}`);
|
||||
|
||||
const dnsResults = await mapLimit(domains, 80, checkDomain);
|
||||
|
||||
const results = [...emailSources.keys()].sort((a, b) => a.localeCompare(b)).map((email) => {
|
||||
const syntaxValid = strictEmailPattern.test(email);
|
||||
const domain = email.includes("@") ? email.split("@")[1] : "";
|
||||
const reserved = /^(example|test|invalid|localhost)(\.|$)/i.test(domain);
|
||||
const dnsResult = dnsResults.get(domain);
|
||||
|
||||
let status = "invalid";
|
||||
let reason = "invalid_syntax";
|
||||
let dnsStatus = "";
|
||||
let mxHosts = "";
|
||||
|
||||
if (syntaxValid && blockedLeadDomains.has(domain)) {
|
||||
reason = "internal_or_generated_domain";
|
||||
} else if (syntaxValid && reserved) {
|
||||
reason = "reserved_or_test_domain";
|
||||
} else if (syntaxValid && dnsResult?.dns_status === "mx") {
|
||||
status = "valid";
|
||||
reason = dnsResult.reason;
|
||||
dnsStatus = dnsResult.dns_status;
|
||||
mxHosts = dnsResult.mx_hosts;
|
||||
} else if (syntaxValid && dnsResult?.dns_status === "a_only") {
|
||||
status = "unknown";
|
||||
reason = dnsResult.reason;
|
||||
dnsStatus = dnsResult.dns_status;
|
||||
} else if (syntaxValid) {
|
||||
reason = dnsResult?.reason || "dns_not_checked";
|
||||
dnsStatus = dnsResult?.dns_status || "";
|
||||
}
|
||||
|
||||
const confidenceResult = getConfidence(status, domain);
|
||||
|
||||
return {
|
||||
email,
|
||||
status,
|
||||
reason,
|
||||
confidence: confidenceResult.confidence,
|
||||
confidence_reason: confidenceResult.confidence_reason,
|
||||
domain,
|
||||
dns_status: dnsStatus,
|
||||
mx_hosts: mxHosts,
|
||||
already_uploaded: excludeEmails.has(email) ? "true" : "false",
|
||||
source_count: emailSources.get(email).length,
|
||||
first_source: emailSources.get(email)[0],
|
||||
};
|
||||
});
|
||||
|
||||
const allOut = path.join(outputDir, `lead_email_validation_all_${dateStamp}.csv`);
|
||||
const validOut = path.join(outputDir, `lead_email_validation_valid_remaining_${dateStamp}.csv`);
|
||||
const highConfidenceOut = path.join(outputDir, `lead_email_validation_high_confidence_remaining_${dateStamp}.csv`);
|
||||
const unknownOut = path.join(outputDir, `lead_email_validation_unknown_remaining_${dateStamp}.csv`);
|
||||
const invalidOut = path.join(outputDir, `lead_email_validation_invalid_${dateStamp}.csv`);
|
||||
const summaryOut = path.join(outputDir, `lead_email_validation_summary_${dateStamp}.txt`);
|
||||
|
||||
const validRemaining = results.filter((row) => row.status === "valid" && row.already_uploaded !== "true");
|
||||
const highConfidenceRemaining = results.filter((row) =>
|
||||
row.status === "valid" &&
|
||||
row.confidence === "high" &&
|
||||
row.already_uploaded !== "true"
|
||||
);
|
||||
const unknownRemaining = results.filter((row) => row.status === "unknown" && row.already_uploaded !== "true");
|
||||
const invalid = results.filter((row) => row.status === "invalid");
|
||||
|
||||
await writeFile(
|
||||
allOut,
|
||||
toCsv(results, ["email", "status", "reason", "confidence", "confidence_reason", "domain", "dns_status", "mx_hosts", "already_uploaded", "source_count", "first_source"]),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(validOut, toCsv(validRemaining.map(({ email }) => ({ email })), ["email"]), "utf8");
|
||||
await writeFile(highConfidenceOut, toCsv(highConfidenceRemaining.map(({ email }) => ({ email })), ["email"]), "utf8");
|
||||
await writeFile(unknownOut, toCsv(unknownRemaining, ["email", "reason", "domain"]), "utf8");
|
||||
await writeFile(invalidOut, toCsv(invalid, ["email", "reason", "domain"]), "utf8");
|
||||
|
||||
const summary = [
|
||||
`Lead email validation summary - ${dateStamp}`,
|
||||
`Lead root: ${leadRoot}`,
|
||||
`Files scanned: ${files.length}`,
|
||||
`Unique emails found: ${results.length}`,
|
||||
`Already uploaded/excluded: ${results.filter((row) => row.already_uploaded === "true").length}`,
|
||||
`Valid total: ${results.filter((row) => row.status === "valid").length}`,
|
||||
`Valid remaining: ${validRemaining.length}`,
|
||||
`High-confidence valid remaining: ${highConfidenceRemaining.length}`,
|
||||
`Unknown remaining: ${unknownRemaining.length}`,
|
||||
`Invalid total: ${invalid.length}`,
|
||||
`All report: ${allOut}`,
|
||||
`Valid remaining upload file: ${validOut}`,
|
||||
`High-confidence upload file: ${highConfidenceOut}`,
|
||||
`Unknown remaining review file: ${unknownOut}`,
|
||||
`Invalid report: ${invalidOut}`,
|
||||
"",
|
||||
].join("\n");
|
||||
await writeFile(summaryOut, summary, "utf8");
|
||||
|
||||
console.log(summary);
|
||||
import { promises as dns } from "node:dns";
|
||||
import { readdir, readFile, mkdir, writeFile, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
const root = process.cwd();
|
||||
const leadRoot = path.resolve(root, process.argv[2] || "Leads");
|
||||
const excludeFile = path.resolve(root, process.argv[3] || "Leads/lead_emails_1000_2026-05-25.csv");
|
||||
const outputDir = path.resolve(root, process.argv[4] || "Leads/validated");
|
||||
const dateStamp = new Date().toISOString().slice(0, 10);
|
||||
|
||||
const emailPattern = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi;
|
||||
const strictEmailPattern = /^[A-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?(?:\.[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?)+$/i;
|
||||
const allowedExtensions = new Set([".csv", ".txt", ".md", ".json"]);
|
||||
const generatedPrefixes = [
|
||||
"lead_email_validation_all_",
|
||||
"lead_email_validation_valid_remaining_",
|
||||
"lead_email_validation_unknown_remaining_",
|
||||
"lead_email_validation_invalid_",
|
||||
"lead_email_validation_summary_",
|
||||
];
|
||||
const blockedLeadDomains = new Set([
|
||||
"qrmaster.net",
|
||||
]);
|
||||
const empiricalHighConfidenceDomains = new Set([
|
||||
"gmail.com",
|
||||
"googlemail.com",
|
||||
"accor.com",
|
||||
"hotelbb.com",
|
||||
"losteria.de",
|
||||
"breizhcafe.com",
|
||||
]);
|
||||
const empiricalLowConfidenceDomains = new Set([
|
||||
"aon.at",
|
||||
"countryinn.com",
|
||||
"hilton.com",
|
||||
"hyatt.com",
|
||||
"motel-one.com",
|
||||
"novum-hotels.de",
|
||||
"riu.com",
|
||||
]);
|
||||
|
||||
function csvCell(value) {
|
||||
const text = String(value ?? "");
|
||||
return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
|
||||
}
|
||||
|
||||
function toCsv(rows, columns) {
|
||||
const lines = [columns.map(csvCell).join(",")];
|
||||
for (const row of rows) {
|
||||
lines.push(columns.map((column) => csvCell(row[column])).join(","));
|
||||
}
|
||||
return `${lines.join("\r\n")}\r\n`;
|
||||
}
|
||||
|
||||
async function collectInputFiles(inputPath) {
|
||||
const inputStat = await stat(inputPath);
|
||||
if (inputStat.isFile()) {
|
||||
return [inputPath];
|
||||
}
|
||||
if (!inputStat.isDirectory()) {
|
||||
throw new Error(`Input path is not a file or directory: ${inputPath}`);
|
||||
}
|
||||
return walkFiles(inputPath);
|
||||
}
|
||||
|
||||
async function walkFiles(dir) {
|
||||
const entries = await readdir(dir, { withFileTypes: true });
|
||||
const files = [];
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...await walkFiles(fullPath));
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) continue;
|
||||
if (!allowedExtensions.has(path.extname(entry.name).toLowerCase())) continue;
|
||||
if (generatedPrefixes.some((prefix) => entry.name.startsWith(prefix))) continue;
|
||||
files.push(fullPath);
|
||||
}
|
||||
return files.sort((a, b) => a.localeCompare(b));
|
||||
}
|
||||
|
||||
async function extractEmailsFromFile(filePath) {
|
||||
try {
|
||||
const content = await readFile(filePath, "utf8");
|
||||
return [...content.matchAll(emailPattern)].map((match) =>
|
||||
match[0].trim().replace(/\.+$/, "").toLowerCase(),
|
||||
);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function loadExcludedEmails(filePathsArg) {
|
||||
const excluded = new Set();
|
||||
const filePaths = String(filePathsArg || "")
|
||||
.split(";")
|
||||
.map((filePath) => filePath.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
for (const filePath of filePaths) {
|
||||
try {
|
||||
await stat(filePath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const emails = await extractEmailsFromFile(filePath);
|
||||
for (const email of emails) excluded.add(email);
|
||||
}
|
||||
|
||||
return excluded;
|
||||
}
|
||||
|
||||
function withTimeout(promise, ms) {
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error("dns_timeout")), ms);
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
async function checkDomain(domain) {
|
||||
try {
|
||||
const mxRecords = await withTimeout(dns.resolveMx(domain), 2500);
|
||||
if (mxRecords.length > 0) {
|
||||
return {
|
||||
dns_status: "mx",
|
||||
mx_hosts: mxRecords
|
||||
.sort((a, b) => a.priority - b.priority)
|
||||
.map((record) => record.exchange)
|
||||
.join(";"),
|
||||
reason: "domain_has_mx",
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Fall through to A lookup. Some domains can receive via address fallback.
|
||||
}
|
||||
|
||||
try {
|
||||
const aRecords = await withTimeout(dns.resolve4(domain), 2000);
|
||||
if (aRecords.length > 0) {
|
||||
return {
|
||||
dns_status: "a_only",
|
||||
mx_hosts: "",
|
||||
reason: "domain_has_a_record_but_no_mx",
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Classified below.
|
||||
}
|
||||
|
||||
return {
|
||||
dns_status: "no_dns",
|
||||
mx_hosts: "",
|
||||
reason: "no_mx_or_a_record",
|
||||
};
|
||||
}
|
||||
|
||||
async function mapLimit(items, limit, worker) {
|
||||
const results = new Map();
|
||||
let index = 0;
|
||||
|
||||
async function runWorker() {
|
||||
while (index < items.length) {
|
||||
const currentIndex = index++;
|
||||
const item = items[currentIndex];
|
||||
if ((currentIndex + 1) % 100 === 0) {
|
||||
console.log(`DNS checked ${currentIndex + 1} / ${items.length} domains...`);
|
||||
}
|
||||
results.set(item, await worker(item));
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, runWorker));
|
||||
return results;
|
||||
}
|
||||
|
||||
function getConfidence(status, domain) {
|
||||
if (status !== "valid") {
|
||||
return {
|
||||
confidence: "reject",
|
||||
confidence_reason: "not_dns_valid",
|
||||
};
|
||||
}
|
||||
|
||||
if (empiricalLowConfidenceDomains.has(domain)) {
|
||||
return {
|
||||
confidence: "low",
|
||||
confidence_reason: "empirical_low_smartlead_valid_rate",
|
||||
};
|
||||
}
|
||||
|
||||
if (empiricalHighConfidenceDomains.has(domain)) {
|
||||
return {
|
||||
confidence: "high",
|
||||
confidence_reason: "empirical_high_smartlead_valid_rate",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
confidence: "medium",
|
||||
confidence_reason: "dns_valid_unproven_domain",
|
||||
};
|
||||
}
|
||||
|
||||
await mkdir(outputDir, { recursive: true });
|
||||
|
||||
const excludeEmails = await loadExcludedEmails(excludeFile);
|
||||
const files = await collectInputFiles(leadRoot);
|
||||
const emailSources = new Map();
|
||||
|
||||
for (const file of files) {
|
||||
const emails = await extractEmailsFromFile(file);
|
||||
for (const email of emails) {
|
||||
if (!emailSources.has(email)) emailSources.set(email, []);
|
||||
const sources = emailSources.get(email);
|
||||
if (sources.length < 5) sources.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
const domains = [...new Set(
|
||||
[...emailSources.keys()]
|
||||
.filter((email) => strictEmailPattern.test(email))
|
||||
.map((email) => email.split("@")[1]),
|
||||
)].sort((a, b) => a.localeCompare(b));
|
||||
|
||||
console.log(`Files scanned: ${files.length}`);
|
||||
console.log(`Unique emails found: ${emailSources.size}`);
|
||||
console.log(`Domains to check: ${domains.length}`);
|
||||
|
||||
const dnsResults = await mapLimit(domains, 80, checkDomain);
|
||||
|
||||
const results = [...emailSources.keys()].sort((a, b) => a.localeCompare(b)).map((email) => {
|
||||
const syntaxValid = strictEmailPattern.test(email);
|
||||
const domain = email.includes("@") ? email.split("@")[1] : "";
|
||||
const reserved = /^(example|test|invalid|localhost)(\.|$)/i.test(domain);
|
||||
const dnsResult = dnsResults.get(domain);
|
||||
|
||||
let status = "invalid";
|
||||
let reason = "invalid_syntax";
|
||||
let dnsStatus = "";
|
||||
let mxHosts = "";
|
||||
|
||||
if (syntaxValid && blockedLeadDomains.has(domain)) {
|
||||
reason = "internal_or_generated_domain";
|
||||
} else if (syntaxValid && reserved) {
|
||||
reason = "reserved_or_test_domain";
|
||||
} else if (syntaxValid && dnsResult?.dns_status === "mx") {
|
||||
status = "valid";
|
||||
reason = dnsResult.reason;
|
||||
dnsStatus = dnsResult.dns_status;
|
||||
mxHosts = dnsResult.mx_hosts;
|
||||
} else if (syntaxValid && dnsResult?.dns_status === "a_only") {
|
||||
status = "unknown";
|
||||
reason = dnsResult.reason;
|
||||
dnsStatus = dnsResult.dns_status;
|
||||
} else if (syntaxValid) {
|
||||
reason = dnsResult?.reason || "dns_not_checked";
|
||||
dnsStatus = dnsResult?.dns_status || "";
|
||||
}
|
||||
|
||||
const confidenceResult = getConfidence(status, domain);
|
||||
|
||||
return {
|
||||
email,
|
||||
status,
|
||||
reason,
|
||||
confidence: confidenceResult.confidence,
|
||||
confidence_reason: confidenceResult.confidence_reason,
|
||||
domain,
|
||||
dns_status: dnsStatus,
|
||||
mx_hosts: mxHosts,
|
||||
already_uploaded: excludeEmails.has(email) ? "true" : "false",
|
||||
source_count: emailSources.get(email).length,
|
||||
first_source: emailSources.get(email)[0],
|
||||
};
|
||||
});
|
||||
|
||||
const allOut = path.join(outputDir, `lead_email_validation_all_${dateStamp}.csv`);
|
||||
const validOut = path.join(outputDir, `lead_email_validation_valid_remaining_${dateStamp}.csv`);
|
||||
const highConfidenceOut = path.join(outputDir, `lead_email_validation_high_confidence_remaining_${dateStamp}.csv`);
|
||||
const unknownOut = path.join(outputDir, `lead_email_validation_unknown_remaining_${dateStamp}.csv`);
|
||||
const invalidOut = path.join(outputDir, `lead_email_validation_invalid_${dateStamp}.csv`);
|
||||
const summaryOut = path.join(outputDir, `lead_email_validation_summary_${dateStamp}.txt`);
|
||||
|
||||
const validRemaining = results.filter((row) => row.status === "valid" && row.already_uploaded !== "true");
|
||||
const highConfidenceRemaining = results.filter((row) =>
|
||||
row.status === "valid" &&
|
||||
row.confidence === "high" &&
|
||||
row.already_uploaded !== "true"
|
||||
);
|
||||
const unknownRemaining = results.filter((row) => row.status === "unknown" && row.already_uploaded !== "true");
|
||||
const invalid = results.filter((row) => row.status === "invalid");
|
||||
|
||||
await writeFile(
|
||||
allOut,
|
||||
toCsv(results, ["email", "status", "reason", "confidence", "confidence_reason", "domain", "dns_status", "mx_hosts", "already_uploaded", "source_count", "first_source"]),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(validOut, toCsv(validRemaining.map(({ email }) => ({ email })), ["email"]), "utf8");
|
||||
await writeFile(highConfidenceOut, toCsv(highConfidenceRemaining.map(({ email }) => ({ email })), ["email"]), "utf8");
|
||||
await writeFile(unknownOut, toCsv(unknownRemaining, ["email", "reason", "domain"]), "utf8");
|
||||
await writeFile(invalidOut, toCsv(invalid, ["email", "reason", "domain"]), "utf8");
|
||||
|
||||
const summary = [
|
||||
`Lead email validation summary - ${dateStamp}`,
|
||||
`Lead root: ${leadRoot}`,
|
||||
`Files scanned: ${files.length}`,
|
||||
`Unique emails found: ${results.length}`,
|
||||
`Already uploaded/excluded: ${results.filter((row) => row.already_uploaded === "true").length}`,
|
||||
`Valid total: ${results.filter((row) => row.status === "valid").length}`,
|
||||
`Valid remaining: ${validRemaining.length}`,
|
||||
`High-confidence valid remaining: ${highConfidenceRemaining.length}`,
|
||||
`Unknown remaining: ${unknownRemaining.length}`,
|
||||
`Invalid total: ${invalid.length}`,
|
||||
`All report: ${allOut}`,
|
||||
`Valid remaining upload file: ${validOut}`,
|
||||
`High-confidence upload file: ${highConfidenceOut}`,
|
||||
`Unknown remaining review file: ${unknownOut}`,
|
||||
`Invalid report: ${invalidOut}`,
|
||||
"",
|
||||
].join("\n");
|
||||
await writeFile(summaryOut, summary, "utf8");
|
||||
|
||||
console.log(summary);
|
||||
|
||||
Reference in New Issue
Block a user