330 lines
11 KiB
JavaScript
330 lines
11 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* Reicht eine kuratierte URL-Liste bei IndexNow (Bing, Yandex, Seznam, Naver)
|
||
* und optional bei der Google Indexing API ein.
|
||
*
|
||
* Keine npm-Dependencies. Braucht Node 18+ (globales fetch).
|
||
*
|
||
* node scripts/submit-changed-urls.mjs --dry-run
|
||
* node scripts/submit-changed-urls.mjs
|
||
* node scripts/submit-changed-urls.mjs --google-only
|
||
* node scripts/submit-changed-urls.mjs --urls scripts/changed-urls-2026-07-27.json
|
||
*
|
||
* Konfiguration (.env oder Umgebung):
|
||
* INDEXNOW_KEY IndexNow-Key, muss als <key>.txt live erreichbar sein
|
||
* GOOGLE_SERVICE_ACCOUNT Pfad zur Service-Account-JSON (Default: ./service_account.json)
|
||
*
|
||
* Hinweis zur Google Indexing API: offiziell unterstuetzt Google damit nur
|
||
* JobPosting und BroadcastEvent. Fuer normale Seiten funktioniert es in der
|
||
* Praxis oft, ist aber nicht zugesichert. Das Tageslimit liegt bei 200 URLs.
|
||
*/
|
||
|
||
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import crypto from 'node:crypto';
|
||
import { fileURLToPath } from 'node:url';
|
||
|
||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||
const repoRoot = path.resolve(__dirname, '..');
|
||
|
||
// ---------------------------------------------------------------- CLI + env
|
||
|
||
const argv = process.argv.slice(2);
|
||
const hasFlag = (name) => argv.includes(`--${name}`);
|
||
const getArg = (name, fallback) => {
|
||
const i = argv.indexOf(`--${name}`);
|
||
return i !== -1 && argv[i + 1] ? argv[i + 1] : fallback;
|
||
};
|
||
|
||
const DRY_RUN = hasFlag('dry-run');
|
||
const SKIP_PREFLIGHT = hasFlag('skip-preflight');
|
||
const GOOGLE_ONLY = hasFlag('google-only');
|
||
const INDEXNOW_ONLY = hasFlag('indexnow-only');
|
||
const DO_INDEXNOW = !GOOGLE_ONLY;
|
||
const DO_GOOGLE = !INDEXNOW_ONLY;
|
||
|
||
loadDotEnv(path.join(repoRoot, '.env'));
|
||
loadDotEnv(path.join(repoRoot, '.env.local'));
|
||
|
||
function loadDotEnv(file) {
|
||
if (!fs.existsSync(file)) return;
|
||
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
|
||
const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)\s*$/i);
|
||
if (!m) continue;
|
||
const key = m[1];
|
||
if (process.env[key] !== undefined) continue;
|
||
process.env[key] = m[2].replace(/^["']|["']$/g, '');
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------- URL-Liste
|
||
|
||
const urlsFile = path.resolve(
|
||
repoRoot,
|
||
getArg('urls', 'scripts/changed-urls-2026-07-27.json')
|
||
);
|
||
|
||
if (!fs.existsSync(urlsFile)) {
|
||
fail(`URL-Datei nicht gefunden: ${urlsFile}`);
|
||
}
|
||
|
||
const config = JSON.parse(fs.readFileSync(urlsFile, 'utf8'));
|
||
const groups = config.groups ?? {};
|
||
const urls = [...new Set(Object.values(groups).flat())];
|
||
|
||
if (urls.length === 0) fail('Die URL-Liste ist leer.');
|
||
|
||
console.log(`\n Quelle ${path.relative(repoRoot, urlsFile)}`);
|
||
console.log(` Host ${config.host}`);
|
||
console.log(` URLs ${urls.length}`);
|
||
for (const [name, list] of Object.entries(groups)) {
|
||
console.log(` ${String(list.length).padStart(3)} ${name}`);
|
||
}
|
||
console.log(` Modus ${DRY_RUN ? 'DRY RUN (es wird nichts gesendet)' : 'LIVE'}`);
|
||
console.log(
|
||
` Kanaele ${[DO_INDEXNOW && 'IndexNow', DO_GOOGLE && 'Google Indexing API']
|
||
.filter(Boolean)
|
||
.join(' + ') || 'keine'}\n`
|
||
);
|
||
|
||
// alle URLs muessen zum konfigurierten Host gehoeren
|
||
const foreign = urls.filter((u) => new URL(u).host !== config.host);
|
||
if (foreign.length) {
|
||
fail(`Diese URLs passen nicht zu host="${config.host}":\n ${foreign.join('\n ')}`);
|
||
}
|
||
|
||
// ---------------------------------------------------------------- Preflight
|
||
|
||
if (!SKIP_PREFLIGHT) {
|
||
console.log(' Preflight: pruefe, ob jede URL live 200 liefert ...');
|
||
const bad = [];
|
||
for (const url of urls) {
|
||
const status = await statusOf(url);
|
||
if (status !== 200) bad.push(`${status} ${url}`);
|
||
}
|
||
if (bad.length) {
|
||
console.error('\n Diese URLs antworten nicht mit 200:');
|
||
for (const b of bad) console.error(` ${b}`);
|
||
fail(
|
||
'Abbruch. Eine URL einzureichen, die 404 oder 500 liefert, schadet mehr als sie nutzt.\n' +
|
||
' Deploy pruefen, dann erneut ausfuehren (oder --skip-preflight setzen).'
|
||
);
|
||
}
|
||
console.log(` Preflight OK: alle ${urls.length} URLs liefern 200.\n`);
|
||
}
|
||
|
||
// ---------------------------------------------------------------- IndexNow
|
||
|
||
if (DO_INDEXNOW) {
|
||
const key = process.env.INDEXNOW_KEY || detectKeyInPublicDir();
|
||
if (!key) {
|
||
fail(
|
||
'Kein IndexNow-Key gefunden.\n' +
|
||
' Entweder INDEXNOW_KEY in .env setzen, oder eine <key>.txt in public/\n' +
|
||
' ablegen, die genau den Key als Inhalt hat (bing.com/indexnow).'
|
||
);
|
||
}
|
||
if (!process.env.INDEXNOW_KEY) {
|
||
console.log(` IndexNow-Key aus public/ erkannt: ${key}`);
|
||
}
|
||
|
||
const keyLocation = `https://${config.host}/${key}.txt`;
|
||
|
||
if (!SKIP_PREFLIGHT) {
|
||
const res = await fetch(keyLocation).catch(() => null);
|
||
const body = res && res.ok ? (await res.text()).trim().replace(/^/, '') : null;
|
||
if (!res || !res.ok) {
|
||
fail(`IndexNow-Key-Datei nicht erreichbar: ${keyLocation}`);
|
||
}
|
||
if (body !== key) {
|
||
fail(
|
||
`IndexNow-Key-Datei enthaelt nicht den erwarteten Key.\n` +
|
||
` ${keyLocation}\n erwartet: ${key}\n gefunden: ${JSON.stringify(body)}`
|
||
);
|
||
}
|
||
console.log(` IndexNow-Key verifiziert: ${keyLocation}`);
|
||
}
|
||
|
||
const payload = {
|
||
host: config.host,
|
||
key,
|
||
keyLocation,
|
||
urlList: urls,
|
||
};
|
||
|
||
if (DRY_RUN) {
|
||
console.log(` [dry-run] POST https://api.indexnow.org/indexnow (${urls.length} URLs)\n`);
|
||
} else {
|
||
const res = await fetch('https://api.indexnow.org/indexnow', {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json; charset=utf-8' },
|
||
body: JSON.stringify(payload),
|
||
});
|
||
// 200 = angenommen, 202 = angenommen, Key wird noch geprueft
|
||
if (res.status === 200 || res.status === 202) {
|
||
console.log(` IndexNow OK (${res.status}): ${urls.length} URLs uebermittelt.\n`);
|
||
} else {
|
||
console.error(` IndexNow fehlgeschlagen (${res.status}): ${await res.text()}\n`);
|
||
process.exitCode = 1;
|
||
}
|
||
}
|
||
}
|
||
|
||
// ------------------------------------------------------- Google Indexing API
|
||
|
||
if (DO_GOOGLE) {
|
||
const saPath = path.resolve(
|
||
repoRoot,
|
||
process.env.GOOGLE_SERVICE_ACCOUNT || 'service_account.json'
|
||
);
|
||
|
||
if (!fs.existsSync(saPath)) {
|
||
console.warn(
|
||
` Google Indexing API uebersprungen: ${path.relative(repoRoot, saPath)} nicht gefunden.\n` +
|
||
` Pfad ueber GOOGLE_SERVICE_ACCOUNT setzen oder --indexnow-only nutzen.\n`
|
||
);
|
||
} else if (urls.length > 200) {
|
||
fail(`Die Google Indexing API erlaubt 200 URLs pro Tag, die Liste hat ${urls.length}.`);
|
||
} else {
|
||
const sa = JSON.parse(fs.readFileSync(saPath, 'utf8'));
|
||
console.log(` Google Indexing API als ${sa.client_email}`);
|
||
|
||
if (DRY_RUN) {
|
||
console.log(` [dry-run] ${urls.length}x urlNotifications:publish (URL_UPDATED)\n`);
|
||
} else {
|
||
const token = await getAccessToken(sa);
|
||
let ok = 0;
|
||
const failures = [];
|
||
|
||
for (const url of urls) {
|
||
const res = await fetch(
|
||
'https://indexing.googleapis.com/v3/urlNotifications:publish',
|
||
{
|
||
method: 'POST',
|
||
headers: {
|
||
Authorization: `Bearer ${token}`,
|
||
'Content-Type': 'application/json',
|
||
},
|
||
body: JSON.stringify({ url, type: 'URL_UPDATED' }),
|
||
}
|
||
);
|
||
|
||
if (res.ok) {
|
||
ok++;
|
||
console.log(` ok ${url}`);
|
||
} else {
|
||
const text = await res.text();
|
||
failures.push(`${res.status} ${url}\n ${text.slice(0, 200)}`);
|
||
console.log(` FEHL ${res.status} ${url}`);
|
||
}
|
||
await sleep(120); // bleibt unter dem Minutenlimit
|
||
}
|
||
|
||
console.log(`\n Google: ${ok}/${urls.length} akzeptiert.`);
|
||
if (failures.length) {
|
||
console.error('\n Fehlgeschlagen:');
|
||
for (const f of failures) console.error(` ${f}`);
|
||
process.exitCode = 1;
|
||
}
|
||
console.log();
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log(' Fertig.\n');
|
||
if (!DRY_RUN) {
|
||
console.log(' Denk dran: Google Search Console hat keine API fuer "Indexierung');
|
||
console.log(' beantragen". Die wichtigsten Seiten dort weiterhin manuell anstossen.\n');
|
||
}
|
||
|
||
// ---------------------------------------------------------------- Helpers
|
||
|
||
/**
|
||
* Sucht in public/ nach einer <key>.txt, deren Inhalt exakt dem Dateinamen
|
||
* entspricht. Genau so verlangt IndexNow die Key-Datei.
|
||
*/
|
||
function detectKeyInPublicDir() {
|
||
const publicDir = path.join(repoRoot, 'public');
|
||
if (!fs.existsSync(publicDir)) return null;
|
||
|
||
const candidates = [];
|
||
for (const file of fs.readdirSync(publicDir)) {
|
||
if (!file.endsWith('.txt')) continue;
|
||
const name = file.slice(0, -4);
|
||
if (!/^[a-f0-9]{8,128}$/i.test(name)) continue;
|
||
let content;
|
||
try {
|
||
content = fs.readFileSync(path.join(publicDir, file), 'utf8');
|
||
} catch {
|
||
continue;
|
||
}
|
||
// BOM und Null-Bytes aus Windows-Editoren wegräumen
|
||
const normalised = content.replace(/ /g, '').replace(/^/, '').trim();
|
||
if (normalised === name) candidates.push(name);
|
||
}
|
||
|
||
if (candidates.length > 1) {
|
||
console.warn(
|
||
` Mehrere gueltige IndexNow-Keys in public/: ${candidates.join(', ')}\n` +
|
||
` Es wird ${candidates[0]} genutzt. Fuer Eindeutigkeit INDEXNOW_KEY setzen.`
|
||
);
|
||
}
|
||
return candidates[0] ?? null;
|
||
}
|
||
|
||
async function statusOf(url) {
|
||
try {
|
||
let res = await fetch(url, { method: 'HEAD', redirect: 'follow' });
|
||
// manche Hosts mögen HEAD nicht
|
||
if (res.status === 405 || res.status === 501) {
|
||
res = await fetch(url, { method: 'GET', redirect: 'follow' });
|
||
}
|
||
return res.status;
|
||
} catch (err) {
|
||
return `ERR ${err.message}`;
|
||
}
|
||
}
|
||
|
||
/** OAuth2 Access Token per signiertem JWT, ohne googleapis-Dependency. */
|
||
async function getAccessToken(sa) {
|
||
const now = Math.floor(Date.now() / 1000);
|
||
const header = { alg: 'RS256', typ: 'JWT' };
|
||
const claim = {
|
||
iss: sa.client_email,
|
||
scope: 'https://www.googleapis.com/auth/indexing',
|
||
aud: 'https://oauth2.googleapis.com/token',
|
||
exp: now + 3600,
|
||
iat: now,
|
||
};
|
||
|
||
const b64 = (obj) => Buffer.from(JSON.stringify(obj)).toString('base64url');
|
||
const unsigned = `${b64(header)}.${b64(claim)}`;
|
||
const signature = crypto
|
||
.createSign('RSA-SHA256')
|
||
.update(unsigned)
|
||
.sign(sa.private_key)
|
||
.toString('base64url');
|
||
|
||
const res = await fetch('https://oauth2.googleapis.com/token', {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||
body: new URLSearchParams({
|
||
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
|
||
assertion: `${unsigned}.${signature}`,
|
||
}),
|
||
});
|
||
|
||
const json = await res.json();
|
||
if (!res.ok) fail(`Google-Auth fehlgeschlagen: ${JSON.stringify(json)}`);
|
||
return json.access_token;
|
||
}
|
||
|
||
function sleep(ms) {
|
||
return new Promise((r) => setTimeout(r, ms));
|
||
}
|
||
|
||
function fail(msg) {
|
||
console.error(`\n ${msg}\n`);
|
||
process.exit(1);
|
||
}
|