diff --git a/docs/greenlens-reference-design/img_a764398e9f55.jpg b/docs/greenlens-reference-design/img_a764398e9f55.jpg new file mode 100644 index 0000000..a48fe36 Binary files /dev/null and b/docs/greenlens-reference-design/img_a764398e9f55.jpg differ diff --git a/docs/greenlens-reference-design/img_aedae96875c7.jpg b/docs/greenlens-reference-design/img_aedae96875c7.jpg new file mode 100644 index 0000000..2a73305 Binary files /dev/null and b/docs/greenlens-reference-design/img_aedae96875c7.jpg differ diff --git a/docs/greenlens-reference-design/img_bef2912f1375.jpg b/docs/greenlens-reference-design/img_bef2912f1375.jpg new file mode 100644 index 0000000..d04adfb Binary files /dev/null and b/docs/greenlens-reference-design/img_bef2912f1375.jpg differ diff --git a/docs/greenlens-reference-design/img_d8791ba62792.jpg b/docs/greenlens-reference-design/img_d8791ba62792.jpg new file mode 100644 index 0000000..df127c5 Binary files /dev/null and b/docs/greenlens-reference-design/img_d8791ba62792.jpg differ diff --git a/docs/greenlens-reference-design/img_db87c9ad89d1.jpg b/docs/greenlens-reference-design/img_db87c9ad89d1.jpg new file mode 100644 index 0000000..4534319 Binary files /dev/null and b/docs/greenlens-reference-design/img_db87c9ad89d1.jpg differ diff --git a/docs/greenlens-reference-design/img_fb8d8229b362.jpg b/docs/greenlens-reference-design/img_fb8d8229b362.jpg new file mode 100644 index 0000000..6331848 Binary files /dev/null and b/docs/greenlens-reference-design/img_fb8d8229b362.jpg differ diff --git a/server/index.js b/server/index.js index f1fa1e9..86cff5c 100644 --- a/server/index.js +++ b/server/index.js @@ -165,8 +165,10 @@ const clamp = (value, min, max) => { const nowIso = () => new Date().toISOString(); const hasImportAdminKey = Boolean(process.env.PLANT_IMPORT_ADMIN_KEY); +// Fail closed: if the admin key is not configured, admin routes deny access +// instead of becoming publicly reachable. const isAuthorizedImport = (request) => { - if (!hasImportAdminKey) return true; + if (!hasImportAdminKey) return false; const provided = request.header('x-admin-key'); return provided === process.env.PLANT_IMPORT_ADMIN_KEY; }; @@ -174,7 +176,7 @@ const isAuthorizedImport = (request) => { // Same admin secret, but also accepted as a query param since browser // navigation to /api/tiktok/connect can't set a custom header. const isAuthorizedAdminNavigation = (request) => { - if (!hasImportAdminKey) return true; + if (!hasImportAdminKey) return false; const provided = request.header('x-admin-key') || request.query?.key; return provided === process.env.PLANT_IMPORT_ADMIN_KEY; }; @@ -332,6 +334,24 @@ const toApiErrorPayload = (error) => { }; } + // TikTok (and similar provider) errors carry an HTTP status and sometimes a + // structured provider body. Preserve that instead of collapsing everything + // into a generic internal error — but never echo secrets or upload URLs. + if (error && typeof error === 'object' && Number.isInteger(error.status) && error.status >= 400) { + const providerError = error.body?.error; + return { + status: error.status >= 500 ? 502 : error.status, + body: { + code: error.status === 416 ? 'TIKTOK_UPLOAD_RANGE_ERROR' : 'PROVIDER_ERROR', + message: error.message || 'Provider request failed.', + provider_status: error.status, + ...(providerError?.code ? { provider_code: providerError.code } : {}), + ...(providerError?.log_id ? { provider_log_id: providerError.log_id } : {}), + retryable: error.status === 429 || error.status >= 500, + }, + }; + } + return { status: 500, body: { @@ -1443,35 +1463,152 @@ const tiktokApi = async (url, options = {}) => { return data; }; -const uploadBinaryToTiktok = async (uploadUrl, buffer, mimeType = 'video/mp4') => { - const res = await fetch(uploadUrl, { - method: 'PUT', - headers: { - 'Content-Type': mimeType, - 'Content-Length': String(buffer.length), - }, - body: buffer, - }); +// TikTok requires chunks between 5 MB and 64 MB; only a file that fits in a +// single chunk may be smaller than 5 MB. total_chunk_count is +// floor(size / chunk_size) — the remainder merges into the final chunk. +const TIKTOK_MAX_CHUNK_BYTES = 64 * 1024 * 1024; +const TIKTOK_CHUNK_TIMEOUT_MS = 120_000; +const TIKTOK_MAX_CHUNK_ATTEMPTS = 3; - const text = await res.text(); - if (!res.ok && res.status !== 201) { - const error = new Error(`TikTok binary upload failed: ${res.status} ${text}`); - error.status = res.status; - throw error; +const sleepMs = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +const tiktokRetryDelayMs = (res, attempt) => { + const retryAfter = res?.headers?.get?.('retry-after'); + if (retryAfter) { + const seconds = Number(retryAfter); + if (Number.isFinite(seconds) && seconds >= 0) return Math.min(seconds * 1000, 30_000); + } + return Math.min(1000 * 2 ** attempt, 15_000) + Math.floor(Math.random() * 500); +}; + +const planTiktokChunks = (totalBytes) => { + if (totalBytes <= TIKTOK_MAX_CHUNK_BYTES) { + return { chunkSize: totalBytes, totalChunkCount: 1 }; + } + const chunkSize = TIKTOK_MAX_CHUNK_BYTES; + return { chunkSize, totalChunkCount: Math.floor(totalBytes / chunkSize) }; +}; + +// Validate video type by magic bytes instead of trusting the client MIME string. +const sniffVideoMimeType = (buffer) => { + if (buffer.length >= 12 && buffer.toString('ascii', 4, 8) === 'ftyp') { + const brand = buffer.toString('ascii', 8, 12); + return brand.startsWith('qt') ? 'video/quicktime' : 'video/mp4'; + } + if (buffer.length >= 4 && buffer[0] === 0x1a && buffer[1] === 0x45 && buffer[2] === 0xdf && buffer[3] === 0xa3) { + return 'video/webm'; + } + return null; +}; + +const putTiktokChunk = async (uploadUrl, chunk, { start, end, total, mimeType, isLast }) => { + let lastError = null; + + for (let attempt = 0; attempt < TIKTOK_MAX_CHUNK_ATTEMPTS; attempt++) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), TIKTOK_CHUNK_TIMEOUT_MS); + let res = null; + try { + res = await fetch(uploadUrl, { + method: 'PUT', + headers: { + 'Content-Type': mimeType, + 'Content-Length': String(chunk.length), + 'Content-Range': `bytes ${start}-${end}/${total}`, + }, + body: chunk, + signal: controller.signal, + }); + const text = await res.text(); + // Complete upload → 201; intermediate chunk → 206. Anything else means + // the upload is not actually complete, even if it is 2xx. + const expected = isLast ? 201 : 206; + if (res.status === expected) { + return { status: res.status, body: text }; + } + lastError = new Error(`TikTok binary upload failed: ${res.status} ${text}`); + lastError.status = res.status; + const retryable = res.status === 429 || res.status >= 500; + if (!retryable) throw lastError; + } catch (err) { + if (Number.isInteger(err?.status) && !(err.status === 429 || err.status >= 500)) throw err; + lastError = Number.isInteger(err?.status) + ? err + : Object.assign(new Error(`TikTok binary upload network error: ${err?.message || err}`), { status: 502 }); + } finally { + clearTimeout(timer); + } + if (attempt < TIKTOK_MAX_CHUNK_ATTEMPTS - 1) await sleepMs(tiktokRetryDelayMs(res, attempt)); } - return { status: res.status, body: text }; + throw lastError || Object.assign(new Error('TikTok binary upload failed.'), { status: 502 }); +}; + +const uploadBinaryToTiktok = async (uploadUrl, buffer, mimeType = 'video/mp4') => { + const total = buffer.length; + if (!total) { + throw Object.assign(new Error('Refusing to upload an empty file to TikTok.'), { status: 400, code: 'BAD_REQUEST' }); + } + + const { chunkSize, totalChunkCount } = planTiktokChunks(total); + let result = { status: 0, body: '' }; + for (let i = 0; i < totalChunkCount; i++) { + const start = i * chunkSize; + const isLast = i === totalChunkCount - 1; + const end = isLast ? total - 1 : start + chunkSize - 1; + result = await putTiktokChunk(uploadUrl, buffer.subarray(start, end + 1), { + start, + end, + total, + mimeType, + isLast, + }); + } + return result; +}; + +// INITIATED is not delivery. Poll with bounded backoff (TikTok allows max 30 +// status calls/min) until the draft reaches the creator's inbox or fails. +const TIKTOK_TERMINAL_STATUSES = new Set(['SEND_TO_USER_INBOX', 'PUBLISH_COMPLETE', 'FAILED']); + +const pollTiktokPublishStatus = async (publishId, { maxWaitMs = 45_000 } = {}) => { + const delays = [2_000, 4_000, 8_000, 15_000, 30_000]; + const startedAt = Date.now(); + let last = { status: 'PROCESSING_UPLOAD', failReason: undefined, raw: null }; + + for (let i = 0; Date.now() - startedAt < maxWaitMs; i++) { + await sleepMs(delays[Math.min(i, delays.length - 1)]); + try { + const result = await tiktokApi('https://open.tiktokapis.com/v2/post/publish/status/fetch/', { + method: 'POST', + body: Buffer.from(JSON.stringify({ publish_id: publishId })), + }); + const data = result?.data || null; + last = { + status: String(data?.status || 'UNKNOWN'), + failReason: typeof data?.fail_reason === 'string' && data.fail_reason ? data.fail_reason : undefined, + raw: data, + }; + if (TIKTOK_TERMINAL_STATUSES.has(last.status)) return last; + } catch { + // A transient status-fetch failure must not fail an already-uploaded + // post; keep the last known state and try again within the window. + } + } + return last; }; // ─── TikTok Video Upload ──────────────────────────────────────────────────── app.post('/api/tiktok/upload/video', async (request, response) => { try { - if (!isAuthorizedAdminNavigation(request)) { + // Uploads are API-only, so the key is accepted via header only (never as + // a URL parameter that could land in logs). + if (!isAuthorizedImport(request)) { return response.status(401).json({ code: 'UNAUTHORIZED', message: 'Invalid or missing admin key.' }); } - const { videoBuffer, videoBase64, mimeType = 'video/mp4' } = request.body || {}; + const { videoBuffer, videoBase64 } = request.body || {}; let uploadBuffer = Buffer.isBuffer(videoBuffer) ? videoBuffer : null; if (!uploadBuffer && typeof videoBase64 === 'string') { const encoded = videoBase64.replace(/^data:video\/[^;]+;base64,/i, '').replace(/\s/g, ''); @@ -1483,17 +1620,26 @@ app.post('/api/tiktok/upload/video', async (request, response) => { if (!uploadBuffer) { return response.status(400).json({ code: 'BAD_REQUEST', message: 'videoBase64 is required.' }); } + if (!uploadBuffer.length) { + return response.status(400).json({ code: 'BAD_REQUEST', message: 'Video data must not be empty.' }); + } if (uploadBuffer.length > 24 * 1024 * 1024) { return response.status(413).json({ code: 'PAYLOAD_TOO_LARGE', message: 'TikTok video must be 24 MB or smaller.' }); } + const sniffedMimeType = sniffVideoMimeType(uploadBuffer); + if (!sniffedMimeType) { + return response.status(400).json({ code: 'BAD_REQUEST', message: 'Video must be MP4, MOV, or WebM (magic-byte check failed).' }); + } + const videoSize = uploadBuffer.length; + const chunkPlan = planTiktokChunks(videoSize); const initBody = { source_info: { source: 'FILE_UPLOAD', video_size: videoSize, - chunk_size: videoSize, - total_chunk_count: 1, + chunk_size: chunkPlan.chunkSize, + total_chunk_count: chunkPlan.totalChunkCount, }, }; @@ -1508,12 +1654,15 @@ app.post('/api/tiktok/upload/video', async (request, response) => { return response.status(502).json({ code: 'PROVIDER_ERROR', message: 'Missing upload_url or publish_id from TikTok.' }); } - await uploadBinaryToTiktok(uploadUrl, uploadBuffer, mimeType || 'video/mp4'); + await uploadBinaryToTiktok(uploadUrl, uploadBuffer, sniffedMimeType); + // Note: the secret upload_url is intentionally not returned to clients. + const finalStatus = await pollTiktokPublishStatus(publishId); response.status(200).json({ publish_id: publishId, - upload_url: uploadUrl, - status: 'INITIATED', + status: finalStatus.status, + delivered: finalStatus.status === 'SEND_TO_USER_INBOX', + ...(finalStatus.failReason ? { fail_reason: finalStatus.failReason } : {}), }); } catch (error) { const payload = toApiErrorPayload(error); @@ -1525,7 +1674,7 @@ app.post('/api/tiktok/upload/video', async (request, response) => { app.post('/api/tiktok/upload/photo', async (request, response) => { try { - if (!isAuthorizedAdminNavigation(request)) { + if (!isAuthorizedImport(request)) { return response.status(401).json({ code: 'UNAUTHORIZED', message: 'Invalid or missing admin key.' }); } @@ -1553,6 +1702,12 @@ app.post('/api/tiktok/upload/photo', async (request, response) => { const title = typeof request.body?.title === 'string' ? request.body.title.trim() : ''; const description = typeof request.body?.description === 'string' ? request.body.description.trim() : ''; + if (title.length > 90) { + return response.status(400).json({ code: 'BAD_REQUEST', message: 'title must be at most 90 characters for TikTok photo posts.' }); + } + if (description.length > 4000) { + return response.status(400).json({ code: 'BAD_REQUEST', message: 'description must be at most 4000 characters for TikTok photo posts.' }); + } const initBody = { media_type: 'PHOTO', @@ -1575,9 +1730,17 @@ app.post('/api/tiktok/upload/photo', async (request, response) => { body: Buffer.from(JSON.stringify(initBody)), }); + const publishId = initResult?.data?.publish_id; + if (!publishId) { + return response.status(502).json({ code: 'PROVIDER_ERROR', message: 'Missing publish_id from TikTok.' }); + } + + const finalStatus = await pollTiktokPublishStatus(publishId); response.status(200).json({ - publish_id: initResult?.data?.publish_id, - status: initResult?.data?.status || 'INITIATED', + publish_id: publishId, + status: finalStatus.status, + delivered: finalStatus.status === 'SEND_TO_USER_INBOX', + ...(finalStatus.failReason ? { fail_reason: finalStatus.failReason } : {}), }); } catch (error) { const payload = toApiErrorPayload(error); @@ -1589,7 +1752,7 @@ app.post('/api/tiktok/upload/photo', async (request, response) => { app.get('/api/tiktok/upload/status', async (request, response) => { try { - if (!isAuthorizedAdminNavigation(request)) { + if (!isAuthorizedImport(request)) { return response.status(401).json({ code: 'UNAUTHORIZED', message: 'Invalid or missing admin key.' }); } diff --git a/server/lib/tiktok-assets.js b/server/lib/tiktok-assets.js index c910d2a..c275afb 100644 --- a/server/lib/tiktok-assets.js +++ b/server/lib/tiktok-assets.js @@ -120,14 +120,54 @@ const downloadRemotePhoto = async (sourceUrl) => { } }; +const sleepMs = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +// Preflight before TikTok ever sees a URL: it must answer 200 without a +// redirect, be JPEG or WebP, and stay within TikTok's 20 MiB limit. Also +// serves as the read-after-write check right after a MinIO upload. Retries a +// couple of times to ride out brief 404/5xx windows after a fresh write. +const verifyPublicPhotoUrl = async (url, { attempts = 3, fetchImpl = fetch } = {}) => { + let lastReason = 'unreachable'; + for (let attempt = 0; attempt < attempts; attempt++) { + if (attempt > 0) await sleepMs(1000 * attempt); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 10_000); + try { + const response = await fetchImpl(url, { method: 'HEAD', redirect: 'error', signal: controller.signal }); + if (response.status !== 200) { + lastReason = `HTTP ${response.status}`; + continue; + } + const contentType = String(response.headers.get('content-type') || '').split(';')[0].trim().toLowerCase(); + if (!['image/jpeg', 'image/webp'].includes(contentType)) { + lastReason = `unsupported content type ${contentType || '(none)'}`; + break; + } + const size = Number(response.headers.get('content-length') || 0); + if (size > MAX_TIKTOK_PHOTO_BYTES) { + lastReason = 'larger than 20 MB'; + break; + } + return; + } catch { + lastReason = 'unreachable or redirected'; + } finally { + clearTimeout(timeout); + } + } + throw createInputError(`Photo URL is not usable for TikTok (${lastReason}): ${url}`); +}; + const prepareTiktokPhotoUrls = async (photos, options) => { const publicBaseUrl = normalizePublicBaseUrl(options.publicBaseUrl); if (!publicBaseUrl) throw new Error('MINIO_PUBLIC_URL is not configured.'); + const verifyUrl = options.verifyPhotoUrl || verifyPublicPhotoUrl; const urls = []; for (const photo of photos) { const sourceUrl = typeof photo === 'string' ? photo.trim() : String(photo?.url || '').trim(); if (sourceUrl && isOwnedTiktokMediaUrl(sourceUrl, publicBaseUrl)) { + await verifyUrl(sourceUrl); urls.push(sourceUrl); continue; } @@ -145,6 +185,7 @@ const prepareTiktokPhotoUrls = async (photos, options) => { if (!isOwnedTiktokMediaUrl(uploaded?.url, publicBaseUrl)) { throw new Error('GreenLens image storage returned an unverified public URL.'); } + await verifyUrl(uploaded.url); urls.push(uploaded.url); } @@ -155,5 +196,6 @@ module.exports = { MAX_TIKTOK_PHOTO_BYTES, isAllowedR2MediaUrl, isOwnedTiktokMediaUrl, + verifyPublicPhotoUrl, prepareTiktokPhotoUrls, }; diff --git a/server/test/tiktok-assets.test.js b/server/test/tiktok-assets.test.js index fe373b3..5673ef9 100644 --- a/server/test/tiktok-assets.test.js +++ b/server/test/tiktok-assets.test.js @@ -1,7 +1,9 @@ const test = require('node:test'); const assert = require('node:assert/strict'); -const { isAllowedR2MediaUrl, isOwnedTiktokMediaUrl, prepareTiktokPhotoUrls } = require('../lib/tiktok-assets'); +const { isAllowedR2MediaUrl, isOwnedTiktokMediaUrl, prepareTiktokPhotoUrls, verifyPublicPhotoUrl } = require('../lib/tiktok-assets'); + +const verifyPhotoUrl = async () => {}; const publicBaseUrl = 'https://greenlenspro.com/storage'; @@ -17,6 +19,7 @@ test('keeps existing GreenLens storage URLs without uploading them again', async const url = 'https://greenlenspro.com/storage/plant-images/slide.jpg'; const result = await prepareTiktokPhotoUrls([url], { publicBaseUrl, + verifyPhotoUrl, uploadImage: async () => { uploads += 1; }, }); @@ -28,6 +31,7 @@ test('hosts base64 photos on the verified GreenLens domain', async () => { const uploaded = []; const result = await prepareTiktokPhotoUrls([{ imageBase64: Buffer.from('source').toString('base64') }], { publicBaseUrl, + verifyPhotoUrl, normalizePhoto: async (buffer) => { assert.equal(buffer.toString(), 'source'); return Buffer.from('jpeg'); @@ -54,6 +58,7 @@ test('rehosts an external R2 photo instead of leaking its URL into the TikTok pa const sourceUrl = 'https://pub-example.r2.dev/slide.png'; const result = await prepareTiktokPhotoUrls([sourceUrl], { publicBaseUrl, + verifyPhotoUrl, downloadPhoto: async (url) => { assert.equal(url, sourceUrl); return Buffer.from('png'); @@ -73,6 +78,7 @@ test('prefers supplied base64 over an external legacy URL', async () => { imageBase64: Buffer.from('local').toString('base64'), }], { publicBaseUrl, + verifyPhotoUrl, downloadPhoto: async () => { downloaded = true; }, normalizePhoto: async (buffer) => { assert.equal(buffer.toString(), 'local'); @@ -89,9 +95,56 @@ test('rejects a storage result outside the verified GreenLens domain', async () await assert.rejects( prepareTiktokPhotoUrls([{ imageBase64: Buffer.from('source').toString('base64') }], { publicBaseUrl, + verifyPhotoUrl, normalizePhoto: async () => Buffer.from('jpeg'), uploadImage: async () => ({ url: 'https://pub-example.r2.dev/slide.jpg' }), }), /unverified public URL/, ); }); + +const okHeaders = (extra = {}) => ({ + get: (name) => ({ 'content-type': 'image/jpeg', 'content-length': '1024', ...extra }[name.toLowerCase()] ?? null), +}); + +test('verifyPublicPhotoUrl accepts a reachable JPEG under 20 MB', async () => { + await verifyPublicPhotoUrl('https://greenlenspro.com/storage/a.jpg', { + fetchImpl: async () => ({ status: 200, headers: okHeaders() }), + }); +}); + +test('verifyPublicPhotoUrl retries transient 404s before failing', async () => { + let calls = 0; + await verifyPublicPhotoUrl('https://greenlenspro.com/storage/a.jpg', { + fetchImpl: async () => { + calls += 1; + return calls < 3 + ? { status: 404, headers: okHeaders() } + : { status: 200, headers: okHeaders() }; + }, + }); + assert.equal(calls, 3); +}); + +test('verifyPublicPhotoUrl rejects unsupported content types without retrying', async () => { + let calls = 0; + await assert.rejects( + verifyPublicPhotoUrl('https://greenlenspro.com/storage/a.gif', { + fetchImpl: async () => { + calls += 1; + return { status: 200, headers: { get: (n) => (n.toLowerCase() === 'content-type' ? 'image/gif' : null) } }; + }, + }), + /unsupported content type/, + ); + assert.equal(calls, 1); +}); + +test('verifyPublicPhotoUrl rejects photos larger than 20 MB', async () => { + await assert.rejects( + verifyPublicPhotoUrl('https://greenlenspro.com/storage/a.jpg', { + fetchImpl: async () => ({ status: 200, headers: okHeaders({ 'content-length': String(21 * 1024 * 1024) }) }), + }), + /larger than 20 MB/, + ); +});