tknuth/Greenlens
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bug fixes

Browse Source
This commit is contained in:
Timo Knuth
2026-07-05 17:52:43 +02:00
parent fb6108bd0c
commit c40531b6ae
12 changed files with 1196 additions and 946 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
server/index.js
View File

@@ -67,6 +67,7 @@ const {
isConfigured: isOpenAiConfigured,
} = require('./lib/openai');
const { applyCatalogGrounding, normalizeText } = require('./lib/scanGrounding');
const { decideReviewOutcome, reviewAgreesWithPrimary } = require('./lib/scanReview');
const { ensureStorageBucketWithRetry, uploadImage, isStorageConfigured } = require('./lib/storage');
const { isPurchaseEventType, notifyPurchase, notifyNewUser } = require('./lib/discord');
const {
@@ -85,6 +86,9 @@ const SCAN_REVIEW_COST = 0;
const SEMANTIC_SEARCH_COST = 2;
const HEALTH_CHECK_COST = 2;
const LOW_CONFIDENCE_REVIEW_THRESHOLD = 0.8;
// Below this the app should treat the identification as uncertain and nudge
// the user toward a clearer photo instead of presenting the name as settled.
const LOW_CONFIDENCE_RESULT_THRESHOLD = 0.6;
let catalogCache = null;
@@ -738,6 +742,7 @@ app.post('/v1/scan', async (request, response) => {
const scanPlan = accountSnapshot.plan === 'pro' ? 'pro' : 'free';
let result = pickCatalogFallback(catalogEntries, imageUri, false, { silent: true });
let usedOpenAi = false;
let rawPrimaryResult = null;
if (isOpenAiConfigured()) {
console.log(`Starting OpenAI identification for user ${userId} using model ${getScanModel(scanPlan)} (plan: ${scanPlan})`);
@@ -753,9 +758,9 @@ app.post('/v1/scan', async (request, response) => {
);
if (openAiPrimary?.result) {
console.log(`OpenAI primary identification successful for user ${userId}: ${openAiPrimary.result.name} (${openAiPrimary.result.confidence}) using ${openAiPrimary.modelUsed}`);
rawPrimaryResult = openAiPrimary.result;
const grounded = applyCatalogGrounding(openAiPrimary.result, catalogEntries, language);
result = grounded.result;
if (!grounded.grounded) result = { ...result, confidence: clamp(Math.max(result.confidence || 0.6, 0.72), 0.05, 0.99) };
usedOpenAi = true;
modelUsed = openAiPrimary.modelUsed || modelUsed;
modelPath.push('openai-primary');
@@ -806,12 +811,23 @@ app.post('/v1/scan', async (request, response) => {
);
if (openAiReview?.result) {
console.log(`OpenAI review identification successful for user ${userId}: ${openAiReview.result.name} (${openAiReview.result.confidence}) using ${openAiReview.modelUsed}`);
const agrees = reviewAgreesWithPrimary(rawPrimaryResult, openAiReview.result);
const grounded = applyCatalogGrounding(openAiReview.result, catalogEntries, language);
result = grounded.result;
if (!grounded.grounded) result = { ...result, confidence: clamp(Math.max(result.confidence || 0.6, 0.72), 0.05, 0.99) };
modelUsed = openAiReview.modelUsed || modelUsed;
modelPath.push('openai-review');
if (grounded.grounded) modelPath.push('catalog-grounded-review');
const decision = decideReviewOutcome({ primaryResult: result, reviewResult: grounded.result, agrees });
if (decision.accept) {
// modelUsed and the grounding marker describe the RESULT the user
// gets, so they only change when the review actually replaces it.
if (decision.replace) {
result = grounded.result;
modelUsed = openAiReview.modelUsed || modelUsed;
if (grounded.grounded) modelPath.push('catalog-grounded-review');
}
modelPath.push('openai-review');
modelPath.push(decision.reason);
} else {
console.log(`OpenAI review disagreed at lower confidence for user ${userId} (${grounded.result.name} ${grounded.result.confidence} vs ${result.name} ${result.confidence}) — keeping primary result.`);
modelPath.push(decision.reason);
}
} else {
console.warn(`OpenAI review identification returned null for user ${userId}.`, {
attemptedModels: openAiReview?.attemptedModels,
@@ -840,6 +856,7 @@ app.post('/v1/scan', async (request, response) => {
const payload = {
result,
lowConfidence: (result.confidence || 0) < LOW_CONFIDENCE_RESULT_THRESHOLD,
creditsCharged,
modelPath,
modelUsed,

220
server/lib/openai.js
View File

@@ -5,6 +5,8 @@ const OPENAI_HEALTH_MODEL = (process.env.OPENAI_HEALTH_MODEL || process.env.EXPO
const OPENAI_SCAN_FALLBACK_MODELS = (process.env.OPENAI_SCAN_FALLBACK_MODELS || process.env.EXPO_PUBLIC_OPENAI_SCAN_FALLBACK_MODELS || 'gpt-5-mini,gpt-4.1-mini').trim();
const OPENAI_SCAN_FALLBACK_MODELS_PRO = (process.env.OPENAI_SCAN_FALLBACK_MODELS_PRO || process.env.EXPO_PUBLIC_OPENAI_SCAN_FALLBACK_MODELS_PRO || OPENAI_SCAN_FALLBACK_MODELS).trim();
const OPENAI_HEALTH_FALLBACK_MODELS = (process.env.OPENAI_HEALTH_FALLBACK_MODELS || process.env.EXPO_PUBLIC_OPENAI_HEALTH_FALLBACK_MODELS || OPENAI_SCAN_FALLBACK_MODELS).trim();
const OPENAI_SCAN_REVIEW_MODEL = (process.env.OPENAI_SCAN_REVIEW_MODEL || process.env.EXPO_PUBLIC_OPENAI_SCAN_REVIEW_MODEL || 'gpt-5').trim();
const OPENAI_SCAN_REVIEW_FALLBACK_MODELS = (process.env.OPENAI_SCAN_REVIEW_FALLBACK_MODELS || process.env.EXPO_PUBLIC_OPENAI_SCAN_REVIEW_FALLBACK_MODELS || 'gpt-5-mini,gpt-4.1-mini').trim();
const OPENAI_CHAT_COMPLETIONS_URL = (process.env.OPENAI_CHAT_COMPLETIONS_URL || 'https://api.openai.com/v1/chat/completions').trim();
const OPENAI_TIMEOUT_MS = (() => {
const raw = (process.env.OPENAI_TIMEOUT_MS || process.env.EXPO_PUBLIC_OPENAI_TIMEOUT_MS || '45000').trim();
@@ -26,19 +28,23 @@ const parseModelChain = (primaryModel, fallbackModels) => {
const OPENAI_SCAN_MODEL_CHAIN = parseModelChain(OPENAI_SCAN_MODEL, OPENAI_SCAN_FALLBACK_MODELS);
const OPENAI_SCAN_MODEL_CHAIN_PRO = parseModelChain(OPENAI_SCAN_MODEL_PRO, OPENAI_SCAN_FALLBACK_MODELS_PRO);
const OPENAI_HEALTH_MODEL_CHAIN = parseModelChain(OPENAI_HEALTH_MODEL, OPENAI_HEALTH_FALLBACK_MODELS);
const OPENAI_SCAN_REVIEW_MODEL_CHAIN = parseModelChain(OPENAI_SCAN_REVIEW_MODEL, OPENAI_SCAN_REVIEW_FALLBACK_MODELS);
const getScanModelChain = (plan) => {
return plan === 'pro' ? OPENAI_SCAN_MODEL_CHAIN_PRO : OPENAI_SCAN_MODEL_CHAIN;
};
const isReasoningModel = (model) => {
const normalized = String(model || '').toLowerCase();
return normalized.startsWith('gpt-5') || normalized.startsWith('o1') || normalized.startsWith('o3') || normalized.startsWith('o4');
};
const clamp = (value, min, max) => {
return Math.min(max, Math.max(min, value));
};
const getScanModelChain = (plan, mode = 'primary') => {
// The review pass exists to catch low-confidence primary IDs, so re-running
// the primary model on the same image adds nothing — use a stronger model.
if (mode === 'review') return OPENAI_SCAN_REVIEW_MODEL_CHAIN;
return plan === 'pro' ? OPENAI_SCAN_MODEL_CHAIN_PRO : OPENAI_SCAN_MODEL_CHAIN;
};
const isReasoningModel = (model) => {
const normalized = String(model || '').toLowerCase();
return normalized.startsWith('gpt-5') || normalized.startsWith('o1') || normalized.startsWith('o3') || normalized.startsWith('o4');
};
const clamp = (value, min, max) => {
return Math.min(max, Math.max(min, value));
};
const toErrorMessage = (error) => {
if (error instanceof Error) return error.message;
@@ -142,10 +148,10 @@ const normalizeIdentifyResult = (raw, language) => {
};
const normalizeHealthAnalysis = (raw, language) => {
const scoreRaw = getNumber(raw.overallHealthScore);
const statusRaw = getString(raw.status);
const analysisSummary = getString(raw.analysisSummary);
const issuesRaw = raw.likelyIssues;
const scoreRaw = getNumber(raw.overallHealthScore);
const statusRaw = getString(raw.status);
const analysisSummary = getString(raw.analysisSummary);
const issuesRaw = raw.likelyIssues;
const actionsNowRaw = getStringArray(raw.actionsNow).slice(0, 8);
const plan7DaysRaw = getStringArray(raw.plan7Days).slice(0, 10);
@@ -181,10 +187,10 @@ const normalizeHealthAnalysis = (raw, language) => {
? 'La IA no pudo extraer senales de salud estables.'
: 'AI could not extract stable health signals.';
return {
overallHealthScore: Math.round(clamp(score, 0, 100)),
status,
analysisSummary: analysisSummary || fallbackIssue,
likelyIssues: [
overallHealthScore: Math.round(clamp(score, 0, 100)),
status,
analysisSummary: analysisSummary || fallbackIssue,
likelyIssues: [
{
title: language === 'de'
? 'Analyse unsicher'
@@ -205,10 +211,10 @@ const normalizeHealthAnalysis = (raw, language) => {
}
return {
overallHealthScore: Math.round(clamp(score, 0, 100)),
status,
analysisSummary,
likelyIssues,
overallHealthScore: Math.round(clamp(score, 0, 100)),
status,
analysisSummary,
likelyIssues,
actionsNow: actionsNowRaw,
plan7Days: plan7DaysRaw,
};
@@ -223,12 +229,12 @@ const buildIdentifyPrompt = (language, mode) => {
? '- "name" must be an English common name only. Never return a German or other non-English common name. If no reliable English common name is known, use "botanicalName" as "name" instead of inventing or translating.'
: `- "name" must be strictly written in ${getLanguageLabel(language)}. If a reliable common name in that language is not known, use "botanicalName" as "name" instead of inventing a localized name.`;
return [
`${reviewInstruction}`,
'If the image does not clearly show a plant (for example a person, animal, room, furniture, or no identifiable foliage), return {"notAPlant":true} and nothing else.',
'Return strict JSON only in this shape:',
'{"name":"...","botanicalName":"...","confidence":0.0,"description":"...","careInfo":{"waterIntervalDays":7,"light":"...","temp":"..."}}',
'Rules:',
return [
`${reviewInstruction}`,
'If the image does not clearly show a plant (for example a person, animal, room, furniture, or no identifiable foliage), return {"notAPlant":true} and nothing else.',
'Return strict JSON only in this shape:',
'{"name":"...","botanicalName":"...","confidence":0.0,"description":"...","careInfo":{"waterIntervalDays":7,"light":"...","temp":"..."}}',
'Rules:',
nameLanguageInstruction,
`- "description" and "careInfo.light" must be written in ${getLanguageLabel(language)}.`,
`- "careInfo.light": short light requirement in ${getLanguageLabel(language)} (e.g. "bright indirect light", "full sun", "partial shade"). Must always be a real value, never "Unknown".`,
@@ -263,13 +269,13 @@ const buildHealthPrompt = (language, plantContext) => {
'Inspect the following in detail: leaf color (yellowing, browning, bleaching, dark spots, necrosis), leaf texture (wilting, crispy edges, curling, drooping), stem condition (rot, soft spots, discoloration), soil surface (dry cracks, mold, pests, waterlogging signs), visible pests (spider mites, fungus gnats, scale insects, aphids, mealybugs), root health (if visible), pot size and drainage.',
'',
'Return strict JSON only in this exact shape:',
'{"overallHealthScore":72,"status":"watch","analysisSummary":"...","likelyIssues":[{"title":"...","confidence":0.64,"details":"..."}],"actionsNow":["..."],"plan7Days":["..."]}',
'{"overallHealthScore":72,"status":"watch","analysisSummary":"...","likelyIssues":[{"title":"...","confidence":0.64,"details":"..."}],"actionsNow":["..."],"plan7Days":["..."]}',
'',
'Rules:',
'- "overallHealthScore": integer 0–100. 100=perfect health, 80–99=minor cosmetic only, 60–79=noticeable issues needing attention, 40–59=significant stress, below 40=severe/critical.',
'- "status": exactly one of "healthy" (score>=80, no active threats), "watch" (score 50–79, needs monitoring), "critical" (score<50, urgent action needed).',
`- "analysisSummary": 6 to 9 precise sentences in ${getLanguageLabel(language)} describing visible condition, symptom pattern, likely root cause, urgency, confidence limits, and what the owner should monitor next.`,
'- "likelyIssues": 2 to 4 items, sorted by confidence descending. Each item:',
`- "analysisSummary": 6 to 9 precise sentences in ${getLanguageLabel(language)} describing visible condition, symptom pattern, likely root cause, urgency, confidence limits, and what the owner should monitor next.`,
'- "likelyIssues": 2 to 4 items, sorted by confidence descending. Each item:',
' - "title": concise issue name (e.g. "Overwatering / Root Rot Risk")',
' - "confidence": float 0.05–0.99 reflecting visual certainty',
' - "details": 2–4 sentence detailed explanation of what you observe visually, what causes it, and what happens if untreated. Be specific — mention leaf color, location, pattern.',
@@ -289,33 +295,33 @@ const extractMessageContent = (payload) => {
.join('')
.trim();
}
return '';
};
const buildRequestBody = ({ model, messages, temperature, maxCompletionTokens }) => {
const body = {
model,
response_format: { type: 'json_object' },
messages,
};
if (typeof temperature === 'number') body.temperature = temperature;
if (isReasoningModel(model)) {
body.reasoning_effort = 'minimal';
body.max_completion_tokens = maxCompletionTokens;
} else {
body.max_tokens = maxCompletionTokens;
}
return body;
};
const postChatCompletion = async ({ modelChain, messages, imageUri, temperature, maxCompletionTokens = 600 }) => {
if (!OPENAI_API_KEY) return null;
if (typeof fetch !== 'function') {
throw new Error('Global fetch is not available in this Node runtime.');
}
return '';
};
const buildRequestBody = ({ model, messages, temperature, maxCompletionTokens }) => {
const body = {
model,
response_format: { type: 'json_object' },
messages,
};
if (typeof temperature === 'number') body.temperature = temperature;
if (isReasoningModel(model)) {
body.reasoning_effort = 'minimal';
body.max_completion_tokens = maxCompletionTokens;
} else {
body.max_tokens = maxCompletionTokens;
}
return body;
};
const postChatCompletion = async ({ modelChain, messages, imageUri, temperature, maxCompletionTokens = 600 }) => {
if (!OPENAI_API_KEY) return null;
if (typeof fetch !== 'function') {
throw new Error('Global fetch is not available in this Node runtime.');
}
const attemptedModels = [];
@@ -324,13 +330,13 @@ const postChatCompletion = async ({ modelChain, messages, imageUri, temperature,
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), OPENAI_TIMEOUT_MS);
try {
const body = buildRequestBody({ model, messages, temperature, maxCompletionTokens });
const response = await fetch(OPENAI_CHAT_COMPLETIONS_URL, {
method: 'POST',
headers: {
try {
const body = buildRequestBody({ model, messages, temperature, maxCompletionTokens });
const response = await fetch(OPENAI_CHAT_COMPLETIONS_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${OPENAI_API_KEY}`,
},
@@ -377,7 +383,7 @@ const postChatCompletion = async ({ modelChain, messages, imageUri, temperature,
const identifyPlant = async ({ imageUri, language, mode = 'primary', plan = 'free' }) => {
if (!OPENAI_API_KEY) return { result: null, modelUsed: null, attemptedModels: [] };
const modelChain = getScanModelChain(plan);
const modelChain = getScanModelChain(plan, mode);
const completion = await postChatCompletion({
modelChain,
imageUri,
@@ -386,16 +392,16 @@ const identifyPlant = async ({ imageUri, language, mode = 'primary', plan = 'fre
role: 'system',
content: 'You are a plant identification assistant. Return strict JSON only.',
},
{
role: 'user',
content: [
{ type: 'text', text: buildIdentifyPrompt(language, mode) },
{ type: 'image_url', image_url: { url: imageUri, detail: 'low' } },
],
},
],
maxCompletionTokens: 600,
});
{
role: 'user',
content: [
{ type: 'text', text: buildIdentifyPrompt(language, mode) },
{ type: 'image_url', image_url: { url: imageUri, detail: 'low' } },
],
},
],
maxCompletionTokens: 600,
});
if (!completion?.payload) {
return {
@@ -416,25 +422,25 @@ const identifyPlant = async ({ imageUri, language, mode = 'primary', plan = 'fre
}
const parsed = parseContentToJson(content);
if (!parsed) {
console.warn('OpenAI identify returned non-JSON content.', {
model: completion.modelUsed || modelChain[0],
mode,
preview: content.slice(0, 220),
});
return { result: null, modelUsed: completion.modelUsed, attemptedModels: completion.attemptedModels };
}
if (parsed.notAPlant === true) {
const error = new Error('Image does not contain a plant.');
error.code = 'NOT_A_PLANT';
throw error;
}
const normalized = normalizeIdentifyResult(parsed, language);
if (!normalized) {
console.warn('OpenAI identify JSON did not match schema.', {
model: completion.modelUsed || modelChain[0],
if (!parsed) {
console.warn('OpenAI identify returned non-JSON content.', {
model: completion.modelUsed || modelChain[0],
mode,
preview: content.slice(0, 220),
});
return { result: null, modelUsed: completion.modelUsed, attemptedModels: completion.attemptedModels };
}
if (parsed.notAPlant === true) {
const error = new Error('Image does not contain a plant.');
error.code = 'NOT_A_PLANT';
throw error;
}
const normalized = normalizeIdentifyResult(parsed, language);
if (!normalized) {
console.warn('OpenAI identify JSON did not match schema.', {
model: completion.modelUsed || modelChain[0],
mode,
keys: Object.keys(parsed),
});
@@ -453,16 +459,16 @@ const analyzePlantHealth = async ({ imageUri, language, plantContext }) => {
role: 'system',
content: 'You are a plant health diagnosis assistant. Return strict JSON only.',
},
{
role: 'user',
content: [
{ type: 'text', text: buildHealthPrompt(language, plantContext) },
{ type: 'image_url', image_url: { url: imageUri, detail: 'low' } },
],
},
],
maxCompletionTokens: 800,
});
{
role: 'user',
content: [
{ type: 'text', text: buildHealthPrompt(language, plantContext) },
{ type: 'image_url', image_url: { url: imageUri, detail: 'low' } },
],
},
],
maxCompletionTokens: 800,
});
if (!completion?.payload) {
return {

2
server/lib/scanGrounding.js
View File

@@ -111,7 +111,7 @@ const applyCatalogGrounding = (aiResult, catalogEntries, language = 'en') => {
result: {
name: useCatalogName ? matchedEntry.name || aiResult.name : aiResult.name,
botanicalName: matchedEntry.botanicalName || aiResult.botanicalName,
confidence: clamp(Math.max(aiResult.confidence || 0.6, 0.78), 0.05, 0.99),
confidence: clamp(aiResult.confidence || 0.6, 0.05, 0.99),
description: aiResult.description || matchedEntry.description || '',
careInfo: {
waterIntervalDays: Math.max(1, Number(matchedEntry.careInfo?.waterIntervalDays) || Number(aiResult.careInfo?.waterIntervalDays) || 7),

46
server/lib/scanReview.js Normal file
View File

@@ -0,0 +1,46 @@
const { normalizeText } = require('./scanGrounding');
// Agreement must be judged on the RAW model answers, not the grounded ones:
// catalog grounding has a genus-level fallback that can collapse two different
// species onto the same catalog entry and fake an agreement.
const reviewAgreesWithPrimary = (rawPrimary, rawReview) => {
if (!rawPrimary || !rawReview) return false;
const primaryBotanical = normalizeText(rawPrimary.botanicalName);
const primaryName = normalizeText(rawPrimary.name);
const botanicalMatch = Boolean(primaryBotanical) && primaryBotanical === normalizeText(rawReview.botanicalName);
const nameMatch = Boolean(primaryName) && primaryName === normalizeText(rawReview.name);
return botanicalMatch || nameMatch;
};
// The review runs on a stronger model chain than the primary, and models are
// not calibrated against each other — so on disagreement the review wins even
// when it trails the primary by up to this margin.
const REVIEW_DISAGREEMENT_MARGIN = 0.05;
// A second low-confidence guess is not a verification: the review may only
// replace the primary when it agrees with it or is (near-)competitively
// confident. On agreement the higher-confidence variant wins (tie goes to
// the review, which runs on the stronger model chain).
const decideReviewOutcome = ({ primaryResult, reviewResult, agrees }) => {
const primaryConfidence = primaryResult?.confidence || 0;
const reviewConfidence = reviewResult?.confidence || 0;
if (agrees) {
return {
accept: true,
replace: reviewConfidence >= primaryConfidence,
reason: 'review-confirmed-primary',
};
}
if (reviewConfidence >= primaryConfidence - REVIEW_DISAGREEMENT_MARGIN) {
return { accept: true, replace: true, reason: 'review-overrode-primary' };
}
return { accept: false, replace: false, reason: 'review-rejected-low-confidence' };
};
module.exports = {
decideReviewOutcome,
reviewAgreesWithPrimary,
};