Files
Greenlens/services/backend/openAiScanService.ts
2026-05-08 13:00:30 +02:00

605 lines
23 KiB
TypeScript

import { CareInfo, IdentificationResult, Language } from '../../types';
import { BackendApiError } from './contracts';
type OpenAiScanMode = 'primary' | 'review';
export interface OpenAiHealthIssue {
title: string;
confidence: number;
details: string;
}
export interface OpenAiHealthAnalysis {
overallHealthScore: number;
status: 'healthy' | 'watch' | 'critical';
analysisSummary?: string;
likelyIssues: OpenAiHealthIssue[];
actionsNow: string[];
plan7Days: string[];
}
const OPENAI_API_KEY = (process.env.EXPO_PUBLIC_OPENAI_API_KEY || '').trim();
const OPENAI_SCAN_MODEL = (process.env.EXPO_PUBLIC_OPENAI_SCAN_MODEL || 'gpt-5-mini').trim();
const OPENAI_SCAN_MODEL_PRO = (process.env.EXPO_PUBLIC_OPENAI_SCAN_MODEL_PRO || OPENAI_SCAN_MODEL).trim();
const OPENAI_HEALTH_MODEL = (process.env.EXPO_PUBLIC_OPENAI_HEALTH_MODEL || OPENAI_SCAN_MODEL).trim();
const OPENAI_SCAN_FALLBACK_MODELS = (process.env.EXPO_PUBLIC_OPENAI_SCAN_FALLBACK_MODELS || 'gpt-5-mini,gpt-4.1-mini').trim();
const OPENAI_SCAN_FALLBACK_MODELS_PRO = (process.env.EXPO_PUBLIC_OPENAI_SCAN_FALLBACK_MODELS_PRO || OPENAI_SCAN_FALLBACK_MODELS).trim();
const OPENAI_HEALTH_FALLBACK_MODELS = (process.env.EXPO_PUBLIC_OPENAI_HEALTH_FALLBACK_MODELS || OPENAI_SCAN_FALLBACK_MODELS).trim();
const OPENAI_CHAT_COMPLETIONS_URL = 'https://api.openai.com/v1/chat/completions';
const OPENAI_TIMEOUT_MS = (() => {
const raw = (process.env.EXPO_PUBLIC_OPENAI_TIMEOUT_MS || '45000').trim();
const parsed = Number.parseInt(raw, 10);
if (Number.isFinite(parsed) && parsed >= 10000) return parsed;
return 45000;
})();
const parseModelChain = (primaryModel: string, fallbackModels: string): string[] => {
const models = [primaryModel];
fallbackModels.split(',').forEach((model) => {
const normalized = model.trim();
if (normalized) models.push(normalized);
});
return [...new Set(models)];
};
const OPENAI_SCAN_MODEL_CHAIN = parseModelChain(OPENAI_SCAN_MODEL, OPENAI_SCAN_FALLBACK_MODELS);
const OPENAI_SCAN_MODEL_CHAIN_PRO = parseModelChain(OPENAI_SCAN_MODEL_PRO, OPENAI_SCAN_FALLBACK_MODELS_PRO);
const OPENAI_HEALTH_MODEL_CHAIN = parseModelChain(OPENAI_HEALTH_MODEL, OPENAI_HEALTH_FALLBACK_MODELS);
const getScanModelChain = (plan: 'free' | 'pro'): string[] => {
return plan === 'pro' ? OPENAI_SCAN_MODEL_CHAIN_PRO : OPENAI_SCAN_MODEL_CHAIN;
};
const clamp = (value: number, min: number, max: number): number => {
return Math.min(max, Math.max(min, value));
};
const toErrorMessage = (error: unknown): string => {
if (error instanceof Error) return error.message;
return String(error);
};
const summarizeImageUri = (imageUri: string): string => {
const trimmed = imageUri.trim();
if (!trimmed) return 'empty';
if (trimmed.startsWith('data:image')) return `data-uri(${Math.round(trimmed.length / 1024)}kb)`;
return trimmed.length > 120 ? `${trimmed.slice(0, 120)}...` : trimmed;
};
const toJsonString = (content: string): string => {
const trimmed = content.trim();
if (!trimmed) return trimmed;
const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
if (fenced?.[1]) return fenced[1].trim();
return trimmed;
};
const parseContentToJson = (content: string): Record<string, unknown> | null => {
try {
const parsed = JSON.parse(toJsonString(content));
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
return parsed as Record<string, unknown>;
}
return null;
} catch {
return null;
}
};
const getString = (value: unknown): string => {
return typeof value === 'string' ? value.trim() : '';
};
const getNumber = (value: unknown): number | null => {
if (typeof value === 'number' && Number.isFinite(value)) return value;
if (typeof value === 'string' && value.trim()) {
const parsed = Number(value);
if (Number.isFinite(parsed)) return parsed;
}
return null;
};
const getStringArray = (value: unknown): string[] => {
if (!Array.isArray(value)) return [];
return value
.map((item) => (typeof item === 'string' ? item.trim() : ''))
.filter(Boolean);
};
const normalizeResult = (
raw: Record<string, unknown>,
language: Language,
): IdentificationResult | null => {
const name = getString(raw.name);
const botanicalName = getString(raw.botanicalName);
const description = getString(raw.description);
const confidenceRaw = getNumber(raw.confidence);
const careInfoRaw = raw.careInfo;
if (!name || !botanicalName || !careInfoRaw || typeof careInfoRaw !== 'object' || Array.isArray(careInfoRaw)) {
return null;
}
const careInfoObj = careInfoRaw as Record<string, unknown>;
const waterIntervalRaw = getNumber(careInfoObj.waterIntervalDays);
const light = getString(careInfoObj.light);
const temp = getString(careInfoObj.temp);
if (waterIntervalRaw == null || !light || !temp) {
return null;
}
const fallbackDescription = language === 'de'
? `${name} wurde per KI erkannt. Pflegehinweise sind unten aufgefuehrt.`
: language === 'es'
? `${name} se detecto con IA. Debajo veras recomendaciones de cuidado.`
: `${name} was identified with AI. Care guidance is shown below.`;
return {
name,
botanicalName,
confidence: clamp(confidenceRaw ?? 0.72, 0.05, 0.99),
description: description || fallbackDescription,
careInfo: {
waterIntervalDays: Math.round(clamp(waterIntervalRaw, 1, 45)),
light,
temp,
},
};
};
const getLanguageLabel = (language: Language): string => {
if (language === 'de') return 'German';
if (language === 'es') return 'Spanish';
return 'English';
};
const buildPrompt = (language: Language, mode: OpenAiScanMode): string => {
const reviewInstruction = mode === 'review'
? 'Re-check your first hypothesis with stricter botanical accuracy and correct any mismatch.'
: 'Identify the most likely houseplant species from this image with conservative confidence.';
const nameLanguageInstruction = language === 'en'
? '- "name" must be an English common name only. Never return a German or other non-English common name. If no reliable English common name is known, use "botanicalName" as "name" instead of inventing or translating.'
: `- "name" must be strictly written in ${getLanguageLabel(language)}. If a reliable common name in that language is not known, use "botanicalName" as "name" instead of inventing a localized name.`;
return [
`${reviewInstruction}`,
`If the image does not clearly show a plant (e.g. it shows a person, animal, object, or has no identifiable plant), return {"notAPlant":true} and nothing else.`,
`Otherwise return strict JSON only in this shape:`,
`{"name":"...","botanicalName":"...","confidence":0.0,"description":"...","careInfo":{"waterIntervalDays":7,"light":"...","temp":"..."}}`,
`Rules:`,
nameLanguageInstruction,
`- "description" and "careInfo.light" must be written in ${getLanguageLabel(language)}.`,
`- "botanicalName" must use accepted Latin scientific naming and must not be invented or misspelled.`,
`- If species is uncertain, prefer genus-level naming (for example: "Calathea sp.").`,
`- "confidence" must be between 0 and 1.`,
`- Keep confidence <= 0.55 when the image is ambiguous, blurred, or partially visible.`,
`- "waterIntervalDays" must be an integer between 1 and 45.`,
`- Do not include markdown, explanations, or extra keys.`,
].join('\n');
};
const buildHealthPrompt = (
language: Language,
plantContext?: {
name: string;
botanicalName: string;
careInfo: CareInfo;
description?: string;
},
): string => {
const contextLines = plantContext
? [
`Plant context:`,
`- name: ${plantContext.name}`,
`- botanicalName: ${plantContext.botanicalName}`,
`- care.light: ${plantContext.careInfo.light}`,
`- care.temp: ${plantContext.careInfo.temp}`,
`- care.waterIntervalDays: ${plantContext.careInfo.waterIntervalDays}`,
`- description: ${plantContext.description || 'n/a'}`,
]
: ['Plant context: not provided'];
return [
`Analyze this plant photo for real health condition signs with focus on yellowing leaves, watering stress, pests, and light stress.`,
`Return strict JSON only in this shape:`,
`{"overallHealthScore":72,"status":"watch","analysisSummary":"...","likelyIssues":[{"title":"...","confidence":0.64,"details":"..."}],"actionsNow":["..."],"plan7Days":["..."]}`,
`Rules:`,
`- "overallHealthScore" must be an integer between 0 and 100.`,
`- "status" must be one of: "healthy", "watch", "critical".`,
`- "analysisSummary" must be 6 to 9 precise sentences. Cover the visible condition, symptom pattern, likely root cause, urgency, what evidence is uncertain, and what the owner should monitor next.`,
`- "likelyIssues" must contain 2 to 4 items sorted by confidence descending.`,
`- "confidence" must be between 0 and 1.`,
`- "title", "details", "analysisSummary", "actionsNow", and "plan7Days" must be written in ${getLanguageLabel(language)}.`,
`- Each issue "details" value must be 2 to 4 sentences and explain visual evidence, likely cause, and risk if untreated.`,
`- "actionsNow" must contain 5 to 8 concrete steps for the next 24 to 48 hours.`,
`- "plan7Days" must contain 7 to 10 day-by-day or milestone steps for the next week.`,
`- Do not include markdown, explanations, or extra keys.`,
...contextLines,
].join('\n');
};
const buildFallbackHealthAnalysis = (
language: Language,
plantContext?: {
name: string;
botanicalName: string;
careInfo: CareInfo;
description?: string;
},
): OpenAiHealthAnalysis => {
if (language === 'de') {
return {
overallHealthScore: 58,
status: 'watch',
analysisSummary: `${plantContext?.name || 'Die Pflanze'} braucht eine erneute Bewertung mit einem scharfen Foto, weil die KI-Antwort nicht stabil genug war. Behandle sie bis dahin vorsichtig als Beobachtungsfall. Vermeide radikale Standortwechsel, grosse Wassermengen und starke Duengung. Pruefe zuerst, ob die auffaelligen Blaetter weich, trocken, fleckig oder eingerollt wirken. Kontrolliere danach die oberen 3 cm Erde und achte auf stehendes Wasser im Uebertopf. Die wichtigsten sichtbaren Signale sollten bei Tageslicht erneut geprueft werden. Wenn sich Blattfarbe oder Spannung innerhalb von 48 Stunden verschlechtern, starte einen neuen Health-Scan mit einem detailreicheren Foto.`,
likelyIssues: [
{
title: 'Eingeschraenkte KI-Analyse',
confidence: 0.42,
details: `${plantContext?.name || 'Die Pflanze'} konnte wegen instabiler Antwort nicht vollstaendig bewertet werden.`,
},
],
actionsNow: [
'Neues Foto bei hellem, indirektem Licht aufnehmen.',
'Blaetter auf Flecken, Schaedlinge und trockene Raender pruefen.',
'Erst giessen, wenn die oberen 2-3 cm Erde trocken sind.',
],
plan7Days: [
'In 2 Tagen mit neuem Foto erneut pruefen.',
'Farbe und Blattspannung taeglich beobachten.',
'Bei Verschlechterung Standort und Giessrhythmus anpassen.',
],
};
}
if (language === 'es') {
return {
overallHealthScore: 58,
status: 'watch',
analysisSummary: `${plantContext?.name || 'La planta'} necesita una nueva evaluacion con una foto mas nitida porque la respuesta de IA no fue suficientemente estable. Hasta entonces tratala como un caso de observacion. Evita cambios bruscos de ubicacion, exceso de agua y fertilizacion fuerte. Revisa si las hojas afectadas estan blandas, secas, manchadas o enrolladas. Comprueba despues los 3 cm superiores del sustrato y busca agua acumulada. Las senales visibles deben revisarse de nuevo con luz natural. Si el color o la firmeza empeoran en 48 horas, inicia otro health-scan con una foto mas detallada.`,
likelyIssues: [
{
title: 'Analisis de IA limitado',
confidence: 0.42,
details: `${plantContext?.name || 'La planta'} no pudo evaluarse por completo por una respuesta inestable.`,
},
],
actionsNow: [
'Tomar una foto nueva con luz brillante e indirecta.',
'Revisar hojas por manchas, plagas y bordes secos.',
'Regar solo si los 2-3 cm superiores del sustrato estan secos.',
],
plan7Days: [
'Revisar otra vez en 2 dias con una foto nueva.',
'Observar color y firmeza de hojas cada dia.',
'Si empeora, ajustar ubicacion y frecuencia de riego.',
],
};
}
return {
overallHealthScore: 58,
status: 'watch',
analysisSummary: `${plantContext?.name || 'This plant'} needs another assessment with a sharper photo because the AI response was not stable enough. Until then, treat it as a watch case. Avoid major placement changes, heavy watering, or strong fertilizing. First inspect whether the unusual leaves look soft, dry, spotted, or curled. Then check the top 3 cm of soil and look for standing water in the outer pot. Re-check the visible signals in daylight before making bigger care changes. If color or leaf firmness gets worse within 48 hours, run a new health scan with a more detailed photo.`,
likelyIssues: [
{
title: 'Limited AI analysis',
confidence: 0.42,
details: `${plantContext?.name || 'This plant'} could not be fully assessed due to an unstable provider response.`,
},
],
actionsNow: [
'Capture a new photo in bright indirect light.',
'Inspect leaves for spots, pests, and dry edges.',
'Water only if the top 2-3 cm of soil is dry.',
],
plan7Days: [
'Re-check in 2 days with a new photo.',
'Track leaf color and firmness daily.',
'If symptoms worsen, adjust placement and watering cadence.',
],
};
};
const normalizeHealthAnalysis = (
raw: Record<string, unknown>,
language: Language,
): OpenAiHealthAnalysis | null => {
const scoreRaw = getNumber(raw.overallHealthScore);
const statusRaw = getString(raw.status);
const analysisSummary = getString(raw.analysisSummary);
const issuesRaw = raw.likelyIssues;
const actionsNowRaw = getStringArray(raw.actionsNow).slice(0, 8);
const plan7DaysRaw = getStringArray(raw.plan7Days).slice(0, 10);
if (scoreRaw == null || !statusRaw || !Array.isArray(issuesRaw)) {
return null;
}
const status: OpenAiHealthAnalysis['status'] = statusRaw === 'healthy' || statusRaw === 'watch' || statusRaw === 'critical'
? statusRaw
: 'watch';
const likelyIssues = issuesRaw
.map((entry) => {
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return null;
const issueObj = entry as Record<string, unknown>;
const title = getString(issueObj.title);
const details = getString(issueObj.details);
const confidenceRaw = getNumber(issueObj.confidence);
if (!title || !details || confidenceRaw == null) return null;
return {
title,
details,
confidence: clamp(confidenceRaw, 0.05, 0.99),
} as OpenAiHealthIssue;
})
.filter((entry): entry is OpenAiHealthIssue => Boolean(entry))
.slice(0, 4);
if (likelyIssues.length === 0 || actionsNowRaw.length === 0 || plan7DaysRaw.length === 0) {
const fallbackIssue = language === 'de'
? 'Die KI konnte keine stabilen Gesundheitsmerkmale extrahieren.'
: language === 'es'
? 'La IA no pudo extraer senales de salud estables.'
: 'AI could not extract stable health signals.';
return {
overallHealthScore: Math.round(clamp(scoreRaw, 0, 100)),
status,
analysisSummary: analysisSummary || fallbackIssue,
likelyIssues: [
{
title: language === 'de'
? 'Analyse unsicher'
: language === 'es'
? 'Analisis incierto'
: 'Uncertain analysis',
confidence: 0.35,
details: fallbackIssue,
},
],
actionsNow: actionsNowRaw.length > 0
? actionsNowRaw
: [language === 'de' ? 'Neues, schaerferes Foto aufnehmen.' : language === 'es' ? 'Tomar una foto nueva y mas nitida.' : 'Capture a new, sharper photo.'],
plan7Days: plan7DaysRaw.length > 0
? plan7DaysRaw
: [language === 'de' ? 'In 2 Tagen erneut pruefen.' : language === 'es' ? 'Volver a revisar en 2 dias.' : 'Re-check in 2 days.'],
};
}
return {
overallHealthScore: Math.round(clamp(scoreRaw, 0, 100)),
status,
analysisSummary,
likelyIssues,
actionsNow: actionsNowRaw,
plan7Days: plan7DaysRaw,
};
};
const extractMessageContent = (payload: unknown): string => {
const response = payload as {
choices?: Array<{
message?: {
content?: string | Array<{ type?: string; text?: string }>;
};
}>;
};
const content = response.choices?.[0]?.message?.content;
if (typeof content === 'string') return content;
if (Array.isArray(content)) {
return content
.map((chunk) => (chunk?.type === 'text' ? chunk.text || '' : ''))
.join('')
.trim();
}
return '';
};
const isReasoningModel = (model: string): boolean => {
const normalized = model.toLowerCase();
return normalized.startsWith('gpt-5') || normalized.startsWith('o1') || normalized.startsWith('o3') || normalized.startsWith('o4');
};
const buildRequestBody = (
model: string,
messages: Array<Record<string, unknown>>,
maxCompletionTokens: number,
): Record<string, unknown> => {
const body: Record<string, unknown> = {
model,
response_format: { type: 'json_object' },
messages,
};
if (isReasoningModel(model)) {
body.reasoning_effort = 'minimal';
body.max_completion_tokens = maxCompletionTokens;
} else {
body.max_tokens = maxCompletionTokens;
}
return body;
};
const postChatCompletion = async (
modelChain: string[],
imageUri: string,
messages: Array<Record<string, unknown>>,
maxCompletionTokens = 600,
): Promise<{ payload: Record<string, unknown> | null; modelUsed: string | null; attemptedModels: string[] }> => {
const attemptedModels: string[] = [];
for (const model of modelChain) {
attemptedModels.push(model);
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), OPENAI_TIMEOUT_MS);
try {
const response = await fetch(OPENAI_CHAT_COMPLETIONS_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${OPENAI_API_KEY}`,
},
body: JSON.stringify(buildRequestBody(model, messages, maxCompletionTokens)),
signal: controller.signal,
});
if (!response.ok) {
const body = await response.text();
console.warn('OpenAI request HTTP error.', {
status: response.status,
model,
image: summarizeImageUri(imageUri),
bodyPreview: body.slice(0, 300),
});
continue;
}
const payload = (await response.json()) as Record<string, unknown>;
return { payload, modelUsed: model, attemptedModels };
} catch (error) {
const isTimeoutAbort = error instanceof Error && error.name === 'AbortError';
console.warn('OpenAI request failed.', {
model,
timeoutMs: OPENAI_TIMEOUT_MS,
aborted: isTimeoutAbort,
error: toErrorMessage(error),
image: summarizeImageUri(imageUri),
});
continue;
} finally {
clearTimeout(timeout);
}
}
return { payload: null, modelUsed: null, attemptedModels };
};
export const openAiScanService = {
isConfigured: (): boolean => Boolean(OPENAI_API_KEY),
identifyPlant: async (
imageUri: string,
language: Language,
mode: OpenAiScanMode = 'primary',
plan: 'free' | 'pro' = 'free',
): Promise<IdentificationResult | null> => {
if (!OPENAI_API_KEY) return null;
const modelChain = getScanModelChain(plan);
const completion = await postChatCompletion(
modelChain,
imageUri,
[
{
role: 'system',
content: 'You are a plant identification assistant. Return strict JSON only.',
},
{
role: 'user',
content: [
{ type: 'text', text: buildPrompt(language, mode) },
{ type: 'image_url', image_url: { url: imageUri, detail: 'low' } },
],
},
],
600,
);
if (!completion.payload) return null;
const content = extractMessageContent(completion.payload);
if (!content) {
console.warn('OpenAI plant scan returned empty message content.', {
model: completion.modelUsed || modelChain[0],
mode,
image: summarizeImageUri(imageUri),
});
return null;
}
const parsed = parseContentToJson(content);
if (!parsed) {
console.warn('OpenAI plant scan returned non-JSON content.', {
model: completion.modelUsed || modelChain[0],
mode,
preview: content.slice(0, 220),
});
return null;
}
if (parsed.notAPlant === true) {
throw new BackendApiError('NOT_A_PLANT', 'Image does not contain a plant.', 422);
}
const normalized = normalizeResult(parsed, language);
if (!normalized) {
console.warn('OpenAI plant scan JSON did not match required schema.', {
model: completion.modelUsed || modelChain[0],
mode,
keys: Object.keys(parsed),
});
}
return normalized;
},
analyzePlantHealth: async (
imageUri: string,
language: Language,
plantContext?: {
name: string;
botanicalName: string;
careInfo: CareInfo;
description?: string;
},
): Promise<OpenAiHealthAnalysis | null> => {
if (!OPENAI_API_KEY) return null;
const completion = await postChatCompletion(
OPENAI_HEALTH_MODEL_CHAIN,
imageUri,
[
{
role: 'system',
content: 'You are a plant health diagnosis assistant. Return strict JSON only.',
},
{
role: 'user',
content: [
{ type: 'text', text: buildHealthPrompt(language, plantContext) },
{ type: 'image_url', image_url: { url: imageUri, detail: 'low' } },
],
},
],
800,
);
if (!completion.payload) return buildFallbackHealthAnalysis(language, plantContext);
const content = extractMessageContent(completion.payload);
if (!content) {
console.warn('OpenAI health check returned empty content.', {
model: completion.modelUsed || OPENAI_HEALTH_MODEL_CHAIN[0],
image: summarizeImageUri(imageUri),
});
return buildFallbackHealthAnalysis(language, plantContext);
}
const parsed = parseContentToJson(content);
if (!parsed) {
console.warn('OpenAI health check returned non-JSON content.', {
model: completion.modelUsed || OPENAI_HEALTH_MODEL_CHAIN[0],
preview: content.slice(0, 220),
});
return buildFallbackHealthAnalysis(language, plantContext);
}
return normalizeHealthAnalysis(parsed, language) || buildFallbackHealthAnalysis(language, plantContext);
},
};