From 9c5eae20f20f8d58d2ca74aca3cf6b712dd4e0db Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 1 Nov 2026 12:50:26 -0500 Subject: [PATCH] Add POST /api/recipes/scan-photo endpoint for multimodal Gemini vision photo parsing --- public/app.js | 95 +++++++++++---------------------------------------- server.js | 85 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 75 deletions(-) diff --git a/public/app.js b/public/app.js index 9356714..a6033f9 100644 --- a/public/app.js +++ b/public/app.js @@ -2708,108 +2708,53 @@ async function scanRecipeFile(file) { showScanLoading(); try { - // 1. Fetch system-wide configuration - const settingsResponse = await fetch('/api/settings', { - headers: getAuthHeaders() - }); - if (settingsResponse.status === 401) { - logout(); - hideScanLoading(); - return; - } - if (!settingsResponse.ok) { - throw new Error('Could not fetch settings. Please ensure system settings are configured.'); - } - const settings = await settingsResponse.json(); - const apiKey = settings.geminiApiKey ? settings.geminiApiKey.trim() : ''; - const rawModel = settings.geminiModel ? settings.geminiModel.trim() : ''; - const geminiModel = (!rawModel || rawModel === 'gemini-2.0-flash') ? 'gemini-2.5-flash' : rawModel; - const isPlaceholder = apiKey === 'PASTE_YOUR_GEMINI_API_KEY_HERE' || apiKey === ''; - - if (!apiKey || isPlaceholder) { - hideScanLoading(); - elements.scanImageInput.value = ''; - alert('Google Gemini API Key is not configured.\n\nPlease click the Settings button at the top right, configure your Google Gemini API Key, and try again.'); - return; - } - - // Step 1 Completed -> Activate Step 2 (Compression) + // Step 1: Initialize elements.loaderStepFetch.className = 'loader-step completed'; setLoaderStepIcon(elements.loaderStepFetch, 'check-circle-2', false); elements.loaderStepParse.className = 'loader-step active'; setLoaderStepIcon(elements.loaderStepParse, 'circle-dashed', true); lucide.createIcons(); - // 2. Compress image using Canvas + // Step 2: Compress image on client const compressed = await compressImage(file, 1600); - // Step 2 Completed -> Activate Step 3 (Gemini transcription) elements.loaderStepParse.className = 'loader-step completed'; setLoaderStepIcon(elements.loaderStepParse, 'check-circle-2', false); elements.loaderStepImages.className = 'loader-step active'; setLoaderStepIcon(elements.loaderStepImages, 'circle-dashed', true); lucide.createIcons(); - // 3. Make direct call to Google Gemini Vision API - const promptText = "You are a professional culinary transcriber. Analyze the image of a recipe (which could be a cookbook page, handwritten note, screenshot, or packaging) and extract it accurately. Fill in all fields as completely as possible. The response must be a JSON object matching this schema:\n{\n \"title\": \"Recipe Title\",\n \"description\": \"Brief description or review\",\n \"servings\": \"e.g., 4\",\n \"prepTime\": \"e.g., 15 mins\",\n \"cookTime\": \"e.g., 30 mins\",\n \"totalTime\": \"e.g., 45 mins\",\n \"ingredients\": [\"ingredient 1\", \"ingredient 2\"],\n \"instructions\": [\"step 1\", \"step 2\"]\n}\nDo not include any markdown styling (like ```json) or explanation. Return only the raw JSON string."; - - const geminiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${apiKey}`; - - const geminiResponse = await fetch(geminiUrl, { + // Step 3: Send photo to server-side scan-photo endpoint + const scanResponse = await fetch('/api/recipes/scan-photo', { method: 'POST', headers: { - 'Content-Type': 'application/json' + 'Content-Type': 'application/json', + ...getAuthHeaders() }, body: JSON.stringify({ - contents: [ - { - parts: [ - { text: promptText }, - { - inlineData: { - mimeType: compressed.mimeType, - data: compressed.base64 - } - } - ] - } - ], - generationConfig: { - responseMimeType: "application/json" - } + imageBase64: compressed.base64, + mimeType: compressed.mimeType }) }); - - if (!geminiResponse.ok) { - const errorObj = await geminiResponse.json().catch(() => ({})); - const status = geminiResponse.status; - let errMsg = (errorObj.error && errorObj.error.message) - ? errorObj.error.message - : 'Google Gemini API call failed.'; - if (status === 400 && (!errorObj.error || !errorObj.error.message)) { - errMsg = `Invalid parameters or model version (${geminiModel}). Please check your settings.`; - } else if (status === 403 && (!errorObj.error || !errorObj.error.message)) { - errMsg = 'Access denied. The Google Gemini API key is invalid or inactive.'; - } else if (status === 429 && (!errorObj.error || !errorObj.error.message)) { - errMsg = 'Rate limit exceeded. You have made too many requests or hit your Gemini quota.'; - } - throw new Error(errMsg); + if (scanResponse.status === 401) { + logout(); + hideScanLoading(); + return; } - - const result = await geminiResponse.json(); - if (!result.candidates || result.candidates.length === 0 || !result.candidates[0].content) { - throw new Error('Gemini API returned an empty response. The text in the image may be unreadable.'); + + if (!scanResponse.ok) { + const errData = await scanResponse.json().catch(() => ({})); + throw new Error(errData.error || 'Server photo scanning failed.'); } - - const rawText = result.candidates[0].content.parts[0].text; - const recipeData = JSON.parse(rawText); + + const recipeData = await scanResponse.json(); // Auto-inject the scanned local picture as a custom visual preview option recipeData.images = [compressed.dataUrl]; recipeData.image = compressed.dataUrl; - // 4. Finalize and trigger draft preview + // Step 4: Finalize and trigger draft preview hideScanLoading(); elements.scanImageInput.value = ''; openDraftPreview(recipeData); @@ -2818,7 +2763,7 @@ async function scanRecipeFile(file) { console.error('Photo scanning error occurred:', error); hideScanLoading(); elements.scanImageInput.value = ''; - alert(`Scanning Failed: ${error.message}\n\nPlease verify your internet connection and check that your Gemini API Key is correct.`); + alert(`Scanning Failed: ${error.message}\n\nPlease check your settings and verify your Google Gemini API configuration.`); } } diff --git a/server.js b/server.js index 83b14a0..b23c8f2 100644 --- a/server.js +++ b/server.js @@ -2305,6 +2305,91 @@ Return ONLY a valid JSON object matching this schema without markdown code block } }); +// API: Scan Recipe Photo using Gemini Multimodal Vision (Cookbook pages, cards, packaging) +app.post('/api/recipes/scan-photo', requireAuth, async (req, res) => { + const { imageBase64, image, mimeType } = req.body; + const rawImage = imageBase64 || image; + + if (!rawImage || typeof rawImage !== 'string') { + return res.status(400).json({ error: 'Image data is required as a base64 string.' }); + } + + // Sanitize base64 data and detect mimeType if provided in data URI format + let cleanBase64 = rawImage.trim(); + let detectedMime = mimeType || 'image/jpeg'; + const dataUriMatch = cleanBase64.match(/^data:([^;]+);base64,(.+)$/s); + if (dataUriMatch) { + detectedMime = dataUriMatch[1]; + cleanBase64 = dataUriMatch[2]; + } + + const settingsRow = db.prepare('SELECT geminiApiKey, geminiModel FROM settings LIMIT 1').get(); + const geminiKey = settingsRow?.geminiApiKey?.trim(); + const geminiModel = settingsRow?.geminiModel?.trim() || 'gemini-2.5-flash'; + const isPlaceholder = !geminiKey || geminiKey === 'PASTE_YOUR_GEMINI_API_KEY_HERE'; + + if (isPlaceholder) { + return res.status(400).json({ + error: 'Google Gemini API Key is not configured on the server. Please open Settings and configure your Gemini API Key.' + }); + } + + try { + const promptText = `You are a professional culinary transcriber and recipe extraction assistant. +Analyze this photo of a recipe (which may be a cookbook page, magazine cutout, handwritten recipe card, packaging, or screen photo). +Carefully read and structure all recipe details into clean, normalized JSON. +- Standardize ingredients onto separate items with clear amounts, units, and ingredient names. +- Turn run-on or multi-part instructions into clean, ordered, numbered steps. +- Infer or estimate prep time, cook time, and servings if mentioned or implied. +- Extract or generate an accurate, appealing recipe title. +- If description/notes are present, include a concise summary. + +Return ONLY a valid JSON object matching this schema without markdown code blocks: +{ + "title": "Recipe Title", + "description": "Brief description or summary", + "servings": "e.g., 4", + "prepTime": "e.g., 15 mins", + "cookTime": "e.g., 30 mins", + "totalTime": "e.g., 45 mins", + "ingredients": ["ingredient line 1", "ingredient line 2"], + "instructions": ["Step 1", "Step 2"] +}`; + + const geminiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${geminiKey}`; + const aiRes = await axios.post(geminiUrl, { + contents: [{ + parts: [ + { text: promptText }, + { + inlineData: { + mimeType: detectedMime, + data: cleanBase64 + } + } + ] + }], + generationConfig: { responseMimeType: "application/json" } + }, { + headers: { 'Content-Type': 'application/json' }, + timeout: 90000 + }); + + if (aiRes.data && aiRes.data.candidates?.[0]?.content?.parts?.[0]?.text) { + const rawText = aiRes.data.candidates[0].content.parts[0].text; + const parsed = JSON.parse(rawText); + parsed._scrapingMethod = 'photo-scan-gemini'; + return res.json(parsed); + } else { + throw new Error('Gemini returned an empty response. The photo text may be unreadable.'); + } + } catch (err) { + log.error('[Scan Photo] Error:', err.response?.data?.error?.message || err.message); + const msg = err.response?.data?.error?.message || err.message; + return res.status(500).json({ error: `Photo recipe scanning failed: ${msg}` }); + } +}); + // API: Load System Settings & User Preferences app.get('/api/settings', requireAuth, (req, res) => { try {