Add POST /api/recipes/scan-photo endpoint for multimodal Gemini vision photo parsing

public/app.js

@@ -2708,108 +2708,53 @@ async function scanRecipeFile(file) {
   showScanLoading();
   
   try {
-    // 1. Fetch system-wide configuration
+    // Step 1: Initialize
-    const settingsResponse = await fetch('/api/settings', {
-      headers: getAuthHeaders()
-    });
-    if (settingsResponse.status === 401) {
-      logout();
-      hideScanLoading();
-      return;
-    }
-    if (!settingsResponse.ok) {
-      throw new Error('Could not fetch settings. Please ensure system settings are configured.');
-    }
-    const settings = await settingsResponse.json();
-    const apiKey = settings.geminiApiKey ? settings.geminiApiKey.trim() : '';
-    const rawModel = settings.geminiModel ? settings.geminiModel.trim() : '';
-    const geminiModel = (!rawModel || rawModel === 'gemini-2.0-flash') ? 'gemini-2.5-flash' : rawModel;
-    const isPlaceholder = apiKey === 'PASTE_YOUR_GEMINI_API_KEY_HERE' || apiKey === '';
-    
-    if (!apiKey || isPlaceholder) {
-      hideScanLoading();
-      elements.scanImageInput.value = '';
-      alert('Google Gemini API Key is not configured.\n\nPlease click the Settings button at the top right, configure your Google Gemini API Key, and try again.');
-      return;
-    }
-    
-    // Step 1 Completed -> Activate Step 2 (Compression)
     elements.loaderStepFetch.className = 'loader-step completed';
     setLoaderStepIcon(elements.loaderStepFetch, 'check-circle-2', false);
     elements.loaderStepParse.className = 'loader-step active';
     setLoaderStepIcon(elements.loaderStepParse, 'circle-dashed', true);
     lucide.createIcons();
     
-    // 2. Compress image using Canvas
+    // Step 2: Compress image on client
     const compressed = await compressImage(file, 1600);
     
-    // Step 2 Completed -> Activate Step 3 (Gemini transcription)
     elements.loaderStepParse.className = 'loader-step completed';
     setLoaderStepIcon(elements.loaderStepParse, 'check-circle-2', false);
     elements.loaderStepImages.className = 'loader-step active';
     setLoaderStepIcon(elements.loaderStepImages, 'circle-dashed', true);
     lucide.createIcons();
     
-    // 3. Make direct call to Google Gemini Vision API
+    // Step 3: Send photo to server-side scan-photo endpoint
-    const promptText = "You are a professional culinary transcriber. Analyze the image of a recipe (which could be a cookbook page, handwritten note, screenshot, or packaging) and extract it accurately. Fill in all fields as completely as possible. The response must be a JSON object matching this schema:\n{\n  \"title\": \"Recipe Title\",\n  \"description\": \"Brief description or review\",\n  \"servings\": \"e.g., 4\",\n  \"prepTime\": \"e.g., 15 mins\",\n  \"cookTime\": \"e.g., 30 mins\",\n  \"totalTime\": \"e.g., 45 mins\",\n  \"ingredients\": [\"ingredient 1\", \"ingredient 2\"],\n  \"instructions\": [\"step 1\", \"step 2\"]\n}\nDo not include any markdown styling (like ```json) or explanation. Return only the raw JSON string.";
+    const scanResponse = await fetch('/api/recipes/scan-photo', {
-    
-    const geminiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${apiKey}`;
-    
-    const geminiResponse = await fetch(geminiUrl, {
       method: 'POST',
       headers: {
-        'Content-Type': 'application/json'
+        'Content-Type': 'application/json',
+        ...getAuthHeaders()
       },
       body: JSON.stringify({
-        contents: [
+        imageBase64: compressed.base64,
-          {
+        mimeType: compressed.mimeType
-            parts: [
-              { text: promptText },
-              {
-                inlineData: {
-                  mimeType: compressed.mimeType,
-                  data: compressed.base64
-                }
-              }
-            ]
-          }
-        ],
-        generationConfig: {
-          responseMimeType: "application/json"
-        }
       })
     });
-    
-    if (!geminiResponse.ok) {
-      const errorObj = await geminiResponse.json().catch(() => ({}));
-      const status = geminiResponse.status;
-      let errMsg = (errorObj.error && errorObj.error.message)
-        ? errorObj.error.message
-        : 'Google Gemini API call failed.';
 
-      if (status === 400 && (!errorObj.error || !errorObj.error.message)) {
+    if (scanResponse.status === 401) {
-        errMsg = `Invalid parameters or model version (${geminiModel}). Please check your settings.`;
+      logout();
-      } else if (status === 403 && (!errorObj.error || !errorObj.error.message)) {
+      hideScanLoading();
-        errMsg = 'Access denied. The Google Gemini API key is invalid or inactive.';
+      return;
-      } else if (status === 429 && (!errorObj.error || !errorObj.error.message)) {
-        errMsg = 'Rate limit exceeded. You have made too many requests or hit your Gemini quota.';
-      }
-      throw new Error(errMsg);
     }
-    
+
-    const result = await geminiResponse.json();
+    if (!scanResponse.ok) {
-    if (!result.candidates || result.candidates.length === 0 || !result.candidates[0].content) {
+      const errData = await scanResponse.json().catch(() => ({}));
-      throw new Error('Gemini API returned an empty response. The text in the image may be unreadable.');
+      throw new Error(errData.error || 'Server photo scanning failed.');
     }
-    
+
-    const rawText = result.candidates[0].content.parts[0].text;
+    const recipeData = await scanResponse.json();
-    const recipeData = JSON.parse(rawText);
     
     // Auto-inject the scanned local picture as a custom visual preview option
     recipeData.images = [compressed.dataUrl];
     recipeData.image = compressed.dataUrl;
     
-    // 4. Finalize and trigger draft preview
+    // Step 4: Finalize and trigger draft preview
     hideScanLoading();
     elements.scanImageInput.value = '';
     openDraftPreview(recipeData);
@@ -2818,7 +2763,7 @@ async function scanRecipeFile(file) {
     console.error('Photo scanning error occurred:', error);
     hideScanLoading();
     elements.scanImageInput.value = '';
-    alert(`Scanning Failed: ${error.message}\n\nPlease verify your internet connection and check that your Gemini API Key is correct.`);
+    alert(`Scanning Failed: ${error.message}\n\nPlease check your settings and verify your Google Gemini API configuration.`);
   }
 }
 

server.js

@@ -2305,6 +2305,91 @@ Return ONLY a valid JSON object matching this schema without markdown code block
   }
 });
 
+// API: Scan Recipe Photo using Gemini Multimodal Vision (Cookbook pages, cards, packaging)
+app.post('/api/recipes/scan-photo', requireAuth, async (req, res) => {
+  const { imageBase64, image, mimeType } = req.body;
+  const rawImage = imageBase64 || image;
+
+  if (!rawImage || typeof rawImage !== 'string') {
+    return res.status(400).json({ error: 'Image data is required as a base64 string.' });
+  }
+
+  // Sanitize base64 data and detect mimeType if provided in data URI format
+  let cleanBase64 = rawImage.trim();
+  let detectedMime = mimeType || 'image/jpeg';
+  const dataUriMatch = cleanBase64.match(/^data:([^;]+);base64,(.+)$/s);
+  if (dataUriMatch) {
+    detectedMime = dataUriMatch[1];
+    cleanBase64 = dataUriMatch[2];
+  }
+
+  const settingsRow = db.prepare('SELECT geminiApiKey, geminiModel FROM settings LIMIT 1').get();
+  const geminiKey = settingsRow?.geminiApiKey?.trim();
+  const geminiModel = settingsRow?.geminiModel?.trim() || 'gemini-2.5-flash';
+  const isPlaceholder = !geminiKey || geminiKey === 'PASTE_YOUR_GEMINI_API_KEY_HERE';
+
+  if (isPlaceholder) {
+    return res.status(400).json({
+      error: 'Google Gemini API Key is not configured on the server. Please open Settings and configure your Gemini API Key.'
+    });
+  }
+
+  try {
+    const promptText = `You are a professional culinary transcriber and recipe extraction assistant.
+Analyze this photo of a recipe (which may be a cookbook page, magazine cutout, handwritten recipe card, packaging, or screen photo).
+Carefully read and structure all recipe details into clean, normalized JSON.
+- Standardize ingredients onto separate items with clear amounts, units, and ingredient names.
+- Turn run-on or multi-part instructions into clean, ordered, numbered steps.
+- Infer or estimate prep time, cook time, and servings if mentioned or implied.
+- Extract or generate an accurate, appealing recipe title.
+- If description/notes are present, include a concise summary.
+
+Return ONLY a valid JSON object matching this schema without markdown code blocks:
+{
+  "title": "Recipe Title",
+  "description": "Brief description or summary",
+  "servings": "e.g., 4",
+  "prepTime": "e.g., 15 mins",
+  "cookTime": "e.g., 30 mins",
+  "totalTime": "e.g., 45 mins",
+  "ingredients": ["ingredient line 1", "ingredient line 2"],
+  "instructions": ["Step 1", "Step 2"]
+}`;
+
+    const geminiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${geminiKey}`;
+    const aiRes = await axios.post(geminiUrl, {
+      contents: [{
+        parts: [
+          { text: promptText },
+          {
+            inlineData: {
+              mimeType: detectedMime,
+              data: cleanBase64
+            }
+          }
+        ]
+      }],
+      generationConfig: { responseMimeType: "application/json" }
+    }, {
+      headers: { 'Content-Type': 'application/json' },
+      timeout: 90000
+    });
+
+    if (aiRes.data && aiRes.data.candidates?.[0]?.content?.parts?.[0]?.text) {
+      const rawText = aiRes.data.candidates[0].content.parts[0].text;
+      const parsed = JSON.parse(rawText);
+      parsed._scrapingMethod = 'photo-scan-gemini';
+      return res.json(parsed);
+    } else {
+      throw new Error('Gemini returned an empty response. The photo text may be unreadable.');
+    }
+  } catch (err) {
+    log.error('[Scan Photo] Error:', err.response?.data?.error?.message || err.message);
+    const msg = err.response?.data?.error?.message || err.message;
+    return res.status(500).json({ error: `Photo recipe scanning failed: ${msg}` });
+  }
+});
+
 // API: Load System Settings & User Preferences
 app.get('/api/settings', requireAuth, (req, res) => {
   try {