zer0interrupt/recipe-manager
1
0
Fork 0
mirror of https://gitlab.com/Zer0interrupt/recipe-manager.git synced 2026-11-01 23:24:11 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add POST /api/recipes/scan-photo endpoint for multimodal Gemini vision photo parsing

Browse source
This commit is contained in:
Chris 2026-11-01 12:50:26 -05:00
parent 633eb4e63c
commit 9c5eae20f2
2 changed files with 105 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
public/app.js
View file

@ -2708,108 +2708,53 @@ async function scanRecipeFile(file) {
showScanLoading();
try {
// 1. Fetch system-wide configuration
const settingsResponse = await fetch('/api/settings', {
headers: getAuthHeaders()
});
if (settingsResponse.status === 401) {
logout();
hideScanLoading();
return;
}
if (!settingsResponse.ok) {
throw new Error('Could not fetch settings. Please ensure system settings are configured.');
}
const settings = await settingsResponse.json();
const apiKey = settings.geminiApiKey ? settings.geminiApiKey.trim() : '';
const rawModel = settings.geminiModel ? settings.geminiModel.trim() : '';
const geminiModel = (!rawModel || rawModel === 'gemini-2.0-flash') ? 'gemini-2.5-flash' : rawModel;
const isPlaceholder = apiKey === 'PASTE_YOUR_GEMINI_API_KEY_HERE' || apiKey === '';
if (!apiKey || isPlaceholder) {
hideScanLoading();
elements.scanImageInput.value = '';
alert('Google Gemini API Key is not configured.\n\nPlease click the Settings button at the top right, configure your Google Gemini API Key, and try again.');
return;
}
// Step 1 Completed -> Activate Step 2 (Compression)
// Step 1: Initialize
elements.loaderStepFetch.className = 'loader-step completed';
setLoaderStepIcon(elements.loaderStepFetch, 'check-circle-2', false);
elements.loaderStepParse.className = 'loader-step active';
setLoaderStepIcon(elements.loaderStepParse, 'circle-dashed', true);
lucide.createIcons();
// 2. Compress image using Canvas
// Step 2: Compress image on client
const compressed = await compressImage(file, 1600);
// Step 2 Completed -> Activate Step 3 (Gemini transcription)
elements.loaderStepParse.className = 'loader-step completed';
setLoaderStepIcon(elements.loaderStepParse, 'check-circle-2', false);
elements.loaderStepImages.className = 'loader-step active';
setLoaderStepIcon(elements.loaderStepImages, 'circle-dashed', true);
lucide.createIcons();
// 3. Make direct call to Google Gemini Vision API
const promptText = "You are a professional culinary transcriber. Analyze the image of a recipe (which could be a cookbook page, handwritten note, screenshot, or packaging) and extract it accurately. Fill in all fields as completely as possible. The response must be a JSON object matching this schema:\n{\n \"title\": \"Recipe Title\",\n \"description\": \"Brief description or review\",\n \"servings\": \"e.g., 4\",\n \"prepTime\": \"e.g., 15 mins\",\n \"cookTime\": \"e.g., 30 mins\",\n \"totalTime\": \"e.g., 45 mins\",\n \"ingredients\": [\"ingredient 1\", \"ingredient 2\"],\n \"instructions\": [\"step 1\", \"step 2\"]\n}\nDo not include any markdown styling (like ```json) or explanation. Return only the raw JSON string.";
const geminiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${apiKey}`;
const geminiResponse = await fetch(geminiUrl, {
// Step 3: Send photo to server-side scan-photo endpoint
const scanResponse = await fetch('/api/recipes/scan-photo', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
'Content-Type': 'application/json',
...getAuthHeaders()
},
body: JSON.stringify({
contents: [
{
parts: [
{ text: promptText },
{
inlineData: {
mimeType: compressed.mimeType,
data: compressed.base64
}
}
]
}
],
generationConfig: {
responseMimeType: "application/json"
}
imageBase64: compressed.base64,
mimeType: compressed.mimeType
})
});
if (!geminiResponse.ok) {
const errorObj = await geminiResponse.json().catch(() => ({}));
const status = geminiResponse.status;
let errMsg = (errorObj.error && errorObj.error.message)
? errorObj.error.message
: 'Google Gemini API call failed.';
if (status === 400 && (!errorObj.error || !errorObj.error.message)) {
errMsg = `Invalid parameters or model version (${geminiModel}). Please check your settings.`;
} else if (status === 403 && (!errorObj.error || !errorObj.error.message)) {
errMsg = 'Access denied. The Google Gemini API key is invalid or inactive.';
} else if (status === 429 && (!errorObj.error || !errorObj.error.message)) {
errMsg = 'Rate limit exceeded. You have made too many requests or hit your Gemini quota.';
}
throw new Error(errMsg);
if (scanResponse.status === 401) {
logout();
hideScanLoading();
return;
}
const result = await geminiResponse.json();
if (!result.candidates || result.candidates.length === 0 || !result.candidates[0].content) {
throw new Error('Gemini API returned an empty response. The text in the image may be unreadable.');
if (!scanResponse.ok) {
const errData = await scanResponse.json().catch(() => ({}));
throw new Error(errData.error || 'Server photo scanning failed.');
}
const rawText = result.candidates[0].content.parts[0].text;
const recipeData = JSON.parse(rawText);
const recipeData = await scanResponse.json();
// Auto-inject the scanned local picture as a custom visual preview option
recipeData.images = [compressed.dataUrl];
recipeData.image = compressed.dataUrl;
// 4. Finalize and trigger draft preview
// Step 4: Finalize and trigger draft preview
hideScanLoading();
elements.scanImageInput.value = '';
openDraftPreview(recipeData);
@ -2818,7 +2763,7 @@ async function scanRecipeFile(file) {
console.error('Photo scanning error occurred:', error);
hideScanLoading();
elements.scanImageInput.value = '';
alert(`Scanning Failed: ${error.message}\n\nPlease verify your internet connection and check that your Gemini API Key is correct.`);
alert(`Scanning Failed: ${error.message}\n\nPlease check your settings and verify your Google Gemini API configuration.`);
}
}

85
server.js
View file

@ -2305,6 +2305,91 @@ Return ONLY a valid JSON object matching this schema without markdown code block
}
});
// API: Scan Recipe Photo using Gemini Multimodal Vision (Cookbook pages, cards, packaging)
app.post('/api/recipes/scan-photo', requireAuth, async (req, res) => {
const { imageBase64, image, mimeType } = req.body;
const rawImage = imageBase64 || image;
if (!rawImage || typeof rawImage !== 'string') {
return res.status(400).json({ error: 'Image data is required as a base64 string.' });
}
// Sanitize base64 data and detect mimeType if provided in data URI format
let cleanBase64 = rawImage.trim();
let detectedMime = mimeType || 'image/jpeg';
const dataUriMatch = cleanBase64.match(/^data:([^;]+);base64,(.+)$/s);
if (dataUriMatch) {
detectedMime = dataUriMatch[1];
cleanBase64 = dataUriMatch[2];
}
const settingsRow = db.prepare('SELECT geminiApiKey, geminiModel FROM settings LIMIT 1').get();
const geminiKey = settingsRow?.geminiApiKey?.trim();
const geminiModel = settingsRow?.geminiModel?.trim() || 'gemini-2.5-flash';
const isPlaceholder = !geminiKey || geminiKey === 'PASTE_YOUR_GEMINI_API_KEY_HERE';
if (isPlaceholder) {
return res.status(400).json({
error: 'Google Gemini API Key is not configured on the server. Please open Settings and configure your Gemini API Key.'
});
}
try {
const promptText = `You are a professional culinary transcriber and recipe extraction assistant.
Analyze this photo of a recipe (which may be a cookbook page, magazine cutout, handwritten recipe card, packaging, or screen photo).
Carefully read and structure all recipe details into clean, normalized JSON.
- Standardize ingredients onto separate items with clear amounts, units, and ingredient names.
- Turn run-on or multi-part instructions into clean, ordered, numbered steps.
- Infer or estimate prep time, cook time, and servings if mentioned or implied.
- Extract or generate an accurate, appealing recipe title.
- If description/notes are present, include a concise summary.
Return ONLY a valid JSON object matching this schema without markdown code blocks:
{
"title": "Recipe Title",
"description": "Brief description or summary",
"servings": "e.g., 4",
"prepTime": "e.g., 15 mins",
"cookTime": "e.g., 30 mins",
"totalTime": "e.g., 45 mins",
"ingredients": ["ingredient line 1", "ingredient line 2"],
"instructions": ["Step 1", "Step 2"]
}`;
const geminiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${geminiKey}`;
const aiRes = await axios.post(geminiUrl, {
contents: [{
parts: [
{ text: promptText },
{
inlineData: {
mimeType: detectedMime,
data: cleanBase64
}
}
]
}],
generationConfig: { responseMimeType: "application/json" }
}, {
headers: { 'Content-Type': 'application/json' },
timeout: 90000
});
if (aiRes.data && aiRes.data.candidates?.[0]?.content?.parts?.[0]?.text) {
const rawText = aiRes.data.candidates[0].content.parts[0].text;
const parsed = JSON.parse(rawText);
parsed._scrapingMethod = 'photo-scan-gemini';
return res.json(parsed);
} else {
throw new Error('Gemini returned an empty response. The photo text may be unreadable.');
}
} catch (err) {
log.error('[Scan Photo] Error:', err.response?.data?.error?.message || err.message);
const msg = err.response?.data?.error?.message || err.message;
return res.status(500).json({ error: `Photo recipe scanning failed: ${msg}` });
}
});
// API: Load System Settings & User Preferences
app.get('/api/settings', requireAuth, (req, res) => {
try {