diff --git a/server.js b/server.js
index 7bde0b6..5690255 100644
--- a/server.js
+++ b/server.js
@@ -1383,6 +1383,10 @@ async function fetchWithPuppeteer(url) {
// TikTok Recipe Extraction (Option 1: Caption & Metadata Extraction)
// -------------------------------------------------------------
+// -------------------------------------------------------------
+// Social Media Recipe Extraction (Instagram & TikTok)
+// -------------------------------------------------------------
+
// Helper: Check if a URL belongs to TikTok
function isTikTokUrl(url) {
try {
@@ -1394,13 +1398,41 @@ function isTikTokUrl(url) {
}
}
-// Helper: Clean raw TikTok caption by stripping trailing hashtags and emojis
-function cleanTikTokCaption(raw) {
+// Helper: Check if a URL belongs to Instagram
+function isInstagramUrl(url) {
+ try {
+ const parsed = new URL(url);
+ const host = parsed.hostname.toLowerCase();
+ return host.includes('instagram.com') || host.includes('instagr.am');
+ } catch (e) {
+ return false;
+ }
+}
+
+// Helper: Clean raw social caption by stripping trailing hashtags and emojis
+function cleanSocialCaption(raw) {
if (!raw) return '';
let text = raw.replace(/(?:\s*#[^\s#]+)+\s*$/g, '').trim();
text = text.replace(/[\uFE00-\uFE0F\uD83C-\uDBFF\uDC00-\uDFFF\u2600-\u26FF\u2700-\u27BF]+$/, '').trim();
return text;
}
+const cleanTikTokCaption = cleanSocialCaption;
+
+// Helper: Clean raw Instagram caption from OpenGraph tags or quotes
+function cleanInstagramCaption(raw) {
+ if (!raw) return '';
+ let text = raw.trim();
+ const quotedMatch = text.match(/(?:on Instagram:\s*["“]|:\s*["“])([\s\S]*?)["”]?$/i);
+ if (quotedMatch && quotedMatch[1]) {
+ text = quotedMatch[1];
+ } else {
+ const colonQuote = text.indexOf(': "');
+ if (colonQuote !== -1) {
+ text = text.slice(colonQuote + 3).replace(/["”]$/, '');
+ }
+ }
+ return cleanSocialCaption(text);
+}
// Helper: Split single-line or unstructured social captions into readable candidate phrases
function splitCaptionIntoPhrases(text) {
@@ -1426,20 +1458,20 @@ function splitCaptionIntoPhrases(text) {
// Helper: Heuristic parser to extract recipe structure from a social video caption
function parseRecipeCaptionHeuristic(rawCaption, author, cover, sourceUrl) {
- const caption = cleanTikTokCaption(rawCaption);
+ const caption = cleanSocialCaption(rawCaption);
if (!caption) return null;
let title = '';
- let description = '';
const ingredients = [];
const instructions = [];
- const ingHeaderMatch = caption.match(/\b(?:ingredients?|what you(?:'ll)? need)\s*[:\-]?/i);
+ const ingHeaderMatch = caption.match(/\b(?:ingredients?|what you(?:'ll)? need)\s*[:\-]?\s*/i);
let remainingText = caption;
if (ingHeaderMatch) {
const rawTitle = caption.slice(0, ingHeaderMatch.index).trim();
- title = rawTitle.replace(/[:\-.]+$/, '').trim();
+ const firstLine = rawTitle.split('\n')[0].trim();
+ title = firstLine.replace(/[:\-.]+$/, '').trim();
remainingText = caption.slice(ingHeaderMatch.index + ingHeaderMatch[0].length).trim();
} else {
const firstSentence = caption.match(/^([^.!?\n]+[.!?])/);
@@ -1447,30 +1479,46 @@ function parseRecipeCaptionHeuristic(rawCaption, author, cover, sourceUrl) {
remainingText = caption.slice(title.length).trim();
}
- const segments = splitCaptionIntoPhrases(remainingText);
+ const methodHeaderMatch = remainingText.match(/\b(?:instructions?|directions?|method|steps?|preparation|how to make(?: it)?)\s*[:\-]?\s*/i);
- const instructionStarters = /^(?:mix|whisk|add|bake|stir|fold|combine|in a|then|swirl|sprinkle|pour|heat|cook|chill|serve|place|cut|slice|dice)\b/i;
- const isIngredientPattern = /^(?:(?:\d+\s*(?:&|\+)?\s*)?(?:\d+[\/\.]\d+|[¼½¾⅓⅔⅛⅜⅝⅞]|\d+)\s*(?:cups?|c|tbsp|tablespoons?|tsp|teaspoons?|oz|ounces?|lbs?|sticks?|large|small|medium|cans?|pinch|pkg)?\b|[•◦⁃∙*\-+])/i;
+ if (methodHeaderMatch) {
+ const ingText = remainingText.slice(0, methodHeaderMatch.index).trim();
+ const instText = remainingText.slice(methodHeaderMatch.index + methodHeaderMatch[0].length).trim();
- for (let seg of segments) {
- seg = seg.trim().replace(/^[:\-•◦⁃∙*\-+]\s*/, '').replace(/#[\w-]+/g, '').trim();
- if (!seg || seg.length < 2) continue;
+ const ingLines = ingText.split('\n').map(s => s.trim().replace(/^[:\-•◦⁃∙*\-+]\s*/, '').replace(/#[\w-]+/g, '').trim()).filter(Boolean);
+ for (const line of ingLines) {
+ if (line.length >= 2) ingredients.push(line);
+ }
- if (instructionStarters.test(seg) || /^\d+[\.\)]\s+/.test(seg)) {
- instructions.push(seg.replace(/^\d+[\.\)]\s*/, ''));
- } else if (isIngredientPattern.test(seg)) {
- if (/batter|pan/i.test(seg)) {
+ const instLines = instText.split('\n').map(s => s.trim().replace(/^\d+[\.\)]\s*/, '').replace(/^[:\-•◦⁃∙*\-+]\s*/, '').replace(/#[\w-]+/g, '').trim()).filter(Boolean);
+ for (const line of instLines) {
+ if (line.length >= 2) instructions.push(line);
+ }
+ } else {
+ const segments = splitCaptionIntoPhrases(remainingText);
+ const instructionStarters = /^(?:mix|whisk|add|bake|stir|fold|combine|in a|then|swirl|sprinkle|pour|heat|cook|chill|serve|place|cut|slice|dice)\b/i;
+ const isIngredientPattern = /^(?:(?:\d+\s*(?:&|\+)?\s*)?(?:\d+[\/\.]\d+|[¼½¾⅓⅔⅛⅜⅝⅞]|\d+)\s*(?:cups?|c|tbsp|tablespoons?|tsp|teaspoons?|oz|ounces?|lbs?|sticks?|large|small|medium|cans?|pinch|pkg)?\b|[•◦⁃∙*\-+])/i;
+
+ for (let seg of segments) {
+ seg = seg.trim().replace(/^[:\-•◦⁃∙*\-+]\s*/, '').replace(/#[\w-]+/g, '').trim();
+ if (!seg || seg.length < 2) continue;
+
+ if (instructionStarters.test(seg) || /^\d+[\.\)]\s+/.test(seg)) {
+ instructions.push(seg.replace(/^\d+[\.\)]\s*/, ''));
+ } else if (isIngredientPattern.test(seg)) {
+ if (/batter|pan/i.test(seg)) {
+ instructions.push(seg);
+ } else {
+ ingredients.push(seg);
+ }
+ } else if (/^(?:glaze|for the )/i.test(seg)) {
instructions.push(seg);
} else {
- ingredients.push(seg);
- }
- } else if (/^(?:glaze|for the )/i.test(seg)) {
- instructions.push(seg);
- } else {
- if (/(?:bowl|pan|oven|bake|mins?|minutes?|degrees?|350|mix|batter)/i.test(seg)) {
- instructions.push(seg);
- } else {
- ingredients.push(seg);
+ if (/(?:bowl|pan|oven|bake|mins?|minutes?|degrees?|350|mix|batter)/i.test(seg)) {
+ instructions.push(seg);
+ } else {
+ ingredients.push(seg);
+ }
}
}
}
@@ -1481,8 +1529,11 @@ function parseRecipeCaptionHeuristic(rawCaption, author, cover, sourceUrl) {
cookTime = bakeMatch[1].trim();
}
+ const isIg = (sourceUrl || '').toLowerCase().includes('instagram');
+ const defaultFallback = isIg ? 'Instagram Recipe' : 'Social Media Recipe';
+
return {
- title: title || (author ? `Recipe by ${author}` : 'TikTok Recipe'),
+ title: title || (author ? `Recipe by ${author}` : defaultFallback),
description: caption.slice(0, 160).trim(),
image: cover || '',
images: cover ? [cover] : [],
@@ -1496,6 +1547,132 @@ function parseRecipeCaptionHeuristic(rawCaption, author, cover, sourceUrl) {
};
}
+// Helper: Scrape recipe from an Instagram post or reel URL
+async function scrapeInstagramRecipe(inputUrl) {
+ log.info('[Instagram] Starting scrape for:', inputUrl);
+
+ let canonicalUrl = inputUrl;
+ let caption = '';
+ let author = '';
+ let cover = '';
+
+ // 1. Resolve redirect for shortened links (e.g., instagr.am)
+ try {
+ const headRes = await axios.get(inputUrl, {
+ maxRedirects: 5,
+ validateStatus: () => true,
+ headers: {
+ 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'
+ },
+ timeout: 8000
+ });
+ if (headRes.request?.res?.responseUrl) {
+ canonicalUrl = headRes.request.res.responseUrl;
+ }
+ } catch (e) {
+ log.warn('[Instagram] Redirect note:', e.message);
+ }
+
+ // 2. Fetch page HTML using crawler user agents (Facebook external hit, Twitterbot)
+ const crawlerUserAgents = [
+ 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
+ 'Twitterbot/1.0',
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
+ ];
+
+ let html = null;
+ for (const ua of crawlerUserAgents) {
+ try {
+ const response = await axios.get(canonicalUrl, {
+ headers: {
+ 'User-Agent': ua,
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+ 'Accept-Language': 'en-US,en;q=0.9'
+ },
+ timeout: 10000,
+ validateStatus: (status) => status < 400 || status === 302
+ });
+
+ if (response.data && typeof response.data === 'string' && response.data.includes('og:')) {
+ html = response.data;
+ break;
+ }
+ } catch (fetchErr) {
+ log.warn(`[Instagram] Crawler fetch failed with UA (${ua.split(' ')[0]}):`, fetchErr.message);
+ }
+ }
+
+ // 3. Extract OpenGraph and JSON-LD metadata if HTML was retrieved
+ if (html) {
+ const $ = cheerio.load(html);
+
+ cover = $('meta[property="og:image"]').attr('content') ||
+ $('meta[property="og:image:secure_url"]').attr('content') ||
+ $('meta[name="twitter:image"]').attr('content') || '';
+
+ const ogTitle = $('meta[property="og:title"]').attr('content') || '';
+ const authorMatch = ogTitle.match(/^(?:.*?on Instagram:\s*["“]?|)(.*?)(?:\s*\(@|\s*on Instagram|$)/i);
+ if (authorMatch && authorMatch[1] && !authorMatch[1].toLowerCase().includes('instagram')) {
+ author = authorMatch[1].trim();
+ }
+
+ const ogDesc = $('meta[property="og:description"]').attr('content') ||
+ $('meta[name="description"]').attr('content') || '';
+
+ if (ogDesc) {
+ caption = cleanInstagramCaption(ogDesc);
+ }
+
+ // Inspect script tags or embedded ld+json
+ $('script[type="application/ld+json"]').each((_, el) => {
+ try {
+ const json = JSON.parse($(el).html());
+ if (json.articleBody || json.description) {
+ const body = json.articleBody || json.description;
+ if (body.length > caption.length) {
+ caption = cleanInstagramCaption(body);
+ }
+ }
+ if (json.author?.name && !author) {
+ author = json.author.name;
+ }
+ if (json.image && !cover) {
+ cover = Array.isArray(json.image) ? json.image[0] : json.image;
+ }
+ } catch (e) {
+ // ignore JSON parse error
+ }
+ });
+ }
+
+ log.info('[Instagram] Extracted metadata - Caption length:', caption.length, 'Author:', author, 'Has cover:', !!cover);
+
+ if (caption && caption.length > 20) {
+ const heuristicResult = parseRecipeCaptionHeuristic(caption, author, cover, canonicalUrl);
+ if (heuristicResult && heuristicResult.ingredients && heuristicResult.ingredients.length > 0) {
+ heuristicResult._scrapingMethod = 'instagram-caption-heuristic';
+ heuristicResult.rawCaption = caption;
+ heuristicResult.author = author;
+ heuristicResult.suggestAiPolish = true;
+ return heuristicResult;
+ }
+ }
+
+ // If automatic caption extraction was blocked or did not contain readable ingredients,
+ // we trigger the seamless caption-paste fallback with the extracted cover and author!
+ const wallError = new Error(
+ caption
+ ? 'Instagram caption was found, but does not appear to contain an ingredients list.'
+ : 'Instagram requires login to view post captions directly.'
+ );
+ wallError.requiresCaption = true;
+ wallError.isInstagram = true;
+ wallError.author = author;
+ wallError.coverImage = cover;
+ wallError.canonicalUrl = canonicalUrl;
+ throw wallError;
+}
+
// Helper: Scrape recipe from a TikTok video URL using caption extraction and AI parsing
async function scrapeTikTokRecipe(inputUrl) {
log.info('[TikTok] Starting scrape for:', inputUrl);
@@ -1545,7 +1722,11 @@ async function scrapeTikTokRecipe(inputUrl) {
}
if (!html) {
- throw new Error('Could not access TikTok video page. Automated access was blocked.');
+ const blockErr = new Error('Could not access TikTok video page. Automated access was blocked.');
+ blockErr.requiresCaption = true;
+ blockErr.isTikTok = true;
+ blockErr.canonicalUrl = canonicalUrl;
+ throw blockErr;
}
let caption = '';
@@ -1588,14 +1769,26 @@ async function scrapeTikTokRecipe(inputUrl) {
log.info('[TikTok] Extracted caption length:', caption.length, 'Author:', author);
if (!caption) {
- throw new Error('No caption or description found on this TikTok video. The creator might not have written recipe details.');
+ const noCapErr = new Error('No caption or description found on this TikTok video. The creator might not have written recipe details.');
+ noCapErr.requiresCaption = true;
+ noCapErr.isTikTok = true;
+ noCapErr.author = author;
+ noCapErr.coverImage = cover;
+ noCapErr.canonicalUrl = canonicalUrl;
+ throw noCapErr;
}
// Fast & Local Heuristic Engine First (Zero API cost, instant execution)
log.info('[TikTok] Running local heuristic parser on video caption...');
const heuristicResult = parseRecipeCaptionHeuristic(caption, author, cover, canonicalUrl);
if (!heuristicResult.ingredients || heuristicResult.ingredients.length === 0) {
- throw new Error('This TikTok caption does not appear to contain an ingredients list. The recipe might be in the audio, video comments, or on-screen text.');
+ const noIngErr = new Error('This TikTok caption does not appear to contain an ingredients list. The recipe might be in the audio, video comments, or on-screen text.');
+ noIngErr.requiresCaption = true;
+ noIngErr.isTikTok = true;
+ noIngErr.author = author;
+ noIngErr.coverImage = cover;
+ noIngErr.canonicalUrl = canonicalUrl;
+ throw noIngErr;
}
heuristicResult._scrapingMethod = 'tiktok-caption-heuristic';
@@ -2448,11 +2641,54 @@ app.post('/api/recipes/:id/revoke-share', requireAuth, (req, res) => {
}
});
-// API: Scrape Recipe from Link
+// API: Scrape Recipe from Link or Social Post Caption
app.post('/api/scrape', requireAuth, async (req, res) => {
- const { url } = req.body;
+ const { url, rawCaption, author: passedAuthor, coverImage: passedCover } = req.body;
+
+ // Direct Social Caption / Text Extraction (Instagram, TikTok, or manual paste)
+ if (rawCaption && typeof rawCaption === 'string' && rawCaption.trim()) {
+ try {
+ const heuristicResult = parseRecipeCaptionHeuristic(
+ rawCaption,
+ passedAuthor || '',
+ passedCover || '',
+ url || ''
+ );
+ if (!heuristicResult || !heuristicResult.ingredients || heuristicResult.ingredients.length === 0) {
+ return res.status(422).json({
+ error: 'Could not detect ingredients in the pasted text. Please make sure the text includes ingredients or try pasting additional details.'
+ });
+ }
+ heuristicResult._scrapingMethod = 'social-caption-paste';
+ heuristicResult.rawCaption = rawCaption;
+ heuristicResult.author = passedAuthor || '';
+ heuristicResult.suggestAiPolish = true;
+ return res.json(heuristicResult);
+ } catch (parseErr) {
+ return res.status(422).json({ error: parseErr.message || 'Failed to parse recipe from caption.' });
+ }
+ }
+
if (!url) {
- return res.status(400).json({ error: 'URL is required' });
+ return res.status(400).json({ error: 'URL or caption is required' });
+ }
+
+ // Specialized Scraper: Instagram Post & Reel Recipes
+ if (isInstagramUrl(url)) {
+ try {
+ const igResult = await scrapeInstagramRecipe(url);
+ return res.json(igResult);
+ } catch (igError) {
+ log.warn('[Instagram Scraper] Scrape note:', igError.message);
+ return res.status(422).json({
+ error: igError.message || 'Instagram post details could not be read automatically.',
+ requiresCaption: true,
+ isInstagram: true,
+ sourceUrl: url,
+ coverImage: igError.coverImage || '',
+ author: igError.author || ''
+ });
+ }
}
// Specialized Scraper: TikTok Video Recipes
@@ -2463,7 +2699,12 @@ app.post('/api/scrape', requireAuth, async (req, res) => {
} catch (tiktokError) {
log.error('[TikTok Scraper] Error:', tiktokError.message);
return res.status(422).json({
- error: tiktokError.message || 'Failed to extract recipe from TikTok video.'
+ error: tiktokError.message || 'Failed to extract recipe from TikTok video.',
+ requiresCaption: true,
+ isTikTok: true,
+ sourceUrl: url,
+ coverImage: tiktokError.coverImage || '',
+ author: tiktokError.author || ''
});
}
}