diff --git a/public/app.js b/public/app.js index 78f5a36..64e3080 100644 --- a/public/app.js +++ b/public/app.js @@ -632,6 +632,8 @@ async function handleScrapeSubmit(e) { isInstagram: errorData.isInstagram, isTikTok: errorData.isTikTok, isFacebook: errorData.isFacebook, + isTruncated: errorData.isTruncated, + partialCaption: errorData.partialCaption || '', author: errorData.author || '', coverImage: errorData.coverImage || '', errorMessage: errorData.error @@ -661,7 +663,7 @@ async function handleScrapeSubmit(e) { // Open Social Caption Extractor Modal (Instagram / TikTok / Facebook) function openSocialCaptionModal(options = {}) { - const { url = '', isInstagram = false, isTikTok = false, isFacebook = false, author = '', coverImage = '' } = options; + const { url = '', isInstagram = false, isTikTok = false, isFacebook = false, isTruncated = false, partialCaption = '', author = '', coverImage = '' } = options; const titleEl = document.getElementById('social-modal-title'); const iconEl = document.getElementById('social-modal-icon'); @@ -704,7 +706,11 @@ function openSocialCaptionModal(options = {}) { iconEl.style.color = '#1877F2'; } if (hintEl) { - hintEl.textContent = 'Facebook restricts automated extraction of Reel captions and comments. Simply copy the caption text or pinned recipe comment from Facebook and paste it below—ForkFolio will parse the recipe and format the ingredients automatically!'; + if (isTruncated && partialCaption) { + hintEl.innerHTML = '⚠️ Truncated Caption Detected
Facebook shortened this caption with an ellipsis. We\'ve pre-filled what was retrieved below—please paste the missing steps or copy the full text from Facebook!'; + } else { + hintEl.textContent = 'Facebook restricts automated extraction of Reel captions and comments. Simply copy the caption text or pinned recipe comment from Facebook and paste it below—ForkFolio will parse the recipe and format the ingredients automatically!'; + } } } else { if (titleEl) titleEl.textContent = 'Social Media Recipe Import'; @@ -720,7 +726,7 @@ function openSocialCaptionModal(options = {}) { if (urlInput) urlInput.value = url || ''; if (authorInput) authorInput.value = author || ''; if (coverInput) coverInput.value = coverImage || ''; - if (captionText) captionText.value = ''; + if (captionText) captionText.value = (isTruncated && partialCaption) ? partialCaption : ''; if (previewBox) { if (url || author || coverImage) { @@ -747,7 +753,12 @@ function openSocialCaptionModal(options = {}) { openModal('social-caption-modal'); lucide.createIcons(); if (captionText) { - setTimeout(() => captionText.focus(), 150); + setTimeout(() => { + captionText.focus(); + if (isTruncated && partialCaption) { + captionText.selectionStart = captionText.selectionEnd = captionText.value.length; + } + }, 150); } } diff --git a/server.js b/server.js index ea0d2b4..0fda361 100644 --- a/server.js +++ b/server.js @@ -1453,6 +1453,68 @@ function cleanFacebookCaption(raw) { return cleanSocialCaption(text); } +// Helper: Detect if a social caption is truncated by platform character limits or ellipsis +function isCaptionTruncated(text) { + if (!text || typeof text !== 'string') return false; + const trimmed = text.trim(); + // 1. Explicit ellipsis or "See more" markers + if (/(?:\.{3,}|…)\s*(?:See\s+more)?$/i.test(trimmed)) return true; + if (/\bSee\s+more$/i.test(trimmed)) return true; + // 2. Typical metadata character limit truncation (~180-320 chars without ending punctuation) + if (trimmed.length >= 180 && trimmed.length <= 320) { + const endsWithPunctuation = /[.!?\n\r"']$/.test(trimmed); + if (!endsWithPunctuation) return true; + } + return false; +} + +// Helper: Extract full untruncated Facebook message from embedded Relay / GraphQL / script tags +function extractFullFacebookMessageFromHtml(html) { + if (!html || typeof html !== 'string') return ''; + let longestText = ''; + + const isBoilerplate = (str) => /log in|watch popular videos|sign up for facebook|connect with friends/i.test(str); + + // 1. Search for JSON Relay message text blobs: "message":{"text":"..."} or "savable_description":{"text":"..."} + const messageRegex = /"(?:message|savable_description)"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g; + let match; + while ((match = messageRegex.exec(html)) !== null) { + try { + const decoded = JSON.parse(`"${match[1]}"`); + if (decoded && decoded.length > longestText.length && !isBoilerplate(decoded)) { + longestText = decoded; + } + } catch (e) { + // Ignore JSON parse errors on malformed sub-strings + } + } + + // 2. Search for creation_story or story messages + const storyRegex = /"creation_story"\s*:\s*\{[\s\S]*?"message"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g; + while ((match = storyRegex.exec(html)) !== null) { + try { + const decoded = JSON.parse(`"${match[1]}"`); + if (decoded && decoded.length > longestText.length && !isBoilerplate(decoded)) { + longestText = decoded; + } + } catch (e) {} + } + + // 3. Check JSON-LD articleBody or description + const ldJsonRegex = /]*>([\s\S]*?)<\/script>/gi; + while ((match = ldJsonRegex.exec(html)) !== null) { + try { + const data = JSON.parse(match[1]); + const body = data.articleBody || data.description || ''; + if (body && body.length > longestText.length && !isBoilerplate(body)) { + longestText = body; + } + } catch (e) {} + } + + return cleanFacebookCaption(longestText); +} + // Helper: Clean raw Instagram caption from OpenGraph tags or quotes function cleanInstagramCaption(raw) { if (!raw) return ''; @@ -1910,34 +1972,49 @@ async function scrapeFacebookRecipe(inputUrl) { caption = cleanFacebookCaption(ogDesc); } - // Inspect script tags or embedded ld+json - $('script[type="application/ld+json"]').each((_, el) => { - try { - const json = JSON.parse($(el).html()); - if (json.articleBody || json.description) { - const body = json.articleBody || json.description; - if (body.length > caption.length && !/log in|sign up for facebook/i.test(body)) { - caption = cleanFacebookCaption(body); - } - } - if (json.author?.name && !author) { - author = json.author.name; - } - if (json.image && !cover) { - cover = Array.isArray(json.image) ? json.image[0] : json.image; - } - if (json.thumbnailUrl && !cover) { - cover = json.thumbnailUrl; - } - } catch (e) { - // ignore JSON parse error - } - }); + // Check embedded JSON / Relay scripts / JSON-LD for full untruncated message + const embeddedFullMessage = extractFullFacebookMessageFromHtml(html); + if (embeddedFullMessage && embeddedFullMessage.length > caption.length) { + log.info(`[Facebook] Found full untruncated message in embedded payload (${embeddedFullMessage.length} chars vs og:desc ${caption.length} chars)`); + caption = embeddedFullMessage; + } } - log.info('[Facebook] Extracted metadata - Caption length:', caption.length, 'Author:', author, 'Has cover:', !!cover); + // Strategy 2: If caption was blocked, empty, or truncated by edge servers, attempt Puppeteer fallback + let isTruncated = isCaptionTruncated(caption); + if (!caption || isTruncated) { + log.info(`[Facebook] Caption is ${!caption ? 'empty' : 'truncated'}. Attempting Puppeteer fallback...`); + try { + const puppeteerHtml = await fetchWithPuppeteer(canonicalUrl); + if (puppeteerHtml && !isBotChallengePage(puppeteerHtml)) { + const pup$ = cheerio.load(puppeteerHtml); + if (!cover) { + cover = pup$('meta[property="og:image"]').attr('content') || + pup$('meta[property="og:image:secure_url"]').attr('content') || ''; + } + if (!author) { + const pupTitle = pup$('meta[property="og:title"]').attr('content') || pup$('title').text().trim() || ''; + const match = pupTitle.match(/^(?:.*?on Facebook:\s*["“]?|)(.*?)(?:\s*[-–|•]\s*Facebook|\s*on Facebook|$)/i); + if (match && match[1] && !match[1].toLowerCase().includes('facebook')) { + author = match[1].trim(); + } + } + const pupEmbedded = extractFullFacebookMessageFromHtml(puppeteerHtml); + if (pupEmbedded && pupEmbedded.length > caption.length) { + log.info(`[Facebook] Puppeteer extracted longer caption (${pupEmbedded.length} chars)`); + caption = pupEmbedded; + isTruncated = isCaptionTruncated(caption); + } + } + } catch (pupErr) { + log.warn('[Facebook] Puppeteer fallback note:', pupErr.message); + } + } - if (caption && caption.length > 20) { + log.info('[Facebook] Extracted metadata - Caption length:', caption.length, 'IsTruncated:', isTruncated, 'Author:', author, 'Has cover:', !!cover); + + // If we have an untruncated caption with ingredients, parse immediately + if (caption && !isTruncated && caption.length > 20) { const heuristicResult = parseRecipeCaptionHeuristic(caption, author, cover, canonicalUrl); if (heuristicResult && heuristicResult.ingredients && heuristicResult.ingredients.length > 0) { heuristicResult._scrapingMethod = 'facebook-caption-heuristic'; @@ -1948,6 +2025,19 @@ async function scrapeFacebookRecipe(inputUrl) { } } + // If caption was truncated, trigger modal with pre-filled partial text + if (isTruncated && caption) { + const truncError = new Error('Facebook truncated this caption. We have pre-filled what was retrieved—please paste the missing steps or full caption.'); + truncError.requiresCaption = true; + truncError.isFacebook = true; + truncError.isTruncated = true; + truncError.partialCaption = caption; + truncError.author = author; + truncError.coverImage = cover; + truncError.canonicalUrl = canonicalUrl; + throw truncError; + } + // Fallback to social-caption-paste modal const wallError = new Error( caption @@ -1956,6 +2046,7 @@ async function scrapeFacebookRecipe(inputUrl) { ); wallError.requiresCaption = true; wallError.isFacebook = true; + wallError.isTruncated = false; wallError.author = author; wallError.coverImage = cover; wallError.canonicalUrl = canonicalUrl; @@ -2884,6 +2975,8 @@ app.post('/api/scrape', requireAuth, async (req, res) => { error: fbError.message || 'Facebook Reel details could not be read automatically.', requiresCaption: true, isFacebook: true, + isTruncated: !!fbError.isTruncated, + partialCaption: fbError.partialCaption || '', sourceUrl: url, coverImage: fbError.coverImage || '', author: fbError.author || '' diff --git a/test_facebook_scraper.js b/test_facebook_scraper.js index 716609d..43e220b 100644 --- a/test_facebook_scraper.js +++ b/test_facebook_scraper.js @@ -1,6 +1,6 @@ const assert = require('assert'); -// 1. Test URL detection helper +// 1. URL detection helper function isFacebookUrl(url) { try { const parsed = new URL(url); @@ -41,6 +41,59 @@ function cleanFacebookCaption(raw) { return cleanSocialCaption(text); } +function isCaptionTruncated(text) { + if (!text || typeof text !== 'string') return false; + const trimmed = text.trim(); + if (/(?:\.{3,}|…)\s*(?:See\s+more)?$/i.test(trimmed)) return true; + if (/\bSee\s+more$/i.test(trimmed)) return true; + if (trimmed.length >= 180 && trimmed.length <= 320) { + const endsWithPunctuation = /[.!?\n\r"']$/.test(trimmed); + if (!endsWithPunctuation) return true; + } + return false; +} + +function extractFullFacebookMessageFromHtml(html) { + if (!html || typeof html !== 'string') return ''; + let longestText = ''; + + const isBoilerplate = (str) => /log in|watch popular videos|sign up for facebook|connect with friends/i.test(str); + + const messageRegex = /"(?:message|savable_description)"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g; + let match; + while ((match = messageRegex.exec(html)) !== null) { + try { + const decoded = JSON.parse(`"${match[1]}"`); + if (decoded && decoded.length > longestText.length && !isBoilerplate(decoded)) { + longestText = decoded; + } + } catch (e) {} + } + + const storyRegex = /"creation_story"\s*:\s*\{[\s\S]*?"message"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g; + while ((match = storyRegex.exec(html)) !== null) { + try { + const decoded = JSON.parse(`"${match[1]}"`); + if (decoded && decoded.length > longestText.length && !isBoilerplate(decoded)) { + longestText = decoded; + } + } catch (e) {} + } + + const ldJsonRegex = /]*>([\s\S]*?)<\/script>/gi; + while ((match = ldJsonRegex.exec(html)) !== null) { + try { + const data = JSON.parse(match[1]); + const body = data.articleBody || data.description || ''; + if (body && body.length > longestText.length && !isBoilerplate(body)) { + longestText = body; + } + } catch (e) {} + } + + return cleanFacebookCaption(longestText); +} + console.log('Testing isFacebookUrl...'); assert.strictEqual(isFacebookUrl('https://www.facebook.com/reel/123456789'), true); assert.strictEqual(isFacebookUrl('https://fb.watch/abcdef123/'), true); @@ -48,17 +101,35 @@ assert.strictEqual(isFacebookUrl('https://m.facebook.com/reel/987654321/'), true assert.strictEqual(isFacebookUrl('https://www.facebook.com/watch/?v=11223344'), true); assert.strictEqual(isFacebookUrl('https://www.facebook.com/share/r/abc123xyz/'), true); assert.strictEqual(isFacebookUrl('https://www.instagram.com/reel/xyz123'), false); -assert.strictEqual(isFacebookUrl('https://www.allrecipes.com/recipe/123/cookies'), false); console.log('✔ isFacebookUrl tests passed!'); -console.log('\nTesting cleanFacebookCaption...'); -const raw1 = 'Chef Gordon on Facebook: "Crispy Garlic Butter Steak Bites. 1 lb sirloin steak, 2 tbsp butter. Cook for 5 mins #food #steak"'; -const cleaned1 = cleanFacebookCaption(raw1); -assert.strictEqual(cleaned1, 'Crispy Garlic Butter Steak Bites. 1 lb sirloin steak, 2 tbsp butter. Cook for 5 mins'); +console.log('\nTesting isCaptionTruncated...'); +assert.strictEqual(isCaptionTruncated('1 cup flour, 2 eggs, 1 tbsp butter...'), true); +assert.strictEqual(isCaptionTruncated('1 cup flour, 2 eggs, 1 tbsp butter…'), true); +assert.strictEqual(isCaptionTruncated('1 cup flour, 2 eggs, 1 tbsp butter... See more'), true); +assert.strictEqual(isCaptionTruncated('1 cup flour, 2 eggs, 1 tbsp butter… See More'), true); +assert.strictEqual(isCaptionTruncated('This is a truncated facebook description that hits around two hundred characters and stops abruptly in the middle of a sentence without any final punctuation mark like a period so it indicates cut off'), true); +assert.strictEqual(isCaptionTruncated('1 cup flour, 2 eggs. Mix thoroughly and bake for 25 minutes at 350F.'), false); +console.log('✔ isCaptionTruncated tests passed!'); -const raw2 = 'Delicious Homemade Pasta Recipe | 2 cups flour, 3 eggs | By Italian Kitchen | Facebook'; -const cleaned2 = cleanFacebookCaption(raw2); -assert.strictEqual(cleaned2, 'Delicious Homemade Pasta Recipe | 2 cups flour, 3 eggs'); -console.log('✔ cleanFacebookCaption tests passed!'); +console.log('\nTesting extractFullFacebookMessageFromHtml...'); +const sampleHtml = ` + + + + + + + + +`; -console.log('\nAll Facebook Reel unit tests passed successfully!'); +const extracted = extractFullFacebookMessageFromHtml(sampleHtml); +assert.ok(extracted.includes('4 cloves garlic, minced')); +assert.ok(extracted.includes('Directions:')); +assert.strictEqual(isCaptionTruncated(extracted), false); +console.log('✔ extractFullFacebookMessageFromHtml successfully retrieved complete message!'); + +console.log('\nAll Facebook Reel scraper & truncation tests passed successfully!');