feat: detect Facebook caption truncation, extract embedded Relay payloads, and pre-fill fallback modal

public/app.js

@@ -632,6 +632,8 @@ async function handleScrapeSubmit(e) {
           isInstagram: errorData.isInstagram,
           isTikTok: errorData.isTikTok,
           isFacebook: errorData.isFacebook,
+          isTruncated: errorData.isTruncated,
+          partialCaption: errorData.partialCaption || '',
           author: errorData.author || '',
           coverImage: errorData.coverImage || '',
           errorMessage: errorData.error
@@ -661,7 +663,7 @@ async function handleScrapeSubmit(e) {
 
 // Open Social Caption Extractor Modal (Instagram / TikTok / Facebook)
 function openSocialCaptionModal(options = {}) {
-  const { url = '', isInstagram = false, isTikTok = false, isFacebook = false, author = '', coverImage = '' } = options;
+  const { url = '', isInstagram = false, isTikTok = false, isFacebook = false, isTruncated = false, partialCaption = '', author = '', coverImage = '' } = options;
 
   const titleEl = document.getElementById('social-modal-title');
   const iconEl = document.getElementById('social-modal-icon');
@@ -704,8 +706,12 @@ function openSocialCaptionModal(options = {}) {
       iconEl.style.color = '#1877F2';
     }
     if (hintEl) {
+      if (isTruncated && partialCaption) {
+        hintEl.innerHTML = '<span style="display: inline-flex; align-items: center; gap: 4px; background: rgba(245, 158, 11, 0.15); color: #d97706; padding: 2px 8px; border-radius: 4px; font-weight: 600; font-size: 0.8rem; margin-bottom: 6px;">⚠️ Truncated Caption Detected</span><br>Facebook shortened this caption with an ellipsis. We\'ve pre-filled what was retrieved below—please paste the missing steps or copy the full text from Facebook!';
+      } else {
         hintEl.textContent = 'Facebook restricts automated extraction of Reel captions and comments. Simply copy the caption text or pinned recipe comment from Facebook and paste it below—ForkFolio will parse the recipe and format the ingredients automatically!';
       }
+    }
   } else {
     if (titleEl) titleEl.textContent = 'Social Media Recipe Import';
     if (iconEl) {
@@ -720,7 +726,7 @@ function openSocialCaptionModal(options = {}) {
   if (urlInput) urlInput.value = url || '';
   if (authorInput) authorInput.value = author || '';
   if (coverInput) coverInput.value = coverImage || '';
-  if (captionText) captionText.value = '';
+  if (captionText) captionText.value = (isTruncated && partialCaption) ? partialCaption : '';
 
   if (previewBox) {
     if (url || author || coverImage) {
@@ -747,7 +753,12 @@ function openSocialCaptionModal(options = {}) {
   openModal('social-caption-modal');
   lucide.createIcons();
   if (captionText) {
-    setTimeout(() => captionText.focus(), 150);
+    setTimeout(() => {
+      captionText.focus();
+      if (isTruncated && partialCaption) {
+        captionText.selectionStart = captionText.selectionEnd = captionText.value.length;
+      }
+    }, 150);
   }
 }
 

server.js

@@ -1453,6 +1453,68 @@ function cleanFacebookCaption(raw) {
   return cleanSocialCaption(text);
 }
 
+// Helper: Detect if a social caption is truncated by platform character limits or ellipsis
+function isCaptionTruncated(text) {
+  if (!text || typeof text !== 'string') return false;
+  const trimmed = text.trim();
+  // 1. Explicit ellipsis or "See more" markers
+  if (/(?:\.{3,}|…)\s*(?:See\s+more)?$/i.test(trimmed)) return true;
+  if (/\bSee\s+more$/i.test(trimmed)) return true;
+  // 2. Typical metadata character limit truncation (~180-320 chars without ending punctuation)
+  if (trimmed.length >= 180 && trimmed.length <= 320) {
+    const endsWithPunctuation = /[.!?\n\r"']$/.test(trimmed);
+    if (!endsWithPunctuation) return true;
+  }
+  return false;
+}
+
+// Helper: Extract full untruncated Facebook message from embedded Relay / GraphQL / script tags
+function extractFullFacebookMessageFromHtml(html) {
+  if (!html || typeof html !== 'string') return '';
+  let longestText = '';
+
+  const isBoilerplate = (str) => /log in|watch popular videos|sign up for facebook|connect with friends/i.test(str);
+
+  // 1. Search for JSON Relay message text blobs: "message":{"text":"..."} or "savable_description":{"text":"..."}
+  const messageRegex = /"(?:message|savable_description)"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g;
+  let match;
+  while ((match = messageRegex.exec(html)) !== null) {
+    try {
+      const decoded = JSON.parse(`"${match[1]}"`);
+      if (decoded && decoded.length > longestText.length && !isBoilerplate(decoded)) {
+        longestText = decoded;
+      }
+    } catch (e) {
+      // Ignore JSON parse errors on malformed sub-strings
+    }
+  }
+
+  // 2. Search for creation_story or story messages
+  const storyRegex = /"creation_story"\s*:\s*\{[\s\S]*?"message"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g;
+  while ((match = storyRegex.exec(html)) !== null) {
+    try {
+      const decoded = JSON.parse(`"${match[1]}"`);
+      if (decoded && decoded.length > longestText.length && !isBoilerplate(decoded)) {
+        longestText = decoded;
+      }
+    } catch (e) {}
+  }
+
+  // 3. Check JSON-LD articleBody or description
+  const ldJsonRegex = /<script\s+type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi;
+  while ((match = ldJsonRegex.exec(html)) !== null) {
+    try {
+      const data = JSON.parse(match[1]);
+      const body = data.articleBody || data.description || '';
+      if (body && body.length > longestText.length && !isBoilerplate(body)) {
+        longestText = body;
+      }
+    } catch (e) {}
+  }
+
+  return cleanFacebookCaption(longestText);
+}
+
 // Helper: Clean raw Instagram caption from OpenGraph tags or quotes
 function cleanInstagramCaption(raw) {
   if (!raw) return '';
@@ -1910,34 +1972,49 @@ async function scrapeFacebookRecipe(inputUrl) {
       caption = cleanFacebookCaption(ogDesc);
     }
 
-    // Inspect script tags or embedded ld+json
+    // Check embedded JSON / Relay scripts / JSON-LD for full untruncated message
-    $('script[type="application/ld+json"]').each((_, el) => {
+    const embeddedFullMessage = extractFullFacebookMessageFromHtml(html);
+    if (embeddedFullMessage && embeddedFullMessage.length > caption.length) {
+      log.info(`[Facebook] Found full untruncated message in embedded payload (${embeddedFullMessage.length} chars vs og:desc ${caption.length} chars)`);
+      caption = embeddedFullMessage;
+    }
+  }
+
+  // Strategy 2: If caption was blocked, empty, or truncated by edge servers, attempt Puppeteer fallback
+  let isTruncated = isCaptionTruncated(caption);
+  if (!caption || isTruncated) {
+    log.info(`[Facebook] Caption is ${!caption ? 'empty' : 'truncated'}. Attempting Puppeteer fallback...`);
     try {
-        const json = JSON.parse($(el).html());
+      const puppeteerHtml = await fetchWithPuppeteer(canonicalUrl);
-        if (json.articleBody || json.description) {
+      if (puppeteerHtml && !isBotChallengePage(puppeteerHtml)) {
-          const body = json.articleBody || json.description;
+        const pup$ = cheerio.load(puppeteerHtml);
-          if (body.length > caption.length && !/log in|sign up for facebook/i.test(body)) {
+        if (!cover) {
-            caption = cleanFacebookCaption(body);
+          cover = pup$('meta[property="og:image"]').attr('content') ||
+                  pup$('meta[property="og:image:secure_url"]').attr('content') || '';
+        }
+        if (!author) {
+          const pupTitle = pup$('meta[property="og:title"]').attr('content') || pup$('title').text().trim() || '';
+          const match = pupTitle.match(/^(?:.*?on Facebook:\s*["“]?|)(.*?)(?:\s*[-–|•]\s*Facebook|\s*on Facebook|$)/i);
+          if (match && match[1] && !match[1].toLowerCase().includes('facebook')) {
+            author = match[1].trim();
           }
         }
-        if (json.author?.name && !author) {
+        const pupEmbedded = extractFullFacebookMessageFromHtml(puppeteerHtml);
-          author = json.author.name;
+        if (pupEmbedded && pupEmbedded.length > caption.length) {
+          log.info(`[Facebook] Puppeteer extracted longer caption (${pupEmbedded.length} chars)`);
+          caption = pupEmbedded;
+          isTruncated = isCaptionTruncated(caption);
         }
-        if (json.image && !cover) {
-          cover = Array.isArray(json.image) ? json.image[0] : json.image;
       }
-        if (json.thumbnailUrl && !cover) {
+    } catch (pupErr) {
-          cover = json.thumbnailUrl;
+      log.warn('[Facebook] Puppeteer fallback note:', pupErr.message);
     }
-      } catch (e) {
-        // ignore JSON parse error
-      }
-    });
   }
 
-  log.info('[Facebook] Extracted metadata - Caption length:', caption.length, 'Author:', author, 'Has cover:', !!cover);
+  log.info('[Facebook] Extracted metadata - Caption length:', caption.length, 'IsTruncated:', isTruncated, 'Author:', author, 'Has cover:', !!cover);
 
-  if (caption && caption.length > 20) {
+  // If we have an untruncated caption with ingredients, parse immediately
+  if (caption && !isTruncated && caption.length > 20) {
     const heuristicResult = parseRecipeCaptionHeuristic(caption, author, cover, canonicalUrl);
     if (heuristicResult && heuristicResult.ingredients && heuristicResult.ingredients.length > 0) {
       heuristicResult._scrapingMethod = 'facebook-caption-heuristic';
@@ -1948,6 +2025,19 @@ async function scrapeFacebookRecipe(inputUrl) {
     }
   }
 
+  // If caption was truncated, trigger modal with pre-filled partial text
+  if (isTruncated && caption) {
+    const truncError = new Error('Facebook truncated this caption. We have pre-filled what was retrieved—please paste the missing steps or full caption.');
+    truncError.requiresCaption = true;
+    truncError.isFacebook = true;
+    truncError.isTruncated = true;
+    truncError.partialCaption = caption;
+    truncError.author = author;
+    truncError.coverImage = cover;
+    truncError.canonicalUrl = canonicalUrl;
+    throw truncError;
+  }
+
   // Fallback to social-caption-paste modal
   const wallError = new Error(
     caption
@@ -1956,6 +2046,7 @@ async function scrapeFacebookRecipe(inputUrl) {
   );
   wallError.requiresCaption = true;
   wallError.isFacebook = true;
+  wallError.isTruncated = false;
   wallError.author = author;
   wallError.coverImage = cover;
   wallError.canonicalUrl = canonicalUrl;
@@ -2884,6 +2975,8 @@ app.post('/api/scrape', requireAuth, async (req, res) => {
         error: fbError.message || 'Facebook Reel details could not be read automatically.',
         requiresCaption: true,
         isFacebook: true,
+        isTruncated: !!fbError.isTruncated,
+        partialCaption: fbError.partialCaption || '',
         sourceUrl: url,
         coverImage: fbError.coverImage || '',
         author: fbError.author || ''

test_facebook_scraper.js

@@ -1,6 +1,6 @@
 const assert = require('assert');
 
-// 1. Test URL detection helper
+// 1. URL detection helper
 function isFacebookUrl(url) {
   try {
     const parsed = new URL(url);
@@ -41,6 +41,59 @@ function cleanFacebookCaption(raw) {
   return cleanSocialCaption(text);
 }
 
+function isCaptionTruncated(text) {
+  if (!text || typeof text !== 'string') return false;
+  const trimmed = text.trim();
+  if (/(?:\.{3,}|…)\s*(?:See\s+more)?$/i.test(trimmed)) return true;
+  if (/\bSee\s+more$/i.test(trimmed)) return true;
+  if (trimmed.length >= 180 && trimmed.length <= 320) {
+    const endsWithPunctuation = /[.!?\n\r"']$/.test(trimmed);
+    if (!endsWithPunctuation) return true;
+  }
+  return false;
+}
+
+function extractFullFacebookMessageFromHtml(html) {
+  if (!html || typeof html !== 'string') return '';
+  let longestText = '';
+
+  const isBoilerplate = (str) => /log in|watch popular videos|sign up for facebook|connect with friends/i.test(str);
+
+  const messageRegex = /"(?:message|savable_description)"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g;
+  let match;
+  while ((match = messageRegex.exec(html)) !== null) {
+    try {
+      const decoded = JSON.parse(`"${match[1]}"`);
+      if (decoded && decoded.length > longestText.length && !isBoilerplate(decoded)) {
+        longestText = decoded;
+      }
+    } catch (e) {}
+  }
+
+  const storyRegex = /"creation_story"\s*:\s*\{[\s\S]*?"message"\s*:\s*\{\s*"text"\s*:\s*"((?:[^"\\]|\\.)*)"/g;
+  while ((match = storyRegex.exec(html)) !== null) {
+    try {
+      const decoded = JSON.parse(`"${match[1]}"`);
+      if (decoded && decoded.length > longestText.length && !isBoilerplate(decoded)) {
+        longestText = decoded;
+      }
+    } catch (e) {}
+  }
+
+  const ldJsonRegex = /<script\s+type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi;
+  while ((match = ldJsonRegex.exec(html)) !== null) {
+    try {
+      const data = JSON.parse(match[1]);
+      const body = data.articleBody || data.description || '';
+      if (body && body.length > longestText.length && !isBoilerplate(body)) {
+        longestText = body;
+      }
+    } catch (e) {}
+  }
+
+  return cleanFacebookCaption(longestText);
+}
+
 console.log('Testing isFacebookUrl...');
 assert.strictEqual(isFacebookUrl('https://www.facebook.com/reel/123456789'), true);
 assert.strictEqual(isFacebookUrl('https://fb.watch/abcdef123/'), true);
@@ -48,17 +101,35 @@ assert.strictEqual(isFacebookUrl('https://m.facebook.com/reel/987654321/'), true
 assert.strictEqual(isFacebookUrl('https://www.facebook.com/watch/?v=11223344'), true);
 assert.strictEqual(isFacebookUrl('https://www.facebook.com/share/r/abc123xyz/'), true);
 assert.strictEqual(isFacebookUrl('https://www.instagram.com/reel/xyz123'), false);
-assert.strictEqual(isFacebookUrl('https://www.allrecipes.com/recipe/123/cookies'), false);
 console.log('✔ isFacebookUrl tests passed!');
 
-console.log('\nTesting cleanFacebookCaption...');
+console.log('\nTesting isCaptionTruncated...');
-const raw1 = 'Chef Gordon on Facebook: "Crispy Garlic Butter Steak Bites. 1 lb sirloin steak, 2 tbsp butter. Cook for 5 mins #food #steak"';
+assert.strictEqual(isCaptionTruncated('1 cup flour, 2 eggs, 1 tbsp butter...'), true);
-const cleaned1 = cleanFacebookCaption(raw1);
+assert.strictEqual(isCaptionTruncated('1 cup flour, 2 eggs, 1 tbsp butter…'), true);
-assert.strictEqual(cleaned1, 'Crispy Garlic Butter Steak Bites. 1 lb sirloin steak, 2 tbsp butter. Cook for 5 mins');
+assert.strictEqual(isCaptionTruncated('1 cup flour, 2 eggs, 1 tbsp butter... See more'), true);
+assert.strictEqual(isCaptionTruncated('1 cup flour, 2 eggs, 1 tbsp butter… See More'), true);
+assert.strictEqual(isCaptionTruncated('This is a truncated facebook description that hits around two hundred characters and stops abruptly in the middle of a sentence without any final punctuation mark like a period so it indicates cut off'), true);
+assert.strictEqual(isCaptionTruncated('1 cup flour, 2 eggs. Mix thoroughly and bake for 25 minutes at 350F.'), false);
+console.log('✔ isCaptionTruncated tests passed!');
 
-const raw2 = 'Delicious Homemade Pasta Recipe | 2 cups flour, 3 eggs | By Italian Kitchen | Facebook';
+console.log('\nTesting extractFullFacebookMessageFromHtml...');
-const cleaned2 = cleanFacebookCaption(raw2);
+const sampleHtml = `
-assert.strictEqual(cleaned2, 'Delicious Homemade Pasta Recipe | 2 cups flour, 3 eggs');
+<html>
-console.log('✔ cleanFacebookCaption tests passed!');
+<head>
+  <meta property="og:description" content="Chef Gordon: Crispy Garlic Butter Steak Bites. 1 lb sirloin steak, 2 tbsp butter...">
+</head>
+<body>
+  <script type="application/json" data-sjs>
+    {"require":[["RelayPrefetchedStreamCache","next",[],[{"message":{"text":"Crispy Garlic Butter Steak Bites.\\n\\nIngredients:\\n- 1 lb sirloin steak\\n- 2 tbsp butter\\n- 4 cloves garlic, minced\\n\\nDirections:\\n1. Sear steak in hot pan for 3 mins.\\n2. Add butter and garlic.\\n3. Serve warm!"}}]]]}
+  </script>
+</body>
+</html>
+`;
 
-console.log('\nAll Facebook Reel unit tests passed successfully!');
+const extracted = extractFullFacebookMessageFromHtml(sampleHtml);
+assert.ok(extracted.includes('4 cloves garlic, minced'));
+assert.ok(extracted.includes('Directions:'));
+assert.strictEqual(isCaptionTruncated(extracted), false);
+console.log('✔ extractFullFacebookMessageFromHtml successfully retrieved complete message!');
+
+console.log('\nAll Facebook Reel scraper & truncation tests passed successfully!');