const express = require('express');
const axios = require('axios');
const cheerio = require('cheerio');
const cors = require('cors');
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');

// Enable stealth mode to bypass bot detection (Cloudflare, Akamai, etc.)
puppeteer.use(StealthPlugin());

const app = express();
const PORT = process.env.PORT || 3000;

// Setup directories for persistence
const DATA_DIR = path.join(__dirname, 'data');
const DATA_FILE = path.join(DATA_DIR, 'recipes.json');
const USERS_FILE = path.join(DATA_DIR, 'users.json');

if (!fs.existsSync(DATA_DIR)) {
  fs.mkdirSync(DATA_DIR, { recursive: true });
}
if (!fs.existsSync(DATA_FILE)) {
  fs.writeFileSync(DATA_FILE, JSON.stringify([], null, 2));
}
if (!fs.existsSync(USERS_FILE)) {
  fs.writeFileSync(USERS_FILE, JSON.stringify([], null, 2));
}

// Ensure default admin user exists
try {
  const usersData = fs.readFileSync(USERS_FILE, 'utf8');
  const existingUsers = JSON.parse(usersData);
  const hasAdmin = existingUsers.some(u => u.isAdmin === true);
  if (!hasAdmin) {
    const salt = crypto.randomBytes(16).toString('hex');
    const passwordHash = crypto.pbkdf2Sync('admin123', salt, 1000, 64, 'sha512').toString('hex');
    existingUsers.push({
      id: 'u_admin',
      username: 'admin',
      salt,
      passwordHash,
      isAdmin: true,
      isRecipeManager: true,
      createdAt: new Date().toISOString()
    });
    fs.writeFileSync(USERS_FILE, JSON.stringify(existingUsers, null, 2), 'utf8');
    console.log('--- ForkFolio Admin Initialized ---');
    console.log('Default Admin Username: admin');
    console.log('Default Admin Password: admin123');
    console.log('--------------------------------------');
  }
} catch (e) {
  console.error('Failed to initialize default admin user:', e);
}

// In-memory sessions storage (token -> { id, username })
const activeSessions = new Map();

app.use(cors());
app.use(express.json({ limit: '10mb' }));
app.use(express.urlencoded({ limit: '10mb', extended: true }));
app.use(express.static(path.join(__dirname, 'public')));

// Helper: Parse ISO 8601 Durations (e.g. PT1H30M to "1 hr 30 mins")
function parseISODuration(duration) {
  if (!duration || typeof duration !== 'string') return '';
  const match = duration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/);
  if (!match) return duration;
  const hours = match[1] ? parseInt(match[1], 10) : 0;
  const minutes = match[2] ? parseInt(match[2], 10) : 0;
  
  let result = '';
  if (hours > 0) result += `${hours} hr${hours > 1 ? 's' : ''} `;
  if (minutes > 0) result += `${minutes} min${minutes > 1 ? 's' : ''}`;
  return result.trim() || duration;
}

// Helper: Extract recipe instructions from schema format
function extractInstructions(instructions) {
  if (!instructions) return [];
  if (typeof instructions === 'string') {
    return instructions.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
  }
  if (Array.isArray(instructions)) {
    const result = [];
    for (const inst of instructions) {
      if (typeof inst === 'string') {
        result.push(inst.trim());
      } else if (inst && typeof inst === 'object') {
        if (inst['@type'] === 'HowToStep') {
          result.push(inst.text || inst.name || '');
        } else if (inst['@type'] === 'HowToSection' && Array.isArray(inst.itemListElement)) {
          if (inst.name) {
            result.push(`### ${inst.name}`); // Prefix section names to identify them in the frontend
          }
          for (const step of inst.itemListElement) {
            if (typeof step === 'string') {
              result.push(step.trim());
            } else if (step && typeof step === 'object') {
              result.push(step.text || step.name || '');
            }
          }
        } else if (inst.text) {
          result.push(inst.text);
        } else if (inst.name) {
          result.push(inst.name);
        }
      }
    }
    return result.filter(Boolean);
  }
  return [];
}

// Helper: Normalize image resources from JSON-LD schema
function normalizeImages(imageObj) {
  if (!imageObj) return [];
  if (typeof imageObj === 'string') return [imageObj];
  if (Array.isArray(imageObj)) {
    return imageObj.map(img => (typeof img === 'string' ? img : img.url)).filter(Boolean);
  }
  if (typeof imageObj === 'object' && imageObj.url) {
    return [imageObj.url];
  }
  return [];
}

// Helper: Retrieve saved users
function readUsers() {
  try {
    const data = fs.readFileSync(USERS_FILE, 'utf8');
    let users = JSON.parse(data);
    let migrated = false;
    users = users.map(u => {
      if (u.isAdmin && !u.isRecipeManager) {
        u.isRecipeManager = true;
        migrated = true;
      }
      return u;
    });
    if (migrated) {
      writeUsers(users);
    }
    return users;
  } catch (e) {
    console.error('Error reading users database:', e);
    return [];
  }
}

// Helper: Save users to storage
function writeUsers(users) {
  try {
    fs.writeFileSync(USERS_FILE, JSON.stringify(users, null, 2), 'utf8');
    return true;
  } catch (e) {
    console.error('Error writing users database:', e);
    return false;
  }
}

// Password Hashing and Salt helpers
function generateSalt() {
  return crypto.randomBytes(16).toString('hex');
}

function hashPassword(password, salt) {
  return crypto.pbkdf2Sync(password, salt, 1000, 64, 'sha512').toString('hex');
}

// Middleware: Authenticate Session Token
function requireAuth(req, res, next) {
  const authHeader = req.headers.authorization;
  if (!authHeader || !authHeader.startsWith('Bearer ')) {
    return res.status(401).json({ error: 'Authorization required. Please log in.' });
  }
  const token = authHeader.substring(7);
  const session = activeSessions.get(token);
  if (!session) {
    return res.status(401).json({ error: 'Session expired. Please log in again.' });
  }
  req.user = session;
  next();
}

// Helper: Retrieve saved recipes with automatic backward-compatibility migration
function readRecipes() {
  try {
    const data = fs.readFileSync(DATA_FILE, 'utf8');
    let recipes = JSON.parse(data);
    let migrated = false;
    recipes = recipes.map(r => {
      if (!r.userId) {
        r.userId = 'system_admin';
        r.createdBy = 'gourmet_system';
        r.isPublic = true;
        migrated = true;
      }
      return r;
    });
    if (migrated) {
      writeRecipes(recipes);
    }
    return recipes;
  } catch (e) {
    console.error('Error reading recipes database:', e);
    return [];
  }
}

// Helper: Save recipes to storage
function writeRecipes(recipes) {
  try {
    fs.writeFileSync(DATA_FILE, JSON.stringify(recipes, null, 2), 'utf8');
    return true;
  } catch (e) {
    console.error('Error writing recipes database:', e);
    return false;
  }
}

// Auth API: User Registration (Requires Admin session)
app.post('/api/auth/register', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }

  const { username, password } = req.body;
  if (!username || !password) {
    return res.status(400).json({ error: 'Username and password are required.' });
  }
  
  const cleanUsername = username.trim().toLowerCase();
  if (!/^[a-z0-9_]{3,20}$/.test(cleanUsername)) {
    return res.status(400).json({ error: 'Username must be 3-20 characters long and contain only alphanumeric characters or underscores.' });
  }
  
  if (password.length < 6) {
    return res.status(400).json({ error: 'Password must be at least 6 characters long.' });
  }
  
  const users = readUsers();
  const existingUser = users.find(u => u.username.toLowerCase() === cleanUsername);
  if (existingUser) {
    return res.status(400).json({ error: 'Username is already taken.' });
  }
  
  const salt = generateSalt();
  const passwordHash = hashPassword(password, salt);
  
  const newUser = {
    id: 'u_' + Date.now().toString(36) + Math.random().toString(36).substr(2, 5),
    username: cleanUsername,
    salt,
    passwordHash,
    isAdmin: false,
    isRecipeManager: false,
    createdAt: new Date().toISOString()
  };
  
  users.push(newUser);
  writeUsers(users);
  
  res.status(201).json({
    user: {
      id: newUser.id,
      username: newUser.username,
      isAdmin: false,
      isRecipeManager: false
    }
  });
});

// Auth API: User Login
app.post('/api/auth/login', (req, res) => {
  const { username, password } = req.body;
  if (!username || !password) {
    return res.status(400).json({ error: 'Username and password are required.' });
  }
  
  const cleanUsername = username.trim().toLowerCase();
  const users = readUsers();
  const user = users.find(u => u.username.toLowerCase() === cleanUsername);
  
  if (!user) {
    return res.status(401).json({ error: 'Invalid username or password.' });
  }
  
  const hash = hashPassword(password, user.salt);
  if (hash !== user.passwordHash) {
    return res.status(401).json({ error: 'Invalid username or password.' });
  }
  
  const token = crypto.randomBytes(32).toString('hex');
  activeSessions.set(token, { 
    id: user.id, 
    username: user.username, 
    isAdmin: user.isAdmin || false,
    isRecipeManager: user.isRecipeManager || false 
  });
  
  res.json({
    token,
    user: {
      id: user.id,
      username: user.username,
      isAdmin: user.isAdmin || false,
      isRecipeManager: user.isRecipeManager || false
    }
  });
});

// Admin API: List all users (Requires Admin session)
app.get('/api/admin/users', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  const users = readUsers().map(u => ({
    id: u.id,
    username: u.username,
    isAdmin: u.isAdmin || false,
    isRecipeManager: u.isRecipeManager || false,
    createdAt: u.createdAt
  }));
  
  res.json(users);
});

// Admin API: Delete user (Requires Admin session)
app.delete('/api/admin/users/:id', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  if (req.params.id === req.user.id) {
    return res.status(400).json({ error: 'You cannot delete your own administrative account.' });
  }
  
  const users = readUsers();
  const userIndex = users.findIndex(u => u.id === req.params.id);
  
  if (userIndex === -1) {
    return res.status(404).json({ error: 'User not found.' });
  }
  
  const deletedUser = users[userIndex];
  const preserveRecipes = req.body.preserveRecipes === true;
  
  // Handle recipe dependencies
  const recipes = readRecipes();
  let updatedRecipes;
  
  if (preserveRecipes) {
    // Keep in Shared Cookbook (reassign to system_admin, set as public)
    updatedRecipes = recipes.map(r => {
      if (r.userId === deletedUser.id) {
        r.userId = 'system_admin';
        r.createdBy = `${deletedUser.username} (former)`;
        r.isPublic = true;
      }
      return r;
    });
  } else {
    // Purge entirely
    updatedRecipes = recipes.filter(r => r.userId !== deletedUser.id);
  }
  
  writeRecipes(updatedRecipes);
  
  // Remove user profile
  users.splice(userIndex, 1);
  writeUsers(users);
  
  // Invalidate any active session tokens of deleted user
  for (const [token, session] of activeSessions.entries()) {
    if (session.id === deletedUser.id) {
      activeSessions.delete(token);
    }
  }
  
  res.json({ success: true });
});

// Admin API: Change user role (Requires Admin session)
app.put('/api/admin/users/:id/role', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  if (req.params.id === req.user.id) {
    return res.status(400).json({ error: 'You cannot change your own administrator role privileges.' });
  }
  
  const { isAdmin } = req.body;
  if (typeof isAdmin !== 'boolean') {
    return res.status(400).json({ error: 'isAdmin property must be a boolean.' });
  }
  
  const users = readUsers();
  const userIndex = users.findIndex(u => u.id === req.params.id);
  
  if (userIndex === -1) {
    return res.status(404).json({ error: 'User not found.' });
  }
  
  users[userIndex].isAdmin = isAdmin;
  writeUsers(users);
  
  // Update active sessions to reflect the privilege change immediately
  for (const [token, session] of activeSessions.entries()) {
    if (session.id === users[userIndex].id) {
      session.isAdmin = isAdmin;
      activeSessions.set(token, session);
    }
  }
  
  res.json({ success: true, user: { id: users[userIndex].id, username: users[userIndex].username, isAdmin } });
});

// Admin API: Toggle recipe manager role (Requires Admin session)
app.put('/api/admin/users/:id/recipe-manager', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  const { isRecipeManager } = req.body;
  if (typeof isRecipeManager !== 'boolean') {
    return res.status(400).json({ error: 'isRecipeManager property must be a boolean.' });
  }
  
  const users = readUsers();
  const userIndex = users.findIndex(u => u.id === req.params.id);
  
  if (userIndex === -1) {
    return res.status(404).json({ error: 'User not found.' });
  }
  
  users[userIndex].isRecipeManager = isRecipeManager;
  writeUsers(users);
  
  // Update active sessions to reflect the privilege change immediately
  for (const [token, session] of activeSessions.entries()) {
    if (session.id === users[userIndex].id) {
      session.isRecipeManager = isRecipeManager;
      activeSessions.set(token, session);
    }
  }
  
  res.json({ success: true, user: { id: users[userIndex].id, username: users[userIndex].username, isRecipeManager } });
});

// API: List authenticated user's private recipes
app.get('/api/recipes', requireAuth, (req, res) => {
  const recipes = readRecipes();
  const userRecipes = recipes.filter(r => r.userId === req.user.id);
  res.json(userRecipes);
});

// API: List all public recipes for Shared Cookbook
app.get('/api/recipes/public', requireAuth, (req, res) => {
  const recipes = readRecipes();
  const publicRecipes = recipes.filter(r => r.isPublic !== false);
  res.json(publicRecipes);
});

// API: Clone a public recipe to user's private collection
app.post('/api/recipes/clone/:id', requireAuth, (req, res) => {
  const recipes = readRecipes();
  const sourceRecipe = recipes.find(r => r.id === req.params.id);
  
  if (!sourceRecipe) {
    return res.status(404).json({ error: 'Recipe not found' });
  }
  
  if (sourceRecipe.isPublic === false) {
    return res.status(403).json({ error: 'This recipe is private and cannot be cloned.' });
  }
  
  const clonedRecipe = {
    ...sourceRecipe,
    id: Date.now().toString(36) + Math.random().toString(36).substr(2, 5),
    userId: req.user.id,
    createdBy: req.user.username,
    isPublic: false, // Default cloned copies to private so users can customize them privately
    createdAt: new Date().toISOString(),
    updatedAt: undefined
  };
  
  recipes.push(clonedRecipe);
  writeRecipes(recipes);
  res.status(201).json(clonedRecipe);
});

// API: Save new recipe
app.post('/api/recipes', requireAuth, (req, res) => {
  const recipes = readRecipes();
  const newRecipe = {
    id: Date.now().toString(36) + Math.random().toString(36).substr(2, 5),
    userId: req.user.id,
    createdBy: req.user.username,
    isPublic: req.body.isPublic !== false,
    title: req.body.title || 'Untitled Recipe',
    description: req.body.description || '',
    ingredients: Array.isArray(req.body.ingredients) ? req.body.ingredients : [],
    instructions: Array.isArray(req.body.instructions) ? req.body.instructions : [],
    servings: req.body.servings || '',
    prepTime: req.body.prepTime || '',
    cookTime: req.body.cookTime || '',
    totalTime: req.body.totalTime || '',
    image: req.body.image || '',
    images: Array.isArray(req.body.images) ? req.body.images : [],
    sourceUrl: req.body.sourceUrl || '',
    createdAt: new Date().toISOString()
  };
  
  recipes.push(newRecipe);
  writeRecipes(recipes);
  res.status(201).json(newRecipe);
});

// API: Update recipe
app.put('/api/recipes/:id', requireAuth, (req, res) => {
  const recipes = readRecipes();
  const index = recipes.findIndex(r => r.id === req.params.id);
  
  if (index === -1) {
    return res.status(404).json({ error: 'Recipe not found' });
  }
  
  if (recipes[index].userId !== req.user.id && !req.user.isRecipeManager && !req.user.isAdmin) {
    return res.status(403).json({ error: 'You do not have permission to modify this recipe.' });
  }
  
  const updatedRecipe = {
    ...recipes[index],
    title: req.body.title || recipes[index].title,
    description: req.body.description !== undefined ? req.body.description : recipes[index].description,
    ingredients: Array.isArray(req.body.ingredients) ? req.body.ingredients : recipes[index].ingredients,
    instructions: Array.isArray(req.body.instructions) ? req.body.instructions : recipes[index].instructions,
    servings: req.body.servings !== undefined ? req.body.servings : recipes[index].servings,
    prepTime: req.body.prepTime !== undefined ? req.body.prepTime : recipes[index].prepTime,
    cookTime: req.body.cookTime !== undefined ? req.body.cookTime : recipes[index].cookTime,
    totalTime: req.body.totalTime !== undefined ? req.body.totalTime : recipes[index].totalTime,
    image: req.body.image !== undefined ? req.body.image : recipes[index].image,
    images: Array.isArray(req.body.images) ? req.body.images : recipes[index].images,
    sourceUrl: req.body.sourceUrl !== undefined ? req.body.sourceUrl : recipes[index].sourceUrl,
    isPublic: req.body.isPublic !== undefined ? req.body.isPublic : recipes[index].isPublic,
    updatedAt: new Date().toISOString()
  };
  
  recipes[index] = updatedRecipe;
  writeRecipes(recipes);
  res.json(updatedRecipe);
});

// API: Delete recipe
app.delete('/api/recipes/:id', requireAuth, (req, res) => {
  let recipes = readRecipes();
  const index = recipes.findIndex(r => r.id === req.params.id);
  
  if (index === -1) {
    return res.status(404).json({ error: 'Recipe not found' });
  }
  
  if (recipes[index].userId !== req.user.id && !req.user.isRecipeManager && !req.user.isAdmin) {
    return res.status(403).json({ error: 'You do not have permission to delete this recipe.' });
  }
  
  const filtered = recipes.filter(r => r.id !== req.params.id);
  writeRecipes(filtered);
  res.json({ success: true });
});

// Helper: Parse recipe data from raw HTML string
function parseRecipeFromHtml(html, url) {
  const $ = cheerio.load(html);

  // Extract JSON-LD structures
  let schemaRecipe = null;
  $('script[type="application/ld+json"]').each((i, el) => {
    try {
      const text = $(el).html();
      if (!text) return;
      const data = JSON.parse(text);
      
      const searchForRecipe = (obj) => {
        if (!obj || typeof obj !== 'object') return;
        if (Array.isArray(obj)) {
          for (const item of obj) {
            searchForRecipe(item);
            if (schemaRecipe) return;
          }
        } else {
          const type = obj['@type'];
          if (type === 'Recipe' || (Array.isArray(type) && type.includes('Recipe'))) {
            schemaRecipe = obj;
            return;
          }
          for (const key in obj) {
            if (obj.hasOwnProperty(key)) {
              searchForRecipe(obj[key]);
              if (schemaRecipe) return;
            }
          }
        }
      };

      searchForRecipe(data);
    } catch (e) {
      // Safe to ignore malformed JSON script elements
    }
  });

  // Populate structured data object
  let recipeData = {
    title: '',
    description: '',
    ingredients: [],
    instructions: [],
    servings: '',
    prepTime: '',
    cookTime: '',
    totalTime: '',
    images: [],
    sourceUrl: url
  };

  // Scrape candidate image arrays from tags to enable user choice
  const extractedImages = [];
  $('img').each((i, el) => {
    const src = $(el).attr('src') || $(el).attr('data-src') || $(el).attr('data-lazy-src');
    if (src && src.startsWith('http')) {
      const width = $(el).attr('width');
      const height = $(el).attr('height');
      const isSmall = (width && parseInt(width, 10) < 150) || (height && parseInt(height, 10) < 150);
      const containsTrashKeywords = /avatar|logo|icon|spacer|ad-|banner|pixel|tracking/i.test(src);
      
      if (!isSmall && !containsTrashKeywords && !extractedImages.includes(src)) {
        extractedImages.push(src);
      }
    }
  });

  if (schemaRecipe) {
    // Standard schema-based extraction
    recipeData.title = schemaRecipe.name || $('title').text().trim() || '';
    recipeData.description = schemaRecipe.description || '';
    
    if (schemaRecipe.recipeYield) {
      recipeData.servings = Array.isArray(schemaRecipe.recipeYield) 
        ? schemaRecipe.recipeYield[0] 
        : schemaRecipe.recipeYield.toString();
    }
    
    recipeData.prepTime = parseISODuration(schemaRecipe.prepTime || '');
    recipeData.cookTime = parseISODuration(schemaRecipe.cookTime || '');
    recipeData.totalTime = parseISODuration(schemaRecipe.totalTime || '');

    const rawIngredients = schemaRecipe.recipeIngredient || schemaRecipe.recipeIngredients || [];
    if (typeof rawIngredients === 'string') {
      recipeData.ingredients = rawIngredients.split(/\r?\n/).map(i => i.trim()).filter(Boolean);
    } else if (Array.isArray(rawIngredients)) {
      recipeData.ingredients = rawIngredients;
    }

    recipeData.instructions = extractInstructions(schemaRecipe.recipeInstructions || schemaRecipe.recipeInstruction);

    const schemaImages = normalizeImages(schemaRecipe.image);
    schemaImages.forEach(img => {
      if (!extractedImages.includes(img)) {
        extractedImages.unshift(img);
      }
    });
  } else {
    // Fallback Scraping strategies if JSON-LD is missing
    recipeData.title = $('meta[property="og:title"]').attr('content') || $('title').text().trim() || '';
    recipeData.description = $('meta[property="og:description"]').attr('content') || $('meta[name="description"]').attr('content') || '';
    
    const ogImg = $('meta[property="og:image"]').attr('content');
    if (ogImg && !extractedImages.includes(ogImg)) {
      extractedImages.unshift(ogImg);
    }

    // Parse ingredients structurally
    const ingredientSelectors = [
      '.wprm-recipe-ingredient',
      '.tasty-recipes-ingredients li',
      '.tasty-recipe-ingredients li',
      '.recipe-ingredients li',
      '.ingredients-list li',
      '.ingredients li',
      '[class*="ingredients"] li',
      '[class*="ingredients"] p',
      '[class*="ingredient"] li',
      '[class*="ingredient"] p',
      'ul[class*="ingredient"] li',
      'ol[class*="ingredient"] li',
      '.cooked-ingredient',
      '.recipe-ingredients-wrap li'
    ];
    
    for (const selector of ingredientSelectors) {
      const items = $(selector);
      if (items.length > 1) {
        items.each((i, el) => {
          const txt = $(el).text().trim().replace(/\s+/g, ' ');
          if (txt && !recipeData.ingredients.includes(txt)) {
            recipeData.ingredients.push(txt);
          }
        });
        if (recipeData.ingredients.length > 1) break;
      }
    }

    // Parse instructions structurally
    const instructionSelectors = [
      '.wprm-recipe-instruction',
      '.tasty-recipes-instructions li',
      '.tasty-recipe-instructions li',
      '.recipe-directions li',
      '.recipe-instructions li',
      '.instructions-list li',
      '.instructions li',
      '.recipe-steps li',
      '[class*="instructions"] li',
      '[class*="instructions"] p',
      '[class*="directions"] li',
      '[class*="directions"] p',
      '[class*="step"] li',
      '[class*="step"] p',
      '.cooked-step',
      '.recipe-method li',
      '.recipe-method p',
      '.recipe-preparation li'
    ];

    for (const selector of instructionSelectors) {
      const items = $(selector);
      if (items.length > 1) {
        items.each((i, el) => {
          const txt = $(el).text().trim().replace(/\s+/g, ' ');
          if (txt && !recipeData.instructions.includes(txt)) {
            recipeData.instructions.push(txt);
          }
        });
        if (recipeData.instructions.length > 1) break;
      }
    }
  }

  // Heuristics Fallback: Heading Sibling crawl if still empty
  if (recipeData.ingredients.length === 0) {
    $('h1, h2, h3, h4, h5').each((i, heading) => {
      const text = $(heading).text().toLowerCase();
      if (text.includes('ingredients') && !text.includes('instruction') && !text.includes('direction')) {
        let next = $(heading).next();
        if (next.length === 0) {
          next = $(heading).parent().next();
        }
        
        let limit = 0;
        while (next.length > 0 && limit < 15) {
          if (/h[1-6]/i.test(next[0].tagName)) break;
          
          const items = next.find('li, p');
          if (items.length > 0) {
            items.each((j, el) => {
              const itemText = $(el).text().trim().replace(/\s+/g, ' ');
              if (itemText && itemText.length > 2 && itemText.length < 200 && !recipeData.ingredients.includes(itemText)) {
                recipeData.ingredients.push(itemText);
              }
            });
          } else {
            const itemText = next.text().trim().replace(/\s+/g, ' ');
            if (itemText && itemText.length > 2 && itemText.length < 200) {
              recipeData.ingredients.push(itemText);
            }
          }
          next = next.next();
          limit++;
        }
        if (recipeData.ingredients.length > 2) return false;
      }
    });
  }

  if (recipeData.instructions.length === 0) {
    $('h1, h2, h3, h4, h5').each((i, heading) => {
      const text = $(heading).text().toLowerCase();
      if (/instruction|direction|method|preparation|steps/i.test(text) && !text.includes('ingredient')) {
        let next = $(heading).next();
        if (next.length === 0) {
          next = $(heading).parent().next();
        }
        
        let limit = 0;
        while (next.length > 0 && limit < 15) {
          if (/h[1-6]/i.test(next[0].tagName)) break;
          
          const items = next.find('li, p');
          if (items.length > 0) {
            items.each((j, el) => {
              const itemText = $(el).text().trim().replace(/\s+/g, ' ');
              if (itemText && itemText.length > 5 && itemText.length < 1000 && !recipeData.instructions.includes(itemText)) {
                recipeData.instructions.push(itemText);
              }
            });
          } else {
            const itemText = next.text().trim().replace(/\s+/g, ' ');
            if (itemText && itemText.length > 5 && itemText.length < 1000) {
              recipeData.instructions.push(itemText);
            }
          }
          next = next.next();
          limit++;
        }
        if (recipeData.instructions.length > 1) return false;
      }
    });
  }

  // Heuristics Fallback: Blockquote / Multi-paragraph text classifier (for vintage/informal blogs)
  if (recipeData.ingredients.length === 0 && recipeData.instructions.length === 0) {
    const isStepText = (text) => {
      if (/^(?:step\s*)?\d+[\.\\)\-]\s/i.test(text)) return true;
      if (/^\d+\s+(?:mix|stir|combine|sift|bake|heat|preheat|fry|drain|serve|boil|chop|pour|place|add|cook|whisk|beat|divide|mould|brush|roll|repeat|prepare|cut)\b/i.test(text)) return true;
      if (/^(?:mix|stir|combine|sift|bake|heat|preheat|fry|drain|serve|boil|chop|pour|place|add|cook|whisk|beat|divide|mould|brush|roll|repeat|prepare|cut)\b/i.test(text) && text.length > 25) return true;
      return false;
    };

    const isTitleOrHeader = (text) => {
      const titleClean = recipeData.title.toLowerCase();
      const textClean = text.toLowerCase();
      if (titleClean.includes(textClean) || textClean.includes(titleClean)) return true;
      if (/^(ingredients|instructions|directions|method|results|notes|conclusion)\b/i.test(textClean)) return true;
      return false;
    };

    let selectedContainer = null;
    if ($('blockquote').length > 0) {
      selectedContainer = $('blockquote');
    } else if ($('.entry-content').length > 0) {
      selectedContainer = $('.entry-content');
    } else if ($('article').length > 0) {
      selectedContainer = $('article');
    }

    if (selectedContainer) {
      selectedContainer.each((i, container) => {
        const paragraphs = $(container).find('p');
        if (paragraphs.length > 2) {
          paragraphs.each((j, el) => {
            const html = $(el).html() || '';
            if (html.includes('<br>')) {
              const lines = html.split(/<br\s*\/?>/i)
                .map(l => cheerio.load(l).text().trim().replace(/\s+/g, ' '))
                .filter(Boolean);
              
              for (const line of lines) {
                if (isTitleOrHeader(line) || line.length < 3) continue;
                if (isStepText(line)) {
                  recipeData.instructions.push(line);
                } else {
                  recipeData.ingredients.push(line);
                }
              }
            } else {
              const txt = $(el).text().trim().replace(/\s+/g, ' ');
              if (!txt || isTitleOrHeader(txt) || txt.length < 3) return;
              
              if (isStepText(txt)) {
                recipeData.instructions.push(txt);
              } else {
                recipeData.ingredients.push(txt);
              }
            }
          });
          if (recipeData.ingredients.length > 1) return false;
        }
      });
    }
  }

  // Clean up lists
  recipeData.ingredients = recipeData.ingredients.map(i => i.trim()).filter(Boolean);
  recipeData.instructions = recipeData.instructions.map(i => i.trim()).filter(Boolean);
  recipeData.images = extractedImages.slice(0, 15);

  return recipeData;
}

// Helper: Detect if HTML response is a bot challenge page rather than real content
function isBotChallengePage(html) {
  if (!html || typeof html !== 'string') return true;
  const lowerHtml = html.toLowerCase();
  // Cloudflare challenge signatures
  if (lowerHtml.includes('cf-browser-verification') || lowerHtml.includes('cf_chl_opt')) return true;
  if (lowerHtml.includes('checking your browser') && lowerHtml.includes('cloudflare')) return true;
  // Akamai bot manager
  if (lowerHtml.includes('akamai') && lowerHtml.includes('bot manager')) return true;
  // Generic CAPTCHA / challenge
  if (lowerHtml.includes('recaptcha') && !lowerHtml.includes('recipeingredient')) return true;
  // Very short response with no recipe-related content
  if (html.length < 500 && !lowerHtml.includes('recipe')) return true;
  return false;
}

// Helper: Fetch HTML using Puppeteer with stealth (headless browser fallback)
async function fetchWithPuppeteer(url) {
  console.log('[Puppeteer] Launching headless browser for:', url);
  let browser = null;
  try {
    browser = await puppeteer.launch({
      headless: 'new',
      args: [
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
        '--disable-accelerated-2d-canvas',
        '--disable-gpu',
        '--window-size=1920,1080'
      ]
    });

    const page = await browser.newPage();

    // Set a realistic viewport and user agent
    await page.setViewport({ width: 1920, height: 1080 });
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');

    // Set extra headers to appear more like a real browser
    await page.setExtraHTTPHeaders({
      'Accept-Language': 'en-US,en;q=0.9',
      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
    });

    // Navigate and wait for the page to fully load (networkidle0 waits for all requests to finish)
    await page.goto(url, { 
      waitUntil: 'networkidle0', 
      timeout: 30000 
    });

    // Give JS-rendered content a moment to hydrate
    await new Promise(resolve => setTimeout(resolve, 2000));

    // Scroll down to trigger any lazy-loaded content
    await page.evaluate(() => {
      window.scrollTo(0, document.body.scrollHeight / 2);
    });
    await new Promise(resolve => setTimeout(resolve, 1000));

    const html = await page.content();
    console.log('[Puppeteer] Successfully retrieved', html.length, 'bytes of HTML');
    return html;
  } finally {
    if (browser) {
      await browser.close();
      console.log('[Puppeteer] Browser closed');
    }
  }
}

// API: Load System Settings
app.get('/api/settings', requireAuth, (req, res) => {
  const settingsFile = path.join(__dirname, 'data', 'settings.json');
  try {
    if (!fs.existsSync(settingsFile)) {
      fs.writeFileSync(settingsFile, JSON.stringify({ geminiApiKey: '' }, null, 2));
    }
    const data = fs.readFileSync(settingsFile, 'utf8');
    res.json(JSON.parse(data));
  } catch (e) {
    res.status(500).json({ error: 'Failed to read settings file' });
  }
});

// API: Save System Settings
app.post('/api/settings', requireAuth, (req, res) => {
  const settingsFile = path.join(__dirname, 'data', 'settings.json');
  const { geminiApiKey } = req.body;
  try {
    const settings = { geminiApiKey: geminiApiKey || '' };
    fs.writeFileSync(settingsFile, JSON.stringify(settings, null, 2), 'utf8');
    res.json(settings);
  } catch (e) {
    res.status(500).json({ error: 'Failed to save settings file' });
  }
});

// API: Scrape Recipe from Link
app.post('/api/scrape', requireAuth, async (req, res) => {
  const { url } = req.body;
  if (!url) {
    return res.status(400).json({ error: 'URL is required' });
  }

  try {
    let html = null;
    let usedPuppeteer = false;

    // Strategy 1: Fast axios fetch
    try {
      const response = await axios.get(url, {
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
          'Accept-Language': 'en-US,en;q=0.9',
          'Accept-Encoding': 'gzip, deflate, br',
          'Connection': 'keep-alive',
          'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
          'sec-ch-ua-mobile': '?0',
          'sec-ch-ua-platform': '"Windows"',
          'Sec-Fetch-Dest': 'document',
          'Sec-Fetch-Mode': 'navigate',
          'Sec-Fetch-Site': 'none',
          'Sec-Fetch-User': '?1',
          'Upgrade-Insecure-Requests': '1'
        },
        timeout: 10000
      });
      html = response.data;

      // Check if we got a bot challenge instead of real content
      if (isBotChallengePage(html)) {
        console.log('[Scraper] Axios response appears to be a bot challenge page, falling back to Puppeteer...');
        html = null; // Force Puppeteer fallback
      }
    } catch (axiosError) {
      const status = axiosError.response ? axiosError.response.status : 0;
      if (status === 403 || status === 503 || status === 429) {
        console.log(`[Scraper] Axios blocked with HTTP ${status}, falling back to Puppeteer...`);
        // Will fall through to Puppeteer below
      } else {
        throw axiosError; // Re-throw non-blocking errors (DNS failures, timeouts, etc.)
      }
    }

    // Strategy 2: Puppeteer with stealth (headless browser fallback)
    if (!html) {
      try {
        html = await fetchWithPuppeteer(url);
        usedPuppeteer = true;
      } catch (puppeteerError) {
        console.error('[Puppeteer] Fallback also failed:', puppeteerError.message);
        return res.status(500).json({
          error: 'This website could not be accessed. Both standard and browser-based scraping methods failed.',
          details: puppeteerError.message,
          isBlocked: true
        });
      }
    }

    // Parse the retrieved HTML into recipe data
    const recipeData = parseRecipeFromHtml(html, url);

    // Include metadata about which method succeeded
    recipeData._scrapingMethod = usedPuppeteer ? 'puppeteer-stealth' : 'axios';

    res.json(recipeData);
  } catch (error) {
    console.error('Error occurred while scraping URL:', error.message);
    const status = error.response ? error.response.status : 500;
    const message = error.response && error.response.status === 403
      ? 'This website is protected and blocks automated access. The Puppeteer fallback was also unavailable.'
      : error.message;
    res.status(status).json({ 
      error: message, 
      isBlocked: status === 403 || status === 503 
    });
  }
});

// Launch server
app.listen(PORT, () => {
  console.log(`Recipe Manager server is active at http://localhost:${PORT}`);
});