const express = require('express');
const axios = require('axios');
const cheerio = require('cheerio');
const cors = require('cors');
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
const Database = require('better-sqlite3');
const rateLimit = require('express-rate-limit');

// Enable stealth mode to bypass bot detection (Cloudflare, Akamai, etc.)
puppeteer.use(StealthPlugin());

const app = express();
const PORT = process.env.PORT || 3000;

// Setup directories for persistence
const DATA_DIR = path.join(__dirname, 'data');
const DB_FILE = path.join(DATA_DIR, 'forkfolio.db');

if (!fs.existsSync(DATA_DIR)) {
  fs.mkdirSync(DATA_DIR, { recursive: true });
}

// Open SQLite Database connection and set WAL journal mode for concurrency speed
const db = new Database(DB_FILE);
db.pragma('journal_mode = WAL');

// Initialize database schema tables
db.prepare(`
  CREATE TABLE IF NOT EXISTS users (
    id TEXT PRIMARY KEY,
    username TEXT UNIQUE NOT NULL,
    salt TEXT NOT NULL,
    passwordHash TEXT NOT NULL,
    isAdmin INTEGER NOT NULL DEFAULT 0,
    isRecipeManager INTEGER NOT NULL DEFAULT 0,
    createdAt TEXT NOT NULL
  )
`).run();

db.prepare(`
  CREATE TABLE IF NOT EXISTS recipes (
    id TEXT PRIMARY KEY,
    userId TEXT NOT NULL,
    createdBy TEXT NOT NULL,
    isPublic INTEGER NOT NULL DEFAULT 1,
    title TEXT NOT NULL,
    description TEXT,
    servings TEXT,
    prepTime TEXT,
    cookTime TEXT,
    totalTime TEXT,
    image TEXT,
    images TEXT,       -- JSON stringified array
    ingredients TEXT,  -- JSON stringified array
    instructions TEXT, -- JSON stringified array
    sourceUrl TEXT,
    shareToken TEXT UNIQUE, -- Unguessable sharing token
    createdAt TEXT NOT NULL,
    updatedAt TEXT,
    FOREIGN KEY (userId) REFERENCES users (id) ON DELETE CASCADE
  )
`).run();

db.prepare(`
  CREATE TABLE IF NOT EXISTS settings (
    geminiApiKey TEXT DEFAULT ''
  )
`).run();

db.prepare(`
  CREATE TABLE IF NOT EXISTS recipe_shares_visits (
    shareToken TEXT NOT NULL,
    visitorIp TEXT NOT NULL,
    visitedAt TEXT NOT NULL,
    PRIMARY KEY (shareToken, visitorIp)
  )
`).run();

// Initialize single settings record if empty
const settingsCount = db.prepare('SELECT COUNT(*) as count FROM settings').get().count;
if (settingsCount === 0) {
  db.prepare('INSERT INTO settings (geminiApiKey) VALUES (?)').run('');
}

// Ensure default system_admin and admin user profiles exist
try {
  // Ensure default system_admin user exists to fulfill recipe foreign key constraints
  const hasSystemAdmin = db.prepare("SELECT COUNT(*) as count FROM users WHERE id = 'system_admin'").get().count;
  if (hasSystemAdmin === 0) {
    db.prepare(`
      INSERT INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, createdAt)
      VALUES (?, ?, ?, ?, ?, ?, ?)
    `).run('system_admin', 'system_admin', 'system_salt', 'system_hash', 0, 0, new Date().toISOString());
  }

  const adminCount = db.prepare("SELECT COUNT(*) as count FROM users WHERE isAdmin = 1").get().count;
  if (adminCount === 0) {
    const salt = crypto.randomBytes(16).toString('hex');
    const passwordHash = crypto.pbkdf2Sync('admin123', salt, 1000, 64, 'sha512').toString('hex');
    db.prepare(`
      INSERT INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, createdAt)
      VALUES (?, ?, ?, ?, ?, ?, ?)
    `).run(
      'u_admin',
      'admin',
      salt,
      passwordHash,
      1,
      1,
      new Date().toISOString()
    );
    console.log('--- ForkFolio Admin Initialized ---');
    console.log('Default Admin Username: admin');
    console.log('Default Admin Password: admin123');
    console.log('--------------------------------------');
  }
} catch (e) {
  console.error('Failed to initialize default admin and system users:', e);
}

// Automatic Migration from legacy JSON databases if they exist
const LEGACY_USERS_FILE = path.join(DATA_DIR, 'users.json');
const LEGACY_RECIPES_FILE = path.join(DATA_DIR, 'recipes.json');
const LEGACY_SETTINGS_FILE = path.join(DATA_DIR, 'settings.json');

try {
  // Temporarily disable foreign keys during bulk migrations to allow flexible insert order
  db.pragma('foreign_keys = OFF');

  // 1. Migrate Users
  if (fs.existsSync(LEGACY_USERS_FILE)) {
    const backupUsersFile = path.join(DATA_DIR, 'users.json.backup');
    const data = fs.readFileSync(LEGACY_USERS_FILE, 'utf8');
    const users = JSON.parse(data);
    if (Array.isArray(users) && users.length > 0) {
      console.log(`[Migration] Found ${users.length} legacy users. Migrating to SQLite...`);
      const insertUser = db.prepare(`
        INSERT OR IGNORE INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, createdAt)
        VALUES (?, ?, ?, ?, ?, ?, ?)
      `);
      
      const transaction = db.transaction((userList) => {
        for (const u of userList) {
          insertUser.run(
            u.id,
            u.username.trim().toLowerCase(),
            u.salt,
            u.passwordHash,
            u.isAdmin ? 1 : 0,
            u.isRecipeManager ? 1 : 0,
            u.createdAt || new Date().toISOString()
          );
        }
      });
      transaction(users);
      console.log(`[Migration] User migration completed.`);
    }
    fs.renameSync(LEGACY_USERS_FILE, backupUsersFile);
  }

  // 2. Migrate Recipes
  if (fs.existsSync(LEGACY_RECIPES_FILE)) {
    const backupRecipesFile = path.join(DATA_DIR, 'recipes.json.backup');
    const data = fs.readFileSync(LEGACY_RECIPES_FILE, 'utf8');
    const recipes = JSON.parse(data);
    if (Array.isArray(recipes) && recipes.length > 0) {
      console.log(`[Migration] Found ${recipes.length} legacy recipes. Migrating to SQLite...`);
      const insertRecipe = db.prepare(`
        INSERT OR IGNORE INTO recipes (
          id, userId, createdBy, isPublic, title, description, servings,
          prepTime, cookTime, totalTime, image, images, ingredients, instructions, sourceUrl, shareToken, createdAt, updatedAt
        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
      `);
      
      const transaction = db.transaction((recipeList) => {
        for (const r of recipeList) {
          const shareToken = r.isPublic !== false && !r.shareToken 
            ? crypto.randomBytes(16).toString('hex') 
            : (r.shareToken || null);
            
          insertRecipe.run(
            r.id,
            r.userId || 'system_admin',
            r.createdBy || 'gourmet_system',
            r.isPublic !== false ? 1 : 0,
            r.title || 'Untitled Recipe',
            r.description || '',
            r.servings || '',
            r.prepTime || '',
            r.cookTime || '',
            r.totalTime || '',
            r.image || '',
            JSON.stringify(r.images || []),
            JSON.stringify(r.ingredients || []),
            JSON.stringify(r.instructions || []),
            r.sourceUrl || '',
            shareToken,
            r.createdAt || new Date().toISOString(),
            r.updatedAt || null
          );
        }
      });
      transaction(recipes);
      console.log(`[Migration] Recipe migration completed.`);
    }
    fs.renameSync(LEGACY_RECIPES_FILE, backupRecipesFile);
  }

  // 3. Migrate Settings
  if (fs.existsSync(LEGACY_SETTINGS_FILE)) {
    const backupSettingsFile = path.join(DATA_DIR, 'settings.json.backup');
    const data = fs.readFileSync(LEGACY_SETTINGS_FILE, 'utf8');
    const settings = JSON.parse(data);
    if (settings && settings.geminiApiKey) {
      console.log('[Migration] Migrating system settings key to SQLite...');
      db.prepare('UPDATE settings SET geminiApiKey = ?').run(settings.geminiApiKey.trim());
      console.log('[Migration] Settings migration completed.');
    }
    fs.renameSync(LEGACY_SETTINGS_FILE, backupSettingsFile);
  }

  // Restore foreign keys enforcement after migrations
  db.pragma('foreign_keys = ON');
} catch (migrationError) {
  // Guarantee foreign keys re-enabled even on crash
  db.pragma('foreign_keys = ON');
  console.error('[Migration] Failed to perform automatic SQLite migration:', migrationError);
}

// In-memory sessions storage (token -> { id, username, isAdmin, isRecipeManager })
const activeSessions = new Map();

app.use(cors());
app.use(express.json({ limit: '10mb' }));
app.use(express.urlencoded({ limit: '10mb', extended: true }));
app.use(express.static(path.join(__dirname, 'public')));

// Configure rate-limiting middleware for public shared routes
const publicShareLimiter = rateLimit({
  windowMs: 15 * 60 * 1000, // 15 minutes
  max: 100, // Limit each IP to 100 requests per 15-minute window
  message: { error: 'Too many queries from this IP. Please try again in 15 minutes.' }
});

// Helper: Parse ISO 8601 Durations (e.g. PT1H30M to "1 hr 30 mins")
function parseISODuration(duration) {
  if (!duration || typeof duration !== 'string') return '';
  const match = duration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/);
  if (!match) return duration;
  const hours = match[1] ? parseInt(match[1], 10) : 0;
  const minutes = match[2] ? parseInt(match[2], 10) : 0;
  
  let result = '';
  if (hours > 0) result += `${hours} hr${hours > 1 ? 's' : ''} `;
  if (minutes > 0) result += `${minutes} min${minutes > 1 ? 's' : ''}`;
  return result.trim() || duration;
}

// Helper: Extract recipe instructions from schema format
function extractInstructions(instructions) {
  if (!instructions) return [];
  if (typeof instructions === 'string') {
    return instructions.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
  }
  if (Array.isArray(instructions)) {
    const result = [];
    for (const inst of instructions) {
      if (typeof inst === 'string') {
        result.push(inst.trim());
      } else if (inst && typeof inst === 'object') {
        if (inst['@type'] === 'HowToStep') {
          result.push(inst.text || inst.name || '');
        } else if (inst['@type'] === 'HowToSection' && Array.isArray(inst.itemListElement)) {
          if (inst.name) {
            result.push(`### ${inst.name}`); // Prefix section names
          }
          for (const step of inst.itemListElement) {
            if (typeof step === 'string') {
              result.push(step.trim());
            } else if (step && typeof step === 'object') {
              result.push(step.text || step.name || '');
            }
          }
        } else if (inst.text) {
          result.push(inst.text);
        } else if (inst.name) {
          result.push(inst.name);
        }
      }
    }
    return result.filter(Boolean);
  }
  return [];
}

// Helper: Normalize image resources from JSON-LD schema
function normalizeImages(imageObj) {
  if (!imageObj) return [];
  if (typeof imageObj === 'string') return [imageObj];
  if (Array.isArray(imageObj)) {
    return imageObj.map(img => (typeof img === 'string' ? img : img.url)).filter(Boolean);
  }
  if (typeof imageObj === 'object' && imageObj.url) {
    return [imageObj.url];
  }
  return [];
}

// Password Hashing and Salt helpers
function generateSalt() {
  return crypto.randomBytes(16).toString('hex');
}

function hashPassword(password, salt) {
  return crypto.pbkdf2Sync(password, salt, 1000, 64, 'sha512').toString('hex');
}

// Middleware: Authenticate Session Token
function requireAuth(req, res, next) {
  const authHeader = req.headers.authorization;
  if (!authHeader || !authHeader.startsWith('Bearer ')) {
    return res.status(401).json({ error: 'Authorization required. Please log in.' });
  }
  const token = authHeader.substring(7);
  const session = activeSessions.get(token);
  if (!session) {
    return res.status(401).json({ error: 'Session expired. Please log in again.' });
  }
  req.user = session;
  next();
}

// Helper: Deserialize JSON columns from SQLite recipe row
function formatRecipe(row) {
  if (!row) return null;
  
  let visitorCount = 0;
  if (row.shareToken) {
    try {
      const res = db.prepare('SELECT COUNT(*) as count FROM recipe_shares_visits WHERE shareToken = ?').get(row.shareToken);
      visitorCount = res ? res.count : 0;
    } catch (e) {
      console.error('Failed to query unique visitor count:', e);
    }
  }

  return {
    ...row,
    isPublic: row.isPublic === 1,
    images: JSON.parse(row.images || '[]'),
    ingredients: JSON.parse(row.ingredients || '[]'),
    instructions: JSON.parse(row.instructions || '[]'),
    visitorCount
  };
}

// Auth API: User Registration (Requires Admin session)
app.post('/api/auth/register', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }

  const { username, password } = req.body;
  if (!username || !password) {
    return res.status(400).json({ error: 'Username and password are required.' });
  }
  
  const cleanUsername = username.trim().toLowerCase();
  if (!/^[a-z0-9_]{3,20}$/.test(cleanUsername)) {
    return res.status(400).json({ error: 'Username must be 3-20 characters long and contain only alphanumeric characters or underscores.' });
  }
  
  if (password.length < 6) {
    return res.status(400).json({ error: 'Password must be at least 6 characters long.' });
  }
  
  try {
    const existing = db.prepare('SELECT id FROM users WHERE username = ?').get(cleanUsername);
    if (existing) {
      return res.status(400).json({ error: 'Username is already taken.' });
    }
    
    const salt = generateSalt();
    const passwordHash = hashPassword(password, salt);
    const id = 'u_' + Date.now().toString(36) + Math.random().toString(36).substr(2, 5);
    
    db.prepare(`
      INSERT INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, createdAt)
      VALUES (?, ?, ?, ?, ?, ?, ?)
    `).run(id, cleanUsername, salt, passwordHash, 0, 0, new Date().toISOString());
    
    res.status(201).json({
      user: {
        id,
        username: cleanUsername,
        isAdmin: false,
        isRecipeManager: false
      }
    });
  } catch (error) {
    console.error('Error during registration:', error);
    res.status(500).json({ error: 'Failed to create account.' });
  }
});

// Auth API: User Login
app.post('/api/auth/login', (req, res) => {
  const { username, password } = req.body;
  if (!username || !password) {
    return res.status(400).json({ error: 'Username and password are required.' });
  }
  
  const cleanUsername = username.trim().toLowerCase();
  try {
    const user = db.prepare('SELECT * FROM users WHERE username = ?').get(cleanUsername);
    if (!user) {
      return res.status(401).json({ error: 'Invalid username or password.' });
    }
    
    const hash = hashPassword(password, user.salt);
    if (hash !== user.passwordHash) {
      return res.status(401).json({ error: 'Invalid username or password.' });
    }
    
    const token = crypto.randomBytes(32).toString('hex');
    activeSessions.set(token, { 
      id: user.id, 
      username: user.username, 
      isAdmin: user.isAdmin === 1,
      isRecipeManager: user.isRecipeManager === 1 
    });
    
    res.json({
      token,
      user: {
        id: user.id,
        username: user.username,
        isAdmin: user.isAdmin === 1,
        isRecipeManager: user.isRecipeManager === 1
      }
    });
  } catch (error) {
    console.error('Error during login:', error);
    res.status(500).json({ error: 'Login failed.' });
  }
});

// Admin API: List all users (Requires Admin session)
app.get('/api/admin/users', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  try {
    const rows = db.prepare('SELECT id, username, isAdmin, isRecipeManager, createdAt FROM users').all();
    const users = rows.map(u => ({
      id: u.id,
      username: u.username,
      isAdmin: u.isAdmin === 1,
      isRecipeManager: u.isRecipeManager === 1,
      createdAt: u.createdAt
    }));
    res.json(users);
  } catch (error) {
    console.error('Error listing users:', error);
    res.status(500).json({ error: 'Failed to retrieve registered users.' });
  }
});

// Admin API: Delete user (Requires Admin session)
app.delete('/api/admin/users/:id', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  if (req.params.id === req.user.id) {
    return res.status(400).json({ error: 'You cannot delete your own administrative account.' });
  }
  
  try {
    const user = db.prepare('SELECT id, username FROM users WHERE id = ?').get(req.params.id);
    if (!user) {
      return res.status(404).json({ error: 'User not found.' });
    }
    
    const preserveRecipes = req.body.preserveRecipes === true;
    
    // SQLite transaction for relational database integrity
    const performDelete = db.transaction(() => {
      if (preserveRecipes) {
        // Reassign user's recipes to system_admin, set as public
        db.prepare(`
          UPDATE recipes
          SET userId = 'system_admin',
              createdBy = ?,
              isPublic = 1
          WHERE userId = ?
        `).run(`${user.username} (former)`, user.id);
      } else {
        // Delete user's recipes
        db.prepare('DELETE FROM recipes WHERE userId = ?').run(user.id);
      }
      
      // Delete user profile
      db.prepare('DELETE FROM users WHERE id = ?').run(user.id);
    });
    
    performDelete();
    
    // Invalidate active sessions
    for (const [token, session] of activeSessions.entries()) {
      if (session.id === user.id) {
        activeSessions.delete(token);
      }
    }
    
    res.json({ success: true });
  } catch (error) {
    console.error('Error deleting user:', error);
    res.status(500).json({ error: 'Failed to delete user.' });
  }
});

// Admin API: Change user role (Requires Admin session)
app.put('/api/admin/users/:id/role', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  if (req.params.id === req.user.id) {
    return res.status(400).json({ error: 'You cannot change your own administrator role privileges.' });
  }
  
  const { isAdmin } = req.body;
  if (typeof isAdmin !== 'boolean') {
    return res.status(400).json({ error: 'isAdmin property must be a boolean.' });
  }
  
  try {
    const user = db.prepare('SELECT id, username FROM users WHERE id = ?').get(req.params.id);
    if (!user) {
      return res.status(404).json({ error: 'User not found.' });
    }
    
    db.prepare('UPDATE users SET isAdmin = ? WHERE id = ?').run(isAdmin ? 1 : 0, user.id);
    
    // Update sessions
    for (const [token, session] of activeSessions.entries()) {
      if (session.id === user.id) {
        session.isAdmin = isAdmin;
        activeSessions.set(token, session);
      }
    }
    
    res.json({ success: true, user: { id: user.id, username: user.username, isAdmin } });
  } catch (error) {
    console.error('Error changing user role:', error);
    res.status(500).json({ error: 'Failed to update user role.' });
  }
});

// Admin API: Toggle recipe manager role (Requires Admin session)
app.put('/api/admin/users/:id/recipe-manager', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  const { isRecipeManager } = req.body;
  if (typeof isRecipeManager !== 'boolean') {
    return res.status(400).json({ error: 'isRecipeManager property must be a boolean.' });
  }
  
  try {
    const user = db.prepare('SELECT id, username FROM users WHERE id = ?').get(req.params.id);
    if (!user) {
      return res.status(404).json({ error: 'User not found.' });
    }
    
    db.prepare('UPDATE users SET isRecipeManager = ? WHERE id = ?').run(isRecipeManager ? 1 : 0, user.id);
    
    // Update sessions
    for (const [token, session] of activeSessions.entries()) {
      if (session.id === user.id) {
        session.isRecipeManager = isRecipeManager;
        activeSessions.set(token, session);
      }
    }
    
    res.json({ success: true, user: { id: user.id, username: user.username, isRecipeManager } });
  } catch (error) {
    console.error('Error toggling manager role:', error);
    res.status(500).json({ error: 'Failed to update manager permission.' });
  }
});

// Admin API: List all public recipe shares and visitor stats (Requires Admin session)
app.get('/api/admin/shared-recipes', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  try {
    const rows = db.prepare(`
      SELECT id, title, createdBy, shareToken, createdAt
      FROM recipes
      WHERE isPublic = 1 AND shareToken IS NOT NULL
      ORDER BY createdAt DESC
    `).all();
    
    const sharedRecipes = rows.map(r => {
      let visitorCount = 0;
      try {
        const stats = db.prepare('SELECT COUNT(*) as count FROM recipe_shares_visits WHERE shareToken = ?').get(r.shareToken);
        visitorCount = stats ? stats.count : 0;
      } catch (err) {
        console.error('Failed to query shared recipe visitor count:', err);
      }
      return {
        id: r.id,
        title: r.title,
        createdBy: r.createdBy,
        shareToken: r.shareToken,
        createdAt: r.createdAt,
        visitorCount
      };
    });
    
    res.json(sharedRecipes);
  } catch (error) {
    console.error('Error fetching admin shared recipes:', error);
    res.status(500).json({ error: 'Failed to retrieve shared recipes.' });
  }
});

// API: List authenticated user's private recipes
app.get('/api/recipes', requireAuth, (req, res) => {
  try {
    const rows = db.prepare('SELECT * FROM recipes WHERE userId = ? ORDER BY createdAt DESC').all(req.user.id);
    const userRecipes = rows.map(formatRecipe);
    res.json(userRecipes);
  } catch (error) {
    console.error('Error fetching recipes:', error);
    res.status(500).json({ error: 'Failed to retrieve recipes.' });
  }
});

// API: List all public recipes for Shared Cookbook
app.get('/api/recipes/public', requireAuth, (req, res) => {
  try {
    const rows = db.prepare('SELECT * FROM recipes WHERE isPublic = 1 ORDER BY createdAt DESC').all();
    const publicRecipes = rows.map(formatRecipe);
    res.json(publicRecipes);
  } catch (error) {
    console.error('Error fetching public recipes:', error);
    res.status(500).json({ error: 'Failed to retrieve community recipes.' });
  }
});

// API: Clone a public recipe to user's private collection
app.post('/api/recipes/clone/:id', requireAuth, (req, res) => {
  try {
    const sourceRecipe = db.prepare('SELECT * FROM recipes WHERE id = ?').get(req.params.id);
    
    if (!sourceRecipe) {
      return res.status(404).json({ error: 'Recipe not found' });
    }
    
    if (sourceRecipe.isPublic === 0) {
      return res.status(403).json({ error: 'This recipe is private and cannot be cloned.' });
    }
    
    const clonedId = Date.now().toString(36) + Math.random().toString(36).substr(2, 5);
    
    db.prepare(`
      INSERT INTO recipes (
        id, userId, createdBy, isPublic, title, description, servings,
        prepTime, cookTime, totalTime, image, images, ingredients, instructions, sourceUrl, shareToken, createdAt, updatedAt
      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `).run(
      clonedId,
      req.user.id,
      req.user.username,
      0, // Default cloned copies to private so users can customize them privately
      sourceRecipe.title || 'Untitled Recipe',
      sourceRecipe.description || '',
      sourceRecipe.servings || '',
      sourceRecipe.prepTime || '',
      sourceRecipe.cookTime || '',
      sourceRecipe.totalTime || '',
      sourceRecipe.image || '',
      sourceRecipe.images || '[]',
      sourceRecipe.ingredients || '[]',
      sourceRecipe.instructions || '[]',
      sourceRecipe.sourceUrl || '',
      null, // Cloned copies start as private, no share token
      new Date().toISOString(),
      null
    );
    
    const clonedObj = formatRecipe(db.prepare('SELECT * FROM recipes WHERE id = ?').get(clonedId));
    res.status(201).json(clonedObj);
  } catch (error) {
    console.error('Error cloning recipe:', error);
    res.status(500).json({ error: 'Failed to clone recipe.' });
  }
});

// API: Save new recipe
app.post('/api/recipes', requireAuth, (req, res) => {
  const isPublic = req.body.isPublic !== false ? 1 : 0;
  const shareToken = isPublic === 1 ? crypto.randomBytes(16).toString('hex') : null;
  const newId = Date.now().toString(36) + Math.random().toString(36).substr(2, 5);
  
  try {
    db.prepare(`
      INSERT INTO recipes (
        id, userId, createdBy, isPublic, title, description, servings,
        prepTime, cookTime, totalTime, image, images, ingredients, instructions, sourceUrl, shareToken, createdAt, updatedAt
      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `).run(
      newId,
      req.user.id,
      req.user.username,
      isPublic,
      req.body.title || 'Untitled Recipe',
      req.body.description || '',
      req.body.servings || '',
      req.body.prepTime || '',
      req.body.cookTime || '',
      req.body.totalTime || '',
      req.body.image || '',
      JSON.stringify(Array.isArray(req.body.images) ? req.body.images : []),
      JSON.stringify(Array.isArray(req.body.ingredients) ? req.body.ingredients : []),
      JSON.stringify(Array.isArray(req.body.instructions) ? req.body.instructions : []),
      req.body.sourceUrl || '',
      shareToken,
      new Date().toISOString(),
      null
    );
    
    const savedObj = formatRecipe(db.prepare('SELECT * FROM recipes WHERE id = ?').get(newId));
    res.status(201).json(savedObj);
  } catch (error) {
    console.error('Error saving new recipe:', error);
    res.status(500).json({ error: 'Failed to save recipe.' });
  }
});

// API: Update recipe
app.put('/api/recipes/:id', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT * FROM recipes WHERE id = ?').get(req.params.id);
    
    if (!existing) {
      return res.status(404).json({ error: 'Recipe not found' });
    }
    
    if (existing.userId !== req.user.id && !req.user.isRecipeManager && !req.user.isAdmin) {
      return res.status(403).json({ error: 'You do not have permission to modify this recipe.' });
    }
    
    const isPublic = req.body.isPublic !== undefined ? (req.body.isPublic ? 1 : 0) : existing.isPublic;
    
    // Generate shareToken if it becomes public and doesn't have one
    let shareToken = existing.shareToken;
    if (isPublic === 1 && !shareToken) {
      shareToken = crypto.randomBytes(16).toString('hex');
    } else if (isPublic === 0) {
      shareToken = null; // Revoke shareToken if it is made private
    }
    
    db.prepare(`
      UPDATE recipes
      SET title = ?,
          description = ?,
          ingredients = ?,
          instructions = ?,
          servings = ?,
          prepTime = ?,
          cookTime = ?,
          totalTime = ?,
          image = ?,
          images = ?,
          sourceUrl = ?,
          isPublic = ?,
          shareToken = ?,
          updatedAt = ?
      WHERE id = ?
    `).run(
      req.body.title || existing.title,
      req.body.description !== undefined ? req.body.description : existing.description,
      Array.isArray(req.body.ingredients) ? JSON.stringify(req.body.ingredients) : existing.ingredients,
      Array.isArray(req.body.instructions) ? JSON.stringify(req.body.instructions) : existing.instructions,
      req.body.servings !== undefined ? req.body.servings : existing.servings,
      req.body.prepTime !== undefined ? req.body.prepTime : existing.prepTime,
      req.body.cookTime !== undefined ? req.body.cookTime : existing.cookTime,
      req.body.totalTime !== undefined ? req.body.totalTime : existing.totalTime,
      req.body.image !== undefined ? req.body.image : existing.image,
      Array.isArray(req.body.images) ? JSON.stringify(req.body.images) : existing.images,
      req.body.sourceUrl !== undefined ? req.body.sourceUrl : existing.sourceUrl,
      isPublic,
      shareToken,
      new Date().toISOString(),
      req.params.id
    );
    
    const updatedObj = formatRecipe(db.prepare('SELECT * FROM recipes WHERE id = ?').get(req.params.id));
    res.json(updatedObj);
  } catch (error) {
    console.error('Error updating recipe:', error);
    res.status(500).json({ error: 'Failed to update recipe.' });
  }
});

// API: Delete recipe
app.delete('/api/recipes/:id', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT userId FROM recipes WHERE id = ?').get(req.params.id);
    
    if (!existing) {
      return res.status(404).json({ error: 'Recipe not found' });
    }
    
    if (existing.userId !== req.user.id && !req.user.isRecipeManager && !req.user.isAdmin) {
      return res.status(403).json({ error: 'You do not have permission to delete this recipe.' });
    }
    
    db.prepare('DELETE FROM recipes WHERE id = ?').run(req.params.id);
    res.json({ success: true });
  } catch (error) {
    console.error('Error deleting recipe:', error);
    res.status(500).json({ error: 'Failed to delete recipe.' });
  }
});

// Helper: Parse recipe data from raw HTML string
function parseRecipeFromHtml(html, url) {
  const $ = cheerio.load(html);

  // Extract JSON-LD structures
  let schemaRecipe = null;
  $('script[type="application/ld+json"]').each((i, el) => {
    try {
      const text = $(el).html();
      if (!text) return;
      const data = JSON.parse(text);
      
      const searchForRecipe = (obj) => {
        if (!obj || typeof obj !== 'object') return;
        if (Array.isArray(obj)) {
          for (const item of obj) {
            searchForRecipe(item);
            if (schemaRecipe) return;
          }
        } else {
          const type = obj['@type'];
          if (type === 'Recipe' || (Array.isArray(type) && type.includes('Recipe'))) {
            schemaRecipe = obj;
            return;
          }
          for (const key in obj) {
            if (obj.hasOwnProperty(key)) {
              searchForRecipe(obj[key]);
              if (schemaRecipe) return;
            }
          }
        }
      };

      searchForRecipe(data);
    } catch (e) {
      // Safe to ignore malformed JSON script elements
    }
  });

  // Populate structured data object
  let recipeData = {
    title: '',
    description: '',
    ingredients: [],
    instructions: [],
    servings: '',
    prepTime: '',
    cookTime: '',
    totalTime: '',
    images: [],
    sourceUrl: url
  };

  // Scrape candidate image arrays from tags to enable user choice
  const extractedImages = [];
  $('img').each((i, el) => {
    const src = $(el).attr('src') || $(el).attr('data-src') || $(el).attr('data-lazy-src');
    if (src && src.startsWith('http')) {
      const width = $(el).attr('width');
      const height = $(el).attr('height');
      const isSmall = (width && parseInt(width, 10) < 150) || (height && parseInt(height, 10) < 150);
      const containsTrashKeywords = /avatar|logo|icon|spacer|ad-|banner|pixel|tracking/i.test(src);
      
      if (!isSmall && !containsTrashKeywords && !extractedImages.includes(src)) {
        extractedImages.push(src);
      }
    }
  });

  if (schemaRecipe) {
    // Standard schema-based extraction
    recipeData.title = schemaRecipe.name || $('title').text().trim() || '';
    recipeData.description = schemaRecipe.description || '';
    
    if (schemaRecipe.recipeYield) {
      recipeData.servings = Array.isArray(schemaRecipe.recipeYield) 
        ? schemaRecipe.recipeYield[0] 
        : schemaRecipe.recipeYield.toString();
    }
    
    recipeData.prepTime = parseISODuration(schemaRecipe.prepTime || '');
    recipeData.cookTime = parseISODuration(schemaRecipe.cookTime || '');
    recipeData.totalTime = parseISODuration(schemaRecipe.totalTime || '');

    const rawIngredients = schemaRecipe.recipeIngredient || schemaRecipe.recipeIngredients || [];
    if (typeof rawIngredients === 'string') {
      recipeData.ingredients = rawIngredients.split(/\r?\n/).map(i => i.trim()).filter(Boolean);
    } else if (Array.isArray(rawIngredients)) {
      recipeData.ingredients = rawIngredients;
    }

    recipeData.instructions = extractInstructions(schemaRecipe.recipeInstructions || schemaRecipe.recipeInstruction);

    const schemaImages = normalizeImages(schemaRecipe.image);
    schemaImages.forEach(img => {
      if (!extractedImages.includes(img)) {
        extractedImages.unshift(img);
      }
    });
  } else {
    // Fallback Scraping strategies if JSON-LD is missing
    recipeData.title = $('meta[property="og:title"]').attr('content') || $('title').text().trim() || '';
    recipeData.description = $('meta[property="og:description"]').attr('content') || $('meta[name="description"]').attr('content') || '';
    
    const ogImg = $('meta[property="og:image"]').attr('content');
    if (ogImg && !extractedImages.includes(ogImg)) {
      extractedImages.unshift(ogImg);
    }

    // Parse ingredients structurally
    const ingredientSelectors = [
      '.wprm-recipe-ingredient',
      '.tasty-recipes-ingredients li',
      '.tasty-recipe-ingredients li',
      '.recipe-ingredients li',
      '.ingredients-list li',
      '.ingredients li',
      '[class*="ingredients"] li',
      '[class*="ingredients"] p',
      '[class*="ingredient"] li',
      '[class*="ingredient"] p',
      'ul[class*="ingredient"] li',
      'ol[class*="ingredient"] li',
      '.cooked-ingredient',
      '.recipe-ingredients-wrap li'
    ];
    
    for (const selector of ingredientSelectors) {
      const items = $(selector);
      if (items.length > 1) {
        items.each((i, el) => {
          const txt = $(el).text().trim().replace(/\s+/g, ' ');
          if (txt && !recipeData.ingredients.includes(txt)) {
            recipeData.ingredients.push(txt);
          }
        });
        if (recipeData.ingredients.length > 1) break;
      }
    }

    // Parse instructions structurally
    const instructionSelectors = [
      '.wprm-recipe-instruction',
      '.tasty-recipes-instructions li',
      '.tasty-recipe-instructions li',
      '.recipe-directions li',
      '.recipe-instructions li',
      '.instructions-list li',
      '.instructions li',
      '.recipe-steps li',
      '[class*="instructions"] li',
      '[class*="instructions"] p',
      '[class*="directions"] li',
      '[class*="directions"] p',
      '[class*="step"] li',
      '[class*="step"] p',
      '.cooked-step',
      '.recipe-method li',
      '.recipe-method p',
      '.recipe-preparation li'
    ];

    for (const selector of instructionSelectors) {
      const items = $(selector);
      if (items.length > 1) {
        items.each((i, el) => {
          const txt = $(el).text().trim().replace(/\s+/g, ' ');
          if (txt && !recipeData.instructions.includes(txt)) {
            recipeData.instructions.push(txt);
          }
        });
        if (recipeData.instructions.length > 1) break;
      }
    }
  }

  // Heuristics Fallback: Heading Sibling crawl if still empty
  if (recipeData.ingredients.length === 0) {
    $('h1, h2, h3, h4, h5').each((i, heading) => {
      const text = $(heading).text().toLowerCase();
      if (text.includes('ingredients') && !text.includes('instruction') && !text.includes('direction')) {
        let next = $(heading).next();
        if (next.length === 0) {
          next = $(heading).parent().next();
        }
        
        let limit = 0;
        while (next.length > 0 && limit < 15) {
          if (/h[1-6]/i.test(next[0].tagName)) break;
          
          const items = next.find('li, p');
          if (items.length > 0) {
            items.each((j, el) => {
              const itemText = $(el).text().trim().replace(/\s+/g, ' ');
              if (itemText && itemText.length > 2 && itemText.length < 200 && !recipeData.ingredients.includes(itemText)) {
                recipeData.ingredients.push(itemText);
              }
            });
          } else {
            const itemText = next.text().trim().replace(/\s+/g, ' ');
            if (itemText && itemText.length > 2 && itemText.length < 200) {
              recipeData.ingredients.push(itemText);
            }
          }
          next = next.next();
          limit++;
        }
        if (recipeData.ingredients.length > 2) return false;
      }
    });
  }

  if (recipeData.instructions.length === 0) {
    $('h1, h2, h3, h4, h5').each((i, heading) => {
      const text = $(heading).text().toLowerCase();
      if (/instruction|direction|method|preparation|steps/i.test(text) && !text.includes('ingredient')) {
        let next = $(heading).next();
        if (next.length === 0) {
          next = $(heading).parent().next();
        }
        
        let limit = 0;
        while (next.length > 0 && limit < 15) {
          if (/h[1-6]/i.test(next[0].tagName)) break;
          
          const items = next.find('li, p');
          if (items.length > 0) {
            items.each((j, el) => {
              const itemText = $(el).text().trim().replace(/\s+/g, ' ');
              if (itemText && itemText.length > 5 && itemText.length < 1000 && !recipeData.instructions.includes(itemText)) {
                recipeData.instructions.push(itemText);
              }
            });
          } else {
            const itemText = next.text().trim().replace(/\s+/g, ' ');
            if (itemText && itemText.length > 5 && itemText.length < 1000) {
              recipeData.instructions.push(itemText);
            }
          }
          next = next.next();
          limit++;
        }
        if (recipeData.instructions.length > 1) return false;
      }
    });
  }

  // Heuristics Fallback: Blockquote / Multi-paragraph text classifier (for vintage/informal blogs)
  if (recipeData.ingredients.length === 0 && recipeData.instructions.length === 0) {
    const isStepText = (text) => {
      if (/^(?:step\s*)?\d+[\.\\)\-]\s/i.test(text)) return true;
      if (/^\d+\s+(?:mix|stir|combine|sift|bake|heat|preheat|fry|drain|serve|boil|chop|pour|place|add|cook|whisk|beat|divide|mould|brush|roll|repeat|prepare|cut)\b/i.test(text)) return true;
      if (/^(?:mix|stir|combine|sift|bake|heat|preheat|fry|drain|serve|boil|chop|pour|place|add|cook|whisk|beat|divide|mould|brush|roll|repeat|prepare|cut)\b/i.test(text) && text.length > 25) return true;
      return false;
    };

    const isTitleOrHeader = (text) => {
      const titleClean = recipeData.title.toLowerCase();
      const textClean = text.toLowerCase();
      if (titleClean.includes(textClean) || textClean.includes(titleClean)) return true;
      if (/^(ingredients|instructions|directions|method|results|notes|conclusion)\b/i.test(textClean)) return true;
      return false;
    };

    let selectedContainer = null;
    if ($('blockquote').length > 0) {
      selectedContainer = $('blockquote');
    } else if ($('.entry-content').length > 0) {
      selectedContainer = $('.entry-content');
    } else if ($('article').length > 0) {
      selectedContainer = $('article');
    }

    if (selectedContainer) {
      selectedContainer.each((i, container) => {
        const paragraphs = $(container).find('p');
        if (paragraphs.length > 2) {
          paragraphs.each((j, el) => {
            const html = $(el).html() || '';
            if (html.includes('<br>')) {
              const lines = html.split(/<br\s*\/?>/i)
                .map(l => cheerio.load(l).text().trim().replace(/\s+/g, ' '))
                .filter(Boolean);
              
              for (const line of lines) {
                if (isTitleOrHeader(line) || line.length < 3) continue;
                if (isStepText(line)) {
                  recipeData.instructions.push(line);
                } else {
                  recipeData.ingredients.push(line);
                }
              }
            } else {
              const txt = $(el).text().trim().replace(/\s+/g, ' ');
              if (!txt || isTitleOrHeader(txt) || txt.length < 3) return;
              
              if (isStepText(txt)) {
                recipeData.instructions.push(txt);
              } else {
                recipeData.ingredients.push(txt);
              }
            }
          });
          if (recipeData.ingredients.length > 1) return false;
        }
      });
    }
  }

  // Clean up lists
  recipeData.ingredients = recipeData.ingredients.map(i => i.trim()).filter(Boolean);
  recipeData.instructions = recipeData.instructions.map(i => i.trim()).filter(Boolean);
  recipeData.images = extractedImages.slice(0, 15);

  return recipeData;
}

// Helper: Detect if HTML response is a bot challenge page rather than real content
function isBotChallengePage(html) {
  if (!html || typeof html !== 'string') return true;
  const lowerHtml = html.toLowerCase();
  if (lowerHtml.includes('cf-browser-verification') || lowerHtml.includes('cf_chl_opt')) return true;
  if (lowerHtml.includes('checking your browser') && lowerHtml.includes('cloudflare')) return true;
  if (lowerHtml.includes('akamai') && lowerHtml.includes('bot manager')) return true;
  if (lowerHtml.includes('px-captcha') || lowerHtml.includes('perimeterx')) return true;
  if (lowerHtml.includes('access to this page has been denied')) return true;
  if (lowerHtml.includes('please verify you are a human') && !lowerHtml.includes('recipeingredient')) return true;
  if (lowerHtml.includes('recaptcha') && !lowerHtml.includes('recipeingredient')) return true;
  if (html.length < 500 && !lowerHtml.includes('recipe')) return true;
  return false;
}

// Helper: Fetch full HTML via Jina Reader HTML mode to bypass edge bot protection (PerimeterX, Cloudflare, etc.)
async function fetchWithReaderProxy(url) {
  console.log('[ReaderProxy] Fetching rendered HTML for:', url);
  const response = await axios.get(`https://r.jina.ai/${url}`, {
    headers: {
      'X-Return-Format': 'html',
      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
    },
    timeout: 20000
  });
  return response.data;
}

// Helper: Fetch HTML using Puppeteer with stealth (headless browser fallback)
async function fetchWithPuppeteer(url) {
  console.log('[Puppeteer] Launching headless browser for:', url);
  let browser = null;
  try {
    browser = await puppeteer.launch({
      headless: 'new',
      args: [
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
        '--disable-accelerated-2d-canvas',
        '--disable-gpu',
        '--window-size=1920,1080'
      ]
    });

    const page = await browser.newPage();
    await page.setViewport({ width: 1920, height: 1080 });
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
    await page.setExtraHTTPHeaders({
      'Accept-Language': 'en-US,en;q=0.9',
      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
    });

    await page.goto(url, { 
      waitUntil: 'networkidle0', 
      timeout: 30000 
    });

    await new Promise(resolve => setTimeout(resolve, 2000));
    await page.evaluate(() => {
      window.scrollTo(0, document.body.scrollHeight / 2);
    });
    await new Promise(resolve => setTimeout(resolve, 1000));

    const html = await page.content();
    console.log('[Puppeteer] Successfully retrieved', html.length, 'bytes of HTML');
    return html;
  } finally {
    if (browser) {
      await browser.close();
      console.log('[Puppeteer] Browser closed');
    }
  }
}

// API: Load System Settings
app.get('/api/settings', requireAuth, (req, res) => {
  try {
    const settings = db.prepare('SELECT geminiApiKey FROM settings LIMIT 1').get();
    res.json({ geminiApiKey: settings ? settings.geminiApiKey : '' });
  } catch (e) {
    console.error('Failed to load settings:', e);
    res.status(500).json({ error: 'Failed to read settings' });
  }
});

// API: Save System Settings
app.post('/api/settings', requireAuth, (req, res) => {
  const { geminiApiKey } = req.body;
  try {
    db.prepare('UPDATE settings SET geminiApiKey = ?').run(geminiApiKey || '');
    res.json({ geminiApiKey: geminiApiKey || '' });
  } catch (e) {
    console.error('Failed to save settings:', e);
    res.status(500).json({ error: 'Failed to save settings' });
  }
});

// API: Sanity-checked, unauthenticated public recipe sharing with rate limiting
app.get('/api/share/recipes/:shareToken', publicShareLimiter, (req, res) => {
  const shareToken = req.params.shareToken;
  if (!shareToken || shareToken.trim() === '') {
    return res.status(400).json({ error: 'Share token is required.' });
  }
  
  try {
    const recipe = db.prepare('SELECT * FROM recipes WHERE shareToken = ? AND isPublic = 1').get(shareToken);
    if (!recipe) {
      return res.status(404).json({ error: 'Recipe not found or is set to private.' });
    }
    
    // Log guest visitor IP dynamically inside the background
    try {
      const ip = req.headers['x-forwarded-for'] || req.socket.remoteAddress;
      db.prepare(`
        INSERT OR IGNORE INTO recipe_shares_visits (shareToken, visitorIp, visitedAt)
        VALUES (?, ?, ?)
      `).run(shareToken, ip, new Date().toISOString());
    } catch (visitErr) {
      console.error('Failed to log shared visitor IP:', visitErr);
    }
    
    // Strip sensitive user identifiers and internal database keys
    const sanitizedRecipe = {
      title: recipe.title,
      description: recipe.description,
      servings: recipe.servings,
      prepTime: recipe.prepTime,
      cookTime: recipe.cookTime,
      totalTime: recipe.totalTime,
      image: recipe.image,
      images: JSON.parse(recipe.images || '[]'),
      ingredients: JSON.parse(recipe.ingredients || '[]'),
      instructions: JSON.parse(recipe.instructions || '[]'),
      sourceUrl: recipe.sourceUrl,
      createdBy: recipe.createdBy,
      createdAt: recipe.createdAt
    };
    
    res.json(sanitizedRecipe);
  } catch (error) {
    console.error('Error serving shared recipe:', error);
    res.status(500).json({ error: 'Internal server error while retrieving shared recipe.' });
  }
});

// API: Revoke public recipe sharing link (requires authenticated ownership or permission)
app.post('/api/recipes/:id/revoke-share', requireAuth, (req, res) => {
  try {
    const recipe = db.prepare('SELECT * FROM recipes WHERE id = ?').get(req.params.id);
    
    if (!recipe) {
      return res.status(404).json({ error: 'Recipe not found.' });
    }
    
    // Verify permissions: owner, recipe manager, or admin
    if (recipe.userId !== req.user.id && !req.user.isRecipeManager && !req.user.isAdmin) {
      return res.status(403).json({ error: 'You do not have permission to revoke this recipe share link.' });
    }
    
    const oldShareToken = recipe.shareToken;
    
    // Relational transaction to clear sharing state and purge visits data
    const performRevocation = db.transaction(() => {
      db.prepare('UPDATE recipes SET isPublic = 0, shareToken = NULL WHERE id = ?').run(req.params.id);
      
      if (oldShareToken) {
        db.prepare('DELETE FROM recipe_shares_visits WHERE shareToken = ?').run(oldShareToken);
      }
    });
    
    performRevocation();
    res.json({ success: true });
  } catch (error) {
    console.error('Error revoking recipe share link:', error);
    res.status(500).json({ error: 'Failed to revoke shared link.' });
  }
});

// API: Scrape Recipe from Link
app.post('/api/scrape', requireAuth, async (req, res) => {
  const { url } = req.body;
  if (!url) {
    return res.status(400).json({ error: 'URL is required' });
  }

  try {
    let html = null;
    let scrapingMethod = 'axios';

    // Strategy 1: Fast axios fetch
    try {
      const response = await axios.get(url, {
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
          'Accept-Language': 'en-US,en;q=0.9',
          'Accept-Encoding': 'gzip, deflate, br',
          'Connection': 'keep-alive',
          'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
          'sec-ch-ua-mobile': '?0',
          'sec-ch-ua-platform': '"Windows"',
          'Sec-Fetch-Dest': 'document',
          'Sec-Fetch-Mode': 'navigate',
          'Sec-Fetch-Site': 'none',
          'Sec-Fetch-User': '?1',
          'Upgrade-Insecure-Requests': '1'
        },
        timeout: 10000
      });
      html = response.data;

      if (isBotChallengePage(html)) {
        console.log('[Scraper] Axios response appears to be a bot challenge page, falling back...');
        html = null;
      }
    } catch (axiosError) {
      const status = axiosError.response ? axiosError.response.status : 0;
      if (status === 403 || status === 503 || status === 429) {
        console.log(`[Scraper] Axios blocked with HTTP ${status}, falling back...`);
      } else {
        throw axiosError;
      }
    }

    // Strategy 2: Reader Proxy HTML fetch (fast bypass for PerimeterX / Cloudflare edge blocks)
    if (!html) {
      try {
        const readerHtml = await fetchWithReaderProxy(url);
        if (!isBotChallengePage(readerHtml)) {
          html = readerHtml;
          scrapingMethod = 'reader-proxy';
        } else {
          console.log('[ReaderProxy] Returned challenge page, falling back to Puppeteer...');
        }
      } catch (readerError) {
        console.warn('[ReaderProxy] Fallback failed, trying Puppeteer:', readerError.message);
      }
    }

    // Strategy 3: Puppeteer with stealth (headless browser fallback)
    if (!html) {
      try {
        const puppeteerHtml = await fetchWithPuppeteer(url);
        if (!isBotChallengePage(puppeteerHtml)) {
          html = puppeteerHtml;
          scrapingMethod = 'puppeteer-stealth';
        } else {
          throw new Error('Blocked by bot challenge page in headless browser.');
        }
      } catch (puppeteerError) {
        console.error('[Puppeteer] Fallback also failed:', puppeteerError.message);
        return res.status(500).json({
          error: 'This website could not be accessed. All scraping methods were blocked.',
          details: puppeteerError.message,
          isBlocked: true
        });
      }
    }

    const recipeData = parseRecipeFromHtml(html, url);
    recipeData._scrapingMethod = scrapingMethod;

    res.json(recipeData);
  } catch (error) {
    console.error('Error occurred while scraping URL:', error.message);
    const status = error.response ? error.response.status : 500;
    const message = error.response && error.response.status === 403
      ? 'This website is protected and blocks automated access. The Puppeteer fallback was also unavailable.'
      : error.message;
    res.status(status).json({ 
      error: message, 
      isBlocked: status === 403 || status === 503 
    });
  }
});

// Launch server
app.listen(PORT, () => {
  console.log(`Recipe Manager server is active at http://localhost:${PORT}`);
});