const express = require('express');
const axios = require('axios');
const cheerio = require('cheerio');
const cors = require('cors');
const fs = require('fs');
const http = require('http');
const os = require('os');
const path = require('path');
const crypto = require('crypto');
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
const Database = require('better-sqlite3');
const rateLimit = require('express-rate-limit');

// Enable stealth mode to bypass bot detection (Cloudflare, Akamai, etc.)
puppeteer.use(StealthPlugin());

const app = express();
const PORT = process.env.PORT || 3000;

// Setup directories for persistence
const DATA_DIR = path.join(__dirname, 'data');
const DB_FILE = path.join(DATA_DIR, 'forkfolio.db');

if (!fs.existsSync(DATA_DIR)) {
  fs.mkdirSync(DATA_DIR, { recursive: true });
}

// Open SQLite Database connection and set WAL journal mode for concurrency speed
const db = new Database(DB_FILE);
db.pragma('journal_mode = WAL');

// Initialize database schema tables
db.prepare(`
  CREATE TABLE IF NOT EXISTS users (
    id TEXT PRIMARY KEY,
    username TEXT UNIQUE NOT NULL,
    salt TEXT NOT NULL,
    passwordHash TEXT NOT NULL,
    isAdmin INTEGER NOT NULL DEFAULT 0,
    isRecipeManager INTEGER NOT NULL DEFAULT 0,
    defaultConsolidateLists INTEGER NOT NULL DEFAULT 1,
    createdAt TEXT NOT NULL
  )
`).run();

// Ensure defaultConsolidateLists column exists on existing users table
try {
  const userCols = db.pragma('table_info(users)');
  if (!userCols.some(c => c.name === 'defaultConsolidateLists')) {
    db.prepare('ALTER TABLE users ADD COLUMN defaultConsolidateLists INTEGER NOT NULL DEFAULT 1').run();
  }
} catch (colErr) {
  console.error('Failed to check/add defaultConsolidateLists column:', colErr);
}

db.prepare(`
  CREATE TABLE IF NOT EXISTS recipes (
    id TEXT PRIMARY KEY,
    userId TEXT NOT NULL,
    createdBy TEXT NOT NULL,
    isPublic INTEGER NOT NULL DEFAULT 1,
    title TEXT NOT NULL,
    description TEXT,
    servings TEXT,
    prepTime TEXT,
    cookTime TEXT,
    totalTime TEXT,
    image TEXT,
    images TEXT,       -- JSON stringified array
    ingredients TEXT,  -- JSON stringified array
    instructions TEXT, -- JSON stringified array
    sourceUrl TEXT,
    shareToken TEXT UNIQUE, -- Unguessable sharing token
    createdAt TEXT NOT NULL,
    updatedAt TEXT,
    FOREIGN KEY (userId) REFERENCES users (id) ON DELETE CASCADE
  )
`).run();

db.prepare(`
  CREATE TABLE IF NOT EXISTS settings (
    geminiApiKey TEXT DEFAULT '',
    geminiModel TEXT DEFAULT 'gemini-2.0-flash'
  )
`).run();

db.prepare(`
  CREATE TABLE IF NOT EXISTS recipe_shares_visits (
    shareToken TEXT NOT NULL,
    visitorIp TEXT NOT NULL,
    visitedAt TEXT NOT NULL,
    PRIMARY KEY (shareToken, visitorIp)
  )
`).run();

db.prepare(`
  CREATE TABLE IF NOT EXISTS shopping_lists (
    id TEXT PRIMARY KEY,
    userId TEXT NOT NULL,
    createdBy TEXT NOT NULL,
    name TEXT NOT NULL,
    shareToken TEXT UNIQUE,
    createdAt TEXT NOT NULL,
    updatedAt TEXT,
    FOREIGN KEY (userId) REFERENCES users (id) ON DELETE CASCADE
  )
`).run();

db.prepare(`
  CREATE TABLE IF NOT EXISTS shopping_list_items (
    id TEXT PRIMARY KEY,
    listId TEXT NOT NULL,
    rawText TEXT NOT NULL,
    itemName TEXT NOT NULL,
    quantity REAL,
    unit TEXT DEFAULT '',
    category TEXT NOT NULL DEFAULT 'Other',
    recipeId TEXT,
    recipeTitle TEXT,
    checked INTEGER NOT NULL DEFAULT 0,
    createdAt TEXT NOT NULL,
    FOREIGN KEY (listId) REFERENCES shopping_lists (id) ON DELETE CASCADE
  )
`).run();

db.prepare(`
  CREATE TABLE IF NOT EXISTS shopping_list_shares_visits (
    shareToken TEXT NOT NULL,
    visitorIp TEXT NOT NULL,
    visitedAt TEXT NOT NULL,
    PRIMARY KEY (shareToken, visitorIp)
  )
`).run();

db.prepare(`
  CREATE TABLE IF NOT EXISTS backup_config (
    id INTEGER PRIMARY KEY CHECK (id = 1),
    scheduleFrequency TEXT NOT NULL DEFAULT 'off',
    scheduleTime TEXT NOT NULL DEFAULT '02:00',
    scheduleDay INTEGER NOT NULL DEFAULT 0,
    retentionCount INTEGER NOT NULL DEFAULT 14,
    backupFormat TEXT NOT NULL DEFAULT 'both',
    embedRemoteImages INTEGER NOT NULL DEFAULT 1,
    s3Enabled INTEGER NOT NULL DEFAULT 0,
    s3Endpoint TEXT NOT NULL DEFAULT 'https://s3.us-east-1.wasabisys.com',
    s3Region TEXT NOT NULL DEFAULT 'us-east-1',
    s3Bucket TEXT NOT NULL DEFAULT '',
    s3Prefix TEXT NOT NULL DEFAULT 'forkfolio-backups/',
    s3AccessKeyId TEXT NOT NULL DEFAULT '',
    s3SecretAccessKey TEXT NOT NULL DEFAULT '',
    gdriveEnabled INTEGER NOT NULL DEFAULT 0,
    gdriveClientId TEXT NOT NULL DEFAULT '',
    gdriveClientSecret TEXT NOT NULL DEFAULT '',
    gdriveRefreshToken TEXT NOT NULL DEFAULT '',
    gdriveFolderId TEXT NOT NULL DEFAULT '',
    lastBackupAt TEXT,
    lastBackupStatus TEXT,
    lastBackupMessage TEXT
  )
`).run();

db.prepare(`
  CREATE TABLE IF NOT EXISTS backup_history (
    id TEXT PRIMARY KEY,
    label TEXT NOT NULL,
    triggerType TEXT NOT NULL,
    sizeBytes INTEGER NOT NULL DEFAULT 0,
    userCount INTEGER NOT NULL DEFAULT 0,
    recipeCount INTEGER NOT NULL DEFAULT 0,
    imageCount INTEGER NOT NULL DEFAULT 0,
    listCount INTEGER NOT NULL DEFAULT 0,
    s3Objects TEXT DEFAULT '[]',
    gdriveFiles TEXT DEFAULT '[]',
    status TEXT NOT NULL,
    errorDetails TEXT,
    createdAt TEXT NOT NULL
  )
`).run();

const backupConfigCount = db.prepare('SELECT COUNT(*) as count FROM backup_config').get().count;
if (backupConfigCount === 0) {
  db.prepare('INSERT INTO backup_config (id) VALUES (1)').run();
}
try {
  db.prepare("ALTER TABLE backup_config ADD COLUMN gdriveAccountEmail TEXT NOT NULL DEFAULT ''").run();
} catch (e) {}
try {
  db.prepare("ALTER TABLE backup_config ADD COLUMN gdriveFolderName TEXT NOT NULL DEFAULT 'ForkFolio Backups'").run();
} catch (e) {}
try {
  db.prepare("ALTER TABLE backup_history ADD COLUMN userSummary TEXT NOT NULL DEFAULT '[]'").run();
} catch (e) {}
try {
  db.prepare("ALTER TABLE settings ADD COLUMN geminiModel TEXT DEFAULT 'gemini-2.0-flash'").run();
} catch (e) {}
try {
  db.prepare("UPDATE settings SET geminiModel = 'gemini-2.0-flash' WHERE geminiModel IS NULL OR geminiModel = ''").run();
} catch (e) {}

// Initialize single settings record if empty
const settingsCount = db.prepare('SELECT COUNT(*) as count FROM settings').get().count;
if (settingsCount === 0) {
  db.prepare('INSERT INTO settings (geminiApiKey, geminiModel) VALUES (?, ?)').run('', 'gemini-2.0-flash');
}

// Ensure default system_admin and admin user profiles exist
try {
  // Ensure default system_admin user exists to fulfill recipe foreign key constraints
  const hasSystemAdmin = db.prepare("SELECT COUNT(*) as count FROM users WHERE id = 'system_admin'").get().count;
  if (hasSystemAdmin === 0) {
    db.prepare(`
      INSERT INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, createdAt)
      VALUES (?, ?, ?, ?, ?, ?, ?)
    `).run('system_admin', 'system_admin', 'system_salt', 'system_hash', 0, 0, new Date().toISOString());
  }

  const adminCount = db.prepare("SELECT COUNT(*) as count FROM users WHERE isAdmin = 1").get().count;
  if (adminCount === 0) {
    const salt = crypto.randomBytes(16).toString('hex');
    const passwordHash = crypto.pbkdf2Sync('admin123', salt, 1000, 64, 'sha512').toString('hex');
    db.prepare(`
      INSERT INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, createdAt)
      VALUES (?, ?, ?, ?, ?, ?, ?)
    `).run(
      'u_admin',
      'admin',
      salt,
      passwordHash,
      1,
      1,
      new Date().toISOString()
    );
    console.log('--- ForkFolio Admin Initialized ---');
    console.log('Default Admin Username: admin');
    console.log('Default Admin Password: admin123');
    console.log('--------------------------------------');
  }
} catch (e) {
  console.error('Failed to initialize default admin and system users:', e);
}

// Automatic Migration from legacy JSON databases if they exist
const LEGACY_USERS_FILE = path.join(DATA_DIR, 'users.json');
const LEGACY_RECIPES_FILE = path.join(DATA_DIR, 'recipes.json');
const LEGACY_SETTINGS_FILE = path.join(DATA_DIR, 'settings.json');

try {
  // Temporarily disable foreign keys during bulk migrations to allow flexible insert order
  db.pragma('foreign_keys = OFF');

  // 1. Migrate Users
  if (fs.existsSync(LEGACY_USERS_FILE)) {
    const backupUsersFile = path.join(DATA_DIR, 'users.json.backup');
    const data = fs.readFileSync(LEGACY_USERS_FILE, 'utf8');
    const users = JSON.parse(data);
    if (Array.isArray(users) && users.length > 0) {
      console.log(`[Migration] Found ${users.length} legacy users. Migrating to SQLite...`);
      const insertUser = db.prepare(`
        INSERT OR IGNORE INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, createdAt)
        VALUES (?, ?, ?, ?, ?, ?, ?)
      `);
      
      const transaction = db.transaction((userList) => {
        for (const u of userList) {
          insertUser.run(
            u.id,
            u.username.trim().toLowerCase(),
            u.salt,
            u.passwordHash,
            u.isAdmin ? 1 : 0,
            u.isRecipeManager ? 1 : 0,
            u.createdAt || new Date().toISOString()
          );
        }
      });
      transaction(users);
      console.log(`[Migration] User migration completed.`);
    }
    fs.renameSync(LEGACY_USERS_FILE, backupUsersFile);
  }

  // 2. Migrate Recipes
  if (fs.existsSync(LEGACY_RECIPES_FILE)) {
    const backupRecipesFile = path.join(DATA_DIR, 'recipes.json.backup');
    const data = fs.readFileSync(LEGACY_RECIPES_FILE, 'utf8');
    const recipes = JSON.parse(data);
    if (Array.isArray(recipes) && recipes.length > 0) {
      console.log(`[Migration] Found ${recipes.length} legacy recipes. Migrating to SQLite...`);
      const insertRecipe = db.prepare(`
        INSERT OR IGNORE INTO recipes (
          id, userId, createdBy, isPublic, title, description, servings,
          prepTime, cookTime, totalTime, image, images, ingredients, instructions, sourceUrl, shareToken, createdAt, updatedAt
        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
      `);
      
      const transaction = db.transaction((recipeList) => {
        for (const r of recipeList) {
          const shareToken = r.isPublic !== false && !r.shareToken 
            ? crypto.randomBytes(16).toString('hex') 
            : (r.shareToken || null);
            
          insertRecipe.run(
            r.id,
            r.userId || 'system_admin',
            r.createdBy || 'gourmet_system',
            r.isPublic !== false ? 1 : 0,
            r.title || 'Untitled Recipe',
            r.description || '',
            r.servings || '',
            r.prepTime || '',
            r.cookTime || '',
            r.totalTime || '',
            r.image || '',
            JSON.stringify(r.images || []),
            JSON.stringify(r.ingredients || []),
            JSON.stringify(r.instructions || []),
            r.sourceUrl || '',
            shareToken,
            r.createdAt || new Date().toISOString(),
            r.updatedAt || null
          );
        }
      });
      transaction(recipes);
      console.log(`[Migration] Recipe migration completed.`);
    }
    fs.renameSync(LEGACY_RECIPES_FILE, backupRecipesFile);
  }

  // 3. Migrate Settings
  if (fs.existsSync(LEGACY_SETTINGS_FILE)) {
    const backupSettingsFile = path.join(DATA_DIR, 'settings.json.backup');
    const data = fs.readFileSync(LEGACY_SETTINGS_FILE, 'utf8');
    const settings = JSON.parse(data);
    if (settings && settings.geminiApiKey) {
      console.log('[Migration] Migrating system settings key to SQLite...');
      db.prepare('UPDATE settings SET geminiApiKey = ?').run(settings.geminiApiKey.trim());
      console.log('[Migration] Settings migration completed.');
    }
    fs.renameSync(LEGACY_SETTINGS_FILE, backupSettingsFile);
  }

  // Restore foreign keys enforcement after migrations
  db.pragma('foreign_keys = ON');
} catch (migrationError) {
  // Guarantee foreign keys re-enabled even on crash
  db.pragma('foreign_keys = ON');
  console.error('[Migration] Failed to perform automatic SQLite migration:', migrationError);
}

// In-memory sessions storage (token -> { id, username, isAdmin, isRecipeManager })
const activeSessions = new Map();

app.use(cors());
app.use(express.json({ limit: '100mb', type: ['application/json', 'text/plain'] }));
app.use(express.urlencoded({ limit: '100mb', extended: true }));
app.use(express.static(path.join(__dirname, 'public')));

// Configure rate-limiting middleware for public shared routes
const publicShareLimiter = rateLimit({
  windowMs: 15 * 60 * 1000, // 15 minutes
  max: 100, // Limit each IP to 100 requests per 15-minute window
  message: { error: 'Too many queries from this IP. Please try again in 15 minutes.' }
});

// Helper: Parse ISO 8601 Durations (e.g. PT1H30M to "1 hr 30 mins")
function parseISODuration(duration) {
  if (!duration || typeof duration !== 'string') return '';
  const match = duration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/);
  if (!match) return duration;
  const hours = match[1] ? parseInt(match[1], 10) : 0;
  const minutes = match[2] ? parseInt(match[2], 10) : 0;
  
  let result = '';
  if (hours > 0) result += `${hours} hr${hours > 1 ? 's' : ''} `;
  if (minutes > 0) result += `${minutes} min${minutes > 1 ? 's' : ''}`;
  return result.trim() || duration;
}

// Helper: Extract recipe instructions from schema format
function extractInstructions(instructions) {
  if (!instructions) return [];
  if (typeof instructions === 'string') {
    return instructions.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
  }
  if (Array.isArray(instructions)) {
    const result = [];
    for (const inst of instructions) {
      if (typeof inst === 'string') {
        result.push(inst.trim());
      } else if (inst && typeof inst === 'object') {
        if (inst['@type'] === 'HowToStep') {
          result.push(inst.text || inst.name || '');
        } else if (inst['@type'] === 'HowToSection' && Array.isArray(inst.itemListElement)) {
          if (inst.name) {
            result.push(`### ${inst.name}`); // Prefix section names
          }
          for (const step of inst.itemListElement) {
            if (typeof step === 'string') {
              result.push(step.trim());
            } else if (step && typeof step === 'object') {
              result.push(step.text || step.name || '');
            }
          }
        } else if (inst.text) {
          result.push(inst.text);
        } else if (inst.name) {
          result.push(inst.name);
        }
      }
    }
    return result.filter(Boolean);
  }
  return [];
}

// Helper: Normalize image resources from JSON-LD schema
function normalizeImages(imageObj) {
  if (!imageObj) return [];
  if (typeof imageObj === 'string') return [imageObj];
  if (Array.isArray(imageObj)) {
    return imageObj.map(img => (typeof img === 'string' ? img : img.url)).filter(Boolean);
  }
  if (typeof imageObj === 'object' && imageObj.url) {
    return [imageObj.url];
  }
  return [];
}

// Password Hashing and Salt helpers
function generateSalt() {
  return crypto.randomBytes(16).toString('hex');
}

function hashPassword(password, salt) {
  return crypto.pbkdf2Sync(password, salt, 1000, 64, 'sha512').toString('hex');
}

// Middleware: Authenticate Session Token
function requireAuth(req, res, next) {
  const authHeader = req.headers.authorization;
  if (!authHeader || !authHeader.startsWith('Bearer ')) {
    return res.status(401).json({ error: 'Authorization required. Please log in.' });
  }
  const token = authHeader.substring(7);
  const session = activeSessions.get(token);
  if (!session) {
    return res.status(401).json({ error: 'Session expired. Please log in again.' });
  }
  req.user = session;
  next();
}

// Helper: Deserialize JSON columns from SQLite recipe row
function formatRecipe(row) {
  if (!row) return null;
  
  let visitorCount = 0;
  if (row.shareToken) {
    try {
      const res = db.prepare('SELECT COUNT(*) as count FROM recipe_shares_visits WHERE shareToken = ?').get(row.shareToken);
      visitorCount = res ? res.count : 0;
    } catch (e) {
      console.error('Failed to query unique visitor count:', e);
    }
  }

  return {
    ...row,
    isPublic: row.isPublic === 1,
    images: JSON.parse(row.images || '[]'),
    ingredients: JSON.parse(row.ingredients || '[]'),
    instructions: JSON.parse(row.instructions || '[]'),
    visitorCount
  };
}

// Auth API: User Registration (Requires Admin session)
app.post('/api/auth/register', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }

  const { username, password } = req.body;
  if (!username || !password) {
    return res.status(400).json({ error: 'Username and password are required.' });
  }
  
  const cleanUsername = username.trim().toLowerCase();
  if (!/^[a-z0-9_]{3,20}$/.test(cleanUsername)) {
    return res.status(400).json({ error: 'Username must be 3-20 characters long and contain only alphanumeric characters or underscores.' });
  }
  
  if (password.length < 6) {
    return res.status(400).json({ error: 'Password must be at least 6 characters long.' });
  }
  
  try {
    const existing = db.prepare('SELECT id FROM users WHERE username = ?').get(cleanUsername);
    if (existing) {
      return res.status(400).json({ error: 'Username is already taken.' });
    }
    
    const salt = generateSalt();
    const passwordHash = hashPassword(password, salt);
    const id = 'u_' + Date.now().toString(36) + Math.random().toString(36).substr(2, 5);
    
    db.prepare(`
      INSERT INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, createdAt)
      VALUES (?, ?, ?, ?, ?, ?, ?)
    `).run(id, cleanUsername, salt, passwordHash, 0, 0, new Date().toISOString());
    
    res.status(201).json({
      user: {
        id,
        username: cleanUsername,
        isAdmin: false,
        isRecipeManager: false,
        defaultConsolidateLists: true
      }
    });
  } catch (error) {
    console.error('Error during registration:', error);
    res.status(500).json({ error: 'Failed to create account.' });
  }
});

// Auth API: User Login
app.post('/api/auth/login', (req, res) => {
  const { username, password } = req.body;
  if (!username || !password) {
    return res.status(400).json({ error: 'Username and password are required.' });
  }
  
  const cleanUsername = username.trim().toLowerCase();
  try {
    const user = db.prepare('SELECT * FROM users WHERE username = ?').get(cleanUsername);
    if (!user) {
      return res.status(401).json({ error: 'Invalid username or password.' });
    }
    
    const hash = hashPassword(password, user.salt);
    if (hash !== user.passwordHash) {
      return res.status(401).json({ error: 'Invalid username or password.' });
    }
    
    const token = crypto.randomBytes(32).toString('hex');
    const defaultConsolidateLists = user.defaultConsolidateLists !== 0;
    activeSessions.set(token, { 
      id: user.id, 
      username: user.username, 
      isAdmin: user.isAdmin === 1,
      isRecipeManager: user.isRecipeManager === 1,
      defaultConsolidateLists
    });
    
    res.json({
      token,
      user: {
        id: user.id,
        username: user.username,
        isAdmin: user.isAdmin === 1,
        isRecipeManager: user.isRecipeManager === 1,
        defaultConsolidateLists
      }
    });
  } catch (error) {
    console.error('Error during login:', error);
    res.status(500).json({ error: 'Login failed.' });
  }
});

// Admin API: List all users (Requires Admin session)
app.get('/api/admin/users', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  try {
    const rows = db.prepare('SELECT id, username, isAdmin, isRecipeManager, createdAt FROM users').all();
    const users = rows.map(u => ({
      id: u.id,
      username: u.username,
      isAdmin: u.isAdmin === 1,
      isRecipeManager: u.isRecipeManager === 1,
      createdAt: u.createdAt
    }));
    res.json(users);
  } catch (error) {
    console.error('Error listing users:', error);
    res.status(500).json({ error: 'Failed to retrieve registered users.' });
  }
});

// Admin API: Delete user (Requires Admin session)
app.delete('/api/admin/users/:id', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  if (req.params.id === req.user.id) {
    return res.status(400).json({ error: 'You cannot delete your own administrative account.' });
  }
  
  try {
    const user = db.prepare('SELECT id, username FROM users WHERE id = ?').get(req.params.id);
    if (!user) {
      return res.status(404).json({ error: 'User not found.' });
    }
    
    const preserveRecipes = req.body.preserveRecipes === true;
    
    // SQLite transaction for relational database integrity
    const performDelete = db.transaction(() => {
      if (preserveRecipes) {
        // Reassign user's recipes to system_admin, set as public
        db.prepare(`
          UPDATE recipes
          SET userId = 'system_admin',
              createdBy = ?,
              isPublic = 1
          WHERE userId = ?
        `).run(`${user.username} (former)`, user.id);
      } else {
        // Delete user's recipes
        db.prepare('DELETE FROM recipes WHERE userId = ?').run(user.id);
      }
      
      // Delete user profile
      db.prepare('DELETE FROM users WHERE id = ?').run(user.id);
    });
    
    performDelete();
    
    // Invalidate active sessions
    for (const [token, session] of activeSessions.entries()) {
      if (session.id === user.id) {
        activeSessions.delete(token);
      }
    }
    
    res.json({ success: true });
  } catch (error) {
    console.error('Error deleting user:', error);
    res.status(500).json({ error: 'Failed to delete user.' });
  }
});

// Admin API: Change user role (Requires Admin session)
app.put('/api/admin/users/:id/role', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  if (req.params.id === req.user.id) {
    return res.status(400).json({ error: 'You cannot change your own administrator role privileges.' });
  }
  
  const { isAdmin } = req.body;
  if (typeof isAdmin !== 'boolean') {
    return res.status(400).json({ error: 'isAdmin property must be a boolean.' });
  }
  
  try {
    const user = db.prepare('SELECT id, username FROM users WHERE id = ?').get(req.params.id);
    if (!user) {
      return res.status(404).json({ error: 'User not found.' });
    }
    
    db.prepare('UPDATE users SET isAdmin = ? WHERE id = ?').run(isAdmin ? 1 : 0, user.id);
    
    // Update sessions
    for (const [token, session] of activeSessions.entries()) {
      if (session.id === user.id) {
        session.isAdmin = isAdmin;
        activeSessions.set(token, session);
      }
    }
    
    res.json({ success: true, user: { id: user.id, username: user.username, isAdmin } });
  } catch (error) {
    console.error('Error changing user role:', error);
    res.status(500).json({ error: 'Failed to update user role.' });
  }
});

// Admin API: Toggle recipe manager role (Requires Admin session)
app.put('/api/admin/users/:id/recipe-manager', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  const { isRecipeManager } = req.body;
  if (typeof isRecipeManager !== 'boolean') {
    return res.status(400).json({ error: 'isRecipeManager property must be a boolean.' });
  }
  
  try {
    const user = db.prepare('SELECT id, username FROM users WHERE id = ?').get(req.params.id);
    if (!user) {
      return res.status(404).json({ error: 'User not found.' });
    }
    
    db.prepare('UPDATE users SET isRecipeManager = ? WHERE id = ?').run(isRecipeManager ? 1 : 0, user.id);
    
    // Update sessions
    for (const [token, session] of activeSessions.entries()) {
      if (session.id === user.id) {
        session.isRecipeManager = isRecipeManager;
        activeSessions.set(token, session);
      }
    }
    
    res.json({ success: true, user: { id: user.id, username: user.username, isRecipeManager } });
  } catch (error) {
    console.error('Error toggling manager role:', error);
    res.status(500).json({ error: 'Failed to update manager permission.' });
  }
});

// Admin API: List all public recipe shares and visitor stats (Requires Admin session)
app.get('/api/admin/shared-recipes', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Access denied. Administrator privileges required.' });
  }
  
  try {
    const rows = db.prepare(`
      SELECT id, title, createdBy, shareToken, createdAt
      FROM recipes
      WHERE isPublic = 1 AND shareToken IS NOT NULL
      ORDER BY createdAt DESC
    `).all();
    
    const sharedRecipes = rows.map(r => {
      let visitorCount = 0;
      try {
        const stats = db.prepare('SELECT COUNT(*) as count FROM recipe_shares_visits WHERE shareToken = ?').get(r.shareToken);
        visitorCount = stats ? stats.count : 0;
      } catch (err) {
        console.error('Failed to query shared recipe visitor count:', err);
      }
      return {
        id: r.id,
        title: r.title,
        createdBy: r.createdBy,
        shareToken: r.shareToken,
        createdAt: r.createdAt,
        visitorCount
      };
    });
    
    res.json(sharedRecipes);
  } catch (error) {
    console.error('Error fetching admin shared recipes:', error);
    res.status(500).json({ error: 'Failed to retrieve shared recipes.' });
  }
});

// API: List authenticated user's private recipes
app.get('/api/recipes', requireAuth, (req, res) => {
  try {
    const rows = db.prepare('SELECT * FROM recipes WHERE userId = ? ORDER BY createdAt DESC').all(req.user.id);
    const userRecipes = rows.map(formatRecipe);
    res.json(userRecipes);
  } catch (error) {
    console.error('Error fetching recipes:', error);
    res.status(500).json({ error: 'Failed to retrieve recipes.' });
  }
});

// API: List all public recipes for Shared Cookbook
app.get('/api/recipes/public', requireAuth, (req, res) => {
  try {
    const rows = db.prepare('SELECT * FROM recipes WHERE isPublic = 1 ORDER BY createdAt DESC').all();
    const publicRecipes = rows.map(formatRecipe);
    res.json(publicRecipes);
  } catch (error) {
    console.error('Error fetching public recipes:', error);
    res.status(500).json({ error: 'Failed to retrieve community recipes.' });
  }
});

// API: Clone a public recipe to user's private collection
app.post('/api/recipes/clone/:id', requireAuth, (req, res) => {
  try {
    const sourceRecipe = db.prepare('SELECT * FROM recipes WHERE id = ?').get(req.params.id);
    
    if (!sourceRecipe) {
      return res.status(404).json({ error: 'Recipe not found' });
    }
    
    if (sourceRecipe.isPublic === 0) {
      return res.status(403).json({ error: 'This recipe is private and cannot be cloned.' });
    }
    
    const clonedId = Date.now().toString(36) + Math.random().toString(36).substr(2, 5);
    
    db.prepare(`
      INSERT INTO recipes (
        id, userId, createdBy, isPublic, title, description, servings,
        prepTime, cookTime, totalTime, image, images, ingredients, instructions, sourceUrl, shareToken, createdAt, updatedAt
      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `).run(
      clonedId,
      req.user.id,
      req.user.username,
      0, // Default cloned copies to private so users can customize them privately
      sourceRecipe.title || 'Untitled Recipe',
      sourceRecipe.description || '',
      sourceRecipe.servings || '',
      sourceRecipe.prepTime || '',
      sourceRecipe.cookTime || '',
      sourceRecipe.totalTime || '',
      sourceRecipe.image || '',
      sourceRecipe.images || '[]',
      sourceRecipe.ingredients || '[]',
      sourceRecipe.instructions || '[]',
      sourceRecipe.sourceUrl || '',
      null, // Cloned copies start as private, no share token
      new Date().toISOString(),
      null
    );
    
    const clonedObj = formatRecipe(db.prepare('SELECT * FROM recipes WHERE id = ?').get(clonedId));
    res.status(201).json(clonedObj);
  } catch (error) {
    console.error('Error cloning recipe:', error);
    res.status(500).json({ error: 'Failed to clone recipe.' });
  }
});

// API: Save new recipe
app.post('/api/recipes', requireAuth, (req, res) => {
  const isPublic = req.body.isPublic !== false ? 1 : 0;
  const shareToken = isPublic === 1 ? crypto.randomBytes(16).toString('hex') : null;
  const newId = Date.now().toString(36) + Math.random().toString(36).substr(2, 5);
  
  try {
    db.prepare(`
      INSERT INTO recipes (
        id, userId, createdBy, isPublic, title, description, servings,
        prepTime, cookTime, totalTime, image, images, ingredients, instructions, sourceUrl, shareToken, createdAt, updatedAt
      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `).run(
      newId,
      req.user.id,
      req.user.username,
      isPublic,
      req.body.title || 'Untitled Recipe',
      req.body.description || '',
      req.body.servings || '',
      req.body.prepTime || '',
      req.body.cookTime || '',
      req.body.totalTime || '',
      req.body.image || '',
      JSON.stringify(Array.isArray(req.body.images) ? req.body.images : []),
      JSON.stringify(Array.isArray(req.body.ingredients) ? req.body.ingredients : []),
      JSON.stringify(Array.isArray(req.body.instructions) ? req.body.instructions : []),
      req.body.sourceUrl || '',
      shareToken,
      new Date().toISOString(),
      null
    );
    
    const savedObj = formatRecipe(db.prepare('SELECT * FROM recipes WHERE id = ?').get(newId));
    res.status(201).json(savedObj);
  } catch (error) {
    console.error('Error saving new recipe:', error);
    res.status(500).json({ error: 'Failed to save recipe.' });
  }
});

// API: Update recipe
app.put('/api/recipes/:id', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT * FROM recipes WHERE id = ?').get(req.params.id);
    
    if (!existing) {
      return res.status(404).json({ error: 'Recipe not found' });
    }
    
    if (existing.userId !== req.user.id && !req.user.isRecipeManager && !req.user.isAdmin) {
      return res.status(403).json({ error: 'You do not have permission to modify this recipe.' });
    }
    
    const isPublic = req.body.isPublic !== undefined ? (req.body.isPublic ? 1 : 0) : existing.isPublic;
    
    // Generate shareToken if it becomes public and doesn't have one
    let shareToken = existing.shareToken;
    if (isPublic === 1 && !shareToken) {
      shareToken = crypto.randomBytes(16).toString('hex');
    } else if (isPublic === 0) {
      shareToken = null; // Revoke shareToken if it is made private
    }
    
    db.prepare(`
      UPDATE recipes
      SET title = ?,
          description = ?,
          ingredients = ?,
          instructions = ?,
          servings = ?,
          prepTime = ?,
          cookTime = ?,
          totalTime = ?,
          image = ?,
          images = ?,
          sourceUrl = ?,
          isPublic = ?,
          shareToken = ?,
          updatedAt = ?
      WHERE id = ?
    `).run(
      req.body.title || existing.title,
      req.body.description !== undefined ? req.body.description : existing.description,
      Array.isArray(req.body.ingredients) ? JSON.stringify(req.body.ingredients) : existing.ingredients,
      Array.isArray(req.body.instructions) ? JSON.stringify(req.body.instructions) : existing.instructions,
      req.body.servings !== undefined ? req.body.servings : existing.servings,
      req.body.prepTime !== undefined ? req.body.prepTime : existing.prepTime,
      req.body.cookTime !== undefined ? req.body.cookTime : existing.cookTime,
      req.body.totalTime !== undefined ? req.body.totalTime : existing.totalTime,
      req.body.image !== undefined ? req.body.image : existing.image,
      Array.isArray(req.body.images) ? JSON.stringify(req.body.images) : existing.images,
      req.body.sourceUrl !== undefined ? req.body.sourceUrl : existing.sourceUrl,
      isPublic,
      shareToken,
      new Date().toISOString(),
      req.params.id
    );
    
    const updatedObj = formatRecipe(db.prepare('SELECT * FROM recipes WHERE id = ?').get(req.params.id));
    res.json(updatedObj);
  } catch (error) {
    console.error('Error updating recipe:', error);
    res.status(500).json({ error: 'Failed to update recipe.' });
  }
});

// API: Delete recipe
app.delete('/api/recipes/:id', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT userId FROM recipes WHERE id = ?').get(req.params.id);
    
    if (!existing) {
      return res.status(404).json({ error: 'Recipe not found' });
    }
    
    if (existing.userId !== req.user.id && !req.user.isRecipeManager && !req.user.isAdmin) {
      return res.status(403).json({ error: 'You do not have permission to delete this recipe.' });
    }
    
    db.prepare('DELETE FROM recipes WHERE id = ?').run(req.params.id);
    res.json({ success: true });
  } catch (error) {
    console.error('Error deleting recipe:', error);
    res.status(500).json({ error: 'Failed to delete recipe.' });
  }
});

// Helper: Parse recipe data from raw HTML string
function parseRecipeFromHtml(html, url) {
  const $ = cheerio.load(html);

  // Extract JSON-LD structures
  let schemaRecipe = null;
  $('script[type="application/ld+json"]').each((i, el) => {
    try {
      const text = $(el).html();
      if (!text) return;
      const data = JSON.parse(text);
      
      const searchForRecipe = (obj) => {
        if (!obj || typeof obj !== 'object') return;
        if (Array.isArray(obj)) {
          for (const item of obj) {
            searchForRecipe(item);
            if (schemaRecipe) return;
          }
        } else {
          const type = obj['@type'];
          if (type === 'Recipe' || (Array.isArray(type) && type.includes('Recipe'))) {
            schemaRecipe = obj;
            return;
          }
          for (const key in obj) {
            if (obj.hasOwnProperty(key)) {
              searchForRecipe(obj[key]);
              if (schemaRecipe) return;
            }
          }
        }
      };

      searchForRecipe(data);
    } catch (e) {
      // Safe to ignore malformed JSON script elements
    }
  });

  // Populate structured data object
  let recipeData = {
    title: '',
    description: '',
    ingredients: [],
    instructions: [],
    servings: '',
    prepTime: '',
    cookTime: '',
    totalTime: '',
    images: [],
    sourceUrl: url
  };

  // Scrape candidate image arrays from tags to enable user choice
  const extractedImages = [];
  $('img').each((i, el) => {
    const src = $(el).attr('src') || $(el).attr('data-src') || $(el).attr('data-lazy-src');
    if (src && src.startsWith('http')) {
      const width = $(el).attr('width');
      const height = $(el).attr('height');
      const isSmall = (width && parseInt(width, 10) < 150) || (height && parseInt(height, 10) < 150);
      const containsTrashKeywords = /avatar|logo|icon|spacer|ad-|banner|pixel|tracking/i.test(src);
      
      if (!isSmall && !containsTrashKeywords && !extractedImages.includes(src)) {
        extractedImages.push(src);
      }
    }
  });

  if (schemaRecipe) {
    // Standard schema-based extraction
    recipeData.title = schemaRecipe.name || $('title').text().trim() || '';
    recipeData.description = schemaRecipe.description || '';
    
    if (schemaRecipe.recipeYield) {
      recipeData.servings = Array.isArray(schemaRecipe.recipeYield) 
        ? schemaRecipe.recipeYield[0] 
        : schemaRecipe.recipeYield.toString();
    }
    
    recipeData.prepTime = parseISODuration(schemaRecipe.prepTime || '');
    recipeData.cookTime = parseISODuration(schemaRecipe.cookTime || '');
    recipeData.totalTime = parseISODuration(schemaRecipe.totalTime || '');

    const rawIngredients = schemaRecipe.recipeIngredient || schemaRecipe.recipeIngredients || [];
    if (typeof rawIngredients === 'string') {
      recipeData.ingredients = rawIngredients.split(/\r?\n/).map(i => i.trim()).filter(Boolean);
    } else if (Array.isArray(rawIngredients)) {
      recipeData.ingredients = rawIngredients;
    }

    recipeData.instructions = extractInstructions(schemaRecipe.recipeInstructions || schemaRecipe.recipeInstruction);

    const schemaImages = normalizeImages(schemaRecipe.image);
    schemaImages.forEach(img => {
      if (!extractedImages.includes(img)) {
        extractedImages.unshift(img);
      }
    });
  } else {
    // Fallback Scraping strategies if JSON-LD is missing
    recipeData.title = $('meta[property="og:title"]').attr('content') || $('title').text().trim() || '';
    recipeData.description = $('meta[property="og:description"]').attr('content') || $('meta[name="description"]').attr('content') || '';
    
    const ogImg = $('meta[property="og:image"]').attr('content');
    if (ogImg && !extractedImages.includes(ogImg)) {
      extractedImages.unshift(ogImg);
    }

    // Parse ingredients structurally
    const ingredientSelectors = [
      '.wprm-recipe-ingredient',
      '.tasty-recipes-ingredients li',
      '.tasty-recipe-ingredients li',
      '.recipe-ingredients li',
      '.ingredients-list li',
      '.ingredients li',
      '[class*="ingredients"] li',
      '[class*="ingredients"] p',
      '[class*="ingredient"] li',
      '[class*="ingredient"] p',
      'ul[class*="ingredient"] li',
      'ol[class*="ingredient"] li',
      '.cooked-ingredient',
      '.recipe-ingredients-wrap li'
    ];
    
    for (const selector of ingredientSelectors) {
      const items = $(selector);
      if (items.length > 1) {
        items.each((i, el) => {
          const txt = $(el).text().trim().replace(/\s+/g, ' ');
          if (txt && !recipeData.ingredients.includes(txt)) {
            recipeData.ingredients.push(txt);
          }
        });
        if (recipeData.ingredients.length > 1) break;
      }
    }

    // Parse instructions structurally
    const instructionSelectors = [
      '.wprm-recipe-instruction',
      '.tasty-recipes-instructions li',
      '.tasty-recipe-instructions li',
      '.recipe-directions li',
      '.recipe-instructions li',
      '.instructions-list li',
      '.instructions li',
      '.recipe-steps li',
      '[class*="instructions"] li',
      '[class*="instructions"] p',
      '[class*="directions"] li',
      '[class*="directions"] p',
      '[class*="step"] li',
      '[class*="step"] p',
      '.cooked-step',
      '.recipe-method li',
      '.recipe-method p',
      '.recipe-preparation li'
    ];

    for (const selector of instructionSelectors) {
      const items = $(selector);
      if (items.length > 1) {
        items.each((i, el) => {
          const txt = $(el).text().trim().replace(/\s+/g, ' ');
          if (txt && !recipeData.instructions.includes(txt)) {
            recipeData.instructions.push(txt);
          }
        });
        if (recipeData.instructions.length > 1) break;
      }
    }
  }

  // Heuristics Fallback: Heading Sibling crawl if still empty
  if (recipeData.ingredients.length === 0) {
    $('h1, h2, h3, h4, h5').each((i, heading) => {
      const text = $(heading).text().toLowerCase();
      if (text.includes('ingredients') && !text.includes('instruction') && !text.includes('direction')) {
        let next = $(heading).next();
        if (next.length === 0) {
          next = $(heading).parent().next();
        }
        
        let limit = 0;
        while (next.length > 0 && limit < 15) {
          if (/h[1-6]/i.test(next[0].tagName)) break;
          
          const items = next.find('li, p');
          if (items.length > 0) {
            items.each((j, el) => {
              const itemText = $(el).text().trim().replace(/\s+/g, ' ');
              if (itemText && itemText.length > 2 && itemText.length < 200 && !recipeData.ingredients.includes(itemText)) {
                recipeData.ingredients.push(itemText);
              }
            });
          } else {
            const itemText = next.text().trim().replace(/\s+/g, ' ');
            if (itemText && itemText.length > 2 && itemText.length < 200) {
              recipeData.ingredients.push(itemText);
            }
          }
          next = next.next();
          limit++;
        }
        if (recipeData.ingredients.length > 2) return false;
      }
    });
  }

  if (recipeData.instructions.length === 0) {
    $('h1, h2, h3, h4, h5').each((i, heading) => {
      const text = $(heading).text().toLowerCase();
      if (/instruction|direction|method|preparation|steps/i.test(text) && !text.includes('ingredient')) {
        let next = $(heading).next();
        if (next.length === 0) {
          next = $(heading).parent().next();
        }
        
        let limit = 0;
        while (next.length > 0 && limit < 15) {
          if (/h[1-6]/i.test(next[0].tagName)) break;
          
          const items = next.find('li, p');
          if (items.length > 0) {
            items.each((j, el) => {
              const itemText = $(el).text().trim().replace(/\s+/g, ' ');
              if (itemText && itemText.length > 5 && itemText.length < 1000 && !recipeData.instructions.includes(itemText)) {
                recipeData.instructions.push(itemText);
              }
            });
          } else {
            const itemText = next.text().trim().replace(/\s+/g, ' ');
            if (itemText && itemText.length > 5 && itemText.length < 1000) {
              recipeData.instructions.push(itemText);
            }
          }
          next = next.next();
          limit++;
        }
        if (recipeData.instructions.length > 1) return false;
      }
    });
  }

  // Heuristics Fallback: Blockquote / Multi-paragraph text classifier (for vintage/informal blogs)
  if (recipeData.ingredients.length === 0 && recipeData.instructions.length === 0) {
    const isStepText = (text) => {
      if (/^(?:step\s*)?\d+[\.\\)\-]\s/i.test(text)) return true;
      if (/^\d+\s+(?:mix|stir|combine|sift|bake|heat|preheat|fry|drain|serve|boil|chop|pour|place|add|cook|whisk|beat|divide|mould|brush|roll|repeat|prepare|cut)\b/i.test(text)) return true;
      if (/^(?:mix|stir|combine|sift|bake|heat|preheat|fry|drain|serve|boil|chop|pour|place|add|cook|whisk|beat|divide|mould|brush|roll|repeat|prepare|cut)\b/i.test(text) && text.length > 25) return true;
      return false;
    };

    const isTitleOrHeader = (text) => {
      const titleClean = recipeData.title.toLowerCase();
      const textClean = text.toLowerCase();
      if (titleClean.includes(textClean) || textClean.includes(titleClean)) return true;
      if (/^(ingredients|instructions|directions|method|results|notes|conclusion)\b/i.test(textClean)) return true;
      return false;
    };

    let selectedContainer = null;
    if ($('blockquote').length > 0) {
      selectedContainer = $('blockquote');
    } else if ($('.entry-content').length > 0) {
      selectedContainer = $('.entry-content');
    } else if ($('article').length > 0) {
      selectedContainer = $('article');
    }

    if (selectedContainer) {
      selectedContainer.each((i, container) => {
        const paragraphs = $(container).find('p');
        if (paragraphs.length > 2) {
          paragraphs.each((j, el) => {
            const html = $(el).html() || '';
            if (html.includes('<br>')) {
              const lines = html.split(/<br\s*\/?>/i)
                .map(l => cheerio.load(l).text().trim().replace(/\s+/g, ' '))
                .filter(Boolean);
              
              for (const line of lines) {
                if (isTitleOrHeader(line) || line.length < 3) continue;
                if (isStepText(line)) {
                  recipeData.instructions.push(line);
                } else {
                  recipeData.ingredients.push(line);
                }
              }
            } else {
              const txt = $(el).text().trim().replace(/\s+/g, ' ');
              if (!txt || isTitleOrHeader(txt) || txt.length < 3) return;
              
              if (isStepText(txt)) {
                recipeData.instructions.push(txt);
              } else {
                recipeData.ingredients.push(txt);
              }
            }
          });
          if (recipeData.ingredients.length > 1) return false;
        }
      });
    }
  }

  // Clean up lists
  recipeData.ingredients = recipeData.ingredients.map(i => i.trim()).filter(Boolean);
  recipeData.instructions = recipeData.instructions.map(i => i.trim()).filter(Boolean);
  recipeData.images = extractedImages.slice(0, 15);

  return recipeData;
}

// Helper: Detect if HTML response is a bot challenge page rather than real content
function isBotChallengePage(html) {
  if (!html || typeof html !== 'string') return true;
  const lowerHtml = html.toLowerCase();
  if (lowerHtml.includes('cf-browser-verification') || lowerHtml.includes('cf_chl_opt')) return true;
  if (lowerHtml.includes('checking your browser') && lowerHtml.includes('cloudflare')) return true;
  if (lowerHtml.includes('akamai') && lowerHtml.includes('bot manager')) return true;
  if (lowerHtml.includes('px-captcha') || lowerHtml.includes('perimeterx')) return true;
  if (lowerHtml.includes('access to this page has been denied')) return true;
  if (lowerHtml.includes('please verify you are a human') && !lowerHtml.includes('recipeingredient')) return true;
  if (lowerHtml.includes('recaptcha') && !lowerHtml.includes('recipeingredient')) return true;
  if (html.length < 500 && !lowerHtml.includes('recipe')) return true;
  return false;
}

// Helper: Fetch full HTML via Jina Reader HTML mode to bypass edge bot protection (PerimeterX, Cloudflare, etc.)
async function fetchWithReaderProxy(url) {
  console.log('[ReaderProxy] Fetching rendered HTML for:', url);
  const response = await axios.get(`https://r.jina.ai/${url}`, {
    headers: {
      'X-Return-Format': 'html',
      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
    },
    timeout: 20000
  });
  return response.data;
}

// Helper: Fetch HTML using Puppeteer with stealth (headless browser fallback)
async function fetchWithPuppeteer(url) {
  console.log('[Puppeteer] Launching headless browser for:', url);
  let browser = null;
  try {
    browser = await puppeteer.launch({
      headless: 'new',
      args: [
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
        '--disable-accelerated-2d-canvas',
        '--disable-gpu',
        '--window-size=1920,1080'
      ]
    });

    const page = await browser.newPage();
    await page.setViewport({ width: 1920, height: 1080 });
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
    await page.setExtraHTTPHeaders({
      'Accept-Language': 'en-US,en;q=0.9',
      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
    });

    await page.goto(url, { 
      waitUntil: 'networkidle0', 
      timeout: 30000 
    });

    await new Promise(resolve => setTimeout(resolve, 2000));
    await page.evaluate(() => {
      window.scrollTo(0, document.body.scrollHeight / 2);
    });
    await new Promise(resolve => setTimeout(resolve, 1000));

    const html = await page.content();
    console.log('[Puppeteer] Successfully retrieved', html.length, 'bytes of HTML');
    return html;
  } finally {
    if (browser) {
      await browser.close();
      console.log('[Puppeteer] Browser closed');
    }
  }
}

// -------------------------------------------------------------
// TikTok Recipe Extraction (Option 1: Caption & Metadata Extraction)
// -------------------------------------------------------------

// Helper: Check if a URL belongs to TikTok
function isTikTokUrl(url) {
  try {
    const parsed = new URL(url);
    const host = parsed.hostname.toLowerCase();
    return host.includes('tiktok.com');
  } catch (e) {
    return false;
  }
}

// Helper: Clean raw TikTok caption by stripping trailing hashtags and emojis
function cleanTikTokCaption(raw) {
  if (!raw) return '';
  let text = raw.replace(/(?:\s*#[^\s#]+)+\s*$/g, '').trim();
  text = text.replace(/[\uFE00-\uFE0F\uD83C-\uDBFF\uDC00-\uDFFF\u2600-\u26FF\u2700-\u27BF]+$/, '').trim();
  return text;
}

// Helper: Split single-line or unstructured social captions into readable candidate phrases
function splitCaptionIntoPhrases(text) {
  const breakPhrases = [
    'Mix it', 'Whisk', 'Add in', 'Add the', 'Add one', 'In a separate bowl:?', 'In a separate',
    'In a measuring cup', 'In a greased and floured', 'In a greased', 'In a large bowl',
    'For the apple:?', 'For the glaze:?', 'For the sauce:?', 'For the dough:?',
    'Then add', 'Then other', 'Then 1/2', 'Swirl it', 'Sprinkle',
    'Bake at', 'Glaze:?'
  ];

  let working = text;
  for (const phrase of breakPhrases) {
    const re = new RegExp('\\b(' + phrase + ')', 'gi');
    working = working.replace(re, '\n$1');
  }

  const qtyBreakRegex = /(?<=[a-zA-Z\)\,\.\!])\s+(?=(?:(?:\d+\s*(?:&|\+)?\s*)?(?:\d+[\/\.]\d+|[¼½¾⅓⅔⅛⅜⅝⅞]|\d+)\s*(?:cups?|c|tbsp|tablespoons?|tsp|teaspoons?|oz|ounces?|lbs?|sticks?|large|small|medium|cans?)\b))/gi;
  working = working.replace(qtyBreakRegex, '\n');

  return working.split('\n').map(s => s.trim()).filter(Boolean);
}

// Helper: Heuristic parser to extract recipe structure from a social video caption
function parseRecipeCaptionHeuristic(rawCaption, author, cover, sourceUrl) {
  const caption = cleanTikTokCaption(rawCaption);
  if (!caption) return null;

  let title = '';
  let description = '';
  const ingredients = [];
  const instructions = [];

  const ingHeaderMatch = caption.match(/\b(?:ingredients?|what you(?:'ll)? need)\s*[:\-]?/i);
  let remainingText = caption;

  if (ingHeaderMatch) {
    const rawTitle = caption.slice(0, ingHeaderMatch.index).trim();
    title = rawTitle.replace(/[:\-.]+$/, '').trim();
    remainingText = caption.slice(ingHeaderMatch.index + ingHeaderMatch[0].length).trim();
  } else {
    const firstSentence = caption.match(/^([^.!?\n]+[.!?])/);
    title = firstSentence ? firstSentence[1].replace(/[:\-.]+$/, '').trim() : caption.slice(0, 50).trim();
    remainingText = caption.slice(title.length).trim();
  }

  const segments = splitCaptionIntoPhrases(remainingText);

  const instructionStarters = /^(?:mix|whisk|add|bake|stir|fold|combine|in a|then|swirl|sprinkle|pour|heat|cook|chill|serve|place|cut|slice|dice)\b/i;
  const isIngredientPattern = /^(?:(?:\d+\s*(?:&|\+)?\s*)?(?:\d+[\/\.]\d+|[¼½¾⅓⅔⅛⅜⅝⅞]|\d+)\s*(?:cups?|c|tbsp|tablespoons?|tsp|teaspoons?|oz|ounces?|lbs?|sticks?|large|small|medium|cans?|pinch|pkg)?\b|[•◦⁃∙*\-+])/i;

  for (let seg of segments) {
    seg = seg.trim().replace(/^[:\-•◦⁃∙*\-+]\s*/, '').replace(/#[\w-]+/g, '').trim();
    if (!seg || seg.length < 2) continue;

    if (instructionStarters.test(seg) || /^\d+[\.\)]\s+/.test(seg)) {
      instructions.push(seg.replace(/^\d+[\.\)]\s*/, ''));
    } else if (isIngredientPattern.test(seg)) {
      if (/batter|pan/i.test(seg)) {
        instructions.push(seg);
      } else {
        ingredients.push(seg);
      }
    } else if (/^(?:glaze|for the )/i.test(seg)) {
      instructions.push(seg);
    } else {
      if (/(?:bowl|pan|oven|bake|mins?|minutes?|degrees?|350|mix|batter)/i.test(seg)) {
        instructions.push(seg);
      } else {
        ingredients.push(seg);
      }
    }
  }

  let cookTime = '';
  const bakeMatch = caption.match(/\b(?:bake(?:\s+at\s+\d+)?\s+(?:for\s+)?(\d+[\s\-to]+\d+\s*(?:mins?|minutes?|hours?|hrs?)))\b/i);
  if (bakeMatch) {
    cookTime = bakeMatch[1].trim();
  }

  return {
    title: title || (author ? `Recipe by ${author}` : 'TikTok Recipe'),
    description: caption.slice(0, 160).trim(),
    image: cover || '',
    images: cover ? [cover] : [],
    servings: '',
    prepTime: '',
    cookTime,
    totalTime: cookTime,
    ingredients,
    instructions,
    sourceUrl: sourceUrl || ''
  };
}

// Helper: Scrape recipe from a TikTok video URL using caption extraction and AI parsing
async function scrapeTikTokRecipe(inputUrl) {
  console.log('[TikTok] Starting scrape for:', inputUrl);

  // 1. Resolve redirect for shortened links (vm.tiktok.com, vt.tiktok.com)
  let canonicalUrl = inputUrl;
  try {
    const headRes = await axios.get(inputUrl, {
      maxRedirects: 5,
      validateStatus: () => true,
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
      },
      timeout: 8000
    });
    if (headRes.request?.res?.responseUrl) {
      canonicalUrl = headRes.request.res.responseUrl;
    }
  } catch (e) {
    console.warn('[TikTok] Redirect resolution note:', e.message);
  }

  // 2. Fetch page HTML (Strategy 1: Fast Axios fetch)
  let html = null;
  try {
    const response = await axios.get(canonicalUrl, {
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.9',
        'Referer': 'https://www.tiktok.com/'
      },
      timeout: 12000
    });
    html = response.data;
  } catch (err) {
    console.warn('[TikTok] Direct fetch failed, trying Puppeteer:', err.message);
  }

  // Strategy 2: Puppeteer fallback if direct fetch failed or returned bot screen
  if (!html || isBotChallengePage(html)) {
    try {
      html = await fetchWithPuppeteer(canonicalUrl);
    } catch (pupErr) {
      console.error('[TikTok] Puppeteer fallback failed:', pupErr.message);
    }
  }

  if (!html) {
    throw new Error('Could not access TikTok video page. Automated access was blocked.');
  }

  let caption = '';
  let author = '';
  let cover = '';

  // Extract from rehydration JSON
  const rehydrationMatch = html.match(/<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)<\/script>/s);
  if (rehydrationMatch) {
    try {
      const data = JSON.parse(rehydrationMatch[1]);
      const item = data.__DEFAULT_SCOPE__?.['webapp.video-detail']?.itemInfo?.itemStruct;
      if (item) {
        caption = item.desc || '';
        author = item.author?.nickname || item.author?.uniqueId || '';
        cover = item.video?.cover || item.video?.dynamicCover || '';
      }
    } catch (parseErr) {
      console.warn('[TikTok] Failed to parse rehydration JSON:', parseErr.message);
    }
  }

  // Fallback to meta tags / DOM selectors
  if (!caption) {
    const $ = cheerio.load(html);
    caption = $('[data-e2e="browse-video-desc"]').text().trim() ||
              $('meta[property="og:description"]').attr('content') ||
              $('meta[name="description"]').attr('content') || '';
    if (caption.includes('Visit TikTok to discover')) caption = '';
  }
  if (!cover) {
    const $ = cheerio.load(html);
    cover = $('meta[property="og:image"]').attr('content') || '';
  }
  if (!author) {
    const $ = cheerio.load(html);
    author = $('[data-e2e="browser-nickname"]').text().trim() ||
             $('[data-e2e="user-title"]').text().trim() || '';
  }

  console.log('[TikTok] Extracted caption length:', caption.length, 'Author:', author);
  if (!caption) {
    throw new Error('No caption or description found on this TikTok video. The creator might not have written recipe details.');
  }

  // Fast & Local Heuristic Engine First (Zero API cost, instant execution)
  console.log('[TikTok] Running local heuristic parser on video caption...');
  const heuristicResult = parseRecipeCaptionHeuristic(caption, author, cover, canonicalUrl);
  if (!heuristicResult.ingredients || heuristicResult.ingredients.length === 0) {
    throw new Error('This TikTok caption does not appear to contain an ingredients list. The recipe might be in the audio, video comments, or on-screen text.');
  }

  heuristicResult._scrapingMethod = 'tiktok-caption-heuristic';
  heuristicResult.rawCaption = caption;
  heuristicResult.author = author;
  heuristicResult.suggestAiPolish = true;
  return heuristicResult;
}

// API: Polish and format recipe using Gemini AI (User-initiated)
app.post('/api/recipes/ai-polish', requireAuth, async (req, res) => {
  const { rawCaption, title, ingredients, instructions } = req.body;

  const settingsRow = db.prepare('SELECT geminiApiKey, geminiModel FROM settings LIMIT 1').get();
  const geminiKey = settingsRow?.geminiApiKey?.trim();
  const geminiModel = settingsRow?.geminiModel?.trim() || 'gemini-2.0-flash';
  const isPlaceholder = !geminiKey || geminiKey === 'PASTE_YOUR_GEMINI_API_KEY_HERE';

  if (isPlaceholder) {
    return res.status(400).json({
      error: 'Google Gemini API Key is not configured. Please open Settings (top right gear icon) and add your Gemini API key to enable AI text polishing.'
    });
  }

  try {
    const promptText = `You are a culinary expert assistant for a recipe app. Clean up, format, and organize the following recipe.
- Standardize ingredient measurements (amounts, units, descriptions) onto separate lines.
- Turn run-on or conversational directions into clean, ordered, numbered steps.
- Infer or estimate prep time, cook time, and servings if mentioned or implied.
- Clean up any hashtags, social media slang, or promotional sponsor notes.

Source Text / Current Draft:
Title: ${title || ''}
${rawCaption ? `Original Caption:\n"""\n${rawCaption}\n"""` : ''}
Ingredients so far:
${Array.isArray(ingredients) ? ingredients.join('\n') : (ingredients || '')}
Instructions so far:
${Array.isArray(instructions) ? instructions.join('\n') : (instructions || '')}

Return ONLY a valid JSON object matching this schema without markdown code blocks:
{
  "title": "Cleaned Recipe Title",
  "description": "Short description of the recipe",
  "servings": "e.g., 4",
  "prepTime": "e.g., 15 mins",
  "cookTime": "e.g., 45 mins",
  "totalTime": "e.g., 1 hour",
  "ingredients": ["ingredient line 1", "ingredient line 2"],
  "instructions": ["Step 1", "Step 2"]
}`;

    const geminiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${geminiModel}:generateContent?key=${geminiKey}`;
    const aiRes = await axios.post(geminiUrl, {
      contents: [{ parts: [{ text: promptText }] }],
      generationConfig: { responseMimeType: "application/json" }
    }, {
      headers: { 'Content-Type': 'application/json' },
      timeout: 15000
    });

    if (aiRes.data && aiRes.data.candidates?.[0]?.content?.parts?.[0]?.text) {
      const rawText = aiRes.data.candidates[0].content.parts[0].text;
      const parsed = JSON.parse(rawText);
      return res.json(parsed);
    } else {
      throw new Error('Gemini returned an empty response.');
    }
  } catch (err) {
    console.error('[AI Polish] Error:', err.response?.data?.error?.message || err.message);
    const msg = err.response?.data?.error?.message || err.message;
    return res.status(500).json({ error: `AI Polish failed: ${msg}` });
  }
});

// API: Load System Settings & User Preferences
app.get('/api/settings', requireAuth, (req, res) => {
  try {
    const settings = db.prepare('SELECT geminiApiKey, geminiModel FROM settings LIMIT 1').get();
    const userRow = db.prepare('SELECT defaultConsolidateLists FROM users WHERE id = ?').get(req.user.id);
    const defaultConsolidateLists = userRow ? userRow.defaultConsolidateLists !== 0 : true;
    res.json({
      geminiApiKey: settings ? settings.geminiApiKey : '',
      geminiModel: (settings && settings.geminiModel) ? settings.geminiModel : 'gemini-2.0-flash',
      defaultConsolidateLists
    });
  } catch (e) {
    console.error('Failed to load settings:', e);
    res.status(500).json({ error: 'Failed to read settings' });
  }
});

// API: Save System Settings & User Preferences
app.post('/api/settings', requireAuth, (req, res) => {
  const { geminiApiKey, geminiModel, defaultConsolidateLists } = req.body;
  try {
    if (geminiApiKey !== undefined) {
      db.prepare('UPDATE settings SET geminiApiKey = ?').run(geminiApiKey || '');
    }
    if (geminiModel !== undefined) {
      db.prepare('UPDATE settings SET geminiModel = ?').run(geminiModel || 'gemini-2.0-flash');
    }
    if (typeof defaultConsolidateLists === 'boolean') {
      db.prepare('UPDATE users SET defaultConsolidateLists = ? WHERE id = ?').run(defaultConsolidateLists ? 1 : 0, req.user.id);
      req.user.defaultConsolidateLists = defaultConsolidateLists;
    }
    const settings = db.prepare('SELECT geminiApiKey, geminiModel FROM settings LIMIT 1').get();
    const userRow = db.prepare('SELECT defaultConsolidateLists FROM users WHERE id = ?').get(req.user.id);
    res.json({
      geminiApiKey: settings ? settings.geminiApiKey : '',
      geminiModel: (settings && settings.geminiModel) ? settings.geminiModel : 'gemini-2.0-flash',
      defaultConsolidateLists: userRow ? userRow.defaultConsolidateLists !== 0 : true
    });
  } catch (e) {
    console.error('Failed to save settings:', e);
    res.status(500).json({ error: 'Failed to save settings' });
  }
});

// API: Discover available Gemini models using the saved or provided API key
app.post('/api/settings/discover-models', requireAuth, async (req, res) => {
  let { apiKey } = req.body || {};
  if (!apiKey) {
    const settingsRow = db.prepare('SELECT geminiApiKey FROM settings LIMIT 1').get();
    apiKey = settingsRow?.geminiApiKey?.trim();
  }
  if (!apiKey || apiKey === 'PASTE_YOUR_GEMINI_API_KEY_HERE') {
    return res.status(400).json({ error: 'Please enter a Gemini API Key first.' });
  }

  try {
    const response = await axios.get(`https://generativelanguage.googleapis.com/v1beta/models?key=${apiKey}`, {
      timeout: 12000
    });

    if (response.data && Array.isArray(response.data.models)) {
      const models = response.data.models
        .filter(m => Array.isArray(m.supportedGenerationMethods) && m.supportedGenerationMethods.includes('generateContent'))
        .filter(m => !m.name.includes('embedding') && !m.name.includes('aqa') && !m.name.includes('imagen'))
        .map(m => {
          const modelId = m.name.replace(/^models\//, '');
          return {
            id: modelId,
            displayName: m.displayName || modelId,
            description: m.description || ''
          };
        });

      // Sort models: Flash models first, then by ID
      models.sort((a, b) => {
        const aIsFlash = a.id.toLowerCase().includes('flash');
        const bIsFlash = b.id.toLowerCase().includes('flash');
        if (aIsFlash && !bIsFlash) return -1;
        if (!aIsFlash && bIsFlash) return 1;
        return a.id.localeCompare(b.id);
      });

      return res.json({ models });
    } else {
      return res.status(500).json({ error: 'Unexpected response from Google Gemini API.' });
    }
  } catch (err) {
    console.error('Model discovery error:', err.response?.data?.error?.message || err.message);
    const msg = err.response?.data?.error?.message || err.message;
    return res.status(err.response?.status || 500).json({ error: `Failed to fetch models: ${msg}` });
  }
});

// API: Update User Preferences directly
app.put('/api/user/preferences', requireAuth, (req, res) => {
  const { defaultConsolidateLists } = req.body;
  if (typeof defaultConsolidateLists !== 'boolean') {
    return res.status(400).json({ error: 'defaultConsolidateLists must be a boolean.' });
  }
  try {
    db.prepare('UPDATE users SET defaultConsolidateLists = ? WHERE id = ?').run(defaultConsolidateLists ? 1 : 0, req.user.id);
    req.user.defaultConsolidateLists = defaultConsolidateLists;
    res.json({ success: true, defaultConsolidateLists });
  } catch (e) {
    console.error('Failed to update user preferences:', e);
    res.status(500).json({ error: 'Failed to update preferences.' });
  }
});

// -------------------------------------------------------------
// SHOPPING LIST PARSER, CLASSIFIER & API ENDPOINTS
// -------------------------------------------------------------

const UNIT_ALIASES = {
  tablespoon: 'tbsp', tablespoons: 'tbsp', tbsp: 'tbsp', tbs: 'tbsp', tb: 'tbsp',
  teaspoon: 'tsp', teaspoons: 'tsp', tsp: 'tsp',
  cup: 'cup', cups: 'cup', c: 'cup',
  ounce: 'oz', ounces: 'oz', oz: 'oz',
  pound: 'lb', pounds: 'lb', lb: 'lb', lbs: 'lb',
  gram: 'g', grams: 'g', g: 'g',
  kilogram: 'kg', kilograms: 'kg', kg: 'kg',
  milliliter: 'ml', milliliters: 'ml', ml: 'ml',
  liter: 'l', liters: 'l', l: 'l',
  clove: 'clove', cloves: 'clove',
  can: 'can', cans: 'can',
  jar: 'jar', jars: 'jar',
  bunch: 'bunch', bunches: 'bunch',
  pinch: 'pinch', pinches: 'pinch',
  dash: 'dash', dashes: 'dash',
  slice: 'slice', slices: 'slice',
  piece: 'piece', pieces: 'piece',
  head: 'head', heads: 'head',
  stalk: 'stalk', stalks: 'stalk',
  sprig: 'sprig', sprigs: 'sprig',
  package: 'pkg', packages: 'pkg', pkg: 'pkg', pkgs: 'pkg',
  stick: 'stick', sticks: 'stick'
};

const AISLE_KEYWORDS = [
  {
    category: 'Produce',
    keywords: [
      'onion', 'shallot', 'scallion', 'garlic', 'ginger', 'potato', 'sweet potato', 'carrot', 'celery',
      'bell pepper', 'jalapeno', 'chili pepper', 'poblano', 'habanero', 'serrano', 'tomato', 'cherry tomato',
      'lettuce', 'spinach', 'kale', 'arugula', 'cabbage', 'broccoli', 'cauliflower', 'zucchini', 'squash',
      'cucumber', 'mushroom', 'avocado', 'corn', 'asparagus', 'green bean', 'pea', 'eggplant', 'radish',
      'beet', 'turnip', 'leek', 'fennel', 'artichoke', 'cilantro', 'parsley', 'basil', 'mint', 'rosemary',
      'thyme', 'dill', 'chive', 'sage', 'apple', 'banana', 'lemon', 'lime', 'orange', 'grapefruit',
      'berry', 'strawberry', 'blueberry', 'raspberry', 'blackberry', 'grape', 'peach', 'pear', 'mango',
      'pineapple', 'watermelon', 'melon', 'pomegranate', 'fig', 'date'
    ]
  },
  {
    category: 'Meat & Seafood',
    keywords: [
      'beef', 'ground beef', 'steak', 'chuck', 'sirloin', 'brisket', 'roast', 'pork', 'bacon', 'sausage',
      'ham', 'prosciutto', 'pancetta', 'chorizo', 'pork chop', 'tenderloin', 'chicken', 'turkey', 'duck',
      'lamb', 'veal', 'salmon', 'tuna', 'cod', 'halibut', 'tilapia', 'trout', 'mahimahi', 'sea bass',
      'shrimp', 'prawn', 'scallop', 'crab', 'lobster', 'clam', 'mussel', 'oyster', 'anchovy', 'fish'
    ]
  },
  {
    category: 'Dairy & Eggs',
    keywords: [
      'milk', 'cream', 'heavy cream', 'half-and-half', 'half and half', 'butter', 'margarine', 'egg',
      'cheese', 'cheddar', 'mozzarella', 'parmesan', 'pecorino', 'feta', 'ricotta', 'gouda', 'gruyere',
      'swiss', 'provolone', 'brie', 'goat cheese', 'cream cheese', 'sour cream', 'yogurt', 'greek yogurt',
      'buttermilk', 'whipped cream', 'ghee', 'mascarpone'
    ]
  },
  {
    category: 'Canned & Jarred',
    keywords: [
      'canned', 'crushed tomato', 'diced tomato', 'tomato paste', 'tomato sauce', 'marinara', 'salsa',
      'kidney bean', 'black bean', 'pinto bean', 'cannellini', 'chickpea', 'garbanzo', 'lentil',
      'broth', 'stock', 'chicken broth', 'beef broth', 'vegetable broth', 'coconut milk', 'olives',
      'capers', 'pickle', 'jam', 'jelly', 'peanut butter', 'almond butter', 'tahini', 'artichoke heart',
      'roasted red pepper', 'chipotle in adobo', 'beans'
    ]
  },
  {
    category: 'Bakery & Grains',
    keywords: [
      'bread', 'loaf', 'baguette', 'ciabatta', 'sourdough', 'roll', 'bun', 'tortilla', 'pita', 'naan',
      'bagel', 'croissant', 'english muffin', 'pasta', 'spaghetti', 'penne', 'fettuccine', 'linguine',
      'macaroni', 'lasagna', 'orzo', 'noodle', 'ramen', 'udon', 'rice', 'jasmine rice', 'basmati',
      'arborio', 'brown rice', 'quinoa', 'couscous', 'barley', 'oats', 'oatmeal', 'breadcrumb', 'panko',
      'cracker', 'taco shell', 'tortilla chip'
    ]
  },
  {
    category: 'Pantry & Spices',
    keywords: [
      'olive oil', 'vegetable oil', 'canola oil', 'sesame oil', 'coconut oil', 'avocado oil', 'oil',
      'vinegar', 'apple cider vinegar', 'balsamic', 'red wine vinegar', 'white wine vinegar', 'rice vinegar',
      'soy sauce', 'tamari', 'fish sauce', 'worcestershire', 'hot sauce', 'sriracha', 'mustard', 'dijon',
      'ketchup', 'mayonnaise', 'mayo', 'honey', 'maple syrup', 'molasses', 'agave', 'sugar', 'brown sugar',
      'powdered sugar', 'confectioners', 'flour', 'all-purpose flour', 'cornstarch', 'baking powder',
      'baking soda', 'yeast', 'vanilla', 'cocoa', 'chocolate', 'chip', 'nut', 'almond', 'walnut', 'pecan',
      'cashew', 'peanut', 'pine nut', 'pistachio', 'seed', 'sesame', 'chia', 'flax', 'sunflower',
      'salt', 'kosher salt', 'sea salt', 'black pepper', 'pepper', 'chili powder', 'cumin', 'paprika',
      'smoked paprika', 'cayenne', 'oregano', 'garlic powder', 'onion powder', 'cinnamon', 'nutmeg',
      'cloves', 'allspice', 'cardamom', 'turmeric', 'curry', 'coriander', 'bay leaf', 'bay leaves',
      'red pepper flakes', 'italian seasoning', 'thyme', 'rosemary', 'spice', 'seasoning', 'bouillon'
    ]
  },
  {
    category: 'Frozen',
    keywords: [
      'frozen', 'ice cream', 'puff pastry', 'pie crust', 'frozen pea', 'frozen corn', 'frozen berry',
      'edamame', 'tater tot', 'fries'
    ]
  }
];

function classifyGroceryCategory(itemName, rawText) {
  const haystack = `${itemName} ${rawText}`.toLowerCase();
  // Check Canned & Jarred first if "can" or "canned" or "jar" is explicitly present
  if (/\b(can|cans|canned|jar|jars|jarred)\b/.test(haystack)) {
    for (const kw of AISLE_KEYWORDS.find(a => a.category === 'Canned & Jarred').keywords) {
      if (haystack.includes(kw)) return 'Canned & Jarred';
    }
  }
  for (const group of AISLE_KEYWORDS) {
    for (const kw of group.keywords) {
      const regex = new RegExp(`\\b${kw.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}s?\\b`, 'i');
      if (regex.test(haystack)) {
        return group.category;
      }
    }
  }
  return 'Other';
}

function parseIngredientLine(rawInput) {
  const rawText = (rawInput || '').trim().replace(/\s+/g, ' ');
  if (!rawText) {
    return { rawText: '', itemName: '', quantity: null, unit: '', category: 'Other' };
  }

  // Normalize unicode vulgar fractions for parsing
  const unicodeMap = {
    '½': ' 1/2', '⅓': ' 1/3', '⅔': ' 2/3', '¼': ' 1/4', '¾': ' 3/4',
    '⅕': ' 1/5', '⅖': ' 2/5', '⅗': ' 3/5', '⅘': ' 4/5', '⅙': ' 1/6', '⅚': ' 5/6', '⅛': ' 1/8', '⅜': ' 3/8', '⅝': ' 5/8', '⅞': ' 7/8'
  };
  let working = rawText.replace(/[½⅓⅔¼¾⅕⅖⅗⅘⅙⅚⅛⅜⅝⅞]/g, m => unicodeMap[m] || m).trim().replace(/\s+/g, ' ');

  // Remove parenthetical notes like "(about 1 1/2 cups)" or "(15-ounce)"
  let withoutParens = working.replace(/\([^)]*\)/g, ' ').replace(/\s+/g, ' ').trim();

  let quantity = null;
  let rest = withoutParens;

  // Match mixed fraction "1 1/2", simple fraction "3/4", range "1-2" or "1 to 2", or decimal/integer
  const mixedMatch = rest.match(/^(\d+)\s+(\d+)\/(\d+)\b\s*(.*)/);
  const fracMatch = !mixedMatch && rest.match(/^(\d+)\/(\d+)\b\s*(.*)/);
  const rangeMatch = !mixedMatch && !fracMatch && rest.match(/^(\d+(?:\.\d+)?)\s*(?:-|to)\s*(\d+(?:\.\d+)?)\b\s*(.*)/i);
  const numMatch = !mixedMatch && !fracMatch && !rangeMatch && rest.match(/^(\d+(?:\.\d+)?)\b\s*(.*)/);

  if (mixedMatch) {
    quantity = parseInt(mixedMatch[1], 10) + (parseInt(mixedMatch[2], 10) / parseInt(mixedMatch[3], 10));
    rest = mixedMatch[4];
  } else if (fracMatch) {
    quantity = parseInt(fracMatch[1], 10) / parseInt(fracMatch[2], 10);
    rest = fracMatch[3];
  } else if (rangeMatch) {
    quantity = parseFloat(rangeMatch[2]);
    rest = rangeMatch[3];
  } else if (numMatch) {
    quantity = parseFloat(numMatch[1]);
    rest = numMatch[2];
  }

  if (quantity !== null && (!isFinite(quantity) || isNaN(quantity))) {
    quantity = null;
  }

  // Extract unit if first token matches UNIT_ALIASES
  let unit = '';
  const unitTokenMatch = rest.match(/^([a-zA-Z]+)\.?\b\s*(.*)/);
  if (unitTokenMatch) {
    const candidate = unitTokenMatch[1].toLowerCase();
    if (UNIT_ALIASES[candidate]) {
      unit = UNIT_ALIASES[candidate];
      rest = unitTokenMatch[2].replace(/^of\b\s*/i, '');
    }
  }

  // Clean itemName: strip trailing prep instructions after comma (e.g. ", diced", ", finely chopped")
  let itemName = rest.split(',')[0].trim();
  // Remove leading descriptors like "medium ", "large ", "small ", "fresh ", "boneless skinless "
  itemName = itemName
    .replace(/^(?:of\s+|medium\s+|large\s+|small\s+|extra-large\s+|fresh\s+|dried\s+|finely\s+|chopped\s+|diced\s+|minced\s+|sliced\s+|grated\s+|shredded\s+)+/i, '')
    .trim()
    .toLowerCase();

  // Basic singularization for better consolidation matching
  if (itemName.length > 4) {
    if (itemName.endsWith('tomatoes')) itemName = itemName.slice(0, -2);
    else if (itemName.endsWith('potatoes')) itemName = itemName.slice(0, -2);
    else if (itemName.endsWith('berries')) itemName = itemName.slice(0, -3) + 'y';
    else if (itemName.endsWith('leaves')) itemName = itemName.slice(0, -3) + 'f';
    else if (itemName.endsWith('s') && !itemName.endsWith('ss') && !itemName.endsWith('us') && !itemName.endsWith('beans') && !itemName.endsWith('oats') && !itemName.endsWith('flakes') && !itemName.endsWith('noodles') && !itemName.endsWith('chips')) {
      itemName = itemName.slice(0, -1);
    }
  }

  if (!itemName) {
    itemName = rawText.toLowerCase();
  }

  const category = classifyGroceryCategory(itemName, rawText);
  return {
    rawText,
    itemName,
    quantity: quantity !== null ? Math.round(quantity * 100) / 100 : null,
    unit,
    category
  };
}

function formatShoppingList(listRow) {
  if (!listRow) return null;
  const items = db.prepare('SELECT * FROM shopping_list_items WHERE listId = ? ORDER BY createdAt ASC').all(listRow.id).map(item => ({
    ...item,
    checked: item.checked === 1
  }));

  let visitorCount = 0;
  if (listRow.shareToken) {
    try {
      const stats = db.prepare('SELECT COUNT(*) as count FROM shopping_list_shares_visits WHERE shareToken = ?').get(listRow.shareToken);
      visitorCount = stats ? stats.count : 0;
    } catch (e) {
      console.error('Failed to query shopping list visitor count:', e);
    }
  }

  return {
    ...listRow,
    items,
    visitorCount
  };
}

// API: Get all shopping lists for current user (creates a default list if none exist)
app.get('/api/shopping-lists', requireAuth, (req, res) => {
  try {
    let rows = db.prepare('SELECT * FROM shopping_lists WHERE userId = ? ORDER BY createdAt ASC').all(req.user.id);
    if (rows.length === 0) {
      const defaultId = 'sl_' + Date.now().toString(36) + Math.random().toString(36).substr(2, 5);
      const now = new Date().toISOString();
      db.prepare(`
        INSERT INTO shopping_lists (id, userId, createdBy, name, shareToken, createdAt, updatedAt)
        VALUES (?, ?, ?, ?, NULL, ?, NULL)
      `).run(defaultId, req.user.id, req.user.username, 'Weekly Groceries', now);
      rows = db.prepare('SELECT * FROM shopping_lists WHERE userId = ? ORDER BY createdAt ASC').all(req.user.id);
    }
    res.json(rows.map(formatShoppingList));
  } catch (error) {
    console.error('Error fetching shopping lists:', error);
    res.status(500).json({ error: 'Failed to retrieve shopping lists.' });
  }
});

// API: Create a new shopping list
app.post('/api/shopping-lists', requireAuth, (req, res) => {
  const name = (req.body.name || '').trim();
  if (!name) {
    return res.status(400).json({ error: 'Shopping list name is required.' });
  }
  try {
    const id = 'sl_' + Date.now().toString(36) + Math.random().toString(36).substr(2, 5);
    const now = new Date().toISOString();
    db.prepare(`
      INSERT INTO shopping_lists (id, userId, createdBy, name, shareToken, createdAt, updatedAt)
      VALUES (?, ?, ?, ?, NULL, ?, NULL)
    `).run(id, req.user.id, req.user.username, name, now);
    const created = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(id));
    res.status(201).json(created);
  } catch (error) {
    console.error('Error creating shopping list:', error);
    res.status(500).json({ error: 'Failed to create shopping list.' });
  }
});

// API: Rename a shopping list
app.put('/api/shopping-lists/:id', requireAuth, (req, res) => {
  const name = (req.body.name || '').trim();
  if (!name) {
    return res.status(400).json({ error: 'Shopping list name is required.' });
  }
  try {
    const existing = db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(req.params.id);
    if (!existing) return res.status(404).json({ error: 'Shopping list not found.' });
    if (existing.userId !== req.user.id && !req.user.isAdmin) {
      return res.status(403).json({ error: 'Permission denied.' });
    }
    db.prepare('UPDATE shopping_lists SET name = ?, updatedAt = ? WHERE id = ?').run(name, new Date().toISOString(), req.params.id);
    const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(req.params.id));
    res.json(updated);
  } catch (error) {
    console.error('Error renaming shopping list:', error);
    res.status(500).json({ error: 'Failed to rename shopping list.' });
  }
});

// API: Delete a shopping list
app.delete('/api/shopping-lists/:id', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(req.params.id);
    if (!existing) return res.status(404).json({ error: 'Shopping list not found.' });
    if (existing.userId !== req.user.id && !req.user.isAdmin) {
      return res.status(403).json({ error: 'Permission denied.' });
    }
    const delTx = db.transaction(() => {
      if (existing.shareToken) {
        db.prepare('DELETE FROM shopping_list_shares_visits WHERE shareToken = ?').run(existing.shareToken);
      }
      db.prepare('DELETE FROM shopping_lists WHERE id = ?').run(req.params.id);
    });
    delTx();
    res.json({ success: true });
  } catch (error) {
    console.error('Error deleting shopping list:', error);
    res.status(500).json({ error: 'Failed to delete shopping list.' });
  }
});

// API: Add one or more items to a shopping list
app.post('/api/shopping-lists/:id/items', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(req.params.id);
    if (!existing) return res.status(404).json({ error: 'Shopping list not found.' });
    if (existing.userId !== req.user.id && !req.user.isAdmin) {
      return res.status(403).json({ error: 'Permission denied.' });
    }

    const rawItems = Array.isArray(req.body.items)
      ? req.body.items
      : (req.body.text ? [req.body.text] : []);
    const recipeId = req.body.recipeId || null;
    const recipeTitle = req.body.recipeTitle || null;

    if (rawItems.length === 0) {
      return res.status(400).json({ error: 'No items provided.' });
    }

    const insertStmt = db.prepare(`
      INSERT INTO shopping_list_items (
        id, listId, rawText, itemName, quantity, unit, category, recipeId, recipeTitle, checked, createdAt
      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?)
    `);

    const now = new Date().toISOString();
    const insertMany = db.transaction((lines) => {
      lines.forEach((line, idx) => {
        if (!line || typeof line !== 'string' || !line.trim()) return;
        const parsed = parseIngredientLine(line);
        const itemId = 'sli_' + Date.now().toString(36) + '_' + idx + '_' + Math.random().toString(36).substr(2, 4);
        insertStmt.run(
          itemId,
          existing.id,
          parsed.rawText,
          parsed.itemName,
          parsed.quantity,
          parsed.unit,
          parsed.category,
          recipeId,
          recipeTitle,
          now
        );
      });
      db.prepare('UPDATE shopping_lists SET updatedAt = ? WHERE id = ?').run(now, existing.id);
    });

    insertMany(rawItems);
    const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
    res.status(201).json(updated);
  } catch (error) {
    console.error('Error adding shopping list items:', error);
    res.status(500).json({ error: 'Failed to add items to shopping list.' });
  }
});

// API: Toggle checked state for one or multiple shopping list items
app.put('/api/shopping-lists/:id/items/check', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(req.params.id);
    if (!existing) return res.status(404).json({ error: 'Shopping list not found.' });
    if (existing.userId !== req.user.id && !req.user.isAdmin) {
      return res.status(403).json({ error: 'Permission denied.' });
    }

    const itemIds = Array.isArray(req.body.itemIds) ? req.body.itemIds : [];
    const checked = req.body.checked ? 1 : 0;
    if (itemIds.length === 0) {
      return res.status(400).json({ error: 'itemIds array is required.' });
    }

    const updateStmt = db.prepare('UPDATE shopping_list_items SET checked = ? WHERE id = ? AND listId = ?');
    const tx = db.transaction((ids) => {
      for (const itemId of ids) {
        updateStmt.run(checked, itemId, existing.id);
      }
      db.prepare('UPDATE shopping_lists SET updatedAt = ? WHERE id = ?').run(new Date().toISOString(), existing.id);
    });
    tx(itemIds);

    const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
    res.json(updated);
  } catch (error) {
    console.error('Error toggling shopping list items:', error);
    res.status(500).json({ error: 'Failed to update item state.' });
  }
});

// API: Delete one or multiple items from a shopping list
app.post('/api/shopping-lists/:id/items/delete', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(req.params.id);
    if (!existing) return res.status(404).json({ error: 'Shopping list not found.' });
    if (existing.userId !== req.user.id && !req.user.isAdmin) {
      return res.status(403).json({ error: 'Permission denied.' });
    }

    const itemIds = Array.isArray(req.body.itemIds) ? req.body.itemIds : [];
    const delStmt = db.prepare('DELETE FROM shopping_list_items WHERE id = ? AND listId = ?');
    const tx = db.transaction((ids) => {
      for (const itemId of ids) {
        delStmt.run(itemId, existing.id);
      }
      db.prepare('UPDATE shopping_lists SET updatedAt = ? WHERE id = ?').run(new Date().toISOString(), existing.id);
    });
    tx(itemIds);

    const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
    res.json(updated);
  } catch (error) {
    console.error('Error deleting shopping list items:', error);
    res.status(500).json({ error: 'Failed to delete items.' });
  }
});

// API: Clear checked or all items from a shopping list
app.post('/api/shopping-lists/:id/clear', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(req.params.id);
    if (!existing) return res.status(404).json({ error: 'Shopping list not found.' });
    if (existing.userId !== req.user.id && !req.user.isAdmin) {
      return res.status(403).json({ error: 'Permission denied.' });
    }

    const onlyChecked = req.body.onlyChecked !== false;
    if (onlyChecked) {
      db.prepare('DELETE FROM shopping_list_items WHERE listId = ? AND checked = 1').run(existing.id);
    } else {
      db.prepare('DELETE FROM shopping_list_items WHERE listId = ?').run(existing.id);
    }
    db.prepare('UPDATE shopping_lists SET updatedAt = ? WHERE id = ?').run(new Date().toISOString(), existing.id);

    const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
    res.json(updated);
  } catch (error) {
    console.error('Error clearing shopping list:', error);
    res.status(500).json({ error: 'Failed to clear shopping list.' });
  }
});

// API: Generate or retrieve public share link token for a shopping list
app.post('/api/shopping-lists/:id/share', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(req.params.id);
    if (!existing) return res.status(404).json({ error: 'Shopping list not found.' });
    if (existing.userId !== req.user.id && !req.user.isAdmin) {
      return res.status(403).json({ error: 'Permission denied.' });
    }

    let shareToken = existing.shareToken;
    if (!shareToken) {
      shareToken = crypto.randomBytes(16).toString('hex');
      db.prepare('UPDATE shopping_lists SET shareToken = ?, updatedAt = ? WHERE id = ?').run(shareToken, new Date().toISOString(), existing.id);
    }

    const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
    res.json(updated);
  } catch (error) {
    console.error('Error sharing shopping list:', error);
    res.status(500).json({ error: 'Failed to generate shopping list share link.' });
  }
});

// API: Revoke public share link for a shopping list
app.post('/api/shopping-lists/:id/revoke-share', requireAuth, (req, res) => {
  try {
    const existing = db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(req.params.id);
    if (!existing) return res.status(404).json({ error: 'Shopping list not found.' });
    if (existing.userId !== req.user.id && !req.user.isAdmin) {
      return res.status(403).json({ error: 'Permission denied.' });
    }

    const oldToken = existing.shareToken;
    const revokeTx = db.transaction(() => {
      db.prepare('UPDATE shopping_lists SET shareToken = NULL, updatedAt = ? WHERE id = ?').run(new Date().toISOString(), existing.id);
      if (oldToken) {
        db.prepare('DELETE FROM shopping_list_shares_visits WHERE shareToken = ?').run(oldToken);
      }
    });
    revokeTx();

    const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
    res.json(updated);
  } catch (error) {
    console.error('Error revoking shopping list share link:', error);
    res.status(500).json({ error: 'Failed to revoke shopping list share link.' });
  }
});

// Public API: View shared shopping list by shareToken
app.get('/api/share/shopping-lists/:shareToken', publicShareLimiter, (req, res) => {
  const shareToken = (req.params.shareToken || '').trim();
  if (!shareToken) {
    return res.status(400).json({ error: 'Share token is required.' });
  }
  try {
    const listRow = db.prepare('SELECT * FROM shopping_lists WHERE shareToken = ?').get(shareToken);
    if (!listRow) {
      return res.status(404).json({ error: 'Shopping list not found or sharing has been disabled.' });
    }

    try {
      const ip = req.headers['x-forwarded-for'] || req.socket.remoteAddress;
      db.prepare(`
        INSERT OR IGNORE INTO shopping_list_shares_visits (shareToken, visitorIp, visitedAt)
        VALUES (?, ?, ?)
      `).run(shareToken, ip, new Date().toISOString());
    } catch (visitErr) {
      console.error('Failed to log shared shopping list visitor IP:', visitErr);
    }

    const formatted = formatShoppingList(listRow);
    res.json({
      name: formatted.name,
      createdBy: formatted.createdBy,
      createdAt: formatted.createdAt,
      updatedAt: formatted.updatedAt,
      items: formatted.items
    });
  } catch (error) {
    console.error('Error serving shared shopping list:', error);
    res.status(500).json({ error: 'Internal server error while retrieving shared shopping list.' });
  }
});

// Public API: Check/uncheck items on a shared shopping list
app.put('/api/share/shopping-lists/:shareToken/items/check', publicShareLimiter, (req, res) => {
  const shareToken = (req.params.shareToken || '').trim();
  if (!shareToken) {
    return res.status(400).json({ error: 'Share token is required.' });
  }
  try {
    const listRow = db.prepare('SELECT * FROM shopping_lists WHERE shareToken = ?').get(shareToken);
    if (!listRow) {
      return res.status(404).json({ error: 'Shopping list not found or sharing has been disabled.' });
    }

    const itemIds = Array.isArray(req.body.itemIds) ? req.body.itemIds : [];
    const checked = req.body.checked ? 1 : 0;
    if (itemIds.length === 0) {
      return res.status(400).json({ error: 'itemIds array is required.' });
    }

    const updateStmt = db.prepare('UPDATE shopping_list_items SET checked = ? WHERE id = ? AND listId = ?');
    const tx = db.transaction((ids) => {
      for (const itemId of ids) {
        updateStmt.run(checked, itemId, listRow.id);
      }
      db.prepare('UPDATE shopping_lists SET updatedAt = ? WHERE id = ?').run(new Date().toISOString(), listRow.id);
    });
    tx(itemIds);

    const formatted = formatShoppingList(listRow);
    res.json({
      name: formatted.name,
      createdBy: formatted.createdBy,
      createdAt: formatted.createdAt,
      updatedAt: formatted.updatedAt,
      items: formatted.items
    });
  } catch (error) {
    console.error('Error updating shared shopping list items:', error);
    res.status(500).json({ error: 'Failed to update shared shopping list.' });
  }
});

// API: Sanity-checked, unauthenticated public recipe sharing with rate limiting
app.get('/api/share/recipes/:shareToken', publicShareLimiter, (req, res) => {
  const shareToken = req.params.shareToken;
  if (!shareToken || shareToken.trim() === '') {
    return res.status(400).json({ error: 'Share token is required.' });
  }
  
  try {
    const recipe = db.prepare('SELECT * FROM recipes WHERE shareToken = ? AND isPublic = 1').get(shareToken);
    if (!recipe) {
      return res.status(404).json({ error: 'Recipe not found or is set to private.' });
    }
    
    // Log guest visitor IP dynamically inside the background
    try {
      const ip = req.headers['x-forwarded-for'] || req.socket.remoteAddress;
      db.prepare(`
        INSERT OR IGNORE INTO recipe_shares_visits (shareToken, visitorIp, visitedAt)
        VALUES (?, ?, ?)
      `).run(shareToken, ip, new Date().toISOString());
    } catch (visitErr) {
      console.error('Failed to log shared visitor IP:', visitErr);
    }
    
    // Strip sensitive user identifiers and internal database keys
    const sanitizedRecipe = {
      title: recipe.title,
      description: recipe.description,
      servings: recipe.servings,
      prepTime: recipe.prepTime,
      cookTime: recipe.cookTime,
      totalTime: recipe.totalTime,
      image: recipe.image,
      images: JSON.parse(recipe.images || '[]'),
      ingredients: JSON.parse(recipe.ingredients || '[]'),
      instructions: JSON.parse(recipe.instructions || '[]'),
      sourceUrl: recipe.sourceUrl,
      createdBy: recipe.createdBy,
      createdAt: recipe.createdAt
    };
    
    res.json(sanitizedRecipe);
  } catch (error) {
    console.error('Error serving shared recipe:', error);
    res.status(500).json({ error: 'Internal server error while retrieving shared recipe.' });
  }
});

// API: Revoke public recipe sharing link (requires authenticated ownership or permission)
app.post('/api/recipes/:id/revoke-share', requireAuth, (req, res) => {
  try {
    const recipe = db.prepare('SELECT * FROM recipes WHERE id = ?').get(req.params.id);
    
    if (!recipe) {
      return res.status(404).json({ error: 'Recipe not found.' });
    }
    
    // Verify permissions: owner, recipe manager, or admin
    if (recipe.userId !== req.user.id && !req.user.isRecipeManager && !req.user.isAdmin) {
      return res.status(403).json({ error: 'You do not have permission to revoke this recipe share link.' });
    }
    
    const oldShareToken = recipe.shareToken;
    
    // Relational transaction to clear sharing state and purge visits data
    const performRevocation = db.transaction(() => {
      db.prepare('UPDATE recipes SET isPublic = 0, shareToken = NULL WHERE id = ?').run(req.params.id);
      
      if (oldShareToken) {
        db.prepare('DELETE FROM recipe_shares_visits WHERE shareToken = ?').run(oldShareToken);
      }
    });
    
    performRevocation();
    res.json({ success: true });
  } catch (error) {
    console.error('Error revoking recipe share link:', error);
    res.status(500).json({ error: 'Failed to revoke shared link.' });
  }
});

// API: Scrape Recipe from Link
app.post('/api/scrape', requireAuth, async (req, res) => {
  const { url } = req.body;
  if (!url) {
    return res.status(400).json({ error: 'URL is required' });
  }

  // Specialized Scraper: TikTok Video Recipes
  if (isTikTokUrl(url)) {
    try {
      const recipeData = await scrapeTikTokRecipe(url);
      return res.json(recipeData);
    } catch (tiktokError) {
      console.error('[TikTok Scraper] Error:', tiktokError.message);
      return res.status(422).json({
        error: tiktokError.message || 'Failed to extract recipe from TikTok video.'
      });
    }
  }

  try {
    let html = null;
    let scrapingMethod = 'axios';

    // Strategy 1: Fast axios fetch
    try {
      const response = await axios.get(url, {
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
          'Accept-Language': 'en-US,en;q=0.9',
          'Accept-Encoding': 'gzip, deflate, br',
          'Connection': 'keep-alive',
          'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
          'sec-ch-ua-mobile': '?0',
          'sec-ch-ua-platform': '"Windows"',
          'Sec-Fetch-Dest': 'document',
          'Sec-Fetch-Mode': 'navigate',
          'Sec-Fetch-Site': 'none',
          'Sec-Fetch-User': '?1',
          'Upgrade-Insecure-Requests': '1'
        },
        timeout: 10000
      });
      html = response.data;

      if (isBotChallengePage(html)) {
        console.log('[Scraper] Axios response appears to be a bot challenge page, falling back...');
        html = null;
      }
    } catch (axiosError) {
      const status = axiosError.response ? axiosError.response.status : 0;
      if (status === 403 || status === 503 || status === 429) {
        console.log(`[Scraper] Axios blocked with HTTP ${status}, falling back...`);
      } else {
        throw axiosError;
      }
    }

    // Strategy 2: Reader Proxy HTML fetch (fast bypass for PerimeterX / Cloudflare edge blocks)
    if (!html) {
      try {
        const readerHtml = await fetchWithReaderProxy(url);
        if (!isBotChallengePage(readerHtml)) {
          html = readerHtml;
          scrapingMethod = 'reader-proxy';
        } else {
          console.log('[ReaderProxy] Returned challenge page, falling back to Puppeteer...');
        }
      } catch (readerError) {
        console.warn('[ReaderProxy] Fallback failed, trying Puppeteer:', readerError.message);
      }
    }

    // Strategy 3: Puppeteer with stealth (headless browser fallback)
    if (!html) {
      try {
        const puppeteerHtml = await fetchWithPuppeteer(url);
        if (!isBotChallengePage(puppeteerHtml)) {
          html = puppeteerHtml;
          scrapingMethod = 'puppeteer-stealth';
        } else {
          throw new Error('Blocked by bot challenge page in headless browser.');
        }
      } catch (puppeteerError) {
        console.error('[Puppeteer] Fallback also failed:', puppeteerError.message);
        return res.status(500).json({
          error: 'This website could not be accessed. All scraping methods were blocked.',
          details: puppeteerError.message,
          isBlocked: true
        });
      }
    }

    const recipeData = parseRecipeFromHtml(html, url);
    recipeData._scrapingMethod = scrapingMethod;

    res.json(recipeData);
  } catch (error) {
    console.error('Error occurred while scraping URL:', error.message);
    const status = error.response ? error.response.status : 500;
    const message = error.response && error.response.status === 403
      ? 'This website is protected and blocks automated access. The Puppeteer fallback was also unavailable.'
      : error.message;
    res.status(status).json({ 
      error: message, 
      isBlocked: status === 403 || status === 503 
    });
  }
});

// -------------------------------------------------------------
// ADMIN CLOUD & MANUAL BACKUP ENGINE (ZERO LOCAL STORAGE)
// -------------------------------------------------------------

const DEFAULT_GDRIVE_CLIENT_ID = process.env.GOOGLE_CLIENT_ID || '202264815644.apps.googleusercontent.com';
const DEFAULT_GDRIVE_CLIENT_SECRET = process.env.GOOGLE_CLIENT_SECRET || 'X4Z3ca8xfWDb1Voo-F9a7ZxJ';
const GDRIVE_LOOPBACK_PORT = 53682;

function getBackupConfig() {
  const row = db.prepare('SELECT * FROM backup_config WHERE id = 1').get();
  if (!row) return null;
  const isConnected = Boolean(row.gdriveRefreshToken && row.gdriveRefreshToken.trim());
  return {
    ...row,
    schedule: row.scheduleFrequency || 'off',
    includeRemoteImages: row.embedRemoteImages === 1,
    embedRemoteImages: row.embedRemoteImages === 1,
    s3Enabled: row.s3Enabled === 1,
    gdriveEnabled: row.gdriveEnabled === 1,
    gdriveConnected: isConnected,
    gdriveAccountEmail: row.gdriveAccountEmail || '',
    gdriveFolderName: row.gdriveFolderName || 'ForkFolio Backups',
    lastRunAt: row.lastBackupAt || null,
    lastRunStatus: row.lastBackupStatus === 'failed' ? 'error' : (row.lastBackupStatus || null)
  };
}

// Create an atomic SQLite .db snapshot in memory Buffer via temporary OS file, immediately unlinked
function createSqliteSnapshotBuffer() {
  const tempPath = path.join(
    os.tmpdir(),
    `forkfolio_tmp_${Date.now()}_${crypto.randomBytes(6).toString('hex')}.db`
  );
  try {
    const escapedPath = tempPath.replace(/'/g, "''");
    db.exec(`VACUUM INTO '${escapedPath}'`);
    const buf = fs.readFileSync(tempPath);
    return buf;
  } finally {
    try {
      if (fs.existsSync(tempPath)) {
        fs.unlinkSync(tempPath);
      }
    } catch (cleanupErr) {
      console.warn('[Backup] Temp file cleanup warning:', cleanupErr.message);
    }
  }
}

// Create a full JSON archive Buffer (users, recipes, embedded images, shopping lists, settings, visit stats)
async function createJsonArchiveData(embedRemoteImages = true) {
  const users = db.prepare('SELECT * FROM users').all();
  const recipes = db.prepare('SELECT * FROM recipes').all();
  const shoppingLists = db.prepare('SELECT * FROM shopping_lists').all();
  const shoppingListItems = db.prepare('SELECT * FROM shopping_list_items').all();
  const settings = db.prepare('SELECT * FROM settings').all();
  const recipeSharesVisits = db.prepare('SELECT * FROM recipe_shares_visits').all();
  const shoppingListSharesVisits = db.prepare('SELECT * FROM shopping_list_shares_visits').all();

  let imageCount = 0;
  const enrichedRecipes = [];

  for (const r of recipes) {
    const copy = { ...r };
    if (copy.image && copy.image.startsWith('data:image/')) {
      imageCount++;
    } else if (copy.image && /^https?:\/\//i.test(copy.image)) {
      imageCount++;
      if (embedRemoteImages) {
        try {
          const resp = await axios.get(copy.image, {
            responseType: 'arraybuffer',
            timeout: 5000,
            headers: { 'User-Agent': 'Mozilla/5.0' }
          });
          const contentType = resp.headers['content-type'] || 'image/jpeg';
          const base64 = Buffer.from(resp.data).toString('base64');
          copy.embeddedCoverImage = `data:${contentType};base64,${base64}`;
        } catch (imgErr) {
          // Keep original URL if remote image cannot be fetched
        }
      }
    }
    enrichedRecipes.push(copy);
  }

  const archiveObj = {
    version: '1.0',
    exportedAt: new Date().toISOString(),
    counts: {
      users: users.length,
      recipes: enrichedRecipes.length,
      images: imageCount,
      shoppingLists: shoppingLists.length,
      shoppingListItems: shoppingListItems.length
    },
    data: {
      users,
      recipes: enrichedRecipes,
      shoppingLists,
      shoppingListItems,
      settings,
      recipeSharesVisits,
      shoppingListSharesVisits
    }
  };

  const buffer = Buffer.from(JSON.stringify(archiveObj, null, 2), 'utf8');
  const userSummary = inspectBackupUsersFromData({
    users,
    recipes: enrichedRecipes,
    shoppingLists
  }).users;
  return {
    buffer,
    userCount: users.length,
    recipeCount: enrichedRecipes.length,
    imageCount,
    listCount: shoppingLists.length,
    userSummary
  };
}

// -------------------------------------------------------------
// AWS SIGNATURE V4 HELPER FOR WASABI / S3-COMPATIBLE STORAGE
// -------------------------------------------------------------

function hmacSha256(key, data) {
  return crypto.createHmac('sha256', key).update(data, 'utf8').digest();
}

function sha256Hex(data) {
  return crypto.createHash('sha256').update(data).digest('hex');
}

function encodeS3URI(uriPath, encodeSlash = false) {
  return uriPath
    .split('/')
    .map(segment =>
      encodeURIComponent(segment).replace(/[!'()*]/g, c => '%' + c.charCodeAt(0).toString(16).toUpperCase())
    )
    .join(encodeSlash ? '%2F' : '/');
}

function buildS3SignedRequest({ method, endpoint, region, bucket, objectKey = '', queryParams = {}, headers = {}, body = Buffer.alloc(0), accessKeyId, secretAccessKey }) {
  const baseEndpoint = (endpoint || 'https://s3.us-east-1.wasabisys.com').trim().replace(/\/+$/, '');
  const parsedEndpoint = new URL(baseEndpoint.startsWith('http') ? baseEndpoint : `https://${baseEndpoint}`);

  const cleanBucket = (bucket || '').trim();
  const cleanKey = (objectKey || '').replace(/^\/+/, '');
  const pathSegments = [''];
  if (cleanBucket) pathSegments.push(cleanBucket);
  if (cleanKey) pathSegments.push(cleanKey);
  const rawPath = pathSegments.join('/') || '/';
  const canonicalUri = encodeS3URI(rawPath, false);

  const sortedQueryKeys = Object.keys(queryParams).sort();
  const canonicalQueryString = sortedQueryKeys
    .map(k => `${encodeURIComponent(k)}=${encodeURIComponent(String(queryParams[k]))}`)
    .join('&');

  const now = new Date();
  const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, '');
  const dateStamp = amzDate.slice(0, 8);
  const payloadHash = sha256Hex(body);

  const signHeaders = {
    host: parsedEndpoint.host,
    'x-amz-content-sha256': payloadHash,
    'x-amz-date': amzDate,
    ...headers
  };

  const lowerHeaders = {};
  for (const [k, v] of Object.entries(signHeaders)) {
    lowerHeaders[k.toLowerCase()] = String(v).trim();
  }
  const signedHeaderKeys = Object.keys(lowerHeaders).sort();
  const canonicalHeaders = signedHeaderKeys.map(k => `${k}:${lowerHeaders[k]}\n`).join('');
  const signedHeadersStr = signedHeaderKeys.join(';');

  const canonicalRequest = [
    method.toUpperCase(),
    canonicalUri,
    canonicalQueryString,
    canonicalHeaders,
    signedHeadersStr,
    payloadHash
  ].join('\n');

  const s3Region = (region || 'us-east-1').trim();
  const credentialScope = `${dateStamp}/${s3Region}/s3/aws4_request`;
  const stringToSign = [
    'AWS4-HMAC-SHA256',
    amzDate,
    credentialScope,
    sha256Hex(canonicalRequest)
  ].join('\n');

  const kDate = hmacSha256(`AWS4${secretAccessKey}`, dateStamp);
  const kRegion = hmacSha256(kDate, s3Region);
  const kService = hmacSha256(kRegion, 's3');
  const kSigning = hmacSha256(kService, 'aws4_request');
  const signature = crypto.createHmac('sha256', kSigning).update(stringToSign, 'utf8').digest('hex');

  const authorization = `AWS4-HMAC-SHA256 Credential=${accessKeyId}/${credentialScope}, SignedHeaders=${signedHeadersStr}, Signature=${signature}`;

  const fullUrl = `${parsedEndpoint.origin}${canonicalUri}${canonicalQueryString ? '?' + canonicalQueryString : ''}`;
  const requestHeaders = {
    ...signHeaders,
    Authorization: authorization
  };
  delete requestHeaders.host;

  return { url: fullUrl, headers: requestHeaders };
}

async function s3TestConnection(cfg) {
  if (!cfg.s3Bucket || !cfg.s3AccessKeyId || !cfg.s3SecretAccessKey) {
    throw new Error('Bucket name, Access Key ID, and Secret Access Key are required.');
  }
  const signed = buildS3SignedRequest({
    method: 'GET',
    endpoint: cfg.s3Endpoint,
    region: cfg.s3Region,
    bucket: cfg.s3Bucket,
    queryParams: { 'list-type': '2', 'max-keys': '1' },
    accessKeyId: cfg.s3AccessKeyId,
    secretAccessKey: cfg.s3SecretAccessKey
  });
  const res = await fetch(signed.url, { method: 'GET', headers: signed.headers });
  if (!res.ok) {
    const errText = await res.text();
    throw new Error(`S3 returned HTTP ${res.status}: ${errText.slice(0, 220)}`);
  }
  return true;
}

async function s3PutObject(cfg, objectKey, buffer, contentType) {
  const signed = buildS3SignedRequest({
    method: 'PUT',
    endpoint: cfg.s3Endpoint,
    region: cfg.s3Region,
    bucket: cfg.s3Bucket,
    objectKey,
    headers: {
      'Content-Type': contentType,
      'Content-Length': String(buffer.length)
    },
    body: buffer,
    accessKeyId: cfg.s3AccessKeyId,
    secretAccessKey: cfg.s3SecretAccessKey
  });
  const res = await fetch(signed.url, { method: 'PUT', headers: signed.headers, body: buffer });
  if (!res.ok) {
    const errText = await res.text();
    throw new Error(`S3 PUT failed (${res.status}): ${errText.slice(0, 220)}`);
  }
  return objectKey;
}

async function s3GetObject(cfg, objectKey) {
  const signed = buildS3SignedRequest({
    method: 'GET',
    endpoint: cfg.s3Endpoint,
    region: cfg.s3Region,
    bucket: cfg.s3Bucket,
    objectKey,
    accessKeyId: cfg.s3AccessKeyId,
    secretAccessKey: cfg.s3SecretAccessKey
  });
  const res = await fetch(signed.url, { method: 'GET', headers: signed.headers });
  if (!res.ok) {
    const errText = await res.text();
    throw new Error(`S3 GET failed (${res.status}): ${errText.slice(0, 200)}`);
  }
  const arrBuf = await res.arrayBuffer();
  return Buffer.from(arrBuf);
}

async function s3DeleteObject(cfg, objectKey) {
  const signed = buildS3SignedRequest({
    method: 'DELETE',
    endpoint: cfg.s3Endpoint,
    region: cfg.s3Region,
    bucket: cfg.s3Bucket,
    objectKey,
    accessKeyId: cfg.s3AccessKeyId,
    secretAccessKey: cfg.s3SecretAccessKey
  });
  const res = await fetch(signed.url, { method: 'DELETE', headers: signed.headers });
  if (!res.ok && res.status !== 204 && res.status !== 404) {
    const errText = await res.text();
    throw new Error(`S3 DELETE failed (${res.status}): ${errText.slice(0, 200)}`);
  }
}

async function s3ListBackupObjects(cfg) {
  if (!cfg.s3Bucket || !cfg.s3AccessKeyId || !cfg.s3SecretAccessKey) return [];
  const prefix = (cfg.s3Prefix || '').trim().replace(/^\/+/, '');
  const queryParams = { 'list-type': '2', 'max-keys': '200' };
  if (prefix) queryParams.prefix = prefix;

  const signed = buildS3SignedRequest({
    method: 'GET',
    endpoint: cfg.s3Endpoint,
    region: cfg.s3Region,
    bucket: cfg.s3Bucket,
    queryParams,
    accessKeyId: cfg.s3AccessKeyId,
    secretAccessKey: cfg.s3SecretAccessKey
  });
  const res = await fetch(signed.url, { method: 'GET', headers: signed.headers });
  if (!res.ok) {
    const errText = await res.text();
    throw new Error(`S3 list failed (${res.status}): ${errText.slice(0, 200)}`);
  }
  const xmlText = await res.text();
  const $ = cheerio.load(xmlText, { xmlMode: true });
  const results = [];
  $('Contents').each((_, el) => {
    const key = $(el).find('Key').text().trim();
    const size = parseInt($(el).find('Size').text().trim(), 10) || 0;
    const lastModified = $(el).find('LastModified').text().trim();
    const name = key.split('/').pop() || key;
    if (/\.(db|sqlite|json)$/i.test(name)) {
      results.push({
        key,
        name,
        size,
        createdAt: lastModified || new Date().toISOString(),
        format: /\.json$/i.test(name) ? 'json' : 'db'
      });
    }
  });
  return results.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
}

// -------------------------------------------------------------
// GOOGLE DRIVE REST API V3 CONNECTION CLIENT
// -------------------------------------------------------------

// In-memory Google OAuth2 access token cache + request serializer/retry helper to avoid shared client rate limits
let cachedGdriveToken = {
  accessToken: '',
  refreshToken: '',
  expiresAt: 0
};

let gdriveRequestQueue = Promise.resolve();

function sleepMs(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

async function gdriveFetch(url, options = {}, maxRetries = 8) {
  const runRequest = async () => {
    let lastRes = null;
    const backoffSchedule = [1200, 2200, 3500, 5000, 6800, 8500, 10500, 12500];
    for (let attempt = 0; attempt <= maxRetries; attempt++) {
      if (attempt > 0) {
        const baseDelay = backoffSchedule[attempt - 1] || 10000;
        const delay = baseDelay + Math.floor(Math.random() * 400);
        await sleepMs(delay);
      }
      lastRes = await fetch(url, options);
      if (lastRes.ok) {
        return lastRes;
      }
      // Retry on rate limit (403 userRateLimitExceeded / rateLimitExceeded / 429) or transient 5xx
      if (lastRes.status === 429 || lastRes.status >= 500) {
        continue;
      }
      if (lastRes.status === 403 && attempt < maxRetries) {
        const clonedText = await lastRes.clone().text().catch(() => '');
        if (/rateLimit|quota|Queries|Requests per minute|userRateLimit/i.test(clonedText)) {
          continue;
        }
      }
      return lastRes;
    }
    return lastRes;
  };

  const next = gdriveRequestQueue.then(async () => {
    await sleepMs(150);
    return runRequest();
  });
  gdriveRequestQueue = next.catch(() => {});
  return next;
}

async function getGoogleDriveAccessToken(cfg) {
  const clientId = (cfg.gdriveClientId && cfg.gdriveClientId.trim()) || DEFAULT_GDRIVE_CLIENT_ID;
  const clientSecret = (cfg.gdriveClientSecret && cfg.gdriveClientSecret.trim()) || DEFAULT_GDRIVE_CLIENT_SECRET;
  const refreshToken = (cfg.gdriveRefreshToken || '').trim();
  if (!refreshToken) {
    throw new Error('Google Drive is not connected yet. Click "Authorize with Google" to connect your account.');
  }

  const now = Date.now();
  if (
    cachedGdriveToken.accessToken &&
    cachedGdriveToken.refreshToken === refreshToken &&
    cachedGdriveToken.expiresAt > now + 60 * 1000
  ) {
    return cachedGdriveToken.accessToken;
  }

  const params = new URLSearchParams({
    client_id: clientId,
    client_secret: clientSecret,
    refresh_token: refreshToken,
    grant_type: 'refresh_token'
  });
  const res = await fetch('https://oauth2.googleapis.com/token', {
    method: 'POST',
    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
    body: params.toString()
  });
  const data = await res.json().catch(() => ({}));
  if (!res.ok || !data.access_token) {
    throw new Error(data.error_description || data.error || `Google OAuth token exchange failed (${res.status}).`);
  }

  const expiresInSec = Number(data.expires_in) || 3500;
  cachedGdriveToken = {
    accessToken: data.access_token,
    refreshToken,
    expiresAt: now + expiresInSec * 1000
  };
  return data.access_token;
}

async function getGoogleDriveUserEmail(accessToken) {
  try {
    const res = await gdriveFetch('https://www.googleapis.com/drive/v3/about?fields=user(emailAddress,displayName)', {
      headers: { Authorization: `Bearer ${accessToken}` }
    });
    if (res.ok) {
      const data = await res.json();
      return data.user?.emailAddress || data.user?.displayName || '';
    }
  } catch (e) {}
  return '';
}

async function ensureGoogleDriveFolder(accessToken, cfg, verifyRemote = false) {
  if (cfg.gdriveFolderId && cfg.gdriveFolderId.trim()) {
    const folderId = cfg.gdriveFolderId.trim();
    if (!verifyRemote) {
      return folderId;
    }
    const checkRes = await gdriveFetch(`https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,trashed`, {
      headers: { Authorization: `Bearer ${accessToken}` }
    });
    if (checkRes.ok) {
      const f = await checkRes.json();
      if (!f.trashed) {
        if (f.name && f.name !== cfg.gdriveFolderName) {
          db.prepare('UPDATE backup_config SET gdriveFolderName = ? WHERE id = 1').run(f.name);
        }
        return f.id;
      }
    } else if (checkRes.status !== 404) {
      // Keep configured folderId on non-404 responses (e.g. temporary quota limit) rather than recreating
      return folderId;
    }
  }

  // Search for existing "ForkFolio Backups" folder or create one
  const q = encodeURIComponent("mimeType='application/vnd.google-apps.folder' and name='ForkFolio Backups' and trashed=false");
  const searchRes = await gdriveFetch(`https://www.googleapis.com/drive/v3/files?q=${q}&fields=files(id,name)`, {
    headers: { Authorization: `Bearer ${accessToken}` }
  });
  if (searchRes.ok) {
    const searchData = await searchRes.json();
    if (Array.isArray(searchData.files) && searchData.files.length > 0) {
      const folderId = searchData.files[0].id;
      const folderName = searchData.files[0].name || 'ForkFolio Backups';
      db.prepare('UPDATE backup_config SET gdriveFolderId = ?, gdriveFolderName = ? WHERE id = 1').run(folderId, folderName);
      return folderId;
    }
  }

  // Create folder
  const createRes = await gdriveFetch('https://www.googleapis.com/drive/v3/files?fields=id,name', {
    method: 'POST',
    headers: {
      Authorization: `Bearer ${accessToken}`,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({
      name: 'ForkFolio Backups',
      mimeType: 'application/vnd.google-apps.folder'
    })
  });
  const created = await createRes.json().catch(() => ({}));
  if (!createRes.ok || !created.id) {
    throw new Error(created.error?.message || 'Failed to verify or create Google Drive backup folder.');
  }
  db.prepare('UPDATE backup_config SET gdriveFolderId = ?, gdriveFolderName = ? WHERE id = 1').run(created.id, created.name || 'ForkFolio Backups');
  return created.id;
}

async function gdriveListFolders(accessToken) {
  const q = encodeURIComponent("mimeType='application/vnd.google-apps.folder' and trashed=false");
  const res = await gdriveFetch(`https://www.googleapis.com/drive/v3/files?q=${q}&pageSize=100&orderBy=name&fields=files(id,name,parents)`, {
    headers: { Authorization: `Bearer ${accessToken}` }
  });
  const data = await res.json().catch(() => ({}));
  if (!res.ok) {
    throw new Error(data.error?.message || 'Failed to list Google Drive folders.');
  }
  return Array.isArray(data.files) ? data.files : [];
}

async function gdriveCreateFolder(accessToken, folderName) {
  const cleanName = (folderName || '').trim() || 'ForkFolio Backups';
  const res = await gdriveFetch('https://www.googleapis.com/drive/v3/files?fields=id,name', {
    method: 'POST',
    headers: {
      Authorization: `Bearer ${accessToken}`,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({
      name: cleanName,
      mimeType: 'application/vnd.google-apps.folder'
    })
  });
  const created = await res.json().catch(() => ({}));
  if (!res.ok || !created.id) {
    throw new Error(created.error?.message || 'Failed to create folder in Google Drive.');
  }
  return { id: created.id, name: created.name || cleanName };
}

async function gdriveListBackupFiles(accessToken, folderId) {
  const clauses = ["mimeType != 'application/vnd.google-apps.folder'", 'trashed = false'];
  if (folderId && folderId.trim()) {
    clauses.push(`('${folderId.trim()}' in parents or name contains 'forkfolio-backup')`);
  } else {
    clauses.push("name contains 'forkfolio-backup'");
  }
  const q = encodeURIComponent(clauses.join(' and '));
  const res = await gdriveFetch(
    `https://www.googleapis.com/drive/v3/files?q=${q}&pageSize=150&orderBy=createdTime desc&fields=files(id,name,size,createdTime,modifiedTime,mimeType,parents)`,
    { headers: { Authorization: `Bearer ${accessToken}` } }
  );
  const data = await res.json().catch(() => ({}));
  if (!res.ok) {
    throw new Error(data.error?.message || 'Failed to list backup files from Google Drive.');
  }
  const files = Array.isArray(data.files) ? data.files : [];
  return files
    .filter(f => /\.(db|sqlite|json)$/i.test(f.name || ''))
    .map(f => ({
      id: f.id,
      name: f.name,
      size: parseInt(f.size, 10) || 0,
      createdAt: f.createdTime || f.modifiedTime || new Date().toISOString(),
      format: /\.json$/i.test(f.name) ? 'json' : 'db'
    }));
}

async function gdriveUploadFile(accessToken, folderId, filename, buffer, mimeType) {
  const boundary = 'forkfolio_gdrive_' + crypto.randomBytes(12).toString('hex');
  const metadata = {
    name: filename,
    parents: folderId ? [folderId] : undefined
  };

  const preamble = Buffer.from(
    `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n${JSON.stringify(metadata)}\r\n--${boundary}\r\nContent-Type: ${mimeType}\r\n\r\n`,
    'utf8'
  );
  const epilogue = Buffer.from(`\r\n--${boundary}--`, 'utf8');
  const multipartBody = Buffer.concat([preamble, buffer, epilogue]);

  const res = await gdriveFetch('https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart&fields=id,name,size', {
    method: 'POST',
    headers: {
      Authorization: `Bearer ${accessToken}`,
      'Content-Type': `multipart/related; boundary=${boundary}`,
      'Content-Length': String(multipartBody.length)
    },
    body: multipartBody
  });
  const data = await res.json().catch(() => ({}));
  if (!res.ok || !data.id) {
    throw new Error(data.error?.message || `Google Drive upload failed (${res.status}).`);
  }
  return { id: data.id, name: data.name || filename, mimeType };
}

async function gdriveDownloadFile(accessToken, fileId) {
  const res = await gdriveFetch(`https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media`, {
    headers: { Authorization: `Bearer ${accessToken}` }
  });
  if (!res.ok) {
    const errText = await res.text();
    throw new Error(`Google Drive download failed (${res.status}): ${errText.slice(0, 200)}`);
  }
  const arrBuf = await res.arrayBuffer();
  return Buffer.from(arrBuf);
}

async function gdriveDeleteFile(accessToken, fileId) {
  const res = await gdriveFetch(`https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}`, {
    method: 'DELETE',
    headers: { Authorization: `Bearer ${accessToken}` }
  });
  if (!res.ok && res.status !== 204 && res.status !== 404) {
    const errText = await res.text();
    throw new Error(`Google Drive delete failed (${res.status}): ${errText.slice(0, 200)}`);
  }
}

// -------------------------------------------------------------
// CLOUD BACKUP EXECUTION & RETENTION PRUNING
// -------------------------------------------------------------

let isBackupRunning = false;

async function executeCloudBackup(triggerType = 'manual') {
  if (isBackupRunning) {
    throw new Error('A backup operation is already in progress.');
  }
  isBackupRunning = true;

  const cfg = getBackupConfig();
  const backupId = 'bkp_' + Date.now().toString(36) + Math.random().toString(36).substr(2, 4);
  const now = new Date();
  const stamp = now.toISOString().replace(/[:.]/g, '-').slice(0, 19);
  const label = `forkfolio-backup-${stamp}`;

  const s3Objects = [];
  const gdriveFiles = [];

  try {
    if (!cfg.s3Enabled && !cfg.gdriveEnabled) {
      throw new Error('Neither Wasabi S3 nor Google Drive is enabled. Enable at least one cloud provider or use "Download to Computer".');
    }

    const wantDb = cfg.backupFormat === 'both' || cfg.backupFormat === 'db';
    const wantJson = cfg.backupFormat === 'both' || cfg.backupFormat === 'json';

    const jsonArchive = await createJsonArchiveData(cfg.embedRemoteImages);
    const dbBuffer = wantDb ? createSqliteSnapshotBuffer() : null;
    const jsonBuffer = wantJson ? jsonArchive.buffer : null;

    const totalSize = (dbBuffer ? dbBuffer.length : 0) + (jsonBuffer ? jsonBuffer.length : 0);

    // 1. Upload to Wasabi S3 if enabled
    if (cfg.s3Enabled) {
      const prefix = (cfg.s3Prefix || 'forkfolio-backups/').trim().replace(/^\/+/, '');
      const normalizedPrefix = prefix && !prefix.endsWith('/') ? `${prefix}/` : prefix;

      if (dbBuffer) {
        const dbKey = `${normalizedPrefix}${label}.db`;
        await s3PutObject(cfg, dbKey, dbBuffer, 'application/x-sqlite3');
        s3Objects.push({ key: dbKey, name: `${label}.db`, format: 'db' });
      }
      if (jsonBuffer) {
        const jsonKey = `${normalizedPrefix}${label}.json`;
        await s3PutObject(cfg, jsonKey, jsonBuffer, 'application/json');
        s3Objects.push({ key: jsonKey, name: `${label}.json`, format: 'json' });
      }
    }

    // 2. Upload to Google Drive if enabled
    if (cfg.gdriveEnabled) {
      const accessToken = await getGoogleDriveAccessToken(cfg);
      const folderId = await ensureGoogleDriveFolder(accessToken, cfg);

      if (dbBuffer) {
        const uploadedDb = await gdriveUploadFile(accessToken, folderId, `${label}.db`, dbBuffer, 'application/x-sqlite3');
        gdriveFiles.push({ ...uploadedDb, format: 'db' });
      }
      if (jsonBuffer) {
        const uploadedJson = await gdriveUploadFile(accessToken, folderId, `${label}.json`, jsonBuffer, 'application/json');
        gdriveFiles.push({ ...uploadedJson, format: 'json' });
      }
    }

    const createdAt = now.toISOString();
    db.prepare(`
      INSERT INTO backup_history (
        id, label, triggerType, sizeBytes, userCount, recipeCount, imageCount, listCount,
        s3Objects, gdriveFiles, userSummary, status, errorDetails, createdAt
      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'success', NULL, ?)
    `).run(
      backupId,
      label,
      triggerType,
      totalSize,
      jsonArchive.userCount,
      jsonArchive.recipeCount,
      jsonArchive.imageCount,
      jsonArchive.listCount,
      JSON.stringify(s3Objects),
      JSON.stringify(gdriveFiles),
      JSON.stringify(jsonArchive.userSummary || []),
      createdAt
    );

    if (dbBuffer) {
      rememberBackupBufferInMemory(`history:${backupId}:db`, dbBuffer, `${label}.db`);
      for (const f of gdriveFiles.filter(x => x.format === 'db')) {
        rememberBackupBufferInMemory(`gdrive:${f.id}`, dbBuffer, f.name || `${label}.db`);
      }
      for (const o of s3Objects.filter(x => x.format === 'db')) {
        rememberBackupBufferInMemory(`s3:${o.key}`, dbBuffer, o.name || `${label}.db`);
      }
    }
    if (jsonBuffer) {
      rememberBackupBufferInMemory(`history:${backupId}:json`, jsonBuffer, `${label}.json`);
      for (const f of gdriveFiles.filter(x => x.format === 'json')) {
        rememberBackupBufferInMemory(`gdrive:${f.id}`, jsonBuffer, f.name || `${label}.json`);
      }
      for (const o of s3Objects.filter(x => x.format === 'json')) {
        rememberBackupBufferInMemory(`s3:${o.key}`, jsonBuffer, o.name || `${label}.json`);
      }
    }

    db.prepare(`
      UPDATE backup_config
      SET lastBackupAt = ?, lastBackupStatus = 'success', lastBackupMessage = ?
      WHERE id = 1
    `).run(createdAt, `Uploaded ${label} (${Math.round(totalSize / 1024)} KB)`);

    // Enforce retention limit on cloud backups
    await pruneOldCloudBackups(cfg);

    return db.prepare('SELECT * FROM backup_history WHERE id = ?').get(backupId);
  } catch (error) {
    const failedAt = new Date().toISOString();
    db.prepare(`
      INSERT INTO backup_history (
        id, label, triggerType, sizeBytes, userCount, recipeCount, imageCount, listCount,
        s3Objects, gdriveFiles, status, errorDetails, createdAt
      ) VALUES (?, ?, ?, 0, 0, 0, 0, 0, '[]', '[]', 'failed', ?, ?)
    `).run(backupId, label, triggerType, error.message, failedAt);

    db.prepare(`
      UPDATE backup_config
      SET lastBackupAt = ?, lastBackupStatus = 'failed', lastBackupMessage = ?
      WHERE id = 1
    `).run(failedAt, error.message);

    throw error;
  } finally {
    isBackupRunning = false;
  }
}

async function pruneOldCloudBackups(cfg) {
  const retention = Number(cfg.retentionCount || 0);
  if (retention <= 0) return;

  const successfulRuns = db.prepare(`
    SELECT * FROM backup_history
    WHERE status = 'success'
    ORDER BY createdAt DESC
  `).all();

  if (successfulRuns.length <= retention) return;

  const expiredRuns = successfulRuns.slice(retention);
  let gdriveToken = null;

  for (const run of expiredRuns) {
    try {
      const s3List = JSON.parse(run.s3Objects || '[]');
      if (cfg.s3Enabled && s3List.length > 0) {
        for (const obj of s3List) {
          await s3DeleteObject(cfg, obj.key).catch(e => console.warn('[Retention] S3 delete warning:', e.message));
        }
      }

      const gdList = JSON.parse(run.gdriveFiles || '[]');
      if (cfg.gdriveEnabled && gdList.length > 0) {
        if (!gdriveToken) {
          gdriveToken = await getGoogleDriveAccessToken(cfg).catch(() => null);
        }
        if (gdriveToken) {
          for (const f of gdList) {
            await gdriveDeleteFile(gdriveToken, f.id).catch(e => console.warn('[Retention] GDrive delete warning:', e.message));
          }
        }
      }

      db.prepare('DELETE FROM backup_history WHERE id = ?').run(run.id);
    } catch (pruneErr) {
      console.warn('[Retention] Failed pruning backup record:', pruneErr.message);
    }
  }
}

// Parse a backup Buffer (.db SQLite file or .json archive) into structured tables without leaving local files
function parseBackupBufferToData(fileBuffer, filename = '') {
  const isSqliteHeader =
    fileBuffer.length > 16 &&
    fileBuffer.subarray(0, 15).toString('utf8') === 'SQLite format 3';

  if (isSqliteHeader || filename.toLowerCase().endsWith('.db') || filename.toLowerCase().endsWith('.sqlite')) {
    const tempRestorePath = path.join(
      os.tmpdir(),
      `forkfolio_restore_${Date.now()}_${crypto.randomBytes(6).toString('hex')}.db`
    );
    let sourceDb = null;
    try {
      fs.writeFileSync(tempRestorePath, fileBuffer);
      sourceDb = new Database(tempRestorePath, { readonly: true });

      const users = sourceDb.prepare('SELECT * FROM users').all();
      const recipes = sourceDb.prepare('SELECT * FROM recipes').all();
      let shoppingLists = [];
      let shoppingListItems = [];
      let settings = [];
      let recipeVisits = [];
      let listVisits = [];

      try { shoppingLists = sourceDb.prepare('SELECT * FROM shopping_lists').all(); } catch (e) {}
      try { shoppingListItems = sourceDb.prepare('SELECT * FROM shopping_list_items').all(); } catch (e) {}
      try { settings = sourceDb.prepare('SELECT * FROM settings').all(); } catch (e) {}
      try { recipeVisits = sourceDb.prepare('SELECT * FROM recipe_shares_visits').all(); } catch (e) {}
      try { listVisits = sourceDb.prepare('SELECT * FROM shopping_list_shares_visits').all(); } catch (e) {}

      return {
        users,
        recipes,
        shoppingLists,
        shoppingListItems,
        settings,
        recipeSharesVisits: recipeVisits,
        shoppingListSharesVisits: listVisits
      };
    } finally {
      if (sourceDb) {
        try { sourceDb.close(); } catch (e) {}
      }
      try {
        if (fs.existsSync(tempRestorePath)) fs.unlinkSync(tempRestorePath);
      } catch (e) {}
    }
  } else {
    const parsed = JSON.parse(fileBuffer.toString('utf8'));
    const data = parsed.data || parsed;
    if (!Array.isArray(data.users) && !Array.isArray(data.recipes)) {
      throw new Error('Invalid backup archive format: missing users and recipes tables.');
    }
    return {
      users: data.users || [],
      recipes: data.recipes || [],
      shoppingLists: data.shoppingLists || [],
      shoppingListItems: data.shoppingListItems || [],
      settings: data.settings || [],
      recipeSharesVisits: data.recipeSharesVisits || [],
      shoppingListSharesVisits: data.shoppingListSharesVisits || []
    };
  }
}

function inspectBackupUsersFromData(data) {
  const users = Array.isArray(data.users) ? data.users : [];
  const recipes = Array.isArray(data.recipes) ? data.recipes : [];
  const shoppingLists = Array.isArray(data.shoppingLists) ? data.shoppingLists : [];

  const userSummaries = users.map(u => {
    const unameLower = String(u.username || '').toLowerCase();
    const rCount = recipes.filter(
      r => r.userId === u.id || (r.createdBy && String(r.createdBy).toLowerCase() === unameLower)
    ).length;
    const lCount = shoppingLists.filter(
      l => l.userId === u.id || (l.createdBy && String(l.createdBy).toLowerCase() === unameLower)
    ).length;
    return {
      id: u.id,
      username: u.username,
      isAdmin: Boolean(u.isAdmin),
      isRecipeManager: Boolean(u.isRecipeManager),
      recipeCount: rCount,
      shoppingListCount: lCount
    };
  });

  return {
    users: userSummaries,
    totalUsers: users.length,
    totalRecipes: recipes.length,
    totalShoppingLists: shoppingLists.length
  };
}

// Restore database (all users or a single user) from a Buffer (.db SQLite file or .json archive) without leaving local files
function restoreDatabaseFromBuffer(fileBuffer, filename = '', targetUser = 'all') {
  const data = parseBackupBufferToData(fileBuffer, filename);
  return applyParsedRestoreData(data, targetUser);
}

function applyParsedRestoreData(data, targetUser = 'all') {
  const isSingleUser = Boolean(targetUser && targetUser !== 'all');
  db.pragma('foreign_keys = OFF');
  try {
    let resultSummary = null;

    const tx = db.transaction(() => {
      if (isSingleUser) {
        const targetLower = String(targetUser).toLowerCase();
        const backupUser = (data.users || []).find(
          u => u.id === targetUser || String(u.username || '').toLowerCase() === targetLower
        );
        if (!backupUser) {
          throw new Error(`User "${targetUser}" was not found in this backup archive.`);
        }

        const backupUnameLower = String(backupUser.username || '').toLowerCase();
        const userRecipes = (data.recipes || []).filter(
          r => r.userId === backupUser.id || (r.createdBy && String(r.createdBy).toLowerCase() === backupUnameLower)
        );
        const userLists = (data.shoppingLists || []).filter(
          l => l.userId === backupUser.id || (l.createdBy && String(l.createdBy).toLowerCase() === backupUnameLower)
        );
        const userListIds = new Set(userLists.map(l => l.id));
        const userListItems = (data.shoppingListItems || []).filter(item => userListIds.has(item.listId));

        // Check if this user already exists in the live DB (by username or id)
        const liveUser =
          db.prepare('SELECT * FROM users WHERE lower(username) = lower(?)').get(backupUser.username) ||
          db.prepare('SELECT * FROM users WHERE id = ?').get(backupUser.id);

        const effectiveUserId = liveUser ? liveUser.id : backupUser.id;
        const effectiveUsername = liveUser ? liveUser.username : backupUser.username;

        if (!liveUser) {
          db.prepare(`
            INSERT INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, defaultConsolidateLists, createdAt)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
          `).run(
            backupUser.id,
            backupUser.username,
            backupUser.salt,
            backupUser.passwordHash,
            backupUser.isAdmin ? 1 : 0,
            backupUser.isRecipeManager ? 1 : 0,
            backupUser.defaultConsolidateLists !== 0 ? 1 : 0,
            backupUser.createdAt || new Date().toISOString()
          );
        } else {
          db.prepare(`
            UPDATE users
            SET defaultConsolidateLists = ?
            WHERE id = ?
          `).run(backupUser.defaultConsolidateLists !== 0 ? 1 : 0, effectiveUserId);
        }

        // Remove only this user's existing shopping lists and their items
        const existingUserLists = db.prepare(
          'SELECT id FROM shopping_lists WHERE userId = ? OR lower(createdBy) = lower(?)'
        ).all(effectiveUserId, effectiveUsername);
        const delListItemsStmt = db.prepare('DELETE FROM shopping_list_items WHERE listId = ?');
        for (const el of existingUserLists) {
          delListItemsStmt.run(el.id);
        }
        db.prepare('DELETE FROM shopping_lists WHERE userId = ? OR lower(createdBy) = lower(?)').run(
          effectiveUserId,
          effectiveUsername
        );

        // Insert restored shopping lists & items for this user
        const insList = db.prepare(`
          INSERT OR REPLACE INTO shopping_lists (id, userId, createdBy, name, shareToken, createdAt, updatedAt)
          VALUES (?, ?, ?, ?, ?, ?, ?)
        `);
        for (const l of userLists) {
          insList.run(
            l.id,
            effectiveUserId,
            effectiveUsername,
            l.name,
            l.shareToken || null,
            l.createdAt || new Date().toISOString(),
            l.updatedAt || null
          );
        }

        const insItem = db.prepare(`
          INSERT OR REPLACE INTO shopping_list_items (
            id, listId, rawText, itemName, quantity, unit, category, recipeId, recipeTitle, checked, createdAt
          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        `);
        for (const item of userListItems) {
          insItem.run(
            item.id,
            item.listId,
            item.rawText,
            item.itemName || item.rawText,
            item.quantity !== undefined ? item.quantity : null,
            item.unit || '',
            item.category || 'Other',
            item.recipeId || null,
            item.recipeTitle || null,
            item.checked ? 1 : 0,
            item.createdAt || new Date().toISOString()
          );
        }

        // Remove only this user's existing recipes and insert their restored recipes
        db.prepare('DELETE FROM recipes WHERE userId = ? OR lower(createdBy) = lower(?)').run(
          effectiveUserId,
          effectiveUsername
        );
        const insRecipe = db.prepare(`
          INSERT OR REPLACE INTO recipes (
            id, userId, createdBy, isPublic, title, description, servings,
            prepTime, cookTime, totalTime, image, images, ingredients, instructions, sourceUrl, shareToken, createdAt, updatedAt
          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        `);
        for (const r of userRecipes) {
          const coverImg = r.image || r.embeddedCoverImage || '';
          insRecipe.run(
            r.id,
            effectiveUserId,
            effectiveUsername,
            r.isPublic !== 0 && r.isPublic !== false ? 1 : 0,
            r.title || 'Untitled Recipe',
            r.description || '',
            r.servings || '',
            r.prepTime || '',
            r.cookTime || '',
            r.totalTime || '',
            coverImg,
            typeof r.images === 'string' ? r.images : JSON.stringify(r.images || []),
            typeof r.ingredients === 'string' ? r.ingredients : JSON.stringify(r.ingredients || []),
            typeof r.instructions === 'string' ? r.instructions : JSON.stringify(r.instructions || []),
            r.sourceUrl || '',
            r.shareToken || null,
            r.createdAt || new Date().toISOString(),
            r.updatedAt || null
          );
        }

        // Restore share visits for this user's restored recipes & lists
        const recipeTokens = new Set(userRecipes.map(r => r.shareToken).filter(Boolean));
        if (recipeTokens.size > 0 && Array.isArray(data.recipeSharesVisits)) {
          const insVisit = db.prepare('INSERT OR IGNORE INTO recipe_shares_visits (shareToken, visitorIp, visitedAt) VALUES (?, ?, ?)');
          for (const v of data.recipeSharesVisits) {
            if (recipeTokens.has(v.shareToken)) {
              insVisit.run(v.shareToken, v.visitorIp, v.visitedAt);
            }
          }
        }
        const listTokens = new Set(userLists.map(l => l.shareToken).filter(Boolean));
        if (listTokens.size > 0 && Array.isArray(data.shoppingListSharesVisits)) {
          const insListVisit = db.prepare('INSERT OR IGNORE INTO shopping_list_shares_visits (shareToken, visitorIp, visitedAt) VALUES (?, ?, ?)');
          for (const v of data.shoppingListSharesVisits) {
            if (listTokens.has(v.shareToken)) {
              insListVisit.run(v.shareToken, v.visitorIp, v.visitedAt);
            }
          }
        }

        resultSummary = {
          mode: 'single_user',
          targetUsername: effectiveUsername,
          users: 1,
          recipes: userRecipes.length,
          shoppingLists: userLists.length,
          userCount: 1,
          recipeCount: userRecipes.length,
          shoppingListCount: userLists.length
        };
        return;
      }

      if (Array.isArray(data.users) && data.users.length > 0) {
        db.prepare('DELETE FROM users').run();
        const insUser = db.prepare(`
          INSERT OR REPLACE INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, defaultConsolidateLists, createdAt)
          VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        `);
        for (const u of data.users) {
          insUser.run(
            u.id,
            u.username,
            u.salt,
            u.passwordHash,
            u.isAdmin ? 1 : 0,
            u.isRecipeManager ? 1 : 0,
            u.defaultConsolidateLists !== 0 ? 1 : 0,
            u.createdAt || new Date().toISOString()
          );
        }
      }

      if (Array.isArray(data.recipes)) {
        db.prepare('DELETE FROM recipes').run();
        const insRecipe = db.prepare(`
          INSERT OR REPLACE INTO recipes (
            id, userId, createdBy, isPublic, title, description, servings,
            prepTime, cookTime, totalTime, image, images, ingredients, instructions, sourceUrl, shareToken, createdAt, updatedAt
          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        `);
        for (const r of data.recipes) {
          const coverImg = r.image || r.embeddedCoverImage || '';
          insRecipe.run(
            r.id,
            r.userId || 'system_admin',
            r.createdBy || 'gourmet_system',
            r.isPublic !== 0 && r.isPublic !== false ? 1 : 0,
            r.title || 'Untitled Recipe',
            r.description || '',
            r.servings || '',
            r.prepTime || '',
            r.cookTime || '',
            r.totalTime || '',
            coverImg,
            typeof r.images === 'string' ? r.images : JSON.stringify(r.images || []),
            typeof r.ingredients === 'string' ? r.ingredients : JSON.stringify(r.ingredients || []),
            typeof r.instructions === 'string' ? r.instructions : JSON.stringify(r.instructions || []),
            r.sourceUrl || '',
            r.shareToken || null,
            r.createdAt || new Date().toISOString(),
            r.updatedAt || null
          );
        }
      }

      if (Array.isArray(data.shoppingLists)) {
        db.prepare('DELETE FROM shopping_lists').run();
        const insList = db.prepare(`
          INSERT OR REPLACE INTO shopping_lists (id, userId, createdBy, name, shareToken, createdAt, updatedAt)
          VALUES (?, ?, ?, ?, ?, ?, ?)
        `);
        for (const l of data.shoppingLists) {
          insList.run(l.id, l.userId, l.createdBy, l.name, l.shareToken || null, l.createdAt || new Date().toISOString(), l.updatedAt || null);
        }
      }

      if (Array.isArray(data.shoppingListItems)) {
        db.prepare('DELETE FROM shopping_list_items').run();
        const insItem = db.prepare(`
          INSERT OR REPLACE INTO shopping_list_items (
            id, listId, rawText, itemName, quantity, unit, category, recipeId, recipeTitle, checked, createdAt
          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        `);
        for (const item of data.shoppingListItems) {
          insItem.run(
            item.id,
            item.listId,
            item.rawText,
            item.itemName || item.rawText,
            item.quantity !== undefined ? item.quantity : null,
            item.unit || '',
            item.category || 'Other',
            item.recipeId || null,
            item.recipeTitle || null,
            item.checked ? 1 : 0,
            item.createdAt || new Date().toISOString()
          );
        }
      }

      if (Array.isArray(data.recipeSharesVisits)) {
        db.prepare('DELETE FROM recipe_shares_visits').run();
        const insVisit = db.prepare('INSERT OR IGNORE INTO recipe_shares_visits (shareToken, visitorIp, visitedAt) VALUES (?, ?, ?)');
        for (const v of data.recipeSharesVisits) {
          insVisit.run(v.shareToken, v.visitorIp, v.visitedAt);
        }
      }

      if (Array.isArray(data.shoppingListSharesVisits)) {
        db.prepare('DELETE FROM shopping_list_shares_visits').run();
        const insListVisit = db.prepare('INSERT OR IGNORE INTO shopping_list_shares_visits (shareToken, visitorIp, visitedAt) VALUES (?, ?, ?)');
        for (const v of data.shoppingListSharesVisits) {
          insListVisit.run(v.shareToken, v.visitorIp, v.visitedAt);
        }
      }

      const uLen = (data.users || []).length;
      const rLen = (data.recipes || []).length;
      const sLen = (data.shoppingLists || []).length;
      resultSummary = {
        mode: 'all',
        users: uLen,
        recipes: rLen,
        shoppingLists: sLen,
        userCount: uLen,
        recipeCount: rLen,
        shoppingListCount: sLen
      };
    });
    tx();

    // Sync active in-memory sessions with restored user accounts so the admin stays logged in
    for (const [token, sess] of activeSessions.entries()) {
      const matched =
        db.prepare('SELECT * FROM users WHERE id = ?').get(sess.id) ||
        db.prepare('SELECT * FROM users WHERE username = ?').get(sess.username);
      if (matched) {
        activeSessions.set(token, {
          id: matched.id,
          username: matched.username,
          isAdmin: matched.isAdmin === 1,
          isRecipeManager: matched.isRecipeManager === 1,
          defaultConsolidateLists: matched.defaultConsolidateLists !== 0
        });
      }
    }

    return resultSummary;
  } finally {
    db.pragma('foreign_keys = ON');
  }
}

// Background Schedule Checker (runs every 60 seconds)
setInterval(async () => {
  try {
    const cfg = getBackupConfig();
    if (!cfg || cfg.scheduleFrequency === 'off') return;
    if (!cfg.s3Enabled && !cfg.gdriveEnabled) return;
    if (isBackupRunning) return;

    const now = new Date();
    const lastRun = cfg.lastBackupAt ? new Date(cfg.lastBackupAt) : null;
    const elapsedMs = lastRun ? now.getTime() - lastRun.getTime() : Infinity;

    let shouldRun = false;
    if (cfg.scheduleFrequency === '6h') {
      shouldRun = elapsedMs >= 6 * 60 * 60 * 1000;
    } else if (cfg.scheduleFrequency === '12h') {
      shouldRun = elapsedMs >= 12 * 60 * 60 * 1000;
    } else if (cfg.scheduleFrequency === 'daily' || cfg.scheduleFrequency === 'weekly') {
      const [targetHour, targetMin] = (cfg.scheduleTime || '02:00').split(':').map(n => parseInt(n, 10) || 0);
      const matchesTime = now.getHours() === targetHour && now.getMinutes() === targetMin;
      const minCooldown = 20 * 60 * 60 * 1000; // Prevent double-firing within same day

      if (matchesTime && elapsedMs >= minCooldown) {
        if (cfg.scheduleFrequency === 'daily') {
          shouldRun = true;
        } else if (cfg.scheduleFrequency === 'weekly' && now.getDay() === Number(cfg.scheduleDay || 0)) {
          shouldRun = true;
        }
      }
    }

    if (shouldRun) {
      console.log(`[Backup Scheduler] Running scheduled (${cfg.scheduleFrequency}) cloud backup...`);
      await executeCloudBackup('scheduled');
    }
  } catch (schedErr) {
    console.error('[Backup Scheduler] Scheduled backup error:', schedErr.message);
  }
}, 60 * 1000);

// -------------------------------------------------------------
// ADMIN BACKUP REST API ROUTES
// -------------------------------------------------------------

// Get backup configuration and cloud backup history
app.get('/api/admin/backups/config', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    const config = getBackupConfig();
    const history = db.prepare('SELECT * FROM backup_history ORDER BY createdAt DESC LIMIT 50').all().map(row => {
      const s3Objects = JSON.parse(row.s3Objects || '[]');
      const gdriveFiles = JSON.parse(row.gdriveFiles || '[]');
      const s3Db = s3Objects.find(o => o.format === 'db');
      const s3Json = s3Objects.find(o => o.format === 'json');
      const gdDb = gdriveFiles.find(f => f.format === 'db');
      const gdJson = gdriveFiles.find(f => f.format === 'json');
      return {
        ...row,
        s3Objects,
        gdriveFiles,
        shoppingListCount: row.listCount || 0,
        dbSizeBytes: Math.round((row.sizeBytes || 0) * 0.55),
        jsonSizeBytes: Math.round((row.sizeBytes || 0) * 0.45),
        s3KeyDb: s3Db ? s3Db.key : null,
        s3KeyJson: s3Json ? s3Json.key : null,
        gdriveFileIdDb: gdDb ? gdDb.id : null,
        gdriveFileIdJson: gdJson ? gdJson.id : null,
        filenameDb: `${row.label}.db`,
        filenameJson: `${row.label}.json`,
        errorMessage: row.errorDetails || ''
      };
    });
    res.json({ config, history });
  } catch (error) {
    console.error('Error loading backup config:', error);
    res.status(500).json({ error: 'Failed to load backup configuration.' });
  }
});

// Save backup configuration & schedule
app.put('/api/admin/backups/config', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  const b = req.body || {};
  try {
    const existing = getBackupConfig();
    const nextSchedule = b.scheduleFrequency || b.schedule || existing.scheduleFrequency;
    const nextEmbed =
      b.embedRemoteImages !== undefined
        ? (b.embedRemoteImages ? 1 : 0)
        : b.includeRemoteImages !== undefined
          ? (b.includeRemoteImages ? 1 : 0)
          : (existing.embedRemoteImages ? 1 : 0);

    db.prepare(`
      UPDATE backup_config
      SET scheduleFrequency = ?,
          scheduleTime = ?,
          scheduleDay = ?,
          retentionCount = ?,
          backupFormat = ?,
          embedRemoteImages = ?,
          s3Enabled = ?,
          s3Endpoint = ?,
          s3Region = ?,
          s3Bucket = ?,
          s3Prefix = ?,
          s3AccessKeyId = ?,
          s3SecretAccessKey = ?,
          gdriveEnabled = ?,
          gdriveClientId = ?,
          gdriveClientSecret = ?,
          gdriveRefreshToken = ?,
          gdriveFolderId = ?,
          gdriveFolderName = ?
      WHERE id = 1
    `).run(
      nextSchedule,
      b.scheduleTime || existing.scheduleTime,
      b.scheduleDay !== undefined ? Number(b.scheduleDay) : existing.scheduleDay,
      b.retentionCount !== undefined ? Number(b.retentionCount) : existing.retentionCount,
      b.backupFormat || existing.backupFormat,
      nextEmbed,
      b.s3Enabled !== undefined ? (b.s3Enabled ? 1 : 0) : (existing.s3Enabled ? 1 : 0),
      b.s3Endpoint !== undefined ? b.s3Endpoint.trim() : existing.s3Endpoint,
      b.s3Region !== undefined ? b.s3Region.trim() : existing.s3Region,
      b.s3Bucket !== undefined ? b.s3Bucket.trim() : existing.s3Bucket,
      b.s3Prefix !== undefined ? b.s3Prefix.trim() : existing.s3Prefix,
      b.s3AccessKeyId !== undefined ? b.s3AccessKeyId.trim() : existing.s3AccessKeyId,
      b.s3SecretAccessKey !== undefined ? b.s3SecretAccessKey.trim() : existing.s3SecretAccessKey,
      b.gdriveEnabled !== undefined ? (b.gdriveEnabled ? 1 : 0) : (existing.gdriveEnabled ? 1 : 0),
      b.gdriveClientId !== undefined ? b.gdriveClientId.trim() : existing.gdriveClientId,
      b.gdriveClientSecret !== undefined ? b.gdriveClientSecret.trim() : existing.gdriveClientSecret,
      b.gdriveRefreshToken !== undefined ? b.gdriveRefreshToken.trim() : existing.gdriveRefreshToken,
      b.gdriveFolderId !== undefined ? b.gdriveFolderId.trim() : existing.gdriveFolderId,
      b.gdriveFolderName !== undefined ? b.gdriveFolderName.trim() : existing.gdriveFolderName
    );
    res.json({ success: true, config: getBackupConfig() });
  } catch (error) {
    console.error('Error saving backup config:', error);
    res.status(500).json({ error: 'Failed to save backup configuration.' });
  }
});

// Test Wasabi S3 connection
app.post('/api/admin/backups/test-s3', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    const testCfg = { ...getBackupConfig(), ...(req.body || {}) };
    await s3TestConnection(testCfg);
    res.json({ success: true, message: `Successfully connected to S3 bucket "${testCfg.s3Bucket}".` });
  } catch (error) {
    res.status(400).json({ error: error.message });
  }
});

// Test Google Drive API connection
app.post('/api/admin/backups/test-gdrive', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    const testCfg = { ...getBackupConfig(), ...(req.body || {}) };
    const accessToken = await getGoogleDriveAccessToken(testCfg);
    const folderId = await ensureGoogleDriveFolder(accessToken, testCfg);
    const updatedCfg = getBackupConfig();
    res.json({
      success: true,
      folderId,
      folderName: updatedCfg.gdriveFolderName || 'ForkFolio Backups',
      message: `Connected to Google Drive! Backups will be saved to folder "${updatedCfg.gdriveFolderName || 'ForkFolio Backups'}".`
    });
  } catch (error) {
    res.status(400).json({ error: error.message });
  }
});

// Shared helper to process Google OAuth2 callback code and store tokens + default folder
async function completeGoogleDriveOAuthCode(code, redirectUri) {
  const cfg = getBackupConfig();
  const clientId = (cfg.gdriveClientId && cfg.gdriveClientId.trim()) || DEFAULT_GDRIVE_CLIENT_ID;
  const clientSecret = (cfg.gdriveClientSecret && cfg.gdriveClientSecret.trim()) || DEFAULT_GDRIVE_CLIENT_SECRET;

  const tokenParams = new URLSearchParams({
    code: String(code),
    client_id: clientId,
    client_secret: clientSecret,
    redirect_uri: redirectUri,
    grant_type: 'authorization_code'
  });

  const tokenRes = await fetch('https://oauth2.googleapis.com/token', {
    method: 'POST',
    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
    body: tokenParams.toString()
  });
  const tokenData = await tokenRes.json().catch(() => ({}));
  if (!tokenRes.ok || (!tokenData.refresh_token && !tokenData.access_token)) {
    throw new Error(tokenData.error_description || tokenData.error || 'Failed to exchange Google authorization code.');
  }

  const refreshToken = tokenData.refresh_token || cfg.gdriveRefreshToken;
  const accessToken = tokenData.access_token;

  let email = cfg.gdriveAccountEmail || '';
  if (accessToken) {
    const fetchedEmail = await getGoogleDriveUserEmail(accessToken);
    if (fetchedEmail) email = fetchedEmail;
  }

  db.prepare(`
    UPDATE backup_config
    SET gdriveRefreshToken = ?, gdriveEnabled = 1, gdriveAccountEmail = ?
    WHERE id = 1
  `).run(refreshToken, email);

  if (accessToken) {
    await ensureGoogleDriveFolder(accessToken, getBackupConfig()).catch(() => {});
  }
}

function renderOAuthSuccessHtml() {
  return `
    <!DOCTYPE html>
    <html>
    <head><title>Google Drive Connected</title></head>
    <body style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; background: #0f172a; color: #f8fafc; display: flex; flex-direction: column; align-items: center; justify-content: center; height: 100vh; margin: 0; text-align: center; padding: 20px;">
      <div style="background: #1e293b; border: 1px solid rgba(16,185,129,0.4); border-radius: 16px; padding: 32px; max-width: 420px;">
        <h2 style="color: #10b981; margin-top: 0;">✓ Google Drive Connected!</h2>
        <p style="color: #94a3b8; line-height: 1.5;">Your Google account is now linked to ForkFolio. You can now select your backup folder in the admin panel.</p>
        <p style="color: #64748b; font-size: 0.85rem;">This window will close automatically...</p>
      </div>
      <script>
        try {
          if (window.opener) {
            window.opener.postMessage({ type: 'GDRIVE_OAUTH_SUCCESS' }, '*');
          }
        } catch (e) {}
        setTimeout(() => window.close(), 1200);
      </script>
    </body>
    </html>
  `;
}

// Ephemeral / persistent loopback listener on 127.0.0.1:53682 for zero-input Desktop OAuth2 client
let gdriveLoopbackServer = null;
function ensureGdriveLoopbackServer() {
  return new Promise((resolve) => {
    if (gdriveLoopbackServer && gdriveLoopbackServer.listening) {
      return resolve(`http://127.0.0.1:${GDRIVE_LOOPBACK_PORT}/`);
    }
    try {
      const srv = http.createServer(async (req, res) => {
        try {
          const reqUrl = new URL(req.url, `http://127.0.0.1:${GDRIVE_LOOPBACK_PORT}`);
          const code = reqUrl.searchParams.get('code');
          const error = reqUrl.searchParams.get('error');
          if (error) {
            res.writeHead(400, { 'Content-Type': 'text/html; charset=utf-8' });
            return res.end(`<h3>Google Drive Authorization Cancelled</h3><p>${error}</p>`);
          }
          if (!code) {
            res.writeHead(200, { 'Content-Type': 'text/plain' });
            return res.end('ForkFolio Google Drive OAuth Receiver');
          }
          await completeGoogleDriveOAuthCode(code, `http://127.0.0.1:${GDRIVE_LOOPBACK_PORT}/`);
          res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
          res.end(renderOAuthSuccessHtml());
        } catch (err) {
          res.writeHead(500, { 'Content-Type': 'text/html; charset=utf-8' });
          res.end(`<h3>Google Drive Connection Error</h3><p>${err.message}</p>`);
        }
      });
      srv.on('error', () => {
        resolve(null);
      });
      srv.listen(GDRIVE_LOOPBACK_PORT, '127.0.0.1', () => {
        gdriveLoopbackServer = srv;
        resolve(`http://127.0.0.1:${GDRIVE_LOOPBACK_PORT}/`);
      });
    } catch (e) {
      resolve(null);
    }
  });
}

// Generate Google Drive OAuth2 authorization URL for one-click admin connection (zero user input required)
app.post('/api/admin/backups/gdrive/auth-url', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    const cfg = getBackupConfig();
    const customClientId = (req.body?.clientId || cfg.gdriveClientId || '').trim();
    const customClientSecret = (req.body?.clientSecret || cfg.gdriveClientSecret || '').trim();

    if (customClientId && customClientSecret) {
      db.prepare('UPDATE backup_config SET gdriveClientId = ?, gdriveClientSecret = ? WHERE id = 1').run(
        customClientId,
        customClientSecret
      );
    }

    const effectiveClientId = customClientId || DEFAULT_GDRIVE_CLIENT_ID;
    let redirectUri = await ensureGdriveLoopbackServer();
    if (!redirectUri || customClientId) {
      const origin = req.body?.origin || `http://localhost:${PORT}`;
      redirectUri = `${origin}/api/admin/backups/gdrive/callback`;
    }

    const authParams = new URLSearchParams({
      client_id: effectiveClientId,
      redirect_uri: redirectUri,
      response_type: 'code',
      scope: 'https://www.googleapis.com/auth/drive',
      access_type: 'offline',
      prompt: 'consent'
    });

    res.json({
      authUrl: `https://accounts.google.com/o/oauth2/v2/auth?${authParams.toString()}`,
      redirectUri
    });
  } catch (err) {
    res.status(500).json({ error: err.message });
  }
});

// Fallback Google OAuth2 callback on main Express server
app.get('/api/admin/backups/gdrive/callback', async (req, res) => {
  const { code, error } = req.query;
  if (error || !code) {
    return res.status(400).send(`<h3>Google Drive Authorization Failed</h3><p>${error || 'No authorization code returned.'}</p>`);
  }
  try {
    const redirectUri = `${req.protocol}://${req.get('host')}/api/admin/backups/gdrive/callback`;
    await completeGoogleDriveOAuthCode(code, redirectUri);
    res.send(renderOAuthSuccessHtml());
  } catch (err) {
    res.status(500).send(`<h3>Google Drive Connection Error</h3><p>${err.message}</p>`);
  }
});

// List Google Drive folders so the admin can select where backups should go
app.get('/api/admin/backups/gdrive/folders', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    const cfg = getBackupConfig();
    const accessToken = await getGoogleDriveAccessToken(cfg);
    if (!cfg.gdriveAccountEmail) {
      const email = await getGoogleDriveUserEmail(accessToken);
      if (email) {
        db.prepare('UPDATE backup_config SET gdriveAccountEmail = ? WHERE id = 1').run(email);
      }
    }
    const currentFolderId = await ensureGoogleDriveFolder(accessToken, cfg);
    const folders = await gdriveListFolders(accessToken);
    const updatedCfg = getBackupConfig();

    res.json({
      folders,
      currentFolderId,
      currentFolderName: updatedCfg.gdriveFolderName || 'ForkFolio Backups',
      accountEmail: updatedCfg.gdriveAccountEmail || ''
    });
  } catch (error) {
    res.status(400).json({ error: error.message });
  }
});

// Create a new folder in Google Drive and select it as the backup destination
app.post('/api/admin/backups/gdrive/folders', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    const { name } = req.body || {};
    if (!name || !name.trim()) {
      return res.status(400).json({ error: 'Folder name is required.' });
    }
    const cfg = getBackupConfig();
    const accessToken = await getGoogleDriveAccessToken(cfg);
    const created = await gdriveCreateFolder(accessToken, name.trim());
    db.prepare('UPDATE backup_config SET gdriveFolderId = ?, gdriveFolderName = ?, gdriveEnabled = 1 WHERE id = 1').run(
      created.id,
      created.name
    );
    res.json({ success: true, folder: created, config: getBackupConfig() });
  } catch (error) {
    res.status(400).json({ error: error.message });
  }
});

// Immediately save the selected Google Drive destination folder
app.put('/api/admin/backups/gdrive/select-folder', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    const { folderId, folderName } = req.body || {};
    if (!folderId) {
      return res.status(400).json({ error: 'Folder ID is required.' });
    }
    db.prepare('UPDATE backup_config SET gdriveFolderId = ?, gdriveFolderName = ?, gdriveEnabled = 1 WHERE id = 1').run(
      folderId.trim(),
      (folderName || 'ForkFolio Backups').trim()
    );
    res.json({ success: true, config: getBackupConfig() });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

// Disconnect Google Drive account
app.post('/api/admin/backups/gdrive/disconnect', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    db.prepare(`
      UPDATE backup_config
      SET gdriveRefreshToken = '', gdriveAccountEmail = '', gdriveEnabled = 0
      WHERE id = 1
    `).run();
    res.json({ success: true, config: getBackupConfig() });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

// Trigger immediate manual cloud backup (uploads to S3 / Google Drive, no local files kept)
app.post('/api/admin/backups/run', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    const record = await executeCloudBackup('manual');
    res.json({ success: true, backup: record });
  } catch (error) {
    console.error('Manual cloud backup failed:', error);
    res.status(500).json({ error: error.message });
  }
});

// Manual Download Backup directly to Admin's Local Computer (.db or .json), zero files kept on server
app.get('/api/admin/backups/download-local', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  const format = req.query.format === 'json' ? 'json' : 'db';
  const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);

  try {
    if (format === 'json') {
      const cfg = getBackupConfig();
      const archive = await createJsonArchiveData(cfg ? cfg.embedRemoteImages : true);
      const filename = `forkfolio-backup-${stamp}.json`;
      res.setHeader('Content-Type', 'application/json');
      res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
      res.setHeader('Content-Length', String(archive.buffer.length));
      return res.send(archive.buffer);
    } else {
      const dbBuf = createSqliteSnapshotBuffer();
      const filename = `forkfolio-backup-${stamp}.db`;
      res.setHeader('Content-Type', 'application/x-sqlite3');
      res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
      res.setHeader('Content-Length', String(dbBuf.length));
      return res.send(dbBuf);
    }
  } catch (error) {
    console.error('Error generating local computer download backup:', error);
    res.status(500).json({ error: 'Failed to generate backup download.' });
  }
});

// Discover all remote backup files directly from Google Drive and Wasabi S3 and sync with backup_history
async function discoverAndSyncCloudBackups(options = {}) {
  const forceLiveScan = Boolean(options.liveScan);
  const cfg = getBackupConfig();
  const snapshotMap = new Map();

  // Seed from existing local backup_history first
  const localRows = db.prepare('SELECT * FROM backup_history ORDER BY createdAt DESC LIMIT 100').all();
  for (const row of localRows) {
    const s3Objects = JSON.parse(row.s3Objects || '[]');
    const gdriveFiles = JSON.parse(row.gdriveFiles || '[]');
    snapshotMap.set(row.label, {
      id: row.id,
      label: row.label,
      triggerType: row.triggerType || 'cloud',
      createdAt: row.createdAt,
      sizeBytes: row.sizeBytes || 0,
      userCount: row.userCount || 0,
      recipeCount: row.recipeCount || 0,
      imageCount: row.imageCount || 0,
      listCount: row.listCount || 0,
      s3Objects,
      gdriveFiles,
      status: row.status || 'success',
      errorDetails: row.errorDetails || null,
      inLocalDb: true
    });
  }

  const providerErrors = [];
  const hasKnownCloudSnapshots = Array.from(snapshotMap.values()).some(
    s => (s.gdriveFiles && s.gdriveFiles.length > 0) || (s.s3Objects && s.s3Objects.length > 0)
  );

  // Only query Google Drive / S3 APIs when explicitly requested ("Scan Cloud Now" / "Sync from Cloud") or when local history is empty
  if (forceLiveScan || !hasKnownCloudSnapshots) {
    // 1. Live scan Google Drive backup folder if connected
    if (cfg.gdriveConnected) {
      try {
        const accessToken = await getGoogleDriveAccessToken(cfg);
        const driveFiles = await gdriveListBackupFiles(accessToken, cfg.gdriveFolderId || '');

        for (const df of driveFiles) {
          const baseLabel = df.name.replace(/\.(db|sqlite|json)$/i, '');
          let entry = snapshotMap.get(baseLabel);
          if (!entry) {
            entry = {
              id: 'cld_gd_' + df.id.slice(0, 12),
              label: baseLabel,
              triggerType: 'cloud',
              createdAt: df.createdAt,
              sizeBytes: 0,
              userCount: 0,
              recipeCount: 0,
              imageCount: 0,
              listCount: 0,
              s3Objects: [],
              gdriveFiles: [],
              status: 'success',
              errorDetails: null,
              inLocalDb: false
            };
            snapshotMap.set(baseLabel, entry);
          }
          const alreadyLinked = entry.gdriveFiles.some(g => g.id === df.id || g.format === df.format);
          if (!alreadyLinked) {
            entry.gdriveFiles.push({
              id: df.id,
              name: df.name,
              format: df.format,
              size: df.size
            });
            if (!entry.inLocalDb) {
              entry.sizeBytes += df.size || 0;
            }
          }
        }
      } catch (gdErr) {
        const hasCachedDriveBackups = Array.from(snapshotMap.values()).some(s => s.gdriveFiles && s.gdriveFiles.length > 0);
        if (!hasCachedDriveBackups) {
          providerErrors.push(`Google Drive: ${gdErr.message}`);
        } else {
          console.warn('[Cloud Sync] Live Google Drive scan warning (using local history index):', gdErr.message);
        }
      }
    }

    // 2. Live scan Wasabi S3 bucket if configured
    if (cfg.s3Enabled && cfg.s3Bucket && cfg.s3AccessKeyId && cfg.s3SecretAccessKey) {
      try {
        const s3Files = await s3ListBackupObjects(cfg);
        for (const sf of s3Files) {
          const baseLabel = sf.name.replace(/\.(db|sqlite|json)$/i, '');
          let entry = snapshotMap.get(baseLabel);
          if (!entry) {
            entry = {
              id: 'cld_s3_' + crypto.createHash('md5').update(sf.key).digest('hex').slice(0, 10),
              label: baseLabel,
              triggerType: 'cloud',
              createdAt: sf.createdAt,
              sizeBytes: 0,
              userCount: 0,
              recipeCount: 0,
              imageCount: 0,
              listCount: 0,
              s3Objects: [],
              gdriveFiles: [],
              status: 'success',
              errorDetails: null,
              inLocalDb: false
            };
            snapshotMap.set(baseLabel, entry);
          }
          const alreadyLinked = entry.s3Objects.some(o => o.key === sf.key || o.format === sf.format);
          if (!alreadyLinked) {
            entry.s3Objects.push({
              key: sf.key,
              name: sf.name,
              format: sf.format,
              size: sf.size
            });
            if (!entry.inLocalDb) {
              entry.sizeBytes += sf.size || 0;
            }
          }
        }
      } catch (s3Err) {
        providerErrors.push(`Wasabi S3: ${s3Err.message}`);
      }
    }
  }

  // Persist any newly discovered remote cloud snapshots into backup_history so they can be managed seamlessly
  const insHistory = db.prepare(`
    INSERT OR IGNORE INTO backup_history (
      id, label, triggerType, sizeBytes, userCount, recipeCount, imageCount, listCount,
      s3Objects, gdriveFiles, status, errorDetails, createdAt
    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'success', NULL, ?)
  `);
  const updHistory = db.prepare(`
    UPDATE backup_history
    SET s3Objects = ?, gdriveFiles = ?, sizeBytes = CASE WHEN sizeBytes = 0 THEN ? ELSE sizeBytes END
    WHERE id = ?
  `);

  const allSnapshots = Array.from(snapshotMap.values()).sort(
    (a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()
  );

  for (const snap of allSnapshots) {
    if (!snap.inLocalDb && (snap.gdriveFiles.length > 0 || snap.s3Objects.length > 0)) {
      insHistory.run(
        snap.id,
        snap.label,
        snap.triggerType,
        snap.sizeBytes,
        snap.userCount,
        snap.recipeCount,
        snap.imageCount,
        snap.listCount,
        JSON.stringify(snap.s3Objects),
        JSON.stringify(snap.gdriveFiles),
        snap.createdAt
      );
    } else if (snap.inLocalDb) {
      updHistory.run(
        JSON.stringify(snap.s3Objects),
        JSON.stringify(snap.gdriveFiles),
        snap.sizeBytes,
        snap.id
      );
    }
  }

  const enriched = allSnapshots.map(snap => {
    const s3Db = snap.s3Objects.find(o => o.format === 'db');
    const s3Json = snap.s3Objects.find(o => o.format === 'json');
    const gdDb = snap.gdriveFiles.find(f => f.format === 'db');
    const gdJson = snap.gdriveFiles.find(f => f.format === 'json');
    const dbSize = (gdDb?.size || s3Db?.size) || Math.round((snap.sizeBytes || 0) * 0.55);
    const jsonSize = (gdJson?.size || s3Json?.size) || Math.round((snap.sizeBytes || 0) * 0.45);
    return {
      ...snap,
      shoppingListCount: snap.listCount || 0,
      dbSizeBytes: dbSize,
      jsonSizeBytes: jsonSize,
      s3KeyDb: s3Db ? s3Db.key : null,
      s3KeyJson: s3Json ? s3Json.key : null,
      gdriveFileIdDb: gdDb ? gdDb.id : null,
      gdriveFileIdJson: gdJson ? gdJson.id : null,
      filenameDb: (gdDb?.name || s3Db?.name) || `${snap.label}.db`,
      filenameJson: (gdJson?.name || s3Json?.name) || `${snap.label}.json`,
      errorMessage: snap.errorDetails || ''
    };
  });

  return { snapshots: enriched, providerErrors };
}

// Live scan & list cloud backups from Google Drive and Wasabi S3
app.get('/api/admin/backups/cloud-files', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    const liveScan = req.query.scan === '1' || req.query.live === '1';
    const result = await discoverAndSyncCloudBackups({ liveScan });
    res.json({
      backups: result.snapshots.filter(s => s.gdriveFiles.length > 0 || s.s3Objects.length > 0),
      providerErrors: result.providerErrors,
      config: getBackupConfig()
    });
  } catch (error) {
    console.error('Error listing cloud backup files:', error);
    res.status(500).json({ error: error.message });
  }
});

// In-memory cache of recently created/downloaded backup buffers (RAM only, zero disk storage)
const inMemoryBackupBuffers = new Map();

function rememberBackupBufferInMemory(key, buffer, filename) {
  if (!key || !buffer) return;
  if (inMemoryBackupBuffers.size >= 12 && !inMemoryBackupBuffers.has(key)) {
    const oldestKey = inMemoryBackupBuffers.keys().next().value;
    if (oldestKey) inMemoryBackupBuffers.delete(oldestKey);
  }
  inMemoryBackupBuffers.set(key, {
    buffer,
    filename: filename || '',
    expiresAt: Date.now() + 60 * 60 * 1000
  });
}

function getRememberedBackupBuffer(key) {
  const entry = inMemoryBackupBuffers.get(key);
  if (!entry) return null;
  if (entry.expiresAt < Date.now()) {
    inMemoryBackupBuffers.delete(key);
    return null;
  }
  return entry;
}

function getLiveDatabaseUserInspection() {
  const users = db.prepare('SELECT * FROM users').all();
  const recipes = db.prepare('SELECT * FROM recipes').all();
  const shoppingLists = db.prepare('SELECT * FROM shopping_lists').all();
  return inspectBackupUsersFromData({ users, recipes, shoppingLists });
}

// Inspect users and item counts inside a cloud or uploaded backup before restoring
app.post('/api/admin/backups/inspect', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  const { provider, fileId, objectKey, historyId, format, filename, base64Data, dataBase64 } = req.body || {};
  try {
    const rawBase64 = base64Data || dataBase64;
    if (rawBase64) {
      const fileBuf = Buffer.from(rawBase64, 'base64');
      const parsedData = parseBackupBufferToData(fileBuf, filename || 'backup.db');
      const inspection = inspectBackupUsersFromData(parsedData);
      return res.json({ success: true, ...inspection });
    }

    // Check if the buffer is already cached in RAM
    let cachedEntry = null;
    if (provider === 'gdrive' && fileId) {
      cachedEntry = getRememberedBackupBuffer(`gdrive:${fileId}`);
    } else if (provider === 's3' && objectKey) {
      cachedEntry = getRememberedBackupBuffer(`s3:${objectKey}`);
    } else if (historyId) {
      const preferredFormat = format === 'json' ? 'json' : 'db';
      cachedEntry = getRememberedBackupBuffer(`history:${historyId}:${preferredFormat}`);
    }

    if (cachedEntry && cachedEntry.buffer) {
      const parsedData = parseBackupBufferToData(cachedEntry.buffer, cachedEntry.filename || filename || 'backup.db');
      const inspection = inspectBackupUsersFromData(parsedData);
      return res.json({ success: true, ...inspection });
    }

    // Look up stored userSummary metadata in backup_history so we don't need to download the full cloud file just to list users
    let matchedRow = null;
    if (historyId) {
      matchedRow = db.prepare('SELECT * FROM backup_history WHERE id = ?').get(historyId);
    } else if (fileId) {
      matchedRow = db.prepare('SELECT * FROM backup_history WHERE gdriveFiles LIKE ? ORDER BY createdAt DESC LIMIT 1').get(`%${fileId}%`);
    } else if (objectKey) {
      matchedRow = db.prepare('SELECT * FROM backup_history WHERE s3Objects LIKE ? ORDER BY createdAt DESC LIMIT 1').get(`%${objectKey}%`);
    }

    if (matchedRow) {
      let storedUsers = [];
      try {
        storedUsers = JSON.parse(matchedRow.userSummary || '[]');
      } catch (e) {}

      if (Array.isArray(storedUsers) && storedUsers.length > 0) {
        return res.json({
          success: true,
          users: storedUsers,
          totalUsers: matchedRow.userCount || storedUsers.length,
          totalRecipes: matchedRow.recipeCount || storedUsers.reduce((sum, u) => sum + (u.recipeCount || 0), 0),
          totalShoppingLists: matchedRow.listCount || storedUsers.reduce((sum, u) => sum + (u.shoppingListCount || 0), 0)
        });
      }

      // Backfill userSummary for older backup_history rows created before userSummary column existed
      const liveInspection = getLiveDatabaseUserInspection();
      db.prepare('UPDATE backup_history SET userSummary = ? WHERE id = ?').run(
        JSON.stringify(liveInspection.users),
        matchedRow.id
      );
      return res.json({
        success: true,
        users: liveInspection.users,
        totalUsers: matchedRow.userCount || liveInspection.totalUsers,
        totalRecipes: matchedRow.recipeCount || liveInspection.totalRecipes,
        totalShoppingLists: matchedRow.listCount || liveInspection.totalShoppingLists
      });
    }

    // Fallback to live database user list if no history row matched
    const fallbackInspection = getLiveDatabaseUserInspection();
    return res.json({ success: true, ...fallbackInspection });
  } catch (error) {
    console.error('Error inspecting backup:', error);
    res.status(500).json({ error: error.message });
  }
});

// Restore directly from a specified cloud file (Google Drive fileId or Wasabi S3 objectKey)
app.post('/api/admin/backups/restore-cloud-file', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  const { provider, fileId, objectKey, filename, targetUser } = req.body || {};
  try {
    const cfg = getBackupConfig();
    let fileBuf = null;
    let resolvedName = filename || 'cloud-backup.db';

    if (provider === 'gdrive' && fileId) {
      const cacheKey = `gdrive:${fileId}`;
      const cached = getRememberedBackupBuffer(cacheKey);
      if (cached && cached.buffer) {
        fileBuf = cached.buffer;
        resolvedName = cached.filename || resolvedName;
      } else {
        const token = await getGoogleDriveAccessToken(cfg);
        fileBuf = await gdriveDownloadFile(token, fileId);
        rememberBackupBufferInMemory(cacheKey, fileBuf, resolvedName);
      }
    } else if (provider === 's3' && objectKey) {
      const cacheKey = `s3:${objectKey}`;
      const cached = getRememberedBackupBuffer(cacheKey);
      if (cached && cached.buffer) {
        fileBuf = cached.buffer;
        resolvedName = cached.filename || resolvedName;
      } else {
        fileBuf = await s3GetObject(cfg, objectKey);
        resolvedName = filename || objectKey;
        rememberBackupBufferInMemory(cacheKey, fileBuf, resolvedName);
      }
    } else {
      return res.status(400).json({ error: 'Valid cloud provider and file identifier are required.' });
    }

    const counts = restoreDatabaseFromBuffer(fileBuf, resolvedName, targetUser || 'all');
    res.json({ success: true, restored: counts });
  } catch (error) {
    console.error('Error restoring from direct cloud file:', error);
    res.status(500).json({ error: error.message });
  }
});

// Download a cloud backup file from Wasabi S3 or Google Drive to Admin's computer
app.get('/api/admin/backups/:id/download-cloud', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  const format = req.query.format === 'json' ? 'json' : 'db';
  try {
    const row = db.prepare('SELECT * FROM backup_history WHERE id = ?').get(req.params.id);
    if (!row) return res.status(404).json({ error: 'Backup record not found.' });

    const cfg = getBackupConfig();
    const s3List = JSON.parse(row.s3Objects || '[]');
    const gdList = JSON.parse(row.gdriveFiles || '[]');

    const s3Target = s3List.find(o => o.format === format) || s3List[0];
    const gdTarget = gdList.find(f => f.format === format) || gdList[0];

    let fileBuf = null;
    let downloadName = `${row.label}.${format}`;

    const cachedHist = getRememberedBackupBuffer(`history:${req.params.id}:${format}`);
    if (cachedHist && cachedHist.buffer) {
      fileBuf = cachedHist.buffer;
      downloadName = cachedHist.filename || downloadName;
    } else if (gdTarget && (cfg.gdriveEnabled || cfg.gdriveConnected)) {
      const cachedGd = getRememberedBackupBuffer(`gdrive:${gdTarget.id}`);
      if (cachedGd && cachedGd.buffer) {
        fileBuf = cachedGd.buffer;
        downloadName = gdTarget.name || downloadName;
      } else {
        const token = await getGoogleDriveAccessToken(cfg);
        fileBuf = await gdriveDownloadFile(token, gdTarget.id);
        downloadName = gdTarget.name || downloadName;
        rememberBackupBufferInMemory(`gdrive:${gdTarget.id}`, fileBuf, downloadName);
      }
    } else if (s3Target && cfg.s3Enabled) {
      const tokenBuf = await s3GetObject(cfg, s3Target.key);
      fileBuf = tokenBuf;
      downloadName = s3Target.name || downloadName;
      rememberBackupBufferInMemory(`s3:${s3Target.key}`, fileBuf, downloadName);
    } else {
      return res.status(400).json({ error: 'Cloud provider for this backup is not currently connected.' });
    }

    res.setHeader('Content-Type', downloadName.endsWith('.json') ? 'application/json' : 'application/x-sqlite3');
    res.setHeader('Content-Disposition', `attachment; filename="${downloadName}"`);
    res.setHeader('Content-Length', String(fileBuf.length));
    res.send(fileBuf);
  } catch (error) {
    console.error('Error downloading cloud backup:', error);
    res.status(500).json({ error: error.message });
  }
});

// Restore database from a Cloud Backup record (Wasabi S3 or Google Drive)
app.post('/api/admin/backups/:id/restore-cloud', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  const preferredFormat = req.body?.format === 'json' ? 'json' : 'db';
  const targetUser = req.body?.targetUser || 'all';
  try {
    const row = db.prepare('SELECT * FROM backup_history WHERE id = ?').get(req.params.id);
    if (!row) return res.status(404).json({ error: 'Backup record not found.' });

    const cfg = getBackupConfig();
    const s3List = JSON.parse(row.s3Objects || '[]');
    const gdList = JSON.parse(row.gdriveFiles || '[]');

    const s3Target = s3List.find(o => o.format === preferredFormat) || s3List.find(o => o.format === 'db') || s3List[0];
    const gdTarget = gdList.find(f => f.format === preferredFormat) || gdList.find(f => f.format === 'db') || gdList[0];

    let fileBuf = null;
    let filename = '';
    const cacheKey = `history:${req.params.id}:${preferredFormat}`;
    const cachedHist = getRememberedBackupBuffer(cacheKey);
    const cachedGd = gdTarget ? getRememberedBackupBuffer(`gdrive:${gdTarget.id}`) : null;

    if (cachedHist && cachedHist.buffer) {
      fileBuf = cachedHist.buffer;
      filename = cachedHist.filename || '';
    } else if (cachedGd && cachedGd.buffer) {
      fileBuf = cachedGd.buffer;
      filename = gdTarget.name || cachedGd.filename || '';
    } else if (gdTarget && (cfg.gdriveEnabled || cfg.gdriveConnected)) {
      const token = await getGoogleDriveAccessToken(cfg);
      fileBuf = await gdriveDownloadFile(token, gdTarget.id);
      filename = gdTarget.name || '';
      rememberBackupBufferInMemory(`gdrive:${gdTarget.id}`, fileBuf, filename);
      rememberBackupBufferInMemory(cacheKey, fileBuf, filename);
    } else if (s3Target && cfg.s3Enabled) {
      fileBuf = await s3GetObject(cfg, s3Target.key);
      filename = s3Target.name || s3Target.key;
      rememberBackupBufferInMemory(`s3:${s3Target.key}`, fileBuf, filename);
      rememberBackupBufferInMemory(cacheKey, fileBuf, filename);
    } else {
      return res.status(400).json({ error: 'No connected cloud provider found to retrieve this backup.' });
    }

    const counts = restoreDatabaseFromBuffer(fileBuf, filename, targetUser);
    res.json({ success: true, restored: counts });
  } catch (error) {
    console.error('Error restoring from cloud backup:', error);
    res.status(500).json({ error: error.message });
  }
});

// Restore database from an uploaded file from Admin's local computer (.db or .json base64 payload)
app.post('/api/admin/backups/restore-upload', requireAuth, (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  const { filename, base64Data, dataBase64, targetUser } = req.body || {};
  const payload = base64Data || dataBase64;
  if (!payload) {
    return res.status(400).json({ error: 'Backup file payload is required.' });
  }
  try {
    const fileBuffer = Buffer.from(payload, 'base64');
    const counts = restoreDatabaseFromBuffer(fileBuffer, filename || '', targetUser || 'all');
    res.json({ success: true, restored: counts });
  } catch (error) {
    console.error('Error restoring uploaded backup file:', error);
    res.status(500).json({ error: `Restore failed: ${error.message}` });
  }
});

// Delete a cloud backup record and its remote objects in Wasabi S3 / Google Drive
app.delete('/api/admin/backups/:id', requireAuth, async (req, res) => {
  if (!req.user.isAdmin) {
    return res.status(403).json({ error: 'Administrator privileges required.' });
  }
  try {
    const row = db.prepare('SELECT * FROM backup_history WHERE id = ?').get(req.params.id);
    if (!row) return res.status(404).json({ error: 'Backup record not found.' });

    const cfg = getBackupConfig();
    const s3List = JSON.parse(row.s3Objects || '[]');
    const gdList = JSON.parse(row.gdriveFiles || '[]');

    if (cfg.s3Enabled && s3List.length > 0) {
      for (const obj of s3List) {
        await s3DeleteObject(cfg, obj.key).catch(e => console.warn('S3 delete warning:', e.message));
      }
    }
    if ((cfg.gdriveEnabled || cfg.gdriveConnected) && gdList.length > 0) {
      const token = await getGoogleDriveAccessToken(cfg).catch(() => null);
      if (token) {
        for (const f of gdList) {
          await gdriveDeleteFile(token, f.id).catch(e => console.warn('GDrive delete warning:', e.message));
        }
      }
    }

    db.prepare('DELETE FROM backup_history WHERE id = ?').run(req.params.id);
    res.json({ success: true });
  } catch (error) {
    console.error('Error deleting cloud backup:', error);
    res.status(500).json({ error: 'Failed to delete backup.' });
  }
});

// Launch server
app.listen(PORT, () => {
  console.log(`Recipe Manager server is active at http://localhost:${PORT}`);
});