diff --git a/public/app.js b/public/app.js index c5938dc..fb24984 100644 --- a/public/app.js +++ b/public/app.js @@ -3116,6 +3116,8 @@ function getBackupFormPayload() { }; } +let lastGdriveFoldersFetchAt = 0; + function renderGdriveConnectionState(cfg) { const isConnected = Boolean(cfg.gdriveConnected); @@ -3152,7 +3154,16 @@ function renderGdriveConnectionState(cfg) { if (elements.backupGdriveFolderSelect) { elements.backupGdriveFolderSelect.disabled = !isConnected; if (!isConnected) { + lastGdriveFoldersFetchAt = 0; elements.backupGdriveFolderSelect.innerHTML = ``; + } else if (cfg.gdriveFolderId && (!elements.backupGdriveFolderSelect.value || elements.backupGdriveFolderSelect.options.length <= 1)) { + elements.backupGdriveFolderSelect.innerHTML = ''; + const opt = document.createElement('option'); + opt.value = cfg.gdriveFolderId; + opt.dataset.folderName = cfg.gdriveFolderName || 'ForkFolio Backups'; + opt.textContent = `📁 ${cfg.gdriveFolderName || 'ForkFolio Backups'}`; + opt.selected = true; + elements.backupGdriveFolderSelect.appendChild(opt); } } if (elements.btnGdriveNewFolder) { @@ -3162,7 +3173,7 @@ function renderGdriveConnectionState(cfg) { elements.btnGdriveRefreshFolders.disabled = !isConnected; } - if (isConnected) { + if (isConnected && Date.now() - lastGdriveFoldersFetchAt > 60 * 1000) { loadGoogleDriveFolders(false, cfg.gdriveFolderId, cfg.gdriveFolderName); } } @@ -3170,6 +3181,7 @@ function renderGdriveConnectionState(cfg) { async function loadGoogleDriveFolders(showFeedback = false, fallbackId = '', fallbackName = 'ForkFolio Backups') { if (!elements.backupGdriveFolderSelect) return; try { + lastGdriveFoldersFetchAt = Date.now(); if (showFeedback) { showGdriveInlineStatus('Loading folders from your Google Drive...', 'info'); } diff --git a/server.js b/server.js index f8df86c..51de31f 100644 --- a/server.js +++ b/server.js @@ -2498,6 +2498,54 @@ async function s3ListBackupObjects(cfg) { // GOOGLE DRIVE REST API V3 CONNECTION CLIENT // ------------------------------------------------------------- +// In-memory Google OAuth2 access token cache + request serializer/retry helper to avoid shared client rate limits +let cachedGdriveToken = { + accessToken: '', + refreshToken: '', + expiresAt: 0 +}; + +let gdriveRequestQueue = Promise.resolve(); + +function sleepMs(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function gdriveFetch(url, options = {}, maxRetries = 4) { + const runRequest = async () => { + let lastRes = null; + for (let attempt = 0; attempt <= maxRetries; attempt++) { + if (attempt > 0) { + const delay = Math.min(800 * Math.pow(1.8, attempt - 1) + Math.floor(Math.random() * 250), 6000); + await sleepMs(delay); + } + lastRes = await fetch(url, options); + if (lastRes.ok) { + return lastRes; + } + // Retry on rate limit (403 userRateLimitExceeded / rateLimitExceeded / 429) or transient 5xx + if (lastRes.status === 429 || lastRes.status >= 500) { + continue; + } + if (lastRes.status === 403 && attempt < maxRetries) { + const clonedText = await lastRes.clone().text().catch(() => ''); + if (/rateLimit|quota|Queries per|userRateLimit/i.test(clonedText)) { + continue; + } + } + return lastRes; + } + return lastRes; + }; + + const next = gdriveRequestQueue.then(async () => { + await sleepMs(120); + return runRequest(); + }); + gdriveRequestQueue = next.catch(() => {}); + return next; +} + async function getGoogleDriveAccessToken(cfg) { const clientId = (cfg.gdriveClientId && cfg.gdriveClientId.trim()) || DEFAULT_GDRIVE_CLIENT_ID; const clientSecret = (cfg.gdriveClientSecret && cfg.gdriveClientSecret.trim()) || DEFAULT_GDRIVE_CLIENT_SECRET; @@ -2505,6 +2553,16 @@ async function getGoogleDriveAccessToken(cfg) { if (!refreshToken) { throw new Error('Google Drive is not connected yet. Click "Authorize with Google" to connect your account.'); } + + const now = Date.now(); + if ( + cachedGdriveToken.accessToken && + cachedGdriveToken.refreshToken === refreshToken && + cachedGdriveToken.expiresAt > now + 60 * 1000 + ) { + return cachedGdriveToken.accessToken; + } + const params = new URLSearchParams({ client_id: clientId, client_secret: clientSecret, @@ -2520,12 +2578,19 @@ async function getGoogleDriveAccessToken(cfg) { if (!res.ok || !data.access_token) { throw new Error(data.error_description || data.error || `Google OAuth token exchange failed (${res.status}).`); } + + const expiresInSec = Number(data.expires_in) || 3500; + cachedGdriveToken = { + accessToken: data.access_token, + refreshToken, + expiresAt: now + expiresInSec * 1000 + }; return data.access_token; } async function getGoogleDriveUserEmail(accessToken) { try { - const res = await fetch('https://www.googleapis.com/drive/v3/about?fields=user(emailAddress,displayName)', { + const res = await gdriveFetch('https://www.googleapis.com/drive/v3/about?fields=user(emailAddress,displayName)', { headers: { Authorization: `Bearer ${accessToken}` } }); if (res.ok) { @@ -2536,10 +2601,13 @@ async function getGoogleDriveUserEmail(accessToken) { return ''; } -async function ensureGoogleDriveFolder(accessToken, cfg) { +async function ensureGoogleDriveFolder(accessToken, cfg, verifyRemote = false) { if (cfg.gdriveFolderId && cfg.gdriveFolderId.trim()) { const folderId = cfg.gdriveFolderId.trim(); - const checkRes = await fetch(`https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,trashed`, { + if (!verifyRemote) { + return folderId; + } + const checkRes = await gdriveFetch(`https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,trashed`, { headers: { Authorization: `Bearer ${accessToken}` } }); if (checkRes.ok) { @@ -2550,12 +2618,15 @@ async function ensureGoogleDriveFolder(accessToken, cfg) { } return f.id; } + } else if (checkRes.status !== 404) { + // Keep configured folderId on non-404 responses (e.g. temporary quota limit) rather than recreating + return folderId; } } // Search for existing "ForkFolio Backups" folder or create one const q = encodeURIComponent("mimeType='application/vnd.google-apps.folder' and name='ForkFolio Backups' and trashed=false"); - const searchRes = await fetch(`https://www.googleapis.com/drive/v3/files?q=${q}&fields=files(id,name)`, { + const searchRes = await gdriveFetch(`https://www.googleapis.com/drive/v3/files?q=${q}&fields=files(id,name)`, { headers: { Authorization: `Bearer ${accessToken}` } }); if (searchRes.ok) { @@ -2569,7 +2640,7 @@ async function ensureGoogleDriveFolder(accessToken, cfg) { } // Create folder - const createRes = await fetch('https://www.googleapis.com/drive/v3/files?fields=id,name', { + const createRes = await gdriveFetch('https://www.googleapis.com/drive/v3/files?fields=id,name', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -2582,7 +2653,7 @@ async function ensureGoogleDriveFolder(accessToken, cfg) { }); const created = await createRes.json().catch(() => ({})); if (!createRes.ok || !created.id) { - throw new Error('Failed to verify or create Google Drive backup folder.'); + throw new Error(created.error?.message || 'Failed to verify or create Google Drive backup folder.'); } db.prepare('UPDATE backup_config SET gdriveFolderId = ?, gdriveFolderName = ? WHERE id = 1').run(created.id, created.name || 'ForkFolio Backups'); return created.id; @@ -2590,7 +2661,7 @@ async function ensureGoogleDriveFolder(accessToken, cfg) { async function gdriveListFolders(accessToken) { const q = encodeURIComponent("mimeType='application/vnd.google-apps.folder' and trashed=false"); - const res = await fetch(`https://www.googleapis.com/drive/v3/files?q=${q}&pageSize=100&orderBy=name&fields=files(id,name,parents)`, { + const res = await gdriveFetch(`https://www.googleapis.com/drive/v3/files?q=${q}&pageSize=100&orderBy=name&fields=files(id,name,parents)`, { headers: { Authorization: `Bearer ${accessToken}` } }); const data = await res.json().catch(() => ({})); @@ -2602,7 +2673,7 @@ async function gdriveListFolders(accessToken) { async function gdriveCreateFolder(accessToken, folderName) { const cleanName = (folderName || '').trim() || 'ForkFolio Backups'; - const res = await fetch('https://www.googleapis.com/drive/v3/files?fields=id,name', { + const res = await gdriveFetch('https://www.googleapis.com/drive/v3/files?fields=id,name', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -2628,7 +2699,7 @@ async function gdriveListBackupFiles(accessToken, folderId) { clauses.push("name contains 'forkfolio-backup'"); } const q = encodeURIComponent(clauses.join(' and ')); - const res = await fetch( + const res = await gdriveFetch( `https://www.googleapis.com/drive/v3/files?q=${q}&pageSize=150&orderBy=createdTime desc&fields=files(id,name,size,createdTime,modifiedTime,mimeType,parents)`, { headers: { Authorization: `Bearer ${accessToken}` } } ); @@ -2662,7 +2733,7 @@ async function gdriveUploadFile(accessToken, folderId, filename, buffer, mimeTyp const epilogue = Buffer.from(`\r\n--${boundary}--`, 'utf8'); const multipartBody = Buffer.concat([preamble, buffer, epilogue]); - const res = await fetch('https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart&fields=id,name,size', { + const res = await gdriveFetch('https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart&fields=id,name,size', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -2679,7 +2750,7 @@ async function gdriveUploadFile(accessToken, folderId, filename, buffer, mimeTyp } async function gdriveDownloadFile(accessToken, fileId) { - const res = await fetch(`https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media`, { + const res = await gdriveFetch(`https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media`, { headers: { Authorization: `Bearer ${accessToken}` } }); if (!res.ok) { @@ -2691,7 +2762,7 @@ async function gdriveDownloadFile(accessToken, fileId) { } async function gdriveDeleteFile(accessToken, fileId) { - const res = await fetch(`https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}`, { + const res = await gdriveFetch(`https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}`, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}` } }); @@ -3797,8 +3868,7 @@ async function discoverAndSyncCloudBackups() { if (cfg.gdriveConnected) { try { const accessToken = await getGoogleDriveAccessToken(cfg); - const folderId = await ensureGoogleDriveFolder(accessToken, cfg); - const driveFiles = await gdriveListBackupFiles(accessToken, folderId); + const driveFiles = await gdriveListBackupFiles(accessToken, cfg.gdriveFolderId || ''); for (const df of driveFiles) { const baseLabel = df.name.replace(/\.(db|sqlite|json)$/i, ''); @@ -3836,7 +3906,12 @@ async function discoverAndSyncCloudBackups() { } } } catch (gdErr) { - providerErrors.push(`Google Drive: ${gdErr.message}`); + const hasCachedDriveBackups = Array.from(snapshotMap.values()).some(s => s.gdriveFiles && s.gdriveFiles.length > 0); + if (!hasCachedDriveBackups) { + providerErrors.push(`Google Drive: ${gdErr.message}`); + } else { + console.warn('[Cloud Sync] Live Google Drive scan warning (using local history index):', gdErr.message); + } } } @@ -3969,6 +4044,14 @@ app.get('/api/admin/backups/cloud-files', requireAuth, async (req, res) => { } }); +// Short-lived in-memory cache of the most recently inspected backup buffer so clicking "Restore Now" is instantaneous +let lastInspectedBackupCache = { + cacheKey: '', + buffer: null, + filename: '', + expiresAt: 0 +}; + // Inspect users and item counts inside a cloud or uploaded backup before restoring app.post('/api/admin/backups/inspect', requireAuth, async (req, res) => { if (!req.user.isAdmin) { @@ -3979,16 +4062,29 @@ app.post('/api/admin/backups/inspect', requireAuth, async (req, res) => { const cfg = getBackupConfig(); let fileBuf = null; let resolvedName = filename || 'backup.db'; + let cacheKey = ''; const rawBase64 = base64Data || dataBase64; if (rawBase64) { fileBuf = Buffer.from(rawBase64, 'base64'); } else if (provider === 'gdrive' && fileId) { - const token = await getGoogleDriveAccessToken(cfg); - fileBuf = await gdriveDownloadFile(token, fileId); + cacheKey = `gdrive:${fileId}`; + if (lastInspectedBackupCache.cacheKey === cacheKey && lastInspectedBackupCache.buffer && lastInspectedBackupCache.expiresAt > Date.now()) { + fileBuf = lastInspectedBackupCache.buffer; + resolvedName = lastInspectedBackupCache.filename || resolvedName; + } else { + const token = await getGoogleDriveAccessToken(cfg); + fileBuf = await gdriveDownloadFile(token, fileId); + } } else if (provider === 's3' && objectKey) { - fileBuf = await s3GetObject(cfg, objectKey); - resolvedName = filename || objectKey; + cacheKey = `s3:${objectKey}`; + if (lastInspectedBackupCache.cacheKey === cacheKey && lastInspectedBackupCache.buffer && lastInspectedBackupCache.expiresAt > Date.now()) { + fileBuf = lastInspectedBackupCache.buffer; + resolvedName = lastInspectedBackupCache.filename || resolvedName; + } else { + fileBuf = await s3GetObject(cfg, objectKey); + resolvedName = filename || objectKey; + } } else if (historyId) { const row = db.prepare('SELECT * FROM backup_history WHERE id = ?').get(historyId); if (!row) return res.status(404).json({ error: 'Backup record not found.' }); @@ -3998,7 +4094,11 @@ app.post('/api/admin/backups/inspect', requireAuth, async (req, res) => { const s3Target = s3List.find(o => o.format === preferredFormat) || s3List.find(o => o.format === 'db') || s3List[0]; const gdTarget = gdList.find(f => f.format === preferredFormat) || gdList.find(f => f.format === 'db') || gdList[0]; - if (gdTarget && (cfg.gdriveEnabled || cfg.gdriveConnected)) { + cacheKey = `history:${historyId}:${preferredFormat}`; + if (lastInspectedBackupCache.cacheKey === cacheKey && lastInspectedBackupCache.buffer && lastInspectedBackupCache.expiresAt > Date.now()) { + fileBuf = lastInspectedBackupCache.buffer; + resolvedName = lastInspectedBackupCache.filename || resolvedName; + } else if (gdTarget && (cfg.gdriveEnabled || cfg.gdriveConnected)) { const token = await getGoogleDriveAccessToken(cfg); fileBuf = await gdriveDownloadFile(token, gdTarget.id); resolvedName = gdTarget.name || resolvedName; @@ -4012,6 +4112,15 @@ app.post('/api/admin/backups/inspect', requireAuth, async (req, res) => { return res.status(400).json({ error: 'Backup source is required to inspect contents.' }); } + if (cacheKey && fileBuf) { + lastInspectedBackupCache = { + cacheKey, + buffer: fileBuf, + filename: resolvedName, + expiresAt: Date.now() + 10 * 60 * 1000 + }; + } + const parsedData = parseBackupBufferToData(fileBuf, resolvedName); const inspection = inspectBackupUsersFromData(parsedData); res.json({ success: true, ...inspection }); @@ -4033,11 +4142,23 @@ app.post('/api/admin/backups/restore-cloud-file', requireAuth, async (req, res) let resolvedName = filename || 'cloud-backup.db'; if (provider === 'gdrive' && fileId) { - const token = await getGoogleDriveAccessToken(cfg); - fileBuf = await gdriveDownloadFile(token, fileId); + const cacheKey = `gdrive:${fileId}`; + if (lastInspectedBackupCache.cacheKey === cacheKey && lastInspectedBackupCache.buffer && lastInspectedBackupCache.expiresAt > Date.now()) { + fileBuf = lastInspectedBackupCache.buffer; + resolvedName = lastInspectedBackupCache.filename || resolvedName; + } else { + const token = await getGoogleDriveAccessToken(cfg); + fileBuf = await gdriveDownloadFile(token, fileId); + } } else if (provider === 's3' && objectKey) { - fileBuf = await s3GetObject(cfg, objectKey); - resolvedName = filename || objectKey; + const cacheKey = `s3:${objectKey}`; + if (lastInspectedBackupCache.cacheKey === cacheKey && lastInspectedBackupCache.buffer && lastInspectedBackupCache.expiresAt > Date.now()) { + fileBuf = lastInspectedBackupCache.buffer; + resolvedName = lastInspectedBackupCache.filename || resolvedName; + } else { + fileBuf = await s3GetObject(cfg, objectKey); + resolvedName = filename || objectKey; + } } else { return res.status(400).json({ error: 'Valid cloud provider and file identifier are required.' }); } @@ -4111,8 +4232,12 @@ app.post('/api/admin/backups/:id/restore-cloud', requireAuth, async (req, res) = let fileBuf = null; let filename = ''; + const cacheKey = `history:${req.params.id}:${preferredFormat}`; - if (gdTarget && (cfg.gdriveEnabled || cfg.gdriveConnected)) { + if (lastInspectedBackupCache.cacheKey === cacheKey && lastInspectedBackupCache.buffer && lastInspectedBackupCache.expiresAt > Date.now()) { + fileBuf = lastInspectedBackupCache.buffer; + filename = lastInspectedBackupCache.filename || ''; + } else if (gdTarget && (cfg.gdriveEnabled || cfg.gdriveConnected)) { const token = await getGoogleDriveAccessToken(cfg); fileBuf = await gdriveDownloadFile(token, gdTarget.id); filename = gdTarget.name || '';