mirror of
https://gitlab.com/Zer0interrupt/recipe-manager.git
synced 2026-11-01 23:24:11 -05:00
Add structured logging (Pino), Docker log rotation, and OpenObserve + Fluent Bit observability stack
- logger.js: Pino JSON logger with console-style helpers and pino-http request logging (static assets skipped, headers not logged, secrets redacted) - server.js: replace 89 console.* calls with log.info/warn/error; add request logging middleware - docker-compose.yml: json-file rotation (10m x 5, compressed), LOG_LEVEL, 'forkfolio' log tag - observability/: standalone OpenObserve + Fluent Bit stack routing tagged container logs to per-app streams - .dockerignore: exclude observability stack from app image
This commit is contained in:
Chris
parent
311c03136c
commit
423334db4d
12 changed files with 558 additions and 89 deletions
2
.dockerignore
Normal file
2
.dockerignore
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
# Keep the standalone observability stack (and its data) out of the app image
|
||||
observability/
|
||||
|
|
@ -12,3 +12,14 @@ services:
|
|||
environment:
|
||||
- PORT=3000
|
||||
- NODE_ENV=production
|
||||
- LOG_LEVEL=info
|
||||
# Rotate container logs (~50 MB max on disk). The "tag" is used by the
|
||||
# observability stack's Fluent Bit collector to route these logs to the
|
||||
# "forkfolio" stream in OpenObserve (see observability/README.md).
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
compress: "true"
|
||||
tag: "forkfolio"
|
||||
|
|
|
|||
89
logger.js
Normal file
89
logger.js
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
/**
|
||||
* Centralized structured logging for ForkFolio.
|
||||
*
|
||||
* - Emits newline-delimited JSON to stdout (collected by Docker, rotated by the
|
||||
* json-file log driver, and forwarded to OpenObserve by Fluent Bit).
|
||||
* - `log.*` helpers accept console-style arguments, so existing
|
||||
* `console.error('Message:', err)` call sites translate 1:1 while Error
|
||||
* objects are serialized with their stack trace under the `err` key.
|
||||
* - `httpLogger` is Express middleware that logs one line per HTTP request.
|
||||
*
|
||||
* Environment variables:
|
||||
* LOG_LEVEL trace | debug | info | warn | error | fatal | silent (default: info)
|
||||
*/
|
||||
const crypto = require('crypto');
|
||||
const util = require('util');
|
||||
const pino = require('pino');
|
||||
const pinoHttp = require('pino-http');
|
||||
|
||||
const logger = pino({
|
||||
level: process.env.LOG_LEVEL || 'info',
|
||||
base: { app: 'forkfolio' },
|
||||
timestamp: pino.stdTimeFunctions.isoTime,
|
||||
formatters: {
|
||||
// Use readable level names ("info") instead of numeric levels (30)
|
||||
level: (label) => ({ level: label })
|
||||
},
|
||||
redact: {
|
||||
paths: ['password', '*.password', 'token', '*.token', 'apiKey', '*.apiKey'],
|
||||
censor: '[REDACTED]'
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Builds a console-compatible log function for the given pino level.
|
||||
* The first Error argument is attached as `err`; remaining arguments are
|
||||
* formatted into the message exactly like console.log would format them.
|
||||
*/
|
||||
function consoleStyle(level) {
|
||||
return (...args) => {
|
||||
const err = args.find((a) => a instanceof Error);
|
||||
const rest = err ? args.filter((a) => a !== err) : args;
|
||||
const msg = util.format(...rest);
|
||||
if (err) {
|
||||
// Drop the trailing "Message:" colon since the error is attached separately
|
||||
logger[level]({ err }, msg.replace(/:\s*$/, ''));
|
||||
} else {
|
||||
logger[level](msg);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const log = {
|
||||
debug: consoleStyle('debug'),
|
||||
info: consoleStyle('info'),
|
||||
warn: consoleStyle('warn'),
|
||||
error: consoleStyle('error')
|
||||
};
|
||||
|
||||
// Static asset requests are high-volume and low-value; skip them in request logs
|
||||
const STATIC_ASSET_RE = /\.(?:js|css|png|jpe?g|gif|svg|ico|webp|woff2?|ttf|map|json|html)$/i;
|
||||
|
||||
const httpLogger = pinoHttp({
|
||||
logger,
|
||||
genReqId: (req, res) => {
|
||||
const id = req.headers['x-request-id'] || crypto.randomUUID();
|
||||
res.setHeader('X-Request-Id', id);
|
||||
return id;
|
||||
},
|
||||
autoLogging: {
|
||||
ignore: (req) => STATIC_ASSET_RE.test((req.url || '').split('?')[0])
|
||||
},
|
||||
customLogLevel: (req, res, err) => {
|
||||
if (err || res.statusCode >= 500) return 'error';
|
||||
if (res.statusCode >= 400) return 'warn';
|
||||
return 'info';
|
||||
},
|
||||
// Keep request logs compact and never record headers (Authorization, cookies)
|
||||
serializers: {
|
||||
req: (req) => ({
|
||||
id: req.id,
|
||||
method: req.method,
|
||||
url: req.url,
|
||||
ip: (req.headers && req.headers['x-forwarded-for']) || req.remoteAddress
|
||||
}),
|
||||
res: (res) => ({ statusCode: res.statusCode })
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = { logger, log, httpLogger };
|
||||
11
observability/.env.example
Normal file
11
observability/.env.example
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# Copy to .env and change the password before first start.
|
||||
# The root user is created on OpenObserve's first launch; changing these later
|
||||
# does not update an existing user (change it in the UI instead).
|
||||
ZO_ROOT_USER_EMAIL=admin@example.com
|
||||
ZO_ROOT_USER_PASSWORD=ChangeMe-StrongPassword123!
|
||||
|
||||
# OpenObserve organization that Fluent Bit writes to
|
||||
ZO_ORG=default
|
||||
|
||||
# Default retention for all streams (days). Per-stream overrides can be set in the UI.
|
||||
ZO_COMPACT_DATA_RETENTION_DAYS=30
|
||||
2
observability/.gitignore
vendored
Normal file
2
observability/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
.env
|
||||
openobserve-data/
|
||||
71
observability/README.md
Normal file
71
observability/README.md
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
# Observability stack (OpenObserve + Fluent Bit)
|
||||
|
||||
A standalone, self-hosted log pipeline shared by every app on this Docker host.
|
||||
|
||||
```
|
||||
ForkFolio (Pino JSON → stdout)
|
||||
└─► Docker json-file driver (rotated: 10 MB × 5, tag "forkfolio")
|
||||
└─► Fluent Bit (tails /var/lib/docker/containers, routes by tag)
|
||||
└─► OpenObserve stream "forkfolio" (search, dashboards, alerts, retention)
|
||||
```
|
||||
|
||||
## First-time setup
|
||||
|
||||
```bash
|
||||
cd observability
|
||||
cp .env.example .env # set a strong ZO_ROOT_USER_PASSWORD
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
1. Open **http://localhost:5080** and sign in with the credentials from `.env`.
|
||||
2. Recreate ForkFolio so its new logging options apply:
|
||||
```bash
|
||||
cd ..
|
||||
docker compose up -d --build
|
||||
```
|
||||
3. In OpenObserve go to **Logs → stream `forkfolio`**. The stream appears after the first log line arrives.
|
||||
|
||||
## Example queries (SQL mode)
|
||||
|
||||
```sql
|
||||
-- All errors
|
||||
SELECT * FROM "forkfolio" WHERE level = 'error' ORDER BY _timestamp DESC
|
||||
|
||||
-- Slow API requests (> 2s)
|
||||
SELECT req_method, req_url, responsetime FROM "forkfolio" WHERE responsetime > 2000
|
||||
|
||||
-- Scraper activity
|
||||
SELECT * FROM "forkfolio" WHERE msg LIKE '%[Puppeteer]%'
|
||||
```
|
||||
|
||||
OpenObserve flattens nested fields, so `req.url` becomes `req_url`. Field names are lowercased.
|
||||
|
||||
## Adding another app
|
||||
|
||||
1. Add to the app's `docker-compose.yml`:
|
||||
```yaml
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
tag: "myapp"
|
||||
```
|
||||
2. Copy the `[OUTPUT]` block in `fluent-bit/fluent-bit.conf`, change `Match` to `app.myapp` and the URI stream to `/api/${ZO_ORG}/myapp/_json`.
|
||||
3. Run `docker compose restart fluent-bit`.
|
||||
|
||||
If you'd rather collect untagged containers too, uncomment the catch-all output, which sends them to a `docker` stream.
|
||||
|
||||
## Retention and rotation
|
||||
|
||||
| Layer | Setting | Default |
|
||||
|---|---|---|
|
||||
| Docker (local disk) | `max-size` × `max-file` in each app's compose file | 10 MB × 5 ≈ 50 MB |
|
||||
| OpenObserve (long-term) | `ZO_COMPACT_DATA_RETENTION_DAYS` in `.env`, or per stream in the UI | 30 days |
|
||||
|
||||
## Notes
|
||||
|
||||
- **Log level:** set `LOG_LEVEL` (`debug`, `info`, `warn`, `error`) in ForkFolio's compose file.
|
||||
- **Docker Desktop (Windows/macOS):** the `/var/lib/docker/containers` mount refers to the Docker VM, and works with the WSL2 backend. On a Linux server it works as-is.
|
||||
- **Exposure:** port 5080 is published on all interfaces. Put it behind a reverse proxy with TLS, or bind it to `127.0.0.1:5080:5080`, if the host is reachable from the internet.
|
||||
- **Data:** stored in `./openobserve-data` (git-ignored). Back it up if you want to keep log history.
|
||||
56
observability/docker-compose.yml
Normal file
56
observability/docker-compose.yml
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
# Standalone observability stack: OpenObserve (storage/search/UI) + Fluent Bit (collector).
|
||||
#
|
||||
# Runs independently of ForkFolio so any number of apps on this Docker host can
|
||||
# ship logs here. Fluent Bit tails every container's json-file log and routes
|
||||
# containers that set a logging "tag" to a matching OpenObserve stream.
|
||||
#
|
||||
# Usage:
|
||||
# cp .env.example .env # then set a strong password
|
||||
# docker compose up -d
|
||||
# open http://localhost:5080
|
||||
name: observability
|
||||
|
||||
services:
|
||||
openobserve:
|
||||
# Pin to a specific release tag for reproducible upgrades if desired
|
||||
image: public.ecr.aws/zinclabs/openobserve:latest
|
||||
container_name: openobserve
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "5080:5080"
|
||||
env_file: .env
|
||||
environment:
|
||||
- ZO_DATA_DIR=/data
|
||||
volumes:
|
||||
- ./openobserve-data:/data
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
fluent-bit:
|
||||
image: fluent/fluent-bit:latest
|
||||
container_name: fluent-bit
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- openobserve
|
||||
env_file: .env
|
||||
environment:
|
||||
- ZO_HOST=openobserve
|
||||
- ZO_PORT=5080
|
||||
volumes:
|
||||
- ./fluent-bit/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf:ro
|
||||
- ./fluent-bit/parsers.conf:/fluent-bit/etc/custom_parsers.conf:ro
|
||||
# Docker's container log files (read-only)
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
# Tail positions, so restarts don't re-send or skip logs
|
||||
- fluent-bit-db:/fluent-bit/db
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
volumes:
|
||||
fluent-bit-db:
|
||||
75
observability/fluent-bit/fluent-bit.conf
Normal file
75
observability/fluent-bit/fluent-bit.conf
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
[SERVICE]
|
||||
Flush 5
|
||||
Log_Level info
|
||||
Parsers_File /fluent-bit/etc/custom_parsers.conf
|
||||
storage.path /fluent-bit/db/buffer
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# INPUT: every container's json-file log on this Docker host
|
||||
# ---------------------------------------------------------------------------
|
||||
[INPUT]
|
||||
Name tail
|
||||
Tag docker.*
|
||||
Path /var/lib/docker/containers/*/*-json.log
|
||||
multiline.parser docker
|
||||
DB /fluent-bit/db/tail.db
|
||||
storage.type filesystem
|
||||
Mem_Buf_Limit 20MB
|
||||
Skip_Long_Lines On
|
||||
Refresh_Interval 10
|
||||
Rotate_Wait 30
|
||||
|
||||
# Apps that log JSON (e.g. ForkFolio via Pino) get their fields lifted to the
|
||||
# top level. Non-JSON lines fail to parse and are kept unchanged in "log".
|
||||
[FILTER]
|
||||
Name parser
|
||||
Match docker.*
|
||||
Key_Name log
|
||||
Parser app_json
|
||||
Reserve_Data On
|
||||
Preserve_Key Off
|
||||
|
||||
# Route by the container's logging "tag" option (docker-compose: logging.options.tag).
|
||||
# A container tagged "forkfolio" becomes "app.forkfolio". Untagged containers
|
||||
# keep their docker.* tag and are dropped unless the catch-all output is enabled.
|
||||
[FILTER]
|
||||
Name rewrite_tag
|
||||
Match docker.*
|
||||
Rule $attrs['tag'] ^([A-Za-z0-9_-]+)$ app.$1 false
|
||||
Emitter_Name app_router
|
||||
Emitter_Storage.type filesystem
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OUTPUTS: one per app -> its own OpenObserve stream
|
||||
# To add an app: set logging.options.tag in its compose file, then copy this
|
||||
# block and change Match (app.<tag>) and the stream name in URI.
|
||||
# ---------------------------------------------------------------------------
|
||||
[OUTPUT]
|
||||
Name http
|
||||
Match app.forkfolio
|
||||
Host ${ZO_HOST}
|
||||
Port ${ZO_PORT}
|
||||
URI /api/${ZO_ORG}/forkfolio/_json
|
||||
Format json
|
||||
Json_date_key _timestamp
|
||||
Json_date_format iso8601
|
||||
HTTP_User ${ZO_ROOT_USER_EMAIL}
|
||||
HTTP_Passwd ${ZO_ROOT_USER_PASSWORD}
|
||||
compress gzip
|
||||
Retry_Limit 10
|
||||
|
||||
# Optional catch-all: uncomment to ship logs from untagged containers to a
|
||||
# shared "docker" stream (includes OpenObserve's and Fluent Bit's own logs).
|
||||
# [OUTPUT]
|
||||
# Name http
|
||||
# Match docker.*
|
||||
# Host ${ZO_HOST}
|
||||
# Port ${ZO_PORT}
|
||||
# URI /api/${ZO_ORG}/docker/_json
|
||||
# Format json
|
||||
# Json_date_key _timestamp
|
||||
# Json_date_format iso8601
|
||||
# HTTP_User ${ZO_ROOT_USER_EMAIL}
|
||||
# HTTP_Passwd ${ZO_ROOT_USER_PASSWORD}
|
||||
# compress gzip
|
||||
# Retry_Limit 10
|
||||
5
observability/fluent-bit/parsers.conf
Normal file
5
observability/fluent-bit/parsers.conf
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
# Parses JSON log lines emitted by applications (e.g. Pino in ForkFolio).
|
||||
# No Time_Key: the record timestamp comes from Docker's log entry.
|
||||
[PARSER]
|
||||
Name app_json
|
||||
Format json
|
||||
142
package-lock.json
generated
142
package-lock.json
generated
|
|
@ -14,11 +14,19 @@
|
|||
"cors": "^2.8.5",
|
||||
"express": "^4.19.2",
|
||||
"express-rate-limit": "^8.5.2",
|
||||
"pino": "^10.4.0",
|
||||
"pino-http": "^11.0.0",
|
||||
"puppeteer": "^25.1.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@pinojs/redact": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
|
||||
"integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@puppeteer/browsers": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.0.4.tgz",
|
||||
|
|
@ -151,6 +159,15 @@
|
|||
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/atomic-sleep": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
|
||||
"integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "1.16.1",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-1.16.1.tgz",
|
||||
|
|
@ -1638,6 +1655,15 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/on-exit-leak-free": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
|
||||
"integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/on-finished": {
|
||||
"version": "2.4.1",
|
||||
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
|
||||
|
|
@ -1732,6 +1758,55 @@
|
|||
"integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pino": {
|
||||
"version": "10.4.0",
|
||||
"resolved": "https://registry.npmjs.org/pino/-/pino-10.4.0.tgz",
|
||||
"integrity": "sha512-bk1ZMTwG/Vymx+zPoa28MhNzucFvXJWp3csrSuxuHPSMeUettaCoQ5Jvg07mtC4JGNpcfTwz41oVCnNgBK9+vA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@pinojs/redact": "^0.4.0",
|
||||
"atomic-sleep": "^1.0.0",
|
||||
"on-exit-leak-free": "^2.1.0",
|
||||
"pino-abstract-transport": "^3.0.0",
|
||||
"pino-std-serializers": "^7.0.0",
|
||||
"process-warning": "^5.0.0",
|
||||
"quick-format-unescaped": "^4.0.3",
|
||||
"real-require": "^1.0.0",
|
||||
"safe-stable-stringify": "^2.3.1",
|
||||
"sonic-boom": "^4.0.1",
|
||||
"thread-stream": "^4.0.0"
|
||||
},
|
||||
"bin": {
|
||||
"pino": "bin.js"
|
||||
}
|
||||
},
|
||||
"node_modules/pino-abstract-transport": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz",
|
||||
"integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"split2": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pino-http": {
|
||||
"version": "11.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pino-http/-/pino-http-11.0.0.tgz",
|
||||
"integrity": "sha512-wqg5XIAGRRIWtTk8qPGxkbrfiwEWz1lgedVLvhLALudKXvg1/L2lTFgTGPJ4Z2e3qcRmxoFxDuSdMdMGNM6I1g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"get-caller-file": "^2.0.5",
|
||||
"pino": "^10.0.0",
|
||||
"pino-std-serializers": "^7.0.0",
|
||||
"process-warning": "^5.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pino-std-serializers": {
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz",
|
||||
"integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/prebuild-install": {
|
||||
"version": "7.1.3",
|
||||
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
|
||||
|
|
@ -1759,6 +1834,22 @@
|
|||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/process-warning": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.1.1.tgz",
|
||||
"integrity": "sha512-oCkNVj4OKnKEcocVBcDfiIugN1/cE6rgfT3nCtzeqZPA8+rIxMaQ0xXzIkGP8Gj2qUaP1UR7uhJ1VD8lUCpzCg==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fastify"
|
||||
},
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/fastify"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/proxy-addr": {
|
||||
"version": "2.0.7",
|
||||
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
|
||||
|
|
@ -2095,6 +2186,12 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/quick-format-unescaped": {
|
||||
"version": "4.0.4",
|
||||
"resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz",
|
||||
"integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/range-parser": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
|
||||
|
|
@ -2160,6 +2257,12 @@
|
|||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/real-require": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz",
|
||||
"integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
|
|
@ -2205,6 +2308,15 @@
|
|||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/safe-stable-stringify": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz",
|
||||
"integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
|
|
@ -2427,6 +2539,24 @@
|
|||
"simple-concat": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/sonic-boom": {
|
||||
"version": "4.2.1",
|
||||
"resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz",
|
||||
"integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"atomic-sleep": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/split2": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
|
||||
"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">= 10.x"
|
||||
}
|
||||
},
|
||||
"node_modules/statuses": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
|
||||
|
|
@ -2508,6 +2638,18 @@
|
|||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/thread-stream": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz",
|
||||
"integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"real-require": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/toidentifier": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
"cors": "^2.8.5",
|
||||
"express": "^4.19.2",
|
||||
"express-rate-limit": "^8.5.2",
|
||||
"pino": "^10.4.0",
|
||||
"pino-http": "^11.0.0",
|
||||
"puppeteer": "^25.1.0",
|
||||
"puppeteer-extra": "^3.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2"
|
||||
|
|
|
|||
181
server.js
181
server.js
|
|
@ -11,6 +11,7 @@ const puppeteer = require('puppeteer-extra');
|
|||
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
||||
const Database = require('better-sqlite3');
|
||||
const rateLimit = require('express-rate-limit');
|
||||
const { log, httpLogger } = require('./logger');
|
||||
|
||||
// Enable stealth mode to bypass bot detection (Cloudflare, Akamai, etc.)
|
||||
puppeteer.use(StealthPlugin());
|
||||
|
|
@ -51,7 +52,7 @@ try {
|
|||
db.prepare('ALTER TABLE users ADD COLUMN defaultConsolidateLists INTEGER NOT NULL DEFAULT 1').run();
|
||||
}
|
||||
} catch (colErr) {
|
||||
console.error('Failed to check/add defaultConsolidateLists column:', colErr);
|
||||
log.error('Failed to check/add defaultConsolidateLists column:', colErr);
|
||||
}
|
||||
|
||||
db.prepare(`
|
||||
|
|
@ -231,13 +232,13 @@ try {
|
|||
1,
|
||||
new Date().toISOString()
|
||||
);
|
||||
console.log('--- ForkFolio Admin Initialized ---');
|
||||
console.log('Default Admin Username: admin');
|
||||
console.log('Default Admin Password: admin123');
|
||||
console.log('--------------------------------------');
|
||||
log.info('--- ForkFolio Admin Initialized ---');
|
||||
log.info('Default Admin Username: admin');
|
||||
log.info('Default Admin Password: admin123');
|
||||
log.info('--------------------------------------');
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to initialize default admin and system users:', e);
|
||||
log.error('Failed to initialize default admin and system users:', e);
|
||||
}
|
||||
|
||||
// Automatic Migration from legacy JSON databases if they exist
|
||||
|
|
@ -255,7 +256,7 @@ try {
|
|||
const data = fs.readFileSync(LEGACY_USERS_FILE, 'utf8');
|
||||
const users = JSON.parse(data);
|
||||
if (Array.isArray(users) && users.length > 0) {
|
||||
console.log(`[Migration] Found ${users.length} legacy users. Migrating to SQLite...`);
|
||||
log.info(`[Migration] Found ${users.length} legacy users. Migrating to SQLite...`);
|
||||
const insertUser = db.prepare(`
|
||||
INSERT OR IGNORE INTO users (id, username, salt, passwordHash, isAdmin, isRecipeManager, createdAt)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
|
|
@ -275,7 +276,7 @@ try {
|
|||
}
|
||||
});
|
||||
transaction(users);
|
||||
console.log(`[Migration] User migration completed.`);
|
||||
log.info(`[Migration] User migration completed.`);
|
||||
}
|
||||
fs.renameSync(LEGACY_USERS_FILE, backupUsersFile);
|
||||
}
|
||||
|
|
@ -286,7 +287,7 @@ try {
|
|||
const data = fs.readFileSync(LEGACY_RECIPES_FILE, 'utf8');
|
||||
const recipes = JSON.parse(data);
|
||||
if (Array.isArray(recipes) && recipes.length > 0) {
|
||||
console.log(`[Migration] Found ${recipes.length} legacy recipes. Migrating to SQLite...`);
|
||||
log.info(`[Migration] Found ${recipes.length} legacy recipes. Migrating to SQLite...`);
|
||||
const insertRecipe = db.prepare(`
|
||||
INSERT OR IGNORE INTO recipes (
|
||||
id, userId, createdBy, isPublic, title, description, servings,
|
||||
|
|
@ -323,7 +324,7 @@ try {
|
|||
}
|
||||
});
|
||||
transaction(recipes);
|
||||
console.log(`[Migration] Recipe migration completed.`);
|
||||
log.info(`[Migration] Recipe migration completed.`);
|
||||
}
|
||||
fs.renameSync(LEGACY_RECIPES_FILE, backupRecipesFile);
|
||||
}
|
||||
|
|
@ -334,9 +335,9 @@ try {
|
|||
const data = fs.readFileSync(LEGACY_SETTINGS_FILE, 'utf8');
|
||||
const settings = JSON.parse(data);
|
||||
if (settings && settings.geminiApiKey) {
|
||||
console.log('[Migration] Migrating system settings key to SQLite...');
|
||||
log.info('[Migration] Migrating system settings key to SQLite...');
|
||||
db.prepare('UPDATE settings SET geminiApiKey = ?').run(settings.geminiApiKey.trim());
|
||||
console.log('[Migration] Settings migration completed.');
|
||||
log.info('[Migration] Settings migration completed.');
|
||||
}
|
||||
fs.renameSync(LEGACY_SETTINGS_FILE, backupSettingsFile);
|
||||
}
|
||||
|
|
@ -346,12 +347,14 @@ try {
|
|||
} catch (migrationError) {
|
||||
// Guarantee foreign keys re-enabled even on crash
|
||||
db.pragma('foreign_keys = ON');
|
||||
console.error('[Migration] Failed to perform automatic SQLite migration:', migrationError);
|
||||
log.error('[Migration] Failed to perform automatic SQLite migration:', migrationError);
|
||||
}
|
||||
|
||||
// In-memory sessions storage (token -> { id, username, isAdmin, isRecipeManager })
|
||||
const activeSessions = new Map();
|
||||
|
||||
// Structured request logging (one JSON line per request; static assets skipped)
|
||||
app.use(httpLogger);
|
||||
app.use(cors());
|
||||
app.use(express.json({ limit: '100mb', type: ['application/json', 'text/plain'] }));
|
||||
app.use(express.urlencoded({ limit: '100mb', extended: true }));
|
||||
|
|
@ -462,7 +465,7 @@ function formatRecipe(row) {
|
|||
const res = db.prepare('SELECT COUNT(*) as count FROM recipe_shares_visits WHERE shareToken = ?').get(row.shareToken);
|
||||
visitorCount = res ? res.count : 0;
|
||||
} catch (e) {
|
||||
console.error('Failed to query unique visitor count:', e);
|
||||
log.error('Failed to query unique visitor count:', e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -521,7 +524,7 @@ app.post('/api/auth/register', requireAuth, (req, res) => {
|
|||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error during registration:', error);
|
||||
log.error('Error during registration:', error);
|
||||
res.status(500).json({ error: 'Failed to create account.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -566,7 +569,7 @@ app.post('/api/auth/login', (req, res) => {
|
|||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error during login:', error);
|
||||
log.error('Error during login:', error);
|
||||
res.status(500).json({ error: 'Login failed.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -588,7 +591,7 @@ app.get('/api/admin/users', requireAuth, (req, res) => {
|
|||
}));
|
||||
res.json(users);
|
||||
} catch (error) {
|
||||
console.error('Error listing users:', error);
|
||||
log.error('Error listing users:', error);
|
||||
res.status(500).json({ error: 'Failed to retrieve registered users.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -642,7 +645,7 @@ app.delete('/api/admin/users/:id', requireAuth, (req, res) => {
|
|||
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error deleting user:', error);
|
||||
log.error('Error deleting user:', error);
|
||||
res.status(500).json({ error: 'Failed to delete user.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -680,7 +683,7 @@ app.put('/api/admin/users/:id/role', requireAuth, (req, res) => {
|
|||
|
||||
res.json({ success: true, user: { id: user.id, username: user.username, isAdmin } });
|
||||
} catch (error) {
|
||||
console.error('Error changing user role:', error);
|
||||
log.error('Error changing user role:', error);
|
||||
res.status(500).json({ error: 'Failed to update user role.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -714,7 +717,7 @@ app.put('/api/admin/users/:id/recipe-manager', requireAuth, (req, res) => {
|
|||
|
||||
res.json({ success: true, user: { id: user.id, username: user.username, isRecipeManager } });
|
||||
} catch (error) {
|
||||
console.error('Error toggling manager role:', error);
|
||||
log.error('Error toggling manager role:', error);
|
||||
res.status(500).json({ error: 'Failed to update manager permission.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -739,7 +742,7 @@ app.get('/api/admin/shared-recipes', requireAuth, (req, res) => {
|
|||
const stats = db.prepare('SELECT COUNT(*) as count FROM recipe_shares_visits WHERE shareToken = ?').get(r.shareToken);
|
||||
visitorCount = stats ? stats.count : 0;
|
||||
} catch (err) {
|
||||
console.error('Failed to query shared recipe visitor count:', err);
|
||||
log.error('Failed to query shared recipe visitor count:', err);
|
||||
}
|
||||
return {
|
||||
id: r.id,
|
||||
|
|
@ -753,7 +756,7 @@ app.get('/api/admin/shared-recipes', requireAuth, (req, res) => {
|
|||
|
||||
res.json(sharedRecipes);
|
||||
} catch (error) {
|
||||
console.error('Error fetching admin shared recipes:', error);
|
||||
log.error('Error fetching admin shared recipes:', error);
|
||||
res.status(500).json({ error: 'Failed to retrieve shared recipes.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -765,7 +768,7 @@ app.get('/api/recipes', requireAuth, (req, res) => {
|
|||
const userRecipes = rows.map(formatRecipe);
|
||||
res.json(userRecipes);
|
||||
} catch (error) {
|
||||
console.error('Error fetching recipes:', error);
|
||||
log.error('Error fetching recipes:', error);
|
||||
res.status(500).json({ error: 'Failed to retrieve recipes.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -777,7 +780,7 @@ app.get('/api/recipes/public', requireAuth, (req, res) => {
|
|||
const publicRecipes = rows.map(formatRecipe);
|
||||
res.json(publicRecipes);
|
||||
} catch (error) {
|
||||
console.error('Error fetching public recipes:', error);
|
||||
log.error('Error fetching public recipes:', error);
|
||||
res.status(500).json({ error: 'Failed to retrieve community recipes.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -826,7 +829,7 @@ app.post('/api/recipes/clone/:id', requireAuth, (req, res) => {
|
|||
const clonedObj = formatRecipe(db.prepare('SELECT * FROM recipes WHERE id = ?').get(clonedId));
|
||||
res.status(201).json(clonedObj);
|
||||
} catch (error) {
|
||||
console.error('Error cloning recipe:', error);
|
||||
log.error('Error cloning recipe:', error);
|
||||
res.status(500).json({ error: 'Failed to clone recipe.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -867,7 +870,7 @@ app.post('/api/recipes', requireAuth, (req, res) => {
|
|||
const savedObj = formatRecipe(db.prepare('SELECT * FROM recipes WHERE id = ?').get(newId));
|
||||
res.status(201).json(savedObj);
|
||||
} catch (error) {
|
||||
console.error('Error saving new recipe:', error);
|
||||
log.error('Error saving new recipe:', error);
|
||||
res.status(500).json({ error: 'Failed to save recipe.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -933,7 +936,7 @@ app.put('/api/recipes/:id', requireAuth, (req, res) => {
|
|||
const updatedObj = formatRecipe(db.prepare('SELECT * FROM recipes WHERE id = ?').get(req.params.id));
|
||||
res.json(updatedObj);
|
||||
} catch (error) {
|
||||
console.error('Error updating recipe:', error);
|
||||
log.error('Error updating recipe:', error);
|
||||
res.status(500).json({ error: 'Failed to update recipe.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -954,7 +957,7 @@ app.delete('/api/recipes/:id', requireAuth, (req, res) => {
|
|||
db.prepare('DELETE FROM recipes WHERE id = ?').run(req.params.id);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error deleting recipe:', error);
|
||||
log.error('Error deleting recipe:', error);
|
||||
res.status(500).json({ error: 'Failed to delete recipe.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -1294,7 +1297,7 @@ function isBotChallengePage(html) {
|
|||
|
||||
// Helper: Fetch full HTML via Jina Reader HTML mode to bypass edge bot protection (PerimeterX, Cloudflare, etc.)
|
||||
async function fetchWithReaderProxy(url) {
|
||||
console.log('[ReaderProxy] Fetching rendered HTML for:', url);
|
||||
log.info('[ReaderProxy] Fetching rendered HTML for:', url);
|
||||
const response = await axios.get(`https://r.jina.ai/${url}`, {
|
||||
headers: {
|
||||
'X-Return-Format': 'html',
|
||||
|
|
@ -1307,7 +1310,7 @@ async function fetchWithReaderProxy(url) {
|
|||
|
||||
// Helper: Fetch HTML using Puppeteer with stealth (headless browser fallback)
|
||||
async function fetchWithPuppeteer(url) {
|
||||
console.log('[Puppeteer] Launching headless browser for:', url);
|
||||
log.info('[Puppeteer] Launching headless browser for:', url);
|
||||
let browser = null;
|
||||
try {
|
||||
browser = await puppeteer.launch({
|
||||
|
|
@ -1342,12 +1345,12 @@ async function fetchWithPuppeteer(url) {
|
|||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
|
||||
const html = await page.content();
|
||||
console.log('[Puppeteer] Successfully retrieved', html.length, 'bytes of HTML');
|
||||
log.info('[Puppeteer] Successfully retrieved', html.length, 'bytes of HTML');
|
||||
return html;
|
||||
} finally {
|
||||
if (browser) {
|
||||
await browser.close();
|
||||
console.log('[Puppeteer] Browser closed');
|
||||
log.info('[Puppeteer] Browser closed');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1471,7 +1474,7 @@ function parseRecipeCaptionHeuristic(rawCaption, author, cover, sourceUrl) {
|
|||
|
||||
// Helper: Scrape recipe from a TikTok video URL using caption extraction and AI parsing
|
||||
async function scrapeTikTokRecipe(inputUrl) {
|
||||
console.log('[TikTok] Starting scrape for:', inputUrl);
|
||||
log.info('[TikTok] Starting scrape for:', inputUrl);
|
||||
|
||||
// 1. Resolve redirect for shortened links (vm.tiktok.com, vt.tiktok.com)
|
||||
let canonicalUrl = inputUrl;
|
||||
|
|
@ -1488,7 +1491,7 @@ async function scrapeTikTokRecipe(inputUrl) {
|
|||
canonicalUrl = headRes.request.res.responseUrl;
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[TikTok] Redirect resolution note:', e.message);
|
||||
log.warn('[TikTok] Redirect resolution note:', e.message);
|
||||
}
|
||||
|
||||
// 2. Fetch page HTML (Strategy 1: Fast Axios fetch)
|
||||
|
|
@ -1505,7 +1508,7 @@ async function scrapeTikTokRecipe(inputUrl) {
|
|||
});
|
||||
html = response.data;
|
||||
} catch (err) {
|
||||
console.warn('[TikTok] Direct fetch failed, trying Puppeteer:', err.message);
|
||||
log.warn('[TikTok] Direct fetch failed, trying Puppeteer:', err.message);
|
||||
}
|
||||
|
||||
// Strategy 2: Puppeteer fallback if direct fetch failed or returned bot screen
|
||||
|
|
@ -1513,7 +1516,7 @@ async function scrapeTikTokRecipe(inputUrl) {
|
|||
try {
|
||||
html = await fetchWithPuppeteer(canonicalUrl);
|
||||
} catch (pupErr) {
|
||||
console.error('[TikTok] Puppeteer fallback failed:', pupErr.message);
|
||||
log.error('[TikTok] Puppeteer fallback failed:', pupErr.message);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1537,7 +1540,7 @@ async function scrapeTikTokRecipe(inputUrl) {
|
|||
cover = item.video?.cover || item.video?.dynamicCover || '';
|
||||
}
|
||||
} catch (parseErr) {
|
||||
console.warn('[TikTok] Failed to parse rehydration JSON:', parseErr.message);
|
||||
log.warn('[TikTok] Failed to parse rehydration JSON:', parseErr.message);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1559,13 +1562,13 @@ async function scrapeTikTokRecipe(inputUrl) {
|
|||
$('[data-e2e="user-title"]').text().trim() || '';
|
||||
}
|
||||
|
||||
console.log('[TikTok] Extracted caption length:', caption.length, 'Author:', author);
|
||||
log.info('[TikTok] Extracted caption length:', caption.length, 'Author:', author);
|
||||
if (!caption) {
|
||||
throw new Error('No caption or description found on this TikTok video. The creator might not have written recipe details.');
|
||||
}
|
||||
|
||||
// Fast & Local Heuristic Engine First (Zero API cost, instant execution)
|
||||
console.log('[TikTok] Running local heuristic parser on video caption...');
|
||||
log.info('[TikTok] Running local heuristic parser on video caption...');
|
||||
const heuristicResult = parseRecipeCaptionHeuristic(caption, author, cover, canonicalUrl);
|
||||
if (!heuristicResult.ingredients || heuristicResult.ingredients.length === 0) {
|
||||
throw new Error('This TikTok caption does not appear to contain an ingredients list. The recipe might be in the audio, video comments, or on-screen text.');
|
||||
|
|
@ -1637,7 +1640,7 @@ Return ONLY a valid JSON object matching this schema without markdown code block
|
|||
throw new Error('Gemini returned an empty response.');
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[AI Polish] Error:', err.response?.data?.error?.message || err.message);
|
||||
log.error('[AI Polish] Error:', err.response?.data?.error?.message || err.message);
|
||||
const msg = err.response?.data?.error?.message || err.message;
|
||||
return res.status(500).json({ error: `AI Polish failed: ${msg}` });
|
||||
}
|
||||
|
|
@ -1655,7 +1658,7 @@ app.get('/api/settings', requireAuth, (req, res) => {
|
|||
defaultConsolidateLists
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Failed to load settings:', e);
|
||||
log.error('Failed to load settings:', e);
|
||||
res.status(500).json({ error: 'Failed to read settings' });
|
||||
}
|
||||
});
|
||||
|
|
@ -1682,7 +1685,7 @@ app.post('/api/settings', requireAuth, (req, res) => {
|
|||
defaultConsolidateLists: userRow ? userRow.defaultConsolidateLists !== 0 : true
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Failed to save settings:', e);
|
||||
log.error('Failed to save settings:', e);
|
||||
res.status(500).json({ error: 'Failed to save settings' });
|
||||
}
|
||||
});
|
||||
|
|
@ -1730,7 +1733,7 @@ app.post('/api/settings/discover-models', requireAuth, async (req, res) => {
|
|||
return res.status(500).json({ error: 'Unexpected response from Google Gemini API.' });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Model discovery error:', err.response?.data?.error?.message || err.message);
|
||||
log.error('Model discovery error:', err.response?.data?.error?.message || err.message);
|
||||
const msg = err.response?.data?.error?.message || err.message;
|
||||
return res.status(err.response?.status || 500).json({ error: `Failed to fetch models: ${msg}` });
|
||||
}
|
||||
|
|
@ -1747,7 +1750,7 @@ app.put('/api/user/preferences', requireAuth, (req, res) => {
|
|||
req.user.defaultConsolidateLists = defaultConsolidateLists;
|
||||
res.json({ success: true, defaultConsolidateLists });
|
||||
} catch (e) {
|
||||
console.error('Failed to update user preferences:', e);
|
||||
log.error('Failed to update user preferences:', e);
|
||||
res.status(500).json({ error: 'Failed to update preferences.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -1977,7 +1980,7 @@ function formatShoppingList(listRow) {
|
|||
const stats = db.prepare('SELECT COUNT(*) as count FROM shopping_list_shares_visits WHERE shareToken = ?').get(listRow.shareToken);
|
||||
visitorCount = stats ? stats.count : 0;
|
||||
} catch (e) {
|
||||
console.error('Failed to query shopping list visitor count:', e);
|
||||
log.error('Failed to query shopping list visitor count:', e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2003,7 +2006,7 @@ app.get('/api/shopping-lists', requireAuth, (req, res) => {
|
|||
}
|
||||
res.json(rows.map(formatShoppingList));
|
||||
} catch (error) {
|
||||
console.error('Error fetching shopping lists:', error);
|
||||
log.error('Error fetching shopping lists:', error);
|
||||
res.status(500).json({ error: 'Failed to retrieve shopping lists.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2024,7 +2027,7 @@ app.post('/api/shopping-lists', requireAuth, (req, res) => {
|
|||
const created = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(id));
|
||||
res.status(201).json(created);
|
||||
} catch (error) {
|
||||
console.error('Error creating shopping list:', error);
|
||||
log.error('Error creating shopping list:', error);
|
||||
res.status(500).json({ error: 'Failed to create shopping list.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2045,7 +2048,7 @@ app.put('/api/shopping-lists/:id', requireAuth, (req, res) => {
|
|||
const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(req.params.id));
|
||||
res.json(updated);
|
||||
} catch (error) {
|
||||
console.error('Error renaming shopping list:', error);
|
||||
log.error('Error renaming shopping list:', error);
|
||||
res.status(500).json({ error: 'Failed to rename shopping list.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2067,7 +2070,7 @@ app.delete('/api/shopping-lists/:id', requireAuth, (req, res) => {
|
|||
delTx();
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error deleting shopping list:', error);
|
||||
log.error('Error deleting shopping list:', error);
|
||||
res.status(500).json({ error: 'Failed to delete shopping list.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2123,7 +2126,7 @@ app.post('/api/shopping-lists/:id/items', requireAuth, (req, res) => {
|
|||
const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
|
||||
res.status(201).json(updated);
|
||||
} catch (error) {
|
||||
console.error('Error adding shopping list items:', error);
|
||||
log.error('Error adding shopping list items:', error);
|
||||
res.status(500).json({ error: 'Failed to add items to shopping list.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2155,7 +2158,7 @@ app.put('/api/shopping-lists/:id/items/check', requireAuth, (req, res) => {
|
|||
const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
|
||||
res.json(updated);
|
||||
} catch (error) {
|
||||
console.error('Error toggling shopping list items:', error);
|
||||
log.error('Error toggling shopping list items:', error);
|
||||
res.status(500).json({ error: 'Failed to update item state.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2182,7 +2185,7 @@ app.post('/api/shopping-lists/:id/items/delete', requireAuth, (req, res) => {
|
|||
const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
|
||||
res.json(updated);
|
||||
} catch (error) {
|
||||
console.error('Error deleting shopping list items:', error);
|
||||
log.error('Error deleting shopping list items:', error);
|
||||
res.status(500).json({ error: 'Failed to delete items.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2207,7 +2210,7 @@ app.post('/api/shopping-lists/:id/clear', requireAuth, (req, res) => {
|
|||
const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
|
||||
res.json(updated);
|
||||
} catch (error) {
|
||||
console.error('Error clearing shopping list:', error);
|
||||
log.error('Error clearing shopping list:', error);
|
||||
res.status(500).json({ error: 'Failed to clear shopping list.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2230,7 +2233,7 @@ app.post('/api/shopping-lists/:id/share', requireAuth, (req, res) => {
|
|||
const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
|
||||
res.json(updated);
|
||||
} catch (error) {
|
||||
console.error('Error sharing shopping list:', error);
|
||||
log.error('Error sharing shopping list:', error);
|
||||
res.status(500).json({ error: 'Failed to generate shopping list share link.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2256,7 +2259,7 @@ app.post('/api/shopping-lists/:id/revoke-share', requireAuth, (req, res) => {
|
|||
const updated = formatShoppingList(db.prepare('SELECT * FROM shopping_lists WHERE id = ?').get(existing.id));
|
||||
res.json(updated);
|
||||
} catch (error) {
|
||||
console.error('Error revoking shopping list share link:', error);
|
||||
log.error('Error revoking shopping list share link:', error);
|
||||
res.status(500).json({ error: 'Failed to revoke shopping list share link.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2280,7 +2283,7 @@ app.get('/api/share/shopping-lists/:shareToken', publicShareLimiter, (req, res)
|
|||
VALUES (?, ?, ?)
|
||||
`).run(shareToken, ip, new Date().toISOString());
|
||||
} catch (visitErr) {
|
||||
console.error('Failed to log shared shopping list visitor IP:', visitErr);
|
||||
log.error('Failed to log shared shopping list visitor IP:', visitErr);
|
||||
}
|
||||
|
||||
const formatted = formatShoppingList(listRow);
|
||||
|
|
@ -2292,7 +2295,7 @@ app.get('/api/share/shopping-lists/:shareToken', publicShareLimiter, (req, res)
|
|||
items: formatted.items
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error serving shared shopping list:', error);
|
||||
log.error('Error serving shared shopping list:', error);
|
||||
res.status(500).json({ error: 'Internal server error while retrieving shared shopping list.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2333,7 +2336,7 @@ app.put('/api/share/shopping-lists/:shareToken/items/check', publicShareLimiter,
|
|||
items: formatted.items
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error updating shared shopping list items:', error);
|
||||
log.error('Error updating shared shopping list items:', error);
|
||||
res.status(500).json({ error: 'Failed to update shared shopping list.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2359,7 +2362,7 @@ app.get('/api/share/recipes/:shareToken', publicShareLimiter, (req, res) => {
|
|||
VALUES (?, ?, ?)
|
||||
`).run(shareToken, ip, new Date().toISOString());
|
||||
} catch (visitErr) {
|
||||
console.error('Failed to log shared visitor IP:', visitErr);
|
||||
log.error('Failed to log shared visitor IP:', visitErr);
|
||||
}
|
||||
|
||||
// Strip sensitive user identifiers and internal database keys
|
||||
|
|
@ -2381,7 +2384,7 @@ app.get('/api/share/recipes/:shareToken', publicShareLimiter, (req, res) => {
|
|||
|
||||
res.json(sanitizedRecipe);
|
||||
} catch (error) {
|
||||
console.error('Error serving shared recipe:', error);
|
||||
log.error('Error serving shared recipe:', error);
|
||||
res.status(500).json({ error: 'Internal server error while retrieving shared recipe.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2414,7 +2417,7 @@ app.post('/api/recipes/:id/revoke-share', requireAuth, (req, res) => {
|
|||
performRevocation();
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error revoking recipe share link:', error);
|
||||
log.error('Error revoking recipe share link:', error);
|
||||
res.status(500).json({ error: 'Failed to revoke shared link.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -2432,7 +2435,7 @@ app.post('/api/scrape', requireAuth, async (req, res) => {
|
|||
const recipeData = await scrapeTikTokRecipe(url);
|
||||
return res.json(recipeData);
|
||||
} catch (tiktokError) {
|
||||
console.error('[TikTok Scraper] Error:', tiktokError.message);
|
||||
log.error('[TikTok Scraper] Error:', tiktokError.message);
|
||||
return res.status(422).json({
|
||||
error: tiktokError.message || 'Failed to extract recipe from TikTok video.'
|
||||
});
|
||||
|
|
@ -2466,13 +2469,13 @@ app.post('/api/scrape', requireAuth, async (req, res) => {
|
|||
html = response.data;
|
||||
|
||||
if (isBotChallengePage(html)) {
|
||||
console.log('[Scraper] Axios response appears to be a bot challenge page, falling back...');
|
||||
log.info('[Scraper] Axios response appears to be a bot challenge page, falling back...');
|
||||
html = null;
|
||||
}
|
||||
} catch (axiosError) {
|
||||
const status = axiosError.response ? axiosError.response.status : 0;
|
||||
if (status === 403 || status === 503 || status === 429) {
|
||||
console.log(`[Scraper] Axios blocked with HTTP ${status}, falling back...`);
|
||||
log.info(`[Scraper] Axios blocked with HTTP ${status}, falling back...`);
|
||||
} else {
|
||||
throw axiosError;
|
||||
}
|
||||
|
|
@ -2486,10 +2489,10 @@ app.post('/api/scrape', requireAuth, async (req, res) => {
|
|||
html = readerHtml;
|
||||
scrapingMethod = 'reader-proxy';
|
||||
} else {
|
||||
console.log('[ReaderProxy] Returned challenge page, falling back to Puppeteer...');
|
||||
log.info('[ReaderProxy] Returned challenge page, falling back to Puppeteer...');
|
||||
}
|
||||
} catch (readerError) {
|
||||
console.warn('[ReaderProxy] Fallback failed, trying Puppeteer:', readerError.message);
|
||||
log.warn('[ReaderProxy] Fallback failed, trying Puppeteer:', readerError.message);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2504,7 +2507,7 @@ app.post('/api/scrape', requireAuth, async (req, res) => {
|
|||
throw new Error('Blocked by bot challenge page in headless browser.');
|
||||
}
|
||||
} catch (puppeteerError) {
|
||||
console.error('[Puppeteer] Fallback also failed:', puppeteerError.message);
|
||||
log.error('[Puppeteer] Fallback also failed:', puppeteerError.message);
|
||||
return res.status(500).json({
|
||||
error: 'This website could not be accessed. All scraping methods were blocked.',
|
||||
details: puppeteerError.message,
|
||||
|
|
@ -2518,7 +2521,7 @@ app.post('/api/scrape', requireAuth, async (req, res) => {
|
|||
|
||||
res.json(recipeData);
|
||||
} catch (error) {
|
||||
console.error('Error occurred while scraping URL:', error.message);
|
||||
log.error('Error occurred while scraping URL:', error.message);
|
||||
const status = error.response ? error.response.status : 500;
|
||||
const message = error.response && error.response.status === 403
|
||||
? 'This website is protected and blocks automated access. The Puppeteer fallback was also unavailable.'
|
||||
|
|
@ -2574,7 +2577,7 @@ function createSqliteSnapshotBuffer() {
|
|||
fs.unlinkSync(tempPath);
|
||||
}
|
||||
} catch (cleanupErr) {
|
||||
console.warn('[Backup] Temp file cleanup warning:', cleanupErr.message);
|
||||
log.warn('[Backup] Temp file cleanup warning:', cleanupErr.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3306,7 +3309,7 @@ async function pruneOldCloudBackups(cfg) {
|
|||
const s3List = JSON.parse(run.s3Objects || '[]');
|
||||
if (cfg.s3Enabled && s3List.length > 0) {
|
||||
for (const obj of s3List) {
|
||||
await s3DeleteObject(cfg, obj.key).catch(e => console.warn('[Retention] S3 delete warning:', e.message));
|
||||
await s3DeleteObject(cfg, obj.key).catch(e => log.warn('[Retention] S3 delete warning:', e.message));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3317,14 +3320,14 @@ async function pruneOldCloudBackups(cfg) {
|
|||
}
|
||||
if (gdriveToken) {
|
||||
for (const f of gdList) {
|
||||
await gdriveDeleteFile(gdriveToken, f.id).catch(e => console.warn('[Retention] GDrive delete warning:', e.message));
|
||||
await gdriveDeleteFile(gdriveToken, f.id).catch(e => log.warn('[Retention] GDrive delete warning:', e.message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
db.prepare('DELETE FROM backup_history WHERE id = ?').run(run.id);
|
||||
} catch (pruneErr) {
|
||||
console.warn('[Retention] Failed pruning backup record:', pruneErr.message);
|
||||
log.warn('[Retention] Failed pruning backup record:', pruneErr.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3779,11 +3782,11 @@ setInterval(async () => {
|
|||
}
|
||||
|
||||
if (shouldRun) {
|
||||
console.log(`[Backup Scheduler] Running scheduled (${cfg.scheduleFrequency}) cloud backup...`);
|
||||
log.info(`[Backup Scheduler] Running scheduled (${cfg.scheduleFrequency}) cloud backup...`);
|
||||
await executeCloudBackup('scheduled');
|
||||
}
|
||||
} catch (schedErr) {
|
||||
console.error('[Backup Scheduler] Scheduled backup error:', schedErr.message);
|
||||
log.error('[Backup Scheduler] Scheduled backup error:', schedErr.message);
|
||||
}
|
||||
}, 60 * 1000);
|
||||
|
||||
|
|
@ -3823,7 +3826,7 @@ app.get('/api/admin/backups/config', requireAuth, (req, res) => {
|
|||
});
|
||||
res.json({ config, history });
|
||||
} catch (error) {
|
||||
console.error('Error loading backup config:', error);
|
||||
log.error('Error loading backup config:', error);
|
||||
res.status(500).json({ error: 'Failed to load backup configuration.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -3889,7 +3892,7 @@ app.put('/api/admin/backups/config', requireAuth, (req, res) => {
|
|||
);
|
||||
res.json({ success: true, config: getBackupConfig() });
|
||||
} catch (error) {
|
||||
console.error('Error saving backup config:', error);
|
||||
log.error('Error saving backup config:', error);
|
||||
res.status(500).json({ error: 'Failed to save backup configuration.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -4194,7 +4197,7 @@ app.post('/api/admin/backups/run', requireAuth, async (req, res) => {
|
|||
const record = await executeCloudBackup('manual');
|
||||
res.json({ success: true, backup: record });
|
||||
} catch (error) {
|
||||
console.error('Manual cloud backup failed:', error);
|
||||
log.error('Manual cloud backup failed:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
|
@ -4225,7 +4228,7 @@ app.get('/api/admin/backups/download-local', requireAuth, async (req, res) => {
|
|||
return res.send(dbBuf);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error generating local computer download backup:', error);
|
||||
log.error('Error generating local computer download backup:', error);
|
||||
res.status(500).json({ error: 'Failed to generate backup download.' });
|
||||
}
|
||||
});
|
||||
|
|
@ -4312,7 +4315,7 @@ async function discoverAndSyncCloudBackups(options = {}) {
|
|||
if (!hasCachedDriveBackups) {
|
||||
providerErrors.push(`Google Drive: ${gdErr.message}`);
|
||||
} else {
|
||||
console.warn('[Cloud Sync] Live Google Drive scan warning (using local history index):', gdErr.message);
|
||||
log.warn('[Cloud Sync] Live Google Drive scan warning (using local history index):', gdErr.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -4443,7 +4446,7 @@ app.get('/api/admin/backups/cloud-files', requireAuth, async (req, res) => {
|
|||
config: getBackupConfig()
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error listing cloud backup files:', error);
|
||||
log.error('Error listing cloud backup files:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
|
@ -4558,7 +4561,7 @@ app.post('/api/admin/backups/inspect', requireAuth, async (req, res) => {
|
|||
const fallbackInspection = getLiveDatabaseUserInspection();
|
||||
return res.json({ success: true, ...fallbackInspection });
|
||||
} catch (error) {
|
||||
console.error('Error inspecting backup:', error);
|
||||
log.error('Error inspecting backup:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
|
@ -4603,7 +4606,7 @@ app.post('/api/admin/backups/restore-cloud-file', requireAuth, async (req, res)
|
|||
const counts = restoreDatabaseFromBuffer(fileBuf, resolvedName, targetUser || 'all');
|
||||
res.json({ success: true, restored: counts });
|
||||
} catch (error) {
|
||||
console.error('Error restoring from direct cloud file:', error);
|
||||
log.error('Error restoring from direct cloud file:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
|
@ -4657,7 +4660,7 @@ app.get('/api/admin/backups/:id/download-cloud', requireAuth, async (req, res) =
|
|||
res.setHeader('Content-Length', String(fileBuf.length));
|
||||
res.send(fileBuf);
|
||||
} catch (error) {
|
||||
console.error('Error downloading cloud backup:', error);
|
||||
log.error('Error downloading cloud backup:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
|
@ -4710,7 +4713,7 @@ app.post('/api/admin/backups/:id/restore-cloud', requireAuth, async (req, res) =
|
|||
const counts = restoreDatabaseFromBuffer(fileBuf, filename, targetUser);
|
||||
res.json({ success: true, restored: counts });
|
||||
} catch (error) {
|
||||
console.error('Error restoring from cloud backup:', error);
|
||||
log.error('Error restoring from cloud backup:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
|
@ -4730,7 +4733,7 @@ app.post('/api/admin/backups/restore-upload', requireAuth, (req, res) => {
|
|||
const counts = restoreDatabaseFromBuffer(fileBuffer, filename || '', targetUser || 'all');
|
||||
res.json({ success: true, restored: counts });
|
||||
} catch (error) {
|
||||
console.error('Error restoring uploaded backup file:', error);
|
||||
log.error('Error restoring uploaded backup file:', error);
|
||||
res.status(500).json({ error: `Restore failed: ${error.message}` });
|
||||
}
|
||||
});
|
||||
|
|
@ -4750,14 +4753,14 @@ app.delete('/api/admin/backups/:id', requireAuth, async (req, res) => {
|
|||
|
||||
if (cfg.s3Enabled && s3List.length > 0) {
|
||||
for (const obj of s3List) {
|
||||
await s3DeleteObject(cfg, obj.key).catch(e => console.warn('S3 delete warning:', e.message));
|
||||
await s3DeleteObject(cfg, obj.key).catch(e => log.warn('S3 delete warning:', e.message));
|
||||
}
|
||||
}
|
||||
if ((cfg.gdriveEnabled || cfg.gdriveConnected) && gdList.length > 0) {
|
||||
const token = await getGoogleDriveAccessToken(cfg).catch(() => null);
|
||||
if (token) {
|
||||
for (const f of gdList) {
|
||||
await gdriveDeleteFile(token, f.id).catch(e => console.warn('GDrive delete warning:', e.message));
|
||||
await gdriveDeleteFile(token, f.id).catch(e => log.warn('GDrive delete warning:', e.message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -4765,12 +4768,12 @@ app.delete('/api/admin/backups/:id', requireAuth, async (req, res) => {
|
|||
db.prepare('DELETE FROM backup_history WHERE id = ?').run(req.params.id);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error deleting cloud backup:', error);
|
||||
log.error('Error deleting cloud backup:', error);
|
||||
res.status(500).json({ error: 'Failed to delete backup.' });
|
||||
}
|
||||
});
|
||||
|
||||
// Launch server
|
||||
app.listen(PORT, () => {
|
||||
console.log(`Recipe Manager server is active at http://localhost:${PORT}`);
|
||||
log.info(`Recipe Manager server is active at http://localhost:${PORT}`);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue