From efa5f0bfe003edce5f96b82b93f854b5c9cb56ad Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 9 Jan 2026 02:51:11 +0100 Subject: [PATCH] feat: improve logs output and docs --- CHANGELOG.md | 1 + docs/cli/index.md | 8 ++ docs/gateway/logging.md | 2 + docs/logging.md | 144 ++++++++++++++++++++++++++++++ src/cli/logs-cli.ts | 154 +++++++++++++++++++++++++++++++-- src/commands/providers/logs.ts | 66 +------------- src/logging/parse-log-line.ts | 63 ++++++++++++++ 7 files changed, 368 insertions(+), 70 deletions(-) create mode 100644 docs/logging.md create mode 100644 src/logging/parse-log-line.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 508717c71..9303094d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- CLI: improve `logs` output (pretty/plain/JSONL), add gateway unreachable hint, and document logging. - WhatsApp: route queued replies to the original sender instead of the bot's own number. (#534) — thanks @mcinteerj - Models: add OAuth expiry checks in doctor, expanded `models status` auth output (missing auth + `--check` exit codes). (#538) — thanks @latitudeki5223 - Deps: bump Pi to 0.40.0 and drop pi-ai patch (upstream 429 fix). (#543) — thanks @mcinteerj diff --git a/docs/cli/index.md b/docs/cli/index.md index c9d19812b..0e73e9120 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -20,6 +20,7 @@ This page describes the current CLI behavior. If commands change, update this do - ANSI colors and progress indicators only render in TTY sessions. - OSC-8 hyperlinks render as clickable links in supported terminals; otherwise we fall back to plain URLs. - `--json` (and `--plain` where supported) disables styling for clean output. +- `--no-color` disables ANSI styling where supported; `NO_COLOR=1` is also respected. - Long-running commands show a progress indicator (OSC 9;4 when supported). ## Color palette @@ -443,10 +444,17 @@ Notes: ### `logs` Tail Gateway file logs via RPC. +Notes: +- TTY sessions render a colorized, structured view; non-TTY falls back to plain text. +- `--json` emits line-delimited JSON (one log event per line). + Examples: ```bash clawdbot logs --follow clawdbot logs --limit 200 +clawdbot logs --plain +clawdbot logs --json +clawdbot logs --no-color ``` ### `gateway ` diff --git a/docs/gateway/logging.md b/docs/gateway/logging.md index 2a8ed3478..0c91e2160 100644 --- a/docs/gateway/logging.md +++ b/docs/gateway/logging.md @@ -7,6 +7,8 @@ read_when: # Logging +For a user-facing overview (CLI + Control UI + config), see [/logging](/logging). + Clawdbot has two log “surfaces”: - **Console output** (what you see in the terminal / Debug UI). diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 000000000..1f718be69 --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,144 @@ +--- +summary: "Logging overview: file logs, console output, CLI tailing, and the Control UI" +read_when: + - You need a beginner-friendly overview of logging + - You want to configure log levels or formats + - You are troubleshooting and need to find logs quickly +--- + +# Logging + +Clawdbot logs in two places: + +- **File logs** (JSON lines) written by the Gateway. +- **Console output** shown in terminals and the Control UI. + +This page explains where logs live, how to read them, and how to configure log +levels and formats. + +## Where logs live + +By default, the Gateway writes a rolling log file under: + +`/tmp/clawdbot/clawdbot-YYYY-MM-DD.log` + +You can override this in `~/.clawdbot/clawdbot.json`: + +```json +{ + "logging": { + "file": "/path/to/clawdbot.log" + } +} +``` + +## How to read logs + +### CLI: live tail (recommended) + +Use the CLI to tail the gateway log file via RPC: + +```bash +clawdbot logs --follow +``` + +Output modes: + +- **TTY sessions**: pretty, colorized, structured log lines. +- **Non-TTY sessions**: plain text. +- `--json`: line-delimited JSON (one log event per line). +- `--plain`: force plain text in TTY sessions. +- `--no-color`: disable ANSI colors. + +In JSON mode, the CLI emits `type`-tagged objects: + +- `meta`: stream metadata (file, cursor, size) +- `log`: parsed log entry +- `notice`: truncation / rotation hints +- `raw`: unparsed log line + +If the Gateway is unreachable, the CLI prints a short hint to run: + +```bash +clawdbot doctor +``` + +### Control UI (web) + +The Control UI’s **Logs** tab tails the same file using `logs.tail`. +See [/web/control-ui](/web/control-ui) for how to open it. + +### Provider-only logs + +To filter provider activity (WhatsApp/Telegram/etc), use: + +```bash +clawdbot providers logs --provider whatsapp +``` + +## Log formats + +### File logs (JSONL) + +Each line in the log file is a JSON object. The CLI and Control UI parse these +entries to render structured output (time, level, subsystem, message). + +### Console output + +Console logs are **TTY-aware** and formatted for readability: + +- Subsystem prefixes (e.g. `gateway/providers/whatsapp`) +- Level coloring (info/warn/error) +- Optional compact or JSON mode + +Console formatting is controlled by `logging.consoleStyle`. + +## Configuring logging + +All logging configuration lives under `logging` in `~/.clawdbot/clawdbot.json`. + +```json +{ + "logging": { + "level": "info", + "file": "/tmp/clawdbot/clawdbot-YYYY-MM-DD.log", + "consoleLevel": "info", + "consoleStyle": "pretty", + "redactSensitive": "tools", + "redactPatterns": [ + "sk-.*" + ] + } +} +``` + +### Log levels + +- `logging.level`: **file logs** (JSONL) level. +- `logging.consoleLevel`: **console** verbosity level. + +`--verbose` only affects console output; it does not change file log levels. + +### Console styles + +`logging.consoleStyle`: + +- `pretty`: human-friendly, colored, with timestamps. +- `compact`: tighter output (best for long sessions). +- `json`: JSON per line (for log processors). + +### Redaction + +Tool summaries can redact sensitive tokens before they hit the console: + +- `logging.redactSensitive`: `off` | `tools` (default: `tools`) +- `logging.redactPatterns`: list of regex strings to override the default set + +Redaction affects **console output only** and does not alter file logs. + +## Troubleshooting tips + +- **Gateway not reachable?** Run `clawdbot doctor` first. +- **Logs empty?** Check that the Gateway is running and writing to the file path + in `logging.file`. +- **Need more detail?** Set `logging.level` to `debug` or `trace` and retry. diff --git a/src/cli/logs-cli.ts b/src/cli/logs-cli.ts index 6cb4dc660..e55222396 100644 --- a/src/cli/logs-cli.ts +++ b/src/cli/logs-cli.ts @@ -1,6 +1,9 @@ import { setTimeout as delay } from "node:timers/promises"; import type { Command } from "commander"; +import { buildGatewayConnectionDetails } from "../gateway/call.js"; +import { parseLogLine } from "../logging/parse-log-line.js"; import { defaultRuntime } from "../runtime.js"; +import { colorize, isRich, theme } from "../terminal/theme.js"; import { addGatewayClientOptions, callGatewayFromCli } from "./gateway-rpc.js"; type LogsTailPayload = { @@ -18,6 +21,8 @@ type LogsCliOptions = { follow?: boolean; interval?: string; json?: boolean; + plain?: boolean; + color?: boolean; url?: string; token?: string; timeout?: string; @@ -47,6 +52,92 @@ async function fetchLogs( return payload as LogsTailPayload; } +function formatLogTimestamp(value?: string, mode: "pretty" | "plain" = "plain") { + if (!value) return ""; + const parsed = new Date(value); + if (Number.isNaN(parsed.getTime())) return value; + if (mode === "pretty") return parsed.toISOString().slice(11, 19); + return parsed.toISOString(); +} + +function formatLogLine( + raw: string, + opts: { + pretty: boolean; + rich: boolean; + }, +): string { + const parsed = parseLogLine(raw); + if (!parsed) return raw; + const label = parsed.subsystem ?? parsed.module ?? ""; + const time = formatLogTimestamp(parsed.time, opts.pretty ? "pretty" : "plain"); + const level = parsed.level ?? ""; + const levelLabel = level.padEnd(5).trim(); + const message = parsed.message || parsed.raw; + + if (!opts.pretty) { + return [time, level, label, message].filter(Boolean).join(" ").trim(); + } + + const timeLabel = colorize(opts.rich, theme.muted, time); + const labelValue = colorize(opts.rich, theme.accent, label); + const levelValue = + level === "error" || level === "fatal" + ? colorize(opts.rich, theme.error, levelLabel) + : level === "warn" + ? colorize(opts.rich, theme.warn, levelLabel) + : level === "debug" || level === "trace" + ? colorize(opts.rich, theme.muted, levelLabel) + : colorize(opts.rich, theme.info, levelLabel); + const messageValue = + level === "error" || level === "fatal" + ? colorize(opts.rich, theme.error, message) + : level === "warn" + ? colorize(opts.rich, theme.warn, message) + : level === "debug" || level === "trace" + ? colorize(opts.rich, theme.muted, message) + : colorize(opts.rich, theme.info, message); + + const head = [timeLabel, levelValue, labelValue].filter(Boolean).join(" "); + return [head, messageValue].filter(Boolean).join(" ").trim(); +} + +function emitJsonLine(payload: Record, toStdErr = false) { + const text = `${JSON.stringify(payload)}\n`; + if (toStdErr) process.stderr.write(text); + else process.stdout.write(text); +} + +function emitGatewayError( + err: unknown, + opts: LogsCliOptions, + mode: "json" | "text", + rich: boolean, +) { + const details = buildGatewayConnectionDetails({ url: opts.url }); + const message = "Gateway not reachable. Is it running and accessible?"; + const hint = "Hint: run `clawdbot doctor`."; + const errorText = err instanceof Error ? err.message : String(err); + + if (mode === "json") { + emitJsonLine( + { + type: "error", + message, + error: errorText, + details, + hint, + }, + true, + ); + return; + } + + defaultRuntime.error(colorize(rich, theme.error, message)); + defaultRuntime.error(details.message); + defaultRuntime.error(colorize(rich, theme.muted, hint)); +} + export function registerLogsCli(program: Command) { const logs = program .command("logs") @@ -55,7 +146,9 @@ export function registerLogsCli(program: Command) { .option("--max-bytes ", "Max bytes to read", "250000") .option("--follow", "Follow log output", false) .option("--interval ", "Polling interval in ms", "1000") - .option("--json", "Emit JSON payloads", false); + .option("--json", "Emit JSON log lines", false) + .option("--plain", "Plain text output (no ANSI styling)", false) + .option("--no-color", "Disable ANSI colors"); addGatewayClientOptions(logs); @@ -63,18 +156,63 @@ export function registerLogsCli(program: Command) { const interval = parsePositiveInt(opts.interval, 1000); let cursor: number | undefined; let first = true; + const jsonMode = Boolean(opts.json); + const pretty = !jsonMode && Boolean(process.stdout.isTTY) && !opts.plain; + const rich = isRich() && opts.color !== false; while (true) { - const payload = await fetchLogs(opts, cursor); + let payload: LogsTailPayload; + try { + payload = await fetchLogs(opts, cursor); + } catch (err) { + emitGatewayError(err, opts, jsonMode ? "json" : "text", rich); + defaultRuntime.exit(1); + return; + } const lines = Array.isArray(payload.lines) ? payload.lines : []; - if (opts.json) { - defaultRuntime.log(JSON.stringify(payload, null, 2)); - } else { - if (first && payload.file) { - defaultRuntime.log(`Log file: ${payload.file}`); + if (jsonMode) { + if (first) { + emitJsonLine({ + type: "meta", + file: payload.file, + cursor: payload.cursor, + size: payload.size, + }); } for (const line of lines) { - defaultRuntime.log(line); + const parsed = parseLogLine(line); + if (parsed) { + emitJsonLine({ type: "log", ...parsed }); + } else { + emitJsonLine({ type: "raw", raw: line }); + } + } + if (payload.truncated) { + emitJsonLine({ + type: "notice", + message: "Log tail truncated (increase --max-bytes).", + }); + } + if (payload.reset) { + emitJsonLine({ + type: "notice", + message: "Log cursor reset (file rotated).", + }); + } + } else { + if (first && payload.file) { + const prefix = pretty + ? colorize(rich, theme.muted, "Log file:") + : "Log file:"; + defaultRuntime.log(`${prefix} ${payload.file}`); + } + for (const line of lines) { + defaultRuntime.log( + formatLogLine(line, { + pretty, + rich, + }), + ); } if (payload.truncated) { defaultRuntime.error("Log tail truncated (increase --max-bytes)."); diff --git a/src/commands/providers/logs.ts b/src/commands/providers/logs.ts index a287c8c5d..642597248 100644 --- a/src/commands/providers/logs.ts +++ b/src/commands/providers/logs.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import { getResolvedLoggerSettings } from "../../logging.js"; +import { parseLogLine } from "../../logging/parse-log-line.js"; import { defaultRuntime, type RuntimeEnv } from "../../runtime.js"; import { theme } from "../../terminal/theme.js"; @@ -10,14 +11,7 @@ export type ProvidersLogsOptions = { json?: boolean; }; -type LogLine = { - time?: string; - level?: string; - subsystem?: string; - module?: string; - message: string; - raw: string; -}; +type LogLine = ReturnType; const DEFAULT_LIMIT = 200; const MAX_BYTES = 1_000_000; @@ -37,59 +31,7 @@ function parseProviderFilter(raw?: string) { return PROVIDERS.has(trimmed) ? trimmed : "all"; } -function extractMessage(value: Record): string { - const parts: string[] = []; - for (const key of Object.keys(value)) { - if (!/^\d+$/.test(key)) continue; - const item = value[key]; - if (typeof item === "string") { - parts.push(item); - } else if (item != null) { - parts.push(JSON.stringify(item)); - } - } - return parts.join(" "); -} - -function parseMetaName(raw?: unknown): { subsystem?: string; module?: string } { - if (typeof raw !== "string") return {}; - try { - const parsed = JSON.parse(raw) as Record; - return { - subsystem: - typeof parsed.subsystem === "string" ? parsed.subsystem : undefined, - module: typeof parsed.module === "string" ? parsed.module : undefined, - }; - } catch { - return {}; - } -} - -function parseLogLine(raw: string): LogLine | null { - try { - const parsed = JSON.parse(raw) as Record; - const meta = parsed._meta as Record | undefined; - const nameMeta = parseMetaName(meta?.name); - return { - time: - typeof parsed.time === "string" - ? parsed.time - : typeof meta?.date === "string" - ? meta.date - : undefined, - level: - typeof meta?.logLevelName === "string" ? meta.logLevelName : undefined, - subsystem: nameMeta.subsystem, - module: nameMeta.module, - message: extractMessage(parsed), - raw, - }; - } catch { - return null; - } -} - -function matchesProvider(line: LogLine, provider: string) { +function matchesProvider(line: NonNullable, provider: string) { if (provider === "all") return true; const needle = `gateway/providers/${provider}`; if (line.subsystem?.includes(needle)) return true; @@ -139,7 +81,7 @@ export async function providersLogsCommand( const rawLines = await readTailLines(file, limit * 4); const parsed = rawLines .map(parseLogLine) - .filter((line): line is LogLine => Boolean(line)); + .filter((line): line is NonNullable => Boolean(line)); const filtered = parsed.filter((line) => matchesProvider(line, provider)); const lines = filtered.slice(Math.max(0, filtered.length - limit)); diff --git a/src/logging/parse-log-line.ts b/src/logging/parse-log-line.ts new file mode 100644 index 000000000..658d27213 --- /dev/null +++ b/src/logging/parse-log-line.ts @@ -0,0 +1,63 @@ +export type ParsedLogLine = { + time?: string; + level?: string; + subsystem?: string; + module?: string; + message: string; + raw: string; +}; + +function extractMessage(value: Record): string { + const parts: string[] = []; + for (const key of Object.keys(value)) { + if (!/^\d+$/.test(key)) continue; + const item = value[key]; + if (typeof item === "string") { + parts.push(item); + } else if (item != null) { + parts.push(JSON.stringify(item)); + } + } + return parts.join(" "); +} + +function parseMetaName( + raw?: unknown, +): { subsystem?: string; module?: string } { + if (typeof raw !== "string") return {}; + try { + const parsed = JSON.parse(raw) as Record; + return { + subsystem: + typeof parsed.subsystem === "string" ? parsed.subsystem : undefined, + module: typeof parsed.module === "string" ? parsed.module : undefined, + }; + } catch { + return {}; + } +} + +export function parseLogLine(raw: string): ParsedLogLine | null { + try { + const parsed = JSON.parse(raw) as Record; + const meta = parsed._meta as Record | undefined; + const nameMeta = parseMetaName(meta?.name); + const levelRaw = + typeof meta?.logLevelName === "string" ? meta.logLevelName : undefined; + return { + time: + typeof parsed.time === "string" + ? parsed.time + : typeof meta?.date === "string" + ? meta.date + : undefined, + level: levelRaw ? levelRaw.toLowerCase() : undefined, + subsystem: nameMeta.subsystem, + module: nameMeta.module, + message: extractMessage(parsed), + raw, + }; + } catch { + return null; + } +}