feat: improve logs output and docs

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- CLI: improve `logs` output (pretty/plain/JSONL), add gateway unreachable hint, and document logging.
 - WhatsApp: route queued replies to the original sender instead of the bot's own number. (#534) — thanks @mcinteerj
 - Models: add OAuth expiry checks in doctor, expanded `models status` auth output (missing auth + `--check` exit codes). (#538) — thanks @latitudeki5223
 - Deps: bump Pi to 0.40.0 and drop pi-ai patch (upstream 429 fix). (#543) — thanks @mcinteerj

docs/cli/index.md

@@ -20,6 +20,7 @@ This page describes the current CLI behavior. If commands change, update this do
 - ANSI colors and progress indicators only render in TTY sessions.
 - OSC-8 hyperlinks render as clickable links in supported terminals; otherwise we fall back to plain URLs.
 - `--json` (and `--plain` where supported) disables styling for clean output.
+- `--no-color` disables ANSI styling where supported; `NO_COLOR=1` is also respected.
 - Long-running commands show a progress indicator (OSC 9;4 when supported).
 
 ## Color palette
@@ -443,10 +444,17 @@ Notes:
 ### `logs`
 Tail Gateway file logs via RPC.
 
+Notes:
+- TTY sessions render a colorized, structured view; non-TTY falls back to plain text.
+- `--json` emits line-delimited JSON (one log event per line).
+
 Examples:
 ```bash
 clawdbot logs --follow
 clawdbot logs --limit 200
+clawdbot logs --plain
+clawdbot logs --json
+clawdbot logs --no-color
 ```
 
 ### `gateway <subcommand>`

docs/gateway/logging.md

@@ -7,6 +7,8 @@ read_when:
 
 # Logging
 
+For a user-facing overview (CLI + Control UI + config), see [/logging](/logging).
+
 Clawdbot has two log “surfaces”:
 
 - **Console output** (what you see in the terminal / Debug UI).

docs/logging.md

@@ -0,0 +1,144 @@
+---
+summary: "Logging overview: file logs, console output, CLI tailing, and the Control UI"
+read_when:
+  - You need a beginner-friendly overview of logging
+  - You want to configure log levels or formats
+  - You are troubleshooting and need to find logs quickly
+---
+
+# Logging
+
+Clawdbot logs in two places:
+
+- **File logs** (JSON lines) written by the Gateway.
+- **Console output** shown in terminals and the Control UI.
+
+This page explains where logs live, how to read them, and how to configure log
+levels and formats.
+
+## Where logs live
+
+By default, the Gateway writes a rolling log file under:
+
+`/tmp/clawdbot/clawdbot-YYYY-MM-DD.log`
+
+You can override this in `~/.clawdbot/clawdbot.json`:
+
+```json
+{
+  "logging": {
+    "file": "/path/to/clawdbot.log"
+  }
+}
+```
+
+## How to read logs
+
+### CLI: live tail (recommended)
+
+Use the CLI to tail the gateway log file via RPC:
+
+```bash
+clawdbot logs --follow
+```
+
+Output modes:
+
+- **TTY sessions**: pretty, colorized, structured log lines.
+- **Non-TTY sessions**: plain text.
+- `--json`: line-delimited JSON (one log event per line).
+- `--plain`: force plain text in TTY sessions.
+- `--no-color`: disable ANSI colors.
+
+In JSON mode, the CLI emits `type`-tagged objects:
+
+- `meta`: stream metadata (file, cursor, size)
+- `log`: parsed log entry
+- `notice`: truncation / rotation hints
+- `raw`: unparsed log line
+
+If the Gateway is unreachable, the CLI prints a short hint to run:
+
+```bash
+clawdbot doctor
+```
+
+### Control UI (web)
+
+The Control UI’s **Logs** tab tails the same file using `logs.tail`.
+See [/web/control-ui](/web/control-ui) for how to open it.
+
+### Provider-only logs
+
+To filter provider activity (WhatsApp/Telegram/etc), use:
+
+```bash
+clawdbot providers logs --provider whatsapp
+```
+
+## Log formats
+
+### File logs (JSONL)
+
+Each line in the log file is a JSON object. The CLI and Control UI parse these
+entries to render structured output (time, level, subsystem, message).
+
+### Console output
+
+Console logs are **TTY-aware** and formatted for readability:
+
+- Subsystem prefixes (e.g. `gateway/providers/whatsapp`)
+- Level coloring (info/warn/error)
+- Optional compact or JSON mode
+
+Console formatting is controlled by `logging.consoleStyle`.
+
+## Configuring logging
+
+All logging configuration lives under `logging` in `~/.clawdbot/clawdbot.json`.
+
+```json
+{
+  "logging": {
+    "level": "info",
+    "file": "/tmp/clawdbot/clawdbot-YYYY-MM-DD.log",
+    "consoleLevel": "info",
+    "consoleStyle": "pretty",
+    "redactSensitive": "tools",
+    "redactPatterns": [
+      "sk-.*"
+    ]
+  }
+}
+```
+
+### Log levels
+
+- `logging.level`: **file logs** (JSONL) level.
+- `logging.consoleLevel`: **console** verbosity level.
+
+`--verbose` only affects console output; it does not change file log levels.
+
+### Console styles
+
+`logging.consoleStyle`:
+
+- `pretty`: human-friendly, colored, with timestamps.
+- `compact`: tighter output (best for long sessions).
+- `json`: JSON per line (for log processors).
+
+### Redaction
+
+Tool summaries can redact sensitive tokens before they hit the console:
+
+- `logging.redactSensitive`: `off` | `tools` (default: `tools`)
+- `logging.redactPatterns`: list of regex strings to override the default set
+
+Redaction affects **console output only** and does not alter file logs.
+
+## Troubleshooting tips
+
+- **Gateway not reachable?** Run `clawdbot doctor` first.
+- **Logs empty?** Check that the Gateway is running and writing to the file path
+  in `logging.file`.
+- **Need more detail?** Set `logging.level` to `debug` or `trace` and retry.

src/cli/logs-cli.ts

@@ -1,6 +1,9 @@
 import { setTimeout as delay } from "node:timers/promises";
 import type { Command } from "commander";
+import { buildGatewayConnectionDetails } from "../gateway/call.js";
+import { parseLogLine } from "../logging/parse-log-line.js";
 import { defaultRuntime } from "../runtime.js";
+import { colorize, isRich, theme } from "../terminal/theme.js";
 import { addGatewayClientOptions, callGatewayFromCli } from "./gateway-rpc.js";
 
 type LogsTailPayload = {
@@ -18,6 +21,8 @@ type LogsCliOptions = {
   follow?: boolean;
   interval?: string;
   json?: boolean;
+  plain?: boolean;
+  color?: boolean;
   url?: string;
   token?: string;
   timeout?: string;
@@ -47,6 +52,92 @@ async function fetchLogs(
   return payload as LogsTailPayload;
 }
 
+function formatLogTimestamp(value?: string, mode: "pretty" | "plain" = "plain") {
+  if (!value) return "";
+  const parsed = new Date(value);
+  if (Number.isNaN(parsed.getTime())) return value;
+  if (mode === "pretty") return parsed.toISOString().slice(11, 19);
+  return parsed.toISOString();
+}
+
+function formatLogLine(
+  raw: string,
+  opts: {
+    pretty: boolean;
+    rich: boolean;
+  },
+): string {
+  const parsed = parseLogLine(raw);
+  if (!parsed) return raw;
+  const label = parsed.subsystem ?? parsed.module ?? "";
+  const time = formatLogTimestamp(parsed.time, opts.pretty ? "pretty" : "plain");
+  const level = parsed.level ?? "";
+  const levelLabel = level.padEnd(5).trim();
+  const message = parsed.message || parsed.raw;
+
+  if (!opts.pretty) {
+    return [time, level, label, message].filter(Boolean).join(" ").trim();
+  }
+
+  const timeLabel = colorize(opts.rich, theme.muted, time);
+  const labelValue = colorize(opts.rich, theme.accent, label);
+  const levelValue =
+    level === "error" || level === "fatal"
+      ? colorize(opts.rich, theme.error, levelLabel)
+      : level === "warn"
+        ? colorize(opts.rich, theme.warn, levelLabel)
+        : level === "debug" || level === "trace"
+          ? colorize(opts.rich, theme.muted, levelLabel)
+          : colorize(opts.rich, theme.info, levelLabel);
+  const messageValue =
+    level === "error" || level === "fatal"
+      ? colorize(opts.rich, theme.error, message)
+      : level === "warn"
+        ? colorize(opts.rich, theme.warn, message)
+        : level === "debug" || level === "trace"
+          ? colorize(opts.rich, theme.muted, message)
+          : colorize(opts.rich, theme.info, message);
+
+  const head = [timeLabel, levelValue, labelValue].filter(Boolean).join(" ");
+  return [head, messageValue].filter(Boolean).join(" ").trim();
+}
+
+function emitJsonLine(payload: Record<string, unknown>, toStdErr = false) {
+  const text = `${JSON.stringify(payload)}\n`;
+  if (toStdErr) process.stderr.write(text);
+  else process.stdout.write(text);
+}
+
+function emitGatewayError(
+  err: unknown,
+  opts: LogsCliOptions,
+  mode: "json" | "text",
+  rich: boolean,
+) {
+  const details = buildGatewayConnectionDetails({ url: opts.url });
+  const message = "Gateway not reachable. Is it running and accessible?";
+  const hint = "Hint: run `clawdbot doctor`.";
+  const errorText = err instanceof Error ? err.message : String(err);
+
+  if (mode === "json") {
+    emitJsonLine(
+      {
+        type: "error",
+        message,
+        error: errorText,
+        details,
+        hint,
+      },
+      true,
+    );
+    return;
+  }
+
+  defaultRuntime.error(colorize(rich, theme.error, message));
+  defaultRuntime.error(details.message);
+  defaultRuntime.error(colorize(rich, theme.muted, hint));
+}
+
 export function registerLogsCli(program: Command) {
   const logs = program
     .command("logs")
@@ -55,7 +146,9 @@ export function registerLogsCli(program: Command) {
     .option("--max-bytes <n>", "Max bytes to read", "250000")
     .option("--follow", "Follow log output", false)
     .option("--interval <ms>", "Polling interval in ms", "1000")
-    .option("--json", "Emit JSON payloads", false);
+    .option("--json", "Emit JSON log lines", false)
+    .option("--plain", "Plain text output (no ANSI styling)", false)
+    .option("--no-color", "Disable ANSI colors");
 
   addGatewayClientOptions(logs);
 
@@ -63,18 +156,63 @@ export function registerLogsCli(program: Command) {
     const interval = parsePositiveInt(opts.interval, 1000);
     let cursor: number | undefined;
     let first = true;
+    const jsonMode = Boolean(opts.json);
+    const pretty = !jsonMode && Boolean(process.stdout.isTTY) && !opts.plain;
+    const rich = isRich() && opts.color !== false;
 
     while (true) {
-      const payload = await fetchLogs(opts, cursor);
+      let payload: LogsTailPayload;
+      try {
+        payload = await fetchLogs(opts, cursor);
+      } catch (err) {
+        emitGatewayError(err, opts, jsonMode ? "json" : "text", rich);
+        defaultRuntime.exit(1);
+        return;
+      }
       const lines = Array.isArray(payload.lines) ? payload.lines : [];
-      if (opts.json) {
-        defaultRuntime.log(JSON.stringify(payload, null, 2));
-      } else {
-        if (first && payload.file) {
-          defaultRuntime.log(`Log file: ${payload.file}`);
+      if (jsonMode) {
+        if (first) {
+          emitJsonLine({
+            type: "meta",
+            file: payload.file,
+            cursor: payload.cursor,
+            size: payload.size,
+          });
         }
         for (const line of lines) {
-          defaultRuntime.log(line);
+          const parsed = parseLogLine(line);
+          if (parsed) {
+            emitJsonLine({ type: "log", ...parsed });
+          } else {
+            emitJsonLine({ type: "raw", raw: line });
+          }
+        }
+        if (payload.truncated) {
+          emitJsonLine({
+            type: "notice",
+            message: "Log tail truncated (increase --max-bytes).",
+          });
+        }
+        if (payload.reset) {
+          emitJsonLine({
+            type: "notice",
+            message: "Log cursor reset (file rotated).",
+          });
+        }
+      } else {
+        if (first && payload.file) {
+          const prefix = pretty
+            ? colorize(rich, theme.muted, "Log file:")
+            : "Log file:";
+          defaultRuntime.log(`${prefix} ${payload.file}`);
+        }
+        for (const line of lines) {
+          defaultRuntime.log(
+            formatLogLine(line, {
+              pretty,
+              rich,
+            }),
+          );
         }
         if (payload.truncated) {
           defaultRuntime.error("Log tail truncated (increase --max-bytes).");

src/commands/providers/logs.ts

@@ -1,6 +1,7 @@
 import fs from "node:fs/promises";
 
 import { getResolvedLoggerSettings } from "../../logging.js";
+import { parseLogLine } from "../../logging/parse-log-line.js";
 import { defaultRuntime, type RuntimeEnv } from "../../runtime.js";
 import { theme } from "../../terminal/theme.js";
 
@@ -10,14 +11,7 @@ export type ProvidersLogsOptions = {
   json?: boolean;
 };
 
-type LogLine = {
-  time?: string;
-  level?: string;
-  subsystem?: string;
-  module?: string;
-  message: string;
-  raw: string;
-};
+type LogLine = ReturnType<typeof parseLogLine>;
 
 const DEFAULT_LIMIT = 200;
 const MAX_BYTES = 1_000_000;
@@ -37,59 +31,7 @@ function parseProviderFilter(raw?: string) {
   return PROVIDERS.has(trimmed) ? trimmed : "all";
 }
 
-function extractMessage(value: Record<string, unknown>): string {
-  const parts: string[] = [];
-  for (const key of Object.keys(value)) {
-    if (!/^\d+$/.test(key)) continue;
-    const item = value[key];
-    if (typeof item === "string") {
-      parts.push(item);
-    } else if (item != null) {
-      parts.push(JSON.stringify(item));
-    }
-  }
-  return parts.join(" ");
-}
-
-function parseMetaName(raw?: unknown): { subsystem?: string; module?: string } {
-  if (typeof raw !== "string") return {};
-  try {
-    const parsed = JSON.parse(raw) as Record<string, unknown>;
-    return {
-      subsystem:
-        typeof parsed.subsystem === "string" ? parsed.subsystem : undefined,
-      module: typeof parsed.module === "string" ? parsed.module : undefined,
-    };
-  } catch {
-    return {};
-  }
-}
-
-function parseLogLine(raw: string): LogLine | null {
-  try {
-    const parsed = JSON.parse(raw) as Record<string, unknown>;
-    const meta = parsed._meta as Record<string, unknown> | undefined;
-    const nameMeta = parseMetaName(meta?.name);
-    return {
-      time:
-        typeof parsed.time === "string"
-          ? parsed.time
-          : typeof meta?.date === "string"
-            ? meta.date
-            : undefined,
-      level:
-        typeof meta?.logLevelName === "string" ? meta.logLevelName : undefined,
-      subsystem: nameMeta.subsystem,
-      module: nameMeta.module,
-      message: extractMessage(parsed),
-      raw,
-    };
-  } catch {
-    return null;
-  }
-}
-
-function matchesProvider(line: LogLine, provider: string) {
+function matchesProvider(line: NonNullable<LogLine>, provider: string) {
   if (provider === "all") return true;
   const needle = `gateway/providers/${provider}`;
   if (line.subsystem?.includes(needle)) return true;
@@ -139,7 +81,7 @@ export async function providersLogsCommand(
   const rawLines = await readTailLines(file, limit * 4);
   const parsed = rawLines
     .map(parseLogLine)
-    .filter((line): line is LogLine => Boolean(line));
+    .filter((line): line is NonNullable<LogLine> => Boolean(line));
   const filtered = parsed.filter((line) => matchesProvider(line, provider));
   const lines = filtered.slice(Math.max(0, filtered.length - limit));
 

src/logging/parse-log-line.ts

@@ -0,0 +1,63 @@
+export type ParsedLogLine = {
+  time?: string;
+  level?: string;
+  subsystem?: string;
+  module?: string;
+  message: string;
+  raw: string;
+};
+
+function extractMessage(value: Record<string, unknown>): string {
+  const parts: string[] = [];
+  for (const key of Object.keys(value)) {
+    if (!/^\d+$/.test(key)) continue;
+    const item = value[key];
+    if (typeof item === "string") {
+      parts.push(item);
+    } else if (item != null) {
+      parts.push(JSON.stringify(item));
+    }
+  }
+  return parts.join(" ");
+}
+
+function parseMetaName(
+  raw?: unknown,
+): { subsystem?: string; module?: string } {
+  if (typeof raw !== "string") return {};
+  try {
+    const parsed = JSON.parse(raw) as Record<string, unknown>;
+    return {
+      subsystem:
+        typeof parsed.subsystem === "string" ? parsed.subsystem : undefined,
+      module: typeof parsed.module === "string" ? parsed.module : undefined,
+    };
+  } catch {
+    return {};
+  }
+}
+
+export function parseLogLine(raw: string): ParsedLogLine | null {
+  try {
+    const parsed = JSON.parse(raw) as Record<string, unknown>;
+    const meta = parsed._meta as Record<string, unknown> | undefined;
+    const nameMeta = parseMetaName(meta?.name);
+    const levelRaw =
+      typeof meta?.logLevelName === "string" ? meta.logLevelName : undefined;
+    return {
+      time:
+        typeof parsed.time === "string"
+          ? parsed.time
+          : typeof meta?.date === "string"
+            ? meta.date
+            : undefined,
+      level: levelRaw ? levelRaw.toLowerCase() : undefined,
+      subsystem: nameMeta.subsystem,
+      module: nameMeta.module,
+      message: extractMessage(parsed),
+      raw,
+    };
+  } catch {
+    return null;
+  }
+}