ouyang/clawdbot
1
0
Fork 0
mirror of https://github.com/clawdbot/clawdbot.git synced 2026-02-01 03:47:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Canvas: click progress + context-rich actions

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-17 20:19:37 +01:00
parent 56d94e6974
commit 9051c5891e
4 changed files with 398 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
apps/macos/Sources/Clawdis/CanvasWindow.swift
View File

@@ -84,6 +84,8 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
const deepLinkKey = \(Self.jsStringLiteral(deepLinkKey));
const sessionKey = \(Self.jsStringLiteral(injectedSessionKey));
const machineName = \(Self.jsStringLiteral(InstanceIdentity.displayName));
const instanceId = \(Self.jsStringLiteral(InstanceIdentity.instanceId));
globalThis.addEventListener('a2uiaction', (evt) => {
try {
@@ -96,6 +98,7 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
const context = Array.isArray(action?.context) ? action.context : [];
const userAction = {
id: (globalThis.crypto?.randomUUID?.() ?? String(Date.now())),
name,
surfaceId: payload.surfaceId ?? 'main',
sourceComponentId: payload.sourceComponentId ?? '',
@@ -117,7 +120,16 @@ final class CanvasWindowController: NSWindowController, WKNavigationDelegate, NS
return;
}
const message = 'A2UI action: ' + name + '\\n\\n```json\\n' + JSON.stringify(userAction, null, 2) + '\\n```';
const ctx = userAction.context ? (' ctx=' + JSON.stringify(userAction.context)) : '';
const message =
'CANVAS_A2UI action=' + userAction.name +
' session=' + sessionKey +
' surface=' + userAction.surfaceId +
' component=' + (userAction.sourceComponentId || '-') +
' host=' + machineName.replace(/\\s+/g, '_') +
' instance=' + instanceId +
ctx +
' default=update_canvas';
const params = new URLSearchParams();
params.set('message', message);
params.set('sessionKey', sessionKey);
@@ -621,23 +633,24 @@ private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHan
guard !userAction.isEmpty else { return }
guard let name = userAction["name"] as? String, !name.isEmpty else { return }
let json: String = {
if let data = try? JSONSerialization.data(withJSONObject: userAction, options: [.prettyPrinted, .sortedKeys]),
let str = String(data: data, encoding: .utf8)
{
return str
}
return ""
}()
let actionId =
(userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty
?? UUID().uuidString
canvasWindowLogger.info("A2UI action \(name, privacy: .public) session=\(self.sessionKey, privacy: .public)")
let text = json.isEmpty
? "A2UI action: \(name)"
: "A2UI action: \(name)\n\n```json\n\(json)\n```"
let surfaceId = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "main"
let sourceComponentId = (userAction["sourceComponentId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "-"
let host = Self.sanitizeTagValue(InstanceIdentity.displayName)
let instanceId = InstanceIdentity.instanceId.lowercased()
let contextJSON = Self.compactJSON(userAction["context"])
let contextSuffix = contextJSON.flatMap { $0.isEmpty ? nil : " ctx=\($0)" } ?? ""
Task {
// Token-efficient and unambiguous. The agent should treat this as a UI event and (by default) update Canvas.
let text =
"CANVAS_A2UI action=\(Self.sanitizeTagValue(name)) session=\(Self.sanitizeTagValue(self.sessionKey)) surface=\(Self.sanitizeTagValue(surfaceId)) component=\(Self.sanitizeTagValue(sourceComponentId)) host=\(host) instance=\(instanceId)\(contextSuffix) default=update_canvas"
Task { [weak webView] in
if AppStateStore.shared.connectionMode == .local {
GatewayProcessManager.shared.setActive(true)
}
@@ -648,13 +661,54 @@ private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHan
thinking: "low",
deliver: false,
to: nil,
channel: .last))
channel: .last,
idempotencyKey: actionId))
await MainActor.run {
guard let webView else { return }
let js = Self.jsDispatchA2UIActionStatus(actionId: actionId, ok: result.ok, error: result.error)
webView.evaluateJavaScript(js) { _, _ in }
}
if !result.ok {
canvasWindowLogger.error(
"A2UI action send failed name=\(name, privacy: .public) error=\(result.error ?? "unknown", privacy: .public)")
}
}
}
private static func sanitizeTagValue(_ value: String) -> String {
let trimmed = value.trimmingCharacters(in: .whitespacesAndNewlines).nonEmpty ?? "-"
let normalized = trimmed.replacingOccurrences(of: " ", with: "_")
let allowed = CharacterSet(charactersIn: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-.:")
let scalars = normalized.unicodeScalars.map { allowed.contains($0) ? Character($0) : "_" }
return String(scalars)
}
private static func compactJSON(_ obj: Any?) -> String? {
guard let obj else { return nil }
guard JSONSerialization.isValidJSONObject(obj) else { return nil }
guard let data = try? JSONSerialization.data(withJSONObject: obj, options: []),
let str = String(data: data, encoding: .utf8)
else { return nil }
return str
}
private static func jsDispatchA2UIActionStatus(actionId: String, ok: Bool, error: String?) -> String {
let payload: [String: Any] = [
"id": actionId,
"ok": ok,
"error": error ?? "",
]
let json: String = {
if let data = try? JSONSerialization.data(withJSONObject: payload, options: []),
let str = String(data: data, encoding: .utf8)
{
return str
}
return "{\"id\":\"\(actionId)\",\"ok\":\(ok ? "true" : "false"),\"error\":\"\"}"
}()
return "window.dispatchEvent(new CustomEvent('clawdis:a2ui-action-status', { detail: \(json) }));"
}
}
// MARK: - Hover chrome container

166
apps/macos/Sources/Clawdis/Resources/CanvasA2UI/a2ui.bundle.js
View File

@@ -17726,17 +17726,25 @@ const clawdisTheme = {
}
};
var ClawdisA2UIHost = class extends i$1 {
static properties = { surfaces: { state: true } };
static properties = {
surfaces: { state: true },
pendingAction: { state: true },
toast: { state: true }
};
#processor = Data.createSignalA2uiMessageProcessor();
#themeProvider = new i$2(this, {
context: themeContext,
initialValue: clawdisTheme
});
surfaces = [];
pendingAction = null;
toast = null;
#statusListener = null;
static styles = i`
:host {
display: block;
height: 100%;
position: relative;
box-sizing: border-box;
padding: 12px;
}
@@ -17749,6 +17757,71 @@ var ClawdisA2UIHost = class extends i$1 {
overflow: auto;
padding-bottom: 24px;
}
.status {
position: absolute;
left: 50%;
transform: translateX(-50%);
top: 12px;
display: inline-flex;
align-items: center;
gap: 8px;
padding: 8px 10px;
border-radius: 12px;
background: rgba(0, 0, 0, 0.45);
border: 1px solid rgba(255, 255, 255, 0.18);
color: rgba(255, 255, 255, 0.92);
font: 13px/1.2 -apple-system, BlinkMacSystemFont, system-ui, sans-serif;
pointer-events: none;
backdrop-filter: blur(14px);
-webkit-backdrop-filter: blur(14px);
box-shadow: 0 10px 24px rgba(0, 0, 0, 0.25);
z-index: 5;
}
.toast {
position: absolute;
left: 50%;
transform: translateX(-50%);
bottom: 12px;
display: inline-flex;
align-items: center;
gap: 8px;
padding: 8px 10px;
border-radius: 12px;
background: rgba(0, 0, 0, 0.45);
border: 1px solid rgba(255, 255, 255, 0.18);
color: rgba(255, 255, 255, 0.92);
font: 13px/1.2 -apple-system, BlinkMacSystemFont, system-ui, sans-serif;
pointer-events: none;
backdrop-filter: blur(14px);
-webkit-backdrop-filter: blur(14px);
box-shadow: 0 10px 24px rgba(0, 0, 0, 0.25);
z-index: 5;
}
.toast.error {
border-color: rgba(255, 109, 109, 0.35);
color: rgba(255, 223, 223, 0.98);
}
.spinner {
width: 12px;
height: 12px;
border-radius: 999px;
border: 2px solid rgba(255, 255, 255, 0.25);
border-top-color: rgba(255, 255, 255, 0.92);
animation: spin 0.75s linear infinite;
}
@keyframes spin {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
`;
connectedCallback() {
super.connectedCallback();
@@ -17758,8 +17831,56 @@ var ClawdisA2UIHost = class extends i$1 {
getSurfaces: () => Array.from(this.#processor.getSurfaces().keys())
};
this.addEventListener("a2uiaction", (evt) => this.#handleA2UIAction(evt));
this.#statusListener = (evt) => this.#handleActionStatus(evt);
globalThis.addEventListener("clawdis:a2ui-action-status", this.#statusListener);
this.#syncSurfaces();
}
disconnectedCallback() {
super.disconnectedCallback();
if (this.#statusListener) {
globalThis.removeEventListener("clawdis:a2ui-action-status", this.#statusListener);
this.#statusListener = null;
}
}
#makeActionId() {
return globalThis.crypto?.randomUUID?.() ?? `a2ui_${Date.now()}_${Math.random().toString(16).slice(2)}`;
}
#setToast(text$1, kind = "ok", timeoutMs = 1400) {
const toast = {
text: text$1,
kind,
expiresAt: Date.now() + timeoutMs
};
this.toast = toast;
this.requestUpdate();
setTimeout(() => {
if (this.toast === toast) {
this.toast = null;
this.requestUpdate();
}
}, timeoutMs + 30);
}
#handleActionStatus(evt) {
const detail = evt?.detail ?? null;
if (!detail || typeof detail.id !== "string") return;
if (!this.pendingAction || this.pendingAction.id !== detail.id) return;
if (detail.ok) {
this.pendingAction = {
...this.pendingAction,
phase: "sent",
sentAt: Date.now()
};
} else {
const msg = typeof detail.error === "string" && detail.error ? detail.error : "send failed";
this.pendingAction = {
...this.pendingAction,
phase: "error",
error: msg
};
this.#setToast(`Failed: ${msg}`, "error", 4500);
}
this.requestUpdate();
}
#handleA2UIAction(evt) {
const payload = evt?.detail ?? evt?.payload ?? null;
if (!payload || payload.eventType !== "a2ui.action") {
@@ -17806,7 +17927,16 @@ var ClawdisA2UIHost = class extends i$1 {
continue;
}
}
const actionId = this.#makeActionId();
this.pendingAction = {
id: actionId,
name,
phase: "sending",
startedAt: Date.now()
};
this.requestUpdate();
const userAction = {
id: actionId,
name,
surfaceId: surfaceId ?? "main",
sourceComponentId,
@@ -17816,7 +17946,28 @@ var ClawdisA2UIHost = class extends i$1 {
globalThis.__clawdisLastA2UIAction = userAction;
const handler = globalThis.webkit?.messageHandlers?.clawdisCanvasA2UIAction;
if (handler?.postMessage) {
handler.postMessage({ userAction });
try {
handler.postMessage({ userAction });
} catch (e$14) {
const msg = String(e$14?.message ?? e$14);
this.pendingAction = {
id: actionId,
name,
phase: "error",
startedAt: Date.now(),
error: msg
};
this.#setToast(`Failed: ${msg}`, "error", 4500);
}
} else {
this.pendingAction = {
id: actionId,
name,
phase: "error",
startedAt: Date.now(),
error: "missing native bridge"
};
this.#setToast("Failed: missing native bridge", "error", 4500);
}
}
applyMessages(messages) {
@@ -17825,6 +17976,10 @@ var ClawdisA2UIHost = class extends i$1 {
}
this.#processor.processMessages(messages);
this.#syncSurfaces();
if (this.pendingAction?.phase === "sent") {
this.#setToast(`Updated: ${this.pendingAction.name}`, "ok", 1100);
this.pendingAction = null;
}
this.requestUpdate();
return {
ok: true,
@@ -17834,6 +17989,7 @@ var ClawdisA2UIHost = class extends i$1 {
reset() {
this.#processor.clearSurfaces();
this.#syncSurfaces();
this.pendingAction = null;
this.requestUpdate();
return { ok: true };
}
@@ -17847,7 +18003,11 @@ var ClawdisA2UIHost = class extends i$1 {
<div>Waiting for A2UI messages…</div>
</div>`;
}
return x`<section id="surfaces">
const statusText = this.pendingAction?.phase === "sent" ? `Working: ${this.pendingAction.name}` : this.pendingAction?.phase === "sending" ? `Sending: ${this.pendingAction.name}` : this.pendingAction?.phase === "error" ? `Failed: ${this.pendingAction.name}` : "";
return x`
${this.pendingAction && this.pendingAction.phase !== "error" ? x`<div class="status"><div class="spinner"></div><div>${statusText}</div></div>` : ""}
${this.toast ? x`<div class="toast ${this.toast.kind === "error" ? "error" : ""}">${this.toast.text}</div>` : ""}
<section id="surfaces">
${c(this.surfaces, ([surfaceId]) => surfaceId, ([surfaceId, surface]) => x`<a2ui-surface
.surfaceId=${surfaceId}
.surface=${surface}

151
apps/macos/Sources/Clawdis/Resources/CanvasA2UI/bootstrap.js vendored
View File

@@ -115,6 +115,8 @@ const clawdisTheme = {
class ClawdisA2UIHost extends LitElement {
static properties = {
surfaces: { state: true },
pendingAction: { state: true },
toast: { state: true },
};
#processor = v0_8.Data.createSignalA2uiMessageProcessor();
@@ -124,11 +126,15 @@ class ClawdisA2UIHost extends LitElement {
});
surfaces = [];
pendingAction = null;
toast = null;
#statusListener = null;
static styles = css`
:host {
display: block;
height: 100%;
position: relative;
box-sizing: border-box;
padding: 12px;
}
@@ -141,6 +147,71 @@ class ClawdisA2UIHost extends LitElement {
overflow: auto;
padding-bottom: 24px;
}
.status {
position: absolute;
left: 50%;
transform: translateX(-50%);
top: 12px;
display: inline-flex;
align-items: center;
gap: 8px;
padding: 8px 10px;
border-radius: 12px;
background: rgba(0, 0, 0, 0.45);
border: 1px solid rgba(255, 255, 255, 0.18);
color: rgba(255, 255, 255, 0.92);
font: 13px/1.2 -apple-system, BlinkMacSystemFont, system-ui, sans-serif;
pointer-events: none;
backdrop-filter: blur(14px);
-webkit-backdrop-filter: blur(14px);
box-shadow: 0 10px 24px rgba(0, 0, 0, 0.25);
z-index: 5;
}
.toast {
position: absolute;
left: 50%;
transform: translateX(-50%);
bottom: 12px;
display: inline-flex;
align-items: center;
gap: 8px;
padding: 8px 10px;
border-radius: 12px;
background: rgba(0, 0, 0, 0.45);
border: 1px solid rgba(255, 255, 255, 0.18);
color: rgba(255, 255, 255, 0.92);
font: 13px/1.2 -apple-system, BlinkMacSystemFont, system-ui, sans-serif;
pointer-events: none;
backdrop-filter: blur(14px);
-webkit-backdrop-filter: blur(14px);
box-shadow: 0 10px 24px rgba(0, 0, 0, 0.25);
z-index: 5;
}
.toast.error {
border-color: rgba(255, 109, 109, 0.35);
color: rgba(255, 223, 223, 0.98);
}
.spinner {
width: 12px;
height: 12px;
border-radius: 999px;
border: 2px solid rgba(255, 255, 255, 0.25);
border-top-color: rgba(255, 255, 255, 0.92);
animation: spin 0.75s linear infinite;
}
@keyframes spin {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
`;
connectedCallback() {
@@ -151,9 +222,50 @@ class ClawdisA2UIHost extends LitElement {
getSurfaces: () => Array.from(this.#processor.getSurfaces().keys()),
};
this.addEventListener("a2uiaction", (evt) => this.#handleA2UIAction(evt));
this.#statusListener = (evt) => this.#handleActionStatus(evt);
globalThis.addEventListener("clawdis:a2ui-action-status", this.#statusListener);
this.#syncSurfaces();
}
disconnectedCallback() {
super.disconnectedCallback();
if (this.#statusListener) {
globalThis.removeEventListener("clawdis:a2ui-action-status", this.#statusListener);
this.#statusListener = null;
}
}
#makeActionId() {
return globalThis.crypto?.randomUUID?.() ?? `a2ui_${Date.now()}_${Math.random().toString(16).slice(2)}`;
}
#setToast(text, kind = "ok", timeoutMs = 1400) {
const toast = { text, kind, expiresAt: Date.now() + timeoutMs };
this.toast = toast;
this.requestUpdate();
setTimeout(() => {
if (this.toast === toast) {
this.toast = null;
this.requestUpdate();
}
}, timeoutMs + 30);
}
#handleActionStatus(evt) {
const detail = evt?.detail ?? null;
if (!detail || typeof detail.id !== "string") return;
if (!this.pendingAction || this.pendingAction.id !== detail.id) return;
if (detail.ok) {
this.pendingAction = { ...this.pendingAction, phase: "sent", sentAt: Date.now() };
} else {
const msg = typeof detail.error === "string" && detail.error ? detail.error : "send failed";
this.pendingAction = { ...this.pendingAction, phase: "error", error: msg };
this.#setToast(`Failed: ${msg}`, "error", 4500);
}
this.requestUpdate();
}
#handleA2UIAction(evt) {
const payload = evt?.detail ?? evt?.payload ?? null;
if (!payload || payload.eventType !== "a2ui.action") {
@@ -208,7 +320,12 @@ class ClawdisA2UIHost extends LitElement {
}
}
const actionId = this.#makeActionId();
this.pendingAction = { id: actionId, name, phase: "sending", startedAt: Date.now() };
this.requestUpdate();
const userAction = {
id: actionId,
name,
surfaceId: surfaceId ?? "main",
sourceComponentId,
@@ -220,7 +337,16 @@ class ClawdisA2UIHost extends LitElement {
const handler = globalThis.webkit?.messageHandlers?.clawdisCanvasA2UIAction;
if (handler?.postMessage) {
handler.postMessage({ userAction });
try {
handler.postMessage({ userAction });
} catch (e) {
const msg = String(e?.message ?? e);
this.pendingAction = { id: actionId, name, phase: "error", startedAt: Date.now(), error: msg };
this.#setToast(`Failed: ${msg}`, "error", 4500);
}
} else {
this.pendingAction = { id: actionId, name, phase: "error", startedAt: Date.now(), error: "missing native bridge" };
this.#setToast("Failed: missing native bridge", "error", 4500);
}
}
@@ -230,6 +356,10 @@ class ClawdisA2UIHost extends LitElement {
}
this.#processor.processMessages(messages);
this.#syncSurfaces();
if (this.pendingAction?.phase === "sent") {
this.#setToast(`Updated: ${this.pendingAction.name}`, "ok", 1100);
this.pendingAction = null;
}
this.requestUpdate();
return { ok: true, surfaces: this.surfaces.map(([id]) => id) };
}
@@ -237,6 +367,7 @@ class ClawdisA2UIHost extends LitElement {
reset() {
this.#processor.clearSurfaces();
this.#syncSurfaces();
this.pendingAction = null;
this.requestUpdate();
return { ok: true };
}
@@ -253,7 +384,23 @@ class ClawdisA2UIHost extends LitElement {
</div>`;
}
return html`<section id="surfaces">
const statusText =
this.pendingAction?.phase === "sent"
? `Working: ${this.pendingAction.name}`
: this.pendingAction?.phase === "sending"
? `Sending: ${this.pendingAction.name}`
: this.pendingAction?.phase === "error"
? `Failed: ${this.pendingAction.name}`
: "";
return html`
${this.pendingAction && this.pendingAction.phase !== "error"
? html`<div class="status"><div class="spinner"></div><div>${statusText}</div></div>`
: ""}
${this.toast
? html`<div class="toast ${this.toast.kind === "error" ? "error" : ""}">${this.toast.text}</div>`
: ""}
<section id="surfaces">
${repeat(
this.surfaces,
([surfaceId]) => surfaceId,

17
docs/refactor/canvas-a2ui.md
View File

@@ -25,6 +25,23 @@
- Now it hops back to the main actor before mutating state.
- Preserve in-page state when closing Canvas (hide the window instead of closing the `WKWebView`).
- Fix another “Canvas looks hung” source: node pairing approval used `NSAlert.runModal()` on the main actor, which stalls Canvas/IPC while the alert is open.
- Add UX feedback + better agent prompting:
- Show a small “Sending/Working” spinner when a button is clicked.
- Show “Updated/Failed” toasts (failures include the gateway error string).
- Send a compact, unambiguous agent message that includes machine identity + Canvas context (instead of a big JSON markdown block).
- Native acks the click back into the page via `clawdis:a2ui-action-status` so the UI can switch from “Sending…” to “Working…” immediately.
## Suggested message format (token-efficient)
We want the model to immediately understand:
- This is a **Canvas UI event** (not user chat).
- It happened on **this specific Mac**.
- Default behavior is to **update the Canvas UI** (unless the button context says otherwise).
Proposed message line (single-line, parseable):
```
CANVAS_A2UI action=<name> session=<sessionKey> surface=<surfaceId> component=<componentId> host=<machine> instance=<instanceId> ctx=<json?> default=update_canvas
```
## Follow-ups
- Add a small “action sent / failed” debug overlay in the A2UI shell (dev-only) to make failures obvious.