Skip to content

Commit 5c4079f

Browse files
steipetesteipete
committed
feat: add diagnostics events and otel exporter
1 parent b1f086b commit 5c4079f

14 files changed

Lines changed: 1030 additions & 13 deletions

File tree

‎docs/logging.md‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,72 @@ Tool summaries can redact sensitive tokens before they hit the console:
136136

137137
Redaction affects **console output only** and does not alter file logs.
138138

139+
## Diagnostics + OpenTelemetry
140+
141+
Diagnostics are **opt-in** structured events for model runs (usage + cost +
142+
context + duration). They do **not** replace logs; they exist to feed metrics,
143+
traces, and other exporters.
144+
145+
Clawdbot currently emits a `model.usage` event after each agent run with:
146+
147+
- Token counts (input/output/cache/prompt/total)
148+
- Estimated cost (USD)
149+
- Context window used/limit
150+
- Duration (ms)
151+
- Provider/channel/model + session identifiers
152+
153+
### Enable diagnostics (no exporter)
154+
155+
Use this if you want diagnostics events available to plugins or custom sinks:
156+
157+
```json
158+
{
159+
"diagnostics": {
160+
"enabled": true
161+
}
162+
}
163+
```
164+
165+
### Export to OpenTelemetry
166+
167+
Diagnostics can be exported via the `diagnostics-otel` plugin (OTLP/HTTP). This
168+
works with any OpenTelemetry collector/backend that accepts OTLP/HTTP.
169+
170+
```json
171+
{
172+
"plugins": {
173+
"allow": ["diagnostics-otel"],
174+
"entries": {
175+
"diagnostics-otel": {
176+
"enabled": true
177+
}
178+
}
179+
},
180+
"diagnostics": {
181+
"enabled": true,
182+
"otel": {
183+
"enabled": true,
184+
"endpoint": "http://otel-collector:4318",
185+
"protocol": "http/protobuf",
186+
"serviceName": "clawdbot-gateway",
187+
"traces": true,
188+
"metrics": true,
189+
"sampleRate": 0.2,
190+
"flushIntervalMs": 60000
191+
}
192+
}
193+
}
194+
```
195+
196+
Notes:
197+
- You can also enable the plugin with `clawdbot plugins enable diagnostics-otel`.
198+
- `protocol` currently supports `http/protobuf`.
199+
- Metrics include token usage, cost, context size, and run duration.
200+
- Traces/metrics can be toggled with `traces` / `metrics` (default: on).
201+
- Set `headers` when your collector requires auth.
202+
- Environment variables supported: `OTEL_EXPORTER_OTLP_ENDPOINT`,
203+
`OTEL_SERVICE_NAME`, `OTEL_EXPORTER_OTLP_PROTOCOL`.
204+
139205
## Troubleshooting tips
140206

141207
- **Gateway not reachable?** Run `clawdbot doctor` first.

‎extensions/diagnostics-otel/clawdbot.plugin.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"id": "diagnostics-otel",
3+
"configSchema": {
4+
"type": "object",
5+
"additionalProperties": false,
6+
"properties": {}
7+
}
8+
}

‎extensions/diagnostics-otel/index.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import type { ClawdbotPluginApi } from "clawdbot/plugin-sdk";
2+
import { emptyPluginConfigSchema } from "clawdbot/plugin-sdk";
3+
4+
import { createDiagnosticsOtelService } from "./src/service.js";
5+
6+
const plugin = {
7+
id: "diagnostics-otel",
8+
name: "Diagnostics OpenTelemetry",
9+
description: "Export diagnostics events to OpenTelemetry",
10+
configSchema: emptyPluginConfigSchema(),
11+
register(api: ClawdbotPluginApi) {
12+
api.registerService(createDiagnosticsOtelService());
13+
},
14+
};
15+
16+
export default plugin;

‎extensions/diagnostics-otel/package.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"name": "@clawdbot/diagnostics-otel",
3+
"version": "2026.1.17-1",
4+
"type": "module",
5+
"description": "Clawdbot diagnostics OpenTelemetry exporter",
6+
"clawdbot": {
7+
"extensions": ["./index.ts"]
8+
},
9+
"dependencies": {
10+
"@opentelemetry/api": "^1.9.0",
11+
"@opentelemetry/exporter-metrics-otlp-http": "^0.210.0",
12+
"@opentelemetry/exporter-trace-otlp-http": "^0.210.0",
13+
"@opentelemetry/resources": "^2.4.0",
14+
"@opentelemetry/sdk-metrics": "^2.4.0",
15+
"@opentelemetry/sdk-node": "^0.210.0",
16+
"@opentelemetry/sdk-trace-base": "^2.4.0",
17+
"@opentelemetry/semantic-conventions": "^1.39.0"
18+
}
19+
}

‎extensions/diagnostics-otel/src/service.ts‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
import { metrics, trace } from "@opentelemetry/api";
2+
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
3+
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
4+
import { Resource } from "@opentelemetry/resources";
5+
import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
6+
import { NodeSDK } from "@opentelemetry/sdk-node";
7+
import { ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
8+
import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
9+
10+
import type {
11+
ClawdbotPluginService,
12+
DiagnosticUsageEvent,
13+
} from "clawdbot/plugin-sdk";
14+
import { onDiagnosticEvent } from "clawdbot/plugin-sdk";
15+
16+
const DEFAULT_SERVICE_NAME = "clawdbot";
17+
18+
function normalizeEndpoint(endpoint?: string): string | undefined {
19+
const trimmed = endpoint?.trim();
20+
return trimmed ? trimmed.replace(/\/+$/, "") : undefined;
21+
}
22+
23+
function resolveOtelUrl(endpoint: string | undefined, path: string): string | undefined {
24+
if (!endpoint) return undefined;
25+
if (endpoint.includes("/v1/")) return endpoint;
26+
return `${endpoint}/${path}`;
27+
}
28+
29+
function resolveSampleRate(value: number | undefined): number | undefined {
30+
if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
31+
if (value < 0 || value > 1) return undefined;
32+
return value;
33+
}
34+
35+
export function createDiagnosticsOtelService(): ClawdbotPluginService {
36+
let sdk: NodeSDK | null = null;
37+
let unsubscribe: (() => void) | null = null;
38+
39+
return {
40+
id: "diagnostics-otel",
41+
async start(ctx) {
42+
const cfg = ctx.config.diagnostics;
43+
const otel = cfg?.otel;
44+
if (!cfg?.enabled || !otel?.enabled) return;
45+
46+
const protocol = otel.protocol ?? process.env.OTEL_EXPORTER_OTLP_PROTOCOL ?? "http/protobuf";
47+
if (protocol !== "http/protobuf") {
48+
ctx.logger.warn(`diagnostics-otel: unsupported protocol ${protocol}`);
49+
return;
50+
}
51+
52+
const endpoint = normalizeEndpoint(otel.endpoint ?? process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
53+
const headers = otel.headers ?? undefined;
54+
const serviceName =
55+
otel.serviceName?.trim() || process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE_NAME;
56+
const sampleRate = resolveSampleRate(otel.sampleRate);
57+
58+
const tracesEnabled = otel.traces !== false;
59+
const metricsEnabled = otel.metrics !== false;
60+
if (!tracesEnabled && !metricsEnabled) return;
61+
62+
const resource = new Resource({
63+
[SemanticResourceAttributes.SERVICE_NAME]: serviceName,
64+
});
65+
66+
const traceUrl = resolveOtelUrl(endpoint, "v1/traces");
67+
const metricUrl = resolveOtelUrl(endpoint, "v1/metrics");
68+
const traceExporter = tracesEnabled
69+
? new OTLPTraceExporter({
70+
...(traceUrl ? { url: traceUrl } : {}),
71+
...(headers ? { headers } : {}),
72+
})
73+
: undefined;
74+
75+
const metricExporter = metricsEnabled
76+
? new OTLPMetricExporter({
77+
...(metricUrl ? { url: metricUrl } : {}),
78+
...(headers ? { headers } : {}),
79+
})
80+
: undefined;
81+
82+
const metricReader = metricExporter
83+
? new PeriodicExportingMetricReader({
84+
exporter: metricExporter,
85+
...(typeof otel.flushIntervalMs === "number"
86+
? { exportIntervalMillis: Math.max(1000, otel.flushIntervalMs) }
87+
: {}),
88+
})
89+
: undefined;
90+
91+
sdk = new NodeSDK({
92+
resource,
93+
...(traceExporter ? { traceExporter } : {}),
94+
...(metricReader ? { metricReader } : {}),
95+
...(sampleRate !== undefined
96+
? {
97+
sampler: new ParentBasedSampler({
98+
root: new TraceIdRatioBasedSampler(sampleRate),
99+
}),
100+
}
101+
: {}),
102+
});
103+
104+
await sdk.start();
105+
106+
const meter = metrics.getMeter("clawdbot");
107+
const tracer = trace.getTracer("clawdbot");
108+
109+
const tokensCounter = meter.createCounter("clawdbot.tokens", {
110+
unit: "1",
111+
description: "Token usage by type",
112+
});
113+
const costCounter = meter.createCounter("clawdbot.cost.usd", {
114+
unit: "1",
115+
description: "Estimated model cost (USD)",
116+
});
117+
const durationHistogram = meter.createHistogram("clawdbot.run.duration_ms", {
118+
unit: "ms",
119+
description: "Agent run duration",
120+
});
121+
const contextHistogram = meter.createHistogram("clawdbot.context.tokens", {
122+
unit: "1",
123+
description: "Context window size and usage",
124+
});
125+
126+
unsubscribe = onDiagnosticEvent((evt) => {
127+
if (evt.type !== "model.usage") return;
128+
const usageEvent = evt as DiagnosticUsageEvent;
129+
const attrs = {
130+
"clawdbot.channel": usageEvent.channel ?? "unknown",
131+
"clawdbot.provider": usageEvent.provider ?? "unknown",
132+
"clawdbot.model": usageEvent.model ?? "unknown",
133+
};
134+
135+
const usage = usageEvent.usage;
136+
if (usage.input) tokensCounter.add(usage.input, { ...attrs, "clawdbot.token": "input" });
137+
if (usage.output)
138+
tokensCounter.add(usage.output, { ...attrs, "clawdbot.token": "output" });
139+
if (usage.cacheRead)
140+
tokensCounter.add(usage.cacheRead, { ...attrs, "clawdbot.token": "cache_read" });
141+
if (usage.cacheWrite)
142+
tokensCounter.add(usage.cacheWrite, { ...attrs, "clawdbot.token": "cache_write" });
143+
if (usage.promptTokens)
144+
tokensCounter.add(usage.promptTokens, { ...attrs, "clawdbot.token": "prompt" });
145+
if (usage.total)
146+
tokensCounter.add(usage.total, { ...attrs, "clawdbot.token": "total" });
147+
148+
if (usageEvent.costUsd) costCounter.add(usageEvent.costUsd, attrs);
149+
if (usageEvent.durationMs) durationHistogram.record(usageEvent.durationMs, attrs);
150+
if (usageEvent.context?.limit)
151+
contextHistogram.record(usageEvent.context.limit, {
152+
...attrs,
153+
"clawdbot.context": "limit",
154+
});
155+
if (usageEvent.context?.used)
156+
contextHistogram.record(usageEvent.context.used, {
157+
...attrs,
158+
"clawdbot.context": "used",
159+
});
160+
161+
if (!tracesEnabled) return;
162+
const spanAttrs: Record<string, string | number> = {
163+
...attrs,
164+
"clawdbot.sessionKey": usageEvent.sessionKey ?? "",
165+
"clawdbot.sessionId": usageEvent.sessionId ?? "",
166+
"clawdbot.tokens.input": usage.input ?? 0,
167+
"clawdbot.tokens.output": usage.output ?? 0,
168+
"clawdbot.tokens.cache_read": usage.cacheRead ?? 0,
169+
"clawdbot.tokens.cache_write": usage.cacheWrite ?? 0,
170+
"clawdbot.tokens.total": usage.total ?? 0,
171+
};
172+
173+
const startTime = usageEvent.durationMs
174+
? Date.now() - Math.max(0, usageEvent.durationMs)
175+
: undefined;
176+
const span = tracer.startSpan("clawdbot.model.usage", {
177+
attributes: spanAttrs,
178+
...(startTime ? { startTime } : {}),
179+
});
180+
span.end();
181+
});
182+
183+
if (otel.logs) {
184+
ctx.logger.warn("diagnostics-otel: logs exporter not wired yet");
185+
}
186+
},
187+
async stop() {
188+
unsubscribe?.();
189+
unsubscribe = null;
190+
if (sdk) {
191+
await sdk.shutdown().catch(() => undefined);
192+
sdk = null;
193+
}
194+
},
195+
} satisfies ClawdbotPluginService;
196+
}

0 commit comments

Comments
 (0)