toknmtr — self-hostable Claude Code usage & analytics dashboard
Server (SvelteKit + SQLite Docker container) + per-machine agent that parses Claude Code JSONL transcripts. docker-compose one-command deploy; raw transcript view + search gated behind SHOW_TRANSCRIPTS (hidden by default). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
71a60ab054
74 changed files with 15613 additions and 0 deletions
15
.dockerignore
Normal file
15
.dockerignore
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
node_modules
|
||||
.git
|
||||
build
|
||||
.svelte-kit
|
||||
data
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
*.db
|
||||
*.db-*
|
||||
*.sqlite
|
||||
*.sqlite-*
|
||||
.vscode
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
13
.env.example
Normal file
13
.env.example
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
# toknmtr server config. Copy to `.env` (gitignored) before `docker compose up`.
|
||||
# Only API_TOKEN is required — docker-compose.yml sets DB_PATH, the port, and body limit.
|
||||
|
||||
# --- server ---
|
||||
API_TOKEN=change-me # Bearer token the agent must present to /api/ingest.
|
||||
# Generate a strong one: openssl rand -hex 32
|
||||
# SHOW_TRANSCRIPTS=true # Unset/false = HIDE the transcript view + full-text search
|
||||
# (raw prompt/response text). Only enable behind auth.
|
||||
|
||||
# --- agent (informational) ---
|
||||
# The per-machine agent reads these from ~/.toknmtr/env, NOT from this file (see ops/README.md):
|
||||
# TOKNMTR_URL=http://<server-host>:3001 # base URL of your deployed server
|
||||
# TOKNMTR_TOKEN=<same value as API_TOKEN above>
|
||||
32
.gitignore
vendored
Normal file
32
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
node_modules
|
||||
|
||||
# Output
|
||||
.output
|
||||
.vercel
|
||||
.netlify
|
||||
.wrangler
|
||||
/.svelte-kit
|
||||
/build
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Env
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
!.env.test
|
||||
|
||||
# Vite
|
||||
vite.config.js.timestamp-*
|
||||
vite.config.ts.timestamp-*
|
||||
|
||||
# toknmtr data — never commit the SQLite DB or secrets
|
||||
/data/
|
||||
*.sqlite
|
||||
*.sqlite-*
|
||||
*.db
|
||||
*.db-*
|
||||
*.pem
|
||||
*.key
|
||||
1
.npmrc
Normal file
1
.npmrc
Normal file
|
|
@ -0,0 +1 @@
|
|||
engine-strict=true
|
||||
9
.prettierignore
Normal file
9
.prettierignore
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
# Package Managers
|
||||
package-lock.json
|
||||
pnpm-lock.yaml
|
||||
yarn.lock
|
||||
bun.lock
|
||||
bun.lockb
|
||||
|
||||
# Miscellaneous
|
||||
/static/
|
||||
15
.prettierrc
Normal file
15
.prettierrc
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"useTabs": true,
|
||||
"singleQuote": true,
|
||||
"trailingComma": "none",
|
||||
"printWidth": 100,
|
||||
"plugins": ["prettier-plugin-svelte"],
|
||||
"overrides": [
|
||||
{
|
||||
"files": "*.svelte",
|
||||
"options": {
|
||||
"parser": "svelte"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
184
DEPLOY.md
Normal file
184
DEPLOY.md
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
# Deploying toknmtr — step by step
|
||||
|
||||
This walks you from a clone to a live dashboard with your own Claude Code usage flowing in.
|
||||
|
||||
toknmtr has **two halves**, and you set them up in order:
|
||||
|
||||
1. **Server** — one Docker container (dashboard + ingest API + SQLite). Run it once, anywhere
|
||||
reachable from your machines (a home server, a NAS, or even your laptop).
|
||||
2. **Agent** — a small script you run on **every machine where you use Claude Code**. It reads
|
||||
that machine's transcripts and pushes them to the server. Nothing shows up until at least
|
||||
one agent has run.
|
||||
|
||||
> **The server image holds no data.** The database is created empty in a Docker volume the
|
||||
> first time the container starts. Everything you see in the dashboard came from *your* agent
|
||||
> pushing *your* transcripts to *your* server.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
**For the server:**
|
||||
- A host with **Docker** + the **Compose plugin** (`docker compose version` works).
|
||||
|
||||
**For the agent (on each machine you use Claude Code):**
|
||||
- **Node.js 24+** (`node --version`) — needed for `--experimental-strip-types`.
|
||||
- A clone of this repo.
|
||||
- `jq` (only if you use the auto-capture hook installer).
|
||||
|
||||
---
|
||||
|
||||
## Part 1 — Run the server
|
||||
|
||||
### 1.1 Clone and configure
|
||||
|
||||
```sh
|
||||
git clone <this-repo-url> toknmtr
|
||||
cd toknmtr
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Open `.env` and set a strong `API_TOKEN` — this is the shared secret between the server and
|
||||
every agent. Generate one:
|
||||
|
||||
```sh
|
||||
openssl rand -hex 32
|
||||
```
|
||||
|
||||
Paste it as `API_TOKEN=...`. Leave `SHOW_TRANSCRIPTS` commented out for now (see
|
||||
[Security](#security--exposure) below).
|
||||
|
||||
### 1.2 Start it
|
||||
|
||||
```sh
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
The first build takes a few minutes (it compiles `better-sqlite3`). When it finishes, the
|
||||
dashboard is at **http://localhost:3001** (or `http://<server-host>:3001` from another
|
||||
machine on your network).
|
||||
|
||||
### 1.3 Verify
|
||||
|
||||
```sh
|
||||
# health check — should print {"ok":true,...}
|
||||
curl -s http://localhost:3001/api/ingest
|
||||
|
||||
# dashboard should return 200
|
||||
curl -s -o /dev/null -w '%{http_code}\n' http://localhost:3001/
|
||||
```
|
||||
|
||||
The dashboard will be empty until you set up an agent (Part 2). The container restarts
|
||||
automatically unless you stop it; the DB persists in the `toknmtr-data` Docker volume across
|
||||
restarts and rebuilds.
|
||||
|
||||
**Common first-run issues**
|
||||
- `API_TOKEN` error from compose → you didn't set it in `.env`.
|
||||
- Port 3001 already in use → change the host side of the mapping in `docker-compose.yml`
|
||||
(`"3001:3000"` → e.g. `"8099:3000"`), then `docker compose up -d`.
|
||||
|
||||
---
|
||||
|
||||
## Part 2 — Feed it data (the agent)
|
||||
|
||||
Do this on **each machine** where you run Claude Code. The agent reads
|
||||
`~/.claude/projects/**/*.jsonl` and POSTs to your server.
|
||||
|
||||
### 2.1 Point the agent at your server
|
||||
|
||||
```sh
|
||||
mkdir -p ~/.toknmtr
|
||||
cat > ~/.toknmtr/env <<'EOF'
|
||||
TOKNMTR_URL=http://<server-host>:3001
|
||||
TOKNMTR_TOKEN=<the same API_TOKEN you put on the server>
|
||||
EOF
|
||||
chmod 600 ~/.toknmtr/env
|
||||
```
|
||||
|
||||
Replace `<server-host>` with the server's hostname or LAN IP (use `localhost` if the agent
|
||||
runs on the same box as the server). `TOKNMTR_TOKEN` **must exactly match** the server's
|
||||
`API_TOKEN`.
|
||||
|
||||
### 2.2 Backfill existing history (one time)
|
||||
|
||||
From your clone of this repo on that machine:
|
||||
|
||||
```sh
|
||||
node --experimental-strip-types agent/run.ts --backfill
|
||||
```
|
||||
|
||||
This ingests every transcript already on disk. Refresh the dashboard — data should appear.
|
||||
|
||||
> Claude Code prunes local transcripts after `cleanupPeriodDays` (default 30), so backfill
|
||||
> only reaches as far back as what's still on disk. From here on, the live hook (next step)
|
||||
> captures everything going forward into the server's permanent DB.
|
||||
|
||||
### 2.3 Turn on live capture
|
||||
|
||||
```sh
|
||||
ops/install-hook.sh
|
||||
```
|
||||
|
||||
This registers a Claude Code `Stop` hook that pushes new activity at the end of every turn —
|
||||
near-live, and near-zero cost to your session. It's additive and reversible
|
||||
(`ops/install-hook.sh --remove`).
|
||||
|
||||
Optionally add a periodic reconcile sweep as a safety net (catches anything a missed hook or
|
||||
a server-down window skipped):
|
||||
|
||||
```sh
|
||||
ops/install-cron.sh
|
||||
```
|
||||
|
||||
Full agent details — the `~/.toknmtr/env` format, the hook-vs-cron tradeoff, and
|
||||
troubleshooting — are in **[ops/README.md](ops/README.md)**.
|
||||
|
||||
Re-ingesting is always safe: the server upserts idempotently on `host + session_id + uuid`,
|
||||
so backfills and overlapping hook/cron sweeps never create duplicates.
|
||||
|
||||
---
|
||||
|
||||
## Security & exposure
|
||||
|
||||
**The dashboard has no login.** Treat it as trusted-network-only unless you add auth.
|
||||
|
||||
- **Raw conversation text is hidden by default.** The transcript view and full-text search —
|
||||
the only surfaces that show verbatim prompts/responses — return `403` unless you set
|
||||
`SHOW_TRANSCRIPTS=true`. Charts, KPIs, and session metadata are always visible. Only enable
|
||||
`SHOW_TRANSCRIPTS` once the whole dashboard is behind authentication.
|
||||
- **Don't put it on the public internet as-is.** Keep it on your LAN/VPN, or front it with a
|
||||
reverse proxy that enforces auth (e.g. Caddy/nginx basic-auth, Authelia, Tailscale, etc.).
|
||||
- The `API_TOKEN` gates ingest only, not the dashboard. Keep it secret; it lives in `.env`
|
||||
(server) and `~/.toknmtr/env` (agents), both of which stay off git.
|
||||
|
||||
To reveal transcripts/search after you've added auth: set `SHOW_TRANSCRIPTS=true` in `.env`
|
||||
and `docker compose up -d` again.
|
||||
|
||||
---
|
||||
|
||||
## Updating
|
||||
|
||||
```sh
|
||||
cd toknmtr
|
||||
git pull
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
Your data is untouched — it lives in the `toknmtr-data` volume, not the image.
|
||||
|
||||
On agent machines, `git pull` in the clone; the hook/cron run the updated code automatically.
|
||||
|
||||
---
|
||||
|
||||
## Uninstalling
|
||||
|
||||
```sh
|
||||
# server
|
||||
docker compose down # stop + remove the container (keeps the data volume)
|
||||
docker compose down -v # ...and DELETE the database volume too
|
||||
|
||||
# agent (per machine)
|
||||
ops/install-hook.sh --remove
|
||||
ops/install-cron.sh --remove
|
||||
rm -rf ~/.toknmtr
|
||||
```
|
||||
23
Dockerfile
Normal file
23
Dockerfile
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# --- build stage ---
|
||||
FROM node:24-slim AS build
|
||||
WORKDIR /app
|
||||
# build tools for better-sqlite3 if a prebuilt binary isn't available
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends python3 make g++ \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
COPY package.json package-lock.json* ./
|
||||
RUN npm install
|
||||
COPY . .
|
||||
RUN npm run build && npm prune --omit=dev
|
||||
|
||||
# --- runtime stage ---
|
||||
FROM node:24-slim
|
||||
WORKDIR /app
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=3000
|
||||
ENV DB_PATH=/data/toknmtr.db
|
||||
COPY --from=build /app/build ./build
|
||||
COPY --from=build /app/node_modules ./node_modules
|
||||
COPY package.json ./
|
||||
VOLUME /data
|
||||
EXPOSE 3000
|
||||
CMD ["node", "build"]
|
||||
101
README.md
Normal file
101
README.md
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
# toknmtr
|
||||
|
||||
Self-hosted **Claude Code usage & analytics dashboard**. An **agent** on each of your
|
||||
machines parses Claude Code's JSONL transcripts into a full event log and pushes it to a
|
||||
**server** (one SvelteKit + SQLite container). The server stores everything — usage, tool
|
||||
calls, commands, raw prompts/responses — and serves a dashboard plus a full-text-searchable
|
||||
session archive.
|
||||
|
||||
It's two halves:
|
||||
|
||||
- **Server** (`src/`, shipped as a Docker image) — the dashboard, the ingest API, the DB.
|
||||
- **Agent** (`agent/` + `ops/`, runs on each machine) — parses `~/.claude/projects/**/*.jsonl`
|
||||
and POSTs to the server. Not part of the image; you run it wherever you use Claude Code.
|
||||
|
||||
The server image contains **no data** — the SQLite DB is created empty in a mounted volume on
|
||||
first run. All conversation content only ever comes from *your* agent pushing *your*
|
||||
transcripts to *your* server.
|
||||
|
||||
> **New here? Follow [DEPLOY.md](DEPLOY.md)** — a full step-by-step walkthrough (prerequisites,
|
||||
> server, agent, security, updating). The sections below are the quick reference.
|
||||
|
||||
---
|
||||
|
||||
## 1. Run the server (Docker)
|
||||
|
||||
Requires Docker with the Compose plugin. From the repo root:
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
# edit .env: set API_TOKEN to a long random secret
|
||||
# openssl rand -hex 32
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
The dashboard is now at **http://localhost:3001**. The DB lives in the `toknmtr-data`
|
||||
Docker volume; the container restarts unless stopped.
|
||||
|
||||
**Privacy note — the dashboard has no auth.** By default the two surfaces that expose
|
||||
verbatim prompt/response text (the per-session **transcript view** and full-text **search**)
|
||||
are **hidden**; charts, KPIs, and session metadata stay visible. Only set
|
||||
`SHOW_TRANSCRIPTS=true` in `.env` once you've put the dashboard behind auth (reverse proxy,
|
||||
VPN, etc.). Don't expose it to the public internet as-is.
|
||||
|
||||
### Config (`.env`)
|
||||
|
||||
| Var | Purpose |
|
||||
| ------------------ | --------------------------------------------------------------- |
|
||||
| `API_TOKEN` | **Required.** Bearer token the agent must send to `/api/ingest`. |
|
||||
| `SHOW_TRANSCRIPTS` | `true` reveals transcript view + search. Unset = hidden (safe). |
|
||||
| `PORT` | Host port is set in `docker-compose.yml` (`3001:3000`). |
|
||||
|
||||
---
|
||||
|
||||
## 2. Feeding it data (the agent)
|
||||
|
||||
The server starts empty. To populate it, run the agent on each machine where you use Claude
|
||||
Code (needs Node 24+ for `--experimental-strip-types`). One-time setup per machine:
|
||||
|
||||
```sh
|
||||
mkdir -p ~/.toknmtr
|
||||
cat > ~/.toknmtr/env <<'EOF'
|
||||
TOKNMTR_URL=http://<server-host>:3001
|
||||
TOKNMTR_TOKEN=<the same API_TOKEN you set on the server>
|
||||
EOF
|
||||
chmod 600 ~/.toknmtr/env
|
||||
```
|
||||
|
||||
Then, from a clone of this repo on that machine:
|
||||
|
||||
```sh
|
||||
# one-time: ingest all transcripts already on disk
|
||||
node --experimental-strip-types agent/run.ts --backfill
|
||||
|
||||
# ongoing capture — register the Stop hook so every turn pushes incrementally
|
||||
ops/install-hook.sh
|
||||
```
|
||||
|
||||
`ops/install-hook.sh` adds a near-zero-cost Claude Code `Stop` hook (additive + reversible
|
||||
with `--remove`). An optional `ops/install-cron.sh` adds a reconcile sweep as a safety net.
|
||||
Full agent/capture docs — the `~/.toknmtr/env` format, the hook-vs-cron tradeoff, backfill,
|
||||
and troubleshooting — are in **[`ops/README.md`](ops/README.md)**.
|
||||
|
||||
Ingest is an idempotent upsert (keyed on `host + session_id + uuid`), so re-running the
|
||||
backfill or overlapping hook/cron sweeps never duplicates data.
|
||||
|
||||
---
|
||||
|
||||
## 3. Development
|
||||
|
||||
Node 24, SvelteKit 2 (Svelte 5), TypeScript, `better-sqlite3`.
|
||||
|
||||
```sh
|
||||
npm install
|
||||
npm run dev # dev server
|
||||
npm run check # typecheck
|
||||
npm run build # production build → build/ (run with `node build`)
|
||||
npm run lint # prettier + eslint
|
||||
```
|
||||
|
||||
Pricing lives server-side in `src/lib/server/pricing.ts` — adding a model is a one-line
|
||||
update. Because the subscription is flat-rate, all `$` figures are *notional* (API-equivalent).
|
||||
66
agent/cursor.ts
Normal file
66
agent/cursor.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
/**
|
||||
* toknmtr agent — per-file byte-offset cursor.
|
||||
*
|
||||
* Persists, per transcript file, the byte offset up to which we've already parsed and
|
||||
* pushed events, so re-runs only emit NEW bytes appended since last time. Stored as a
|
||||
* single JSON file at ~/.toknmtr/cursors.json (one process at a time is assumed — there's
|
||||
* no file locking).
|
||||
*
|
||||
* Handles truncation/rotation: if a file's current size is smaller than the recorded
|
||||
* offset, the file was truncated or replaced (e.g. a session id got reused, or the file
|
||||
* was edited out from under us) — getOffset() resets to 0 so the whole file is reparsed.
|
||||
*/
|
||||
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
export const CURSOR_FILE = join(homedir(), '.toknmtr', 'cursors.json');
|
||||
|
||||
interface CursorEntry {
|
||||
offset: number;
|
||||
size: number;
|
||||
}
|
||||
|
||||
type CursorMap = Record<string, CursorEntry>;
|
||||
|
||||
let cache: CursorMap | null = null;
|
||||
|
||||
function load(): CursorMap {
|
||||
if (cache) return cache;
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(CURSOR_FILE, 'utf8')) as unknown;
|
||||
cache = parsed && typeof parsed === 'object' ? (parsed as CursorMap) : {};
|
||||
} catch {
|
||||
cache = {};
|
||||
}
|
||||
return cache;
|
||||
}
|
||||
|
||||
/** Persist the in-memory cursor map to disk. Call once after a batch of setOffset() calls. */
|
||||
export function save(): void {
|
||||
const map = load();
|
||||
mkdirSync(dirname(CURSOR_FILE), { recursive: true });
|
||||
writeFileSync(CURSOR_FILE, JSON.stringify(map, null, 2));
|
||||
}
|
||||
|
||||
/**
|
||||
* Byte offset to resume reading `filePath` from, given its current size on disk.
|
||||
* Returns 0 (full reparse) if there's no recorded cursor, or if the file shrank since
|
||||
* last run (truncated/rotated).
|
||||
*/
|
||||
export function getOffset(filePath: string, currentSize: number): number {
|
||||
const entry = load()[filePath];
|
||||
if (!entry) return 0;
|
||||
if (entry.offset > currentSize) return 0;
|
||||
return entry.offset;
|
||||
}
|
||||
|
||||
/** Record the new offset/size for `filePath` in memory. Call save() to persist. */
|
||||
export function setOffset(filePath: string, offset: number, size: number): void {
|
||||
load()[filePath] = { offset, size };
|
||||
}
|
||||
|
||||
/** Forget a file's cursor entirely (forces a full reparse on next run). */
|
||||
export function resetOffset(filePath: string): void {
|
||||
delete load()[filePath];
|
||||
}
|
||||
74
agent/hooks/toknmtr-capture.sh
Executable file
74
agent/hooks/toknmtr-capture.sh
Executable file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# toknmtr-capture.sh — Claude Code hook that triggers an incremental toknmtr agent sweep.
|
||||
#
|
||||
# Registered (by ops/install-hook.sh) under the Claude Code 'Stop' hook event, so it fires
|
||||
# at the end of every assistant turn — see ops/install-hook.sh for the Stop-vs-SessionEnd
|
||||
# tradeoff writeup.
|
||||
#
|
||||
# FAIL-OPEN CONTRACT — this script must NEVER block, slow down, or fail the Claude Code
|
||||
# session it's attached to:
|
||||
# - The actual agent run (`node agent/run.ts --once`) is launched with `nohup ... &` and
|
||||
# disowned, with stdio redirected to a log file (never inherited from the hook's own
|
||||
# stdio), so this script returns to Claude Code in a handful of milliseconds — it does
|
||||
# NOT wait for the parse+push to finish.
|
||||
# - `timeout` wraps the backgrounded agent run so a hung/unreachable server can never
|
||||
# leave an orphaned process running forever.
|
||||
# - Every prerequisite (project dir, config file, node, timeout) is individually checked;
|
||||
# any miss just skips capture for this turn silently.
|
||||
# - This script prints NOTHING to its own stdout/stderr (Claude Code parses Stop-hook
|
||||
# stdout as potential JSON — e.g. {"decision":"block"} would force the session to keep
|
||||
# going — so silence here is required, not just polite). All diagnostic output goes to
|
||||
# $LOG_FILE instead.
|
||||
# - Always exits 0, intentionally not using `set -e`: every step below is already
|
||||
# individually guarded, so a failure anywhere means "skip capture this turn", never
|
||||
# "fail the hook" / block the Stop event.
|
||||
#
|
||||
# Config: ~/.toknmtr/env (TOKNMTR_URL, TOKNMTR_TOKEN — see ops/README.md). Kept out of
|
||||
# ~/.claude/settings.json so secrets aren't sitting in a file that's more likely to be
|
||||
# shared, synced, or dumped for support/debugging.
|
||||
#
|
||||
# Override knobs (env, all optional):
|
||||
# TOKNMTR_PROJECT_DIR path to the toknmtr repo checkout (default: ~/claude/projects/toknmtr)
|
||||
# TOKNMTR_ENV_FILE path to the config file (default: ~/.toknmtr/env)
|
||||
# TOKNMTR_LOG_FILE where backgrounded output is logged (default: ~/.toknmtr/capture.log)
|
||||
# TOKNMTR_HOOK_TIMEOUT_S max seconds the backgrounded sweep may run (default: 25)
|
||||
|
||||
PROJECT_DIR="${TOKNMTR_PROJECT_DIR:-$HOME/claude/projects/toknmtr}"
|
||||
CONFIG_FILE="${TOKNMTR_ENV_FILE:-$HOME/.toknmtr/env}"
|
||||
LOG_FILE="${TOKNMTR_LOG_FILE:-$HOME/.toknmtr/capture.log}"
|
||||
TIMEOUT_S="${TOKNMTR_HOOK_TIMEOUT_S:-25}"
|
||||
|
||||
# Claude Code hooks receive a JSON payload on stdin describing the event. We don't need its
|
||||
# contents (the agent re-walks transcripts itself from disk), but drain it anyway so we
|
||||
# never leave the pipe half-read.
|
||||
cat >/dev/null 2>&1 || true
|
||||
|
||||
# Bail out quietly (still exit 0) if any prerequisite is missing — never surface a hook
|
||||
# failure to the session over a merely-unconfigured machine.
|
||||
[ -d "$PROJECT_DIR" ] || exit 0
|
||||
[ -f "$PROJECT_DIR/agent/run.ts" ] || exit 0
|
||||
[ -f "$CONFIG_FILE" ] || exit 0
|
||||
command -v node >/dev/null 2>&1 || exit 0
|
||||
|
||||
mkdir -p "$(dirname "$LOG_FILE")" 2>/dev/null || exit 0
|
||||
|
||||
# Build the backgrounded command as a single string for `bash -c` so it can `cd`, source
|
||||
# the config file (exporting TOKNMTR_URL/TOKNMTR_TOKEN into its own environment), and then
|
||||
# exec node — all inside the detached child, never the foreground hook process.
|
||||
read -r -d '' INNER_CMD <<EOF || true
|
||||
cd "$PROJECT_DIR" || exit 0
|
||||
set -a
|
||||
. "$CONFIG_FILE"
|
||||
set +a
|
||||
exec node --experimental-strip-types agent/run.ts --once
|
||||
EOF
|
||||
|
||||
if command -v timeout >/dev/null 2>&1; then
|
||||
nohup timeout "${TIMEOUT_S}s" bash -c "$INNER_CMD" >>"$LOG_FILE" 2>&1 &
|
||||
else
|
||||
nohup bash -c "$INNER_CMD" >>"$LOG_FILE" 2>&1 &
|
||||
fi
|
||||
disown 2>/dev/null || true
|
||||
|
||||
exit 0
|
||||
384
agent/parse.ts
Normal file
384
agent/parse.ts
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
/**
|
||||
* toknmtr agent — JSONL parser.
|
||||
*
|
||||
* Walks ~/.claude/projects/<encoded-cwd>/<session-id>.jsonl and turns every line
|
||||
* into a typed event the server can ingest. NOT bundled by Vite — this runs
|
||||
* standalone via the capture hook + cron (e.g. `node --experimental-strip-types
|
||||
* agent/run.ts`).
|
||||
*
|
||||
* Real JSONL line shape (see CLAUDE.md / the shared agent contract):
|
||||
* - Not every physical line is a trackable event: `mode`, `file-history-snapshot`,
|
||||
* `attachment`, `ai-title`, `last-prompt`, etc. carry no top-level `uuid` and are
|
||||
* skipped (parseLine returns null), but they (and every other line) may still carry
|
||||
* `sessionId`/`cwd`/`gitBranch`/`version`/`entrypoint`/`timestamp`, which we scan for
|
||||
* session metadata regardless of whether the line itself becomes an event.
|
||||
* - Assistant turns STREAM: the same `message.id` repeats across several consecutive
|
||||
* physical lines (each with its own unique top-level `uuid`, chained via `parentUuid`),
|
||||
* each carrying a different slice of `message.content` (one block per line in
|
||||
* practice: a `thinking` line, then a `text` line, then one `tool_use` line per tool
|
||||
* call). Every physical line is still stored as its own `events` row.
|
||||
* - `message.content` is either a plain string (typed user prompt) or an array of
|
||||
* blocks: `{type:'text', text}`, `{type:'thinking', thinking}`,
|
||||
* `{type:'tool_use', id, name, input}`, and on USER lines
|
||||
* `{type:'tool_result', tool_use_id, content, is_error}` (content is a string or a
|
||||
* content-block array).
|
||||
*/
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
export const TRANSCRIPT_ROOT = join(homedir(), '.claude', 'projects');
|
||||
|
||||
export interface ParsedEvent {
|
||||
host: string;
|
||||
session_id: string;
|
||||
uuid: string;
|
||||
parent_uuid: string | null;
|
||||
ts_utc: string;
|
||||
type: string;
|
||||
role: string | null;
|
||||
model: string | null;
|
||||
request_id: string | null;
|
||||
message_id: string | null;
|
||||
is_sidechain: boolean;
|
||||
/** True for the one row per (session_id, message_id, request_id) group that should
|
||||
* count toward token totals — the final/max-output_tokens streamed line. */
|
||||
is_usage_canonical: boolean;
|
||||
stop_reason: string | null;
|
||||
/** ms between this line's timestamp and its parent line's timestamp (assistant lines
|
||||
* only, when the parent is known within the parsed chunk). */
|
||||
latency_ms: number | null;
|
||||
input_tokens: number | null;
|
||||
output_tokens: number | null;
|
||||
cache_creation_tokens: number | null;
|
||||
cache_read_tokens: number | null;
|
||||
web_search_requests: number | null;
|
||||
web_fetch_requests: number | null;
|
||||
/** Flattened visible text (joined `text` content blocks, or the raw string content of
|
||||
* a plain-string user message). Does NOT include `thinking` or `tool_result` content. */
|
||||
text: string | null;
|
||||
}
|
||||
|
||||
/** Mirrors the `tool_calls` table. */
|
||||
export interface ToolCall {
|
||||
host: string;
|
||||
session_id: string;
|
||||
tool_use_id: string;
|
||||
event_uuid: string | null;
|
||||
tool_name: string;
|
||||
input_json: string | null;
|
||||
is_error: boolean | null;
|
||||
result_bytes: number | null;
|
||||
duration_ms: number | null;
|
||||
ts_utc: string | null;
|
||||
}
|
||||
|
||||
/** Mirrors the `sessions` table. */
|
||||
export interface SessionMeta {
|
||||
host: string;
|
||||
session_id: string;
|
||||
project: string | null;
|
||||
git_branch: string | null;
|
||||
cc_version: string | null;
|
||||
entrypoint: string | null;
|
||||
started_at: string | null;
|
||||
ended_at: string | null;
|
||||
}
|
||||
|
||||
interface ContentBlock {
|
||||
type?: string;
|
||||
text?: string;
|
||||
thinking?: string;
|
||||
id?: string;
|
||||
name?: string;
|
||||
input?: unknown;
|
||||
tool_use_id?: string;
|
||||
content?: unknown;
|
||||
is_error?: boolean;
|
||||
}
|
||||
|
||||
function asContentBlocks(content: unknown): ContentBlock[] {
|
||||
return Array.isArray(content) ? (content as ContentBlock[]) : [];
|
||||
}
|
||||
|
||||
/** Join all `text`-type content blocks, or return a plain-string message as-is. */
|
||||
function flattenText(content: unknown): string | null {
|
||||
if (typeof content === 'string') return content.length > 0 ? content : null;
|
||||
const texts = asContentBlocks(content)
|
||||
.filter(
|
||||
(b): b is ContentBlock & { text: string } => b.type === 'text' && typeof b.text === 'string'
|
||||
)
|
||||
.map((b) => b.text);
|
||||
return texts.length > 0 ? texts.join('\n\n') : null;
|
||||
}
|
||||
|
||||
/** Byte length of tool_result content, which may be a string or a content-block array. */
|
||||
function byteLengthOf(content: unknown): number {
|
||||
if (typeof content === 'string') return Buffer.byteLength(content, 'utf8');
|
||||
if (content === undefined || content === null) return 0;
|
||||
try {
|
||||
return Buffer.byteLength(JSON.stringify(content), 'utf8');
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
interface ParsedToolUse {
|
||||
tool_use_id: string;
|
||||
tool_name: string;
|
||||
input_json: string;
|
||||
}
|
||||
|
||||
interface ParsedToolResult {
|
||||
tool_use_id: string;
|
||||
is_error: boolean;
|
||||
result_bytes: number;
|
||||
}
|
||||
|
||||
export interface LineParseResult {
|
||||
event: ParsedEvent;
|
||||
toolUses: ParsedToolUse[];
|
||||
toolResults: ParsedToolResult[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single JSONL line into a ParsedEvent (plus any tool_use/tool_result blocks it
|
||||
* carries), or null if the line isn't a trackable transcript event (no `type`/`uuid`) or
|
||||
* isn't valid JSON at all (workflow journal files, a truncated in-flight line, etc.).
|
||||
*/
|
||||
export function parseLine(host: string, line: string): LineParseResult | null {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) return null;
|
||||
|
||||
let d: Record<string, unknown>;
|
||||
try {
|
||||
d = JSON.parse(trimmed) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const type = d.type as string | undefined;
|
||||
if (!type || typeof d.uuid !== 'string') return null;
|
||||
|
||||
const message = (d.message ?? {}) as Record<string, unknown>;
|
||||
const usage = (message.usage ?? {}) as Record<string, unknown>;
|
||||
const serverTool = (usage.server_tool_use ?? {}) as Record<string, unknown>;
|
||||
const content = message.content;
|
||||
|
||||
const toolUses: ParsedToolUse[] = [];
|
||||
const toolResults: ParsedToolResult[] = [];
|
||||
for (const block of asContentBlocks(content)) {
|
||||
if (
|
||||
block.type === 'tool_use' &&
|
||||
typeof block.id === 'string' &&
|
||||
typeof block.name === 'string'
|
||||
) {
|
||||
toolUses.push({
|
||||
tool_use_id: block.id,
|
||||
tool_name: block.name,
|
||||
input_json: JSON.stringify(block.input ?? {})
|
||||
});
|
||||
} else if (block.type === 'tool_result' && typeof block.tool_use_id === 'string') {
|
||||
toolResults.push({
|
||||
tool_use_id: block.tool_use_id,
|
||||
is_error: Boolean(block.is_error),
|
||||
result_bytes: byteLengthOf(block.content)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const numOrNull = (v: unknown): number | null =>
|
||||
typeof v === 'number' && Number.isFinite(v) ? v : null;
|
||||
|
||||
const event: ParsedEvent = {
|
||||
host,
|
||||
session_id: (d.sessionId as string) ?? '',
|
||||
uuid: d.uuid,
|
||||
parent_uuid: typeof d.parentUuid === 'string' ? d.parentUuid : null,
|
||||
ts_utc: typeof d.timestamp === 'string' ? d.timestamp : '',
|
||||
type,
|
||||
role: typeof message.role === 'string' ? message.role : null,
|
||||
model: typeof message.model === 'string' ? message.model : null,
|
||||
request_id: typeof d.requestId === 'string' ? d.requestId : null,
|
||||
message_id: typeof message.id === 'string' ? message.id : null,
|
||||
is_sidechain: Boolean(d.isSidechain),
|
||||
is_usage_canonical: false, // set by parseTranscript, which sees the whole batch
|
||||
stop_reason: typeof message.stop_reason === 'string' ? message.stop_reason : null,
|
||||
latency_ms: null, // set by parseTranscript, which has sibling-line context
|
||||
input_tokens: numOrNull(usage.input_tokens),
|
||||
output_tokens: numOrNull(usage.output_tokens),
|
||||
cache_creation_tokens: numOrNull(usage.cache_creation_input_tokens),
|
||||
cache_read_tokens: numOrNull(usage.cache_read_input_tokens),
|
||||
web_search_requests: numOrNull(serverTool.web_search_requests),
|
||||
web_fetch_requests: numOrNull(serverTool.web_fetch_requests),
|
||||
text: flattenText(content)
|
||||
};
|
||||
|
||||
return { event, toolUses, toolResults };
|
||||
}
|
||||
|
||||
export interface ParseResult {
|
||||
events: ParsedEvent[];
|
||||
toolCalls: ToolCall[];
|
||||
session: SessionMeta;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a chunk of JSONL text (a whole file, or just the bytes appended since the last
|
||||
* cursor position — see cursor.ts/run.ts) into events, tool calls, and session metadata.
|
||||
*
|
||||
* `sessionIdFallback` should be the session id derived from the filename (transcripts are
|
||||
* named `<session-id>.jsonl`); it's used for lines that for some reason omit `sessionId`,
|
||||
* and as the session's id before any line with `sessionId` has been seen.
|
||||
*
|
||||
* Session metadata (project/branch/version/entrypoint/started_at/ended_at) is scanned
|
||||
* across ALL lines in the chunk, including ones that aren't trackable events themselves
|
||||
* (e.g. the leading `mode`/`file-history-snapshot` lines carry `cwd`/`gitBranch`). On an
|
||||
* incremental (non-backfill) chunk that doesn't include the start of the file, some of
|
||||
* these fields may come back null — the ingest server COALESCEs session fields so that's
|
||||
* safe (an earlier sweep's non-null values are preserved).
|
||||
*/
|
||||
export function parseTranscript(
|
||||
host: string,
|
||||
sessionIdFallback: string,
|
||||
text: string
|
||||
): ParseResult {
|
||||
const events: ParsedEvent[] = [];
|
||||
const toolCalls: ToolCall[] = [];
|
||||
|
||||
const tsByUuid = new Map<string, string>();
|
||||
const pendingToolCalls = new Map<string, ToolCall>();
|
||||
const usageGroups = new Map<string, ParsedEvent[]>();
|
||||
|
||||
let sessionId = sessionIdFallback;
|
||||
let project: string | null = null;
|
||||
let gitBranch: string | null = null;
|
||||
let ccVersion: string | null = null;
|
||||
let entrypoint: string | null = null;
|
||||
let minTs: string | null = null;
|
||||
let maxTs: string | null = null;
|
||||
|
||||
for (const rawLine of text.split('\n')) {
|
||||
const trimmed = rawLine.trim();
|
||||
if (!trimmed) continue;
|
||||
|
||||
let d: Record<string, unknown>;
|
||||
try {
|
||||
d = JSON.parse(trimmed) as Record<string, unknown>;
|
||||
} catch {
|
||||
continue; // malformed/truncated line — skip gracefully
|
||||
}
|
||||
|
||||
// Pull session metadata from every line that has it, not just trackable events.
|
||||
if (typeof d.sessionId === 'string' && d.sessionId.length > 0) sessionId = d.sessionId;
|
||||
if (typeof d.cwd === 'string') project = d.cwd;
|
||||
if (typeof d.gitBranch === 'string') gitBranch = d.gitBranch;
|
||||
if (typeof d.version === 'string') ccVersion = d.version;
|
||||
if (typeof d.entrypoint === 'string') entrypoint = d.entrypoint;
|
||||
if (typeof d.timestamp === 'string' && d.timestamp.length > 0) {
|
||||
if (!minTs || d.timestamp < minTs) minTs = d.timestamp;
|
||||
if (!maxTs || d.timestamp > maxTs) maxTs = d.timestamp;
|
||||
}
|
||||
|
||||
const parsed = parseLine(host, trimmed);
|
||||
if (!parsed) continue;
|
||||
const { event, toolUses, toolResults } = parsed;
|
||||
if (!event.session_id) event.session_id = sessionId;
|
||||
|
||||
tsByUuid.set(event.uuid, event.ts_utc);
|
||||
if (event.type === 'assistant' && event.parent_uuid && event.ts_utc) {
|
||||
const parentTs = tsByUuid.get(event.parent_uuid);
|
||||
if (parentTs) {
|
||||
const dt = Date.parse(event.ts_utc) - Date.parse(parentTs);
|
||||
if (Number.isFinite(dt) && dt >= 0) event.latency_ms = dt;
|
||||
}
|
||||
}
|
||||
|
||||
events.push(event);
|
||||
|
||||
if (event.message_id) {
|
||||
const key = `${event.session_id} | ||||