toknmtr — self-hostable Claude Code usage & analytics dashboard
Server (SvelteKit + SQLite Docker container) + per-machine agent that parses Claude Code JSONL transcripts. docker-compose one-command deploy; raw transcript view + search gated behind SHOW_TRANSCRIPTS (hidden by default). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
71a60ab054
74 changed files with 15613 additions and 0 deletions
414
src/routes/api/ingest/+server.ts
Normal file
414
src/routes/api/ingest/+server.ts
Normal file
|
|
@ -0,0 +1,414 @@
|
|||
import { json, error } from '@sveltejs/kit';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { db } from '$lib/server/db';
|
||||
import type { RequestHandler } from './$types';
|
||||
|
||||
/**
|
||||
* Ingest endpoint. The agent POSTs batches of parsed events here.
|
||||
* Auth: `Authorization: Bearer $API_TOKEN`.
|
||||
*
|
||||
* Accepted body shape (all top-level keys optional except `events`):
|
||||
*
|
||||
* {
|
||||
* host?: string, // fallback host for entries that omit their own `host`
|
||||
* events: IncomingEvent[], // see agent/parse.ts ParsedEvent — read defensively, coerced
|
||||
* toolCalls?: IncomingToolCall[],
|
||||
* sessions?: IncomingSession[] // if omitted, session rows are derived from event timestamps
|
||||
* }
|
||||
*
|
||||
* Everything is read off `unknown` and coerced (str/num/bool01 below) because agent/parse.ts is still
|
||||
* being finalized in parallel — this endpoint must not break if a field is missing, null, or mistyped.
|
||||
*
|
||||
* Idempotency: events upsert by (host, session_id, uuid) — re-posting the same physical JSONL line just
|
||||
* overwrites the row. tool_calls upsert by (host, session_id, tool_use_id) and *merge* (COALESCE) so a
|
||||
* tool_use line and its later tool_result line can arrive in separate batches without blanking fields.
|
||||
* content_fts is a standalone fts5 table, so re-ingesting a uuid does delete+insert to avoid duplicate rows.
|
||||
*
|
||||
* is_usage_canonical: trusted from the agent when the field is present (even if false) on at least one row
|
||||
* of a (session_id, message_id, request_id) group; otherwise recomputed server-side as a safety net — the
|
||||
* row with the max output_tokens in that group (across the whole DB, not just this batch) wins.
|
||||
*/
|
||||
|
||||
interface IncomingEvent {
|
||||
host?: unknown;
|
||||
session_id?: unknown;
|
||||
uuid?: unknown;
|
||||
parent_uuid?: unknown;
|
||||
ts_utc?: unknown;
|
||||
type?: unknown;
|
||||
role?: unknown;
|
||||
model?: unknown;
|
||||
request_id?: unknown;
|
||||
message_id?: unknown;
|
||||
is_sidechain?: unknown;
|
||||
is_usage_canonical?: unknown;
|
||||
stop_reason?: unknown;
|
||||
latency_ms?: unknown;
|
||||
input_tokens?: unknown;
|
||||
output_tokens?: unknown;
|
||||
cache_creation_tokens?: unknown;
|
||||
cache_read_tokens?: unknown;
|
||||
web_search_requests?: unknown;
|
||||
web_fetch_requests?: unknown;
|
||||
text?: unknown;
|
||||
}
|
||||
|
||||
interface IncomingToolCall {
|
||||
host?: unknown;
|
||||
session_id?: unknown;
|
||||
tool_use_id?: unknown;
|
||||
event_uuid?: unknown;
|
||||
tool_name?: unknown;
|
||||
input_json?: unknown;
|
||||
input?: unknown; // accepted as an alternative to input_json — JSON.stringify'd if input_json is absent
|
||||
is_error?: unknown;
|
||||
result_bytes?: unknown;
|
||||
duration_ms?: unknown;
|
||||
ts_utc?: unknown;
|
||||
}
|
||||
|
||||
interface IncomingSession {
|
||||
host?: unknown;
|
||||
session_id?: unknown;
|
||||
project?: unknown;
|
||||
git_branch?: unknown;
|
||||
cc_version?: unknown;
|
||||
entrypoint?: unknown;
|
||||
started_at?: unknown;
|
||||
ended_at?: unknown;
|
||||
}
|
||||
|
||||
interface IngestBody {
|
||||
host?: unknown;
|
||||
events?: unknown;
|
||||
toolCalls?: unknown;
|
||||
sessions?: unknown;
|
||||
}
|
||||
|
||||
function str(v: unknown): string | null {
|
||||
return typeof v === 'string' && v.length > 0 ? v : null;
|
||||
}
|
||||
|
||||
function num(v: unknown): number | null {
|
||||
return typeof v === 'number' && Number.isFinite(v) ? v : null;
|
||||
}
|
||||
|
||||
function bool01(v: unknown): number {
|
||||
return v ? 1 : 0;
|
||||
}
|
||||
|
||||
// Prepared statements are built lazily on first ingest so that *importing* this
|
||||
// module (e.g. SvelteKit's build-time analyse pass, which has no writable DB dir)
|
||||
// never opens the database. They are created once, then reused for the process life.
|
||||
type Stmt = import('better-sqlite3').Statement<unknown[]>;
|
||||
|
||||
let prepared = false;
|
||||
let upsertSessionStmt!: Stmt;
|
||||
let upsertEventStmt!: Stmt;
|
||||
let upsertToolCallStmt!: Stmt;
|
||||
let upsertContentStmt!: Stmt;
|
||||
let deleteFtsStmt!: Stmt;
|
||||
let insertFtsStmt!: Stmt;
|
||||
let selectUsageGroupStmt!: Stmt;
|
||||
let setCanonicalStmt!: Stmt;
|
||||
let clearCanonicalStmt!: Stmt;
|
||||
let runIngest!: (body: IngestBody) => { events: number; tool_calls: number; sessions: number };
|
||||
|
||||
function ensurePrepared() {
|
||||
if (prepared) return;
|
||||
const conn = db();
|
||||
|
||||
upsertSessionStmt = conn.prepare(`
|
||||
INSERT INTO sessions (host, session_id, project, git_branch, cc_version, entrypoint, started_at, ended_at)
|
||||
VALUES (@host, @session_id, @project, @git_branch, @cc_version, @entrypoint, @started_at, @ended_at)
|
||||
ON CONFLICT(host, session_id) DO UPDATE SET
|
||||
project = COALESCE(excluded.project, project),
|
||||
git_branch = COALESCE(excluded.git_branch, git_branch),
|
||||
cc_version = COALESCE(excluded.cc_version, cc_version),
|
||||
entrypoint = COALESCE(excluded.entrypoint, entrypoint),
|
||||
started_at = COALESCE(MIN(started_at, excluded.started_at), started_at, excluded.started_at),
|
||||
ended_at = COALESCE(MAX(ended_at, excluded.ended_at), ended_at, excluded.ended_at)
|
||||
`);
|
||||
|
||||
upsertEventStmt = conn.prepare(`
|
||||
INSERT INTO events (
|
||||
host, session_id, uuid, parent_uuid, ts_utc, type, role, model, request_id, message_id,
|
||||
is_sidechain, is_usage_canonical, stop_reason, latency_ms,
|
||||
input_tokens, output_tokens, cache_creation_tokens, cache_read_tokens,
|
||||
web_search_requests, web_fetch_requests
|
||||
) VALUES (
|
||||
@host, @session_id, @uuid, @parent_uuid, @ts_utc, @type, @role, @model, @request_id, @message_id,
|
||||
@is_sidechain, @is_usage_canonical, @stop_reason, @latency_ms,
|
||||
@input_tokens, @output_tokens, @cache_creation_tokens, @cache_read_tokens,
|
||||
@web_search_requests, @web_fetch_requests
|
||||
)
|
||||
ON CONFLICT(host, session_id, uuid) DO UPDATE SET
|
||||
parent_uuid = excluded.parent_uuid,
|
||||
ts_utc = excluded.ts_utc,
|
||||
type = excluded.type,
|
||||
role = excluded.role,
|
||||
model = excluded.model,
|
||||
request_id = excluded.request_id,
|
||||
message_id = excluded.message_id,
|
||||
is_sidechain = excluded.is_sidechain,
|
||||
is_usage_canonical = excluded.is_usage_canonical,
|
||||
stop_reason = excluded.stop_reason,
|
||||
latency_ms = excluded.latency_ms,
|
||||
input_tokens = excluded.input_tokens,
|
||||
output_tokens = excluded.output_tokens,
|
||||
cache_creation_tokens = excluded.cache_creation_tokens,
|
||||
cache_read_tokens = excluded.cache_read_tokens,
|
||||
web_search_requests = excluded.web_search_requests,
|
||||
web_fetch_requests = excluded.web_fetch_requests
|
||||
`);
|
||||
|
||||
upsertToolCallStmt = conn.prepare(`
|
||||
INSERT INTO tool_calls (
|
||||
host, session_id, tool_use_id, event_uuid, tool_name, input_json, is_error, result_bytes, duration_ms, ts_utc
|
||||
) VALUES (
|
||||
@host, @session_id, @tool_use_id, @event_uuid, @tool_name, @input_json, @is_error, @result_bytes, @duration_ms, @ts_utc
|
||||
)
|
||||
ON CONFLICT(host, session_id, tool_use_id) DO UPDATE SET
|
||||
event_uuid = COALESCE(excluded.event_uuid, event_uuid),
|
||||
tool_name = COALESCE(excluded.tool_name, tool_name),
|
||||
input_json = COALESCE(excluded.input_json, input_json),
|
||||
is_error = COALESCE(excluded.is_error, is_error),
|
||||
result_bytes = COALESCE(excluded.result_bytes, result_bytes),
|
||||
duration_ms = COALESCE(excluded.duration_ms, duration_ms),
|
||||
ts_utc = COALESCE(excluded.ts_utc, ts_utc)
|
||||
`);
|
||||
|
||||
upsertContentStmt = conn.prepare(`
|
||||
INSERT INTO content (host, session_id, uuid, role, text)
|
||||
VALUES (@host, @session_id, @uuid, @role, @text)
|
||||
ON CONFLICT(host, session_id, uuid) DO UPDATE SET
|
||||
role = excluded.role,
|
||||
text = excluded.text
|
||||
`);
|
||||
deleteFtsStmt = conn.prepare(
|
||||
`DELETE FROM content_fts WHERE host = ? AND session_id = ? AND uuid = ?`
|
||||
);
|
||||
insertFtsStmt = conn.prepare(
|
||||
`INSERT INTO content_fts (text, host, session_id, uuid) VALUES (?, ?, ?, ?)`
|
||||
);
|
||||
|
||||
selectUsageGroupStmt = conn.prepare(`
|
||||
SELECT uuid, output_tokens FROM events
|
||||
WHERE host = ? AND session_id = ? AND message_id = ? AND request_id = ? AND output_tokens IS NOT NULL
|
||||
`);
|
||||
setCanonicalStmt = conn.prepare(
|
||||
`UPDATE events SET is_usage_canonical = 1 WHERE host = ? AND session_id = ? AND uuid = ?`
|
||||
);
|
||||
clearCanonicalStmt = conn.prepare(`
|
||||
UPDATE events SET is_usage_canonical = 0
|
||||
WHERE host = ? AND session_id = ? AND message_id = ? AND request_id = ? AND uuid != ?
|
||||
`);
|
||||
|
||||
runIngest = conn.transaction(ingestBatch);
|
||||
prepared = true;
|
||||
}
|
||||
|
||||
interface SessionBounds {
|
||||
host: string;
|
||||
session_id: string;
|
||||
started_at: string | null;
|
||||
ended_at: string | null;
|
||||
}
|
||||
|
||||
interface UsageGroupKey {
|
||||
host: string;
|
||||
session_id: string;
|
||||
message_id: string;
|
||||
request_id: string;
|
||||
}
|
||||
|
||||
function groupKey(g: UsageGroupKey): string {
|
||||
return `${g.host}${g.session_id}${g.message_id}${g.request_id}`;
|
||||
}
|
||||
|
||||
function ingestBatch(body: IngestBody) {
|
||||
const fallbackHost = str(body.host) ?? '';
|
||||
const rawEvents = Array.isArray(body.events) ? (body.events as IncomingEvent[]) : [];
|
||||
const rawToolCalls = Array.isArray(body.toolCalls) ? (body.toolCalls as IncomingToolCall[]) : [];
|
||||
const rawSessions = Array.isArray(body.sessions) ? (body.sessions as IncomingSession[]) : [];
|
||||
|
||||
let eventCount = 0;
|
||||
let toolCallCount = 0;
|
||||
let sessionCount = 0;
|
||||
|
||||
// groups the agent explicitly flagged (is_usage_canonical present on >=1 row) — skip recompute for these
|
||||
const agentSetGroups = new Set<string>();
|
||||
// groups touched by this batch that have usage but no explicit flag — candidates for recompute
|
||||
const needsRecompute = new Map<string, UsageGroupKey>();
|
||||
// derived session bounds (host|session_id -> min/max ts_utc), used when `sessions` wasn't supplied
|
||||
const bounds = new Map<string, SessionBounds>();
|
||||
|
||||
for (const e of rawEvents) {
|
||||
const host = str(e.host) ?? fallbackHost;
|
||||
const session_id = str(e.session_id);
|
||||
const uuid = str(e.uuid);
|
||||
if (!host || !session_id || !uuid) continue; // incomplete PK, can't store
|
||||
|
||||
const ts_utc = str(e.ts_utc) ?? '';
|
||||
const role = str(e.role);
|
||||
const message_id = str(e.message_id);
|
||||
const request_id = str(e.request_id);
|
||||
const hasExplicitCanonical =
|
||||
e.is_usage_canonical !== undefined && e.is_usage_canonical !== null;
|
||||
const is_usage_canonical = hasExplicitCanonical ? bool01(e.is_usage_canonical) : 0;
|
||||
const output_tokens = num(e.output_tokens);
|
||||
|
||||
upsertEventStmt.run({
|
||||
host,
|
||||
session_id,
|
||||
uuid,
|
||||
parent_uuid: str(e.parent_uuid),
|
||||
ts_utc,
|
||||
type: str(e.type) ?? 'unknown',
|
||||
role,
|
||||
model: str(e.model),
|
||||
request_id,
|
||||
message_id,
|
||||
is_sidechain: bool01(e.is_sidechain),
|
||||
is_usage_canonical,
|
||||
stop_reason: str(e.stop_reason),
|
||||
latency_ms: num(e.latency_ms),
|
||||
input_tokens: num(e.input_tokens),
|
||||
output_tokens,
|
||||
cache_creation_tokens: num(e.cache_creation_tokens),
|
||||
cache_read_tokens: num(e.cache_read_tokens),
|
||||
web_search_requests: num(e.web_search_requests),
|
||||
web_fetch_requests: num(e.web_fetch_requests)
|
||||
});
|
||||
eventCount++;
|
||||
|
||||
if (message_id && request_id) {
|
||||
const key = groupKey({ host, session_id, message_id, request_id });
|
||||
if (hasExplicitCanonical) {
|
||||
agentSetGroups.add(key);
|
||||
} else if (output_tokens !== null) {
|
||||
needsRecompute.set(key, { host, session_id, message_id, request_id });
|
||||
}
|
||||
}
|
||||
|
||||
const text = str(e.text);
|
||||
if (text && (role === 'user' || role === 'assistant')) {
|
||||
upsertContentStmt.run({ host, session_id, uuid, role, text });
|
||||
// standalone fts5 table — manual delete+insert keeps re-ingest of the same uuid from duplicating rows
|
||||
deleteFtsStmt.run(host, session_id, uuid);
|
||||
insertFtsStmt.run(text, host, session_id, uuid);
|
||||
}
|
||||
|
||||
if (ts_utc) {
|
||||
const bkey = `${host}${session_id}`;
|
||||
const existing = bounds.get(bkey);
|
||||
if (!existing) {
|
||||
bounds.set(bkey, { host, session_id, started_at: ts_utc, ended_at: ts_utc });
|
||||
} else {
|
||||
if (existing.started_at === null || ts_utc < existing.started_at)
|
||||
existing.started_at = ts_utc;
|
||||
if (existing.ended_at === null || ts_utc > existing.ended_at) existing.ended_at = ts_utc;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const t of rawToolCalls) {
|
||||
const host = str(t.host) ?? fallbackHost;
|
||||
const session_id = str(t.session_id);
|
||||
const tool_use_id = str(t.tool_use_id);
|
||||
if (!host || !session_id || !tool_use_id) continue;
|
||||
|
||||
let input_json = str(t.input_json);
|
||||
if (input_json === null && t.input !== undefined && t.input !== null) {
|
||||
try {
|
||||
input_json = JSON.stringify(t.input);
|
||||
} catch {
|
||||
input_json = null;
|
||||
}
|
||||
}
|
||||
|
||||
upsertToolCallStmt.run({
|
||||
host,
|
||||
session_id,
|
||||
tool_use_id,
|
||||
event_uuid: str(t.event_uuid),
|
||||
tool_name: str(t.tool_name) ?? 'unknown',
|
||||
input_json,
|
||||
is_error: t.is_error === undefined || t.is_error === null ? null : bool01(t.is_error),
|
||||
result_bytes: num(t.result_bytes),
|
||||
duration_ms: num(t.duration_ms),
|
||||
ts_utc: str(t.ts_utc)
|
||||
});
|
||||
toolCallCount++;
|
||||
}
|
||||
|
||||
for (const s of rawSessions) {
|
||||
const host = str(s.host) ?? fallbackHost;
|
||||
const session_id = str(s.session_id);
|
||||
if (!host || !session_id) continue;
|
||||
|
||||
upsertSessionStmt.run({
|
||||
host,
|
||||
session_id,
|
||||
project: str(s.project),
|
||||
git_branch: str(s.git_branch),
|
||||
cc_version: str(s.cc_version),
|
||||
entrypoint: str(s.entrypoint),
|
||||
started_at: str(s.started_at),
|
||||
ended_at: str(s.ended_at)
|
||||
});
|
||||
sessionCount++;
|
||||
bounds.delete(`${host}${session_id}`); // an explicit session row wins over derived bounds
|
||||
}
|
||||
|
||||
// derive session rows (start/end bounds only) for any session not explicitly supplied above
|
||||
for (const b of bounds.values()) {
|
||||
upsertSessionStmt.run({
|
||||
host: b.host,
|
||||
session_id: b.session_id,
|
||||
project: null,
|
||||
git_branch: null,
|
||||
cc_version: null,
|
||||
entrypoint: null,
|
||||
started_at: b.started_at,
|
||||
ended_at: b.ended_at
|
||||
});
|
||||
sessionCount++;
|
||||
}
|
||||
|
||||
// safety net: recompute is_usage_canonical for groups this batch touched but the agent didn't flag
|
||||
for (const [key, g] of needsRecompute) {
|
||||
if (agentSetGroups.has(key)) continue; // agent already flagged this group — trust it
|
||||
const rows = selectUsageGroupStmt.all(
|
||||
g.host,
|
||||
g.session_id,
|
||||
g.message_id,
|
||||
g.request_id
|
||||
) as Array<{
|
||||
uuid: string;
|
||||
output_tokens: number;
|
||||
}>;
|
||||
if (rows.length === 0) continue;
|
||||
let best = rows[0];
|
||||
for (const r of rows) if (r.output_tokens > best.output_tokens) best = r;
|
||||
setCanonicalStmt.run(g.host, g.session_id, best.uuid);
|
||||
clearCanonicalStmt.run(g.host, g.session_id, g.message_id, g.request_id, best.uuid);
|
||||
}
|
||||
|
||||
return { events: eventCount, tool_calls: toolCallCount, sessions: sessionCount };
|
||||
}
|
||||
|
||||
export const POST: RequestHandler = async ({ request }) => {
|
||||
const token = env.API_TOKEN;
|
||||
if (!token) throw error(500, 'server missing API_TOKEN');
|
||||
if (request.headers.get('authorization') !== `Bearer ${token}`) throw error(401, 'unauthorized');
|
||||
|
||||
const body = (await request.json().catch(() => null)) as IngestBody | null;
|
||||
if (!body || typeof body !== 'object') throw error(400, 'invalid JSON body');
|
||||
|
||||
ensurePrepared();
|
||||
const result = runIngest(body);
|
||||
return json({ ok: true, ...result });
|
||||
};
|
||||
|
||||
export const GET: RequestHandler = async () => json({ ok: true, service: 'toknmtr ingest' });
|
||||
110
src/routes/api/search/+server.ts
Normal file
110
src/routes/api/search/+server.ts
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
import { json, error } from '@sveltejs/kit';
|
||||
import { db } from '$lib/server/db';
|
||||
import { showTranscripts } from '$lib/server/config';
|
||||
import type { RequestHandler } from './$types';
|
||||
|
||||
/**
|
||||
* Full-text search over the session archive (`content_fts`, fts5).
|
||||
* GET ?q=<query>&limit=<n>
|
||||
*
|
||||
* Read-only, LAN-only, no auth (matches /api/stats). Empty/whitespace-only `q` short-circuits
|
||||
* to an empty result set instead of hitting SQLite (an unqualified `MATCH ''` is a syntax error).
|
||||
*
|
||||
* The `q` string is never spliced into the MATCH expression as-is: each whitespace-separated term
|
||||
* is individually double-quoted (with embedded `"` doubled, fts5's own escape) so user input can't
|
||||
* inject fts5 query syntax (`OR`, `NOT`, `NEAR`, column filters, dangling `*`, etc). Multiple quoted
|
||||
* terms are matched with fts5's default implicit AND.
|
||||
*
|
||||
* Snippets come from `snippet()`, wrapped in control-character markers (not HTML) so the client can
|
||||
* split + highlight without ever needing `{@html}` on raw transcript content.
|
||||
*/
|
||||
|
||||
// Control characters (never appear in normal transcript text) used to delimit highlighted spans in
|
||||
// the returned snippet, so the client can split on them without ever rendering raw HTML.
|
||||
const SNIPPET_MARK_START = '';
|
||||
const SNIPPET_MARK_END = '';
|
||||
const SNIPPET_TOKENS = 12;
|
||||
|
||||
/** Turn free-text user input into a safe, quoted fts5 MATCH expression, or null if there's nothing to search. */
|
||||
function buildFtsQuery(raw: string): string | null {
|
||||
const terms = raw
|
||||
.trim()
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.map((term) => `"${term.replace(/"/g, '""')}"`);
|
||||
return terms.length > 0 ? terms.join(' ') : null;
|
||||
}
|
||||
|
||||
interface SearchRow {
|
||||
host: string;
|
||||
session_id: string;
|
||||
uuid: string;
|
||||
snippet: string;
|
||||
role: string | null;
|
||||
type: string | null;
|
||||
ts_utc: string | null;
|
||||
project: string | null;
|
||||
}
|
||||
|
||||
export interface SearchResult {
|
||||
host: string;
|
||||
sessionId: string;
|
||||
uuid: string;
|
||||
snippet: string;
|
||||
role: string | null;
|
||||
type: string | null;
|
||||
tsUtc: string | null;
|
||||
project: string | null;
|
||||
}
|
||||
|
||||
export const GET: RequestHandler = async ({ url }) => {
|
||||
// Returns verbatim transcript snippets — refuse while conversation content is hidden.
|
||||
if (!showTranscripts) {
|
||||
throw error(403, 'Search is disabled');
|
||||
}
|
||||
|
||||
const q = url.searchParams.get('q') ?? '';
|
||||
const limitParam = Number(url.searchParams.get('limit'));
|
||||
const limit =
|
||||
Number.isFinite(limitParam) && limitParam > 0 ? Math.min(200, Math.floor(limitParam)) : 30;
|
||||
|
||||
const ftsQuery = buildFtsQuery(q);
|
||||
if (!ftsQuery) {
|
||||
return json({ query: q, count: 0, results: [] satisfies SearchResult[] });
|
||||
}
|
||||
|
||||
const dbh = db();
|
||||
let rows: SearchRow[];
|
||||
try {
|
||||
rows = dbh
|
||||
.prepare(
|
||||
`SELECT f.host AS host, f.session_id AS session_id, f.uuid AS uuid,
|
||||
snippet(content_fts, 0, ?, ?, ' … ', ${SNIPPET_TOKENS}) AS snippet,
|
||||
e.role AS role, e.type AS type, e.ts_utc AS ts_utc, s.project AS project
|
||||
FROM content_fts f
|
||||
JOIN events e ON e.host = f.host AND e.session_id = f.session_id AND e.uuid = f.uuid
|
||||
LEFT JOIN sessions s ON s.host = f.host AND s.session_id = f.session_id
|
||||
WHERE content_fts MATCH ?
|
||||
ORDER BY rank
|
||||
LIMIT ?`
|
||||
)
|
||||
.all(SNIPPET_MARK_START, SNIPPET_MARK_END, ftsQuery, limit) as SearchRow[];
|
||||
} catch {
|
||||
// Malformed MATCH expression (shouldn't happen given the quoting above, but fts5 can still
|
||||
// reject pathological input like a lone `""`) — treat as "no results" rather than a 500.
|
||||
return json({ query: q, count: 0, results: [] satisfies SearchResult[] });
|
||||
}
|
||||
|
||||
const results: SearchResult[] = rows.map((r) => ({
|
||||
host: r.host,
|
||||
sessionId: r.session_id,
|
||||
uuid: r.uuid,
|
||||
snippet: r.snippet,
|
||||
role: r.role,
|
||||
type: r.type,
|
||||
tsUtc: r.ts_utc,
|
||||
project: r.project
|
||||
}));
|
||||
|
||||
return json({ query: q, count: results.length, results });
|
||||
};
|
||||
40
src/routes/api/stats/+server.ts
Normal file
40
src/routes/api/stats/+server.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import { json } from '@sveltejs/kit';
|
||||
import {
|
||||
overviewStats,
|
||||
usageSeries,
|
||||
usageByModel,
|
||||
topTools,
|
||||
recentSessions,
|
||||
hourOfDayActivity,
|
||||
cacheEfficiency,
|
||||
usageGauges,
|
||||
type TimeWindow
|
||||
} from '$lib/server/queries';
|
||||
import type { RequestHandler } from './$types';
|
||||
|
||||
/**
|
||||
* Read-only stats bundle. LAN-only, no auth.
|
||||
* Query params: `?days=30` — window size in days (default 30). Series buckets by
|
||||
* hour when days <= 2, else by day.
|
||||
*/
|
||||
export const GET: RequestHandler = async ({ url }) => {
|
||||
const daysParam = Number(url.searchParams.get('days'));
|
||||
const days = Number.isFinite(daysParam) && daysParam > 0 ? Math.floor(daysParam) : 30;
|
||||
|
||||
const untilIso = new Date().toISOString();
|
||||
const sinceIso = new Date(Date.now() - days * 86_400_000).toISOString();
|
||||
const window: TimeWindow = { since: sinceIso, until: untilIso };
|
||||
const bucket = days <= 2 ? 'hour' : 'day';
|
||||
|
||||
return json({
|
||||
days,
|
||||
overview: overviewStats(window),
|
||||
series: usageSeries(sinceIso, untilIso, bucket),
|
||||
byModel: usageByModel(window),
|
||||
hourly: hourOfDayActivity(window),
|
||||
cache: cacheEfficiency(window),
|
||||
topTools: topTools(10, window),
|
||||
recentSessions: recentSessions(20, window),
|
||||
gauges: usageGauges()
|
||||
});
|
||||
};
|
||||
94
src/routes/api/usage/+server.ts
Normal file
94
src/routes/api/usage/+server.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
import { json, error } from '@sveltejs/kit';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { db } from '$lib/server/db';
|
||||
import type { RequestHandler } from './$types';
|
||||
|
||||
/**
|
||||
* Usage-gauge ingest. The agent PTY-drives `claude /usage` (see agent/usage.ts) and POSTs the
|
||||
* three subscription bars here. Separate from /api/ingest because the shape and cadence differ
|
||||
* (a slow, ~5-9s scrape done once per run, not per JSONL sweep).
|
||||
*
|
||||
* Auth: `Authorization: Bearer $API_TOKEN` (same token as /api/ingest).
|
||||
*
|
||||
* Accepted body — either a single gauge object or a batch:
|
||||
* { host, ts_utc, session_pct?, week_all_pct?, week_sonnet_pct? }
|
||||
* { host?, gauges: GaugeRow[] } // host is a fallback for rows that omit their own
|
||||
*
|
||||
* Rows upsert by (host, ts_utc) — re-posting the same scrape is idempotent.
|
||||
*/
|
||||
|
||||
interface IncomingGauge {
|
||||
host?: unknown;
|
||||
ts_utc?: unknown;
|
||||
session_pct?: unknown;
|
||||
week_all_pct?: unknown;
|
||||
week_sonnet_pct?: unknown;
|
||||
}
|
||||
|
||||
interface UsageBody extends IncomingGauge {
|
||||
gauges?: unknown;
|
||||
}
|
||||
|
||||
function str(v: unknown): string | null {
|
||||
return typeof v === 'string' && v.length > 0 ? v : null;
|
||||
}
|
||||
|
||||
function num(v: unknown): number | null {
|
||||
return typeof v === 'number' && Number.isFinite(v) ? v : null;
|
||||
}
|
||||
|
||||
// Lazily prepared on first request so importing this module (e.g. the build's
|
||||
// analyse pass, which has no writable DB dir) never opens the database.
|
||||
type Stmt = import('better-sqlite3').Statement<unknown[]>;
|
||||
|
||||
let prepared = false;
|
||||
let upsertGaugeStmt!: Stmt;
|
||||
let insertMany!: (rows: IncomingGauge[], fallbackHost: string | null) => number;
|
||||
|
||||
function ensurePrepared() {
|
||||
if (prepared) return;
|
||||
const conn = db();
|
||||
upsertGaugeStmt = conn.prepare(`
|
||||
INSERT INTO usage_gauges (host, ts_utc, session_pct, week_all_pct, week_sonnet_pct)
|
||||
VALUES (@host, @ts_utc, @session_pct, @week_all_pct, @week_sonnet_pct)
|
||||
ON CONFLICT(host, ts_utc) DO UPDATE SET
|
||||
session_pct = COALESCE(excluded.session_pct, session_pct),
|
||||
week_all_pct = COALESCE(excluded.week_all_pct, week_all_pct),
|
||||
week_sonnet_pct = COALESCE(excluded.week_sonnet_pct, week_sonnet_pct)
|
||||
`);
|
||||
|
||||
insertMany = conn.transaction((rows: IncomingGauge[], fallbackHost: string | null) => {
|
||||
let count = 0;
|
||||
for (const g of rows) {
|
||||
const host = str(g.host) ?? fallbackHost;
|
||||
const ts_utc = str(g.ts_utc);
|
||||
if (!host || !ts_utc) continue; // incomplete PK — skip
|
||||
upsertGaugeStmt.run({
|
||||
host,
|
||||
ts_utc,
|
||||
session_pct: num(g.session_pct),
|
||||
week_all_pct: num(g.week_all_pct),
|
||||
week_sonnet_pct: num(g.week_sonnet_pct)
|
||||
});
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
});
|
||||
prepared = true;
|
||||
}
|
||||
|
||||
export const POST: RequestHandler = async ({ request }) => {
|
||||
const token = env.API_TOKEN;
|
||||
if (!token) throw error(500, 'server missing API_TOKEN');
|
||||
if (request.headers.get('authorization') !== `Bearer ${token}`) throw error(401, 'unauthorized');
|
||||
|
||||
const body = (await request.json().catch(() => null)) as UsageBody | null;
|
||||
if (!body || typeof body !== 'object') throw error(400, 'invalid JSON body');
|
||||
|
||||
ensurePrepared();
|
||||
const rows = Array.isArray(body.gauges) ? (body.gauges as IncomingGauge[]) : [body];
|
||||
const count = insertMany(rows, str(body.host));
|
||||
return json({ ok: true, gauges: count });
|
||||
};
|
||||
|
||||
export const GET: RequestHandler = async () => json({ ok: true, service: 'toknmtr usage' });
|
||||
Loading…
Add table
Add a link
Reference in a new issue