/** * toknmtr agent — pusher. * * POSTs parsed batches to the server's /api/ingest. The ingest endpoint * (src/routes/api/ingest/+server.ts) accepts `{ host?, events, toolCalls?, sessions? }` — * events are the documented minimal contract; toolCalls/sessions are additional top-level * keys the ingest agent reads to populate the tool_calls/sessions tables. Idempotent * upsert on the server means re-pushing (e.g. retrying after a failed request) is always * safe. Config via env: TOKNMTR_URL, TOKNMTR_TOKEN. */ import type { ParsedEvent, SessionMeta, ToolCall } from './parse.ts'; import type { UsageGauges } from './usage.ts'; const URL_BASE = process.env.TOKNMTR_URL ?? 'http://localhost:3001'; const TOKEN = process.env.TOKNMTR_TOKEN ?? ''; /** Max events per ingest request — keeps request bodies (and the server's single * transaction per request) a reasonable size for large backfills. Events carry raw * prompt/response text, so a batch of 500 can run into many MB; 200 keeps each POST * comfortably under the server's BODY_SIZE_LIMIT (see ops/README.md — set it generously, * e.g. BODY_SIZE_LIMIT=64M, on the deployed container). */ const CHUNK_SIZE = 200; export interface PushBatch { host?: string; events: ParsedEvent[]; toolCalls?: ToolCall[]; sessions?: SessionMeta[]; } export interface PushResult { ok: boolean; received: number; requests: number; } interface IngestResponse { ok: boolean; events?: number; tool_calls?: number; sessions?: number; received?: number; // older/stub server shape } async function postChunk(body: Record): Promise { const res = await fetch(`${URL_BASE}/api/ingest`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${TOKEN}` }, body: JSON.stringify(body) }); if (!res.ok) throw new Error(`ingest failed: ${res.status} ${await res.text()}`); const parsed = (await res.json()) as IngestResponse; return parsed.events ?? parsed.received ?? 0; } /** * Push a full batch (events + tool calls + session metadata), chunking `events` into * groups of CHUNK_SIZE per request. toolCalls/sessions (typically much smaller) are sent * once, attached to the first request. */ export async function pushBatch(batch: PushBatch): Promise { const { host, events, toolCalls = [], sessions = [] } = batch; if (events.length === 0 && toolCalls.length === 0 && sessions.length === 0) { return { ok: true, received: 0, requests: 0 }; } let received = 0; let requests = 0; const chunkCount = events.length > 0 ? Math.ceil(events.length / CHUNK_SIZE) : 1; for (let i = 0; i < chunkCount; i++) { const chunk = events.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE); const body: Record = { events: chunk }; if (host) body.host = host; if (i === 0) { if (toolCalls.length > 0) body.toolCalls = toolCalls; if (sessions.length > 0) body.sessions = sessions; } received += await postChunk(body); requests++; } return { ok: true, received, requests }; } /** Convenience wrapper for pushing just events (no tool calls / session metadata). */ export async function pushEvents( events: ParsedEvent[] ): Promise<{ ok: boolean; received: number }> { const result = await pushBatch({ events }); return { ok: result.ok, received: result.received }; } /** * POST a single subscription-usage gauge reading to the server's /api/usage endpoint. * Returns true on a 2xx. Caller decides whether/when to scrape (see agent/usage.ts) — this * just ships whatever it's handed. Idempotent on the server (upsert by host + ts_utc). */ export async function pushUsageGauges(gauges: UsageGauges): Promise { const res = await fetch(`${URL_BASE}/api/usage`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${TOKEN}` }, body: JSON.stringify(gauges) }); if (!res.ok) throw new Error(`usage push failed: ${res.status} ${await res.text()}`); return true; }