Add AI receipt parsing with provider API key settings

- Settings → AI: configure Anthropic or OpenAI provider with encrypted API key
- Sparkle button on each attachment in transaction drawer sends image/PDF to AI
- AI extracts merchant, amount, date, description, category hint
- "Apply to transaction" button patches the transaction with parsed fields
- Anthropic supports images and PDFs; OpenAI supports images only
- API key stored AES-256-GCM encrypted in users table (migration 0003)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
megaproxy 2026-04-22 17:07:24 +00:00
parent fe4e69b9ad
commit 22fc1ce2f1
8 changed files with 507 additions and 31 deletions

View file

@ -0,0 +1,23 @@
"""add ai_provider and ai_api_key_enc to users
Revision ID: 0003
Revises: 0002
Create Date: 2026-04-22
"""
from alembic import op
import sqlalchemy as sa
revision = "0003"
down_revision = "0002"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("users", sa.Column("ai_provider", sa.Text, nullable=True))
op.add_column("users", sa.Column("ai_api_key_enc", sa.LargeBinary, nullable=True))
def downgrade() -> None:
op.drop_column("users", "ai_api_key_enc")
op.drop_column("users", "ai_provider")

View file

@ -1,6 +1,6 @@
from fastapi import APIRouter
from app.api.v1 import auth, users, accounts, categories, transactions, budgets, reports, investments, predictions, admin
from app.api.v1 import auth, users, accounts, categories, transactions, budgets, reports, investments, predictions, admin, settings
router = APIRouter()
router.include_router(auth.router, prefix="/auth", tags=["auth"])
@ -13,3 +13,4 @@ router.include_router(reports.router)
router.include_router(investments.router)
router.include_router(predictions.router)
router.include_router(admin.router)
router.include_router(settings.router)

View file

@ -0,0 +1,69 @@
"""
User-level settings: AI provider configuration.
"""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
from sqlalchemy import update
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.security import decrypt_field, encrypt_field
from app.dependencies import get_current_user, get_db
from app.db.models.user import User
router = APIRouter(prefix="/settings", tags=["settings"])
SUPPORTED_PROVIDERS = {"anthropic", "openai"}
class AiSettingsResponse(BaseModel):
provider: str | None
has_api_key: bool
class AiSettingsSave(BaseModel):
provider: str
api_key: str
@router.get("/ai", response_model=AiSettingsResponse)
async def get_ai_settings(user: User = Depends(get_current_user)):
return AiSettingsResponse(
provider=user.ai_provider,
has_api_key=bool(user.ai_api_key_enc),
)
@router.put("/ai", response_model=AiSettingsResponse)
async def save_ai_settings(
body: AiSettingsSave,
db: AsyncSession = Depends(get_db),
user: User = Depends(get_current_user),
):
if body.provider not in SUPPORTED_PROVIDERS:
raise HTTPException(status_code=400, detail=f"Unsupported provider. Choose: {', '.join(SUPPORTED_PROVIDERS)}")
if not body.api_key.strip():
raise HTTPException(status_code=400, detail="api_key must not be empty")
encrypted = encrypt_field(body.api_key.strip())
await db.execute(
update(User)
.where(User.id == user.id)
.values(ai_provider=body.provider, ai_api_key_enc=encrypted)
)
await db.commit()
return AiSettingsResponse(provider=body.provider, has_api_key=True)
@router.delete("/ai", status_code=204)
async def clear_ai_settings(
db: AsyncSession = Depends(get_db),
user: User = Depends(get_current_user),
):
await db.execute(
update(User)
.where(User.id == user.id)
.values(ai_provider=None, ai_api_key_enc=None)
)
await db.commit()

View file

@ -278,6 +278,140 @@ async def delete_attachment(
await db.commit()
@router.post("/{txn_id}/attachments/{attachment_id}/parse")
async def parse_attachment(
txn_id: uuid.UUID,
attachment_id: str,
db: AsyncSession = Depends(get_db),
user=Depends(get_current_user),
):
"""Send an attachment to the configured AI provider and extract receipt fields."""
import base64
import json
import httpx
from sqlalchemy import select
from app.db.models.transaction import Transaction as TxnModel
from app.db.models.user import User as UserModel
from app.core.security import decrypt_field
settings = get_settings()
# Reload user to get ai fields (get_current_user may be cached without them)
user_row = await db.get(UserModel, user.id)
if not user_row or not user_row.ai_provider or not user_row.ai_api_key_enc:
raise HTTPException(status_code=400, detail="No AI provider configured. Add your API key in Settings → AI.")
api_key = decrypt_field(user_row.ai_api_key_enc)
result = await db.execute(
select(TxnModel).where(TxnModel.id == txn_id, TxnModel.user_id == user.id)
)
txn_row = result.scalar_one_or_none()
if not txn_row:
raise HTTPException(status_code=404, detail="Transaction not found")
ref = next((r for r in (txn_row.attachment_refs or []) if r["id"] == attachment_id), None)
if not ref:
raise HTTPException(status_code=404, detail="Attachment not found")
mime_type = ref["mime_type"]
path = Path(settings.upload_dir) / str(user.id) / ref["stored_name"]
if not path.exists():
raise HTTPException(status_code=404, detail="Attachment file missing")
file_bytes = path.read_bytes()
b64 = base64.standard_b64encode(file_bytes).decode()
prompt = (
"You are a receipt parser. Extract information from this receipt and return ONLY a JSON object "
"with exactly these keys (use null for any field you cannot determine):\n"
'{"merchant": "store name", "amount": 0.00, "currency": "GBP", '
'"date": "YYYY-MM-DD", "description": "brief description", '
'"category": "one of: Food & Drink, Transport, Shopping, Entertainment, Health, Travel, Bills & Utilities, Other"}\n'
"Return ONLY the JSON object. No markdown, no explanation, no code fences."
)
try:
if user_row.ai_provider == "anthropic":
if mime_type == "application/pdf":
content_block = {
"type": "document",
"source": {"type": "base64", "media_type": "application/pdf", "data": b64},
}
else:
content_block = {
"type": "image",
"source": {"type": "base64", "media_type": mime_type, "data": b64},
}
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.post(
"https://api.anthropic.com/v1/messages",
headers={
"x-api-key": api_key,
"anthropic-version": "2023-06-01",
"content-type": "application/json",
},
json={
"model": "claude-haiku-4-5-20251001",
"max_tokens": 512,
"messages": [{"role": "user", "content": [content_block, {"type": "text", "text": prompt}]}],
},
)
resp.raise_for_status()
text = resp.json()["content"][0]["text"].strip()
elif user_row.ai_provider == "openai":
if mime_type == "application/pdf":
raise HTTPException(status_code=400, detail="PDF parsing is not supported with the OpenAI provider. Use an image format or switch to Anthropic.")
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.post(
"https://api.openai.com/v1/chat/completions",
headers={"Authorization": f"Bearer {api_key}", "content-type": "application/json"},
json={
"model": "gpt-4o-mini",
"max_tokens": 512,
"messages": [{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{b64}"}},
{"type": "text", "text": prompt},
],
}],
},
)
resp.raise_for_status()
text = resp.json()["choices"][0]["message"]["content"].strip()
else:
raise HTTPException(status_code=400, detail="Unknown provider")
except httpx.HTTPStatusError as e:
raise HTTPException(status_code=502, detail=f"AI provider error: {e.response.status_code}")
except httpx.RequestError:
raise HTTPException(status_code=502, detail="Could not reach AI provider")
# Strip markdown fences if model wrapped the JSON
if text.startswith("```"):
text = text.split("```")[1]
if text.startswith("json"):
text = text[4:]
text = text.strip()
try:
parsed = json.loads(text)
except json.JSONDecodeError:
raise HTTPException(status_code=502, detail="AI returned an unexpected response. Try again.")
return {
"merchant": parsed.get("merchant"),
"amount": parsed.get("amount"),
"currency": parsed.get("currency"),
"date": parsed.get("date"),
"description": parsed.get("description"),
"category": parsed.get("category"),
}
@router.post("/import")
async def import_transactions(
file: UploadFile = File(...),

View file

@ -1,7 +1,7 @@
import uuid
from datetime import datetime
from sqlalchemy import Boolean, DateTime, Integer, String, Text
from sqlalchemy import Boolean, DateTime, Integer, LargeBinary, String, Text
from sqlalchemy.dialects.postgresql import INET, UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
@ -29,5 +29,8 @@ class User(Base):
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
ai_provider: Mapped[str | None] = mapped_column(Text, nullable=True)
ai_api_key_enc: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True)
accounts: Mapped[list["Account"]] = relationship(back_populates="user", lazy="noload") # type: ignore[name-defined]
sessions: Mapped[list["Session"]] = relationship(back_populates="user", lazy="noload") # type: ignore[name-defined]

View file

@ -0,0 +1,34 @@
import { api } from "./client";
export interface AiSettings {
provider: string | null;
has_api_key: boolean;
}
export interface ParsedReceipt {
merchant: string | null;
amount: number | null;
currency: string | null;
date: string | null;
description: string | null;
category: string | null;
}
export async function getAiSettings(): Promise<AiSettings> {
const { data } = await api.get("/settings/ai");
return data;
}
export async function saveAiSettings(provider: string, api_key: string): Promise<AiSettings> {
const { data } = await api.put("/settings/ai", { provider, api_key });
return data;
}
export async function clearAiSettings(): Promise<void> {
await api.delete("/settings/ai");
}
export async function parseReceipt(txnId: string, attachmentId: string): Promise<ParsedReceipt> {
const { data } = await api.post(`/transactions/${txnId}/attachments/${attachmentId}/parse`);
return data;
}

View file

@ -8,13 +8,14 @@ import {
changePassword, updateProfile, exportData, getMe,
} from "@/api/auth";
import { listBackups, triggerBackup, downloadBackup, restoreBackup } from "@/api/admin";
import { getAiSettings, saveAiSettings, clearAiSettings } from "@/api/settings";
import type { BackupFile } from "@/api/admin";
import { cn } from "@/utils/cn";
import { format } from "date-fns";
import {
User, Shield, MonitorSmartphone, Download, HardDrive,
Loader2, CheckCircle, Eye, EyeOff, Trash2,
LogOut, QrCode, KeyRound, AlertTriangle, RefreshCw, RotateCcw,
LogOut, QrCode, KeyRound, AlertTriangle, RefreshCw, RotateCcw, Sparkles,
} from "lucide-react";
const SECTIONS = [
@ -23,6 +24,7 @@ const SECTIONS = [
{ id: "sessions", label: "Sessions", icon: MonitorSmartphone },
{ id: "data", label: "Data", icon: Download },
{ id: "backups", label: "Backups", icon: HardDrive },
{ id: "ai", label: "AI", icon: Sparkles },
] as const;
type Section = (typeof SECTIONS)[number]["id"];
@ -64,6 +66,7 @@ export default function SettingsPage() {
{section === "sessions" && <SessionsSection />}
{section === "data" && <DataSection />}
{section === "backups" && <BackupsSection />}
{section === "ai" && <AiSection />}
</div>
</div>
</div>
@ -648,6 +651,130 @@ function BackupsSection() {
);
}
// ─── AI ───────────────────────────────────────────────────────────────────────
function AiSection() {
const qc = useQueryClient();
const [provider, setProvider] = useState("anthropic");
const [apiKey, setApiKey] = useState("");
const [showKey, setShowKey] = useState(false);
const [success, setSuccess] = useState("");
const { data: current, isLoading } = useQuery({
queryKey: ["ai-settings"],
queryFn: getAiSettings,
onSuccess: (d: any) => { if (d.provider) setProvider(d.provider); },
} as any);
const saveMutation = useMutation({
mutationFn: () => saveAiSettings(provider, apiKey),
onSuccess: () => {
qc.invalidateQueries({ queryKey: ["ai-settings"] });
setApiKey("");
setSuccess("API key saved");
setTimeout(() => setSuccess(""), 3000);
},
});
const clearMutation = useMutation({
mutationFn: clearAiSettings,
onSuccess: () => {
qc.invalidateQueries({ queryKey: ["ai-settings"] });
setSuccess("API key removed");
setTimeout(() => setSuccess(""), 3000);
},
});
return (
<div className={cardCls}>
<div className="flex items-center gap-2">
<Sparkles className="w-4 h-4 text-muted-foreground" />
<SectionTitle>AI Receipt Parsing</SectionTitle>
{!isLoading && current?.has_api_key && (
<span className="ml-auto text-xs px-2 py-0.5 rounded-full font-medium bg-success/15 text-success">
Configured
</span>
)}
</div>
<p className="text-sm text-muted-foreground">
Upload a receipt photo to any transaction and use AI to automatically extract the merchant, amount, date, and description.
Your API key is stored encrypted on your server and never shared.
</p>
{success && <SuccessBanner message={success} />}
{(saveMutation.isError || clearMutation.isError) && (
<ErrorBanner message={
(saveMutation.error as any)?.response?.data?.detail ??
(clearMutation.error as any)?.response?.data?.detail ??
"Failed"
} />
)}
<div>
<label className="text-sm font-medium block mb-1.5">Provider</label>
<select
value={provider}
onChange={e => setProvider(e.target.value)}
className={inputCls}
>
<option value="anthropic">Anthropic (Claude)</option>
<option value="openai">OpenAI (GPT-4o mini)</option>
</select>
</div>
<div>
<label className="text-sm font-medium block mb-1.5">
API Key {current?.has_api_key && <span className="text-muted-foreground font-normal">(leave blank to keep existing)</span>}
</label>
<div className="relative">
<input
type={showKey ? "text" : "password"}
value={apiKey}
onChange={e => setApiKey(e.target.value)}
className={cn(inputCls, "pr-10")}
placeholder={current?.has_api_key ? "••••••••••••••••" : provider === "anthropic" ? "sk-ant-..." : "sk-..."}
/>
<button
type="button"
onClick={() => setShowKey(v => !v)}
className="absolute right-3 top-1/2 -translate-y-1/2 text-muted-foreground hover:text-foreground"
>
{showKey ? <EyeOff className="w-4 h-4" /> : <Eye className="w-4 h-4" />}
</button>
</div>
<p className="text-xs text-muted-foreground mt-1">
{provider === "anthropic"
? "Get your key at console.anthropic.com → API Keys"
: "Get your key at platform.openai.com → API Keys"}
</p>
</div>
<div className="flex gap-3">
<button
onClick={() => saveMutation.mutate()}
disabled={saveMutation.isPending || (!apiKey && !current?.has_api_key)}
className="flex items-center gap-2 bg-primary text-primary-foreground px-4 py-2 rounded-lg text-sm font-medium hover:bg-primary/90 disabled:opacity-50 transition-colors"
>
{saveMutation.isPending && <Loader2 className="w-4 h-4 animate-spin" />}
{current?.has_api_key && !apiKey ? "Update provider" : "Save API key"}
</button>
{current?.has_api_key && (
<button
onClick={() => clearMutation.mutate()}
disabled={clearMutation.isPending}
className="flex items-center gap-2 border border-destructive/40 text-destructive px-4 py-2 rounded-lg text-sm font-medium hover:bg-destructive/10 disabled:opacity-50 transition-colors"
>
{clearMutation.isPending && <Loader2 className="w-4 h-4 animate-spin" />}
Remove key
</button>
)}
</div>
</div>
);
}
// ─── Data ─────────────────────────────────────────────────────────────────────
function DataSection() {

View file

@ -3,12 +3,14 @@ import { useMutation, useQueryClient } from "@tanstack/react-query";
import { format } from "date-fns";
import {
X, Paperclip, Upload, Trash2, FileText, ImageIcon, Loader2,
ArrowUpCircle, ArrowDownCircle, ArrowLeftRight, TrendingUp,
ArrowUpCircle, ArrowDownCircle, ArrowLeftRight, TrendingUp, Sparkles, CheckCircle,
} from "lucide-react";
import { cn } from "@/utils/cn";
import { formatCurrency } from "@/utils/currency";
import type { Transaction, AttachmentRef } from "@/api/transactions";
import { uploadAttachment, deleteAttachment, getAttachmentUrl } from "@/api/transactions";
import { uploadAttachment, deleteAttachment, getAttachmentUrl, updateTransaction } from "@/api/transactions";
import { parseReceipt } from "@/api/settings";
import type { ParsedReceipt } from "@/api/settings";
const TYPE_COLORS = {
income: "text-success",
@ -54,6 +56,9 @@ export default function TransactionDetailDrawer({ transaction, accountName, cate
const [attachments, setAttachments] = useState<AttachmentRef[]>(transaction.attachment_refs ?? []);
const [dragging, setDragging] = useState(false);
const [uploadError, setUploadError] = useState<string | null>(null);
const [parseResult, setParseResult] = useState<{ attId: string; data: ParsedReceipt } | null>(null);
const [parseError, setParseError] = useState<string | null>(null);
const [applySuccess, setApplySuccess] = useState(false);
const fileInputRef = useRef<HTMLInputElement>(null);
const Icon = TYPE_ICONS[transaction.type] ?? ArrowDownCircle;
@ -74,10 +79,37 @@ export default function TransactionDetailDrawer({ transaction, accountName, cate
mutationFn: (attachmentId: string) => deleteAttachment(transaction.id, attachmentId),
onSuccess: (_data, attachmentId) => {
setAttachments((prev) => prev.filter((a) => a.id !== attachmentId));
if (parseResult?.attId === attachmentId) setParseResult(null);
qc.invalidateQueries({ queryKey: ["transactions"] });
},
});
const parseMutation = useMutation({
mutationFn: (attachmentId: string) => parseReceipt(transaction.id, attachmentId),
onSuccess: (data, attachmentId) => {
setParseResult({ attId: attachmentId, data });
setParseError(null);
},
onError: (err: any) => {
setParseError(err?.response?.data?.detail ?? "Parse failed");
},
});
const applyMutation = useMutation({
mutationFn: (d: ParsedReceipt) => updateTransaction(transaction.id, {
...(d.merchant ? { merchant: d.merchant } : {}),
...(d.amount ? { amount: d.amount } : {}),
...(d.date ? { date: d.date } : {}),
...(d.description ? { description: d.description } : {}),
}),
onSuccess: () => {
qc.invalidateQueries({ queryKey: ["transactions"] });
setParseResult(null);
setApplySuccess(true);
setTimeout(() => setApplySuccess(false), 3000);
},
});
const handleFiles = useCallback((files: FileList | null) => {
if (!files) return;
setUploadError(null);
@ -176,10 +208,8 @@ export default function TransactionDetailDrawer({ transaction, accountName, cate
{attachments.length > 0 && (
<div className="space-y-2 mb-3">
{attachments.map((att) => (
<div
key={att.id}
className="flex items-center gap-3 bg-secondary/50 rounded-lg px-3 py-2 group"
>
<div key={att.id} className="space-y-2">
<div className="flex items-center gap-3 bg-secondary/50 rounded-lg px-3 py-2 group">
<FileIcon mimeType={att.mime_type} />
<div className="flex-1 min-w-0">
<a
@ -193,6 +223,18 @@ export default function TransactionDetailDrawer({ transaction, accountName, cate
</a>
<p className="text-xs text-muted-foreground">{humanFileSize(att.size)}</p>
</div>
<button
onClick={() => { setParseError(null); parseMutation.mutate(att.id); }}
disabled={parseMutation.isPending}
title="Parse with AI"
className="shrink-0 text-muted-foreground hover:text-primary opacity-100 sm:opacity-0 sm:group-hover:opacity-100 transition-all p-1 rounded"
>
{parseMutation.isPending && parseMutation.variables === att.id ? (
<Loader2 className="w-3.5 h-3.5 animate-spin" />
) : (
<Sparkles className="w-3.5 h-3.5" />
)}
</button>
<button
onClick={() => deleteMutation.mutate(att.id)}
disabled={deleteMutation.isPending}
@ -205,6 +247,39 @@ export default function TransactionDetailDrawer({ transaction, accountName, cate
)}
</button>
</div>
{/* Parse result card */}
{parseResult?.attId === att.id && (
<div className="bg-primary/5 border border-primary/20 rounded-lg p-3 space-y-2">
<p className="text-xs font-semibold text-primary flex items-center gap-1.5">
<Sparkles className="w-3.5 h-3.5" /> AI parsed result
</p>
<div className="space-y-1 text-xs">
{parseResult.data.merchant && <div className="flex justify-between"><span className="text-muted-foreground">Merchant</span><span className="font-medium">{parseResult.data.merchant}</span></div>}
{parseResult.data.amount && <div className="flex justify-between"><span className="text-muted-foreground">Amount</span><span className="font-medium">{parseResult.data.amount} {parseResult.data.currency ?? ""}</span></div>}
{parseResult.data.date && <div className="flex justify-between"><span className="text-muted-foreground">Date</span><span className="font-medium">{parseResult.data.date}</span></div>}
{parseResult.data.description && <div className="flex justify-between gap-4"><span className="text-muted-foreground shrink-0">Description</span><span className="font-medium text-right">{parseResult.data.description}</span></div>}
{parseResult.data.category && <div className="flex justify-between"><span className="text-muted-foreground">Category hint</span><span className="font-medium">{parseResult.data.category}</span></div>}
</div>
<div className="flex gap-2 pt-1">
<button
onClick={() => applyMutation.mutate(parseResult.data)}
disabled={applyMutation.isPending}
className="flex items-center gap-1.5 bg-primary text-primary-foreground text-xs px-3 py-1.5 rounded-lg font-medium hover:bg-primary/90 disabled:opacity-50 transition-colors"
>
{applyMutation.isPending ? <Loader2 className="w-3 h-3 animate-spin" /> : <CheckCircle className="w-3 h-3" />}
Apply to transaction
</button>
<button
onClick={() => setParseResult(null)}
className="text-xs text-muted-foreground hover:text-foreground px-2 py-1.5 rounded-lg hover:bg-secondary transition-colors"
>
Dismiss
</button>
</div>
</div>
)}
</div>
))}
</div>
)}
@ -250,6 +325,16 @@ export default function TransactionDetailDrawer({ transaction, accountName, cate
{uploadError && (
<p className="text-destructive text-xs mt-2">{uploadError}</p>
)}
{parseError && (
<p className="text-destructive text-xs mt-2">{parseError}</p>
)}
{applySuccess && (
<div className="flex items-center gap-1.5 text-success text-xs mt-2">
<CheckCircle className="w-3.5 h-3.5" /> Transaction updated from receipt
</div>
)}
</div>
</div>
</div>