Add AI receipt parsing with provider API key settings
- Settings → AI: configure Anthropic or OpenAI provider with encrypted API key - Sparkle button on each attachment in transaction drawer sends image/PDF to AI - AI extracts merchant, amount, date, description, category hint - "Apply to transaction" button patches the transaction with parsed fields - Anthropic supports images and PDFs; OpenAI supports images only - API key stored AES-256-GCM encrypted in users table (migration 0003) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
fe4e69b9ad
commit
22fc1ce2f1
8 changed files with 507 additions and 31 deletions
|
|
@ -1,6 +1,6 @@
|
|||
from fastapi import APIRouter
|
||||
|
||||
from app.api.v1 import auth, users, accounts, categories, transactions, budgets, reports, investments, predictions, admin
|
||||
from app.api.v1 import auth, users, accounts, categories, transactions, budgets, reports, investments, predictions, admin, settings
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(auth.router, prefix="/auth", tags=["auth"])
|
||||
|
|
@ -13,3 +13,4 @@ router.include_router(reports.router)
|
|||
router.include_router(investments.router)
|
||||
router.include_router(predictions.router)
|
||||
router.include_router(admin.router)
|
||||
router.include_router(settings.router)
|
||||
|
|
|
|||
69
backend/app/api/v1/settings.py
Normal file
69
backend/app/api/v1/settings.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
"""
|
||||
User-level settings: AI provider configuration.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.security import decrypt_field, encrypt_field
|
||||
from app.dependencies import get_current_user, get_db
|
||||
from app.db.models.user import User
|
||||
|
||||
router = APIRouter(prefix="/settings", tags=["settings"])
|
||||
|
||||
SUPPORTED_PROVIDERS = {"anthropic", "openai"}
|
||||
|
||||
|
||||
class AiSettingsResponse(BaseModel):
|
||||
provider: str | None
|
||||
has_api_key: bool
|
||||
|
||||
|
||||
class AiSettingsSave(BaseModel):
|
||||
provider: str
|
||||
api_key: str
|
||||
|
||||
|
||||
@router.get("/ai", response_model=AiSettingsResponse)
|
||||
async def get_ai_settings(user: User = Depends(get_current_user)):
|
||||
return AiSettingsResponse(
|
||||
provider=user.ai_provider,
|
||||
has_api_key=bool(user.ai_api_key_enc),
|
||||
)
|
||||
|
||||
|
||||
@router.put("/ai", response_model=AiSettingsResponse)
|
||||
async def save_ai_settings(
|
||||
body: AiSettingsSave,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user: User = Depends(get_current_user),
|
||||
):
|
||||
if body.provider not in SUPPORTED_PROVIDERS:
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported provider. Choose: {', '.join(SUPPORTED_PROVIDERS)}")
|
||||
if not body.api_key.strip():
|
||||
raise HTTPException(status_code=400, detail="api_key must not be empty")
|
||||
|
||||
encrypted = encrypt_field(body.api_key.strip())
|
||||
await db.execute(
|
||||
update(User)
|
||||
.where(User.id == user.id)
|
||||
.values(ai_provider=body.provider, ai_api_key_enc=encrypted)
|
||||
)
|
||||
await db.commit()
|
||||
return AiSettingsResponse(provider=body.provider, has_api_key=True)
|
||||
|
||||
|
||||
@router.delete("/ai", status_code=204)
|
||||
async def clear_ai_settings(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user: User = Depends(get_current_user),
|
||||
):
|
||||
await db.execute(
|
||||
update(User)
|
||||
.where(User.id == user.id)
|
||||
.values(ai_provider=None, ai_api_key_enc=None)
|
||||
)
|
||||
await db.commit()
|
||||
|
|
@ -278,6 +278,140 @@ async def delete_attachment(
|
|||
await db.commit()
|
||||
|
||||
|
||||
@router.post("/{txn_id}/attachments/{attachment_id}/parse")
|
||||
async def parse_attachment(
|
||||
txn_id: uuid.UUID,
|
||||
attachment_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user=Depends(get_current_user),
|
||||
):
|
||||
"""Send an attachment to the configured AI provider and extract receipt fields."""
|
||||
import base64
|
||||
import json
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from app.db.models.transaction import Transaction as TxnModel
|
||||
from app.db.models.user import User as UserModel
|
||||
from app.core.security import decrypt_field
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
# Reload user to get ai fields (get_current_user may be cached without them)
|
||||
user_row = await db.get(UserModel, user.id)
|
||||
if not user_row or not user_row.ai_provider or not user_row.ai_api_key_enc:
|
||||
raise HTTPException(status_code=400, detail="No AI provider configured. Add your API key in Settings → AI.")
|
||||
|
||||
api_key = decrypt_field(user_row.ai_api_key_enc)
|
||||
|
||||
result = await db.execute(
|
||||
select(TxnModel).where(TxnModel.id == txn_id, TxnModel.user_id == user.id)
|
||||
)
|
||||
txn_row = result.scalar_one_or_none()
|
||||
if not txn_row:
|
||||
raise HTTPException(status_code=404, detail="Transaction not found")
|
||||
|
||||
ref = next((r for r in (txn_row.attachment_refs or []) if r["id"] == attachment_id), None)
|
||||
if not ref:
|
||||
raise HTTPException(status_code=404, detail="Attachment not found")
|
||||
|
||||
mime_type = ref["mime_type"]
|
||||
path = Path(settings.upload_dir) / str(user.id) / ref["stored_name"]
|
||||
if not path.exists():
|
||||
raise HTTPException(status_code=404, detail="Attachment file missing")
|
||||
|
||||
file_bytes = path.read_bytes()
|
||||
b64 = base64.standard_b64encode(file_bytes).decode()
|
||||
|
||||
prompt = (
|
||||
"You are a receipt parser. Extract information from this receipt and return ONLY a JSON object "
|
||||
"with exactly these keys (use null for any field you cannot determine):\n"
|
||||
'{"merchant": "store name", "amount": 0.00, "currency": "GBP", '
|
||||
'"date": "YYYY-MM-DD", "description": "brief description", '
|
||||
'"category": "one of: Food & Drink, Transport, Shopping, Entertainment, Health, Travel, Bills & Utilities, Other"}\n'
|
||||
"Return ONLY the JSON object. No markdown, no explanation, no code fences."
|
||||
)
|
||||
|
||||
try:
|
||||
if user_row.ai_provider == "anthropic":
|
||||
if mime_type == "application/pdf":
|
||||
content_block = {
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": b64},
|
||||
}
|
||||
else:
|
||||
content_block = {
|
||||
"type": "image",
|
||||
"source": {"type": "base64", "media_type": mime_type, "data": b64},
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.post(
|
||||
"https://api.anthropic.com/v1/messages",
|
||||
headers={
|
||||
"x-api-key": api_key,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": "claude-haiku-4-5-20251001",
|
||||
"max_tokens": 512,
|
||||
"messages": [{"role": "user", "content": [content_block, {"type": "text", "text": prompt}]}],
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
text = resp.json()["content"][0]["text"].strip()
|
||||
|
||||
elif user_row.ai_provider == "openai":
|
||||
if mime_type == "application/pdf":
|
||||
raise HTTPException(status_code=400, detail="PDF parsing is not supported with the OpenAI provider. Use an image format or switch to Anthropic.")
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.post(
|
||||
"https://api.openai.com/v1/chat/completions",
|
||||
headers={"Authorization": f"Bearer {api_key}", "content-type": "application/json"},
|
||||
json={
|
||||
"model": "gpt-4o-mini",
|
||||
"max_tokens": 512,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{b64}"}},
|
||||
{"type": "text", "text": prompt},
|
||||
],
|
||||
}],
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
text = resp.json()["choices"][0]["message"]["content"].strip()
|
||||
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="Unknown provider")
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise HTTPException(status_code=502, detail=f"AI provider error: {e.response.status_code}")
|
||||
except httpx.RequestError:
|
||||
raise HTTPException(status_code=502, detail="Could not reach AI provider")
|
||||
|
||||
# Strip markdown fences if model wrapped the JSON
|
||||
if text.startswith("```"):
|
||||
text = text.split("```")[1]
|
||||
if text.startswith("json"):
|
||||
text = text[4:]
|
||||
text = text.strip()
|
||||
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(status_code=502, detail="AI returned an unexpected response. Try again.")
|
||||
|
||||
return {
|
||||
"merchant": parsed.get("merchant"),
|
||||
"amount": parsed.get("amount"),
|
||||
"currency": parsed.get("currency"),
|
||||
"date": parsed.get("date"),
|
||||
"description": parsed.get("description"),
|
||||
"category": parsed.get("category"),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/import")
|
||||
async def import_transactions(
|
||||
file: UploadFile = File(...),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue