Add AI receipt parsing with provider API key settings

- Settings → AI: configure Anthropic or OpenAI provider with encrypted API key
- Sparkle button on each attachment in transaction drawer sends image/PDF to AI
- AI extracts merchant, amount, date, description, category hint
- "Apply to transaction" button patches the transaction with parsed fields
- Anthropic supports images and PDFs; OpenAI supports images only
- API key stored AES-256-GCM encrypted in users table (migration 0003)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
megaproxy 2026-04-22 17:07:24 +00:00
parent fe4e69b9ad
commit 22fc1ce2f1
8 changed files with 507 additions and 31 deletions

View file

@ -278,6 +278,140 @@ async def delete_attachment(
await db.commit()
@router.post("/{txn_id}/attachments/{attachment_id}/parse")
async def parse_attachment(
txn_id: uuid.UUID,
attachment_id: str,
db: AsyncSession = Depends(get_db),
user=Depends(get_current_user),
):
"""Send an attachment to the configured AI provider and extract receipt fields."""
import base64
import json
import httpx
from sqlalchemy import select
from app.db.models.transaction import Transaction as TxnModel
from app.db.models.user import User as UserModel
from app.core.security import decrypt_field
settings = get_settings()
# Reload user to get ai fields (get_current_user may be cached without them)
user_row = await db.get(UserModel, user.id)
if not user_row or not user_row.ai_provider or not user_row.ai_api_key_enc:
raise HTTPException(status_code=400, detail="No AI provider configured. Add your API key in Settings → AI.")
api_key = decrypt_field(user_row.ai_api_key_enc)
result = await db.execute(
select(TxnModel).where(TxnModel.id == txn_id, TxnModel.user_id == user.id)
)
txn_row = result.scalar_one_or_none()
if not txn_row:
raise HTTPException(status_code=404, detail="Transaction not found")
ref = next((r for r in (txn_row.attachment_refs or []) if r["id"] == attachment_id), None)
if not ref:
raise HTTPException(status_code=404, detail="Attachment not found")
mime_type = ref["mime_type"]
path = Path(settings.upload_dir) / str(user.id) / ref["stored_name"]
if not path.exists():
raise HTTPException(status_code=404, detail="Attachment file missing")
file_bytes = path.read_bytes()
b64 = base64.standard_b64encode(file_bytes).decode()
prompt = (
"You are a receipt parser. Extract information from this receipt and return ONLY a JSON object "
"with exactly these keys (use null for any field you cannot determine):\n"
'{"merchant": "store name", "amount": 0.00, "currency": "GBP", '
'"date": "YYYY-MM-DD", "description": "brief description", '
'"category": "one of: Food & Drink, Transport, Shopping, Entertainment, Health, Travel, Bills & Utilities, Other"}\n'
"Return ONLY the JSON object. No markdown, no explanation, no code fences."
)
try:
if user_row.ai_provider == "anthropic":
if mime_type == "application/pdf":
content_block = {
"type": "document",
"source": {"type": "base64", "media_type": "application/pdf", "data": b64},
}
else:
content_block = {
"type": "image",
"source": {"type": "base64", "media_type": mime_type, "data": b64},
}
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.post(
"https://api.anthropic.com/v1/messages",
headers={
"x-api-key": api_key,
"anthropic-version": "2023-06-01",
"content-type": "application/json",
},
json={
"model": "claude-haiku-4-5-20251001",
"max_tokens": 512,
"messages": [{"role": "user", "content": [content_block, {"type": "text", "text": prompt}]}],
},
)
resp.raise_for_status()
text = resp.json()["content"][0]["text"].strip()
elif user_row.ai_provider == "openai":
if mime_type == "application/pdf":
raise HTTPException(status_code=400, detail="PDF parsing is not supported with the OpenAI provider. Use an image format or switch to Anthropic.")
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.post(
"https://api.openai.com/v1/chat/completions",
headers={"Authorization": f"Bearer {api_key}", "content-type": "application/json"},
json={
"model": "gpt-4o-mini",
"max_tokens": 512,
"messages": [{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{b64}"}},
{"type": "text", "text": prompt},
],
}],
},
)
resp.raise_for_status()
text = resp.json()["choices"][0]["message"]["content"].strip()
else:
raise HTTPException(status_code=400, detail="Unknown provider")
except httpx.HTTPStatusError as e:
raise HTTPException(status_code=502, detail=f"AI provider error: {e.response.status_code}")
except httpx.RequestError:
raise HTTPException(status_code=502, detail="Could not reach AI provider")
# Strip markdown fences if model wrapped the JSON
if text.startswith("```"):
text = text.split("```")[1]
if text.startswith("json"):
text = text[4:]
text = text.strip()
try:
parsed = json.loads(text)
except json.JSONDecodeError:
raise HTTPException(status_code=502, detail="AI returned an unexpected response. Try again.")
return {
"merchant": parsed.get("merchant"),
"amount": parsed.get("amount"),
"currency": parsed.get("currency"),
"date": parsed.get("date"),
"description": parsed.get("description"),
"category": parsed.get("category"),
}
@router.post("/import")
async def import_transactions(
file: UploadFile = File(...),