Add Scan Receipt button to create transactions from receipt photos
- New backend endpoint POST /transactions/parse-receipt (file upload, no existing txn needed) - Refactored AI call logic into shared _call_ai_parse helper (no duplication) - Scan Receipt button in transactions toolbar → file picker → AI parse → pre-filled form - TransactionFormModal accepts initialValues prop to pre-populate fields from receipt - "Fields pre-filled from receipt" banner shown in form when AI-populated - Scan error displayed inline with dismiss button - Supports JPEG, PNG, WebP, PDF (Anthropic) or images (OpenAI) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
d6118bac54
commit
024a8330fa
4 changed files with 171 additions and 82 deletions
|
|
@ -278,59 +278,28 @@ async def delete_attachment(
|
|||
await db.commit()
|
||||
|
||||
|
||||
@router.post("/{txn_id}/attachments/{attachment_id}/parse")
|
||||
async def parse_attachment(
|
||||
txn_id: uuid.UUID,
|
||||
attachment_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user=Depends(get_current_user),
|
||||
):
|
||||
"""Send an attachment to the configured AI provider and extract receipt fields."""
|
||||
_RECEIPT_PROMPT = (
|
||||
"You are a receipt parser. Extract information from this receipt and return ONLY a JSON object "
|
||||
"with exactly these keys (use null for any field you cannot determine):\n"
|
||||
'{"merchant": "store name", "amount": 0.00, "currency": "GBP", '
|
||||
'"date": "YYYY-MM-DD", "description": "brief description", '
|
||||
'"category": "one of: Food & Drink, Transport, Shopping, Entertainment, Health, Travel, Bills & Utilities, Other"}\n'
|
||||
"Return ONLY the JSON object. No markdown, no explanation, no code fences."
|
||||
)
|
||||
|
||||
|
||||
async def _call_ai_parse(file_bytes: bytes, mime_type: str, user_row) -> dict:
|
||||
"""Call the configured AI provider and return parsed receipt fields."""
|
||||
import base64
|
||||
import json
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from app.db.models.transaction import Transaction as TxnModel
|
||||
from app.db.models.user import User as UserModel
|
||||
from app.core.security import decrypt_field
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
# Reload user to get ai fields (get_current_user may be cached without them)
|
||||
user_row = await db.get(UserModel, user.id)
|
||||
if not user_row or not user_row.ai_provider or not user_row.ai_api_key_enc:
|
||||
if not user_row.ai_provider or not user_row.ai_api_key_enc:
|
||||
raise HTTPException(status_code=400, detail="No AI provider configured. Add your API key in Settings → AI.")
|
||||
|
||||
api_key = decrypt_field(user_row.ai_api_key_enc)
|
||||
|
||||
result = await db.execute(
|
||||
select(TxnModel).where(TxnModel.id == txn_id, TxnModel.user_id == user.id)
|
||||
)
|
||||
txn_row = result.scalar_one_or_none()
|
||||
if not txn_row:
|
||||
raise HTTPException(status_code=404, detail="Transaction not found")
|
||||
|
||||
ref = next((r for r in (txn_row.attachment_refs or []) if r["id"] == attachment_id), None)
|
||||
if not ref:
|
||||
raise HTTPException(status_code=404, detail="Attachment not found")
|
||||
|
||||
mime_type = ref["mime_type"]
|
||||
path = Path(settings.upload_dir) / str(user.id) / ref["stored_name"]
|
||||
if not path.exists():
|
||||
raise HTTPException(status_code=404, detail="Attachment file missing")
|
||||
|
||||
file_bytes = path.read_bytes()
|
||||
b64 = base64.standard_b64encode(file_bytes).decode()
|
||||
|
||||
prompt = (
|
||||
"You are a receipt parser. Extract information from this receipt and return ONLY a JSON object "
|
||||
"with exactly these keys (use null for any field you cannot determine):\n"
|
||||
'{"merchant": "store name", "amount": 0.00, "currency": "GBP", '
|
||||
'"date": "YYYY-MM-DD", "description": "brief description", '
|
||||
'"category": "one of: Food & Drink, Transport, Shopping, Entertainment, Health, Travel, Bills & Utilities, Other"}\n'
|
||||
"Return ONLY the JSON object. No markdown, no explanation, no code fences."
|
||||
)
|
||||
|
||||
custom_base_url = (user_row.ai_base_url or "").rstrip("/")
|
||||
custom_model = (user_row.ai_model or "").strip()
|
||||
|
||||
|
|
@ -339,28 +308,14 @@ async def parse_attachment(
|
|||
base_url = custom_base_url or "https://api.anthropic.com"
|
||||
model = custom_model or "claude-haiku-4-5-20251001"
|
||||
if mime_type == "application/pdf":
|
||||
content_block = {
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": b64},
|
||||
}
|
||||
content_block = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": b64}}
|
||||
else:
|
||||
content_block = {
|
||||
"type": "image",
|
||||
"source": {"type": "base64", "media_type": mime_type, "data": b64},
|
||||
}
|
||||
content_block = {"type": "image", "source": {"type": "base64", "media_type": mime_type, "data": b64}}
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
resp = await client.post(
|
||||
f"{base_url}/v1/messages",
|
||||
headers={
|
||||
"x-api-key": api_key,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": model,
|
||||
"max_tokens": 512,
|
||||
"messages": [{"role": "user", "content": [content_block, {"type": "text", "text": prompt}]}],
|
||||
},
|
||||
headers={"x-api-key": api_key, "anthropic-version": "2023-06-01", "content-type": "application/json"},
|
||||
json={"model": model, "max_tokens": 512, "messages": [{"role": "user", "content": [content_block, {"type": "text", "text": _RECEIPT_PROMPT}]}]},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
text = resp.json()["content"][0]["text"].strip()
|
||||
|
|
@ -374,17 +329,10 @@ async def parse_attachment(
|
|||
resp = await client.post(
|
||||
f"{base_url}/v1/chat/completions",
|
||||
headers={"Authorization": f"Bearer {api_key}", "content-type": "application/json"},
|
||||
json={
|
||||
"model": model,
|
||||
"max_tokens": 512,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{b64}"}},
|
||||
{"type": "text", "text": prompt},
|
||||
],
|
||||
}],
|
||||
},
|
||||
json={"model": model, "max_tokens": 512, "messages": [{"role": "user", "content": [
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{b64}"}},
|
||||
{"type": "text", "text": _RECEIPT_PROMPT},
|
||||
]}]},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
text = resp.json()["choices"][0]["message"]["content"].strip()
|
||||
|
|
@ -397,7 +345,6 @@ async def parse_attachment(
|
|||
except httpx.RequestError:
|
||||
raise HTTPException(status_code=502, detail="Could not reach AI provider")
|
||||
|
||||
# Strip markdown fences if model wrapped the JSON
|
||||
if text.startswith("```"):
|
||||
text = text.split("```")[1]
|
||||
if text.startswith("json"):
|
||||
|
|
@ -419,6 +366,65 @@ async def parse_attachment(
|
|||
}
|
||||
|
||||
|
||||
@router.post("/parse-receipt")
|
||||
async def parse_receipt_upload(
|
||||
file: UploadFile = File(...),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user=Depends(get_current_user),
|
||||
):
|
||||
"""Upload a receipt image and parse it with AI — no existing transaction required."""
|
||||
from app.db.models.user import User as UserModel
|
||||
|
||||
settings = get_settings()
|
||||
filename = file.filename or "upload"
|
||||
ext = Path(filename).suffix.lower()
|
||||
if ext not in ALLOWED_EXTENSIONS:
|
||||
raise HTTPException(status_code=400, detail="Unsupported file type. Allowed: JPG, PNG, WebP, PDF")
|
||||
|
||||
content = await file.read(settings.max_attachment_bytes + 1)
|
||||
if len(content) > settings.max_attachment_bytes:
|
||||
raise HTTPException(status_code=413, detail="File too large (max 10 MB)")
|
||||
|
||||
import magic
|
||||
mime_type = magic.from_buffer(content[:2048], mime=True)
|
||||
if mime_type not in ALLOWED_MIME_TYPES:
|
||||
raise HTTPException(status_code=400, detail="File content does not match an allowed type")
|
||||
|
||||
user_row = await db.get(UserModel, user.id)
|
||||
return await _call_ai_parse(content, mime_type, user_row)
|
||||
|
||||
|
||||
@router.post("/{txn_id}/attachments/{attachment_id}/parse")
|
||||
async def parse_attachment(
|
||||
txn_id: uuid.UUID,
|
||||
attachment_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user=Depends(get_current_user),
|
||||
):
|
||||
"""Parse an already-uploaded attachment with AI."""
|
||||
from sqlalchemy import select
|
||||
from app.db.models.transaction import Transaction as TxnModel
|
||||
from app.db.models.user import User as UserModel
|
||||
|
||||
settings = get_settings()
|
||||
user_row = await db.get(UserModel, user.id)
|
||||
|
||||
result = await db.execute(select(TxnModel).where(TxnModel.id == txn_id, TxnModel.user_id == user.id))
|
||||
txn_row = result.scalar_one_or_none()
|
||||
if not txn_row:
|
||||
raise HTTPException(status_code=404, detail="Transaction not found")
|
||||
|
||||
ref = next((r for r in (txn_row.attachment_refs or []) if r["id"] == attachment_id), None)
|
||||
if not ref:
|
||||
raise HTTPException(status_code=404, detail="Attachment not found")
|
||||
|
||||
path = Path(settings.upload_dir) / str(user.id) / ref["stored_name"]
|
||||
if not path.exists():
|
||||
raise HTTPException(status_code=404, detail="Attachment file missing")
|
||||
|
||||
return await _call_ai_parse(path.read_bytes(), ref["mime_type"], user_row)
|
||||
|
||||
|
||||
@router.post("/import")
|
||||
async def import_transactions(
|
||||
file: UploadFile = File(...),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue