Add Scan Receipt button to create transactions from receipt photos
- New backend endpoint POST /transactions/parse-receipt (file upload, no existing txn needed) - Refactored AI call logic into shared _call_ai_parse helper (no duplication) - Scan Receipt button in transactions toolbar → file picker → AI parse → pre-filled form - TransactionFormModal accepts initialValues prop to pre-populate fields from receipt - "Fields pre-filled from receipt" banner shown in form when AI-populated - Scan error displayed inline with dismiss button - Supports JPEG, PNG, WebP, PDF (Anthropic) or images (OpenAI) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
d6118bac54
commit
024a8330fa
4 changed files with 171 additions and 82 deletions
|
|
@ -278,59 +278,28 @@ async def delete_attachment(
|
|||
await db.commit()
|
||||
|
||||
|
||||
@router.post("/{txn_id}/attachments/{attachment_id}/parse")
|
||||
async def parse_attachment(
|
||||
txn_id: uuid.UUID,
|
||||
attachment_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user=Depends(get_current_user),
|
||||
):
|
||||
"""Send an attachment to the configured AI provider and extract receipt fields."""
|
||||
_RECEIPT_PROMPT = (
|
||||
"You are a receipt parser. Extract information from this receipt and return ONLY a JSON object "
|
||||
"with exactly these keys (use null for any field you cannot determine):\n"
|
||||
'{"merchant": "store name", "amount": 0.00, "currency": "GBP", '
|
||||
'"date": "YYYY-MM-DD", "description": "brief description", '
|
||||
'"category": "one of: Food & Drink, Transport, Shopping, Entertainment, Health, Travel, Bills & Utilities, Other"}\n'
|
||||
"Return ONLY the JSON object. No markdown, no explanation, no code fences."
|
||||
)
|
||||
|
||||
|
||||
async def _call_ai_parse(file_bytes: bytes, mime_type: str, user_row) -> dict:
|
||||
"""Call the configured AI provider and return parsed receipt fields."""
|
||||
import base64
|
||||
import json
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from app.db.models.transaction import Transaction as TxnModel
|
||||
from app.db.models.user import User as UserModel
|
||||
from app.core.security import decrypt_field
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
# Reload user to get ai fields (get_current_user may be cached without them)
|
||||
user_row = await db.get(UserModel, user.id)
|
||||
if not user_row or not user_row.ai_provider or not user_row.ai_api_key_enc:
|
||||
if not user_row.ai_provider or not user_row.ai_api_key_enc:
|
||||
raise HTTPException(status_code=400, detail="No AI provider configured. Add your API key in Settings → AI.")
|
||||
|
||||
api_key = decrypt_field(user_row.ai_api_key_enc)
|
||||
|
||||
result = await db.execute(
|
||||
select(TxnModel).where(TxnModel.id == txn_id, TxnModel.user_id == user.id)
|
||||
)
|
||||
txn_row = result.scalar_one_or_none()
|
||||
if not txn_row:
|
||||
raise HTTPException(status_code=404, detail="Transaction not found")
|
||||
|
||||
ref = next((r for r in (txn_row.attachment_refs or []) if r["id"] == attachment_id), None)
|
||||
if not ref:
|
||||
raise HTTPException(status_code=404, detail="Attachment not found")
|
||||
|
||||
mime_type = ref["mime_type"]
|
||||
path = Path(settings.upload_dir) / str(user.id) / ref["stored_name"]
|
||||
if not path.exists():
|
||||
raise HTTPException(status_code=404, detail="Attachment file missing")
|
||||
|
||||
file_bytes = path.read_bytes()
|
||||
b64 = base64.standard_b64encode(file_bytes).decode()
|
||||
|
||||
prompt = (
|
||||
"You are a receipt parser. Extract information from this receipt and return ONLY a JSON object "
|
||||
"with exactly these keys (use null for any field you cannot determine):\n"
|
||||
'{"merchant": "store name", "amount": 0.00, "currency": "GBP", '
|
||||
'"date": "YYYY-MM-DD", "description": "brief description", '
|
||||
'"category": "one of: Food & Drink, Transport, Shopping, Entertainment, Health, Travel, Bills & Utilities, Other"}\n'
|
||||
"Return ONLY the JSON object. No markdown, no explanation, no code fences."
|
||||
)
|
||||
|
||||
custom_base_url = (user_row.ai_base_url or "").rstrip("/")
|
||||
custom_model = (user_row.ai_model or "").strip()
|
||||
|
||||
|
|
@ -339,28 +308,14 @@ async def parse_attachment(
|
|||
base_url = custom_base_url or "https://api.anthropic.com"
|
||||
model = custom_model or "claude-haiku-4-5-20251001"
|
||||
if mime_type == "application/pdf":
|
||||
content_block = {
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": b64},
|
||||
}
|
||||
content_block = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": b64}}
|
||||
else:
|
||||
content_block = {
|
||||
"type": "image",
|
||||
"source": {"type": "base64", "media_type": mime_type, "data": b64},
|
||||
}
|
||||
content_block = {"type": "image", "source": {"type": "base64", "media_type": mime_type, "data": b64}}
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
resp = await client.post(
|
||||
f"{base_url}/v1/messages",
|
||||
headers={
|
||||
"x-api-key": api_key,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": model,
|
||||
"max_tokens": 512,
|
||||
"messages": [{"role": "user", "content": [content_block, {"type": "text", "text": prompt}]}],
|
||||
},
|
||||
headers={"x-api-key": api_key, "anthropic-version": "2023-06-01", "content-type": "application/json"},
|
||||
json={"model": model, "max_tokens": 512, "messages": [{"role": "user", "content": [content_block, {"type": "text", "text": _RECEIPT_PROMPT}]}]},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
text = resp.json()["content"][0]["text"].strip()
|
||||
|
|
@ -374,17 +329,10 @@ async def parse_attachment(
|
|||
resp = await client.post(
|
||||
f"{base_url}/v1/chat/completions",
|
||||
headers={"Authorization": f"Bearer {api_key}", "content-type": "application/json"},
|
||||
json={
|
||||
"model": model,
|
||||
"max_tokens": 512,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{b64}"}},
|
||||
{"type": "text", "text": prompt},
|
||||
],
|
||||
}],
|
||||
},
|
||||
json={"model": model, "max_tokens": 512, "messages": [{"role": "user", "content": [
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{b64}"}},
|
||||
{"type": "text", "text": _RECEIPT_PROMPT},
|
||||
]}]},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
text = resp.json()["choices"][0]["message"]["content"].strip()
|
||||
|
|
@ -397,7 +345,6 @@ async def parse_attachment(
|
|||
except httpx.RequestError:
|
||||
raise HTTPException(status_code=502, detail="Could not reach AI provider")
|
||||
|
||||
# Strip markdown fences if model wrapped the JSON
|
||||
if text.startswith("```"):
|
||||
text = text.split("```")[1]
|
||||
if text.startswith("json"):
|
||||
|
|
@ -419,6 +366,65 @@ async def parse_attachment(
|
|||
}
|
||||
|
||||
|
||||
@router.post("/parse-receipt")
|
||||
async def parse_receipt_upload(
|
||||
file: UploadFile = File(...),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user=Depends(get_current_user),
|
||||
):
|
||||
"""Upload a receipt image and parse it with AI — no existing transaction required."""
|
||||
from app.db.models.user import User as UserModel
|
||||
|
||||
settings = get_settings()
|
||||
filename = file.filename or "upload"
|
||||
ext = Path(filename).suffix.lower()
|
||||
if ext not in ALLOWED_EXTENSIONS:
|
||||
raise HTTPException(status_code=400, detail="Unsupported file type. Allowed: JPG, PNG, WebP, PDF")
|
||||
|
||||
content = await file.read(settings.max_attachment_bytes + 1)
|
||||
if len(content) > settings.max_attachment_bytes:
|
||||
raise HTTPException(status_code=413, detail="File too large (max 10 MB)")
|
||||
|
||||
import magic
|
||||
mime_type = magic.from_buffer(content[:2048], mime=True)
|
||||
if mime_type not in ALLOWED_MIME_TYPES:
|
||||
raise HTTPException(status_code=400, detail="File content does not match an allowed type")
|
||||
|
||||
user_row = await db.get(UserModel, user.id)
|
||||
return await _call_ai_parse(content, mime_type, user_row)
|
||||
|
||||
|
||||
@router.post("/{txn_id}/attachments/{attachment_id}/parse")
|
||||
async def parse_attachment(
|
||||
txn_id: uuid.UUID,
|
||||
attachment_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user=Depends(get_current_user),
|
||||
):
|
||||
"""Parse an already-uploaded attachment with AI."""
|
||||
from sqlalchemy import select
|
||||
from app.db.models.transaction import Transaction as TxnModel
|
||||
from app.db.models.user import User as UserModel
|
||||
|
||||
settings = get_settings()
|
||||
user_row = await db.get(UserModel, user.id)
|
||||
|
||||
result = await db.execute(select(TxnModel).where(TxnModel.id == txn_id, TxnModel.user_id == user.id))
|
||||
txn_row = result.scalar_one_or_none()
|
||||
if not txn_row:
|
||||
raise HTTPException(status_code=404, detail="Transaction not found")
|
||||
|
||||
ref = next((r for r in (txn_row.attachment_refs or []) if r["id"] == attachment_id), None)
|
||||
if not ref:
|
||||
raise HTTPException(status_code=404, detail="Attachment not found")
|
||||
|
||||
path = Path(settings.upload_dir) / str(user.id) / ref["stored_name"]
|
||||
if not path.exists():
|
||||
raise HTTPException(status_code=404, detail="Attachment file missing")
|
||||
|
||||
return await _call_ai_parse(path.read_bytes(), ref["mime_type"], user_row)
|
||||
|
||||
|
||||
@router.post("/import")
|
||||
async def import_transactions(
|
||||
file: UploadFile = File(...),
|
||||
|
|
|
|||
|
|
@ -41,3 +41,12 @@ export async function parseReceipt(txnId: string, attachmentId: string): Promise
|
|||
const { data } = await api.post(`/transactions/${txnId}/attachments/${attachmentId}/parse`);
|
||||
return data;
|
||||
}
|
||||
|
||||
export async function parseReceiptFile(file: File): Promise<ParsedReceipt> {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
const { data } = await api.post("/transactions/parse-receipt", form, {
|
||||
headers: { "Content-Type": "multipart/form-data" },
|
||||
});
|
||||
return data;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { useForm } from "react-hook-form";
|
|||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import { z } from "zod";
|
||||
import { format } from "date-fns";
|
||||
import { X, Loader2 } from "lucide-react";
|
||||
import { X, Loader2, Sparkles } from "lucide-react";
|
||||
import type { Account } from "@/api/accounts";
|
||||
|
||||
const schema = z.object({
|
||||
|
|
@ -21,22 +21,35 @@ const schema = z.object({
|
|||
|
||||
type Form = z.infer<typeof schema>;
|
||||
|
||||
export interface TransactionInitialValues {
|
||||
description?: string;
|
||||
merchant?: string;
|
||||
amount?: number;
|
||||
date?: string;
|
||||
currency?: string;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
accounts: Account[];
|
||||
categories: { id: string; name: string; type: string }[];
|
||||
onClose: () => void;
|
||||
onSubmit: (data: any) => void;
|
||||
isLoading: boolean;
|
||||
initialValues?: TransactionInitialValues;
|
||||
parsedFromReceipt?: boolean;
|
||||
}
|
||||
|
||||
export default function TransactionFormModal({ accounts, categories, onClose, onSubmit, isLoading }: Props) {
|
||||
export default function TransactionFormModal({ accounts, categories, onClose, onSubmit, isLoading, initialValues, parsedFromReceipt }: Props) {
|
||||
const { register, handleSubmit, watch, formState: { errors } } = useForm<Form>({
|
||||
resolver: zodResolver(schema),
|
||||
defaultValues: {
|
||||
type: "expense",
|
||||
status: "cleared",
|
||||
currency: "GBP",
|
||||
date: format(new Date(), "yyyy-MM-dd"),
|
||||
currency: initialValues?.currency ?? "GBP",
|
||||
date: initialValues?.date ?? format(new Date(), "yyyy-MM-dd"),
|
||||
description: initialValues?.description ?? "",
|
||||
merchant: initialValues?.merchant ?? "",
|
||||
amount: initialValues?.amount ? Math.abs(initialValues.amount) : undefined,
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -70,6 +83,13 @@ export default function TransactionFormModal({ accounts, categories, onClose, on
|
|||
</div>
|
||||
|
||||
<form onSubmit={handleSubmit(handleFormSubmit)} className="p-6 space-y-4">
|
||||
{parsedFromReceipt && (
|
||||
<div className="flex items-center gap-2 bg-primary/10 border border-primary/20 rounded-lg px-3 py-2 text-xs text-primary">
|
||||
<Sparkles className="w-3.5 h-3.5 shrink-0" />
|
||||
Fields pre-filled from receipt — review before saving
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Type */}
|
||||
<div>
|
||||
<label className="text-sm font-medium block mb-1.5">Type</label>
|
||||
|
|
|
|||
|
|
@ -1,16 +1,19 @@
|
|||
import { useState } from "react";
|
||||
import { useRef, useState } from "react";
|
||||
import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { getTransactions, deleteTransaction, createTransaction, getCategories } from "@/api/transactions";
|
||||
import type { Transaction } from "@/api/transactions";
|
||||
import { getAccounts } from "@/api/accounts";
|
||||
import { parseReceiptFile } from "@/api/settings";
|
||||
import type { ParsedReceipt } from "@/api/settings";
|
||||
import { formatCurrency } from "@/utils/currency";
|
||||
import { cn } from "@/utils/cn";
|
||||
import { format, startOfMonth, subMonths, startOfYear } from "date-fns";
|
||||
import {
|
||||
Plus, Trash2, Search, ChevronLeft, ChevronRight, Upload,
|
||||
ArrowUpCircle, ArrowDownCircle, ArrowLeftRight, TrendingUp, Paperclip, RefreshCw
|
||||
ArrowUpCircle, ArrowDownCircle, ArrowLeftRight, TrendingUp, Paperclip, RefreshCw, ScanLine, Loader2,
|
||||
} from "lucide-react";
|
||||
import TransactionFormModal from "./TransactionFormModal";
|
||||
import type { TransactionInitialValues } from "./TransactionFormModal";
|
||||
import TransactionDetailDrawer from "./TransactionDetailDrawer";
|
||||
import { Link } from "react-router-dom";
|
||||
|
||||
|
|
@ -49,6 +52,10 @@ export default function TransactionList() {
|
|||
const qc = useQueryClient();
|
||||
const [showForm, setShowForm] = useState(false);
|
||||
const [selectedTxn, setSelectedTxn] = useState<Transaction | null>(null);
|
||||
const [receiptParsed, setReceiptParsed] = useState<ParsedReceipt | null>(null);
|
||||
const [scanError, setScanError] = useState<string | null>(null);
|
||||
const [scanning, setScanning] = useState(false);
|
||||
const receiptInputRef = useRef<HTMLInputElement>(null);
|
||||
const [search, setSearch] = useState("");
|
||||
const [filterType, setFilterType] = useState("");
|
||||
const [filterAccount, setFilterAccount] = useState("");
|
||||
|
|
@ -89,9 +96,25 @@ export default function TransactionList() {
|
|||
qc.invalidateQueries({ queryKey: ["transactions"] });
|
||||
qc.invalidateQueries({ queryKey: ["accounts"] });
|
||||
setShowForm(false);
|
||||
setReceiptParsed(null);
|
||||
},
|
||||
});
|
||||
|
||||
async function handleReceiptFile(file: File) {
|
||||
setScanning(true);
|
||||
setScanError(null);
|
||||
try {
|
||||
const parsed = await parseReceiptFile(file);
|
||||
setReceiptParsed(parsed);
|
||||
setShowForm(true);
|
||||
} catch (e: any) {
|
||||
setScanError(e?.response?.data?.detail ?? "Could not parse receipt. Check your AI settings.");
|
||||
} finally {
|
||||
setScanning(false);
|
||||
if (receiptInputRef.current) receiptInputRef.current.value = "";
|
||||
}
|
||||
}
|
||||
|
||||
const accountMap = Object.fromEntries(accounts.map((a) => [a.id, a]));
|
||||
const categoryMap = Object.fromEntries(categories.map((c) => [c.id, c]));
|
||||
|
||||
|
|
@ -114,15 +137,38 @@ export default function TransactionList() {
|
|||
<span className="hidden sm:inline">Import CSV</span>
|
||||
</Link>
|
||||
<button
|
||||
onClick={() => setShowForm(true)}
|
||||
onClick={() => receiptInputRef.current?.click()}
|
||||
disabled={scanning}
|
||||
title="Scan receipt with AI"
|
||||
className="flex items-center gap-2 border border-border px-3 py-2 rounded-lg text-sm hover:bg-secondary disabled:opacity-50 transition-colors"
|
||||
>
|
||||
{scanning ? <Loader2 className="w-4 h-4 animate-spin" /> : <ScanLine className="w-4 h-4" />}
|
||||
<span className="hidden sm:inline">Scan Receipt</span>
|
||||
</button>
|
||||
<button
|
||||
onClick={() => { setReceiptParsed(null); setShowForm(true); }}
|
||||
className="flex items-center gap-2 bg-primary text-primary-foreground px-4 py-2 rounded-lg text-sm font-medium hover:bg-primary/90 transition-colors"
|
||||
>
|
||||
<Plus className="w-4 h-4" />
|
||||
<span className="hidden sm:inline">Add</span>
|
||||
</button>
|
||||
</div>
|
||||
<input
|
||||
ref={receiptInputRef}
|
||||
type="file"
|
||||
accept=".jpg,.jpeg,.png,.webp,.pdf,image/jpeg,image/png,image/webp,application/pdf"
|
||||
className="sr-only"
|
||||
onChange={(e) => { const f = e.target.files?.[0]; if (f) handleReceiptFile(f); }}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{scanError && (
|
||||
<div className="flex items-center gap-2 bg-destructive/10 border border-destructive/30 text-destructive rounded-lg px-3 py-2 text-sm">
|
||||
{scanError}
|
||||
<button onClick={() => setScanError(null)} className="ml-auto text-destructive/70 hover:text-destructive">✕</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Date presets */}
|
||||
<div className="flex gap-1 flex-wrap">
|
||||
{DATE_PRESETS.map((p) => (
|
||||
|
|
@ -301,9 +347,17 @@ export default function TransactionList() {
|
|||
<TransactionFormModal
|
||||
accounts={accounts}
|
||||
categories={categories}
|
||||
onClose={() => setShowForm(false)}
|
||||
onClose={() => { setShowForm(false); setReceiptParsed(null); }}
|
||||
onSubmit={(data) => createMutation.mutate(data)}
|
||||
isLoading={createMutation.isPending}
|
||||
initialValues={receiptParsed ? {
|
||||
description: receiptParsed.description ?? undefined,
|
||||
merchant: receiptParsed.merchant ?? undefined,
|
||||
amount: receiptParsed.amount ?? undefined,
|
||||
date: receiptParsed.date ?? undefined,
|
||||
currency: receiptParsed.currency ?? undefined,
|
||||
} : undefined}
|
||||
parsedFromReceipt={!!receiptParsed}
|
||||
/>
|
||||
)}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue