Add AI receipt scanning with OCR pipeline and debug toggle
- OCR pipeline: Tesseract (images) + pdfplumber (PDFs) → AI text prompt → rule-based regex fallback; works with any text model, not just vision models - Scan Receipt toolbar button parses a photo and pre-fills the transaction form; receipt image is automatically attached to the created transaction - AI settings page: provider, API key (AES-256-GCM encrypted), custom URL, model, and per-user debug toggle that gates the OCR/AI debug panel - Fix CSRF cookie secure=False so HTTP deployments work; add 7-day max_age - Fix attachment_refs missing from _to_response (attachments never appeared in UI) - Fix multipart boundary lost when Content-Type was set manually in axios calls - nginx: raise client_max_body_size to 15 MB, add 120s proxy timeout for OCR - Migration 0005: add ai_debug boolean to users table - Update README and CLAUDE.md with AI scanning docs and architecture notes Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
a7c54ca61c
commit
26e2a055db
16 changed files with 397 additions and 99 deletions
|
|
@ -5,6 +5,7 @@ export interface AiSettings {
|
|||
has_api_key: boolean;
|
||||
base_url: string | null;
|
||||
model: string | null;
|
||||
debug: boolean;
|
||||
}
|
||||
|
||||
export interface AiSettingsSave {
|
||||
|
|
@ -12,6 +13,7 @@ export interface AiSettingsSave {
|
|||
api_key?: string;
|
||||
base_url?: string;
|
||||
model?: string;
|
||||
debug?: boolean;
|
||||
}
|
||||
|
||||
export interface ParsedReceipt {
|
||||
|
|
@ -22,6 +24,7 @@ export interface ParsedReceipt {
|
|||
description: string | null;
|
||||
category: string | null;
|
||||
raw: string | null;
|
||||
ocr_text: string | null;
|
||||
}
|
||||
|
||||
export async function getAiSettings(): Promise<AiSettings> {
|
||||
|
|
@ -51,8 +54,7 @@ export async function parseReceipt(txnId: string, attachmentId: string): Promise
|
|||
export async function parseReceiptFile(file: File): Promise<ParsedReceipt> {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
const { data } = await api.post("/transactions/parse-receipt", form, {
|
||||
headers: { "Content-Type": "multipart/form-data" },
|
||||
});
|
||||
// Do NOT set Content-Type manually — axios sets it with the multipart boundary automatically
|
||||
const { data } = await api.post("/transactions/parse-receipt", form);
|
||||
return data;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -113,9 +113,8 @@ export async function importCsv(
|
|||
export async function uploadAttachment(txnId: string, file: File): Promise<AttachmentRef> {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
const res = await api.post(`/transactions/${txnId}/attachments`, form, {
|
||||
headers: { "Content-Type": "multipart/form-data" },
|
||||
});
|
||||
// Do NOT set Content-Type manually — axios sets it with the multipart boundary automatically
|
||||
const res = await api.post(`/transactions/${txnId}/attachments`, form);
|
||||
return res.data;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -660,6 +660,7 @@ function AiSection() {
|
|||
const [apiKey, setApiKey] = useState("");
|
||||
const [baseUrl, setBaseUrl] = useState("");
|
||||
const [model, setModel] = useState("");
|
||||
const [debug, setDebug] = useState(false);
|
||||
const [showKey, setShowKey] = useState(false);
|
||||
const [success, setSuccess] = useState("");
|
||||
|
||||
|
|
@ -670,6 +671,7 @@ function AiSection() {
|
|||
if (d.provider) setProvider(d.provider);
|
||||
if (d.base_url) setBaseUrl(d.base_url);
|
||||
if (d.model) setModel(d.model);
|
||||
setDebug(d.debug);
|
||||
return d;
|
||||
},
|
||||
});
|
||||
|
|
@ -680,6 +682,7 @@ function AiSection() {
|
|||
api_key: apiKey,
|
||||
base_url: baseUrl,
|
||||
model,
|
||||
debug,
|
||||
}),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ["ai-settings"] });
|
||||
|
|
@ -800,6 +803,28 @@ function AiSection() {
|
|||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between py-1">
|
||||
<div>
|
||||
<p className="text-sm font-medium">Debug mode</p>
|
||||
<p className="text-xs text-muted-foreground">Show OCR text and raw AI responses when scanning receipts</p>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
role="switch"
|
||||
aria-checked={debug}
|
||||
onClick={() => setDebug(v => !v)}
|
||||
className={cn(
|
||||
"relative inline-flex h-6 w-11 items-center rounded-full transition-colors focus:outline-none",
|
||||
debug ? "bg-primary" : "bg-secondary border border-input"
|
||||
)}
|
||||
>
|
||||
<span className={cn(
|
||||
"inline-block h-4 w-4 transform rounded-full bg-white shadow transition-transform",
|
||||
debug ? "translate-x-6" : "translate-x-1"
|
||||
)} />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="flex gap-3 flex-wrap">
|
||||
<button
|
||||
onClick={() => saveMutation.mutate()}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@ export interface TransactionInitialValues {
|
|||
amount?: number;
|
||||
date?: string;
|
||||
currency?: string;
|
||||
raw?: string | null;
|
||||
ocr_text?: string | null;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
|
|
@ -37,9 +39,10 @@ interface Props {
|
|||
isLoading: boolean;
|
||||
initialValues?: TransactionInitialValues;
|
||||
parsedFromReceipt?: boolean;
|
||||
showAiDebug?: boolean;
|
||||
}
|
||||
|
||||
export default function TransactionFormModal({ accounts, categories, onClose, onSubmit, isLoading, initialValues, parsedFromReceipt }: Props) {
|
||||
export default function TransactionFormModal({ accounts, categories, onClose, onSubmit, isLoading, initialValues, parsedFromReceipt, showAiDebug }: Props) {
|
||||
const { register, handleSubmit, watch, formState: { errors } } = useForm<Form>({
|
||||
resolver: zodResolver(schema),
|
||||
defaultValues: {
|
||||
|
|
@ -83,12 +86,38 @@ export default function TransactionFormModal({ accounts, categories, onClose, on
|
|||
</div>
|
||||
|
||||
<form onSubmit={handleSubmit(handleFormSubmit)} className="p-6 space-y-4">
|
||||
{parsedFromReceipt && (
|
||||
{parsedFromReceipt && !showAiDebug && (
|
||||
<div className="flex items-center gap-2 bg-primary/10 border border-primary/20 rounded-lg px-3 py-2 text-xs text-primary">
|
||||
<Sparkles className="w-3.5 h-3.5 shrink-0" />
|
||||
Fields pre-filled from receipt — review before saving
|
||||
</div>
|
||||
)}
|
||||
{parsedFromReceipt && showAiDebug && (
|
||||
<div className="bg-primary/5 border border-primary/20 rounded-lg p-3 space-y-2">
|
||||
<p className="text-xs font-semibold text-primary flex items-center gap-1.5">
|
||||
<Sparkles className="w-3.5 h-3.5" /> AI scan result — review before saving
|
||||
</p>
|
||||
<div className="grid grid-cols-2 gap-x-4 gap-y-1 text-xs">
|
||||
<span className="text-muted-foreground">Merchant</span><span className="font-medium truncate">{initialValues?.merchant ?? <span className="text-destructive/70 italic">not detected</span>}</span>
|
||||
<span className="text-muted-foreground">Amount</span><span className="font-medium">{initialValues?.amount != null ? initialValues.amount : <span className="text-destructive/70 italic">not detected</span>}</span>
|
||||
<span className="text-muted-foreground">Date</span><span className="font-medium">{initialValues?.date ?? <span className="text-destructive/70 italic">not detected</span>}</span>
|
||||
<span className="text-muted-foreground">Currency</span><span className="font-medium">{initialValues?.currency ?? <span className="text-destructive/70 italic">not detected</span>}</span>
|
||||
<span className="text-muted-foreground">Description</span><span className="font-medium truncate">{initialValues?.description ?? <span className="text-destructive/70 italic">not detected</span>}</span>
|
||||
</div>
|
||||
{initialValues?.ocr_text && (
|
||||
<details className="text-xs">
|
||||
<summary className="text-muted-foreground cursor-pointer hover:text-foreground select-none">OCR extracted text</summary>
|
||||
<pre className="mt-1.5 bg-background border border-border rounded p-2 text-xs overflow-x-auto whitespace-pre-wrap break-all">{initialValues.ocr_text}</pre>
|
||||
</details>
|
||||
)}
|
||||
{initialValues?.raw && (
|
||||
<details className="text-xs">
|
||||
<summary className="text-muted-foreground cursor-pointer hover:text-foreground select-none">Raw AI response</summary>
|
||||
<pre className="mt-1.5 bg-background border border-border rounded p-2 text-xs overflow-x-auto whitespace-pre-wrap break-all">{initialValues.raw}</pre>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Type */}
|
||||
<div>
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import { useRef, useState } from "react";
|
||||
import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { getTransactions, deleteTransaction, createTransaction, getCategories } from "@/api/transactions";
|
||||
import { getTransactions, deleteTransaction, createTransaction, getCategories, uploadAttachment } from "@/api/transactions";
|
||||
import type { Transaction } from "@/api/transactions";
|
||||
import { getAccounts } from "@/api/accounts";
|
||||
import { parseReceiptFile } from "@/api/settings";
|
||||
import { parseReceiptFile, getAiSettings } from "@/api/settings";
|
||||
import type { ParsedReceipt } from "@/api/settings";
|
||||
import { formatCurrency } from "@/utils/currency";
|
||||
import { cn } from "@/utils/cn";
|
||||
|
|
@ -52,6 +52,7 @@ export default function TransactionList() {
|
|||
const [showForm, setShowForm] = useState(false);
|
||||
const [selectedTxn, setSelectedTxn] = useState<Transaction | null>(null);
|
||||
const [receiptParsed, setReceiptParsed] = useState<ParsedReceipt | null>(null);
|
||||
const receiptFileRef = useRef<File | null>(null);
|
||||
const [scanError, setScanError] = useState<string | null>(null);
|
||||
const [scanning, setScanning] = useState(false);
|
||||
const receiptInputRef = useRef<HTMLInputElement>(null);
|
||||
|
|
@ -80,6 +81,7 @@ export default function TransactionList() {
|
|||
|
||||
const { data: accounts = [] } = useQuery({ queryKey: ["accounts"], queryFn: getAccounts });
|
||||
const { data: categories = [] } = useQuery({ queryKey: ["categories"], queryFn: getCategories });
|
||||
const { data: aiSettings } = useQuery({ queryKey: ["ai-settings"], queryFn: getAiSettings });
|
||||
|
||||
const deleteMutation = useMutation({
|
||||
mutationFn: deleteTransaction,
|
||||
|
|
@ -89,30 +91,47 @@ export default function TransactionList() {
|
|||
},
|
||||
});
|
||||
|
||||
const createMutation = useMutation({
|
||||
mutationFn: createTransaction,
|
||||
onSuccess: () => {
|
||||
const createMutation = useMutation({ mutationFn: createTransaction });
|
||||
|
||||
async function handleCreateTransaction(data: any) {
|
||||
try {
|
||||
const txn = await createMutation.mutateAsync(data);
|
||||
if (receiptFileRef.current) {
|
||||
try {
|
||||
await uploadAttachment(txn.id, receiptFileRef.current);
|
||||
} catch (e: any) {
|
||||
setScanError(`Transaction saved but receipt attachment failed: ${e?.response?.data?.detail ?? e?.message ?? "unknown error"}`);
|
||||
}
|
||||
receiptFileRef.current = null;
|
||||
}
|
||||
qc.invalidateQueries({ queryKey: ["transactions"] });
|
||||
qc.invalidateQueries({ queryKey: ["accounts"] });
|
||||
setShowForm(false);
|
||||
setReceiptParsed(null);
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
// Transaction creation failed — createMutation.error has the detail, form stays open
|
||||
}
|
||||
}
|
||||
|
||||
async function handleReceiptFile(file: File) {
|
||||
setScanning(true);
|
||||
setScanError(null);
|
||||
try {
|
||||
const parsed = await parseReceiptFile(file);
|
||||
const hasAnyField = parsed.merchant || parsed.amount || parsed.description || parsed.date;
|
||||
if (!hasAnyField && parsed.raw) {
|
||||
setScanError(`AI couldn't extract any fields. Raw response: "${parsed.raw}"`);
|
||||
} else {
|
||||
setReceiptParsed(parsed);
|
||||
setShowForm(true);
|
||||
}
|
||||
// Always open the form — the modal shows a debug panel with what was/wasn't detected
|
||||
setReceiptParsed(parsed);
|
||||
receiptFileRef.current = file;
|
||||
setShowForm(true);
|
||||
} catch (e: any) {
|
||||
setScanError(e?.response?.data?.detail ?? "Could not parse receipt. Check your AI settings.");
|
||||
const detail = e?.response?.data?.detail;
|
||||
const status = e?.response?.status;
|
||||
if (detail) {
|
||||
setScanError(typeof detail === "string" ? detail : `HTTP ${status}: ${JSON.stringify(detail)}`);
|
||||
} else if (status) {
|
||||
setScanError(`Server error ${status} — check backend logs (docker compose logs backend).`);
|
||||
} else {
|
||||
setScanError(`Network error — backend may be unreachable. ${e?.message ?? ""}`);
|
||||
}
|
||||
} finally {
|
||||
setScanning(false);
|
||||
if (receiptInputRef.current) receiptInputRef.current.value = "";
|
||||
|
|
@ -351,8 +370,8 @@ export default function TransactionList() {
|
|||
<TransactionFormModal
|
||||
accounts={accounts}
|
||||
categories={categories}
|
||||
onClose={() => { setShowForm(false); setReceiptParsed(null); }}
|
||||
onSubmit={(data) => createMutation.mutate(data)}
|
||||
onClose={() => { setShowForm(false); setReceiptParsed(null); receiptFileRef.current = null; }}
|
||||
onSubmit={handleCreateTransaction}
|
||||
isLoading={createMutation.isPending}
|
||||
initialValues={receiptParsed ? {
|
||||
description: receiptParsed.description ?? undefined,
|
||||
|
|
@ -360,8 +379,11 @@ export default function TransactionList() {
|
|||
amount: receiptParsed.amount ?? undefined,
|
||||
date: receiptParsed.date ?? undefined,
|
||||
currency: receiptParsed.currency ?? undefined,
|
||||
raw: receiptParsed.raw,
|
||||
ocr_text: receiptParsed.ocr_text,
|
||||
} : undefined}
|
||||
parsedFromReceipt={!!receiptParsed}
|
||||
showAiDebug={aiSettings?.debug ?? false}
|
||||
/>
|
||||
)}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue