feat: implement PP-Structure table extraction pipeline with GPU runtime configuration support

2026-04-27 00:51:23 +07:00
parent 9d969e61fd
commit 6d793758ff
12 changed files with 896 additions and 31 deletions
--- a/docs/FRONTEND-INTEGRATION.md
+++ b/docs/FRONTEND-INTEGRATION.md
@@ -0,0 +1,537 @@
+# Frontend Integration Guide
+
+Dokumen ini menjelaskan kontrak API yang perlu dipakai frontend untuk upload dokumen sprint, menampilkan hasil OCR, menjalankan review manual, dan approve hasil final.
+
+## Base URL
+
+Default local API:
+
+```text
+http://localhost:8000/api/v1
+```
+
+Untuk frontend, simpan URL di environment variable:
+
+```env
+VITE_OCR_API_BASE_URL=http://localhost:8000/api/v1
+```
+
+Jika `API_KEYS` di backend diisi, semua endpoint protected membutuhkan header:
+
+```http
+X-API-Key: <api-key>
+```
+
+Catatan: jangan expose API key production di frontend publik. Untuk deployment internal, gunakan reverse proxy atau session backend-for-frontend jika aksesnya tidak sepenuhnya trusted.
+
+## Health Check
+
+```http
+GET /health
+GET /health/ready
+```
+
+Contoh response `/health`:
+
+```json
+{
+  "status": "ok",
+  "version": "0.1.0"
+}
+```
+
+Contoh response `/health/ready`:
+
+```json
+{
+  "status": "ready",
+  "version": "0.1.0",
+  "models": {
+    "paddleocr": "ready",
+    "pp_structure": "disabled"
+  }
+}
+```
+
+Gunakan `/health/ready` untuk disable upload button sampai model OCR siap.
+
+## Upload Dokumen
+
+Endpoint:
+
+```http
+POST /documents
+POST /documents?sync=true
+```
+
+Body harus `multipart/form-data` dengan field `file`.
+
+Backend menerima PDF dan format image umum. Default max upload mengikuti backend config `BLOB_MAX_UPLOAD_MB`, saat ini 25 MB.
+
+### Recommended Flow
+
+Untuk frontend production, gunakan async flow:
+
+1. `POST /documents`
+2. Jika status HTTP `202`, ambil `job_id`
+3. Poll `GET /documents/{job_id}` setiap 1-3 detik
+4. Stop polling saat status `completed`, `needs_review`, atau `failed`
+
+Untuk local dev sederhana, `POST /documents?sync=true` boleh dipakai, tetapi request bisa lama karena OCR berjalan inline.
+
+### Upload Example
+
+```ts
+const API_BASE = import.meta.env.VITE_OCR_API_BASE_URL;
+const API_KEY = import.meta.env.VITE_OCR_API_KEY;
+
+async function uploadDocument(file: File) {
+  const form = new FormData();
+  form.append("file", file);
+
+  const res = await fetch(`${API_BASE}/documents`, {
+    method: "POST",
+    headers: API_KEY ? { "X-API-Key": API_KEY } : undefined,
+    body: form,
+  });
+
+  if (!res.ok) {
+    throw await readApiError(res);
+  }
+
+  return (await res.json()) as DocumentResponse;
+}
+```
+
+## Polling Job
+
+Endpoint:
+
+```http
+GET /documents/{job_id}
+```
+
+```ts
+const TERMINAL_STATUSES = new Set(["completed", "needs_review", "failed"]);
+
+async function getDocument(jobId: string) {
+  const res = await fetch(`${API_BASE}/documents/${jobId}`, {
+    headers: API_KEY ? { "X-API-Key": API_KEY } : undefined,
+  });
+
+  if (!res.ok) {
+    throw await readApiError(res);
+  }
+
+  return (await res.json()) as DocumentResponse;
+}
+
+async function pollDocument(jobId: string, onUpdate: (doc: DocumentResponse) => void) {
+  while (true) {
+    const doc = await getDocument(jobId);
+    onUpdate(doc);
+
+    if (TERMINAL_STATUSES.has(doc.status)) {
+      return doc;
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+  }
+}
+```
+
+## Response Schema
+
+### DocumentResponse
+
+```ts
+type DocumentStatus =
+  | "pending"
+  | "processing"
+  | "completed"
+  | "needs_review"
+  | "failed";
+
+type DocumentResponse = {
+  job_id: string;
+  status: DocumentStatus;
+  confidence: number | null;
+  data: ExtractionResult | null;
+  review_flags: ReviewFlag[];
+  error: string | null;
+  approved: boolean;
+  reviewed_by: string | null;
+  reviewed_at: string | null;
+};
+```
+
+### ExtractionResult
+
+```ts
+type ExtractionResult = {
+  header: HeaderFields;
+  personel: PersonnelEntry[];
+  untuk: string[];
+  ttd: Signatory;
+  raw_text: string;
+  confidence: number;
+  review_flags: ReviewFlag[];
+};
+
+type HeaderFields = {
+  nomor_sprint: string | null;
+  tanggal: string | null; // YYYY-MM-DD
+  satuan_penerbit: string | null;
+  perihal: string | null;
+  dasar: string[];
+};
+
+type PersonnelEntry = {
+  no: number | null;
+  pangkat: string | null;
+  nrp: string | null;
+  nama: string | null;
+  jabatan_dinas: string | null;
+  jabatan_sprint: string | null;
+  keterangan: string | null;
+  confidence: number;
+};
+
+type Signatory = {
+  nama: string | null;
+  pangkat: string | null;
+  nrp: string | null;
+  jabatan: string | null;
+};
+```
+
+### Review Flags
+
+```ts
+type ReviewFlag =
+  | "low_ocr_confidence"
+  | "missing_field"
+  | "invalid_nrp"
+  | "unknown_pangkat"
+  | "personnel_count_mismatch"
+  | "date_parse_failed"
+  | "llm_fallback"
+  | "llm_unavailable"
+  | "personnel_text_fallback"
+  | "personnel_text_fallback_no_nrp"
+  | "incomplete_personnel_row";
+```
+
+Recommended UI labels:
+
+| Flag | Label |
+|---|---|
+| `low_ocr_confidence` | Confidence OCR rendah |
+| `missing_field` | Field wajib belum lengkap |
+| `invalid_nrp` | NRP tidak valid |
+| `unknown_pangkat` | Pangkat tidak dikenali |
+| `personnel_count_mismatch` | Jumlah personel perlu dicek |
+| `date_parse_failed` | Tanggal gagal dibaca |
+| `llm_fallback` | Sebagian field diisi fallback LLM |
+| `llm_unavailable` | LLM tidak tersedia |
+| `personnel_text_fallback` | Personel dibaca dari fallback teks |
+| `personnel_text_fallback_no_nrp` | Personel dibaca tanpa NRP |
+| `incomplete_personnel_row` | Baris personel belum lengkap |
+
+## Example Final Response
+
+```json
+{
+  "job_id": "e21e83ed-a42c-4672-baec-914e5c60cc5a",
+  "status": "needs_review",
+  "confidence": 0.82,
+  "data": {
+    "header": {
+      "nomor_sprint": "Sprin/123/IV/2026",
+      "tanggal": "2026-04-21",
+      "satuan_penerbit": "POLRES BANJAR",
+      "perihal": "Instruktur Ops Pekat I Lodaya 2026",
+      "dasar": []
+    },
+    "personel": [
+      {
+        "no": 1,
+        "pangkat": "IPDA",
+        "nrp": "12345678",
+        "nama": "BUDI SANTOSO",
+        "jabatan_dinas": "KANIT",
+        "jabatan_sprint": "INSTRUKTUR",
+        "keterangan": null,
+        "confidence": 0.91
+      }
+    ],
+    "untuk": ["Melaksanakan kegiatan sesuai surat perintah."],
+    "ttd": {
+      "nama": "AGUS",
+      "pangkat": "AKBP",
+      "nrp": "87654321",
+      "jabatan": "KAPOLRES"
+    },
+    "raw_text": "full OCR text...",
+    "confidence": 0.82,
+    "review_flags": ["low_ocr_confidence"]
+  },
+  "review_flags": ["low_ocr_confidence"],
+  "error": null,
+  "approved": false,
+  "reviewed_by": null,
+  "reviewed_at": null
+}
+```
+
+`raw_text` bisa panjang. Tampilkan di collapsible/debug panel, bukan di layar utama.
+
+## Review dan Koreksi HITL
+
+Frontend review screen sebaiknya mengizinkan editor untuk:
+
+- Header: nomor sprint, tanggal, satuan penerbit, perihal, dasar
+- Personel: pangkat, NRP, nama, jabatan dinas, jabatan sprint, keterangan
+- Untuk: daftar tugas
+- TTD: nama, pangkat, NRP, jabatan
+
+### Patch Corrections
+
+Endpoint:
+
+```http
+PATCH /documents/{job_id}
+```
+
+Body:
+
+```json
+{
+  "corrections": [
+    {
+      "path": "header.perihal",
+      "value": "Pelaksanaan Operasi Pekat I Lodaya 2026",
+      "reason": "OCR membaca perihal tidak lengkap"
+    },
+    {
+      "path": "personel[0].nama",
+      "value": "BUDI SANTOSO",
+      "reason": "Perbaikan nama"
+    }
+  ]
+}
+```
+
+Header opsional untuk audit trail:
+
+```http
+X-User-Id: reviewer-a
+```
+
+Path yang umum dipakai:
+
+```text
+header.nomor_sprint
+header.tanggal
+header.satuan_penerbit
+header.perihal
+header.dasar
+ttd.nama
+ttd.pangkat
+ttd.nrp
+ttd.jabatan
+personel[0].pangkat
+personel[0].nrp
+personel[0].nama
+personel[0].jabatan_dinas
+personel[0].jabatan_sprint
+personel[0].keterangan
+untuk
+```
+
+Semua correction dalam satu request bersifat atomic. Jika satu path invalid, seluruh batch ditolak dan tidak ada perubahan disimpan.
+
+### Patch Example
+
+```ts
+async function patchDocument(jobId: string, corrections: FieldCorrection[], userId?: string) {
+  const headers: Record<string, string> = { "Content-Type": "application/json" };
+  if (API_KEY) headers["X-API-Key"] = API_KEY;
+  if (userId) headers["X-User-Id"] = userId;
+
+  const res = await fetch(`${API_BASE}/documents/${jobId}`, {
+    method: "PATCH",
+    headers,
+    body: JSON.stringify({ corrections }),
+  });
+
+  if (!res.ok) {
+    throw await readApiError(res);
+  }
+
+  return (await res.json()) as DocumentResponse;
+}
+
+type FieldCorrection = {
+  path: string;
+  value: unknown;
+  reason?: string | null;
+};
+```
+
+## Correction History
+
+Endpoint:
+
+```http
+GET /documents/{job_id}/history
+```
+
+Response:
+
+```ts
+type CorrectionEventResponse = {
+  id: number;
+  job_id: string;
+  field_path: string;
+  old_value: unknown | null;
+  new_value: unknown | null;
+  corrected_by: string | null;
+  reason: string | null;
+  corrected_at: string;
+};
+```
+
+Gunakan endpoint ini untuk audit panel di halaman review.
+
+## Approve Final Result
+
+Endpoint:
+
+```http
+POST /documents/{job_id}/approve
+```
+
+Header opsional:
+
+```http
+X-User-Id: reviewer-a
+```
+
+Response:
+
+```json
+{
+  "job_id": "e21e83ed-a42c-4672-baec-914e5c60cc5a",
+  "approved": true,
+  "reviewed_by": "reviewer-a",
+  "reviewed_at": "2026-04-26T16:30:00"
+}
+```
+
+Setelah approved, `PATCH /documents/{job_id}` akan ditolak dengan `409`.
+
+## Error Handling
+
+Application errors:
+
+```json
+{
+  "error": "UnsupportedDocumentError",
+  "message": "Uploaded file is empty."
+}
+```
+
+FastAPI validation errors memakai shape standar:
+
+```json
+{
+  "detail": [
+    {
+      "type": "missing",
+      "loc": ["body", "file"],
+      "msg": "Field required"
+    }
+  ]
+}
+```
+
+Helper error:
+
+```ts
+async function readApiError(res: Response) {
+  let payload: unknown = null;
+  try {
+    payload = await res.json();
+  } catch {
+    payload = await res.text();
+  }
+
+  return {
+    status: res.status,
+    payload,
+  };
+}
+```
+
+Recommended UI handling:
+
+| HTTP Status | UI Handling |
+|---|---|
+| `400` | Tampilkan pesan validasi/upload |
+| `401` | Session/API key tidak valid |
+| `404` | Job tidak ditemukan |
+| `409` | Job belum selesai atau sudah approved |
+| `422` | Form correction tidak valid |
+| `500` | Tampilkan error umum dan minta operator cek log backend |
+
+## Ground Truth Admin
+
+Endpoint ini opsional untuk dashboard admin/training data.
+
+```http
+GET /ground-truth/stats?top_n=10
+GET /ground-truth/export?approved_only=true&has_corrections=true&limit=1000
+```
+
+`/ground-truth/export` mengembalikan `application/x-ndjson`, satu JSON per baris. Frontend biasanya cukup menyediakan tombol download, bukan parse seluruh stream di browser.
+
+## Recommended Screens
+
+1. Upload screen
+   - Dropzone file PDF/image
+   - Health readiness badge
+   - Upload progress
+   - Processing state setelah `job_id` diterima
+
+2. Result screen
+   - Status badge
+   - Confidence score
+   - Review flags
+   - Header summary
+   - Personnel table
+   - Untuk list
+   - TTD section
+   - Raw OCR collapsible
+
+3. Review screen
+   - Editable fields
+   - Dirty-state tracking
+   - Correction reason input
+   - Save corrections via `PATCH`
+   - History panel
+   - Approve button
+
+4. Admin screen
+   - Health/ready status
+   - Ground-truth stats
+   - Export approved samples
+
+## UX Rules
+
+- Jangan tunggu `POST /documents?sync=true` untuk production UI; gunakan async + polling.
+- Disable approve kalau status masih `pending` atau `processing`.
+- Tampilkan `needs_review` sebagai hasil yang berhasil diproses tetapi perlu validasi manusia.
+- Jangan render `raw_text` sebagai konten utama.
+- Pada `failed`, tampilkan `error` dari response jika ada.
+- Pada confidence rendah, arahkan user ke review fields yang punya flag terkait.
--- a/docs/OCR-RUNTIME-MODES.md
+++ b/docs/OCR-RUNTIME-MODES.md
@@ -0,0 +1,49 @@
+# OCR Runtime Modes
+
+Backend OCR bisa dijalankan dalam mode CPU atau GPU lewat konfigurasi `OCR_USE_GPU`.
+
+## Cara Pakai
+
+Mode CPU:
+
+```powershell
+.\update.ps1 -OcrMode cpu
+```
+
+Mode GPU:
+
+```powershell
+.\update.ps1 -OcrMode gpu
+```
+
+Jika parameter tidak diberikan, `update.ps1` memakai nilai yang sudah ada di `.env`.
+
+```env
+OCR_USE_GPU=false
+```
+
+atau:
+
+```env
+OCR_USE_GPU=true
+```
+
+## Perilaku Script
+
+- `-OcrMode cpu` menyimpan `OCR_USE_GPU=false` ke `.env`.
+- `-OcrMode gpu` menyimpan `OCR_USE_GPU=true` ke `.env`.
+- Script tidak menghapus package Paddle/CUDA yang sudah terpasang.
+- Dalam mode GPU, script akan memasang `paddlepaddle-gpu` dan runtime cuDNN/cuBLAS jika belum ada.
+- Dalam mode CPU, script hanya memasang `paddlepaddle` CPU jika belum ada runtime Paddle sama sekali.
+
+## Catatan
+
+Mode CPU tidak membutuhkan CUDA, cuDNN, atau driver NVIDIA.
+
+Mode GPU membutuhkan NVIDIA driver dan runtime CUDA/cuDNN yang cocok. Pada Windows, backend juga menambahkan folder DLL NVIDIA dari `.venv` secara otomatis sebelum PaddleOCR diinisialisasi.
+
+`TABLES_ENABLED` adalah konfigurasi terpisah dari mode CPU/GPU. Jika PP-Structure belum stabil di environment lokal, biarkan:
+
+```env
+TABLES_ENABLED=false
+```