Improve pdfoxide parser with text preprocessing

pdfparser/pdfoxide_parser.py

@@ -5,6 +5,7 @@
 It is optimized for performance and multiprocessing safety.
 """
 
+import re
 from pathlib import Path
 from typing import Any, Dict
 
@@ -13,6 +14,48 @@
 from pdfparser.utils import extract_metadata, extract_summary_totals, extract_transactions
 
 
+def preprocess_text(text: str) -> str:
+    """
+    Preprocess text extracted by pdf_oxide to separate smashed columns.
+
+    pdf_oxide often extracts text in draw-order, causing columns to be merged
+    without spaces or newlines. This function inserts newlines to separate
+    fields, especially amounts and user IDs.
+
+    Args:
+        text: Raw text extracted from PDF page
+
+    Returns:
+        Processed text with inserted newlines to separate fields
+    """
+    if not text:
+        return ""
+
+    # 1. Split smashed amounts (e.g. "0.001,000.00" -> "0.00\n1,000.00")
+    # Amounts always end in .00 in these statements
+    # Match .xx followed immediately by digit or comma
+    # Apply twice to handle chains of 3+ amounts (e.g. A.00B.00C.00)
+    # because re.sub doesn't handle overlapping matches in one pass
+    text = re.sub(r"(\.\d{2})([\d,])", r"\1\n\2", text)
+    text = re.sub(r"(\.\d{2})([\d,])", r"\1\n\2", text)
+
+    # 2. Split space-separated amounts to new lines
+    # e.g. "Description 100.00" -> "Description\n100.00"
+    # Matches space followed by number with decimal
+    text = re.sub(r" (\d[\d,]*\.\d{2})", r"\n\1", text)
+
+    # 3. Split smashed User IDs from text
+    # e.g. "BRImo8888049" -> "BRImo\n8888049"
+    # User IDs are typically 6-8 digits
+    text = re.sub(r"([a-zA-Z])(\d{6,8})(?=\s|$)", r"\1\n\2", text)
+
+    # 4. Split Date/Time from Description if on the same line
+    # e.g. "02/05/25 09:38:58 Transfer..." -> "02/05/25 09:38:58\nTransfer..."
+    text = re.sub(r"^(\d{2}/\d{2}/\d{2}\s+\d{2}:\d{2}:\d{2})\s+", r"\1\n", text, flags=re.MULTILINE)
+
+    return text
+
+
 def parse_pdf_pdfoxide(path: str) -> Dict[str, Any]:
     """
     Parse Indonesian bank statement PDF using pdf_oxide.
@@ -60,8 +103,6 @@ def parse_pdf_pdfoxide(path: str) -> Dict[str, Any]:
 
         # Fallback: extract account_no from filename if not found in text
         if not metadata.get("account_no"):
-            import re
-
             acct_match = re.search(r"(\d{10,16})", path_obj.stem)
             if acct_match:
                 metadata["account_no"] = acct_match.group(1)
@@ -70,6 +111,8 @@ def parse_pdf_pdfoxide(path: str) -> Dict[str, Any]:
         all_text = ""
         for page_num in range(page_count):
             page_text = doc.extract_text(page_num)  # type: ignore[attr-defined] or ""
+            # Preprocess the page text to fix layout issues
+            page_text = preprocess_text(page_text)
             all_text += page_text + "\n"
         transactions = extract_transactions(all_text)