Feature/pdf to markdown agent (#6271)

Co-authored-by: James Brunton <[email protected]>

app/common/build.gradle

@@ -51,4 +51,12 @@ dependencies {
     api 'org.simplejavamail:outlook-module:8.12.6' // MSG file support
     api 'jakarta.mail:jakarta.mail-api:2.1.5'
     runtimeOnly 'org.eclipse.angus:angus-mail:2.0.5'
+
+    // Tabula table extraction — used by the shared PDF parser and directly by downstream modules.
+    // api-scoped so downstream modules (core, proprietary) retain it on their compile classpath.
+    api ('technology.tabula:tabula:1.0.5') {
+        exclude group: 'org.slf4j', module: 'slf4j-simple'
+        exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
+        exclude group: 'com.google.code.gson', module: 'gson'
+    }
 }

app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java

@@ -0,0 +1,73 @@
+package stirling.software.SPDF.pdf.parser;
+
+import static stirling.software.SPDF.pdf.parser.PdfModels.*;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Service;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link
+ * LineAlignmentTableParser}. The first parser returning a result above {@link
+ * #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page.
+ */
+@Service
+@Primary
+@RequiredArgsConstructor
+@Slf4j
+public class CompositeTableParser implements TableParser {
+
+    /** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */
+    static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f;
+
+    private final TabulaTableParser tabulaParser;
+    private final LineAlignmentTableParser lineAlignmentParser;
+
+    @Override
+    public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
+        // Step 1: Tabula lattice mode (ruled/bordered tables).
+        List<TableFragment> latticeResults = filterConfident(tabulaParser.parse(document, rawPage));
+        if (!latticeResults.isEmpty()) {
+            log.debug(
+                    "Page {}: using Tabula lattice ({} table(s))",
+                    rawPage.pageNumber(),
+                    latticeResults.size());
+            return latticeResults;
+        }
+
+        // Step 2: Tabula stream mode (borderless/whitespace-delimited tables).
+        // parseStream is not on the TableParser interface — this intentionally couples to the
+        // concrete TabulaTableParser since stream mode is a Tabula-specific concept.
+        List<TableFragment> streamResults =
+                filterConfident(tabulaParser.parseStream(document, rawPage));
+        if (!streamResults.isEmpty()) {
+            log.debug(
+                    "Page {}: using Tabula stream ({} table(s))",
+                    rawPage.pageNumber(),
+                    streamResults.size());
+            return streamResults;
+        }
+
+        // Step 3: Geometry-based line-alignment fallback.
+        List<TableFragment> lineResults = lineAlignmentParser.parse(document, rawPage);
+        if (!lineResults.isEmpty()) {
+            log.debug(
+                    "Page {}: using LineAlignment ({} table(s))",
+                    rawPage.pageNumber(),
+                    lineResults.size());
+            return lineResults;
+        }
+
+        return List.of();
+    }
+
+    private List<TableFragment> filterConfident(List<TableFragment> tables) {
+        return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList();
+    }
+}

app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java

@@ -0,0 +1,528 @@
+package stirling.software.SPDF.pdf.parser;
+
+import static stirling.software.SPDF.pdf.parser.PdfModels.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.TreeMap;
+import java.util.regex.Pattern;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.springframework.stereotype.Service;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Fallback {@link TableParser} for borderless financial tables using text geometry.
+ *
+ * <p>Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge
+ * positions, groups vertically proximate anchor lines into table candidates, then scores each group
+ * on column consistency and anchor density (confidence ceiling 0.85).
+ */
+@Service
+@Slf4j
+public class LineAlignmentTableParser implements TableParser {
+
+    /** Width in points of each column position bucket. */
+    static final float COLUMN_BUCKET_PT = 5f;
+
+    /** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */
+    private static final int COLUMN_MATCH_BUCKETS = 2;
+
+    /** Maximum gap (as a multiple of modal line spacing) before splitting a group. */
+    private static final float MAX_GAP_FACTOR = 2.5f;
+
+    /** Minimum anchor rows (numeric-heavy) to form a valid table. */
+    static final int MIN_TABLE_ROWS = 3;
+
+    /** Minimum confirmed column positions to form a valid table. */
+    static final int MIN_COLUMNS = 2;
+
+    /**
+     * Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A
+     * rows).
+     */
+    private static final double COLUMN_MIN_FREQUENCY = 0.40;
+
+    /**
+     * Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency,
+     * percent, nil dashes.
+     */
+    private static final Pattern NUMERIC =
+            Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$");
+
+    /**
+     * Lines within this y-distance are merged into one row (restores rows split by LineBuilder's
+     * column-gap logic).
+     */
+    static final float ROW_MERGE_TOLERANCE_PT = 2f;
+
+    // ── public API ───────────────────────────────────────────────────────────────────────────────
+
+    @Override
+    public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
+        List<RawLine> lines = rawPage.lines();
+        if (lines.size() < MIN_TABLE_ROWS) return List.of();
+
+        float modalSpacing = computeModalSpacing(lines);
+        List<TokenizedLine> tokenized =
+                mergeCoincidentLines(lines.stream().map(this::tokenize).toList());
+
+        List<TokenizedLine> anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList();
+
+        if (anchors.size() < MIN_TABLE_ROWS) return List.of();
+
+        List<Float> columnGrid = buildColumnGrid(anchors);
+        if (columnGrid.size() < MIN_COLUMNS) {
+            log.debug(
+                    "Page {}: LineAlignment — fewer than {} confirmed columns, skipping",
+                    rawPage.pageNumber(),
+                    MIN_COLUMNS);
+            return List.of();
+        }
+
+        List<List<TokenizedLine>> groups = groupRows(tokenized, columnGrid, modalSpacing);
+
+        List<TableFragment> results = new ArrayList<>();
+        for (int i = 0; i < groups.size(); i++) {
+            buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i)
+                    .ifPresent(results::add);
+        }
+
+        log.debug(
+                "Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)",
+                rawPage.pageNumber(),
+                results.size(),
+                anchors.size(),
+                columnGrid.size());
+        return results;
+    }
+
+    // ── coincident-line merging ──────────────────────────────────────────────────────────────────
+
+    /**
+     * Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves
+     * split by LineBuilder.
+     */
+    List<TokenizedLine> mergeCoincidentLines(List<TokenizedLine> tokenized) {
+        if (tokenized.size() < 2) return tokenized;
+
+        List<TokenizedLine> result = new ArrayList<>();
+        int i = 0;
+
+        while (i < tokenized.size()) {
+            float baseY = tokenized.get(i).line().bounds().y();
+            int j = i + 1;
+            while (j < tokenized.size()
+                    && Math.abs(tokenized.get(j).line().bounds().y() - baseY)
+                            <= ROW_MERGE_TOLERANCE_PT) {
+                j++;
+            }
+
+            if (j == i + 1) {
+                result.add(tokenized.get(i));
+            } else {
+                result.add(mergeGroup(tokenized.subList(i, j)));
+            }
+            i = j;
+        }
+
+        return result;
+    }
+
+    private TokenizedLine mergeGroup(List<TokenizedLine> group) {
+        List<TextFragment> mergedFragments =
+                group.stream()
+                        .flatMap(tl -> tl.line().fragments().stream())
+                        .sorted(Comparator.comparingDouble(f -> f.bounds().x()))
+                        .toList();
+
+        Bounds mergedBounds =
+                group.stream()
+                        .map(tl -> tl.line().bounds())
+                        .reduce(Bounds::merge)
+                        .orElse(group.get(0).line().bounds());
+
+        RawLine mergedLine =
+                new RawLine(
+                        group.get(0).line().lineId(),
+                        mergedFragments,
+                        mergedBounds,
+                        group.get(0).line().pageNumber());
+
+        return tokenize(mergedLine);
+    }
+
+    // ── tokenisation ─────────────────────────────────────────────────────────────────────────────
+
+    /**
+     * Splits fragments into word-level tokens; x-positions are estimated linearly within each
+     * fragment.
+     */
+    TokenizedLine tokenize(RawLine line) {
+        List<LineToken> tokens = new ArrayList<>();
+        for (TextFragment frag : line.fragments()) {
+            tokens.addAll(tokensFromFragment(frag));
+        }
+        List<LineToken> numeric = tokens.stream().filter(LineToken::numeric).toList();
+        return new TokenizedLine(line, tokens, numeric);
+    }
+
+    private List<LineToken> tokensFromFragment(TextFragment frag) {
+        String raw = frag.text();
+        if (raw == null || raw.isBlank()) return List.of();
+
+        float fragX = frag.bounds().x();
+        float fragWidth = frag.bounds().width();
+        int rawLen = raw.length();
+
+        List<LineToken> result = new ArrayList<>();
+        int offset = 0;
+        for (String part : raw.split("\\s+")) {
+            if (part.isEmpty()) {
+                offset++;
+                continue;
+            }
+            int idx = raw.indexOf(part, offset);
+            if (idx < 0) idx = offset;
+
+            float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX;
+            float tokenRight =
+                    rawLen > 0
+                            ? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth
+                            : fragX + fragWidth;
+
+            result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches()));
+            offset = idx + part.length();
+        }
+        return result;
+    }
+
+    // ── column grid ──────────────────────────────────────────────────────────────────────────────
+
+    /**
+     * Returns confirmed column right-edge positions — those appearing on ≥ {@value
+     * #COLUMN_MIN_FREQUENCY} × N anchor lines.
+     */
+    private List<Float> buildColumnGrid(List<TokenizedLine> anchors) {
+        // bucket → set of line indices that contributed a numeric token to that bucket
+        Map<Integer, List<Integer>> bucketLines = new HashMap<>();
+        for (int i = 0; i < anchors.size(); i++) {
+            for (LineToken t : anchors.get(i).numeric()) {
+                int bucket = bucket(t.right());
+                bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i);
+            }
+        }
+
+        int minHits =
+                Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY));
+
+        // Confirmed buckets → average right-edge for that bucket
+        TreeMap<Integer, Float> confirmed = new TreeMap<>();
+        for (Map.Entry<Integer, List<Integer>> entry : bucketLines.entrySet()) {
+            // Count distinct lines
+            long distinctLines = entry.getValue().stream().distinct().count();
+            if (distinctLines >= minHits) {
+                double avg =
+                        entry.getValue().stream()
+                                .distinct() // weight each line equally regardless of token count
+                                .mapToDouble(
+                                        lineIdx ->
+                                                avgRightEdgeForBucket(
+                                                        anchors, lineIdx, entry.getKey()))
+                                .average()
+                                .orElse(entry.getKey() * (double) COLUMN_BUCKET_PT);
+                confirmed.put(entry.getKey(), (float) avg);
+            }
+        }
+
+        return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right)
+    }
+
+    /**
+     * Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code
+     * targetBucket}, falling back to the bucket's nominal centre when no tokens match.
+     */
+    private double avgRightEdgeForBucket(
+            List<TokenizedLine> anchors, int lineIdx, int targetBucket) {
+        return anchors.get(lineIdx).numeric().stream()
+                .filter(t -> bucket(t.right()) == targetBucket)
+                .mapToDouble(LineToken::right)
+                .average()
+                .orElse(targetBucket * (double) COLUMN_BUCKET_PT);
+    }
+
+    // ── grouping ─────────────────────────────────────────────────────────────────────────────────
+
+    /**
+     * Groups anchor lines into table candidates, including adjacent label rows; a gap &gt;
+     * MAX_GAP_FACTOR × modal spacing splits groups.
+     */
+    private List<List<TokenizedLine>> groupRows(
+            List<TokenizedLine> all, List<Float> columnGrid, float modalSpacing) {
+        float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f;
+
+        List<List<TokenizedLine>> groups = new ArrayList<>();
+        List<TokenizedLine> current = new ArrayList<>();
+
+        for (int i = 0; i < all.size(); i++) {
+            TokenizedLine tl = all.get(i);
+            boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid);
+
+            if (current.isEmpty()) {
+                if (fits) current.add(tl);
+                continue;
+            }
+
+            float gap =
+                    tl.line().bounds().y()
+                            - current.get(current.size() - 1).line().bounds().bottom();
+
+            if (gap > maxGap) {
+                groups.add(current);
+                current = new ArrayList<>();
+                if (fits) current.add(tl);
+                continue;
+            }
+
+            if (fits) {
+                current.add(tl);
+            } else if (!tl.line().text().isBlank()) {
+                // Include non-anchor lines (labels) only if they have text and are within
+                // proximity.
+                current.add(tl);
+            }
+        }
+
+        if (!current.isEmpty()) groups.add(current);
+
+        return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList();
+    }
+
+    private boolean hasEnoughAnchorRows(List<TokenizedLine> group, List<Float> columnGrid) {
+        return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count()
+                >= MIN_TABLE_ROWS;
+    }
+
+    /** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */
+    private boolean matchesGrid(TokenizedLine tl, List<Float> columnGrid) {
+        if (tl.numeric().isEmpty()) return false;
+        long matches =
+                tl.numeric().stream()
+                        .filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
+                        .count();
+        return (double) matches / tl.numeric().size() >= 0.60;
+    }
+
+    private boolean hasInconsistentColumnMatch(TokenizedLine tl, List<Float> columnGrid) {
+        if (tl.numeric().isEmpty()) return false;
+        long hits =
+                tl.numeric().stream()
+                        .filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
+                        .count();
+        return (double) hits / tl.numeric().size() < 0.60;
+    }
+
+    // ── fragment assembly ────────────────────────────────────────────────────────────────────────
+
+    private Optional<TableFragment> buildFragment(
+            List<TokenizedLine> group, List<Float> columnGrid, int pageNumber, int tableIndex) {
+
+        long anchorCount =
+                group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
+        if (anchorCount < MIN_TABLE_ROWS) return Optional.empty();
+
+        List<String> warnings = new ArrayList<>();
+        List<List<String>> rawRows = new ArrayList<>();
+        List<TableRow> rows = new ArrayList<>();
+
+        for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) {
+            TokenizedLine tl = group.get(rowIdx);
+            List<String> rawRow = buildRawRow(tl, columnGrid);
+            rawRows.add(Collections.unmodifiableList(rawRow));
+            rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid));
+        }
+
+        // Column count = 1 label column + confirmed numeric columns
+        int colCount = columnGrid.size() + 1;
+        Bounds bounds = computeGroupBounds(group);
+        float confidence = computeConfidence(group, columnGrid, warnings);
+
+        return Optional.of(
+                new TableFragment(
+                        "tbl-la-p" + pageNumber + "-" + tableIndex,
+                        pageNumber,
+                        bounds,
+                        List.of(),
+                        Collections.unmodifiableList(rows),
+                        Collections.unmodifiableList(rawRows),
+                        colCount,
+                        confidence,
+                        Collections.unmodifiableList(warnings),
+                        null));
+    }
+
+    /**
+     * Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values.
+     */
+    private List<String> buildRawRow(TokenizedLine tl, List<Float> columnGrid) {
+        String[] cells = new String[columnGrid.size() + 1];
+        Arrays.fill(cells, "");
+
+        // Separate label tokens (those not landing in any confirmed column) from column tokens.
+        List<String> labelParts = new ArrayList<>();
+        for (LineToken token : tl.all()) {
+            int col = nearestColumnIndex(token.right(), columnGrid);
+            if (col >= 0 && token.numeric()) {
+                int cellIdx = col + 1;
+                cells[cellIdx] =
+                        cells[cellIdx].isEmpty()
+                                ? token.text()
+                                : cells[cellIdx] + " " + token.text();
+            } else {
+                labelParts.add(token.text());
+            }
+        }
+        cells[0] = String.join(" ", labelParts).trim();
+        return Arrays.asList(cells);
+    }
+
+    private TableRow buildTableRow(
+            int rowIdx, TokenizedLine tl, List<String> rawRow, List<Float> columnGrid) {
+        List<TableCell> cells = new ArrayList<>(rawRow.size());
+
+        // Label cell: use the line's full bounds as an approximation.
+        cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds()));
+
+        for (int col = 0; col < columnGrid.size(); col++) {
+            String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : "";
+            float right = columnGrid.get(col);
+            float left = col > 0 ? columnGrid.get(col - 1) : right - 50f;
+            Bounds cellBounds =
+                    new Bounds(
+                            left,
+                            tl.line().bounds().y(),
+                            right - left,
+                            tl.line().bounds().height());
+            cells.add(TableCell.of(col + 1, text, cellBounds));
+        }
+        return new TableRow(rowIdx, Collections.unmodifiableList(cells));
+    }
+
+    // ── confidence scoring ───────────────────────────────────────────────────────────────────────
+
+    /**
+     * Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at
+     * 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if
+     * &gt;30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors.
+     */
+    private float computeConfidence(
+            List<TokenizedLine> group, List<Float> columnGrid, List<String> warnings) {
+        float score = 0.70f;
+
+        long anchorCount =
+                group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
+        long totalRows = group.size();
+
+        // More columns
+        int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2);
+        score += extraCols * 0.05f;
+
+        // More anchor rows
+        if (anchorCount >= 5) score += 0.05f;
+        if (anchorCount >= 8) score += 0.05f;
+
+        // Inconsistent column matching
+        long inconsistent =
+                group.stream()
+                        .filter(TokenizedLine::isAnchor)
+                        .filter(tl -> hasInconsistentColumnMatch(tl, columnGrid))
+                        .count();
+        if (inconsistent > anchorCount * 0.30) {
+            score -= 0.15f;
+            warnings.add(
+                    "Column match inconsistent on "
+                            + inconsistent
+                            + "/"
+                            + anchorCount
+                            + " anchor rows");
+        }
+
+        // Label-heavy
+        long nonAnchor = totalRows - anchorCount;
+        if (nonAnchor > anchorCount) {
+            score -= 0.10f;
+            warnings.add(
+                    "Non-anchor rows ("
+                            + nonAnchor
+                            + ") outnumber anchor rows ("
+                            + anchorCount
+                            + ")");
+        }
+
+        return Math.max(0f, Math.min(0.85f, score));
+    }
+
+    // ── utility ──────────────────────────────────────────────────────────────────────────────────
+
+    /**
+     * Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value
+     * #COLUMN_MATCH_BUCKETS} buckets.
+     */
+    private int nearestColumnIndex(float rightEdge, List<Float> grid) {
+        int nearest = -1;
+        float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f;
+        for (int i = 0; i < grid.size(); i++) {
+            float dist = Math.abs(rightEdge - grid.get(i));
+            if (dist < minDist) {
+                minDist = dist;
+                nearest = i;
+            }
+        }
+        return nearest;
+    }
+
+    private Bounds computeGroupBounds(List<TokenizedLine> group) {
+        return group.stream()
+                .map(tl -> tl.line().bounds())
+                .reduce(Bounds::merge)
+                .orElse(new Bounds(0, 0, 0, 0));
+    }
+
+    /** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */
+    private float computeModalSpacing(List<RawLine> lines) {
+        if (lines.size() < 2) return 0f;
+        Map<Float, Long> freq = new HashMap<>();
+        for (int i = 1; i < lines.size(); i++) {
+            float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom();
+            if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum);
+        }
+        return freq.entrySet().stream()
+                .max(Map.Entry.comparingByValue())
+                .map(Map.Entry::getKey)
+                .orElse(0f);
+    }
+
+    private static int bucket(float x) {
+        return Math.round(x / COLUMN_BUCKET_PT);
+    }
+
+    // ── private data types ───────────────────────────────────────────────────────────────────────
+
+    /** A word-level token with an approximate right-edge x-position. */
+    record LineToken(String text, float x, float right, boolean numeric) {}
+
+    /** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */
+    record TokenizedLine(RawLine line, List<LineToken> all, List<LineToken> numeric) {
+        boolean isAnchor() {
+            return numeric.size() >= 2;
+        }
+    }
+}

app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java

@@ -0,0 +1,139 @@
+package stirling.software.SPDF.pdf.parser;
+
+import static stirling.software.SPDF.pdf.parser.PdfModels.*;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity.
+ *
+ * <p>Fragments are on the same line when their baselines are within a font-size-derived tolerance.
+ * A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code
+ * max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text.
+ */
+@Service
+@Slf4j
+public class LineBuilder {
+
+    /** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */
+    private static final float BASELINE_TOLERANCE_FACTOR = 0.5f;
+
+    /** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */
+    private static final float MIN_BASELINE_TOLERANCE = 2f;
+
+    /**
+     * Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below
+     * two-column gutters.
+     */
+    static final float COLUMN_GAP_RATIO = 0.10f;
+
+    /** Floor for the column-gap threshold so narrow pages don't over-split lines. */
+    static final float COLUMN_GAP_MIN_PT = 40f;
+
+    public List<RawLine> build(List<TextFragment> fragments, int pageNumber) {
+        if (fragments.isEmpty()) return List.of();
+
+        float effectiveWidth = inferEffectiveWidth(fragments);
+        float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT);
+        log.debug(
+                "LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt",
+                pageNumber,
+                effectiveWidth,
+                columnGapThreshold);
+
+        // Sort top-to-bottom first, then left-to-right within the same baseline band.
+        List<TextFragment> sorted =
+                fragments.stream()
+                        .sorted(
+                                Comparator.comparingDouble(TextFragment::baseline)
+                                        .thenComparingDouble(f -> f.bounds().x()))
+                        .toList();
+
+        List<List<TextFragment>> groups = groupByBaseline(sorted, columnGapThreshold);
+
+        List<RawLine> lines = new ArrayList<>(groups.size());
+        for (int i = 0; i < groups.size(); i++) {
+            List<TextFragment> group =
+                    groups.get(i).stream()
+                            .sorted(Comparator.comparingDouble(f -> f.bounds().x()))
+                            .toList();
+
+            Bounds lineBounds =
+                    group.stream()
+                            .map(TextFragment::bounds)
+                            .reduce(Bounds::merge)
+                            .orElse(new Bounds(0, 0, 0, 0));
+
+            lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber));
+        }
+        return lines;
+    }
+
+    private List<List<TextFragment>> groupByBaseline(
+            List<TextFragment> sorted, float columnGapThreshold) {
+        List<List<TextFragment>> groups = new ArrayList<>();
+        List<TextFragment> current = new ArrayList<>();
+        float currentBaseline = Float.NaN;
+
+        for (TextFragment fragment : sorted) {
+            if (current.isEmpty()) {
+                current.add(fragment);
+                currentBaseline = fragment.baseline();
+                continue;
+            }
+
+            float maxFontSize =
+                    Math.max(
+                            fragment.fontSize(),
+                            (float)
+                                    current.stream()
+                                            .mapToDouble(TextFragment::fontSize)
+                                            .max()
+                                            .orElse(0));
+            float tolerance =
+                    Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE);
+
+            boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance;
+            boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold);
+
+            if (sameBaseline && !columnGap) {
+                current.add(fragment);
+                // Anchor to the weighted mean baseline so long lines stay stable.
+                currentBaseline =
+                        (currentBaseline * (current.size() - 1) + fragment.baseline())
+                                / current.size();
+            } else {
+                groups.add(current);
+                current = new ArrayList<>();
+                current.add(fragment);
+                currentBaseline = fragment.baseline();
+            }
+        }
+
+        if (!current.isEmpty()) groups.add(current);
+        return groups;
+    }
+
+    /**
+     * True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code
+     * threshold}.
+     */
+    private static boolean hasColumnGap(
+            TextFragment next, List<TextFragment> group, float threshold) {
+        float lastRight = group.get(group.size() - 1).bounds().right();
+        return next.bounds().x() - lastRight > threshold;
+    }
+
+    /** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */
+    private static float inferEffectiveWidth(List<TextFragment> fragments) {
+        double maxRight =
+                fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0);
+        return (float) maxRight * 1.10f;
+    }
+}

app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java

@@ -0,0 +1,79 @@
+package stirling.software.SPDF.pdf.parser;
+
+import static stirling.software.SPDF.pdf.parser.PdfModels.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.springframework.stereotype.Service;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} →
+ * {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the
+ * {@link PDDocument} lifecycle.
+ */
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class PdfIngester {
+
+    private final LineBuilder lineBuilder;
+    private final TableParser tableParser;
+
+    public List<ParsedPage> parse(PDDocument document) throws IOException {
+        return parse(document, document.getNumberOfPages());
+    }
+
+    public List<ParsedPage> parse(PDDocument document, int maxPages) throws IOException {
+        int pageCount = Math.min(document.getNumberOfPages(), maxPages);
+        List<ParsedPage> pages = new ArrayList<>(pageCount);
+        long fragmentsMs = 0;
+        long tablesMs = 0;
+        long t0 = System.currentTimeMillis();
+
+        for (int p = 1; p <= pageCount; p++) {
+            long ft = System.currentTimeMillis();
+            List<TextFragment> fragments = extractFragments(document, p);
+            fragmentsMs += System.currentTimeMillis() - ft;
+
+            PDPage page = document.getPage(p - 1);
+            PDRectangle mediaBox = page.getMediaBox();
+            List<RawLine> lines = lineBuilder.build(fragments, p);
+            RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines);
+
+            long tt = System.currentTimeMillis();
+            List<TableFragment> tables = tableParser.parse(document, rawPage);
+            tablesMs += System.currentTimeMillis() - tt;
+
+            log.debug(
+                    "Page {}: {} fragments → {} lines, {} table(s)",
+                    p,
+                    fragments.size(),
+                    lines.size(),
+                    tables.size());
+            pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines));
+        }
+
+        log.info(
+                "[timing] parse pages={} total={}ms fragments={}ms tables={}ms",
+                pageCount,
+                System.currentTimeMillis() - t0,
+                fragmentsMs,
+                tablesMs);
+        return pages;
+    }
+
+    private List<TextFragment> extractFragments(PDDocument document, int pageNumber)
+            throws IOException {
+        WordExtractingStripper stripper = new WordExtractingStripper(pageNumber);
+        stripper.getText(document);
+        return stripper.getFragments();
+    }
+}

app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfModels.java

@@ -0,0 +1,148 @@
+package stirling.software.SPDF.pdf.parser;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+
+/**
+ * All PDF parser model types and the table-parser contract in one place.
+ *
+ * <p>Import as {@code import static stirling.software.SPDF.pdf.parser.PdfModels.*;} to use all
+ * nested types without qualification.
+ */
+public final class PdfModels {
+
+    private PdfModels() {}
+
+    // ── Geometry ──────────────────────────────────────────────────────────────
+
+    public record Bounds(float x, float y, float width, float height) {
+
+        public float right() {
+            return x + width;
+        }
+
+        public float bottom() {
+            return y + height;
+        }
+
+        public static Bounds merge(Bounds a, Bounds b) {
+            float x = Math.min(a.x, b.x);
+            float y = Math.min(a.y, b.y);
+            return new Bounds(
+                    x, y, Math.max(a.right(), b.right()) - x, Math.max(a.bottom(), b.bottom()) - y);
+        }
+    }
+
+    // ── Text fragments and lines ──────────────────────────────────────────────
+
+    /**
+     * A contiguous run of text from a single PDF content-stream string operation. {@code baseline}
+     * is preserved separately from bounds for line-grouping — characters of different sizes on the
+     * same visual line share a baseline but differ in top Y.
+     */
+    public record TextFragment(
+            String fragmentId,
+            String text,
+            Bounds bounds,
+            float baseline,
+            float fontSize,
+            String fontName,
+            boolean bold) {}
+
+    public record RawLine(
+            String lineId, List<TextFragment> fragments, Bounds bounds, int pageNumber) {
+
+        public String text() {
+            if (fragments.isEmpty()) return "";
+            StringBuilder sb = new StringBuilder();
+            TextFragment prev = null;
+            for (TextFragment f : fragments) {
+                if (prev != null) {
+                    float gap = f.bounds().x() - prev.bounds().right();
+                    float avgCharWidth = prev.bounds().width() / Math.max(prev.text().length(), 1);
+                    if (gap > avgCharWidth * 0.5f) sb.append(' ');
+                }
+                sb.append(f.text());
+                prev = f;
+            }
+            return sb.toString();
+        }
+
+        public float dominantFontSize() {
+            return fragments.stream()
+                    .collect(Collectors.groupingBy(TextFragment::fontSize, Collectors.counting()))
+                    .entrySet()
+                    .stream()
+                    .max(Map.Entry.comparingByValue())
+                    .map(Map.Entry::getKey)
+                    .orElse(0f);
+        }
+
+        public boolean hasBold() {
+            return fragments.stream().anyMatch(TextFragment::bold);
+        }
+    }
+
+    public record RawPage(int pageNumber, float widthPt, float heightPt, List<RawLine> lines) {}
+
+    // ── Table model ───────────────────────────────────────────────────────────
+
+    /**
+     * A single cell within a {@link TableRow}. {@code colSpan} and {@code rowSpan} are always 1 in
+     * v1 — span detection is deferred but the fields keep the schema stable.
+     */
+    public record TableCell(int colIndex, String text, Bounds bounds, int colSpan, int rowSpan) {
+
+        public static TableCell of(int colIndex, String text, Bounds bounds) {
+            return new TableCell(colIndex, text, bounds, 1, 1);
+        }
+    }
+
+    public record TableRow(int rowIndex, List<TableCell> cells) {}
+
+    /**
+     * A table as it appears on a single page. {@code headers} is empty in v1 — all rows are in
+     * {@code rows}. {@code rawRows} preserves exact Tabula text output for diagnostics. {@code
+     * confidence} is a heuristic score in [0.0, 1.0]. {@code continuedFromPage} is null in v1.
+     */
+    public record TableFragment(
+            String tableId,
+            int pageNumber,
+            Bounds bounds,
+            List<TableRow> headers,
+            List<TableRow> rows,
+            List<List<String>> rawRows,
+            int columnCount,
+            float confidence,
+            List<String> warnings,
+            Integer continuedFromPage) {}
+
+    // ── Page output ───────────────────────────────────────────────────────────
+
+    public record ParsedPage(
+            int pageNumber,
+            float widthPt,
+            float heightPt,
+            List<TableFragment> tables,
+            List<RawLine> layoutLines) {}
+
+    // ── Parser contract ───────────────────────────────────────────────────────
+
+    /**
+     * Extracts tables from a single page of a PDF. The caller owns the document lifecycle;
+     * implementations must not close it.
+     */
+    public interface TableParser {
+
+        /**
+         * @param document open PDF; must not be closed by the implementation
+         * @param rawPage page metadata and lines for the page to process
+         * @return zero or more table fragments found on the page, never null
+         */
+        List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException;
+    }
+}

app/common/src/main/java/stirling/software/SPDF/pdf/parser/TabulaTableParser.java

@@ -0,0 +1,256 @@
+package stirling.software.SPDF.pdf.parser;
+
+import static stirling.software.SPDF.pdf.parser.PdfModels.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.springframework.stereotype.Service;
+
+import lombok.extern.slf4j.Slf4j;
+
+import technology.tabula.ObjectExtractor;
+import technology.tabula.Page;
+import technology.tabula.RectangularTextContainer;
+import technology.tabula.Table;
+import technology.tabula.extractors.BasicExtractionAlgorithm;
+import technology.tabula.extractors.ExtractionAlgorithm;
+import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
+
+/**
+ * Primary {@link TableParser} implementation using Tabula's lattice-mode extraction.
+ *
+ * <h3>Algorithm</h3>
+ *
+ * Uses {@link SpreadsheetExtractionAlgorithm} (lattice mode), which detects tables from ruled lines
+ * (horizontal and vertical PDF path operators). This is reliable for tables with visible borders.
+ * Borderless or whitespace-delimited tables will not be detected — that requires stream mode, which
+ * is deferred to a later iteration.
+ *
+ * <h3>Coordinate system</h3>
+ *
+ * Tabula normalises page coordinates so that (0,0) is the top-left corner and Y increases downward,
+ * matching the {@link Bounds} convention used throughout this parser. No coordinate transformation
+ * is needed when mapping Tabula cell bounds to {@link Bounds}.
+ *
+ * <h3>Known limitations</h3>
+ *
+ * <ul>
+ *   <li>Borderless tables are not detected (use stream mode, deferred).
+ *   <li>Colspan and rowspan are not detected; all cells report colSpan=1, rowSpan=1.
+ *   <li>Header rows are not identified; all rows appear in {@code rows}, headers is empty.
+ *   <li>Cross-page table linking is not performed; each page is independent.
+ *   <li>Rotated tables (90°/270° pages) may produce incorrect bounds.
+ * </ul>
+ */
+@Service
+@Slf4j
+public class TabulaTableParser implements TableParser {
+
+    /** Lattice mode — reliable for tables with visible ruled borders. */
+    @Override
+    public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
+        return parseWithAlgorithm(
+                document, rawPage, new SpreadsheetExtractionAlgorithm(), "lattice");
+    }
+
+    /**
+     * Convenience overload for callers that only have a page number, not a full {@link RawPage}.
+     */
+    public List<TableFragment> parse(PDDocument document, int pageNumber) throws IOException {
+        return parse(document, new RawPage(pageNumber, 0f, 0f, List.of()));
+    }
+
+    /** Stream mode — whitespace-based column detection for borderless tables. */
+    public List<TableFragment> parseStream(PDDocument document, RawPage rawPage)
+            throws IOException {
+        return parseWithAlgorithm(document, rawPage, new BasicExtractionAlgorithm(), "stream");
+    }
+
+    private List<TableFragment> parseWithAlgorithm(
+            PDDocument document, RawPage rawPage, ExtractionAlgorithm algorithm, String modeName)
+            throws IOException {
+        int pageNumber = rawPage.pageNumber();
+
+        List<Table> tabulaTables;
+        try {
+            // Do NOT use try-with-resources: ObjectExtractor.close() closes the underlying
+            // PDDocument, which we don't own. The extractor holds no resources of its own.
+            ObjectExtractor extractor = new ObjectExtractor(document);
+            Page page = extractor.extract(pageNumber);
+            tabulaTables = new ArrayList<>(algorithm.extract(page));
+        } catch (Exception e) {
+            log.warn(
+                    "Tabula {} extraction failed on page {}: {}",
+                    modeName,
+                    pageNumber,
+                    e.getMessage());
+            return List.of();
+        }
+
+        if (tabulaTables.isEmpty()) {
+            log.debug("Page {}: no tables detected by Tabula ({})", pageNumber, modeName);
+            return List.of();
+        }
+
+        log.debug(
+                "Page {}: Tabula ({}) detected {} table(s)",
+                pageNumber,
+                modeName,
+                tabulaTables.size());
+
+        List<TableFragment> fragments = new ArrayList<>(tabulaTables.size());
+        for (int i = 0; i < tabulaTables.size(); i++) {
+            fragments.add(toFragment(tabulaTables.get(i), pageNumber, i));
+        }
+        return fragments;
+    }
+
+    // ── private helpers ──────────────────────────────────────────────────────────────────────────
+
+    private TableFragment toFragment(Table table, int pageNumber, int tableIndex) {
+        List<List<RectangularTextContainer>> tabulaRows = table.getRows();
+        List<String> warnings = new ArrayList<>();
+
+        List<List<String>> rawRows = buildRawRows(tabulaRows);
+        int colCount = inferColumnCount(rawRows, warnings);
+        List<TableRow> rows = buildRows(tabulaRows, colCount, warnings);
+        float confidence = computeConfidence(rawRows, colCount, warnings);
+        Bounds bounds = tableBounds(table);
+
+        String tableId = "tbl-p" + pageNumber + "-" + tableIndex;
+
+        if (!warnings.isEmpty()) {
+            log.warn("Page {}, table {}: {}", pageNumber, tableIndex, warnings);
+        }
+
+        return new TableFragment(
+                tableId,
+                pageNumber,
+                bounds,
+                List.of(), // headers: deferred to v2
+                rows,
+                rawRows,
+                colCount,
+                confidence,
+                Collections.unmodifiableList(warnings),
+                null); // continuedFromPage: deferred to v2
+    }
+
+    private List<List<String>> buildRawRows(List<List<RectangularTextContainer>> tabulaRows) {
+        List<List<String>> rawRows = new ArrayList<>(tabulaRows.size());
+        for (List<RectangularTextContainer> tabulaRow : tabulaRows) {
+            List<String> cells = new ArrayList<>(tabulaRow.size());
+            for (RectangularTextContainer cell : tabulaRow) {
+                cells.add(normaliseText(cell.getText()));
+            }
+            rawRows.add(Collections.unmodifiableList(cells));
+        }
+        return rawRows;
+    }
+
+    private List<TableRow> buildRows(
+            List<List<RectangularTextContainer>> tabulaRows, int colCount, List<String> warnings) {
+        List<TableRow> rows = new ArrayList<>(tabulaRows.size());
+        for (int rowIdx = 0; rowIdx < tabulaRows.size(); rowIdx++) {
+            List<RectangularTextContainer> tabulaRow = tabulaRows.get(rowIdx);
+            List<TableCell> cells = new ArrayList<>(tabulaRow.size());
+
+            for (int colIdx = 0; colIdx < tabulaRow.size(); colIdx++) {
+                RectangularTextContainer c = tabulaRow.get(colIdx);
+                Bounds cellBounds =
+                        new Bounds(
+                                (float) c.getX(),
+                                (float) c.getY(),
+                                (float) c.getWidth(),
+                                (float) c.getHeight());
+                cells.add(TableCell.of(colIdx, normaliseText(c.getText()), cellBounds));
+            }
+
+            if (tabulaRow.size() != colCount) {
+                warnings.add(
+                        "Row "
+                                + rowIdx
+                                + " has "
+                                + tabulaRow.size()
+                                + " cells; expected "
+                                + colCount);
+            }
+
+            rows.add(new TableRow(rowIdx, Collections.unmodifiableList(cells)));
+        }
+        return rows;
+    }
+
+    /**
+     * The canonical column count for a table is the size of the widest row. Tabula can produce
+     * uneven rows when a cell's ruling lines are partially missing.
+     */
+    private int inferColumnCount(List<List<String>> rawRows, List<String> warnings) {
+        if (rawRows.isEmpty()) return 0;
+        int max = rawRows.stream().mapToInt(List::size).max().orElse(0);
+        int mode =
+                rawRows.stream()
+                        .collect(
+                                java.util.stream.Collectors.groupingBy(
+                                        List::size, java.util.stream.Collectors.counting()))
+                        .entrySet()
+                        .stream()
+                        .max(java.util.Map.Entry.comparingByValue())
+                        .map(java.util.Map.Entry::getKey)
+                        .orElse(0);
+        if (max != mode) {
+            warnings.add("Inconsistent column count: modal=" + mode + " max=" + max);
+        }
+        return mode > 0 ? mode : max;
+    }
+
+    /**
+     * Heuristic confidence score in [0.0, 1.0].
+     *
+     * <ul>
+     *   <li>Starts at 1.0.
+     *   <li>-0.3 if only one column (single-column tables are usually not real tables).
+     *   <li>-0.1 per row with an inconsistent column count, capped at -0.4.
+     *   <li>-0.3 if the empty-cell ratio across all cells exceeds 80%.
+     * </ul>
+     */
+    private float computeConfidence(
+            List<List<String>> rawRows, int colCount, List<String> warnings) {
+        if (rawRows.isEmpty() || colCount == 0) return 0f;
+
+        float score = 1.0f;
+
+        if (colCount == 1) score -= 0.3f;
+
+        long inconsistentRows = rawRows.stream().filter(r -> r.size() != colCount).count();
+        score -= Math.min(inconsistentRows * 0.1f, 0.4f);
+
+        long totalCells = rawRows.stream().mapToLong(List::size).sum();
+        long emptyCells =
+                rawRows.stream().flatMap(Collection::stream).filter(String::isBlank).count();
+        if (totalCells > 0 && (float) emptyCells / totalCells > 0.8f) {
+            score -= 0.3f;
+        }
+
+        return Math.max(0f, Math.min(1f, score));
+    }
+
+    private Bounds tableBounds(Table table) {
+        return new Bounds(
+                (float) table.getX(),
+                (float) table.getY(),
+                (float) table.getWidth(),
+                (float) table.getHeight());
+    }
+
+    private String normaliseText(String raw) {
+        if (raw == null) return "";
+        // Tabula wraps multi-line cell content with \r\n — collapse to a single space.
+        return raw.replace("\r\n", " ").replace("\n", " ").replace("\r", " ").trim();
+    }
+}

app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java

@@ -0,0 +1,113 @@
+package stirling.software.SPDF.pdf.parser;
+
+import static stirling.software.SPDF.pdf.parser.PdfModels.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.font.PDFont;
+import org.apache.pdfbox.text.PDFTextStripper;
+import org.apache.pdfbox.text.TextPosition;
+
+/**
+ * Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata.
+ *
+ * <p>Overrides {@link #writeString} to split each content-stream string into word-level {@link
+ * TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in
+ * PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline.
+ */
+class WordExtractingStripper extends PDFTextStripper {
+
+    private final int targetPage;
+    private final List<TextFragment> fragments = new ArrayList<>();
+    private int fragmentIndex = 0;
+
+    WordExtractingStripper(int pageNumber) throws IOException {
+        this.targetPage = pageNumber;
+        setStartPage(pageNumber);
+        setEndPage(pageNumber);
+        setSortByPosition(true);
+    }
+
+    @Override
+    protected void startPage(PDPage page) throws IOException {
+        super.startPage(page);
+        fragments.clear();
+        fragmentIndex = 0;
+    }
+
+    @Override
+    protected void writeString(String text, List<TextPosition> textPositions) throws IOException {
+        if (text == null || text.isBlank()) return;
+
+        // Fast path: no whitespace → emit one fragment (most financial PDFs have each
+        // number as its own string operation, so this is the common case).
+        if (text.indexOf(' ') < 0) {
+            emitFragment(text, textPositions);
+            return;
+        }
+
+        // Per-word splitting requires 1:1 text-char to TextPosition correspondence.
+        // Fall back to one fragment when sizes differ (ligatures, encoding edge cases).
+        if (textPositions.size() != text.length()) {
+            emitFragment(text, textPositions);
+            return;
+        }
+
+        // Emit one TextFragment per whitespace-delimited word with accurate per-word bounds.
+        int start = 0;
+        for (int i = 0; i <= text.length(); i++) {
+            if (i == text.length() || text.charAt(i) == ' ') {
+                if (start < i) {
+                    emitFragment(text.substring(start, i), textPositions.subList(start, i));
+                }
+                start = i + 1;
+            }
+        }
+    }
+
+    private void emitFragment(String text, List<TextPosition> positions) {
+        if (positions.isEmpty()) return;
+
+        float minX = Float.MAX_VALUE;
+        float minY = Float.MAX_VALUE;
+        float maxRight = -Float.MAX_VALUE;
+        float maxBaseline = -Float.MAX_VALUE;
+        TextPosition first = null;
+
+        for (TextPosition tp : positions) {
+            if (tp == null) continue;
+            if (first == null) first = tp;
+
+            float x = tp.getX();
+            // getY() is the baseline; top of character = getY() - getHeight().
+            float top = tp.getY() - tp.getHeight();
+            float right = x + tp.getWidth();
+            float baseline = tp.getY();
+
+            minX = Math.min(minX, x);
+            minY = Math.min(minY, top);
+            maxRight = Math.max(maxRight, right);
+            maxBaseline = Math.max(maxBaseline, baseline);
+        }
+
+        if (first == null) return;
+
+        PDFont font = first.getFont();
+        String fontName = font != null ? font.getName() : "";
+        boolean bold = fontName != null && fontName.toLowerCase().contains("bold");
+        // getHeight() gives the rendered glyph height, which is the most reliable visual size.
+        float fontSize = first.getHeight();
+
+        Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY);
+        String id = "tf-p" + targetPage + "-" + fragmentIndex++;
+        fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold));
+    }
+
+    List<TextFragment> getFragments() {
+        return Collections.unmodifiableList(fragments);
+    }
+}

app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java

@@ -0,0 +1,153 @@
+package stirling.software.SPDF.pdf.parser;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static stirling.software.SPDF.pdf.parser.PdfModels.*;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and
+ * column-grid construction.
+ */
+class LineAlignmentTableParserTest {
+
+    private final LineAlignmentTableParser parser = new LineAlignmentTableParser();
+
+    // ── mergeCoincidentLines ─────────────────────────────────────────────────────────────────────
+
+    @Test
+    void mergeCoincidentLines_singleLine_unchanged() {
+        var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue")));
+        assertThat(parser.mergeCoincidentLines(lines)).hasSize(1);
+    }
+
+    @Test
+    void mergeCoincidentLines_distinctYLines_unchanged() {
+        // Two lines at different y positions — must NOT be merged.
+        var lines =
+                List.of(
+                        tokenized(rawLine(10f, 100f, "Revenue")),
+                        tokenized(rawLine(10f, 115f, "Cost")));
+        assertThat(parser.mergeCoincidentLines(lines)).hasSize(2);
+    }
+
+    @Test
+    void mergeCoincidentLines_sameY_merged() {
+        // Simulates a financial-table row split by LineBuilder at the column gap:
+        //   label fragment at x=72  → "Revenue"
+        //   value fragment at x=350 → "1,234"
+        // Both have y=100. After merge they should form one TokenizedLine.
+        var label = rawLine(72f, 100f, "Revenue");
+        var value = rawLine(350f, 100f, "1,234");
+
+        var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
+
+        assertThat(merged).hasSize(1);
+        // The merged line should contain tokens from both halves.
+        var tokens = merged.get(0).all();
+        assertThat(tokens.stream().map(t -> t.text()).toList())
+                .containsExactlyInAnyOrder("Revenue", "1,234");
+    }
+
+    @Test
+    void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() {
+        var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114
+        var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380
+
+        var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
+
+        var bounds = merged.get(0).line().bounds();
+        assertThat(bounds.x()).isEqualTo(72f);
+        assertThat(bounds.right()).isEqualTo(380f);
+    }
+
+    @Test
+    void mergeCoincidentLines_withinTolerance_merged() {
+        // Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge.
+        var a = rawLine(10f, 100.0f, "Alpha");
+        var b = rawLine(200f, 101.5f, "99");
+
+        var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
+        assertThat(merged).hasSize(1);
+    }
+
+    @Test
+    void mergeCoincidentLines_beyondTolerance_notMerged() {
+        // Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge.
+        var a = rawLine(10f, 100.0f, "Alpha");
+        var b = rawLine(200f, 103.0f, "99");
+
+        var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
+        assertThat(merged).hasSize(2);
+    }
+
+    @Test
+    void mergeCoincidentLines_threeCoincident_allMerged() {
+        // Three fragments at the same y (e.g. wide financial table with two value columns).
+        var a = rawLine(72f, 100f, "Revenue");
+        var b = rawLine(300f, 100f, "1,234");
+        var c = rawLine(400f, 100f, "5,678");
+
+        var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
+        assertThat(merged).hasSize(1);
+        assertThat(merged.get(0).all()).hasSize(3);
+    }
+
+    @Test
+    void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() {
+        var a = rawLine(72f, 100f, "Revenue");
+        var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a
+        var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate
+
+        var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
+        assertThat(merged).hasSize(2);
+    }
+
+    @Test
+    void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() {
+        // After merging, the combined line should be an anchor (≥2 numeric tokens).
+        // "Revenue" alone → not an anchor. "1,234  567" alone → anchor.
+        // Merged → anchor with at least 2 numerics.
+        var label = rawLine(72f, 100f, "Revenue");
+        var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f);
+
+        var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values)));
+
+        assertThat(merged).hasSize(1);
+        assertThat(merged.get(0).isAnchor()).isTrue();
+    }
+
+    // ── helpers ──────────────────────────────────────────────────────────────────────────────────
+
+    /** Creates a RawLine with a single TextFragment of the given text at the given position. */
+    private static RawLine rawLine(float x, float y, String text) {
+        float width = text.length() * 6f; // ~6pt per char — rough but consistent
+        float height = 12f;
+        Bounds bounds = new Bounds(x, y, width, height);
+        TextFragment fragment =
+                new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false);
+        return new RawLine("ln-test", List.of(fragment), bounds, 1);
+    }
+
+    /**
+     * Creates a RawLine with two TextFragments representing two words separated by a small gap.
+     * Used to simulate a values-only line with multiple numeric tokens.
+     */
+    private static RawLine rawLineMultiWord(
+            float x, float y, String word1, float w1, String word2, float w2) {
+        float height = 12f;
+        Bounds b1 = new Bounds(x, y, w1, height);
+        Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height);
+        TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false);
+        TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false);
+        Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height);
+        return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1);
+    }
+
+    /** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */
+    private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) {
+        return parser.tokenize(line);
+    }
+}

app/core/build.gradle

@@ -87,12 +87,6 @@ dependencies {
     implementation 'com.sun.xml.bind:jaxb-core:4.0.7'
     implementation 'org.apache.poi:poi-ooxml:5.5.1'
 
-    // https://mvnrepository.com/artifact/technology.tabula/tabula
-    implementation ('technology.tabula:tabula:1.0.5')  {
-        exclude group: 'org.slf4j', module: 'slf4j-simple'
-        exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
-        exclude group: 'com.google.code.gson', module: 'gson'
-    }
     // CVE-2022-25647: Explicit gson 2.13.2 to prevent unsafe deserialization (tabula would pull 2.8.7)
     implementation 'com.google.code.gson:gson:2.13.2'
     implementation 'org.apache.pdfbox:jbig2-imageio:3.0.4'

app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractCSVController.java

@@ -4,13 +4,13 @@ import java.io.ByteArrayOutputStream;
 import java.io.StringWriter;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
 import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.csv.QuoteMode;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.http.ContentDisposition;
@@ -26,24 +26,21 @@ import lombok.extern.slf4j.Slf4j;
 
 import stirling.software.SPDF.config.swagger.CsvConversionResponse;
 import stirling.software.SPDF.model.api.PDFWithPageNums;
-import stirling.software.SPDF.pdf.FlexibleCSVWriter;
+import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
+import stirling.software.SPDF.pdf.parser.TabulaTableParser;
 import stirling.software.common.annotations.AutoJobPostMapping;
 import stirling.software.common.annotations.api.ConvertApi;
 import stirling.software.common.service.CustomPDFDocumentFactory;
 import stirling.software.common.util.GeneralUtils;
 import stirling.software.common.util.WebResponseUtils;
 
-import technology.tabula.ObjectExtractor;
-import technology.tabula.Page;
-import technology.tabula.Table;
-import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
-
 @ConvertApi
 @Slf4j
 @RequiredArgsConstructor
 public class ExtractCSVController {
 
     private final CustomPDFDocumentFactory pdfDocumentFactory;
+    private final TabulaTableParser tabulaTableParser;
 
     @AutoJobPostMapping(value = "/pdf/csv", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
     @CsvConversionResponse
@@ -58,24 +55,23 @@ public class ExtractCSVController {
 
         try (PDDocument document = pdfDocumentFactory.load(request)) {
             List<Integer> pages = request.getPageNumbersList(document, true);
-            SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
             CSVFormat format =
                     CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
 
             for (int pageNum : pages) {
-                try (ObjectExtractor extractor = new ObjectExtractor(document)) {
-                    log.info("{}", pageNum);
-                    Page page = extractor.extract(pageNum);
-                    List<Table> tables = sea.extract(page);
+                log.info("{}", pageNum);
+                List<TableFragment> fragments = tabulaTableParser.parse(document, pageNum);
 
-                    for (int i = 0; i < tables.size(); i++) {
-                        StringWriter sw = new StringWriter();
-                        FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
-                        csvWriter.write(sw, Collections.singletonList(tables.get(i)));
-
-                        String entryName = generateEntryName(baseName, pageNum, i + 1);
-                        csvEntries.add(new CsvEntry(entryName, sw.toString()));
+                for (int i = 0; i < fragments.size(); i++) {
+                    StringWriter sw = new StringWriter();
+                    try (CSVPrinter printer = format.print(sw)) {
+                        for (List<String> row : fragments.get(i).rawRows()) {
+                            printer.printRecord(row);
+                        }
                     }
+                    csvEntries.add(
+                            new CsvEntry(
+                                    generateEntryName(baseName, pageNum, i + 1), sw.toString()));
                 }
             }
 

app/core/src/main/java/stirling/software/SPDF/pdf/FlexibleCSVWriter.java

@@ -1,16 +0,0 @@
-package stirling.software.SPDF.pdf;
-
-import org.apache.commons.csv.CSVFormat;
-
-import technology.tabula.writers.CSVWriter;
-
-public class FlexibleCSVWriter extends CSVWriter {
-
-    public FlexibleCSVWriter() {
-        super();
-    }
-
-    public FlexibleCSVWriter(CSVFormat csvFormat) {
-        super(csvFormat);
-    }
-}

app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ExtractCSVControllerTest.java

@@ -20,6 +20,7 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.mock.web.MockMultipartFile;
 
 import stirling.software.SPDF.model.api.PDFWithPageNums;
+import stirling.software.SPDF.pdf.parser.TabulaTableParser;
 import stirling.software.common.service.CustomPDFDocumentFactory;
 import stirling.software.common.util.GeneralUtils;
 
@@ -27,6 +28,7 @@ import stirling.software.common.util.GeneralUtils;
 class ExtractCSVControllerTest {
 
     @Mock private CustomPDFDocumentFactory pdfDocumentFactory;
+    @Mock private TabulaTableParser tabulaTableParser;
 
     @InjectMocks private ExtractCSVController controller;
 

app/core/src/test/java/stirling/software/SPDF/pdf/FlexibleCSVWriterTest.java

@@ -1,25 +0,0 @@
-package stirling.software.SPDF.pdf;
-
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-import org.apache.commons.csv.CSVFormat;
-import org.junit.jupiter.api.Test;
-
-class FlexibleCSVWriterTest {
-
-    @Test
-    void testDefaultConstructor() {
-        FlexibleCSVWriter writer = new FlexibleCSVWriter();
-        assertNotNull(writer, "The FlexibleCSVWriter instance should not be null");
-    }
-
-    @Test
-    void testConstructorWithCSVFormat() {
-        CSVFormat csvFormat = CSVFormat.DEFAULT;
-        FlexibleCSVWriter writer = new FlexibleCSVWriter(csvFormat);
-        assertNotNull(
-                writer,
-                "The FlexibleCSVWriter instance should not be null when initialized with"
-                        + " CSVFormat");
-    }
-}

app/proprietary/build.gradle

@@ -59,12 +59,6 @@ dependencies {
     // https://mvnrepository.com/artifact/com.bucket4j/bucket4j_jdk17
     implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion"
 
-    // Tabula table extraction — used by MathAuditorOrchestrator
-    implementation ('technology.tabula:tabula:1.0.5') {
-        exclude group: 'org.slf4j', module: 'slf4j-simple'
-        exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
-        exclude group: 'com.google.code.gson', module: 'gson'
-    }
     implementation 'com.google.code.gson:gson:2.13.2'
 
     api 'io.micrometer:micrometer-registry-prometheus'

app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java

@@ -20,7 +20,8 @@ public enum AiWorkflowOutcome {
     TOOL_CALL("tool_call"),
     COMPLETED("completed"),
     UNSUPPORTED_CAPABILITY("unsupported_capability"),
-    CANNOT_CONTINUE("cannot_continue");
+    CANNOT_CONTINUE("cannot_continue"),
+    GENERATE_FILE("generate_file");
 
     private final String value;
 

app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResponse.java

@@ -4,6 +4,8 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 import io.swagger.v3.oas.annotations.media.Schema;
 
 import lombok.Data;
@@ -20,6 +22,14 @@ public class AiWorkflowResponse {
     @Schema(description = "Answer returned by the AI workflow when applicable")
     private String answer;
 
+    @JsonProperty("content")
+    @Schema(description = "Text content to package as a file (generate_file outcomes)")
+    private String generatedContent;
+
+    @JsonProperty("filename")
+    @Schema(description = "Desired output filename for generate_file outcomes")
+    private String generatedFilename;
+
     @Schema(description = "Summary returned by the AI workflow when applicable")
     private String summary;
 

app/proprietary/src/main/java/stirling/software/proprietary/pdf/FlexibleCSVWriter.java

@@ -1,17 +0,0 @@
-package stirling.software.proprietary.pdf;
-
-import org.apache.commons.csv.CSVFormat;
-
-import technology.tabula.writers.CSVWriter;
-
-/** Exposes Tabula's protected {@link CSVWriter#CSVWriter(CSVFormat)} constructor. */
-public class FlexibleCSVWriter extends CSVWriter {
-
-    public FlexibleCSVWriter() {
-        super();
-    }
-
-    public FlexibleCSVWriter(CSVFormat csvFormat) {
-        super(csvFormat);
-    }
-}

app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java

@@ -160,6 +160,7 @@ public class AiWorkflowService {
             case TOOL_CALL -> onToolCall(response, filesById, listener);
             case PLAN -> onPlan(response, filesById, request, listener);
             case ANSWER -> onAnswer(response, filesById, request, listener);
+            case GENERATE_FILE -> onGenerateFile(response, listener);
             case NOT_FOUND,
                     NEED_CLARIFICATION,
                     CANNOT_DO,
@@ -364,6 +365,29 @@ public class AiWorkflowService {
         return new WorkflowState.Terminal(response);
     }
 
+    private WorkflowState onGenerateFile(AiWorkflowResponse response, ProgressListener listener)
+            throws IOException {
+        String content = response.getGeneratedContent();
+        String filename = response.getGeneratedFilename();
+        if (content == null || filename == null || filename.isBlank()) {
+            return new WorkflowState.Terminal(
+                    cannotContinue(
+                            "AI engine returned generate_file without content or filename."));
+        }
+        listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.PROCESSING));
+        String safeFilename = Filenames.toSimpleFileName(filename);
+        byte[] bytes = content.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        org.springframework.core.io.Resource resource =
+                new org.springframework.core.io.ByteArrayResource(bytes) {
+                    @Override
+                    public String getFilename() {
+                        return safeFilename;
+                    }
+                };
+        return new WorkflowState.Terminal(
+                buildCompletedResponse(response.getSummary(), List.of(resource), List.of(), null));
+    }
+
     @SuppressWarnings("unchecked")
     private WorkflowState runPlan(
             List<Map<String, Object>> steps,

app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java

@@ -3,7 +3,6 @@ package stirling.software.proprietary.service;
 import java.io.IOException;
 import java.io.StringWriter;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -12,6 +11,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.csv.QuoteMode;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.text.PDFTextStripper;
@@ -21,25 +21,30 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonValue;
 
 import lombok.Data;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+import stirling.software.SPDF.pdf.parser.PdfIngester;
+import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage;
+import stirling.software.SPDF.pdf.parser.PdfModels.RawLine;
+import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
+import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment;
+import stirling.software.SPDF.pdf.parser.TabulaTableParser;
 import stirling.software.common.util.ExceptionUtils;
 import stirling.software.common.util.PdfUtils;
 import stirling.software.proprietary.model.api.ai.AiPdfContentType;
 import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest;
 import stirling.software.proprietary.model.api.ai.AiWorkflowTextSelection;
 import stirling.software.proprietary.model.api.ai.FolioType;
-import stirling.software.proprietary.pdf.FlexibleCSVWriter;
-
-import technology.tabula.ObjectExtractor;
-import technology.tabula.Page;
-import technology.tabula.Table;
-import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
 
 @Slf4j
 @Service
+@RequiredArgsConstructor
 public class PdfContentExtractor {
 
+    private final TabulaTableParser tabulaTableParser;
+    private final PdfIngester pdfIngester;
+
     private static final int MAX_CHARACTERS_PER_PAGE = 4_000;
 
     private static final int TEXT_PRESENCE_THRESHOLD = 20;
@@ -101,21 +106,21 @@ public class PdfContentExtractor {
      * @return list of CSV strings (one per table), empty if no tables found
      */
     public List<String> extractTablesAsCsv(PDDocument document, int pageNumber) throws IOException {
-        SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
+        List<TableFragment> fragments = tabulaTableParser.parse(document, pageNumber);
+        if (fragments.isEmpty()) return List.of();
+
         CSVFormat format =
                 CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
         List<String> csvStrings = new ArrayList<>();
 
-        try (ObjectExtractor extractor = new ObjectExtractor(document)) {
-            Page tabulaPage = extractor.extract(pageNumber);
-            List<Table> tables = sea.extract(tabulaPage);
-
-            for (Table table : tables) {
-                StringWriter sw = new StringWriter();
-                FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
-                csvWriter.write(sw, Collections.singletonList(table));
-                csvStrings.add(sw.toString());
+        for (TableFragment fragment : fragments) {
+            StringWriter sw = new StringWriter();
+            try (CSVPrinter printer = format.print(sw)) {
+                for (List<String> row : fragment.rawRows()) {
+                    printer.printRecord(row);
+                }
             }
+            csvStrings.add(sw.toString());
         }
         return csvStrings;
     }
@@ -183,6 +188,8 @@ public class PdfContentExtractor {
             case PAGE_TEXT, FULL_TEXT ->
                     Optional.<PdfContentResult>ofNullable(
                             extractText(lf, fileReq, remainingPages, remainingCharacters));
+            case PAGE_LAYOUT ->
+                    Optional.<PdfContentResult>ofNullable(extractPageLayout(lf, remainingPages));
             default -> {
                 log.warn(
                         "Content type {} not yet implemented, skipping for {}",
@@ -207,6 +214,35 @@ public class PdfContentExtractor {
         return extracted.isEmpty() ? null : buildExtractedFileText(lf.fileName(), extracted);
     }
 
+    private PageLayoutFileResult extractPageLayout(LoadedFile lf, int maxPages) throws IOException {
+        List<ParsedPage> parsedPages = pdfIngester.parse(lf.document(), maxPages);
+        List<LayoutPage> pages = new ArrayList<>();
+        for (ParsedPage pp : parsedPages) {
+            if (pp.layoutLines().isEmpty()) continue;
+            List<LayoutLine> lines = new ArrayList<>();
+            for (RawLine rawLine : pp.layoutLines()) {
+                List<LayoutFragment> fragments = new ArrayList<>();
+                for (TextFragment tf : rawLine.fragments()) {
+                    fragments.add(
+                            new LayoutFragment(
+                                    tf.text(),
+                                    tf.bounds().x(),
+                                    tf.bounds().y(),
+                                    tf.bounds().width(),
+                                    tf.fontSize(),
+                                    tf.bold()));
+                }
+                lines.add(new LayoutLine(rawLine.bounds().y(), fragments));
+            }
+            pages.add(new LayoutPage(pp.pageNumber(), lines));
+        }
+        if (pages.isEmpty()) return null;
+        PageLayoutFileResult result = new PageLayoutFileResult();
+        result.setFileName(lf.fileName());
+        result.setPages(pages);
+        return result;
+    }
+
     private WorkflowArtifact buildArtifact(ArtifactKind kind, List<PdfContentResult> results) {
         return switch (kind) {
             case EXTRACTED_TEXT -> {
@@ -214,10 +250,12 @@ public class PdfContentExtractor {
                 artifact.setFiles(results.stream().map(ExtractedFileText.class::cast).toList());
                 yield artifact;
             }
+            case PAGE_LAYOUT -> {
+                PageLayoutArtifact artifact = new PageLayoutArtifact();
+                artifact.setFiles(results.stream().map(PageLayoutFileResult.class::cast).toList());
+                yield artifact;
+            }
             case TOOL_REPORT ->
-                    // TOOL_REPORT artifacts don't come from PDF content extraction — they're
-                    // built by AiWorkflowService from tool-response metadata. Never reached
-                    // from this code path; presence in the enum is to satisfy the switch.
                     throw new IllegalArgumentException(
                             "TOOL_REPORT artifacts are not produced by PdfContentExtractor");
         };
@@ -331,6 +369,7 @@ public class PdfContentExtractor {
      */
     enum ArtifactKind {
         EXTRACTED_TEXT("extracted_text"),
+        PAGE_LAYOUT("page_layout"),
         TOOL_REPORT("tool_report");
 
         private final String value;
@@ -394,4 +433,40 @@ public class PdfContentExtractor {
             this.report = report;
         }
     }
+
+    // Serialization contract with the Python engine — see PageLayoutArtifactContractTest.
+
+    /** One text fragment with its bounding-box geometry and font properties. */
+    record LayoutFragment(
+            String text, float x, float y, float width, float fontSize, boolean bold) {}
+
+    /** A visual line on the page: y-coordinate and all fragments on that line. */
+    record LayoutLine(float y, List<LayoutFragment> fragments) {}
+
+    /** All layout lines for a single page. */
+    record LayoutPage(int pageNumber, List<LayoutLine> lines) {}
+
+    /** Page layout data for one file, as a PdfContentResult. */
+    @Data
+    static final class PageLayoutFileResult implements PdfContentResult {
+        private String fileName;
+        private List<LayoutPage> pages = new ArrayList<>();
+
+        @Override
+        public ArtifactKind getArtifactKind() {
+            return ArtifactKind.PAGE_LAYOUT;
+        }
+
+        @Override
+        public int pagesConsumed() {
+            return pages.size();
+        }
+    }
+
+    /** Artifact carrying full spatial page layout for all input files. */
+    @Data
+    static final class PageLayoutArtifact implements WorkflowArtifact {
+        private final ArtifactKind kind = ArtifactKind.PAGE_LAYOUT;
+        private List<PageLayoutFileResult> files = new ArrayList<>();
+    }
 }

app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java

@@ -408,6 +408,31 @@ class AiWorkflowServiceTest {
         verify(internalApiClient, times(1)).post(eq(COMPRESS_ENDPOINT), any());
     }
 
+    @Test
+    void generateFileStoresContentDirectlyWithoutToolCall() throws IOException {
+        MockMultipartFile input = pdf("report.pdf", "bytes");
+        stubOrchestrator(
+                """
+                {
+                  "outcome":"generate_file",
+                  "content":"# Hello\\n\\nWorld",
+                  "filename":"report-reconstruction.md",
+                  "summary":"Reconstructed the document as a Markdown file."
+                }
+                """);
+        AtomicInteger ids = stubFileStorage();
+
+        AiWorkflowResponse result = service.orchestrate(requestFor(input, "convert to markdown"));
+
+        assertEquals(AiWorkflowOutcome.COMPLETED, result.getOutcome());
+        assertEquals(1, result.getResultFiles().size());
+        assertEquals("report-reconstruction.md", result.getResultFiles().get(0).getFileName());
+        assertEquals("file-1", result.getResultFiles().get(0).getFileId());
+        assertEquals(1, ids.get());
+        // No tool endpoint should be called — content goes directly to file storage.
+        verify(internalApiClient, never()).post(anyString(), any());
+    }
+
     @Test
     void toolCallWithoutEndpointFallsBackToCannotContinue() throws IOException {
         MockMultipartFile input = pdf("input.pdf", "bytes");

app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java

@@ -0,0 +1,66 @@
+package stirling.software.proprietary.service;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import stirling.software.proprietary.service.PdfContentExtractor.LayoutFragment;
+import stirling.software.proprietary.service.PdfContentExtractor.LayoutLine;
+import stirling.software.proprietary.service.PdfContentExtractor.LayoutPage;
+import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutArtifact;
+import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutFileResult;
+
+import tools.jackson.databind.JsonNode;
+import tools.jackson.databind.json.JsonMapper;
+
+/**
+ * Contract test: verifies that {@link PageLayoutArtifact} serializes to the JSON field names that
+ * the Python engine expects in {@code engine/src/stirling/contracts/pdf_to_markdown.py}.
+ *
+ * <p>The companion Python test in {@code tests/test_pdf_to_markdown.py} deserializes the same JSON
+ * literal and asserts field values. If either side renames a field, one of these tests fails.
+ */
+class PageLayoutArtifactContractTest {
+
+    static final String CONTRACT_JSON =
+            """
+            {"kind":"page_layout","files":[{"fileName":"test.pdf","pages":[{"pageNumber":1,"lines":[{"y":10.0,"fragments":[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}""";
+
+    @Test
+    void pageLayoutArtifact_serialisesToExpectedJson() throws Exception {
+        LayoutFragment fragment = new LayoutFragment("Hello", 1.0f, 2.0f, 30.0f, 12.0f, true);
+        LayoutLine line = new LayoutLine(10.0f, List.of(fragment));
+        LayoutPage page = new LayoutPage(1, List.of(line));
+
+        PageLayoutFileResult fileResult = new PageLayoutFileResult();
+        fileResult.setFileName("test.pdf");
+        fileResult.setPages(List.of(page));
+
+        PageLayoutArtifact artifact = new PageLayoutArtifact();
+        artifact.setFiles(List.of(fileResult));
+
+        JsonNode json = new JsonMapper().valueToTree(artifact);
+
+        assertEquals("page_layout", json.get("kind").asText());
+
+        JsonNode file = json.get("files").get(0);
+        assertEquals("test.pdf", file.get("fileName").asText());
+
+        JsonNode pg = file.get("pages").get(0);
+        assertEquals(1, pg.get("pageNumber").asInt());
+
+        JsonNode ln = pg.get("lines").get(0);
+        assertEquals(10.0, ln.get("y").asDouble(), 0.001);
+
+        JsonNode frag = ln.get("fragments").get(0);
+        assertEquals("Hello", frag.get("text").asText());
+        assertEquals(1.0, frag.get("x").asDouble(), 0.001);
+        assertEquals(2.0, frag.get("y").asDouble(), 0.001);
+        assertEquals(30.0, frag.get("width").asDouble(), 0.001);
+        assertEquals(12.0, frag.get("fontSize").asDouble(), 0.001);
+        assertTrue(frag.get("bold").asBoolean());
+    }
+}