Feature/pdf to markdown agent (#6271)

Co-authored-by: James Brunton <[email protected]>

app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java

@@ -20,7 +20,8 @@ public enum AiWorkflowOutcome {
     TOOL_CALL("tool_call"),
     COMPLETED("completed"),
     UNSUPPORTED_CAPABILITY("unsupported_capability"),
-    CANNOT_CONTINUE("cannot_continue");
+    CANNOT_CONTINUE("cannot_continue"),
+    GENERATE_FILE("generate_file");
 
     private final String value;
 

app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResponse.java

@@ -4,6 +4,8 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 import io.swagger.v3.oas.annotations.media.Schema;
 
 import lombok.Data;
@@ -20,6 +22,14 @@ public class AiWorkflowResponse {
     @Schema(description = "Answer returned by the AI workflow when applicable")
     private String answer;
 
+    @JsonProperty("content")
+    @Schema(description = "Text content to package as a file (generate_file outcomes)")
+    private String generatedContent;
+
+    @JsonProperty("filename")
+    @Schema(description = "Desired output filename for generate_file outcomes")
+    private String generatedFilename;
+
     @Schema(description = "Summary returned by the AI workflow when applicable")
     private String summary;
 

app/proprietary/src/main/java/stirling/software/proprietary/pdf/FlexibleCSVWriter.java

@@ -1,17 +0,0 @@
-package stirling.software.proprietary.pdf;
-
-import org.apache.commons.csv.CSVFormat;
-
-import technology.tabula.writers.CSVWriter;
-
-/** Exposes Tabula's protected {@link CSVWriter#CSVWriter(CSVFormat)} constructor. */
-public class FlexibleCSVWriter extends CSVWriter {
-
-    public FlexibleCSVWriter() {
-        super();
-    }
-
-    public FlexibleCSVWriter(CSVFormat csvFormat) {
-        super(csvFormat);
-    }
-}

app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java

@@ -160,6 +160,7 @@ public class AiWorkflowService {
             case TOOL_CALL -> onToolCall(response, filesById, listener);
             case PLAN -> onPlan(response, filesById, request, listener);
             case ANSWER -> onAnswer(response, filesById, request, listener);
+            case GENERATE_FILE -> onGenerateFile(response, listener);
             case NOT_FOUND,
                     NEED_CLARIFICATION,
                     CANNOT_DO,
@@ -364,6 +365,29 @@ public class AiWorkflowService {
         return new WorkflowState.Terminal(response);
     }
 
+    private WorkflowState onGenerateFile(AiWorkflowResponse response, ProgressListener listener)
+            throws IOException {
+        String content = response.getGeneratedContent();
+        String filename = response.getGeneratedFilename();
+        if (content == null || filename == null || filename.isBlank()) {
+            return new WorkflowState.Terminal(
+                    cannotContinue(
+                            "AI engine returned generate_file without content or filename."));
+        }
+        listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.PROCESSING));
+        String safeFilename = Filenames.toSimpleFileName(filename);
+        byte[] bytes = content.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+        org.springframework.core.io.Resource resource =
+                new org.springframework.core.io.ByteArrayResource(bytes) {
+                    @Override
+                    public String getFilename() {
+                        return safeFilename;
+                    }
+                };
+        return new WorkflowState.Terminal(
+                buildCompletedResponse(response.getSummary(), List.of(resource), List.of(), null));
+    }
+
     @SuppressWarnings("unchecked")
     private WorkflowState runPlan(
             List<Map<String, Object>> steps,

app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java

@@ -3,7 +3,6 @@ package stirling.software.proprietary.service;
 import java.io.IOException;
 import java.io.StringWriter;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -12,6 +11,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.csv.QuoteMode;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.text.PDFTextStripper;
@@ -21,25 +21,30 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonValue;
 
 import lombok.Data;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+import stirling.software.SPDF.pdf.parser.PdfIngester;
+import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage;
+import stirling.software.SPDF.pdf.parser.PdfModels.RawLine;
+import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
+import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment;
+import stirling.software.SPDF.pdf.parser.TabulaTableParser;
 import stirling.software.common.util.ExceptionUtils;
 import stirling.software.common.util.PdfUtils;
 import stirling.software.proprietary.model.api.ai.AiPdfContentType;
 import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest;
 import stirling.software.proprietary.model.api.ai.AiWorkflowTextSelection;
 import stirling.software.proprietary.model.api.ai.FolioType;
-import stirling.software.proprietary.pdf.FlexibleCSVWriter;
-
-import technology.tabula.ObjectExtractor;
-import technology.tabula.Page;
-import technology.tabula.Table;
-import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
 
 @Slf4j
 @Service
+@RequiredArgsConstructor
 public class PdfContentExtractor {
 
+    private final TabulaTableParser tabulaTableParser;
+    private final PdfIngester pdfIngester;
+
     private static final int MAX_CHARACTERS_PER_PAGE = 4_000;
 
     private static final int TEXT_PRESENCE_THRESHOLD = 20;
@@ -101,21 +106,21 @@ public class PdfContentExtractor {
      * @return list of CSV strings (one per table), empty if no tables found
      */
     public List<String> extractTablesAsCsv(PDDocument document, int pageNumber) throws IOException {
-        SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
+        List<TableFragment> fragments = tabulaTableParser.parse(document, pageNumber);
+        if (fragments.isEmpty()) return List.of();
+
         CSVFormat format =
                 CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
         List<String> csvStrings = new ArrayList<>();
 
-        try (ObjectExtractor extractor = new ObjectExtractor(document)) {
-            Page tabulaPage = extractor.extract(pageNumber);
-            List<Table> tables = sea.extract(tabulaPage);
-
-            for (Table table : tables) {
-                StringWriter sw = new StringWriter();
-                FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
-                csvWriter.write(sw, Collections.singletonList(table));
-                csvStrings.add(sw.toString());
+        for (TableFragment fragment : fragments) {
+            StringWriter sw = new StringWriter();
+            try (CSVPrinter printer = format.print(sw)) {
+                for (List<String> row : fragment.rawRows()) {
+                    printer.printRecord(row);
+                }
             }
+            csvStrings.add(sw.toString());
         }
         return csvStrings;
     }
@@ -183,6 +188,8 @@ public class PdfContentExtractor {
             case PAGE_TEXT, FULL_TEXT ->
                     Optional.<PdfContentResult>ofNullable(
                             extractText(lf, fileReq, remainingPages, remainingCharacters));
+            case PAGE_LAYOUT ->
+                    Optional.<PdfContentResult>ofNullable(extractPageLayout(lf, remainingPages));
             default -> {
                 log.warn(
                         "Content type {} not yet implemented, skipping for {}",
@@ -207,6 +214,35 @@ public class PdfContentExtractor {
         return extracted.isEmpty() ? null : buildExtractedFileText(lf.fileName(), extracted);
     }
 
+    private PageLayoutFileResult extractPageLayout(LoadedFile lf, int maxPages) throws IOException {
+        List<ParsedPage> parsedPages = pdfIngester.parse(lf.document(), maxPages);
+        List<LayoutPage> pages = new ArrayList<>();
+        for (ParsedPage pp : parsedPages) {
+            if (pp.layoutLines().isEmpty()) continue;
+            List<LayoutLine> lines = new ArrayList<>();
+            for (RawLine rawLine : pp.layoutLines()) {
+                List<LayoutFragment> fragments = new ArrayList<>();
+                for (TextFragment tf : rawLine.fragments()) {
+                    fragments.add(
+                            new LayoutFragment(
+                                    tf.text(),
+                                    tf.bounds().x(),
+                                    tf.bounds().y(),
+                                    tf.bounds().width(),
+                                    tf.fontSize(),
+                                    tf.bold()));
+                }
+                lines.add(new LayoutLine(rawLine.bounds().y(), fragments));
+            }
+            pages.add(new LayoutPage(pp.pageNumber(), lines));
+        }
+        if (pages.isEmpty()) return null;
+        PageLayoutFileResult result = new PageLayoutFileResult();
+        result.setFileName(lf.fileName());
+        result.setPages(pages);
+        return result;
+    }
+
     private WorkflowArtifact buildArtifact(ArtifactKind kind, List<PdfContentResult> results) {
         return switch (kind) {
             case EXTRACTED_TEXT -> {
@@ -214,10 +250,12 @@ public class PdfContentExtractor {
                 artifact.setFiles(results.stream().map(ExtractedFileText.class::cast).toList());
                 yield artifact;
             }
+            case PAGE_LAYOUT -> {
+                PageLayoutArtifact artifact = new PageLayoutArtifact();
+                artifact.setFiles(results.stream().map(PageLayoutFileResult.class::cast).toList());
+                yield artifact;
+            }
             case TOOL_REPORT ->
-                    // TOOL_REPORT artifacts don't come from PDF content extraction — they're
-                    // built by AiWorkflowService from tool-response metadata. Never reached
-                    // from this code path; presence in the enum is to satisfy the switch.
                     throw new IllegalArgumentException(
                             "TOOL_REPORT artifacts are not produced by PdfContentExtractor");
         };
@@ -331,6 +369,7 @@ public class PdfContentExtractor {
      */
     enum ArtifactKind {
         EXTRACTED_TEXT("extracted_text"),
+        PAGE_LAYOUT("page_layout"),
         TOOL_REPORT("tool_report");
 
         private final String value;
@@ -394,4 +433,40 @@ public class PdfContentExtractor {
             this.report = report;
         }
     }
+
+    // Serialization contract with the Python engine — see PageLayoutArtifactContractTest.
+
+    /** One text fragment with its bounding-box geometry and font properties. */
+    record LayoutFragment(
+            String text, float x, float y, float width, float fontSize, boolean bold) {}
+
+    /** A visual line on the page: y-coordinate and all fragments on that line. */
+    record LayoutLine(float y, List<LayoutFragment> fragments) {}
+
+    /** All layout lines for a single page. */
+    record LayoutPage(int pageNumber, List<LayoutLine> lines) {}
+
+    /** Page layout data for one file, as a PdfContentResult. */
+    @Data
+    static final class PageLayoutFileResult implements PdfContentResult {
+        private String fileName;
+        private List<LayoutPage> pages = new ArrayList<>();
+
+        @Override
+        public ArtifactKind getArtifactKind() {
+            return ArtifactKind.PAGE_LAYOUT;
+        }
+
+        @Override
+        public int pagesConsumed() {
+            return pages.size();
+        }
+    }
+
+    /** Artifact carrying full spatial page layout for all input files. */
+    @Data
+    static final class PageLayoutArtifact implements WorkflowArtifact {
+        private final ArtifactKind kind = ArtifactKind.PAGE_LAYOUT;
+        private List<PageLayoutFileResult> files = new ArrayList<>();
+    }
 }

app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java

@@ -408,6 +408,31 @@ class AiWorkflowServiceTest {
         verify(internalApiClient, times(1)).post(eq(COMPRESS_ENDPOINT), any());
     }
 
+    @Test
+    void generateFileStoresContentDirectlyWithoutToolCall() throws IOException {
+        MockMultipartFile input = pdf("report.pdf", "bytes");
+        stubOrchestrator(
+                """
+                {
+                  "outcome":"generate_file",
+                  "content":"# Hello\\n\\nWorld",
+                  "filename":"report-reconstruction.md",
+                  "summary":"Reconstructed the document as a Markdown file."
+                }
+                """);
+        AtomicInteger ids = stubFileStorage();
+
+        AiWorkflowResponse result = service.orchestrate(requestFor(input, "convert to markdown"));
+
+        assertEquals(AiWorkflowOutcome.COMPLETED, result.getOutcome());
+        assertEquals(1, result.getResultFiles().size());
+        assertEquals("report-reconstruction.md", result.getResultFiles().get(0).getFileName());
+        assertEquals("file-1", result.getResultFiles().get(0).getFileId());
+        assertEquals(1, ids.get());
+        // No tool endpoint should be called — content goes directly to file storage.
+        verify(internalApiClient, never()).post(anyString(), any());
+    }
+
     @Test
     void toolCallWithoutEndpointFallsBackToCannotContinue() throws IOException {
         MockMultipartFile input = pdf("input.pdf", "bytes");

app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java

@@ -0,0 +1,66 @@
+package stirling.software.proprietary.service;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import stirling.software.proprietary.service.PdfContentExtractor.LayoutFragment;
+import stirling.software.proprietary.service.PdfContentExtractor.LayoutLine;
+import stirling.software.proprietary.service.PdfContentExtractor.LayoutPage;
+import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutArtifact;
+import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutFileResult;
+
+import tools.jackson.databind.JsonNode;
+import tools.jackson.databind.json.JsonMapper;
+
+/**
+ * Contract test: verifies that {@link PageLayoutArtifact} serializes to the JSON field names that
+ * the Python engine expects in {@code engine/src/stirling/contracts/pdf_to_markdown.py}.
+ *
+ * <p>The companion Python test in {@code tests/test_pdf_to_markdown.py} deserializes the same JSON
+ * literal and asserts field values. If either side renames a field, one of these tests fails.
+ */
+class PageLayoutArtifactContractTest {
+
+    static final String CONTRACT_JSON =
+            """
+            {"kind":"page_layout","files":[{"fileName":"test.pdf","pages":[{"pageNumber":1,"lines":[{"y":10.0,"fragments":[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}""";
+
+    @Test
+    void pageLayoutArtifact_serialisesToExpectedJson() throws Exception {
+        LayoutFragment fragment = new LayoutFragment("Hello", 1.0f, 2.0f, 30.0f, 12.0f, true);
+        LayoutLine line = new LayoutLine(10.0f, List.of(fragment));
+        LayoutPage page = new LayoutPage(1, List.of(line));
+
+        PageLayoutFileResult fileResult = new PageLayoutFileResult();
+        fileResult.setFileName("test.pdf");
+        fileResult.setPages(List.of(page));
+
+        PageLayoutArtifact artifact = new PageLayoutArtifact();
+        artifact.setFiles(List.of(fileResult));
+
+        JsonNode json = new JsonMapper().valueToTree(artifact);
+
+        assertEquals("page_layout", json.get("kind").asText());
+
+        JsonNode file = json.get("files").get(0);
+        assertEquals("test.pdf", file.get("fileName").asText());
+
+        JsonNode pg = file.get("pages").get(0);
+        assertEquals(1, pg.get("pageNumber").asInt());
+
+        JsonNode ln = pg.get("lines").get(0);
+        assertEquals(10.0, ln.get("y").asDouble(), 0.001);
+
+        JsonNode frag = ln.get("fragments").get(0);
+        assertEquals("Hello", frag.get("text").asText());
+        assertEquals(1.0, frag.get("x").asDouble(), 0.001);
+        assertEquals(2.0, frag.get("y").asDouble(), 0.001);
+        assertEquals(30.0, frag.get("width").asDouble(), 0.001);
+        assertEquals(12.0, frag.get("fontSize").asDouble(), 0.001);
+        assertTrue(frag.get("bold").asBoolean());
+    }
+}