Feature/pdf to markdown agent (#6271)

Co-authored-by: James Brunton <[email protected]>

app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractCSVController.java

@@ -4,13 +4,13 @@ import java.io.ByteArrayOutputStream;
 import java.io.StringWriter;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
 import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.csv.QuoteMode;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.http.ContentDisposition;
@@ -26,24 +26,21 @@ import lombok.extern.slf4j.Slf4j;
 
 import stirling.software.SPDF.config.swagger.CsvConversionResponse;
 import stirling.software.SPDF.model.api.PDFWithPageNums;
-import stirling.software.SPDF.pdf.FlexibleCSVWriter;
+import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
+import stirling.software.SPDF.pdf.parser.TabulaTableParser;
 import stirling.software.common.annotations.AutoJobPostMapping;
 import stirling.software.common.annotations.api.ConvertApi;
 import stirling.software.common.service.CustomPDFDocumentFactory;
 import stirling.software.common.util.GeneralUtils;
 import stirling.software.common.util.WebResponseUtils;
 
-import technology.tabula.ObjectExtractor;
-import technology.tabula.Page;
-import technology.tabula.Table;
-import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
-
 @ConvertApi
 @Slf4j
 @RequiredArgsConstructor
 public class ExtractCSVController {
 
     private final CustomPDFDocumentFactory pdfDocumentFactory;
+    private final TabulaTableParser tabulaTableParser;
 
     @AutoJobPostMapping(value = "/pdf/csv", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
     @CsvConversionResponse
@@ -58,24 +55,23 @@ public class ExtractCSVController {
 
         try (PDDocument document = pdfDocumentFactory.load(request)) {
             List<Integer> pages = request.getPageNumbersList(document, true);
-            SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
             CSVFormat format =
                     CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
 
             for (int pageNum : pages) {
-                try (ObjectExtractor extractor = new ObjectExtractor(document)) {
-                    log.info("{}", pageNum);
-                    Page page = extractor.extract(pageNum);
-                    List<Table> tables = sea.extract(page);
+                log.info("{}", pageNum);
+                List<TableFragment> fragments = tabulaTableParser.parse(document, pageNum);
 
-                    for (int i = 0; i < tables.size(); i++) {
-                        StringWriter sw = new StringWriter();
-                        FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
-                        csvWriter.write(sw, Collections.singletonList(tables.get(i)));
-
-                        String entryName = generateEntryName(baseName, pageNum, i + 1);
-                        csvEntries.add(new CsvEntry(entryName, sw.toString()));
+                for (int i = 0; i < fragments.size(); i++) {
+                    StringWriter sw = new StringWriter();
+                    try (CSVPrinter printer = format.print(sw)) {
+                        for (List<String> row : fragments.get(i).rawRows()) {
+                            printer.printRecord(row);
+                        }
                     }
+                    csvEntries.add(
+                            new CsvEntry(
+                                    generateEntryName(baseName, pageNum, i + 1), sw.toString()));
                 }
             }
 

app/core/src/main/java/stirling/software/SPDF/pdf/FlexibleCSVWriter.java

@@ -1,16 +0,0 @@
-package stirling.software.SPDF.pdf;
-
-import org.apache.commons.csv.CSVFormat;
-
-import technology.tabula.writers.CSVWriter;
-
-public class FlexibleCSVWriter extends CSVWriter {
-
-    public FlexibleCSVWriter() {
-        super();
-    }
-
-    public FlexibleCSVWriter(CSVFormat csvFormat) {
-        super(csvFormat);
-    }
-}

app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ExtractCSVControllerTest.java

@@ -20,6 +20,7 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.mock.web.MockMultipartFile;
 
 import stirling.software.SPDF.model.api.PDFWithPageNums;
+import stirling.software.SPDF.pdf.parser.TabulaTableParser;
 import stirling.software.common.service.CustomPDFDocumentFactory;
 import stirling.software.common.util.GeneralUtils;
 
@@ -27,6 +28,7 @@ import stirling.software.common.util.GeneralUtils;
 class ExtractCSVControllerTest {
 
     @Mock private CustomPDFDocumentFactory pdfDocumentFactory;
+    @Mock private TabulaTableParser tabulaTableParser;
 
     @InjectMocks private ExtractCSVController controller;
 

app/core/src/test/java/stirling/software/SPDF/pdf/FlexibleCSVWriterTest.java

@@ -1,25 +0,0 @@
-package stirling.software.SPDF.pdf;
-
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-import org.apache.commons.csv.CSVFormat;
-import org.junit.jupiter.api.Test;
-
-class FlexibleCSVWriterTest {
-
-    @Test
-    void testDefaultConstructor() {
-        FlexibleCSVWriter writer = new FlexibleCSVWriter();
-        assertNotNull(writer, "The FlexibleCSVWriter instance should not be null");
-    }
-
-    @Test
-    void testConstructorWithCSVFormat() {
-        CSVFormat csvFormat = CSVFormat.DEFAULT;
-        FlexibleCSVWriter writer = new FlexibleCSVWriter(csvFormat);
-        assertNotNull(
-                writer,
-                "The FlexibleCSVWriter instance should not be null when initialized with"
-                        + " CSVFormat");
-    }
-}