arsvendg/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/arsvendg/Stirling-PDF.git synced 2026-07-16 19:33:11 +02:00
Code Issues Actions 4 Packages Projects Releases 1 Wiki Activity

Feature/pdf to markdown agent (#6271)

Browse Source 
Co-authored-by: James Brunton <[email protected]>
This commit is contained in:
EthanHealy01
2026-05-14 16:20:45 +00:00
committed by GitHub
co-authored by James Brunton
parent 5b9ef852ab
commit ece1bb6865
36 changed files with 4081 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/core/build.gradle
-6
View File
@@ -87,12 +87,6 @@ dependencies {
implementation 'com.sun.xml.bind:jaxb-core:4.0.7'
implementation 'org.apache.poi:poi-ooxml:5.5.1'
// https://mvnrepository.com/artifact/technology.tabula/tabula
implementation ('technology.tabula:tabula:1.0.5') {
exclude group: 'org.slf4j', module: 'slf4j-simple'
exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on'
exclude group: 'com.google.code.gson', module: 'gson'
}
// CVE-2022-25647: Explicit gson 2.13.2 to prevent unsafe deserialization (tabula would pull 2.8.7)
implementation 'com.google.code.gson:gson:2.13.2'
implementation 'org.apache.pdfbox:jbig2-imageio:3.0.4'
app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractCSVController.java
+15 -19
View File
@@ -4,13 +4,13 @@ import java.io.ByteArrayOutputStream;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.QuoteMode;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.http.ContentDisposition;
@@ -26,24 +26,21 @@ import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.config.swagger.CsvConversionResponse;
import stirling.software.SPDF.model.api.PDFWithPageNums;
import stirling.software.SPDF.pdf.FlexibleCSVWriter;
import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.ConvertApi;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.GeneralUtils;
import stirling.software.common.util.WebResponseUtils;
import technology.tabula.ObjectExtractor;
import technology.tabula.Page;
import technology.tabula.Table;
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
@ConvertApi
@Slf4j
@RequiredArgsConstructor
public class ExtractCSVController {
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final TabulaTableParser tabulaTableParser;
@AutoJobPostMapping(value = "/pdf/csv", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@CsvConversionResponse
@@ -58,24 +55,23 @@ public class ExtractCSVController {
try (PDDocument document = pdfDocumentFactory.load(request)) {
List<Integer> pages = request.getPageNumbersList(document, true);
SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
CSVFormat format =
CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
for (int pageNum : pages) {
try (ObjectExtractor extractor = new ObjectExtractor(document)) {
log.info("{}", pageNum);
Page page = extractor.extract(pageNum);
List<Table> tables = sea.extract(page);
log.info("{}", pageNum);
List<TableFragment> fragments = tabulaTableParser.parse(document, pageNum);
for (int i = 0; i < tables.size(); i++) {
StringWriter sw = new StringWriter();
FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
csvWriter.write(sw, Collections.singletonList(tables.get(i)));
String entryName = generateEntryName(baseName, pageNum, i + 1);
csvEntries.add(new CsvEntry(entryName, sw.toString()));
for (int i = 0; i < fragments.size(); i++) {
StringWriter sw = new StringWriter();
try (CSVPrinter printer = format.print(sw)) {
for (List<String> row : fragments.get(i).rawRows()) {
printer.printRecord(row);
}
}
csvEntries.add(
new CsvEntry(
generateEntryName(baseName, pageNum, i + 1), sw.toString()));
}
}
app/core/src/main/java/stirling/software/SPDF/pdf/FlexibleCSVWriter.java
-16
View File
@@ -1,16 +0,0 @@
package stirling.software.SPDF.pdf;
import org.apache.commons.csv.CSVFormat;
import technology.tabula.writers.CSVWriter;
public class FlexibleCSVWriter extends CSVWriter {
public FlexibleCSVWriter() {
super();
}
public FlexibleCSVWriter(CSVFormat csvFormat) {
super(csvFormat);
}
}
app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ExtractCSVControllerTest.java
+2
View File
@@ -20,6 +20,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import stirling.software.SPDF.model.api.PDFWithPageNums;
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.GeneralUtils;
@@ -27,6 +28,7 @@ import stirling.software.common.util.GeneralUtils;
class ExtractCSVControllerTest {
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
@Mock private TabulaTableParser tabulaTableParser;
@InjectMocks private ExtractCSVController controller;
app/core/src/test/java/stirling/software/SPDF/pdf/FlexibleCSVWriterTest.java
-25
View File
@@ -1,25 +0,0 @@
package stirling.software.SPDF.pdf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.apache.commons.csv.CSVFormat;
import org.junit.jupiter.api.Test;
class FlexibleCSVWriterTest {
@Test
void testDefaultConstructor() {
FlexibleCSVWriter writer = new FlexibleCSVWriter();
assertNotNull(writer, "The FlexibleCSVWriter instance should not be null");
}
@Test
void testConstructorWithCSVFormat() {
CSVFormat csvFormat = CSVFormat.DEFAULT;
FlexibleCSVWriter writer = new FlexibleCSVWriter(csvFormat);
assertNotNull(
writer,
"The FlexibleCSVWriter instance should not be null when initialized with"
+ " CSVFormat");
}
}