mirror of
https://github.com/arsvendg/Stirling-PDF.git
synced 2026-07-16 19:33:11 +02:00
Feat/math validation agent (#6012)
Co-authored-by: James Brunton <[email protected]> Co-authored-by: EthanHealy01 <[email protected]>
This commit is contained in:
co-authored by
James Brunton
EthanHealy01
parent
688f7f2013
commit
de8c483054
+97
@@ -0,0 +1,97 @@
|
||||
package stirling.software.proprietary.controller.api;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.math.BigDecimal;
|
||||
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.proprietary.model.api.ai.Verdict;
|
||||
import stirling.software.proprietary.service.MathAuditorOrchestrator;
|
||||
|
||||
/**
|
||||
* Public entry point for the Math Auditor Agent (mathAuditorAgent).
|
||||
*
|
||||
* <p>Accepts a PDF from the client, hands it to the {@link MathAuditorOrchestrator} which runs the
|
||||
* multi-round Java-Python negotiation, and returns the Auditor's {@link Verdict}.
|
||||
*
|
||||
* <p>The raw PDF never leaves Java. Python receives only structured text and CSV data.
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/ai")
|
||||
@RequiredArgsConstructor
|
||||
@Tag(name = "AI Engine", description = "AI-powered document analysis endpoints.")
|
||||
public class MathAuditorAgentController {
|
||||
|
||||
private final MathAuditorOrchestrator orchestrator;
|
||||
|
||||
@PostMapping(value = "/math-auditor-agent", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@Operation(
|
||||
summary = "Validate mathematical calculations in a PDF",
|
||||
description =
|
||||
"""
|
||||
Analyses a PDF document for mathematical errors using the Math Auditor Agent.
|
||||
|
||||
The auditor checks:
|
||||
- Table row and column totals (tally errors)
|
||||
- Inline arithmetic expressions (e.g. "100 + 200 = 300")
|
||||
- Cross-page figure consistency (same figure cited differently on different pages)
|
||||
- Prose claims about percentages, growth rates, and comparisons
|
||||
|
||||
The PDF is processed entirely on the Java side; only extracted text and table data
|
||||
are sent to the AI engine.
|
||||
|
||||
Input: PDF Output: JSON Type: SISO
|
||||
""")
|
||||
public ResponseEntity<Verdict> mathAuditorAgent(
|
||||
@Parameter(description = "The PDF document to audit", required = true)
|
||||
@RequestParam("fileInput")
|
||||
MultipartFile fileInput,
|
||||
@Parameter(
|
||||
description =
|
||||
"Arithmetic tolerance — differences smaller than this are"
|
||||
+ " ignored (default: 0.01)")
|
||||
@RequestParam(value = "tolerance", defaultValue = "0.01")
|
||||
BigDecimal tolerance) {
|
||||
|
||||
String contentType = fileInput.getContentType();
|
||||
if (contentType == null || !contentType.equals("application/pdf")) {
|
||||
return ResponseEntity.badRequest().build();
|
||||
}
|
||||
|
||||
if (tolerance.compareTo(BigDecimal.ZERO) < 0) {
|
||||
return ResponseEntity.badRequest().build();
|
||||
}
|
||||
|
||||
String safeName =
|
||||
fileInput.getOriginalFilename() != null
|
||||
? fileInput.getOriginalFilename().replaceAll("[\\r\\n]", "_")
|
||||
: "<unnamed>";
|
||||
log.info("[math-auditor-agent] request file={} tolerance={}", safeName, tolerance);
|
||||
|
||||
try {
|
||||
Verdict verdict = orchestrator.audit(fileInput, tolerance);
|
||||
return ResponseEntity.ok(verdict);
|
||||
} catch (IOException e) {
|
||||
log.error("[math-auditor-agent] IO error during audit", e);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
|
||||
} catch (Exception e) {
|
||||
log.error("[math-auditor-agent] unexpected error during audit", e);
|
||||
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
|
||||
}
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
/**
|
||||
* A single mathematical error found by the Python Auditor.
|
||||
*
|
||||
* @param page 0-indexed page number where the discrepancy appears.
|
||||
* @param kind Category of the discrepancy.
|
||||
* @param severity Whether this is a definite mistake or a possible ambiguity.
|
||||
* @param description Human-readable explanation of the error.
|
||||
* @param stated The value as it appears in the document.
|
||||
* @param expected The value the Auditor calculated.
|
||||
* @param context Surrounding text or table fragment for traceability.
|
||||
*/
|
||||
public record AuditDiscrepancy(
|
||||
int page,
|
||||
DiscrepancyKind kind,
|
||||
AuditSeverity severity,
|
||||
String description,
|
||||
String stated,
|
||||
String expected,
|
||||
String context) {}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
|
||||
/**
|
||||
* Severity of a mathematical discrepancy. Mirrors the Python {@code Severity} enum in {@code
|
||||
* contracts/ledger.py}.
|
||||
*/
|
||||
public enum AuditSeverity {
|
||||
ERROR,
|
||||
WARNING;
|
||||
|
||||
@JsonValue
|
||||
public String toJson() {
|
||||
return name().toLowerCase();
|
||||
}
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
|
||||
/**
|
||||
* Category of a mathematical discrepancy found by the auditor. Mirrors the Python {@code
|
||||
* DiscrepancyKind} enum in {@code contracts/ledger.py}.
|
||||
*/
|
||||
public enum DiscrepancyKind {
|
||||
TALLY,
|
||||
ARITHMETIC,
|
||||
CONSISTENCY,
|
||||
STATEMENT;
|
||||
|
||||
@JsonValue
|
||||
public String toJson() {
|
||||
return name().toLowerCase();
|
||||
}
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Java's fulfilment package: the extracted content the Python Auditor asked for.
|
||||
*
|
||||
* <p>Sent after Java has fulfilled a {@link Requisition}. When {@code finalRound} is {@code true},
|
||||
* the Auditor must return a {@link Verdict} — Java will not honour further Requisitions.
|
||||
*
|
||||
* @param sessionId Matches the session opened by the original client request.
|
||||
* @param folios The extracted page content for each page in the Requisition.
|
||||
* @param round Which negotiation round this Evidence belongs to (1–3).
|
||||
* @param finalRound When {@code true}, the Auditor must commit to a Verdict this round.
|
||||
* @param unauditablePages Pages that were requested but could not be fulfilled — e.g. OCR was asked
|
||||
* for but is not yet wired. The Auditor echoes these into {@link Verdict#unauditablePages()} so
|
||||
* the client knows coverage is incomplete.
|
||||
*/
|
||||
public record Evidence(
|
||||
String sessionId,
|
||||
List<Folio> folios,
|
||||
int round,
|
||||
boolean finalRound,
|
||||
List<Integer> unauditablePages) {}
|
||||
@@ -0,0 +1,17 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* One page's worth of extracted content, assembled by Java in response to a {@link Requisition}.
|
||||
*
|
||||
* <p>Only the fields explicitly requested will be populated; unused fields are {@code null}.
|
||||
*
|
||||
* @param page 0-indexed page number.
|
||||
* @param text PDFBox plain-text extraction result (null if not requested).
|
||||
* @param tables Tabula CSV strings, one per table found on the page (null if not requested).
|
||||
* @param ocrText OCRmyPDF output text (null if not requested or OCR not available).
|
||||
* @param ocrConfidence Mean character confidence from OCRmyPDF, 0.0–1.0 (null if OCR not run).
|
||||
*/
|
||||
public record Folio(
|
||||
int page, String text, List<String> tables, String ocrText, Double ocrConfidence) {}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Java's opening move in the audit negotiation.
|
||||
*
|
||||
* <p>Built from a cheap PDFBox scan (character count + image detection) with no OCR or Tabula
|
||||
* involved. Sent to the Python Examiner, which replies with a {@link Requisition}.
|
||||
*
|
||||
* @param sessionId Opaque handle Java uses to locate the PDF on disk during this audit session.
|
||||
* @param pageCount Total number of pages in the document.
|
||||
* @param folioTypes One {@link FolioType} per page (0-indexed). {@code folioTypes.size() ==
|
||||
* pageCount}.
|
||||
* @param round Which negotiation round this manifest belongs to (1–3).
|
||||
*/
|
||||
public record FolioManifest(
|
||||
String sessionId, int pageCount, List<FolioType> folioTypes, int round) {}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
|
||||
/**
|
||||
* Java's classification of a single PDF page after a cheap PDFBox character-count scan. Mirrors the
|
||||
* Python {@code FolioType} enum in {@code ledger/models.py}.
|
||||
*/
|
||||
public enum FolioType {
|
||||
/** Selectable text layer is present — PDFBox can extract text directly. */
|
||||
TEXT,
|
||||
/** Image-only page — OCRmyPDF is required before any text is available. */
|
||||
IMAGE,
|
||||
/** Partial text layer plus embedded images — both PDFBox and OCRmyPDF may be useful. */
|
||||
MIXED;
|
||||
|
||||
@JsonValue
|
||||
public String toJson() {
|
||||
return name().toLowerCase();
|
||||
}
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* The Python Examiner's shopping list: which pages Java must extract before the Auditor can form an
|
||||
* opinion.
|
||||
*
|
||||
* <p>Java parses this from the Examiner's response, fulfils it (text / tables / OCR), and sends the
|
||||
* results back as an {@link Evidence} payload.
|
||||
*
|
||||
* @param type Discriminator — always {@code "requisition"}.
|
||||
* @param needText 0-indexed page numbers requiring PDFBox plain-text extraction.
|
||||
* @param needTables 0-indexed page numbers requiring Tabula CSV extraction.
|
||||
* @param needOcr 0-indexed page numbers requiring OCRmyPDF.
|
||||
* @param rationale Human-readable reason logged for observability.
|
||||
*/
|
||||
public record Requisition(
|
||||
String type,
|
||||
List<Integer> needText,
|
||||
List<Integer> needTables,
|
||||
List<Integer> needOcr,
|
||||
String rationale) {
|
||||
|
||||
public boolean isEmpty() {
|
||||
return (needText == null || needText.isEmpty())
|
||||
&& (needTables == null || needTables.isEmpty())
|
||||
&& (needOcr == null || needOcr.isEmpty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package stirling.software.proprietary.model.api.ai;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* The Auditor's final opinion on the document's mathematical integrity.
|
||||
*
|
||||
* <p>This is the terminal message in the audit negotiation; Java returns it to the client once
|
||||
* received from Python.
|
||||
*
|
||||
* @param type Discriminator — always {@code "verdict"}.
|
||||
* @param sessionId Matches the session opened by the original client request.
|
||||
* @param discrepancies Every mathematical error found, sorted by page.
|
||||
* @param pagesExamined 0-indexed page numbers the Auditor actually inspected.
|
||||
* @param roundsTaken How many negotiation rounds were needed (1–3).
|
||||
* @param summary One or two sentences suitable for the end user.
|
||||
* @param clean {@code true} iff no errors were found (warnings are tolerated).
|
||||
* @param unauditablePages Pages that could not be audited — typically image-only pages for which
|
||||
* OCR was requested but is not yet wired. The client should indicate that these pages were not
|
||||
* checked.
|
||||
*/
|
||||
public record Verdict(
|
||||
String type,
|
||||
String sessionId,
|
||||
List<AuditDiscrepancy> discrepancies,
|
||||
List<Integer> pagesExamined,
|
||||
int roundsTaken,
|
||||
String summary,
|
||||
boolean clean,
|
||||
List<Integer> unauditablePages) {
|
||||
|
||||
public long errorCount() {
|
||||
return discrepancies == null
|
||||
? 0
|
||||
: discrepancies.stream().filter(d -> d.severity() == AuditSeverity.ERROR).count();
|
||||
}
|
||||
|
||||
public long warningCount() {
|
||||
return discrepancies == null
|
||||
? 0
|
||||
: discrepancies.stream().filter(d -> d.severity() == AuditSeverity.WARNING).count();
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package stirling.software.proprietary.pdf;
|
||||
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
|
||||
import technology.tabula.writers.CSVWriter;
|
||||
|
||||
/** Exposes Tabula's protected {@link CSVWriter#CSVWriter(CSVFormat)} constructor. */
|
||||
public class FlexibleCSVWriter extends CSVWriter {
|
||||
|
||||
public FlexibleCSVWriter() {
|
||||
super();
|
||||
}
|
||||
|
||||
public FlexibleCSVWriter(CSVFormat csvFormat) {
|
||||
super(csvFormat);
|
||||
}
|
||||
}
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
package stirling.software.proprietary.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.math.BigDecimal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.proprietary.model.api.ai.Evidence;
|
||||
import stirling.software.proprietary.model.api.ai.Folio;
|
||||
import stirling.software.proprietary.model.api.ai.FolioManifest;
|
||||
import stirling.software.proprietary.model.api.ai.FolioType;
|
||||
import stirling.software.proprietary.model.api.ai.Requisition;
|
||||
import stirling.software.proprietary.model.api.ai.Verdict;
|
||||
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Orchestrator for the Math Auditor Agent (mathAuditorAgent).
|
||||
*
|
||||
* <p>Manages a four-step Java-Python protocol:
|
||||
*
|
||||
* <ol>
|
||||
* <li>Classify all pages cheaply with PDFBox (no OCR or Tabula yet).
|
||||
* <li>Send the {@link FolioManifest} to the Python Examiner; receive a {@link Requisition}.
|
||||
* <li>Fulfil the Requisition (text / tables / OCR) for only the requested pages.
|
||||
* <li>Send the {@link Evidence} to the Python Auditor; receive a {@link Verdict}.
|
||||
* </ol>
|
||||
*
|
||||
* <p>The raw PDF never leaves Java. Python only receives structured text and CSV data.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class MathAuditorOrchestrator {
|
||||
|
||||
private static final String EXAMINE_PATH = "/api/v1/ai/math-auditor-agent/examine";
|
||||
private static final String DELIBERATE_PATH = "/api/v1/ai/math-auditor-agent/deliberate";
|
||||
|
||||
private final AiEngineClient aiEngineClient;
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final PdfContentExtractor pdfContentExtractor;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
/**
|
||||
* Run a full math audit against the supplied PDF file.
|
||||
*
|
||||
* @param pdfFile The uploaded PDF to audit.
|
||||
* @param tolerance Arithmetic tolerance — differences smaller than this are ignored.
|
||||
* @return The Auditor's final Verdict.
|
||||
*/
|
||||
public Verdict audit(MultipartFile pdfFile, BigDecimal tolerance) throws IOException {
|
||||
String sessionId = UUID.randomUUID().toString();
|
||||
log.info(
|
||||
"[math-auditor-agent] audit started session={} file={} tolerance={}",
|
||||
sessionId,
|
||||
pdfFile.getOriginalFilename(),
|
||||
tolerance);
|
||||
|
||||
try (PDDocument document = pdfDocumentFactory.load(pdfFile)) {
|
||||
// Round 1: classify pages cheaply; send manifest; get requisition
|
||||
List<FolioType> folioTypes = classifyPages(document);
|
||||
FolioManifest manifest =
|
||||
new FolioManifest(sessionId, document.getNumberOfPages(), folioTypes, 1);
|
||||
|
||||
Requisition requisition = callExamine(manifest);
|
||||
log.info(
|
||||
"[math-auditor-agent] session={} requisition received: {}",
|
||||
sessionId,
|
||||
requisition.rationale());
|
||||
|
||||
// Round 2: fulfil the requisition and get verdict
|
||||
Evidence evidence = fulfil(document, sessionId, requisition, 2, true);
|
||||
Verdict verdict = callDeliberate(evidence, tolerance);
|
||||
|
||||
if (verdict == null) {
|
||||
log.error(
|
||||
"[math-auditor-agent] session={} null Verdict from deliberate", sessionId);
|
||||
throw new IllegalStateException("Math Auditor Agent returned null Verdict");
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[math-auditor-agent] session={} verdict: {} errors, {} warnings,"
|
||||
+ " clean={}",
|
||||
sessionId,
|
||||
verdict.errorCount(),
|
||||
verdict.warningCount(),
|
||||
verdict.clean());
|
||||
return verdict;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Python engine calls
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private Requisition callExamine(FolioManifest manifest) throws IOException {
|
||||
String requestBody = objectMapper.writeValueAsString(manifest);
|
||||
log.info(
|
||||
"[math-auditor-agent] POST {} session={} round={}",
|
||||
EXAMINE_PATH,
|
||||
manifest.sessionId(),
|
||||
manifest.round());
|
||||
String responseBody = aiEngineClient.post(EXAMINE_PATH, requestBody);
|
||||
return objectMapper.readValue(responseBody, Requisition.class);
|
||||
}
|
||||
|
||||
private Verdict callDeliberate(Evidence evidence, BigDecimal tolerance) throws IOException {
|
||||
String path = DELIBERATE_PATH + "?tolerance=" + tolerance.toPlainString();
|
||||
String requestBody = objectMapper.writeValueAsString(evidence);
|
||||
log.info(
|
||||
"[math-auditor-agent] POST {} session={} round={} final={}",
|
||||
path,
|
||||
evidence.sessionId(),
|
||||
evidence.round(),
|
||||
evidence.finalRound());
|
||||
String responseBody = aiEngineClient.post(path, requestBody);
|
||||
return objectMapper.readValue(responseBody, Verdict.class);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Page classification
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private List<FolioType> classifyPages(PDDocument document) throws IOException {
|
||||
List<FolioType> types = new ArrayList<>();
|
||||
for (int page = 1; page <= document.getNumberOfPages(); page++) {
|
||||
types.add(pdfContentExtractor.classifyPage(document, page));
|
||||
}
|
||||
return types;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Requisition fulfilment
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private Evidence fulfil(
|
||||
PDDocument document,
|
||||
String sessionId,
|
||||
Requisition requisition,
|
||||
int round,
|
||||
boolean finalRound)
|
||||
throws IOException {
|
||||
|
||||
List<Integer> allPages =
|
||||
union(requisition.needText(), requisition.needTables(), requisition.needOcr());
|
||||
int totalPages = document.getNumberOfPages();
|
||||
allPages.removeIf(page -> page < 0 || page >= totalPages);
|
||||
if (allPages.isEmpty()) {
|
||||
log.warn(
|
||||
"[math-auditor-agent] session={} all requested pages are out of bounds",
|
||||
sessionId);
|
||||
}
|
||||
List<Folio> folios = new ArrayList<>();
|
||||
List<Integer> unauditablePages = new ArrayList<>();
|
||||
|
||||
for (int page : allPages) {
|
||||
// Page indices from Python are 0-based; PdfContentExtractor uses 1-based
|
||||
int pageNumber = page + 1;
|
||||
String text = null;
|
||||
List<String> tables = null;
|
||||
String ocrText = null;
|
||||
|
||||
if (contains(requisition.needText(), page)) {
|
||||
text = pdfContentExtractor.extractPageTextRaw(document, pageNumber);
|
||||
}
|
||||
if (contains(requisition.needTables(), page)) {
|
||||
tables = pdfContentExtractor.extractTablesAsCsv(document, pageNumber);
|
||||
}
|
||||
if (contains(requisition.needOcr(), page)) {
|
||||
log.warn(
|
||||
"[math-auditor-agent] session={} OCR requested for page {} but not yet"
|
||||
+ " wired - marking unauditable",
|
||||
sessionId,
|
||||
page);
|
||||
unauditablePages.add(page);
|
||||
}
|
||||
|
||||
if (text != null || tables != null) {
|
||||
folios.add(new Folio(page, text, tables, ocrText, null));
|
||||
}
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[math-auditor-agent] session={} fulfilled round {} with {} folios, {}"
|
||||
+ " unauditable pages",
|
||||
sessionId,
|
||||
round,
|
||||
folios.size(),
|
||||
unauditablePages.size());
|
||||
return new Evidence(sessionId, folios, round, finalRound, unauditablePages);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@SafeVarargs
|
||||
private static List<Integer> union(List<Integer>... lists) {
|
||||
List<Integer> result = new ArrayList<>();
|
||||
for (List<Integer> list : lists) {
|
||||
if (list != null) {
|
||||
for (int page : list) {
|
||||
if (!result.contains(page)) {
|
||||
result.add(page);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Collections.sort(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static boolean contains(List<Integer> list, int value) {
|
||||
return list != null && list.contains(value);
|
||||
}
|
||||
}
|
||||
+88
@@ -1,7 +1,9 @@
|
||||
package stirling.software.proprietary.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -9,6 +11,8 @@ import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
import org.apache.commons.csv.QuoteMode;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -20,9 +24,17 @@ import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
import stirling.software.proprietary.model.api.ai.AiPdfContentType;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowTextSelection;
|
||||
import stirling.software.proprietary.model.api.ai.FolioType;
|
||||
import stirling.software.proprietary.pdf.FlexibleCSVWriter;
|
||||
|
||||
import technology.tabula.ObjectExtractor;
|
||||
import technology.tabula.Page;
|
||||
import technology.tabula.Table;
|
||||
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -30,8 +42,84 @@ public class PdfContentExtractor {
|
||||
|
||||
private static final int MAX_CHARACTERS_PER_PAGE = 4_000;
|
||||
|
||||
private static final int TEXT_PRESENCE_THRESHOLD = 20;
|
||||
|
||||
record LoadedFile(String fileName, PDDocument document) {}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Low-level extraction methods (usable by any agent)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Classify a single page as TEXT, IMAGE, or MIXED.
|
||||
*
|
||||
* @param document the open PDF
|
||||
* @param pageNumber 1-based page number
|
||||
*/
|
||||
public FolioType classifyPage(PDDocument document, int pageNumber) throws IOException {
|
||||
PDFTextStripper stripper = new PDFTextStripper();
|
||||
stripper.setStartPage(pageNumber);
|
||||
stripper.setEndPage(pageNumber);
|
||||
String text = stripper.getText(document).trim();
|
||||
|
||||
boolean hasText = text.length() > TEXT_PRESENCE_THRESHOLD;
|
||||
boolean hasImages = PdfUtils.hasImagesOnPage(document.getPage(pageNumber - 1));
|
||||
|
||||
if (hasText && hasImages) {
|
||||
return FolioType.MIXED;
|
||||
} else if (hasText) {
|
||||
return FolioType.TEXT;
|
||||
} else {
|
||||
return FolioType.IMAGE;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract plain text from a single page, clipped to {@link #MAX_CHARACTERS_PER_PAGE}.
|
||||
*
|
||||
* @param document the open PDF
|
||||
* @param pageNumber 1-based page number
|
||||
* @return trimmed text, or empty string if the page has no extractable text
|
||||
*/
|
||||
public String extractPageTextRaw(PDDocument document, int pageNumber) throws IOException {
|
||||
PDFTextStripper stripper = new PDFTextStripper();
|
||||
stripper.setStartPage(pageNumber);
|
||||
stripper.setEndPage(pageNumber);
|
||||
String text = stripper.getText(document).trim();
|
||||
return clip(text, MAX_CHARACTERS_PER_PAGE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all tables from a single page as CSV strings.
|
||||
*
|
||||
* @param document the open PDF
|
||||
* @param pageNumber 1-based page number
|
||||
* @return list of CSV strings (one per table), empty if no tables found
|
||||
*/
|
||||
public List<String> extractTablesAsCsv(PDDocument document, int pageNumber) throws IOException {
|
||||
SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
|
||||
CSVFormat format =
|
||||
CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
|
||||
List<String> csvStrings = new ArrayList<>();
|
||||
|
||||
try (ObjectExtractor extractor = new ObjectExtractor(document)) {
|
||||
Page tabulaPage = extractor.extract(pageNumber);
|
||||
List<Table> tables = sea.extract(tabulaPage);
|
||||
|
||||
for (Table table : tables) {
|
||||
StringWriter sw = new StringWriter();
|
||||
FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
|
||||
csvWriter.write(sw, Collections.singletonList(table));
|
||||
csvStrings.add(sw.toString());
|
||||
}
|
||||
}
|
||||
return csvStrings;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Workflow extraction (used by AiWorkflowService)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Extracts content from the loaded files according to the requested content types and budget
|
||||
* constraints.
|
||||
|
||||
Reference in New Issue
Block a user