Feat/math validation agent (#6012)

Co-authored-by: James Brunton <[email protected]>
Co-authored-by: EthanHealy01 <[email protected]>
This commit is contained in:
ConnorYoh
2026-04-17 10:36:45 +01:00
committed by GitHub
co-authored by James Brunton EthanHealy01
parent 688f7f2013
commit de8c483054
49 changed files with 3726 additions and 17 deletions
@@ -0,0 +1,97 @@
package stirling.software.proprietary.controller.api;
import java.io.IOException;
import java.math.BigDecimal;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.proprietary.model.api.ai.Verdict;
import stirling.software.proprietary.service.MathAuditorOrchestrator;
/**
* Public entry point for the Math Auditor Agent (mathAuditorAgent).
*
* <p>Accepts a PDF from the client, hands it to the {@link MathAuditorOrchestrator} which runs the
* multi-round Java-Python negotiation, and returns the Auditor's {@link Verdict}.
*
* <p>The raw PDF never leaves Java. Python receives only structured text and CSV data.
*/
@Slf4j
@RestController
@RequestMapping("/api/v1/ai")
@RequiredArgsConstructor
@Tag(name = "AI Engine", description = "AI-powered document analysis endpoints.")
public class MathAuditorAgentController {
private final MathAuditorOrchestrator orchestrator;
@PostMapping(value = "/math-auditor-agent", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@Operation(
summary = "Validate mathematical calculations in a PDF",
description =
"""
Analyses a PDF document for mathematical errors using the Math Auditor Agent.
The auditor checks:
- Table row and column totals (tally errors)
- Inline arithmetic expressions (e.g. "100 + 200 = 300")
- Cross-page figure consistency (same figure cited differently on different pages)
- Prose claims about percentages, growth rates, and comparisons
The PDF is processed entirely on the Java side; only extracted text and table data
are sent to the AI engine.
Input: PDF Output: JSON Type: SISO
""")
public ResponseEntity<Verdict> mathAuditorAgent(
@Parameter(description = "The PDF document to audit", required = true)
@RequestParam("fileInput")
MultipartFile fileInput,
@Parameter(
description =
"Arithmetic tolerance — differences smaller than this are"
+ " ignored (default: 0.01)")
@RequestParam(value = "tolerance", defaultValue = "0.01")
BigDecimal tolerance) {
String contentType = fileInput.getContentType();
if (contentType == null || !contentType.equals("application/pdf")) {
return ResponseEntity.badRequest().build();
}
if (tolerance.compareTo(BigDecimal.ZERO) < 0) {
return ResponseEntity.badRequest().build();
}
String safeName =
fileInput.getOriginalFilename() != null
? fileInput.getOriginalFilename().replaceAll("[\\r\\n]", "_")
: "<unnamed>";
log.info("[math-auditor-agent] request file={} tolerance={}", safeName, tolerance);
try {
Verdict verdict = orchestrator.audit(fileInput, tolerance);
return ResponseEntity.ok(verdict);
} catch (IOException e) {
log.error("[math-auditor-agent] IO error during audit", e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
} catch (Exception e) {
log.error("[math-auditor-agent] unexpected error during audit", e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
}
}
}
@@ -0,0 +1,21 @@
package stirling.software.proprietary.model.api.ai;
/**
* A single mathematical error found by the Python Auditor.
*
* @param page 0-indexed page number where the discrepancy appears.
* @param kind Category of the discrepancy.
* @param severity Whether this is a definite mistake or a possible ambiguity.
* @param description Human-readable explanation of the error.
* @param stated The value as it appears in the document.
* @param expected The value the Auditor calculated.
* @param context Surrounding text or table fragment for traceability.
*/
public record AuditDiscrepancy(
int page,
DiscrepancyKind kind,
AuditSeverity severity,
String description,
String stated,
String expected,
String context) {}
@@ -0,0 +1,17 @@
package stirling.software.proprietary.model.api.ai;
import com.fasterxml.jackson.annotation.JsonValue;
/**
* Severity of a mathematical discrepancy. Mirrors the Python {@code Severity} enum in {@code
* contracts/ledger.py}.
*/
public enum AuditSeverity {
ERROR,
WARNING;
@JsonValue
public String toJson() {
return name().toLowerCase();
}
}
@@ -0,0 +1,19 @@
package stirling.software.proprietary.model.api.ai;
import com.fasterxml.jackson.annotation.JsonValue;
/**
* Category of a mathematical discrepancy found by the auditor. Mirrors the Python {@code
* DiscrepancyKind} enum in {@code contracts/ledger.py}.
*/
public enum DiscrepancyKind {
TALLY,
ARITHMETIC,
CONSISTENCY,
STATEMENT;
@JsonValue
public String toJson() {
return name().toLowerCase();
}
}
@@ -0,0 +1,24 @@
package stirling.software.proprietary.model.api.ai;
import java.util.List;
/**
* Java's fulfilment package: the extracted content the Python Auditor asked for.
*
* <p>Sent after Java has fulfilled a {@link Requisition}. When {@code finalRound} is {@code true},
* the Auditor must return a {@link Verdict} — Java will not honour further Requisitions.
*
* @param sessionId Matches the session opened by the original client request.
* @param folios The extracted page content for each page in the Requisition.
* @param round Which negotiation round this Evidence belongs to (13).
* @param finalRound When {@code true}, the Auditor must commit to a Verdict this round.
* @param unauditablePages Pages that were requested but could not be fulfilled — e.g. OCR was asked
* for but is not yet wired. The Auditor echoes these into {@link Verdict#unauditablePages()} so
* the client knows coverage is incomplete.
*/
public record Evidence(
String sessionId,
List<Folio> folios,
int round,
boolean finalRound,
List<Integer> unauditablePages) {}
@@ -0,0 +1,17 @@
package stirling.software.proprietary.model.api.ai;
import java.util.List;
/**
* One page's worth of extracted content, assembled by Java in response to a {@link Requisition}.
*
* <p>Only the fields explicitly requested will be populated; unused fields are {@code null}.
*
* @param page 0-indexed page number.
* @param text PDFBox plain-text extraction result (null if not requested).
* @param tables Tabula CSV strings, one per table found on the page (null if not requested).
* @param ocrText OCRmyPDF output text (null if not requested or OCR not available).
* @param ocrConfidence Mean character confidence from OCRmyPDF, 0.01.0 (null if OCR not run).
*/
public record Folio(
int page, String text, List<String> tables, String ocrText, Double ocrConfidence) {}
@@ -0,0 +1,18 @@
package stirling.software.proprietary.model.api.ai;
import java.util.List;
/**
* Java's opening move in the audit negotiation.
*
* <p>Built from a cheap PDFBox scan (character count + image detection) with no OCR or Tabula
* involved. Sent to the Python Examiner, which replies with a {@link Requisition}.
*
* @param sessionId Opaque handle Java uses to locate the PDF on disk during this audit session.
* @param pageCount Total number of pages in the document.
* @param folioTypes One {@link FolioType} per page (0-indexed). {@code folioTypes.size() ==
* pageCount}.
* @param round Which negotiation round this manifest belongs to (13).
*/
public record FolioManifest(
String sessionId, int pageCount, List<FolioType> folioTypes, int round) {}
@@ -0,0 +1,21 @@
package stirling.software.proprietary.model.api.ai;
import com.fasterxml.jackson.annotation.JsonValue;
/**
* Java's classification of a single PDF page after a cheap PDFBox character-count scan. Mirrors the
* Python {@code FolioType} enum in {@code ledger/models.py}.
*/
public enum FolioType {
/** Selectable text layer is present — PDFBox can extract text directly. */
TEXT,
/** Image-only page — OCRmyPDF is required before any text is available. */
IMAGE,
/** Partial text layer plus embedded images — both PDFBox and OCRmyPDF may be useful. */
MIXED;
@JsonValue
public String toJson() {
return name().toLowerCase();
}
}
@@ -0,0 +1,30 @@
package stirling.software.proprietary.model.api.ai;
import java.util.List;
/**
* The Python Examiner's shopping list: which pages Java must extract before the Auditor can form an
* opinion.
*
* <p>Java parses this from the Examiner's response, fulfils it (text / tables / OCR), and sends the
* results back as an {@link Evidence} payload.
*
* @param type Discriminator — always {@code "requisition"}.
* @param needText 0-indexed page numbers requiring PDFBox plain-text extraction.
* @param needTables 0-indexed page numbers requiring Tabula CSV extraction.
* @param needOcr 0-indexed page numbers requiring OCRmyPDF.
* @param rationale Human-readable reason logged for observability.
*/
public record Requisition(
String type,
List<Integer> needText,
List<Integer> needTables,
List<Integer> needOcr,
String rationale) {
public boolean isEmpty() {
return (needText == null || needText.isEmpty())
&& (needTables == null || needTables.isEmpty())
&& (needOcr == null || needOcr.isEmpty());
}
}
@@ -0,0 +1,43 @@
package stirling.software.proprietary.model.api.ai;
import java.util.List;
/**
* The Auditor's final opinion on the document's mathematical integrity.
*
* <p>This is the terminal message in the audit negotiation; Java returns it to the client once
* received from Python.
*
* @param type Discriminator — always {@code "verdict"}.
* @param sessionId Matches the session opened by the original client request.
* @param discrepancies Every mathematical error found, sorted by page.
* @param pagesExamined 0-indexed page numbers the Auditor actually inspected.
* @param roundsTaken How many negotiation rounds were needed (13).
* @param summary One or two sentences suitable for the end user.
* @param clean {@code true} iff no errors were found (warnings are tolerated).
* @param unauditablePages Pages that could not be audited — typically image-only pages for which
* OCR was requested but is not yet wired. The client should indicate that these pages were not
* checked.
*/
public record Verdict(
String type,
String sessionId,
List<AuditDiscrepancy> discrepancies,
List<Integer> pagesExamined,
int roundsTaken,
String summary,
boolean clean,
List<Integer> unauditablePages) {
public long errorCount() {
return discrepancies == null
? 0
: discrepancies.stream().filter(d -> d.severity() == AuditSeverity.ERROR).count();
}
public long warningCount() {
return discrepancies == null
? 0
: discrepancies.stream().filter(d -> d.severity() == AuditSeverity.WARNING).count();
}
}
@@ -0,0 +1,17 @@
package stirling.software.proprietary.pdf;
import org.apache.commons.csv.CSVFormat;
import technology.tabula.writers.CSVWriter;
/** Exposes Tabula's protected {@link CSVWriter#CSVWriter(CSVFormat)} constructor. */
public class FlexibleCSVWriter extends CSVWriter {
public FlexibleCSVWriter() {
super();
}
public FlexibleCSVWriter(CSVFormat csvFormat) {
super(csvFormat);
}
}
@@ -0,0 +1,226 @@
package stirling.software.proprietary.service;
import java.io.IOException;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.proprietary.model.api.ai.Evidence;
import stirling.software.proprietary.model.api.ai.Folio;
import stirling.software.proprietary.model.api.ai.FolioManifest;
import stirling.software.proprietary.model.api.ai.FolioType;
import stirling.software.proprietary.model.api.ai.Requisition;
import stirling.software.proprietary.model.api.ai.Verdict;
import tools.jackson.databind.ObjectMapper;
/**
* Orchestrator for the Math Auditor Agent (mathAuditorAgent).
*
* <p>Manages a four-step Java-Python protocol:
*
* <ol>
* <li>Classify all pages cheaply with PDFBox (no OCR or Tabula yet).
* <li>Send the {@link FolioManifest} to the Python Examiner; receive a {@link Requisition}.
* <li>Fulfil the Requisition (text / tables / OCR) for only the requested pages.
* <li>Send the {@link Evidence} to the Python Auditor; receive a {@link Verdict}.
* </ol>
*
* <p>The raw PDF never leaves Java. Python only receives structured text and CSV data.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class MathAuditorOrchestrator {
private static final String EXAMINE_PATH = "/api/v1/ai/math-auditor-agent/examine";
private static final String DELIBERATE_PATH = "/api/v1/ai/math-auditor-agent/deliberate";
private final AiEngineClient aiEngineClient;
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final PdfContentExtractor pdfContentExtractor;
private final ObjectMapper objectMapper;
/**
* Run a full math audit against the supplied PDF file.
*
* @param pdfFile The uploaded PDF to audit.
* @param tolerance Arithmetic tolerance — differences smaller than this are ignored.
* @return The Auditor's final Verdict.
*/
public Verdict audit(MultipartFile pdfFile, BigDecimal tolerance) throws IOException {
String sessionId = UUID.randomUUID().toString();
log.info(
"[math-auditor-agent] audit started session={} file={} tolerance={}",
sessionId,
pdfFile.getOriginalFilename(),
tolerance);
try (PDDocument document = pdfDocumentFactory.load(pdfFile)) {
// Round 1: classify pages cheaply; send manifest; get requisition
List<FolioType> folioTypes = classifyPages(document);
FolioManifest manifest =
new FolioManifest(sessionId, document.getNumberOfPages(), folioTypes, 1);
Requisition requisition = callExamine(manifest);
log.info(
"[math-auditor-agent] session={} requisition received: {}",
sessionId,
requisition.rationale());
// Round 2: fulfil the requisition and get verdict
Evidence evidence = fulfil(document, sessionId, requisition, 2, true);
Verdict verdict = callDeliberate(evidence, tolerance);
if (verdict == null) {
log.error(
"[math-auditor-agent] session={} null Verdict from deliberate", sessionId);
throw new IllegalStateException("Math Auditor Agent returned null Verdict");
}
log.info(
"[math-auditor-agent] session={} verdict: {} errors, {} warnings,"
+ " clean={}",
sessionId,
verdict.errorCount(),
verdict.warningCount(),
verdict.clean());
return verdict;
}
}
// -----------------------------------------------------------------------
// Python engine calls
// -----------------------------------------------------------------------
private Requisition callExamine(FolioManifest manifest) throws IOException {
String requestBody = objectMapper.writeValueAsString(manifest);
log.info(
"[math-auditor-agent] POST {} session={} round={}",
EXAMINE_PATH,
manifest.sessionId(),
manifest.round());
String responseBody = aiEngineClient.post(EXAMINE_PATH, requestBody);
return objectMapper.readValue(responseBody, Requisition.class);
}
private Verdict callDeliberate(Evidence evidence, BigDecimal tolerance) throws IOException {
String path = DELIBERATE_PATH + "?tolerance=" + tolerance.toPlainString();
String requestBody = objectMapper.writeValueAsString(evidence);
log.info(
"[math-auditor-agent] POST {} session={} round={} final={}",
path,
evidence.sessionId(),
evidence.round(),
evidence.finalRound());
String responseBody = aiEngineClient.post(path, requestBody);
return objectMapper.readValue(responseBody, Verdict.class);
}
// -----------------------------------------------------------------------
// Page classification
// -----------------------------------------------------------------------
private List<FolioType> classifyPages(PDDocument document) throws IOException {
List<FolioType> types = new ArrayList<>();
for (int page = 1; page <= document.getNumberOfPages(); page++) {
types.add(pdfContentExtractor.classifyPage(document, page));
}
return types;
}
// -----------------------------------------------------------------------
// Requisition fulfilment
// -----------------------------------------------------------------------
private Evidence fulfil(
PDDocument document,
String sessionId,
Requisition requisition,
int round,
boolean finalRound)
throws IOException {
List<Integer> allPages =
union(requisition.needText(), requisition.needTables(), requisition.needOcr());
int totalPages = document.getNumberOfPages();
allPages.removeIf(page -> page < 0 || page >= totalPages);
if (allPages.isEmpty()) {
log.warn(
"[math-auditor-agent] session={} all requested pages are out of bounds",
sessionId);
}
List<Folio> folios = new ArrayList<>();
List<Integer> unauditablePages = new ArrayList<>();
for (int page : allPages) {
// Page indices from Python are 0-based; PdfContentExtractor uses 1-based
int pageNumber = page + 1;
String text = null;
List<String> tables = null;
String ocrText = null;
if (contains(requisition.needText(), page)) {
text = pdfContentExtractor.extractPageTextRaw(document, pageNumber);
}
if (contains(requisition.needTables(), page)) {
tables = pdfContentExtractor.extractTablesAsCsv(document, pageNumber);
}
if (contains(requisition.needOcr(), page)) {
log.warn(
"[math-auditor-agent] session={} OCR requested for page {} but not yet"
+ " wired - marking unauditable",
sessionId,
page);
unauditablePages.add(page);
}
if (text != null || tables != null) {
folios.add(new Folio(page, text, tables, ocrText, null));
}
}
log.info(
"[math-auditor-agent] session={} fulfilled round {} with {} folios, {}"
+ " unauditable pages",
sessionId,
round,
folios.size(),
unauditablePages.size());
return new Evidence(sessionId, folios, round, finalRound, unauditablePages);
}
// -----------------------------------------------------------------------
// Helpers
// -----------------------------------------------------------------------
@SafeVarargs
private static List<Integer> union(List<Integer>... lists) {
List<Integer> result = new ArrayList<>();
for (List<Integer> list : lists) {
if (list != null) {
for (int page : list) {
if (!result.contains(page)) {
result.add(page);
}
}
}
}
Collections.sort(result);
return result;
}
private static boolean contains(List<Integer> list, int value) {
return list != null && list.contains(value);
}
}
@@ -1,7 +1,9 @@
package stirling.software.proprietary.service;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -9,6 +11,8 @@ import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.QuoteMode;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;
import org.springframework.stereotype.Service;
@@ -20,9 +24,17 @@ import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.PdfUtils;
import stirling.software.proprietary.model.api.ai.AiPdfContentType;
import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest;
import stirling.software.proprietary.model.api.ai.AiWorkflowTextSelection;
import stirling.software.proprietary.model.api.ai.FolioType;
import stirling.software.proprietary.pdf.FlexibleCSVWriter;
import technology.tabula.ObjectExtractor;
import technology.tabula.Page;
import technology.tabula.Table;
import technology.tabula.extractors.SpreadsheetExtractionAlgorithm;
@Slf4j
@Service
@@ -30,8 +42,84 @@ public class PdfContentExtractor {
private static final int MAX_CHARACTERS_PER_PAGE = 4_000;
private static final int TEXT_PRESENCE_THRESHOLD = 20;
record LoadedFile(String fileName, PDDocument document) {}
// -----------------------------------------------------------------------
// Low-level extraction methods (usable by any agent)
// -----------------------------------------------------------------------
/**
* Classify a single page as TEXT, IMAGE, or MIXED.
*
* @param document the open PDF
* @param pageNumber 1-based page number
*/
public FolioType classifyPage(PDDocument document, int pageNumber) throws IOException {
PDFTextStripper stripper = new PDFTextStripper();
stripper.setStartPage(pageNumber);
stripper.setEndPage(pageNumber);
String text = stripper.getText(document).trim();
boolean hasText = text.length() > TEXT_PRESENCE_THRESHOLD;
boolean hasImages = PdfUtils.hasImagesOnPage(document.getPage(pageNumber - 1));
if (hasText && hasImages) {
return FolioType.MIXED;
} else if (hasText) {
return FolioType.TEXT;
} else {
return FolioType.IMAGE;
}
}
/**
* Extract plain text from a single page, clipped to {@link #MAX_CHARACTERS_PER_PAGE}.
*
* @param document the open PDF
* @param pageNumber 1-based page number
* @return trimmed text, or empty string if the page has no extractable text
*/
public String extractPageTextRaw(PDDocument document, int pageNumber) throws IOException {
PDFTextStripper stripper = new PDFTextStripper();
stripper.setStartPage(pageNumber);
stripper.setEndPage(pageNumber);
String text = stripper.getText(document).trim();
return clip(text, MAX_CHARACTERS_PER_PAGE);
}
/**
* Extract all tables from a single page as CSV strings.
*
* @param document the open PDF
* @param pageNumber 1-based page number
* @return list of CSV strings (one per table), empty if no tables found
*/
public List<String> extractTablesAsCsv(PDDocument document, int pageNumber) throws IOException {
SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm();
CSVFormat format =
CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build();
List<String> csvStrings = new ArrayList<>();
try (ObjectExtractor extractor = new ObjectExtractor(document)) {
Page tabulaPage = extractor.extract(pageNumber);
List<Table> tables = sea.extract(tabulaPage);
for (Table table : tables) {
StringWriter sw = new StringWriter();
FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format);
csvWriter.write(sw, Collections.singletonList(table));
csvStrings.add(sw.toString());
}
}
return csvStrings;
}
// -----------------------------------------------------------------------
// Workflow extraction (used by AiWorkflowService)
// -----------------------------------------------------------------------
/**
* Extracts content from the loaded files according to the requested content types and budget
* constraints.