arsvendg/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/arsvendg/Stirling-PDF.git synced 2026-07-14 10:34:06 +02:00
Code Issues Actions 4 Packages Projects Releases 1 Wiki Activity

Merge branch 'main' into SaaS

Browse Source
This commit is contained in:
Anthony Stirling
2026-06-10 14:58:44 +01:00
committed by GitHub
parent be0db3fd8a 611468b972
commit 9a1804ce04
93 changed files with 6398 additions and 2421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java
-73
View File
@@ -1,73 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link
* LineAlignmentTableParser}. The first parser returning a result above {@link
* #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page.
*/
@Service
@Primary
@RequiredArgsConstructor
@Slf4j
public class CompositeTableParser implements TableParser {
/** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */
static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f;
private final TabulaTableParser tabulaParser;
private final LineAlignmentTableParser lineAlignmentParser;
@Override
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
// Step 1: Tabula lattice mode (ruled/bordered tables).
List<TableFragment> latticeResults = filterConfident(tabulaParser.parse(document, rawPage));
if (!latticeResults.isEmpty()) {
log.debug(
"Page {}: using Tabula lattice ({} table(s))",
rawPage.pageNumber(),
latticeResults.size());
return latticeResults;
}
// Step 2: Tabula stream mode (borderless/whitespace-delimited tables).
// parseStream is not on the TableParser interface — this intentionally couples to the
// concrete TabulaTableParser since stream mode is a Tabula-specific concept.
List<TableFragment> streamResults =
filterConfident(tabulaParser.parseStream(document, rawPage));
if (!streamResults.isEmpty()) {
log.debug(
"Page {}: using Tabula stream ({} table(s))",
rawPage.pageNumber(),
streamResults.size());
return streamResults;
}
// Step 3: Geometry-based line-alignment fallback.
List<TableFragment> lineResults = lineAlignmentParser.parse(document, rawPage);
if (!lineResults.isEmpty()) {
log.debug(
"Page {}: using LineAlignment ({} table(s))",
rawPage.pageNumber(),
lineResults.size());
return lineResults;
}
return List.of();
}
private List<TableFragment> filterConfident(List<TableFragment> tables) {
return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList();
}
}
app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java
-528
View File
@@ -1,528 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
import java.util.regex.Pattern;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
/**
* Fallback {@link TableParser} for borderless financial tables using text geometry.
*
* <p>Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge
* positions, groups vertically proximate anchor lines into table candidates, then scores each group
* on column consistency and anchor density (confidence ceiling 0.85).
*/
@Service
@Slf4j
public class LineAlignmentTableParser implements TableParser {
/** Width in points of each column position bucket. */
static final float COLUMN_BUCKET_PT = 5f;
/** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */
private static final int COLUMN_MATCH_BUCKETS = 2;
/** Maximum gap (as a multiple of modal line spacing) before splitting a group. */
private static final float MAX_GAP_FACTOR = 2.5f;
/** Minimum anchor rows (numeric-heavy) to form a valid table. */
static final int MIN_TABLE_ROWS = 3;
/** Minimum confirmed column positions to form a valid table. */
static final int MIN_COLUMNS = 2;
/**
* Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A
* rows).
*/
private static final double COLUMN_MIN_FREQUENCY = 0.40;
/**
* Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency,
* percent, nil dashes.
*/
private static final Pattern NUMERIC =
Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$");
/**
* Lines within this y-distance are merged into one row (restores rows split by LineBuilder's
* column-gap logic).
*/
static final float ROW_MERGE_TOLERANCE_PT = 2f;
// ── public API ───────────────────────────────────────────────────────────────────────────────
@Override
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
List<RawLine> lines = rawPage.lines();
if (lines.size() < MIN_TABLE_ROWS) return List.of();
float modalSpacing = computeModalSpacing(lines);
List<TokenizedLine> tokenized =
mergeCoincidentLines(lines.stream().map(this::tokenize).toList());
List<TokenizedLine> anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList();
if (anchors.size() < MIN_TABLE_ROWS) return List.of();
List<Float> columnGrid = buildColumnGrid(anchors);
if (columnGrid.size() < MIN_COLUMNS) {
log.debug(
"Page {}: LineAlignment — fewer than {} confirmed columns, skipping",
rawPage.pageNumber(),
MIN_COLUMNS);
return List.of();
}
List<List<TokenizedLine>> groups = groupRows(tokenized, columnGrid, modalSpacing);
List<TableFragment> results = new ArrayList<>();
for (int i = 0; i < groups.size(); i++) {
buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i)
.ifPresent(results::add);
}
log.debug(
"Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)",
rawPage.pageNumber(),
results.size(),
anchors.size(),
columnGrid.size());
return results;
}
// ── coincident-line merging ──────────────────────────────────────────────────────────────────
/**
* Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves
* split by LineBuilder.
*/
List<TokenizedLine> mergeCoincidentLines(List<TokenizedLine> tokenized) {
if (tokenized.size() < 2) return tokenized;
List<TokenizedLine> result = new ArrayList<>();
int i = 0;
while (i < tokenized.size()) {
float baseY = tokenized.get(i).line().bounds().y();
int j = i + 1;
while (j < tokenized.size()
&& Math.abs(tokenized.get(j).line().bounds().y() - baseY)
<= ROW_MERGE_TOLERANCE_PT) {
j++;
}
if (j == i + 1) {
result.add(tokenized.get(i));
} else {
result.add(mergeGroup(tokenized.subList(i, j)));
}
i = j;
}
return result;
}
private TokenizedLine mergeGroup(List<TokenizedLine> group) {
List<TextFragment> mergedFragments =
group.stream()
.flatMap(tl -> tl.line().fragments().stream())
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
.toList();
Bounds mergedBounds =
group.stream()
.map(tl -> tl.line().bounds())
.reduce(Bounds::merge)
.orElse(group.get(0).line().bounds());
RawLine mergedLine =
new RawLine(
group.get(0).line().lineId(),
mergedFragments,
mergedBounds,
group.get(0).line().pageNumber());
return tokenize(mergedLine);
}
// ── tokenisation ─────────────────────────────────────────────────────────────────────────────
/**
* Splits fragments into word-level tokens; x-positions are estimated linearly within each
* fragment.
*/
TokenizedLine tokenize(RawLine line) {
List<LineToken> tokens = new ArrayList<>();
for (TextFragment frag : line.fragments()) {
tokens.addAll(tokensFromFragment(frag));
}
List<LineToken> numeric = tokens.stream().filter(LineToken::numeric).toList();
return new TokenizedLine(line, tokens, numeric);
}
private List<LineToken> tokensFromFragment(TextFragment frag) {
String raw = frag.text();
if (raw == null || raw.isBlank()) return List.of();
float fragX = frag.bounds().x();
float fragWidth = frag.bounds().width();
int rawLen = raw.length();
List<LineToken> result = new ArrayList<>();
int offset = 0;
for (String part : raw.split("\\s+")) {
if (part.isEmpty()) {
offset++;
continue;
}
int idx = raw.indexOf(part, offset);
if (idx < 0) idx = offset;
float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX;
float tokenRight =
rawLen > 0
? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth
: fragX + fragWidth;
result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches()));
offset = idx + part.length();
}
return result;
}
// ── column grid ──────────────────────────────────────────────────────────────────────────────
/**
* Returns confirmed column right-edge positions — those appearing on ≥ {@value
* #COLUMN_MIN_FREQUENCY} × N anchor lines.
*/
private List<Float> buildColumnGrid(List<TokenizedLine> anchors) {
// bucket → set of line indices that contributed a numeric token to that bucket
Map<Integer, List<Integer>> bucketLines = new HashMap<>();
for (int i = 0; i < anchors.size(); i++) {
for (LineToken t : anchors.get(i).numeric()) {
int bucket = bucket(t.right());
bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i);
}
}
int minHits =
Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY));
// Confirmed buckets → average right-edge for that bucket
TreeMap<Integer, Float> confirmed = new TreeMap<>();
for (Map.Entry<Integer, List<Integer>> entry : bucketLines.entrySet()) {
// Count distinct lines
long distinctLines = entry.getValue().stream().distinct().count();
if (distinctLines >= minHits) {
double avg =
entry.getValue().stream()
.distinct() // weight each line equally regardless of token count
.mapToDouble(
lineIdx ->
avgRightEdgeForBucket(
anchors, lineIdx, entry.getKey()))
.average()
.orElse(entry.getKey() * (double) COLUMN_BUCKET_PT);
confirmed.put(entry.getKey(), (float) avg);
}
}
return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right)
}
/**
* Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code
* targetBucket}, falling back to the bucket's nominal centre when no tokens match.
*/
private double avgRightEdgeForBucket(
List<TokenizedLine> anchors, int lineIdx, int targetBucket) {
return anchors.get(lineIdx).numeric().stream()
.filter(t -> bucket(t.right()) == targetBucket)
.mapToDouble(LineToken::right)
.average()
.orElse(targetBucket * (double) COLUMN_BUCKET_PT);
}
// ── grouping ─────────────────────────────────────────────────────────────────────────────────
/**
* Groups anchor lines into table candidates, including adjacent label rows; a gap &gt;
* MAX_GAP_FACTOR × modal spacing splits groups.
*/
private List<List<TokenizedLine>> groupRows(
List<TokenizedLine> all, List<Float> columnGrid, float modalSpacing) {
float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f;
List<List<TokenizedLine>> groups = new ArrayList<>();
List<TokenizedLine> current = new ArrayList<>();
for (int i = 0; i < all.size(); i++) {
TokenizedLine tl = all.get(i);
boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid);
if (current.isEmpty()) {
if (fits) current.add(tl);
continue;
}
float gap =
tl.line().bounds().y()
- current.get(current.size() - 1).line().bounds().bottom();
if (gap > maxGap) {
groups.add(current);
current = new ArrayList<>();
if (fits) current.add(tl);
continue;
}
if (fits) {
current.add(tl);
} else if (!tl.line().text().isBlank()) {
// Include non-anchor lines (labels) only if they have text and are within
// proximity.
current.add(tl);
}
}
if (!current.isEmpty()) groups.add(current);
return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList();
}
private boolean hasEnoughAnchorRows(List<TokenizedLine> group, List<Float> columnGrid) {
return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count()
>= MIN_TABLE_ROWS;
}
/** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */
private boolean matchesGrid(TokenizedLine tl, List<Float> columnGrid) {
if (tl.numeric().isEmpty()) return false;
long matches =
tl.numeric().stream()
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
.count();
return (double) matches / tl.numeric().size() >= 0.60;
}
private boolean hasInconsistentColumnMatch(TokenizedLine tl, List<Float> columnGrid) {
if (tl.numeric().isEmpty()) return false;
long hits =
tl.numeric().stream()
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
.count();
return (double) hits / tl.numeric().size() < 0.60;
}
// ── fragment assembly ────────────────────────────────────────────────────────────────────────
private Optional<TableFragment> buildFragment(
List<TokenizedLine> group, List<Float> columnGrid, int pageNumber, int tableIndex) {
long anchorCount =
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
if (anchorCount < MIN_TABLE_ROWS) return Optional.empty();
List<String> warnings = new ArrayList<>();
List<List<String>> rawRows = new ArrayList<>();
List<TableRow> rows = new ArrayList<>();
for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) {
TokenizedLine tl = group.get(rowIdx);
List<String> rawRow = buildRawRow(tl, columnGrid);
rawRows.add(Collections.unmodifiableList(rawRow));
rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid));
}
// Column count = 1 label column + confirmed numeric columns
int colCount = columnGrid.size() + 1;
Bounds bounds = computeGroupBounds(group);
float confidence = computeConfidence(group, columnGrid, warnings);
return Optional.of(
new TableFragment(
"tbl-la-p" + pageNumber + "-" + tableIndex,
pageNumber,
bounds,
List.of(),
Collections.unmodifiableList(rows),
Collections.unmodifiableList(rawRows),
colCount,
confidence,
Collections.unmodifiableList(warnings),
null));
}
/**
* Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values.
*/
private List<String> buildRawRow(TokenizedLine tl, List<Float> columnGrid) {
String[] cells = new String[columnGrid.size() + 1];
Arrays.fill(cells, "");
// Separate label tokens (those not landing in any confirmed column) from column tokens.
List<String> labelParts = new ArrayList<>();
for (LineToken token : tl.all()) {
int col = nearestColumnIndex(token.right(), columnGrid);
if (col >= 0 && token.numeric()) {
int cellIdx = col + 1;
cells[cellIdx] =
cells[cellIdx].isEmpty()
? token.text()
: cells[cellIdx] + " " + token.text();
} else {
labelParts.add(token.text());
}
}
cells[0] = String.join(" ", labelParts).trim();
return Arrays.asList(cells);
}
private TableRow buildTableRow(
int rowIdx, TokenizedLine tl, List<String> rawRow, List<Float> columnGrid) {
List<TableCell> cells = new ArrayList<>(rawRow.size());
// Label cell: use the line's full bounds as an approximation.
cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds()));
for (int col = 0; col < columnGrid.size(); col++) {
String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : "";
float right = columnGrid.get(col);
float left = col > 0 ? columnGrid.get(col - 1) : right - 50f;
Bounds cellBounds =
new Bounds(
left,
tl.line().bounds().y(),
right - left,
tl.line().bounds().height());
cells.add(TableCell.of(col + 1, text, cellBounds));
}
return new TableRow(rowIdx, Collections.unmodifiableList(cells));
}
// ── confidence scoring ───────────────────────────────────────────────────────────────────────
/**
* Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at
* 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if
* &gt;30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors.
*/
private float computeConfidence(
List<TokenizedLine> group, List<Float> columnGrid, List<String> warnings) {
float score = 0.70f;
long anchorCount =
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
long totalRows = group.size();
// More columns
int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2);
score += extraCols * 0.05f;
// More anchor rows
if (anchorCount >= 5) score += 0.05f;
if (anchorCount >= 8) score += 0.05f;
// Inconsistent column matching
long inconsistent =
group.stream()
.filter(TokenizedLine::isAnchor)
.filter(tl -> hasInconsistentColumnMatch(tl, columnGrid))
.count();
if (inconsistent > anchorCount * 0.30) {
score -= 0.15f;
warnings.add(
"Column match inconsistent on "
+ inconsistent
+ "/"
+ anchorCount
+ " anchor rows");
}
// Label-heavy
long nonAnchor = totalRows - anchorCount;
if (nonAnchor > anchorCount) {
score -= 0.10f;
warnings.add(
"Non-anchor rows ("
+ nonAnchor
+ ") outnumber anchor rows ("
+ anchorCount
+ ")");
}
return Math.max(0f, Math.min(0.85f, score));
}
// ── utility ──────────────────────────────────────────────────────────────────────────────────
/**
* Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value
* #COLUMN_MATCH_BUCKETS} buckets.
*/
private int nearestColumnIndex(float rightEdge, List<Float> grid) {
int nearest = -1;
float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f;
for (int i = 0; i < grid.size(); i++) {
float dist = Math.abs(rightEdge - grid.get(i));
if (dist < minDist) {
minDist = dist;
nearest = i;
}
}
return nearest;
}
private Bounds computeGroupBounds(List<TokenizedLine> group) {
return group.stream()
.map(tl -> tl.line().bounds())
.reduce(Bounds::merge)
.orElse(new Bounds(0, 0, 0, 0));
}
/** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */
private float computeModalSpacing(List<RawLine> lines) {
if (lines.size() < 2) return 0f;
Map<Float, Long> freq = new HashMap<>();
for (int i = 1; i < lines.size(); i++) {
float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom();
if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum);
}
return freq.entrySet().stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse(0f);
}
private static int bucket(float x) {
return Math.round(x / COLUMN_BUCKET_PT);
}
// ── private data types ───────────────────────────────────────────────────────────────────────
/** A word-level token with an approximate right-edge x-position. */
record LineToken(String text, float x, float right, boolean numeric) {}
/** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */
record TokenizedLine(RawLine line, List<LineToken> all, List<LineToken> numeric) {
boolean isAnchor() {
return numeric.size() >= 2;
}
}
}
app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java
-139
View File
@@ -1,139 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
/**
* Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity.
*
* <p>Fragments are on the same line when their baselines are within a font-size-derived tolerance.
* A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code
* max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text.
*/
@Service
@Slf4j
public class LineBuilder {
/** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */
private static final float BASELINE_TOLERANCE_FACTOR = 0.5f;
/** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */
private static final float MIN_BASELINE_TOLERANCE = 2f;
/**
* Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below
* two-column gutters.
*/
static final float COLUMN_GAP_RATIO = 0.10f;
/** Floor for the column-gap threshold so narrow pages don't over-split lines. */
static final float COLUMN_GAP_MIN_PT = 40f;
public List<RawLine> build(List<TextFragment> fragments, int pageNumber) {
if (fragments.isEmpty()) return List.of();
float effectiveWidth = inferEffectiveWidth(fragments);
float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT);
log.debug(
"LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt",
pageNumber,
effectiveWidth,
columnGapThreshold);
// Sort top-to-bottom first, then left-to-right within the same baseline band.
List<TextFragment> sorted =
fragments.stream()
.sorted(
Comparator.comparingDouble(TextFragment::baseline)
.thenComparingDouble(f -> f.bounds().x()))
.toList();
List<List<TextFragment>> groups = groupByBaseline(sorted, columnGapThreshold);
List<RawLine> lines = new ArrayList<>(groups.size());
for (int i = 0; i < groups.size(); i++) {
List<TextFragment> group =
groups.get(i).stream()
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
.toList();
Bounds lineBounds =
group.stream()
.map(TextFragment::bounds)
.reduce(Bounds::merge)
.orElse(new Bounds(0, 0, 0, 0));
lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber));
}
return lines;
}
private List<List<TextFragment>> groupByBaseline(
List<TextFragment> sorted, float columnGapThreshold) {
List<List<TextFragment>> groups = new ArrayList<>();
List<TextFragment> current = new ArrayList<>();
float currentBaseline = Float.NaN;
for (TextFragment fragment : sorted) {
if (current.isEmpty()) {
current.add(fragment);
currentBaseline = fragment.baseline();
continue;
}
float maxFontSize =
Math.max(
fragment.fontSize(),
(float)
current.stream()
.mapToDouble(TextFragment::fontSize)
.max()
.orElse(0));
float tolerance =
Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE);
boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance;
boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold);
if (sameBaseline && !columnGap) {
current.add(fragment);
// Anchor to the weighted mean baseline so long lines stay stable.
currentBaseline =
(currentBaseline * (current.size() - 1) + fragment.baseline())
/ current.size();
} else {
groups.add(current);
current = new ArrayList<>();
current.add(fragment);
currentBaseline = fragment.baseline();
}
}
if (!current.isEmpty()) groups.add(current);
return groups;
}
/**
* True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code
* threshold}.
*/
private static boolean hasColumnGap(
TextFragment next, List<TextFragment> group, float threshold) {
float lastRight = group.get(group.size() - 1).bounds().right();
return next.bounds().x() - lastRight > threshold;
}
/** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */
private static float inferEffectiveWidth(List<TextFragment> fragments) {
double maxRight =
fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0);
return (float) maxRight * 1.10f;
}
}
app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java
-79
View File
@@ -1,79 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} →
* {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the
* {@link PDDocument} lifecycle.
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class PdfIngester {
private final LineBuilder lineBuilder;
private final TableParser tableParser;
public List<ParsedPage> parse(PDDocument document) throws IOException {
return parse(document, document.getNumberOfPages());
}
public List<ParsedPage> parse(PDDocument document, int maxPages) throws IOException {
int pageCount = Math.min(document.getNumberOfPages(), maxPages);
List<ParsedPage> pages = new ArrayList<>(pageCount);
long fragmentsMs = 0;
long tablesMs = 0;
long t0 = System.currentTimeMillis();
for (int p = 1; p <= pageCount; p++) {
long ft = System.currentTimeMillis();
List<TextFragment> fragments = extractFragments(document, p);
fragmentsMs += System.currentTimeMillis() - ft;
PDPage page = document.getPage(p - 1);
PDRectangle mediaBox = page.getMediaBox();
List<RawLine> lines = lineBuilder.build(fragments, p);
RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines);
long tt = System.currentTimeMillis();
List<TableFragment> tables = tableParser.parse(document, rawPage);
tablesMs += System.currentTimeMillis() - tt;
log.debug(
"Page {}: {} fragments → {} lines, {} table(s)",
p,
fragments.size(),
lines.size(),
tables.size());
pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines));
}
log.info(
"[timing] parse pages={} total={}ms fragments={}ms tables={}ms",
pageCount,
System.currentTimeMillis() - t0,
fragmentsMs,
tablesMs);
return pages;
}
private List<TextFragment> extractFragments(PDDocument document, int pageNumber)
throws IOException {
WordExtractingStripper stripper = new WordExtractingStripper(pageNumber);
stripper.getText(document);
return stripper.getFragments();
}
}
app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java
-113
View File
@@ -1,113 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.TextPosition;
/**
* Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata.
*
* <p>Overrides {@link #writeString} to split each content-stream string into word-level {@link
* TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in
* PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline.
*/
class WordExtractingStripper extends PDFTextStripper {
private final int targetPage;
private final List<TextFragment> fragments = new ArrayList<>();
private int fragmentIndex = 0;
WordExtractingStripper(int pageNumber) throws IOException {
this.targetPage = pageNumber;
setStartPage(pageNumber);
setEndPage(pageNumber);
setSortByPosition(true);
}
@Override
protected void startPage(PDPage page) throws IOException {
super.startPage(page);
fragments.clear();
fragmentIndex = 0;
}
@Override
protected void writeString(String text, List<TextPosition> textPositions) throws IOException {
if (text == null || text.isBlank()) return;
// Fast path: no whitespace → emit one fragment (most financial PDFs have each
// number as its own string operation, so this is the common case).
if (text.indexOf(' ') < 0) {
emitFragment(text, textPositions);
return;
}
// Per-word splitting requires 1:1 text-char to TextPosition correspondence.
// Fall back to one fragment when sizes differ (ligatures, encoding edge cases).
if (textPositions.size() != text.length()) {
emitFragment(text, textPositions);
return;
}
// Emit one TextFragment per whitespace-delimited word with accurate per-word bounds.
int start = 0;
for (int i = 0; i <= text.length(); i++) {
if (i == text.length() || text.charAt(i) == ' ') {
if (start < i) {
emitFragment(text.substring(start, i), textPositions.subList(start, i));
}
start = i + 1;
}
}
}
private void emitFragment(String text, List<TextPosition> positions) {
if (positions.isEmpty()) return;
float minX = Float.MAX_VALUE;
float minY = Float.MAX_VALUE;
float maxRight = -Float.MAX_VALUE;
float maxBaseline = -Float.MAX_VALUE;
TextPosition first = null;
for (TextPosition tp : positions) {
if (tp == null) continue;
if (first == null) first = tp;
float x = tp.getX();
// getY() is the baseline; top of character = getY() - getHeight().
float top = tp.getY() - tp.getHeight();
float right = x + tp.getWidth();
float baseline = tp.getY();
minX = Math.min(minX, x);
minY = Math.min(minY, top);
maxRight = Math.max(maxRight, right);
maxBaseline = Math.max(maxBaseline, baseline);
}
if (first == null) return;
PDFont font = first.getFont();
String fontName = font != null ? font.getName() : "";
boolean bold = fontName != null && fontName.toLowerCase().contains("bold");
// getHeight() gives the rendered glyph height, which is the most reliable visual size.
float fontSize = first.getHeight();
Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY);
String id = "tf-p" + targetPage + "-" + fragmentIndex++;
fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold));
}
List<TextFragment> getFragments() {
return Collections.unmodifiableList(fragments);
}
}
app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java
+40
View File
@@ -80,6 +80,7 @@ public class ApplicationProperties {
private Mcp mcp = new Mcp();
private InternalApi internalApi = new InternalApi();
private Cluster cluster = new Cluster();
private Policies policies = new Policies();
@Bean
public PropertySource<?> dynamicYamlPropertySource(ConfigurableEnvironment environment)
@@ -203,6 +204,45 @@ public class ApplicationProperties {
}
}
@Data
public static class Policies {
/**
* Absolute directories that policy folder input sources and output sinks may read from or
* write to. Empty (the default) disables folder access entirely, so a policy can never be
* pointed at an arbitrary server path. Stirling's own config directory is always
* off-limits, and folder access is always disabled in SaaS mode regardless of this list.
*/
private List<String> allowedFolderRoots = new java.util.ArrayList<>();
/** How often (seconds) the schedule trigger checks for policies whose schedule is due. */
private long scheduleSweepSeconds = 60;
/**
* How often (seconds) the folder-watch trigger reconciles its watch registrations and
* re-runs every folder-watch policy as a safety net for filesystem events that were missed
* (NFS, bind mounts, inotify-queue overflow).
*/
private long watchReconcileSeconds = 300;
/**
* How long (milliseconds) the folder-watch trigger keeps draining filesystem events after
* the first, so a burst from a single file copy coalesces into one run instead of many.
*/
private long watchQuietPeriodMs = 500;
/**
* SSE emitter timeout (milliseconds) for streamed runs; generous for long multi-step runs.
*/
private long streamTimeoutMs = 1800000;
/**
* How long (minutes) a finished run's in-memory state is retained before eviction,
* mirroring the job-result expiry so rich run state does not outlive the process. Active
* and paused runs are kept regardless of age.
*/
private int runExpiryMinutes = 30;
}
@Data
public static class PdfEditor {
private Cache cache = new Cache();
app/common/src/main/java/stirling/software/common/pdf/HeadingDetector.java
+191
View File
@@ -0,0 +1,191 @@
package stirling.software.common.pdf;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import stirling.software.jpdfium.text.PageText;
import stirling.software.jpdfium.text.TextChar;
import stirling.software.jpdfium.text.TextLine;
import stirling.software.jpdfium.text.TextWord;
final class HeadingDetector {
private HeadingDetector() {}
/** A heading is at most this many words; longer lines are treated as body text. */
private static final int MAX_HEADING_WORDS = 12;
/**
* Returns the Markdown heading prefix for a line. The decision combines several signals, never
* text matching, so a plain line that merely shares text with a heading is never promoted:
*
* <ul>
* <li><b>Size</b> — dominant glyph font size vs. the document body median (primary signal).
* Some PDFs encode visual size in the text matrix, so every glyph reports ~1.0; for those
* the line height is used as the proxy instead.
* <li><b>Brevity</b> — headings are short labels; a line over {@value #MAX_HEADING_WORDS}
* words is body text regardless of size.
* <li><b>Not a sentence</b> — a line ending in {@code . ! ?} reads as prose, not a heading.
* </ul>
*
* <p>Boldness is deliberately <em>not</em> a heading signal — a bold-but-not-larger line is
* emphasis, not a heading (see {@link #isBoldLabel}); promoting it to {@code #}/{@code ##} is
* the main source of false-positive headings.
*
* <ul>
* <li>size &gt; baseline * 1.4 → {@code "# "}
* <li>size &gt; baseline * 1.2 → {@code "## "}
* <li>otherwise → {@code ""}
* </ul>
*/
static String headingPrefix(TextLine line, float medianBodySize, float medianBodyHeight) {
String text = line.text().strip();
if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) {
return "";
}
float dominant = dominantFontSize(line);
float value;
float baseline;
if (dominant > 2f && medianBodySize > 2f) {
value = dominant;
baseline = medianBodySize;
} else {
value = line.height();
baseline = medianBodyHeight;
}
if (baseline <= 0f) {
return "";
}
float ratio = value / baseline;
if (ratio > 1.4f) {
return "# ";
}
if (ratio > 1.2f) {
return "## ";
}
return "";
}
/**
* True when a line should be emphasised as bold (rendered {@code **like this**}) rather than
* promoted to a heading: it is bold, short, and not a full sentence. Used for bold labels that
* are not large enough to be headings.
*/
static boolean isBoldLabel(TextLine line) {
String text = line.text().strip();
if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) {
return false;
}
return isBold(line);
}
private static int wordCount(String text) {
return text.split("\\s+").length;
}
private static boolean endsLikeSentence(String text) {
char last = text.charAt(text.length() - 1);
return last == '.' || last == '!' || last == '?';
}
/** True when the line's dominant font is bold, inferred from PostScript font names. */
private static boolean isBold(TextLine line) {
Map<String, Integer> counts = new HashMap<>();
for (TextWord word : line.words()) {
for (TextChar ch : word.chars()) {
if (ch.isWhitespace() || ch.isNewline()) {
continue;
}
String name = ch.fontName();
if (name != null && !name.isBlank()) {
counts.merge(name, 1, Integer::sum);
}
}
}
String dominantFont = "";
int max = -1;
for (Map.Entry<String, Integer> e : counts.entrySet()) {
if (e.getValue() > max) {
max = e.getValue();
dominantFont = e.getKey();
}
}
String lower = dominantFont.toLowerCase(java.util.Locale.ROOT);
return lower.contains("bold")
|| lower.contains("black")
|| lower.contains("heavy")
|| lower.contains("semibold");
}
/** Computes the median glyph font size across all pages. */
static float medianFontSize(List<PageText> allPages) {
List<Float> sizes = new ArrayList<>();
for (PageText page : allPages) {
for (TextChar ch : page.chars()) {
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
sizes.add(ch.fontSize());
}
}
}
return median(sizes, 12f);
}
/** Computes the median TextLine height across all pages. Used when font size is degenerate. */
static float medianLineHeight(List<PageText> allPages) {
List<Float> heights = new ArrayList<>();
for (PageText page : allPages) {
for (TextLine line : page.lines()) {
if (line.height() > 0f && !line.text().isBlank()) {
heights.add(line.height());
}
}
}
return median(heights, 12f);
}
private static float median(List<Float> values, float fallback) {
if (values.isEmpty()) {
return fallback;
}
Collections.sort(values);
int mid = values.size() / 2;
if (values.size() % 2 == 0) {
return (values.get(mid - 1) + values.get(mid)) / 2f;
}
return values.get(mid);
}
/**
* Returns the font size that appears most often (by character count) in the given line. Ties
* are broken in favour of the larger size.
*/
private static float dominantFontSize(TextLine line) {
Map<Float, Integer> counts = new HashMap<>();
for (TextWord word : line.words()) {
for (TextChar ch : word.chars()) {
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
counts.merge(ch.fontSize(), 1, Integer::sum);
}
}
}
if (counts.isEmpty()) {
return 0f;
}
float dominant = 0f;
int maxCount = -1;
for (Map.Entry<Float, Integer> entry : counts.entrySet()) {
int count = entry.getValue();
float size = entry.getKey();
if (count > maxCount || (count == maxCount && size > dominant)) {
maxCount = count;
dominant = size;
}
}
return dominant;
}
}
app/common/src/main/java/stirling/software/common/pdf/PdfMarkdownConverter.java
+1043
View File
File diff suppressed because it is too large Load Diff
app/common/src/main/java/stirling/software/common/pdf/TableRenderer.java
+82
View File
@@ -0,0 +1,82 @@
package stirling.software.common.pdf;
import stirling.software.jpdfium.text.Table;
final class TableRenderer {
private TableRenderer() {}
/** Renders a Table as a GitHub-Flavoured Markdown table string. */
static String render(Table table) {
if (table.rowCount() == 0) {
return "";
}
String[][] grid = table.asGrid();
if (table.rowCount() < 2) {
// No separator row possible — return plain lines
StringBuilder sb = new StringBuilder();
for (int c = 0; c < grid[0].length; c++) {
if (c > 0) sb.append('\n');
sb.append(escape(grid[0][c].trim()));
}
return sb.toString();
}
int cols = grid[0].length;
// Compute column widths: max(3, max content length across all rows)
int[] widths = new int[cols];
for (int c = 0; c < cols; c++) {
widths[c] = 3;
}
for (String[] row : grid) {
for (int c = 0; c < cols; c++) {
String cell = c < row.length ? row[c].trim() : "";
widths[c] = Math.max(widths[c], escape(cell).length());
}
}
StringBuilder sb = new StringBuilder();
// Header row
sb.append(buildRow(grid[0], widths, cols));
sb.append('\n');
// Separator row
sb.append('|');
for (int c = 0; c < cols; c++) {
sb.append('-').append("-".repeat(widths[c])).append('-').append('|');
}
sb.append('\n');
// Data rows
for (int r = 1; r < grid.length; r++) {
sb.append(buildRow(grid[r], widths, cols));
if (r < grid.length - 1) {
sb.append('\n');
}
}
return sb.toString();
}
private static String buildRow(String[] row, int[] widths, int cols) {
StringBuilder sb = new StringBuilder();
sb.append('|');
for (int c = 0; c < cols; c++) {
String cell = c < row.length ? escape(row[c].trim()) : "";
sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|');
}
return sb.toString();
}
private static String escape(String cell) {
return cell.replace("|", "\\|");
}
private static String padRight(String s, int width) {
if (s.length() >= width) return s;
return s + " ".repeat(width - s.length());
}
}
app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java
-153
View File
@@ -1,153 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static org.assertj.core.api.Assertions.assertThat;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.util.List;
import org.junit.jupiter.api.Test;
/**
* Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and
* column-grid construction.
*/
class LineAlignmentTableParserTest {
private final LineAlignmentTableParser parser = new LineAlignmentTableParser();
// ── mergeCoincidentLines ─────────────────────────────────────────────────────────────────────
@Test
void mergeCoincidentLines_singleLine_unchanged() {
var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue")));
assertThat(parser.mergeCoincidentLines(lines)).hasSize(1);
}
@Test
void mergeCoincidentLines_distinctYLines_unchanged() {
// Two lines at different y positions — must NOT be merged.
var lines =
List.of(
tokenized(rawLine(10f, 100f, "Revenue")),
tokenized(rawLine(10f, 115f, "Cost")));
assertThat(parser.mergeCoincidentLines(lines)).hasSize(2);
}
@Test
void mergeCoincidentLines_sameY_merged() {
// Simulates a financial-table row split by LineBuilder at the column gap:
// label fragment at x=72 → "Revenue"
// value fragment at x=350 → "1,234"
// Both have y=100. After merge they should form one TokenizedLine.
var label = rawLine(72f, 100f, "Revenue");
var value = rawLine(350f, 100f, "1,234");
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
assertThat(merged).hasSize(1);
// The merged line should contain tokens from both halves.
var tokens = merged.get(0).all();
assertThat(tokens.stream().map(t -> t.text()).toList())
.containsExactlyInAnyOrder("Revenue", "1,234");
}
@Test
void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() {
var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114
var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
var bounds = merged.get(0).line().bounds();
assertThat(bounds.x()).isEqualTo(72f);
assertThat(bounds.right()).isEqualTo(380f);
}
@Test
void mergeCoincidentLines_withinTolerance_merged() {
// Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge.
var a = rawLine(10f, 100.0f, "Alpha");
var b = rawLine(200f, 101.5f, "99");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
assertThat(merged).hasSize(1);
}
@Test
void mergeCoincidentLines_beyondTolerance_notMerged() {
// Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge.
var a = rawLine(10f, 100.0f, "Alpha");
var b = rawLine(200f, 103.0f, "99");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
assertThat(merged).hasSize(2);
}
@Test
void mergeCoincidentLines_threeCoincident_allMerged() {
// Three fragments at the same y (e.g. wide financial table with two value columns).
var a = rawLine(72f, 100f, "Revenue");
var b = rawLine(300f, 100f, "1,234");
var c = rawLine(400f, 100f, "5,678");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
assertThat(merged).hasSize(1);
assertThat(merged.get(0).all()).hasSize(3);
}
@Test
void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() {
var a = rawLine(72f, 100f, "Revenue");
var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a
var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
assertThat(merged).hasSize(2);
}
@Test
void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() {
// After merging, the combined line should be an anchor (≥2 numeric tokens).
// "Revenue" alone → not an anchor. "1,234 567" alone → anchor.
// Merged → anchor with at least 2 numerics.
var label = rawLine(72f, 100f, "Revenue");
var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f);
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values)));
assertThat(merged).hasSize(1);
assertThat(merged.get(0).isAnchor()).isTrue();
}
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
/** Creates a RawLine with a single TextFragment of the given text at the given position. */
private static RawLine rawLine(float x, float y, String text) {
float width = text.length() * 6f; // ~6pt per char — rough but consistent
float height = 12f;
Bounds bounds = new Bounds(x, y, width, height);
TextFragment fragment =
new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false);
return new RawLine("ln-test", List.of(fragment), bounds, 1);
}
/**
* Creates a RawLine with two TextFragments representing two words separated by a small gap.
* Used to simulate a values-only line with multiple numeric tokens.
*/
private static RawLine rawLineMultiWord(
float x, float y, String word1, float w1, String word2, float w2) {
float height = 12f;
Bounds b1 = new Bounds(x, y, w1, height);
Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height);
TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false);
TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false);
Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height);
return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1);
}
/** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */
private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) {
return parser.tokenize(line);
}
}
app/common/src/test/java/stirling/software/common/pdf/PdfMarkdownConverterTest.java
+269
View File
@@ -0,0 +1,269 @@
package stirling.software.common.pdf;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import stirling.software.jpdfium.PdfDocument;
import stirling.software.jpdfium.text.TextLine;
import stirling.software.jpdfium.text.TextWord;
/**
* Accuracy and robustness tests for {@link PdfMarkdownConverter}, comparing conversion output
* against hand-authored golden Markdown for a set of owned/synthetic fixtures.
*
* <p>The {@link #gatedFixtures()} set is enforced in CI: those fixtures currently convert within
* the accuracy threshold and guard against regressions. Fixtures still being iterated on live in
* {@link #wipFixtures()} under a {@link Disabled} test so the goldens stay in the tree without
* breaking the build. Enable the WIP test locally to see per-fixture scores while working on the
* converter.
*/
class PdfMarkdownConverterTest {
/** Accuracy threshold: output must share at least this fraction of content with the golden. */
private static final double THRESHOLD = 0.95;
@TempDir Path tmp;
/** Fixtures that meet the accuracy threshold today and therefore gate CI. */
static Stream<Arguments> gatedFixtures() {
return Stream.of(
Arguments.of("multi-column-test_lorem.pdf", "multi-column-test_lorem.md"),
Arguments.of("bordered-table-test_widget.pdf", "bordered-table-test_widget.md"),
Arguments.of("many-tables-test_stress.pdf", "many-tables-test_stress.md"));
}
/** Fixtures still below the threshold; tracked here, enable locally to iterate. */
static Stream<Arguments> wipFixtures() {
return Stream.of(
Arguments.of(
"wrapped-cell-test_expense-report.pdf",
"wrapped-cell-test_expense-report.md"));
}
@ParameterizedTest(name = "{0}")
@MethodSource("gatedFixtures")
void convertMatchesGoldenMarkdown(String pdfName, String mdName) throws IOException {
assertConversionMatchesGolden(pdfName, mdName);
}
@Disabled("WIP fixtures below the accuracy threshold; enable locally to iterate")
@ParameterizedTest(name = "{0}")
@MethodSource("wipFixtures")
void convertMatchesGoldenMarkdownWip(String pdfName, String mdName) throws IOException {
assertConversionMatchesGolden(pdfName, mdName);
}
/**
* Degenerate/extreme geometry must not crash the converter. A crafted or malformed PDF can
* position text anywhere via a text matrix, so a row's words can span from near the origin to a
* coordinate beyond {@link Integer#MAX_VALUE}. The old column-detection code sized an {@code
* int[]} straight from {@code (int) Math.ceil(maxX) - lo}, which either allocated a multi-GB
* array (OutOfMemoryError) or overflowed to a negative length (NegativeArraySizeException) —
* taking down the request thread. Detection must instead bail out and return no columns.
*/
@Test
void columnDetectionSurvivesDegenerateGeometry() {
// x ≈ 2.5e9 is past Integer.MAX_VALUE; combined with a near-origin word it yields an
// implausible span that the pre-fix code turned into a fatal array allocation.
List<TextLine> rows = new ArrayList<>();
for (int r = 0; r < 4; r++) {
float y = 400f - r * 12f;
TextWord near = new TextWord(List.of(), 50f, y, 30f, 10f);
TextWord far = new TextWord(List.of(), 2_500_000_000f, y, 30f, 10f);
rows.add(new TextLine(List.of(near, far), 50f, y, 2_499_999_980f, 10f));
}
List<float[]> columns =
assertDoesNotThrow(() -> PdfMarkdownConverter.findColumnRangesFromLines(rows));
assertTrue(
columns.isEmpty(),
"implausible page span should disable column detection, not allocate from it");
}
private void assertConversionMatchesGolden(String pdfName, String mdName) throws IOException {
Path pdfPath = tmp.resolve(pdfName);
try (InputStream in =
getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + pdfName)) {
if (in == null) {
fail("Fixture not found on classpath: /pdf-ingestion-fixtures/" + pdfName);
}
Files.copy(in, pdfPath);
}
String actual;
try (PdfDocument doc = PdfDocument.open(pdfPath)) {
actual = new PdfMarkdownConverter().convert(doc);
}
String expected;
try (InputStream in = getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + mdName)) {
if (in == null) {
fail("Golden file not found on classpath: /pdf-ingestion-fixtures/" + mdName);
}
expected = new String(in.readAllBytes(), StandardCharsets.UTF_8);
}
// Image placeholders are not scored: their body text is a TODO ("ideally, add the info
// available about the image...") rather than real content, so comparing it would penalise
// output for matching a placeholder we intend to replace. Drop those lines from both sides.
expected = stripImagePlaceholders(expected);
actual = stripImagePlaceholders(actual);
double similarity = similarity(expected, actual);
if (similarity < THRESHOLD) {
fail(
String.format(
"Markdown output differs from golden file '%s' by %.1f%% (threshold %.0f%%):%n%s",
mdName,
(1.0 - similarity) * 100,
(1.0 - THRESHOLD) * 100,
unifiedDiff(expected, actual)));
}
}
/** Substring identifying an image-placeholder line, which is excluded from scoring. */
private static final String IMAGE_PLACEHOLDER_MARKER = "Image intentionally redacted";
/**
* Removes non-content lines from the comparison: image placeholders (TODO text we intend to
* replace) and GFM table separator rows (the {@code |---|---|} divider, whose exact dash count
* is cosmetic — any run of three or more dashes is valid Markdown).
*/
private static String stripImagePlaceholders(String md) {
StringBuilder sb = new StringBuilder();
for (String line : md.split("\n", -1)) {
if (line.contains(IMAGE_PLACEHOLDER_MARKER)
|| line.strip().startsWith("<image redacted")
|| isTableSeparatorRow(line)) {
continue;
}
if (sb.length() > 0) {
sb.append('\n');
}
sb.append(line);
}
return sb.toString();
}
/** True for a GFM table separator row, e.g. {@code |---|:--:|---|} (only |, -, :, space). */
private static boolean isTableSeparatorRow(String line) {
String t = line.strip();
if (!t.contains("-")) {
return false;
}
return t.chars().allMatch(c -> c == '|' || c == '-' || c == ':' || c == ' ');
}
/**
* Character-level similarity: proportion of expected characters that appear in the LCS. O(n*m)
* but golden files are small enough that this is fine.
*/
private static double similarity(String expected, String actual) {
if (expected.isEmpty() && actual.isEmpty()) return 1.0;
if (expected.isEmpty() || actual.isEmpty()) return 0.0;
// Strip all whitespace for a content-focused comparison
String e = expected.replaceAll("\\s+", " ").strip();
String a = actual.replaceAll("\\s+", " ").strip();
int lcs = lcsLength(e, a);
return (double) lcs / Math.max(e.length(), a.length());
}
private static int lcsLength(String a, String b) {
// Use two-row DP to keep memory reasonable
int m = a.length(), n = b.length();
int[] prev = new int[n + 1];
int[] curr = new int[n + 1];
for (int i = 1; i <= m; i++) {
for (int j = 1; j <= n; j++) {
if (a.charAt(i - 1) == b.charAt(j - 1)) {
curr[j] = prev[j - 1] + 1;
} else {
curr[j] = Math.max(curr[j - 1], prev[j]);
}
}
int[] tmp = prev;
prev = curr;
curr = tmp;
java.util.Arrays.fill(curr, 0);
}
return prev[n];
}
private static String unifiedDiff(String expected, String actual) {
String[] expectedLines = expected.split("\n", -1);
String[] actualLines = actual.split("\n", -1);
List<String> diff = new ArrayList<>();
diff.add("--- expected");
diff.add("+++ actual");
int maxLines = Math.max(expectedLines.length, actualLines.length);
int context = 3;
boolean inHunk = false;
int hunkStart = -1;
List<String> hunkLines = new ArrayList<>();
for (int i = 0; i < maxLines; i++) {
String exp = i < expectedLines.length ? expectedLines[i] : null;
String act = i < actualLines.length ? actualLines[i] : null;
boolean changed = exp == null || act == null || !exp.equals(act);
if (changed) {
if (!inHunk) {
inHunk = true;
hunkStart = Math.max(0, i - context);
// add context lines before change
for (int c = hunkStart; c < i; c++) {
hunkLines.add(" " + (c < expectedLines.length ? expectedLines[c] : ""));
}
}
if (exp != null) hunkLines.add("-" + exp);
if (act != null) hunkLines.add("+" + act);
} else {
if (inHunk) {
hunkLines.add(" " + exp);
// check if we're far enough past the last change to close the hunk
boolean moreChanges = false;
for (int j = i + 1; j < Math.min(i + context, maxLines); j++) {
String e2 = j < expectedLines.length ? expectedLines[j] : null;
String a2 = j < actualLines.length ? actualLines[j] : null;
if (e2 == null || a2 == null || !e2.equals(a2)) {
moreChanges = true;
break;
}
}
if (!moreChanges && (i - hunkStart) >= context) {
diff.add("@@ -" + (hunkStart + 1) + " @@");
diff.addAll(hunkLines);
hunkLines.clear();
inHunk = false;
}
}
}
}
if (inHunk && !hunkLines.isEmpty()) {
diff.add("@@ -" + (hunkStart + 1) + " @@");
diff.addAll(hunkLines);
}
return String.join("\n", diff);
}
}
app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.md
+10
View File
@@ -0,0 +1,10 @@
# Widget Inventory Report
This report lists current stock levels for each warehouse.
| Region | Units | Status |
|---|---|---|
| North | 1200 | OK |
| South | 950 | Low |
| East | 1430 | OK |
| West | 875 | Low |
app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.pdf
+74
View File
@@ -0,0 +1,74 @@
%PDF-1.4
%“Œ‹ž ReportLab Generated PDF document (opensource)
1 0 obj
<<
/F1 2 0 R /F2 3 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
5 0 obj
<<
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
>>
endobj
6 0 obj
<<
/Author (\(anonymous\)) /CreationDate (D:20260603003133+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603003133+01'00') /Producer (ReportLab PDF Library - \(opensource\))
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
>>
endobj
7 0 obj
<<
/Count 1 /Kids [ 4 0 R ] /Type /Pages
>>
endobj
8 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 500
>>
stream
Gas1[9i&Y\%))C:pc(6t3;pQ8%0T@tD+-Nf,MP8j(><jDruNO^VsZ'm=ue7^Ia=;l(`h-+jUU3'Knh#YjH-MEIFj6r!oqf+FjMh;(rCq>R4tMCHL4NI*\1+UNiI'V9NC%VeJKn/YI0J];XQt&X83?=ihrg<*Mcn1n!1nWcDaQPe\P"9gnJuHl(jf]JQgZ[,&^uobI4QF',k"*^S)3c;)GMWC(T'=",ErnS#U=YCUN0&q4+*KmK1Zd*NI\GQDiZUG7;PTja8lulb"\PWWO#WcfI[ZB:6s*3g$be%?JH<b[c(?4J1!2SLDmbmkh8.c&CU)B?Xu,cp$<hP@=fLc>(n`oaEJ[XE'%QW=HE04M<,;ERm[MS=uYF=nN3jG'f@#?O48Ia,6Y-3m&tTWVq1?DeiBkp.Ug*;lVZX`Z=P.eklHhNV;!R_?QOuoeJ<0%7idG7GM8boU$^>N.N,2^;25]0Z8M<<]XMCct>noC'Qfb?`*[Mo+,F9#>t~>endstream
endobj
xref
0 9
0000000000 65535 f
0000000061 00000 n
0000000102 00000 n
0000000209 00000 n
0000000321 00000 n
0000000514 00000 n
0000000582 00000 n
0000000862 00000 n
0000000921 00000 n
trailer
<<
/ID
[<ee376a59c53e3f2af87024f65fd4222c><ee376a59c53e3f2af87024f65fd4222c>]
% ReportLab generated PDF document -- digest (opensource)
/Info 6 0 R
/Root 5 0 R
/Size 9
>>
startxref
1511
%%EOF
app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.md
+222
View File
@@ -0,0 +1,222 @@
Intro paragraph for section 1.
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
# Section 2 Heading
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
| golf | 301 | india |
## Section 3 Heading
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
| golf | 301 | india | 23 |
| juliet | 401 | lima | 33 |
Intro paragraph for section 4.
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
| golf | 301 | india | 23 | kilo |
| juliet | 401 | lima | 33 | november |
| mike | 501 | oscar | 43 | alpha |
# Section 5 Heading
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
| golf | 301 |
| juliet | 401 |
| mike | 501 |
| papa | 601 |
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
# Section 7 Heading
Intro paragraph for section 7.
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
| golf | 301 | india | 23 |
## Section 8 Heading
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
| golf | 301 | india | 23 | kilo |
| juliet | 401 | lima | 33 | november |
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
| golf | 301 |
| juliet | 401 |
| mike | 501 |
# Section 10 Heading
Intro paragraph for section 10.
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
| golf | 301 | india |
| juliet | 401 | lima |
| mike | 501 | oscar |
| papa | 601 | bravo |
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
# Section 12 Heading
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
| golf | 301 | india | 23 | kilo |
## Section 13 Heading
Intro paragraph for section 13.
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
| golf | 301 |
| juliet | 401 |
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
| golf | 301 | india |
| juliet | 401 | lima |
| mike | 501 | oscar |
# Section 15 Heading
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
| golf | 301 | india | 23 |
| juliet | 401 | lima | 33 |
| mike | 501 | oscar | 43 |
| papa | 601 | bravo | 53 |
Intro paragraph for section 16.
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
# Section 17 Heading
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
| golf | 301 |
## Section 18 Heading
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
| golf | 301 | india |
| juliet | 401 | lima |
Intro paragraph for section 19.
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
| golf | 301 | india | 23 |
| juliet | 401 | lima | 33 |
| mike | 501 | oscar | 43 |
# Section 20 Heading
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
| golf | 301 | india | 23 | kilo |
| juliet | 401 | lima | 33 | november |
| mike | 501 | oscar | 43 | alpha |
| papa | 601 | bravo | 53 | delta |
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
# Section 22 Heading
Intro paragraph for section 22.
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
| golf | 301 | india |
## Section 23 Heading
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
| golf | 301 | india | 23 |
| juliet | 401 | lima | 33 |
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
| golf | 301 | india | 23 | kilo |
| juliet | 401 | lima | 33 | november |
| mike | 501 | oscar | 43 | alpha |
# Section 25 Heading
Intro paragraph for section 25.
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
| golf | 301 |
| juliet | 401 |
| mike | 501 |
| papa | 601 |
app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.pdf
+169
View File
@@ -0,0 +1,169 @@
%PDF-1.4
%“Œ‹ž ReportLab Generated PDF document (opensource)
1 0 obj
<<
/F1 2 0 R /F2 3 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/Contents 13 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
5 0 obj
<<
/Contents 14 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
6 0 obj
<<
/Contents 15 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
7 0 obj
<<
/Contents 16 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
8 0 obj
<<
/Contents 17 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
9 0 obj
<<
/Contents 18 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
10 0 obj
<<
/PageMode /UseNone /Pages 12 0 R /Type /Catalog
>>
endobj
11 0 obj
<<
/Author (\(anonymous\)) /CreationDate (D:20260603005358+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603005358+01'00') /Producer (ReportLab PDF Library - \(opensource\))
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
>>
endobj
12 0 obj
<<
/Count 6 /Kids [ 4 0 R 5 0 R 6 0 R 7 0 R 8 0 R 9 0 R ] /Type /Pages
>>
endobj
13 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1007
>>
stream
GauHKgN)$k&:O:Sn<io8&IUqd[51N1e*/]GIR%m`>DZmeMUK]*(BdhV"2[X,j0;*\%_E*0HXJRc`<Z.PlPcH-)*5ON#Qr4-#h+U(ll5egXa[FW7^<l-+Mko1pU57W'%??d'1Q-,]\qWS80;JOp0HFGa1T[+D>q%4S#/VM`O[Rh4i5T.Phi,$]aT$05!q:PBuCQU/.p2]@1koN*Tb*K9k5G\ht+Dr\K=+8\NZ"alMaOEo**@OK:8-.O1X3-?Gg`@m3%,ti3'">T-&c=M&Wuu?cDbGDp.gO<KniopM)A\M"ajn9N]6p@tefHlX67:iW3SE/[O\LV1TKe.i9es,%kWY+1-s,ft.u27]:^>F0!r6&&CLC$tM?fIR"M37["/*k9@YkpKKSS@Np"1/4#R]`I^(g*1Sc,9L3Qt6N(T@F`A>oBGgL-!r#Y4)R\G`D,iVtFeJUE9u4iuUQ?D%C&SB4kkp.>D5tii>nDKJ"Y07jANhOb$R(_$=U7Stjs)-/KZ(6IBm`6<Z:IO1/g<Q6n^@fdK*]SO#g&Wj4B)g1,#O?30",57V_ONl0p`%uQN]'EAZV$475Z"'DApH2)Sg6UQ75XC?4^'<&6&d?YC`OF9F3eE;WIMt@a?'q)c\3oWQ_BD]d`4=O\,'aSUFV4HsE7)O:lPOi6Ht1,[dTm6JqQM6es[?9G4VC(YieF*)uM6[SZT#[TN2,BIir^1;/Ku"E1laH[\"M+F'8=n0!;?_#hHd!hkpV0u\Ij\4VC-km.[F&JhGmbc`6@I1=mf>u<3;i.Bh4+MFJ"H:.XWUQX6%LU(sg4Tt$_":5`p.2,gZkpUfdg,Hd(qR1)9\ltF2^8b5,,XbY%VfhD52O7A.c.u]dhTc5t%0<0L5E38`Bq+i;"%J7kcc#@i)@okdN-qiaA"33DdPgPrUs7;j%+47_cnGN@&ug9/dqGOAHA4f.N*&guHspf?E;GIc-Wt>:<(m1AmcS_Zc2VlEI'_S_>@#!MF^m1$$<mB1`gt\X~>endstream
endobj
14 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 987
>>
stream
GatU3gN&c;&:O:SkV;H,eI!J\f?X"LSPIZ'"4Y.F#oAAY?N.Y?8Iu:s)eV;,aNG1Lea>G$!MSc\rU6m7jF'AOI09V1,^TSTi$A+f4sZVQ%2^>uj$40\AI>WbZo9q!NL^CS9P3E9X.:XEMb$a/X"qH9hN6fY,]D!OEOVT,722%RJnVqK4f'[4d3(/hbJWJRs,>>28jk5A:nCl/%FlC#LntsAC5cE$q`QO_@Pk%Lp>7V23")NNrFkcXfuoI=SJJ-`g)_+K\@,SQ@8ORGg(_(B.[u[WXD%Q8@I";9d(]M62K_)Q+-<\kHY5,)o99%BB2bcQVkd:+MZVH=$Z`S@BhH]-XfPANk@qBN.:Jc'?.Kn\p7(aPI<Cb@g:XRZ9Km+RD-e6S\N*kQMA?P5>QOI(du7U]WDrNNPW%d"8P_j^Y8d%G<t1\CJ_5-m+>,'JJLXrV.UD!ah19&9b#*fgR2o730<9)QX)X0"EK6u;mBJs3/fl\d&K$B3bXI5R>F*E,^\%+b?0o8s$o.CuX4uDAT_?G0(p4N3La,8qfi'j?e`e88KrZ8*NDgc:62'icCb],4B]Z"SO&e/BRC*C$2PmbSKAjc\$FLD8?&qHVH3G4p834/77n[%()-p/JDRBapcO2b\,3dW%#U9u!ilSkd%2'rr&g[u".1O]W%0J<@#(ptUD;%0:^.^ls:.#.V6NgR[`2\RUW!k$-*GZIf3DO<K8np=hCA>BiB_Mu"=LVGlO\1Z]0pF(@UaiVQ=dh?h5kfQ_gj>$jHW$8TCt:C.jVU98ne.XoiMtoV;q8XhnlOE;3a]=dGJ8KL&+Khj"&P4Q=P-:nHsDprY"B4+#A3dF,<W=NC>;b&gf(sT\Ts*iH)f!KhfHV;\mMSHej.9.XrBG41_<:b)MG;-Y(:&nCKS0F]r&CGl>EfMc0td\0GHScl3Lr;?OUoo6619Qdr)S8S);]7L5rmWW&(5t4Dil"USDhhY/Y=!=!X:85**$:~>endstream
endobj
15 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 976
>>
stream
GatU3gN&c;'R]XVkgAc"70atdYFXp#3h7VV#7/=-%N$"G?N.YO&iX#cVsS_l_5i8)eum:1-)(7`mXJ7LnhqY0hGZal,We=q^e"$M]Lu;_=4AoIN$QOjOF.@i*Sg!_rLL=-'?kk;m;M&R`$D>'HCL7<qMHOgP.\NgCm:Zf5tLAgarh6YCKLONS%?!`3T6F7YKNJ(dBFh(U1"e7D6YdC:71/`?cjR"QrKtsGg*:'=Ck>]$A,-S`H1I/1T]4\B3=gQ9:I4j9f]+af*Z7*GH5H+;j73Z-T#DAcgpO<SFuA['l7@MF*6c^<pB]o84rG$&I@Cs3`KIG,N/bG0f<k/M@gUKb"%\W-lZ#AU+i=gUKkm46K-b>W(/'b?XF9+X+'-Dqi"+1U7?6=ZRJNBSJ6ojDTa+]o4RF^26C+i0s(9G2o;$@RNbs\(f75],L(SVpI0M!'7JVhZ"$t/%kb1+)F\p54/@jpVJF2+)7^F=gVcp8(h*m%F#'p9c%9Ep9?b7g^F5>(>I3t^d_"f'aue%^E\X4kN:g\P[;sC=g3h,nD/n9*`94uoM?sBCL<kr$AOqh:,Y&c90,npSDXN*bEfY(k^0LZ_\I@Y+L2nn64-qsuX5D'=o'B2"8rCsRJ?h/-4Ur6khb;ph%X"8-mXTUU5-!Uh$iVuRi1.=>4;XilfMPT4]&FC.U%DI(c6uf3*XPJP2MRdq@Ag3uYF().nqLFAE2Nih'(^p\5FnSNa>n+cI?U65A61_N1<9ZXH4Xr)Fgq(6md3H9[MQ3Q^%aq>E?XM:O:RKGjc+QV2MB!.I.17QpSt]X[C*6ap.`#'o;8F(ebZ.VMM6k`BkZB\I,2MNoX]J"k]Qd";,@($aWX&29q"6k;>Let)@]QHm:i/lf[DB_&U@ROq-0MLQp;@j+]?$"E)brXVun52AVQo$"[a7@!?!LT*>$&:5X_.R;9)'$l-S.>WjLQIVTo.JU%(H\,-*pl_kP9~>endstream
endobj
16 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1024
>>
stream
GauHKflEQ9'Rf^WgrHc4<#\/Qm7c-rFIIq+TFSD%Z#L'6o(Nl&^_!k0ds=.%-s&pt#i0QBKE91*<^/MJJCcfoHA_e;aL?\&_BAj]Dt;HW$6(8Ql\%O7-5%uT8<qrc9!'@hZ!'`C3a&iu\ZDZi+5lF*E@>0ZIL#\l)(+[ABOTNKmq!2F,S:.F93Z6QIu[nf\Q[qCE`Y-=.8k$67iCR!7=El+50a@f&b^6'Yg'mqlcRQ-5j:%[K@'5!A_m0L)&VlW1T50^X)"3Ma3,U1P8Di$>uPZ)Zj_igBc0lm]WE0#e>*M53(Yh6\9jHYh6B+3bM\b[Q9j'O_9:!:W0<K\bX;'ZV[%iHrL=:rH.Wh5O-II8-Rh3..Sd^\dNItUUd<nli^`$=A@?\WW4/7Wnp>$Ijm,#(bKn_D?UYMGJ7LLqC0\[$l<8U*O5^GnN;!N6YB'?d[d8DAPt^>+E:r0<2$'VL/TtuNg;C@5iM#%KdS-GU->0]`/20eLXVW#qnjsU.P:Xj&=enVE[mpqDCn2!qDRuQhI/#cZ-#m6`U9'SI4"9\<WSWoVPaf@]<NRl?FrTPOd:V_=>"r:4p.\W>IK_$#C,EkU#uO1g6*HoUK':XiJjtQfI'Sdc)B3J1eqgGJ+l?"$<J+^?VHgiq^90uD"@6ar+/_WhKG+8t!Slo>//&C$^f'TA!2:K!0pd;^Zif)c[:f!C/8hNKc7+'+1WbOFKr9T7c_G$s*O634O2Zs:"iN>j,pWU+:kEQ>5YbF=$27*59iJE<glrr=EAMssR&N"lVT1:*a,OcI=8\"6/>A//7(KsL\pUG.G[+JB-r8t;Sih+(W.A'8Q*LZK*I9WjCLjEt>lT%2RFJZg0Ps:5c,HdK#tSNI:#Rp!]O$Ic=T%K0[._E_Fcjn>OcM,iSAf9iF7,,+Qe:@o'Y4o_:54t*l+TAg<i_F?ke9:\G<n<b*t@"_`_;08ft@[9;qtKC#H0Z6cJ+d)MdidY-OFV1K"=!=K=^?hK-Q.s+?\[I98%IXRiAI.!:AVTf`~>endstream
endobj
17 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1075
>>
stream
GauHKgN&c;&:O:SkV;H,6&Q<FCM0)BVUK5trr=4p-3<-<P.#jI+b[BoQ`]$,GaorC,Bfp>Z[5)3-pL]0bpl':D91fR,o"F49.1/bfmFt32ljt6eO[j7!K]\nZ:^QQASu[l@3no(6*HL'@3I!Qfi7%,.N\.>A91O)D\n5*o@JWSOI@ng5p/"_I`g*Y$j0+5PWaso'25ltcglFd,QIaO'dPO0Yr\h5/,=m\PiP:P+Gg0&N;_5iLN$]BD.#VL+h,tSe\fL&,;+2fg&%\pAJiV#eVhq@ro%%/`[(&8n29YiYd>bDF^Bg[AVf/n>[FE(au"ZAe&>%Q[7/n3-QhsPXuPbp#<$d"UR8u-nBnr!Q$Wjd1;9WN/,KG8])Ca5rrDrI'Ue'*I0C%pn+f`Hd"4sB_&2*)S$pX=F/6"DC"TU`bMtd/m7u]?]=J5L[SNBuE:6Ri,NmW7.*qUp)97Tt+a84b\os`Ti8/uRD<mg2;n&q)23L#\f:[QJk+sYYhIC_O\TNS/;3<qfN@GE^L#jl^i&)"7_/%Cb;-M;;ooT&`R;C&\WTZ4"\YIcTZDa;0CKTX+?@ra\94$H:Q<l)r*NHFJWjMr7[tCBb^PX,G]H!fs0g3,5^'AZ%<4Zqp(JCd+;^lkoDJ%q(Bb84V=:PCgOtM#Rr-=oHW&DmO?/!mYrV;g]MR?14cGuH_H*]73!34(()[A.)c<V3:@I>lhrl*Xrn(a0\,%4%*k^rL\k3%6ako`&A3MoN^CXV77D!Qg<S_OEMIh5(T*nFt'&4]_38'Q<s*V7!;A%[email protected]/(oCu;?^7kb[#R/-P*GF@KHDm6E*dLMg/4Q$W(<Ta[fVpGB5td>%=j9Ge+0B@dZDHELb2fDU(,iiXXRl^*U,U#Fld!R72UIpK<KU`cn&R/_eYS//1PE/$d7p:t-BP62fL<3/QQ4%=NoD?Tlf:A,GpJc<]*uYaL6fSAqm"-E.#%h]idd9$6\DZKKp8Vd[>GWu#-DXi653a?U$fqCs18gt(5/#U%!u,gkMK;>OT_/='S:kn,iD@u]<8-rlTRu'+\7L$U7-#!5-5\WQn(n:q@-p24u!~>endstream
endobj
18 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 590
>>
stream
GasbXgMWc?&;KY!MRgt!"n`^Bf\9$c*`Q-Sb6tfd8P(q-dT1eng=QV-!OG*\kiWo2AGBdKLqI^%h,XNB)-gDk+GFVBa?g*a%:!J\97Vc8-jH8>8P>0S@Wr)o*"Hu<6qPp`EF:M3'l4@S\c2]`&[J%dO@5p$<(([H6:SlB;FS8L`&pO%6Y\V"/O7F]E%L@d.%>L@"4bK_XXs&";n_.W^Nr847HH,X@$.pC"4bYC?9RH,dN?h8<BTH9l89:QMq:es\ePUpFRc4?hV'h3b4`sE_b4r->a+8/!?Q=D\F41Nc@'B`';4XgMeh!;U23C+>AbMQD-o--lJ/":m#(Yt,X5(`1KL,GTMBLlr6=-1mi#k.-Ou\\T4$Pun2dEU(\?$&;1@T4dm^t!KuOD-U@N_AMr"&uVpsGm,+8I7B*f!%9.o4cC1<Nr^$d5!dH-#UIr,n6Z;JUFYWCTVA\,or>a[CZ12(hd>1*0bU`k2-MXo1[Gor#kmXGIM'#R49X#NSAOpdf'0ilUH4:M(^Snc3;m/+><UlJ1jSN&ZuoQ68F8`c/Y-e0[!aljuioC5fR7-VUAX`%g<0P5J7"CFlQJT1~>endstream
endobj
xref
0 19
0000000000 65535 f
0000000061 00000 n
0000000102 00000 n
0000000209 00000 n
0000000321 00000 n
0000000516 00000 n
0000000711 00000 n
0000000906 00000 n
0000001101 00000 n
0000001296 00000 n
0000001491 00000 n
0000001561 00000 n
0000001842 00000 n
0000001932 00000 n
0000003031 00000 n
0000004109 00000 n
0000005176 00000 n
0000006292 00000 n
0000007459 00000 n
trailer
<<
/ID
[<ba877f458edd6b06b69a7e843c67586d><ba877f458edd6b06b69a7e843c67586d>]
% ReportLab generated PDF document -- digest (opensource)
/Info 11 0 R
/Root 10 0 R
/Size 19
>>
startxref
8140
%%EOF
app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.md
+25
View File
@@ -0,0 +1,25 @@
# Lorem Ipsum in Two Columns
## 1. Origins
Lorem ipsum dolor sit amet consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
## 2. Structure
Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
## 3. Usage
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
## 4. Variations
Excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum.
## 5. Typography
Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam various turpis et commodo pharetra est.
## 6. Conclusion
Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum.
app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.pdf
+74
View File
@@ -0,0 +1,74 @@
%PDF-1.4
%“Œ‹ž ReportLab Generated PDF document (opensource)
1 0 obj
<<
/F1 2 0 R /F2 3 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
5 0 obj
<<
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
>>
endobj
6 0 obj
<<
/Author (\(anonymous\)) /CreationDate (D:20260603021636+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603021636+01'00') /Producer (ReportLab PDF Library - \(opensource\))
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
>>
endobj
7 0 obj
<<
/Count 1 /Kids [ 4 0 R ] /Type /Pages
>>
endobj
8 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 861
>>
stream
Gat=i>Ak00&;B$5/*8Q/Z)fmNp(^2^OK'MC[dT6#Oq.&b^4c4;1No6>+D%TC.n+fice+l9R0[L%'(o<DJBpB+r$jsRcu23$RjI].E7rS*BB"3t\qRKaS-SYt[_<-]),Cp@'$i;6dQuJ#Q*+lGY?X<phgEWPa?op]_1g@kf_>[email protected]:T=#%ZIIl18hB7Zf<piVi-V+;]ct^jS<=Z4l:IOd[,o"%q.hK[[k]r1!^oX'%44os7T9[m%YrbZ3KU/;aR0$cV$;D._".OT8)jeEBKU'l<A`'aD&R'L"Bo]Q)R+=cKOEld8s#M0ZYZN4[lRm1*F6^d9-Gfl5>I(hVU+?`]UK%;aabAH:q>9NUGQq^.=u])Wt-ETI))C'a[FE@elj!RSrf2Q'F>URAO.C,!DneTPqrj#4e2kb9%1"4qfZ)"#&^0j9nHQ?9nF!j7mVPP5\*Uq'_jMVS]9%`kQB\8*AF_bpr/hGj;HCUOSQU-%5:6S79Ud\b!*tPbr_'pCr$Ea#(FYP31NFhSX.-("1M:$cgH#hX8L(2]R3Q>'BYHCS%pI!;=WdJp,'ii[`QPZ_9mcd\baZ2U(_;c\-p,8EoIEpQ*lstL>]LE;C#\dLnT2R:)BM-fTc['3_He[U,k'!Bo".uERd>SkhRj^J+koSIrZ_dEf_5L'/1h.`+DTK(R:P,WH)h5\se=SZ"L/5b8b..,e/E\o+4YQ+*im^C>AERG/TieEK\)#>U@HXnJ,H0A9-MqhhkDp8%.6Lr,OrK*lih;B<-opZ8%EU?,$r^jmCDAQ`-0/-8/`[p]7Fm%0f:E&S*FV)DX2>#q\bRqA=^_`43#8EA$u%8r6F5`rc9>K@q>E4q^*~>endstream
endobj
xref
0 9
0000000000 65535 f
0000000061 00000 n
0000000102 00000 n
0000000209 00000 n
0000000321 00000 n
0000000514 00000 n
0000000582 00000 n
0000000862 00000 n
0000000921 00000 n
trailer
<<
/ID
[<21a9fbd0a0991a91b6e6e2db0856056e><21a9fbd0a0991a91b6e6e2db0856056e>]
% ReportLab generated PDF document -- digest (opensource)
/Info 6 0 R
/Root 5 0 R
/Size 9
>>
startxref
1872
%%EOF
app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.md
+62
View File
@@ -0,0 +1,62 @@
# Employee Expense Report
Reimbursement Request
EMP-1047
**Report Header**
| Employee Name | Michael Tran |
|---|---|
| Employee ID | EMP-1047 |
| Department | Client Services |
| Report Date | January 20th, 2026 |
| Reporting Period | January 5th–16th, 2026 |
| Manager Approver | Laura Simmons |
**Company Information**
| Company | Summit Consulting Partners |
|---|---|
| Company Address | 88 Riverside Plaza, Suite 1400, New York, NY 10069 |
| Accounting Department Email | expenses@example.com |
**Trip Purpose**
The trip was undertaken for client onsite meetings with Atlantic Energy Solutions in Boston, MA.
**Expense Details**
| Description | Amount | Date | Category |
|---|---|---|---|
| Flight (NYC to Boston roundtrip) | $325.40 | January 5th, 2026 | Airline ticket |
| Hotel (3 nights at Harborview Hotel) | $822.75 | January 5th–8th, 2026 | Lodging |
| Taxi from airport to hotel | $48.00 | January 5th, 2026 | Ground transportation |
| Client dinner (3 attendees) | $186.20 | January 6th, 2026 | Meals |
| Parking at JFK Airport | $72.00 | January 5th–8th, 2026 | Parking |
| Breakfast (per diem not used) | $18.50 | January 7th, 2026 | Meals |
| Description | Amount | Date | Category |
|---|---|---|---|
| Uber to client office | $22.10 | January 7th, 2026 | Ground transportation |
| Printing + presentation materials | $46.90 | January 8th, 2026 | Materials |
| Lunch with client | $39.75 | January 8th, 2026 | Meals |
| Office supplies (notebooks, pens) | $27.60 | January 10th, 2026 | Supplies |
| Mileage reimbursement (client visit in NJ, 42 miles @ $0.67/mile) | $28.14 | January 14th, 2026 | Mileage |
| Team lunch meeting (internal) | $64.30 | January 15th, 2026 | Meals |
Total Expenses $1,701.64
Reimbursement Method
Reimbursement method Direct deposit
Notes
All receipts are attached. Expenses are business-related and comply with company travel policy.
**Approval**
Michael Tran, Employee
Laura Simmons, Manager
app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.pdf
BIN
View File
Binary file not shown.
app/core/src/main/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdown.java
+27 -5
View File
@@ -1,11 +1,13 @@
package stirling.software.SPDF.model.api.converters;
import org.springframework.core.io.Resource;
import java.nio.charset.StandardCharsets;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.multipart.MultipartFile;
import io.github.pixee.security.Filenames;
import io.swagger.v3.oas.annotations.Operation;
import lombok.RequiredArgsConstructor;
@@ -15,8 +17,11 @@ import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.ConvertApi;
import stirling.software.common.enumeration.ResourceWeight;
import stirling.software.common.model.api.PDFFile;
import stirling.software.common.util.PDFToFile;
import stirling.software.common.pdf.PdfMarkdownConverter;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
import stirling.software.jpdfium.PdfDocument;
@ConvertApi
@RequiredArgsConstructor
@@ -33,10 +38,27 @@ public class ConvertPDFToMarkdown {
summary = "Convert PDF to Markdown",
description =
"This endpoint converts a PDF file to Markdown format. Input:PDF Output:Markdown Type:SISO")
public ResponseEntity<Resource> processPdfToMarkdown(@ModelAttribute PDFFile file)
public ResponseEntity<byte[]> processPdfToMarkdown(@ModelAttribute PDFFile file)
throws Exception {
MultipartFile inputFile = file.getFileInput();
PDFToFile pdfToFile = new PDFToFile(tempFileManager);
return pdfToFile.processPdfToMarkdown(inputFile);
String originalName = Filenames.toSimpleFileName(inputFile.getOriginalFilename());
String baseName =
originalName.contains(".")
? originalName.substring(0, originalName.lastIndexOf('.'))
: originalName;
String markdown;
try (TempFile tempInput = new TempFile(tempFileManager, ".pdf")) {
inputFile.transferTo(tempInput.getFile());
try (PdfDocument doc = PdfDocument.open(tempInput.getPath())) {
markdown = new PdfMarkdownConverter().convert(doc);
}
}
return WebResponseUtils.bytesToWebResponse(
markdown.getBytes(StandardCharsets.UTF_8),
baseName + ".md",
MediaType.valueOf("text/markdown"));
}
}
app/core/src/main/resources/settings.yml.template
+12
View File
@@ -364,6 +364,18 @@ aiEngine:
url: http://localhost:5001 # URL of the Python AI engine
timeoutSeconds: 120 # Timeout in seconds for AI engine requests
policies:
# Folder automations can read from and write to the directories you allow here, so treat this as a
# security boundary. Leave allowedFolderRoots empty (default) to disable folder sources/outputs
# entirely; list absolute directories to permit folder access only within them. Stirling's own
# config directory is always off-limits, and folder access is always disabled in SaaS mode.
allowedFolderRoots: [] # e.g. ["/data/inbox", "/data/outbox"]
scheduleSweepSeconds: 60 # How often (seconds) scheduled policies are checked for being due
watchReconcileSeconds: 300 # How often (seconds) folder-watch re-syncs watches and re-runs as a safety net for missed events
watchQuietPeriodMs: 500 # How long (ms) folder-watch coalesces a burst of file events into a single run
streamTimeoutMs: 1800000 # SSE timeout (ms) for live run-progress streams
runExpiryMinutes: 30 # How long (minutes) a finished run's in-memory state is kept before eviction
# Model Context Protocol (MCP) server. Exposes Stirling's PDF tools (grouped by namespace)
# plus the AI agents to MCP clients (Inspector, Claude Desktop, custom). OAuth-protected.
# Disabled by default - enable explicitly per deployment after configuring mcp.auth.
app/core/src/test/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdownTest.java
+48 -46
View File
@@ -1,16 +1,17 @@
package stirling.software.SPDF.model.api.converters;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.MockedConstruction;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
@@ -21,9 +22,10 @@ import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.common.util.PDFToFile;
import stirling.software.common.pdf.PdfMarkdownConverter;
import stirling.software.common.util.TempFile;
import stirling.software.jpdfium.PdfDocument;
class ConvertPDFToMarkdownTest {
@@ -47,68 +49,68 @@ class ConvertPDFToMarkdownTest {
@Test
void pdfToMarkdownReturnsMarkdownBytes() throws Exception {
byte[] md = "# heading\n\ncontent\n".getBytes(StandardCharsets.UTF_8);
String expectedMd = "# heading\n\ncontent\n";
try (MockedConstruction<PDFToFile> construction =
Mockito.mockConstruction(
PDFToFile.class,
(mock, ctx) -> {
when(mock.processPdfToMarkdown(any(MultipartFile.class)))
.thenAnswer(
inv ->
ResponseEntity.ok()
.header("Content-Type", "text/markdown")
.body(new ByteArrayResource(md)));
})) {
File tmpFile = File.createTempFile("test", ".pdf");
tmpFile.deleteOnExit();
MockMvc mvc = mockMvc();
try (MockedConstruction<TempFile> tempMock =
Mockito.mockConstruction(
TempFile.class,
(mock, ctx) -> {
when(mock.getFile()).thenReturn(tmpFile);
when(mock.getPath()).thenReturn(tmpFile.toPath());
});
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
MockedConstruction<PdfMarkdownConverter> converterMock =
Mockito.mockConstruction(
PdfMarkdownConverter.class,
(mock, ctx) -> when(mock.convert(any())).thenReturn(expectedMd))) {
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
MockMultipartFile file =
new MockMultipartFile(
"fileInput", // must match the field name in PDFFile
"input.pdf",
"application/pdf",
new byte[] {1, 2, 3});
"fileInput", "input.pdf", "application/pdf", new byte[] {1, 2, 3});
// ResponseEntity<Resource> is written synchronously on the request thread,
// so there is no async dispatch to wait for (unlike the old StreamingResponseBody
// path).
mvc.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
mockMvc()
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
.andExpect(status().isOk())
.andExpect(header().string("Content-Type", "text/markdown"))
.andExpect(content().bytes(md));
// Verify that exactly one instance was created
assert construction.constructed().size() == 1;
// And that the uploaded file was passed to processPdfToMarkdown()
PDFToFile created = construction.constructed().get(0);
ArgumentCaptor<MultipartFile> captor = ArgumentCaptor.forClass(MultipartFile.class);
verify(created, times(1)).processPdfToMarkdown(captor.capture());
MultipartFile passed = captor.getValue();
// Minimal plausibility checks
assertEquals("input.pdf", passed.getOriginalFilename());
assertEquals("application/pdf", passed.getContentType());
}
}
@Test
void pdfToMarkdownWhenServiceThrowsReturns500() throws Exception {
try (MockedConstruction<PDFToFile> ignored =
Mockito.mockConstruction(
PDFToFile.class,
(mock, ctx) -> {
when(mock.processPdfToMarkdown(any(MultipartFile.class)))
.thenThrow(new RuntimeException("boom"));
})) {
File tmpFile = File.createTempFile("test", ".pdf");
tmpFile.deleteOnExit();
MockMvc mvc = mockMvc();
try (MockedConstruction<TempFile> tempMock =
Mockito.mockConstruction(
TempFile.class,
(mock, ctx) -> {
when(mock.getFile()).thenReturn(tmpFile);
when(mock.getPath()).thenReturn(tmpFile.toPath());
});
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
MockedConstruction<PdfMarkdownConverter> converterMock =
Mockito.mockConstruction(
PdfMarkdownConverter.class,
(mock, ctx) ->
when(mock.convert(any()))
.thenThrow(new RuntimeException("boom")))) {
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
MockMultipartFile file =
new MockMultipartFile(
"fileInput", "x.pdf", "application/pdf", new byte[] {0x01});
mvc.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
mockMvc()
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
.andExpect(status().isInternalServerError());
}
}
app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java
+2 -1
View File
@@ -21,7 +21,8 @@ public enum AiWorkflowOutcome {
COMPLETED("completed"),
UNSUPPORTED_CAPABILITY("unsupported_capability"),
CANNOT_CONTINUE("cannot_continue"),
GENERATE_FILE("generate_file");
GENERATE_FILE("generate_file"),
CONVERT_MARKDOWN("convert_markdown");
private final String value;
app/proprietary/src/main/java/stirling/software/proprietary/policy/config/FolderAccessGuard.java
+106
View File
@@ -0,0 +1,106 @@
package stirling.software.proprietary.policy.config;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
import stirling.software.common.configuration.InstallationPathConfig;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.model.Policy;
/**
* The single authority on which filesystem locations a policy may read from or write to. Folder
* sources and sinks take a configured directory, so without this a user who can save a policy could
* point one at Stirling's own config/secrets directory and exfiltrate (or overwrite) it. Every
* folder source and sink runs its directory through {@link #requirePermitted(Path)} at save time
* and again at run time.
*
* <p>Enforced fail-closed, in order:
*
* <ul>
* <li><b>Disabled in SaaS</b> - folder access is never allowed when the {@code saas} profile is
* active; a tenant must not reach the host filesystem at all.
* <li><b>Protected paths</b> - Stirling's own config directory (settings, database, keys,
* backups) is always rejected, even if an allowed root were misconfigured to contain it.
* <li><b>Allowlist</b> - the directory must resolve within one of {@code
* policies.allowedFolderRoots}; with none configured, all folder access is refused.
* </ul>
*
* <p>Paths are compared after normalisation, so {@code ..} segments cannot walk out of an allowed
* root. (Symlink escape is not defended here; an operator who configures an allowed root containing
* a symlink to a sensitive location is trusted.)
*/
@Component
public class FolderAccessGuard {
public static final String FOLDER_TYPE = "folder";
private final boolean saasActive;
private final List<Path> allowedRoots;
private final List<Path> protectedRoots;
public FolderAccessGuard(ApplicationProperties applicationProperties, Environment environment) {
this.saasActive = Arrays.asList(environment.getActiveProfiles()).contains("saas");
this.allowedRoots =
normalizeAll(applicationProperties.getPolicies().getAllowedFolderRoots());
this.protectedRoots = List.of(normalize(Path.of(InstallationPathConfig.getConfigPath())));
}
/**
* Check that {@code dir} is a permitted folder location, returning its normalised absolute
* form.
*
* @throws IllegalArgumentException if folder access is disabled (SaaS or no roots configured),
* the path is inside a protected directory, or it falls outside every allowed root
*/
public Path requirePermitted(Path dir) {
if (saasActive) {
throw new IllegalArgumentException(
"folder sources and outputs are not available in SaaS mode");
}
Path normalized = normalize(dir);
for (Path protectedRoot : protectedRoots) {
if (normalized.startsWith(protectedRoot)) {
throw new IllegalArgumentException(
"folder may not point inside a protected Stirling directory");
}
}
if (allowedRoots.isEmpty()) {
throw new IllegalArgumentException(
"folder access is disabled; set policies.allowedFolderRoots to permit it");
}
boolean within = allowedRoots.stream().anyMatch(normalized::startsWith);
if (!within) {
throw new IllegalArgumentException(
"folder '" + normalized + "' is outside the allowed folder roots");
}
return normalized;
}
/** Whether this policy reads from or writes to a folder, and so is subject to these rules. */
public boolean usesFolderAccess(Policy policy) {
boolean readsFolder =
policy.sources().stream().anyMatch(spec -> FOLDER_TYPE.equals(spec.type()));
boolean writesFolder =
policy.output() != null && FOLDER_TYPE.equals(policy.output().type());
return readsFolder || writesFolder;
}
private static List<Path> normalizeAll(List<String> roots) {
List<Path> result = new ArrayList<>();
for (String root : roots) {
if (root != null && !root.isBlank()) {
result.add(normalize(Path.of(root)));
}
}
return result;
}
private static Path normalize(Path path) {
return path.toAbsolutePath().normalize();
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java
+44 -12
View File
@@ -6,7 +6,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus;
@@ -34,11 +33,16 @@ import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.job.JobResponse;
import stirling.software.common.service.UserServiceInterface;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.proprietary.policy.config.FolderAccessGuard;
import stirling.software.proprietary.policy.engine.PolicyRunHandle;
import stirling.software.proprietary.policy.engine.PolicyRunRegistry;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.engine.PolicyValidator;
import stirling.software.proprietary.policy.model.PipelineDefinition;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
@@ -47,7 +51,6 @@ import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.model.PolicyRunView;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
import stirling.software.proprietary.policy.store.PolicyStore;
import stirling.software.proprietary.policy.trigger.ManualTrigger;
import stirling.software.proprietary.security.config.PremiumEndpoint;
import tools.jackson.core.JacksonException;
@@ -71,16 +74,16 @@ import tools.jackson.databind.ObjectMapper;
@Tag(name = "Policies", description = "Run tool pipelines on the backend")
public class PolicyController {
private final ManualTrigger manualTrigger;
private final PolicyRunner policyRunner;
private final PolicyRunRegistry runRegistry;
private final PolicyStore policyStore;
private final PolicyValidator policyValidator;
private final FolderAccessGuard folderAccessGuard;
private final UserServiceInterface userService;
private final ApplicationProperties applicationProperties;
private final ObjectMapper objectMapper;
private final TempFileManager tempFileManager;
/** SSE emitter timeout, generous enough for long multi-step runs on large files. */
@Value("${stirling.policies.streamTimeoutMs:1800000}")
private long streamTimeoutMs;
@PostMapping(value = "/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@Operation(
summary = "Run a tool pipeline",
@@ -95,7 +98,8 @@ public class PolicyController {
throws IOException {
PipelineDefinition definition = parseDefinition(json);
PolicyInputs inputs = collectInputs(request);
String runId = manualTrigger.fire(definition, inputs, PolicyProgressListener.NOOP).runId();
String runId =
policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP).runId();
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
}
@@ -112,10 +116,11 @@ public class PolicyController {
PipelineDefinition definition = parseDefinition(json);
PolicyInputs inputs = collectInputs(request);
SseEmitter emitter = new SseEmitter(streamTimeoutMs);
SseEmitter emitter =
new SseEmitter(applicationProperties.getPolicies().getStreamTimeoutMs());
emitter.onError(e -> log.warn("Policy run SSE emitter error", e));
PolicyRunHandle handle = manualTrigger.fire(definition, inputs, streamListener(emitter));
PolicyRunHandle handle = policyRunner.runAdHoc(definition, inputs, streamListener(emitter));
// Close the stream with a terminal event once the run finishes. whenComplete runs on the
// engine's worker thread after the run is done, so this never races the step events.
handle.completion()
@@ -155,7 +160,34 @@ public class PolicyController {
"Stores a policy (trigger config + steps + output + metadata). A blank id is"
+ " assigned; returns the stored policy with its id.")
public ResponseEntity<Policy> savePolicy(@RequestBody String json) {
return ResponseEntity.ok(policyStore.save(parsePolicy(json)));
Policy policy = parsePolicy(json);
requireAuthorizedForFolderAccess(policy);
try {
policyValidator.validate(policy);
} catch (IllegalArgumentException e) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
}
return ResponseEntity.ok(policyStore.save(policy));
}
/**
* A policy that reads from or writes to a server folder grants whoever saves it access to that
* path, so restrict it to administrators on multi-user deployments. Single-user deployments
* (login disabled, e.g. desktop) trust the local operator. The {@link FolderAccessGuard} still
* enforces SaaS-off and the path allowlist during validation regardless of who saves.
*/
private void requireAuthorizedForFolderAccess(Policy policy) {
if (!folderAccessGuard.usesFolderAccess(policy)) {
return;
}
if (!applicationProperties.getSecurity().isEnableLogin()) {
return;
}
if (!userService.isCurrentUserAdmin()) {
throw new ResponseStatusException(
HttpStatus.FORBIDDEN,
"Folder sources and outputs may only be configured by an administrator");
}
}
@GetMapping
@@ -199,7 +231,7 @@ public class PolicyController {
new ResponseStatusException(
HttpStatus.NOT_FOUND, "No policy: " + policyId));
PolicyInputs inputs = collectInputs(request);
String runId = manualTrigger.run(policy, inputs, PolicyProgressListener.NOOP).runId();
String runId = policyRunner.runWith(policy, inputs, PolicyProgressListener.NOOP).runId();
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java
+6 -6
View File
@@ -9,13 +9,13 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import jakarta.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.model.PolicyRun;
/**
@@ -24,9 +24,9 @@ import stirling.software.proprietary.policy.model.PolicyRun;
* TaskManager}.
*
* <p>Finished runs are evicted on a fixed interval once they age past {@code
* stirling.policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a
* run's rich in-memory state does not outlive the process. Only terminal runs are evicted; active
* and paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not
* policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a run's
* rich in-memory state does not outlive the process. Only terminal runs are evicted; active and
* paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not
* touched here: a run shares its runId with a {@code TaskManager} job, which owns file-lifecycle
* cleanup, so eviction only frees this map's entry.
*/
@@ -41,8 +41,8 @@ public class PolicyRunRegistry {
Executors.newSingleThreadScheduledExecutor(
Thread.ofVirtual().name("policy-run-cleanup-", 0).factory());
public PolicyRunRegistry(
@Value("${stirling.policies.runExpiryMinutes:30}") int runExpiryMinutes) {
public PolicyRunRegistry(ApplicationProperties applicationProperties) {
int runExpiryMinutes = applicationProperties.getPolicies().getRunExpiryMinutes();
this.runExpiry = Duration.ofMinutes(runExpiryMinutes);
cleanupExecutor.scheduleAtFixedRate(this::evictExpiredRuns, 10, 10, TimeUnit.MINUTES);
log.debug(
app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java
+116
View File
@@ -0,0 +1,116 @@
package stirling.software.proprietary.policy.engine;
import java.io.IOException;
import java.util.List;
import java.util.function.Consumer;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.input.ResolvedInput;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.PipelineDefinition;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
/**
* Runs policies, and is the one place that knows how to turn a policy's configured {@link InputSpec
* sources} into actual runs. Triggers (schedule, and future webhook/folder-watch) decide
* <em>when</em> to run and call {@link #run(Policy)}; they never touch sources themselves. The
* controller uses the supplied-input and ad-hoc entry points for on-demand work.
*
* <p>This is the seam that keeps triggers and sources independent: a trigger depends on the runner,
* the runner depends on the {@link InputSource} beans, and a source depends on neither - it just
* yields {@link ResolvedInput units of work}, each carrying its own completion hook.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class PolicyRunner {
private final PolicyEngine policyEngine;
private final List<InputSource> inputSources;
/**
* Run a policy by pulling from every source it configures: each source yields zero or more
* units of work, and each unit becomes its own run so one failure does not affect the others. A
* policy with no sources runs once with no input files (a generator pipeline). Used by
* automatic triggers.
*/
public void run(Policy policy) {
List<InputSpec> sources = policy.sources();
if (sources.isEmpty()) {
startRun(policy, PolicyInputs.of(List.of()), unused -> {});
return;
}
for (InputSpec spec : sources) {
pullAndRun(policy, spec);
}
}
/**
* Run a stored policy on files supplied directly by the caller (e.g. a manual run with
* uploads), bypassing its configured sources. Returns the run handle so callers can stream
* progress.
*/
public PolicyRunHandle runWith(
Policy policy, PolicyInputs inputs, PolicyProgressListener listener) {
return policyEngine.runPolicy(policy, inputs, listener);
}
/** Run an ad-hoc pipeline with no stored policy (AI/Automate one-offs). */
public PolicyRunHandle runAdHoc(
PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) {
return policyEngine.submit(definition, inputs, listener);
}
private void pullAndRun(Policy policy, InputSpec spec) {
InputSource source = sourceFor(spec);
if (source == null) {
log.warn(
"No input source for type '{}' (policy {}); skipping",
spec.type(),
policy.id());
return;
}
List<ResolvedInput> work;
try {
work = source.resolve(spec);
} catch (IOException | RuntimeException e) {
log.warn(
"Failed to resolve source '{}' for policy {}: {}",
spec.type(),
policy.id(),
e.getMessage());
return;
}
for (ResolvedInput unit : work) {
startRun(policy, unit.inputs(), unit.onComplete());
}
}
private void startRun(Policy policy, PolicyInputs inputs, Consumer<Boolean> onComplete) {
log.info("Running policy {} ({})", policy.id(), policy.name());
PolicyRunHandle handle =
policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP);
handle.completion()
.whenComplete((run, throwable) -> onComplete.accept(succeeded(run, throwable)));
}
private static boolean succeeded(PolicyRun run, Throwable throwable) {
return throwable == null && run != null && run.getStatus() == PolicyRunStatus.COMPLETED;
}
private InputSource sourceFor(InputSpec spec) {
return inputSources.stream()
.filter(source -> source.supports(spec))
.findFirst()
.orElse(null);
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyValidator.java
+74
View File
@@ -0,0 +1,74 @@
package stirling.software.proprietary.policy.engine;
import java.util.List;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.output.PolicyOutputSink;
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
/**
* Validates a policy's trigger, sources, and output configuration by delegating each facet to the
* bean that handles its type. Called when a policy is saved so a misconfigured schedule, missing
* folder directory, or unknown type fails fast instead of silently misbehaving at run time.
*
* <p>The trigger is optional (a {@code null} trigger is a manual-only policy and needs no
* validation); every configured source is validated.
*/
@Service
@RequiredArgsConstructor
public class PolicyValidator {
private final List<PolicyTrigger> triggers;
private final List<InputSource> inputSources;
private final List<PolicyOutputSink> outputSinks;
/**
* @throws IllegalArgumentException if any facet's type is unknown or its configuration is
* invalid
*/
public void validate(Policy policy) {
if (policy.trigger() != null) {
triggerFor(policy.trigger()).validate(policy);
}
for (InputSpec source : policy.sources()) {
inputSourceFor(source).validate(source);
}
outputSinkFor(policy.output()).validate(policy.output());
}
private PolicyTrigger triggerFor(TriggerConfig config) {
return triggers.stream()
.filter(trigger -> trigger.type().equals(config.type()))
.findFirst()
.orElseThrow(
() ->
new IllegalArgumentException(
"unknown trigger type: " + config.type()));
}
private InputSource inputSourceFor(InputSpec spec) {
return inputSources.stream()
.filter(source -> source.supports(spec))
.findFirst()
.orElseThrow(
() ->
new IllegalArgumentException(
"unknown input source type: " + spec.type()));
}
private PolicyOutputSink outputSinkFor(OutputSpec spec) {
return outputSinks.stream()
.filter(sink -> sink.supports(spec))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException("unknown output type: " + spec.type()));
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java
+186
View File
@@ -0,0 +1,186 @@
package stirling.software.proprietary.policy.input;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.util.FileReadinessChecker;
import stirling.software.proprietary.policy.config.FolderAccessGuard;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.PolicyInputs;
/**
* Reads input files from a directory. Each ready file becomes its own unit of work (one run per
* file) so a failure on one file does not affect the others.
*
* <p>Two modes via the {@code mode} option:
*
* <ul>
* <li>{@code "consume"} (default) - claim each file by moving it into {@code
* .stirling/processing}, then route it to {@code .stirling/done} or {@code .stirling/error}
* when its run finishes. Each file is processed once; right for "process new arrivals" (and
* the basis of watched folders).
* <li>{@code "snapshot"} - read the directory's current files without moving them; every run sees
* the full set again. Right for "always regenerate from a fixed input set".
* </ul>
*
* Readiness is checked first (via {@link FileReadinessChecker}) so files mid-write are skipped.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class FolderInputSource implements InputSource {
private static final String TYPE = FolderAccessGuard.FOLDER_TYPE;
// Bookkeeping lives under one hidden namespace dir so the watched folder stays tidy.
private static final String WORK_SUBDIR = ".stirling";
private static final String PROCESSING_SUBDIR = "processing";
private static final String DONE_SUBDIR = "done";
private static final String ERROR_SUBDIR = "error";
private final FileReadinessChecker readinessChecker;
private final FolderAccessGuard accessGuard;
@Override
public String type() {
return TYPE;
}
@Override
public boolean supports(InputSpec spec) {
return spec != null && TYPE.equals(spec.type());
}
@Override
public void validate(InputSpec spec) {
accessGuard.requirePermitted(FolderConfig.from(spec.options()).directory());
}
@Override
public List<Path> watchTargets(InputSpec spec) {
return List.of(FolderConfig.from(spec.options()).directory());
}
@Override
public List<ResolvedInput> resolve(InputSpec spec) throws IOException {
FolderConfig config = FolderConfig.from(spec.options());
Path inputDir = accessGuard.requirePermitted(config.directory());
if (!Files.isDirectory(inputDir)) {
log.debug("Folder input dir does not exist: {}", inputDir);
return List.of();
}
List<Path> ready = new ArrayList<>();
try (Stream<Path> entries = Files.list(inputDir)) {
entries.filter(Files::isRegularFile)
.filter(readinessChecker::isReady)
.forEach(ready::add);
}
List<ResolvedInput> work = new ArrayList<>();
for (Path file : ready) {
if (config.snapshot()) {
work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file)))));
} else {
Path claimed = claim(inputDir, file);
if (claimed == null) {
continue; // another sweep/process grabbed it
}
work.add(
new ResolvedInput(
PolicyInputs.of(List.of(fileResource(claimed))),
success -> route(inputDir, claimed, success)));
}
}
return work;
}
private Path claim(Path inputDir, Path file) {
try {
Path processingDir = workDir(inputDir, PROCESSING_SUBDIR);
Files.createDirectories(processingDir);
Path claimed = uniqueTarget(processingDir, file.getFileName().toString());
Files.move(file, claimed, StandardCopyOption.ATOMIC_MOVE);
return claimed;
} catch (IOException e) {
log.debug("Could not claim {}: {}", file, e.getMessage());
return null;
}
}
private void route(Path inputDir, Path claimed, boolean success) {
String subdir = success ? DONE_SUBDIR : ERROR_SUBDIR;
try {
Path destDir = workDir(inputDir, subdir);
Files.createDirectories(destDir);
Files.move(
claimed,
uniqueTarget(destDir, claimed.getFileName().toString()),
StandardCopyOption.ATOMIC_MOVE);
} catch (IOException e) {
log.warn(
"Could not move processed input {} to {}: {}", claimed, subdir, e.getMessage());
}
}
/** A bookkeeping subdirectory under the watched folder's {@code .stirling} namespace. */
private static Path workDir(Path inputDir, String subdir) {
return inputDir.resolve(WORK_SUBDIR).resolve(subdir);
}
private static Resource fileResource(Path path) {
String name = path.getFileName().toString();
return new FileSystemResource(path.toFile()) {
@Override
public String getFilename() {
return name;
}
};
}
private static Path uniqueTarget(Path dir, String filename) {
Path candidate = dir.resolve(filename);
if (!Files.exists(candidate)) {
return candidate;
}
int dot = filename.lastIndexOf('.');
String base = dot < 0 ? filename : filename.substring(0, dot);
String ext = dot < 0 ? "" : filename.substring(dot);
for (int n = 1; ; n++) {
Path next = dir.resolve(base + " (" + n + ")" + ext);
if (!Files.exists(next)) {
return next;
}
}
}
/** The typed, validated form of a folder source's options: the directory and dedup mode. */
record FolderConfig(Path directory, boolean snapshot) {
private static final String DIRECTORY_OPTION = "directory";
private static final String MODE_OPTION = "mode";
private static final String MODE_SNAPSHOT = "snapshot";
static FolderConfig from(Map<String, Object> options) {
Object directory = options.get(DIRECTORY_OPTION);
if (directory == null || directory.toString().isBlank()) {
throw new IllegalArgumentException("folder input requires a 'directory' option");
}
Object mode = options.get(MODE_OPTION);
boolean snapshot = mode != null && MODE_SNAPSHOT.equals(mode.toString());
return new FolderConfig(Path.of(directory.toString()), snapshot);
}
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java
+49
View File
@@ -0,0 +1,49 @@
package stirling.software.proprietary.policy.input;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import stirling.software.proprietary.policy.model.InputSpec;
/**
* Resolves one of a policy's {@link InputSpec sources} into the files to run on - answering
* <em>where</em> a run's files come from, independent of <em>when</em> it runs. The counterpart of
* {@code PolicyOutputSink}: implementations are beans selected by {@link #supports(InputSpec)}, so
* a new source kind (folder, S3) is just a new bean.
*
* <p>Driven by the {@code PolicyRunner}, which a trigger calls when a policy is due; a source is
* passive and knows nothing about what triggered the run. A manual run may instead supply files
* directly and bypass sources entirely.
*/
public interface InputSource {
/** Stable identifier for this source, matching {@code InputSpec.type()} (e.g. "folder"). */
String type();
/** Whether this source can handle the given spec. */
boolean supports(InputSpec spec);
/**
* Check that an input spec is usable, throwing {@link IllegalArgumentException} if not. Called
* when a policy is saved so misconfiguration fails fast rather than at run time.
*/
default void validate(InputSpec spec) {}
/**
* Resolve the spec into zero or more units of work, each carrying the files for one run and a
* completion hook. Returning an empty list means there is nothing to run right now.
*/
List<ResolvedInput> resolve(InputSpec spec) throws IOException;
/**
* The local filesystem directories this source draws from, if any, for triggers that want to
* react to changes there (the folder-watch trigger) rather than poll. Advisory only: it merely
* tells a trigger <em>where</em> to watch; resolving the spec into files is still this source's
* job via {@link #resolve}. Non-filesystem sources (S3, ...) return an empty list and so are
* simply not watchable. Default: nothing to watch.
*/
default List<Path> watchTargets(InputSpec spec) {
return List.of();
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolvedInput.java
+23
View File
@@ -0,0 +1,23 @@
package stirling.software.proprietary.policy.input;
import java.util.function.Consumer;
import stirling.software.proprietary.policy.model.PolicyInputs;
/**
* One unit of work produced by an {@link InputSource}: the files to run plus a completion callback
* invoked with the run's success once it finishes (e.g. a folder source routes the input to {@code
* .stirling/done} or {@code .stirling/error}). A source may return several of these (e.g. one per
* file).
*/
public record ResolvedInput(PolicyInputs inputs, Consumer<Boolean> onComplete) {
public ResolvedInput {
onComplete = onComplete == null ? success -> {} : onComplete;
}
/** A unit of work with no completion side effect. */
public static ResolvedInput of(PolicyInputs inputs) {
return new ResolvedInput(inputs, success -> {});
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/InputSpec.java
+26
View File
@@ -0,0 +1,26 @@
package stirling.software.proprietary.policy.model;
import java.util.Map;
/**
* One source a policy's input files come from. {@code type} selects an {@code InputSource}
* ("folder", and in future "s3", ...); {@code options} carries source-specific configuration (a
* directory, a bucket, a dedup mode, ...).
*
* <p>A policy holds a list of these; a run pulls from every one. An empty list means the policy
* runs with no input files (a generator pipeline, or files supplied directly to a manual run).
*
* <p>Data-driven and parallel to {@link OutputSpec}/{@link TriggerConfig}: a new source kind is a
* new {@code type} handled by a new {@code InputSource} bean.
*/
public record InputSpec(String type, Map<String, Object> options) {
public InputSpec {
options = options == null ? Map.of() : options;
}
/** Read input files from a directory on disk. */
public static InputSpec folder(String directory) {
return new InputSpec("folder", Map.of("directory", directory));
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java
+5
View File
@@ -18,4 +18,9 @@ public record OutputSpec(String type, Map<String, Object> options) {
public static OutputSpec inline() {
return new OutputSpec("inline", Map.of());
}
/** Write outputs to a directory on disk. */
public static OutputSpec folder(String directory) {
return new OutputSpec("folder", Map.of("directory", directory));
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java
+22 -11
View File
@@ -3,18 +3,16 @@ package stirling.software.proprietary.policy.model;
import java.util.List;
/**
* A stored, owned automation: how it is triggered, the ordered tool steps to run, and where its
* output goes, plus identity and metadata.
* A stored automation: an ordered chain of tool steps, the sources its input files come from, and
* an output destination for the results.
*
* <p>This is the central object of the feature. Everything that runs a chain of tools is a use of a
* Policy: a watched folder is a Policy with a folder {@link TriggerConfig} and a folder {@link
* OutputSpec}; a scheduled job is a Policy with a schedule trigger; manual/Automate/AI runs execute
* a Policy (or an ad-hoc {@link PipelineDefinition}) on demand. The engine itself only ever
* executes the {@link PipelineDefinition} this exposes via {@link #toDefinition()} - it is
* trigger-agnostic.
* <p>Every policy can always be run on demand (manually). It may additionally carry one automatic
* {@link TriggerConfig} - usually a schedule - that fires it without a person asking; a {@code
* null} trigger means manual-only. A trigger decides <em>when</em> a run happens and a {@link
* InputSpec source} decides <em>where</em> its files come from; the two are independent, and a run
* pulls from every configured source.
*
* <p>{@code enabled} gates automatic triggering (a disabled policy is not picked up by its
* trigger); it does not block an explicit manual run.
* <p>This is the feature's central configuration object - what a user defines and the engine runs.
*/
public record Policy(
String id,
@@ -22,15 +20,28 @@ public record Policy(
String owner,
boolean enabled,
TriggerConfig trigger,
List<InputSpec> sources,
List<PipelineStep> steps,
OutputSpec output) {
public Policy {
trigger = trigger == null ? TriggerConfig.manual() : trigger;
sources = sources == null ? List.of() : List.copyOf(sources);
steps = steps == null ? List.of() : steps;
output = output == null ? OutputSpec.inline() : output;
}
/** A policy with no configured sources (a generator, or files supplied directly to a run). */
public Policy(
String id,
String name,
String owner,
boolean enabled,
TriggerConfig trigger,
List<PipelineStep> steps,
OutputSpec output) {
this(id, name, owner, enabled, trigger, List.of(), steps, output);
}
/** The engine-level, trigger-agnostic view of this policy's pipeline. */
public PipelineDefinition toDefinition() {
return new PipelineDefinition(name, steps, output);
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Schedule.java
+141
View File
@@ -0,0 +1,141 @@
package stirling.software.proprietary.policy.model;
import java.time.DayOfWeek;
import java.time.LocalTime;
import java.time.ZonedDateTime;
import java.util.EnumSet;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* When a scheduled policy should fire, expressed as intent ("every day at 02:00") rather than a
* cron string. A frontend builds one of these from a friendly picker; the schedule trigger asks it
* for the next firing after a given moment.
*
* <p>Sealed so every cadence is an explicit, self-validating shape and adding a new one forces a
* new subtype rather than another string convention to learn. The {@code type} discriminator stays
* on the wire so the frontend can switch on it without knowing the Java hierarchy.
*
* <p>Wall-clock kinds ({@link Daily}, {@link Weekly}, {@link Monthly}) are evaluated in the zone of
* the {@code after} argument; {@link Every} is a fixed offset and ignores wall-clock time.
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
@JsonSubTypes({
@JsonSubTypes.Type(value = Schedule.Every.class, name = "every"),
@JsonSubTypes.Type(value = Schedule.Daily.class, name = "daily"),
@JsonSubTypes.Type(value = Schedule.Weekly.class, name = "weekly"),
@JsonSubTypes.Type(value = Schedule.Monthly.class, name = "monthly"),
})
@JsonIgnoreProperties(ignoreUnknown = true)
public sealed interface Schedule {
/** The next firing strictly after {@code after}, evaluated in {@code after}'s zone. */
ZonedDateTime nextAfter(ZonedDateTime after);
/** The granularities a fixed-interval schedule can repeat on. */
enum Unit {
MINUTES,
HOURS,
DAYS
}
/**
* A fixed cadence repeating from when the policy was last seen: "every 15 minutes", "every 6
* hours". Time of day is irrelevant.
*/
record Every(long count, Unit unit) implements Schedule {
public Every {
if (count <= 0) {
throw new IllegalArgumentException("'every' schedule needs a positive count");
}
if (unit == null) {
throw new IllegalArgumentException("'every' schedule needs a unit");
}
}
@Override
public ZonedDateTime nextAfter(ZonedDateTime after) {
return switch (unit) {
case MINUTES -> after.plusMinutes(count);
case HOURS -> after.plusHours(count);
case DAYS -> after.plusDays(count);
};
}
}
/** Once a day at a wall-clock time: "every day at 02:00". */
record Daily(LocalTime at) implements Schedule {
public Daily {
requireTime(at);
}
@Override
public ZonedDateTime nextAfter(ZonedDateTime after) {
ZonedDateTime today = after.with(at);
return today.isAfter(after) ? today : today.plusDays(1);
}
}
/** On chosen weekdays at a wall-clock time: "every Monday and Thursday at 09:00". */
record Weekly(Set<DayOfWeek> days, LocalTime at) implements Schedule {
public Weekly {
if (days == null || days.isEmpty()) {
throw new IllegalArgumentException("'weekly' schedule needs at least one day");
}
requireTime(at);
days = EnumSet.copyOf(days);
}
@Override
public ZonedDateTime nextAfter(ZonedDateTime after) {
// The soonest of the next 7 days that lands on a chosen weekday, at the configured
// time.
for (int i = 0; i <= 7; i++) {
ZonedDateTime candidate = after.plusDays(i).with(at);
if (candidate.isAfter(after) && days.contains(candidate.getDayOfWeek())) {
return candidate;
}
}
throw new IllegalStateException("unreachable: a chosen weekday recurs within 8 days");
}
}
/**
* On a day of the month at a wall-clock time: "the 1st at 00:00". Months too short for the
* chosen day (e.g. the 31st in February) are skipped, not clamped.
*/
record Monthly(int dayOfMonth, LocalTime at) implements Schedule {
public Monthly {
if (dayOfMonth < 1 || dayOfMonth > 31) {
throw new IllegalArgumentException("'monthly' day-of-month must be 1-31");
}
requireTime(at);
}
@Override
public ZonedDateTime nextAfter(ZonedDateTime after) {
ZonedDateTime firstOfMonth = after.withDayOfMonth(1).with(at);
// Scan forward a few years' worth of months to skip ones without the chosen day.
for (int i = 0; i < 48; i++) {
ZonedDateTime month = firstOfMonth.plusMonths(i);
if (month.toLocalDate().lengthOfMonth() >= dayOfMonth) {
ZonedDateTime fire = month.withDayOfMonth(dayOfMonth);
if (fire.isAfter(after)) {
return fire;
}
}
}
throw new IllegalStateException(
"unreachable: a month with the chosen day recurs yearly");
}
}
private static void requireTime(LocalTime at) {
if (at == null) {
throw new IllegalArgumentException("schedule needs a time of day ('at')");
}
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java
+9 -11
View File
@@ -3,23 +3,21 @@ package stirling.software.proprietary.policy.model;
import java.util.Map;
/**
* How a {@link Policy} is automatically triggered. {@code type} selects a trigger kind ("manual",
* "folder", "schedule", "s3"); {@code options} carries type-specific configuration (a folder path,
* a cron expression, a bucket, ...).
* A {@link Policy}'s optional automatic trigger - what fires it without a person asking. {@code
* type} selects a trigger kind ("schedule", and in future "webhook", "folder-watch", ...); {@code
* options} carries type-specific configuration (a {@link Schedule}, a webhook secret, ...).
*
* <p>Manual running is <em>not</em> a trigger kind: every policy can always be run on demand, so a
* policy with no automatic trigger simply has a {@code null} {@code TriggerConfig}. A trigger
* answers only "when"; where a run's files come from is a separate concern owned by the policy's
* {@link InputSpec sources}.
*
* <p>Data-driven and parallel to {@link OutputSpec}: new trigger kinds are new {@code type} values
* handled by a new trigger bean, with no change to the model. {@code "manual"} means there is no
* automatic trigger - the policy is only ever run on demand.
* handled by a new trigger bean, with no change to the model.
*/
public record TriggerConfig(String type, Map<String, Object> options) {
public TriggerConfig {
type = type == null || type.isBlank() ? "manual" : type;
options = options == null ? Map.of() : options;
}
/** No automatic trigger; the policy is run on demand only. */
public static TriggerConfig manual() {
return new TriggerConfig("manual", Map.of());
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java
+130
View File
@@ -0,0 +1,130 @@
package stirling.software.proprietary.policy.output;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import org.apache.commons.io.FilenameUtils;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
import org.springframework.http.MediaTypeFactory;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.job.ResultFile;
import stirling.software.proprietary.policy.config.FolderAccessGuard;
import stirling.software.proprietary.policy.model.OutputSpec;
/**
* Writes a run's output files to a directory on disk. The destination is the {@code directory}
* option of the {@link OutputSpec}.
*
* <p>Files are streamed to disk (so large outputs are not buffered) and given unique names to avoid
* clobbering existing files. The returned {@link ResultFile}s describe what was written (path +
* size); they carry a synthetic id because the deliverable is the file on disk, not a {@code
* FileStorage} entry, so folder outputs are not downloadable via {@code /files/{id}}.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class FolderOutputSink implements PolicyOutputSink {
static final String TYPE = FolderAccessGuard.FOLDER_TYPE;
static final String DIRECTORY_OPTION = "directory";
private final FolderAccessGuard accessGuard;
@Override
public String type() {
return TYPE;
}
@Override
public boolean supports(OutputSpec spec) {
return spec != null && TYPE.equals(spec.type());
}
@Override
public void validate(OutputSpec spec) {
accessGuard.requirePermitted(directoryOf(spec));
}
@Override
public List<ResultFile> deliver(String runId, List<Resource> outputs, OutputSpec spec)
throws IOException {
Path targetDir = accessGuard.requirePermitted(directoryOf(spec));
Files.createDirectories(targetDir);
List<ResultFile> results = new ArrayList<>();
for (int i = 0; i < outputs.size(); i++) {
Resource resource = outputs.get(i);
String name = safeName(resource.getFilename(), i);
Path target = uniqueTarget(targetDir, name);
try (InputStream is = resource.getInputStream()) {
Files.copy(is, target);
}
long size = Files.size(target);
String contentType =
MediaTypeFactory.getMediaType(name)
.orElse(MediaType.APPLICATION_OCTET_STREAM)
.toString();
results.add(
ResultFile.builder()
.fileId(UUID.randomUUID().toString())
.fileName(target.toString())
.contentType(contentType)
.fileSize(size)
.build());
log.debug("Wrote policy run {} output to {}", runId, target);
}
return results;
}
private static Path directoryOf(OutputSpec spec) {
Object directory = spec.options().get(DIRECTORY_OPTION);
if (directory == null || directory.toString().isBlank()) {
throw new IllegalArgumentException(
"folder output requires a '" + DIRECTORY_OPTION + "' option");
}
return Path.of(directory.toString());
}
/**
* The resource's filename reduced to a bare, traversal-free name: any directory component or
* "../" is stripped so a crafted output name cannot escape {@code targetDir}. Falls back to a
* synthetic name when the filename is absent or reduces to nothing usable.
*/
private static String safeName(String filename, int index) {
if (filename == null || filename.isBlank()) {
return "output-" + index;
}
String name = FilenameUtils.getName(filename);
if (name.isBlank() || ".".equals(name) || "..".equals(name)) {
return "output-" + index;
}
return name;
}
/** Resolve a non-colliding path in {@code dir}, appending " (n)" before the extension. */
private static Path uniqueTarget(Path dir, String filename) {
Path candidate = dir.resolve(filename);
if (!Files.exists(candidate)) {
return candidate;
}
String base = FilenameUtils.getBaseName(filename);
String ext = FilenameUtils.getExtension(filename);
String suffix = ext.isEmpty() ? "" : "." + ext;
for (int n = 1; ; n++) {
Path next = dir.resolve(base + " (" + n + ")" + suffix);
if (!Files.exists(next)) {
return next;
}
}
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java
+6
View File
@@ -23,6 +23,12 @@ public interface PolicyOutputSink {
/** Whether this sink can handle the given output spec. */
boolean supports(OutputSpec spec);
/**
* Check that an output spec is usable, throwing {@link IllegalArgumentException} if not. Called
* when a policy is saved so misconfiguration fails fast rather than at run time.
*/
default void validate(OutputSpec spec) {}
/**
* Persist/deliver the output files and return their descriptors.
*
app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java
+2
View File
@@ -30,6 +30,7 @@ public class InProcessPolicyStore implements PolicyStore {
policy.owner(),
policy.enabled(),
policy.trigger(),
policy.sources(),
policy.steps(),
policy.output());
policies.put(id, stored);
@@ -50,6 +51,7 @@ public class InProcessPolicyStore implements PolicyStore {
public List<Policy> findByTriggerType(String triggerType) {
return policies.values().stream()
.filter(Policy::enabled)
.filter(policy -> policy.trigger() != null)
.filter(policy -> triggerType.equals(policy.trigger().type()))
.toList();
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java
+2 -1
View File
@@ -37,6 +37,7 @@ public class JpaPolicyStore implements PolicyStore {
policy.owner(),
policy.enabled(),
policy.trigger(),
policy.sources(),
policy.steps(),
policy.output());
@@ -45,7 +46,7 @@ public class JpaPolicyStore implements PolicyStore {
entity.setName(stored.name());
entity.setOwner(stored.owner());
entity.setEnabled(stored.enabled());
entity.setTriggerType(stored.trigger().type());
entity.setTriggerType(stored.trigger() == null ? null : stored.trigger().type());
entity.setPolicyJson(objectMapper.writeValueAsString(stored));
repository.save(entity);
return stored;
app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java
+302
View File
@@ -0,0 +1,302 @@
package stirling.software.proprietary.policy.trigger;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import java.io.IOException;
import java.nio.file.ClosedWatchServiceException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.store.PolicyStore;
/**
* Fires policies the moment a file lands in one of their folder sources, instead of polling on a
* timer. The trigger only reads that location; turning it into files is still the source's job.
*
* <p>The watcher is a latency optimisation, not a source of truth, so this pairs an event watch
* with a low-frequency <b>reconcile</b> sweep ({@code watchReconcileSeconds}). The reconcile both
* (a) re-syncs which directories are watched as policies are created/edited/deleted and folders
* appear on disk, and (b) runs every folder-watch policy once, catching files that pre-dated the
* watch, events lost to inotify-queue overflow, and changes on filesystems that do not deliver
* events at all (NFS, many container bind mounts). Both paths just call {@link PolicyRunner#run};
* the {@link InputSource} does the claiming, so a redundant run finds nothing to claim and is
* harmless.
*
* <p>Like {@link ScheduleTrigger}, watch state is per-node and in memory; this assumes a single
* node and rebuilds its registrations on restart from the {@link PolicyStore}.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class FolderWatchTrigger implements PolicyTrigger {
private static final String TYPE = "folder-watch";
private final PolicyStore policyStore;
private final PolicyRunner policyRunner;
private final List<InputSource> inputSources;
private final ApplicationProperties applicationProperties;
private final Map<Path, WatchKey> keysByDir = new ConcurrentHashMap<>();
private final Map<WatchKey, Path> dirByKey = new ConcurrentHashMap<>();
private volatile boolean running;
// Package-visible (not private) so tests can drive syncRegistrations() against a real service.
volatile WatchService watchService;
private volatile ScheduledExecutorService reconciler;
@Override
public String type() {
return TYPE;
}
@Override
public void validate(Policy policy) {
if (watchDirsOf(policy).isEmpty()) {
throw new IllegalArgumentException(
"folder-watch trigger requires at least one watchable (folder) input source");
}
}
@Override
public synchronized void start() {
if (watchService != null) {
return;
}
try {
watchService = FileSystems.getDefault().newWatchService();
} catch (IOException e) {
log.error("Could not start folder-watch trigger: {}", e.getMessage(), e);
return;
}
running = true;
Thread.ofVirtual().name("policy-folder-watch").start(this::watchLoop);
long reconcileSeconds = applicationProperties.getPolicies().getWatchReconcileSeconds();
reconciler =
Executors.newSingleThreadScheduledExecutor(
Thread.ofVirtual().name("policy-folder-reconcile-", 0).factory());
// First reconcile runs immediately so pre-existing files are picked up at startup.
reconciler.scheduleAtFixedRate(this::safeReconcile, 0, reconcileSeconds, TimeUnit.SECONDS);
log.info("Folder-watch trigger started (reconcile every {}s)", reconcileSeconds);
}
@Override
public synchronized void stop() {
running = false;
if (reconciler != null) {
reconciler.shutdownNow();
reconciler = null;
}
if (watchService != null) {
try {
watchService.close(); // wakes the watch loop with ClosedWatchServiceException
} catch (IOException e) {
log.debug("Error closing folder watch service: {}", e.getMessage());
}
watchService = null;
}
keysByDir.clear();
dirByKey.clear();
}
private void watchLoop() {
// Capture the service once: stop() may null the field, and a local avoids racing that to an
// NPE (close() still wakes take()/poll() on this same instance).
WatchService watcher = watchService;
if (watcher == null) {
return;
}
while (running) {
WatchKey first;
try {
first = watcher.take();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
} catch (ClosedWatchServiceException e) {
return;
}
runForChangedDirs(drainBurst(watcher, first));
}
}
/**
* Collect the directories touched by {@code first} and any further events that arrive within
* the quiet period, so a burst of file-system events becomes a single set of affected
* directories. The event kinds are irrelevant: any event on a watched dir just means "go look".
*/
private Set<Path> drainBurst(WatchService watcher, WatchKey first) {
long quietPeriodMs = applicationProperties.getPolicies().getWatchQuietPeriodMs();
Set<Path> changed = new HashSet<>();
WatchKey key = first;
while (key != null) {
key.pollEvents();
Path dir = dirByKey.get(key);
if (dir != null) {
changed.add(dir);
}
key.reset();
try {
key = watcher.poll(quietPeriodMs, TimeUnit.MILLISECONDS);
} catch (ClosedWatchServiceException | InterruptedException e) {
break;
}
}
return changed;
}
/** Run every folder-watch policy that draws from one of the changed directories. */
void runForChangedDirs(Set<Path> changedDirs) {
if (changedDirs.isEmpty()) {
return;
}
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
List<Path> dirs;
try {
dirs = watchDirsOf(policy);
} catch (RuntimeException e) {
log.warn(
"Folder-watch policy {} is misconfigured: {}", policy.id(), e.getMessage());
continue;
}
if (dirs.stream().anyMatch(changedDirs::contains)) {
log.debug("Folder-watch policy {} ({}) saw activity", policy.id(), policy.name());
policyRunner.run(policy);
}
}
}
private void safeReconcile() {
try {
syncRegistrations();
runAll();
} catch (RuntimeException e) {
log.error("Folder-watch reconcile failed: {}", e.getMessage(), e);
}
}
/** The reconcile safety net: run every folder-watch policy regardless of watch events. */
void runAll() {
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
try {
policyRunner.run(policy);
} catch (RuntimeException e) {
log.warn(
"Folder-watch reconcile run failed for policy {}: {}",
policy.id(),
e.getMessage());
}
}
}
/**
* Bring the set of watched directories in line with the current folder-watch policies: register
* newly required directories that exist on disk, and cancel ones no longer wanted.
*/
synchronized void syncRegistrations() {
if (watchService == null) {
return;
}
Set<Path> desired = desiredDirs();
keysByDir
.entrySet()
.removeIf(
entry -> {
if (desired.contains(entry.getKey())) {
return false;
}
entry.getValue().cancel();
dirByKey.remove(entry.getValue());
return true;
});
for (Path dir : desired) {
if (keysByDir.containsKey(dir)) {
continue;
}
try {
WatchKey key = dir.register(watchService, ENTRY_CREATE, ENTRY_MODIFY);
keysByDir.put(dir, key);
dirByKey.put(key, dir);
log.info("Watching {} for folder-watch policies", dir);
} catch (IOException | RuntimeException e) {
log.warn("Could not watch {}: {}", dir, e.getMessage());
}
}
}
/** The directories currently registered with the watch service. Visible for tests. */
Set<Path> watchedDirs() {
return Set.copyOf(keysByDir.keySet());
}
/** Every existing directory any current folder-watch policy wants watched. */
private Set<Path> desiredDirs() {
Set<Path> dirs = new HashSet<>();
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
try {
for (Path dir : watchDirsOf(policy)) {
if (Files.isDirectory(dir)) {
dirs.add(dir);
}
}
} catch (RuntimeException e) {
log.warn(
"Folder-watch policy {} is misconfigured: {}", policy.id(), e.getMessage());
}
}
return dirs;
}
/**
* The normalised, absolute directories this policy's sources expose to watch. Normalisation
* makes registration keys and event-time matching comparable regardless of how the path was
* configured.
*/
private List<Path> watchDirsOf(Policy policy) {
List<Path> dirs = new ArrayList<>();
for (InputSpec spec : policy.sources()) {
InputSource source = sourceFor(spec);
if (source == null) {
continue;
}
for (Path dir : source.watchTargets(spec)) {
dirs.add(dir.toAbsolutePath().normalize());
}
}
return dirs;
}
private InputSource sourceFor(InputSpec spec) {
return inputSources.stream()
.filter(source -> source.supports(spec))
.findFirst()
.orElse(null);
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java
-41
View File
@@ -1,41 +0,0 @@
package stirling.software.proprietary.policy.trigger;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import stirling.software.proprietary.policy.engine.PolicyEngine;
import stirling.software.proprietary.policy.engine.PolicyRunHandle;
import stirling.software.proprietary.policy.model.PipelineDefinition;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
/**
* Runs policies on demand, in response to a request (the {@code PolicyController} endpoints, an AI,
* or another automation). It is the request-driven trigger: no background lifecycle, it just
* forwards to the engine. Any policy can be run manually regardless of its configured trigger type.
*/
@Service
@RequiredArgsConstructor
public class ManualTrigger implements PolicyTrigger {
private final PolicyEngine policyEngine;
@Override
public String type() {
return "manual";
}
/** Run a stored policy immediately and return its run handle. */
public PolicyRunHandle run(
Policy policy, PolicyInputs inputs, PolicyProgressListener listener) {
return policyEngine.runPolicy(policy, inputs, listener);
}
/** Run an ad-hoc pipeline (no stored policy), e.g. for AI or Automate one-offs. */
public PolicyRunHandle fire(
PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) {
return policyEngine.submit(definition, inputs, listener);
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java
+23 -9
View File
@@ -1,22 +1,36 @@
package stirling.software.proprietary.policy.trigger;
import stirling.software.proprietary.policy.model.Policy;
/**
* Activates policies of one trigger type. A trigger owns a {@link #type()} (matching {@code
* TriggerConfig.type()}); when its condition fires it runs the relevant {@code Policy} through the
* {@code PolicyEngine}.
* An automatic trigger: the thing that decides <em>when</em> a policy runs without a person asking.
* A trigger owns a {@link #type()} (matching {@code TriggerConfig.type()}); when its condition
* fires it hands the policy to the {@code PolicyRunner}, which pulls the policy's sources and
* starts the runs. A trigger never resolves sources itself.
*
* <p>Background triggers (folder watcher, schedule) are driven by configuration: on {@link
* #start()} they begin watching/scheduling for the policies returned by {@code
* PolicyStore.findByTriggerType(type())}, and stop on {@link #stop()}. Request-driven triggers
* (manual) have no background lifecycle and run a policy directly in response to a call. New
* trigger kinds are new beans of this type; the engine and the {@code Policy} model do not change.
* <p>Triggers are background, configuration-driven beans (schedule, and in future webhook or
* folder-watch): on {@link #start()} they begin watching/scheduling for the policies returned by
* {@code PolicyStore.findByTriggerType(type())}, and stop on {@link #stop()}. New trigger kinds are
* new beans of this type; the runner and the {@code Policy} model do not change.
*
* <p>Manual running is not a trigger - every policy can always be run on demand via the {@code
* PolicyRunner} regardless of whether it has a trigger.
*/
public interface PolicyTrigger {
/** Stable identifier for this trigger kind, matching {@code TriggerConfig.type()}. */
String type();
/** Begin activating policies of this type (e.g. start a folder watcher). No-op for manual. */
/**
* Check that this trigger is usable for the given policy, throwing {@link
* IllegalArgumentException} if not. Called when a policy is saved so misconfiguration fails
* fast rather than at fire time. Receives the whole {@link Policy} (not just its {@code
* TriggerConfig}) so a trigger whose firing depends on the policy's sources (folder-watch) can
* assert that relationship; most triggers only inspect {@code policy.trigger()}.
*/
default void validate(Policy policy) {}
/** Begin activating policies of this type (e.g. start the schedule sweep). */
default void start() {}
/** Stop activating and release any resources. */
app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManager.java
+56
View File
@@ -0,0 +1,56 @@
package stirling.software.proprietary.policy.trigger;
import java.util.List;
import org.springframework.context.SmartLifecycle;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Starts and stops every {@link PolicyTrigger} with the application lifecycle. Background triggers
* (schedule, and future folder/S3) begin watching on {@link #start()} and release resources on
* {@link #stop()}; request-driven triggers (manual) are no-ops.
*
* <p>This is the single activation point for triggers - a new background trigger only has to be a
* {@link PolicyTrigger} bean.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class PolicyTriggerManager implements SmartLifecycle {
private final List<PolicyTrigger> triggers;
private volatile boolean running;
@Override
public void start() {
for (PolicyTrigger trigger : triggers) {
try {
trigger.start();
} catch (RuntimeException e) {
log.error("Failed to start trigger '{}': {}", trigger.type(), e.getMessage(), e);
}
}
running = true;
}
@Override
public void stop() {
for (PolicyTrigger trigger : triggers) {
try {
trigger.stop();
} catch (RuntimeException e) {
log.error("Failed to stop trigger '{}': {}", trigger.type(), e.getMessage(), e);
}
}
running = false;
}
@Override
public boolean isRunning() {
return running;
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java
+158
View File
@@ -0,0 +1,158 @@
package stirling.software.proprietary.policy.trigger;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.Schedule;
import stirling.software.proprietary.policy.store.PolicyStore;
import tools.jackson.databind.ObjectMapper;
/**
* Fires policies on a {@link Schedule}. On {@link #start()} it sweeps on a fixed interval; each
* sweep finds the enabled "schedule" policies and runs any whose next firing has come due since it
* last fired.
*
* <p>The trigger only decides <em>when</em>: once a policy is due it hands it to the {@link
* PolicyRunner}, which pulls from the policy's configured sources and starts the runs. The trigger
* knows nothing about folders, buckets, or how many runs a sweep produces.
*
* <p>Caveat: last-fire times are tracked <b>in memory</b>, so this assumes a single node and resets
* on restart; cluster-wide coordination (leader election) is a follow-up.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class ScheduleTrigger implements PolicyTrigger {
private static final String TYPE = "schedule";
private final PolicyStore policyStore;
private final PolicyRunner policyRunner;
private final ObjectMapper objectMapper;
private final ApplicationProperties applicationProperties;
private final Map<String, Instant> lastFiredByPolicy = new ConcurrentHashMap<>();
private volatile ScheduledExecutorService scheduler;
@Override
public String type() {
return TYPE;
}
@Override
public void validate(Policy policy) {
ScheduleConfig.from(objectMapper, policy.trigger().options());
}
@Override
public synchronized void start() {
if (scheduler != null) {
return;
}
long sweepSeconds = applicationProperties.getPolicies().getScheduleSweepSeconds();
scheduler =
Executors.newSingleThreadScheduledExecutor(
Thread.ofVirtual().name("policy-schedule-", 0).factory());
scheduler.scheduleAtFixedRate(
this::safeSweep, sweepSeconds, sweepSeconds, TimeUnit.SECONDS);
log.info("Schedule trigger started (sweep every {}s)", sweepSeconds);
}
@Override
public synchronized void stop() {
if (scheduler != null) {
scheduler.shutdownNow();
scheduler = null;
}
}
private void safeSweep() {
try {
sweep(Instant.now());
} catch (RuntimeException e) {
log.error("Schedule sweep failed: {}", e.getMessage(), e);
}
}
/** Fire every scheduled policy that is due as of {@code now}. Package-visible for testing. */
void sweep(Instant now) {
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
ScheduleConfig config;
try {
config = ScheduleConfig.from(objectMapper, policy.trigger().options());
} catch (IllegalArgumentException e) {
log.warn("Scheduled policy {} is misconfigured: {}", policy.id(), e.getMessage());
continue;
}
// First time we see a policy, baseline its last-fire to now so it does not fire
// immediately; subsequent sweeps fire it once its next firing has passed.
Instant last = lastFiredByPolicy.computeIfAbsent(policy.id(), id -> now);
ZonedDateTime next = config.schedule().nextAfter(last.atZone(config.zone()));
if (!next.toInstant().isAfter(now)) {
lastFiredByPolicy.put(policy.id(), now);
log.info("Scheduled policy {} ({}) is due", policy.id(), policy.name());
policyRunner.run(policy);
}
}
}
/**
* The typed, validated form of a schedule trigger's options: the {@link Schedule} and the zone
* its wall-clock kinds are evaluated in (default UTC). Construction fails for a missing/invalid
* schedule or zone.
*/
record ScheduleConfig(Schedule schedule, ZoneId zone) {
private static final String SCHEDULE_OPTION = "schedule";
private static final String ZONE_OPTION = "zone";
static ScheduleConfig from(ObjectMapper mapper, Map<String, Object> options) {
Object scheduleNode = options.get(SCHEDULE_OPTION);
if (scheduleNode == null) {
throw new IllegalArgumentException("schedule trigger requires a 'schedule'");
}
Schedule schedule;
try {
schedule = mapper.convertValue(scheduleNode, Schedule.class);
} catch (RuntimeException e) {
throw new IllegalArgumentException("invalid schedule: " + rootMessage(e), e);
}
ZoneId zone = ZoneOffset.UTC;
Object zoneNode = options.get(ZONE_OPTION);
if (zoneNode != null && !zoneNode.toString().isBlank()) {
try {
zone = ZoneId.of(zoneNode.toString());
} catch (RuntimeException e) {
throw new IllegalArgumentException("invalid zone '" + zoneNode + "'");
}
}
return new ScheduleConfig(schedule, zone);
}
private static String rootMessage(Throwable t) {
Throwable cause = t;
while (cause.getCause() != null) {
cause = cause.getCause();
}
return cause.getMessage();
}
}
}
app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java
+74 -10
View File
@@ -66,6 +66,7 @@ import tools.jackson.databind.ObjectMapper;
public class AiWorkflowService {
private static final String DOCUMENTS_ENDPOINT = "/api/v1/documents";
private static final String PDF_TO_MARKDOWN_ENDPOINT = "/api/v1/convert/pdf/markdown";
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final AiEngineClient aiEngineClient;
@@ -194,6 +195,7 @@ public class AiWorkflowService {
return switch (response.getOutcome()) {
case NEED_CONTENT -> onNeedContent(response, filesById, request, listener);
case NEED_INGEST -> onNeedIngest(response, filesById, request, listener);
case CONVERT_MARKDOWN -> onConvertMarkdown(response, filesById, listener);
case TOOL_CALL -> onToolCall(response, filesById, listener);
case PLAN -> onPlan(response, filesById, request, listener);
case ANSWER -> onAnswer(response, filesById, request, listener);
@@ -330,6 +332,77 @@ public class AiWorkflowService {
return new WorkflowState.Pending(nextRequest);
}
/**
* Deterministically convert each requested PDF to Markdown via the {@code
* /convert/pdf/markdown} endpoint (backed by {@code PdfMarkdownConverter}) and return the
* {@code .md} file(s) as a completed result. No AI resume — the conversion output is the final
* answer.
*/
private WorkflowState onConvertMarkdown(
AiWorkflowResponse response,
Map<String, MultipartFile> filesById,
ProgressListener listener) {
List<AiFile> filesToConvert = response.getFilesToIngest();
if (filesToConvert == null || filesToConvert.isEmpty()) {
return new WorkflowState.Terminal(
cannotContinue(
"AI engine requested markdown conversion without listing any files."));
}
try {
List<Resource> resultFiles = new ArrayList<>();
List<String> inputNames = new ArrayList<>();
for (int i = 0; i < filesToConvert.size(); i++) {
AiFile file = filesToConvert.get(i);
MultipartFile multipartFile = filesById.get(file.getId());
if (multipartFile == null) {
return new WorkflowState.Terminal(
cannotContinue(
"AI engine requested markdown conversion for unknown file: "
+ file.getName()));
}
listener.onProgress(
AiWorkflowProgressEvent.executingTool(
PDF_TO_MARKDOWN_ENDPOINT, i + 1, filesToConvert.size()));
Resource input = toResource(multipartFile);
PipelineDefinition definition =
new PipelineDefinition(
"convert-markdown",
List.of(new PipelineStep(PDF_TO_MARKDOWN_ENDPOINT, Map.of())),
null);
PolicyExecutionResult result =
policyExecutor.execute(
definition,
PolicyInputs.of(List.of(input)),
PolicyProgressListener.NOOP);
resultFiles.addAll(result.files());
inputNames.add(multipartFile.getOriginalFilename());
}
return new WorkflowState.Terminal(
buildCompletedResponse(null, resultFiles, inputNames, null));
} catch (InternalApiTimeoutException e) {
log.error("PDF to Markdown conversion timed out: {}", e.getMessage());
return new WorkflowState.Terminal(
cannotContinue(toolTimeoutMessage(PDF_TO_MARKDOWN_ENDPOINT, e)));
} catch (Exception e) {
log.error("Failed to convert PDF to Markdown: {}", e.getMessage(), e);
return new WorkflowState.Terminal(
cannotContinue(toolFailureMessage(PDF_TO_MARKDOWN_ENDPOINT, e)));
}
}
private Resource toResource(MultipartFile file) throws IOException {
TempFile tempFile = tempFileManager.createManagedTempFile("ai-workflow");
file.transferTo(tempFile.getPath());
final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename());
return new FileSystemResource(tempFile.getFile()) {
@Override
public String getFilename() {
return originalName;
}
};
}
private void ingestFile(AiFile file, MultipartFile multipartFile) throws IOException {
List<AiPageText> pages = new ArrayList<>();
try (PDDocument document = pdfDocumentFactory.load(multipartFile, true)) {
@@ -551,16 +624,7 @@ public class AiWorkflowService {
private List<Resource> toResources(Map<String, MultipartFile> filesById) throws IOException {
List<Resource> resources = new ArrayList<>();
for (MultipartFile file : filesById.values()) {
TempFile tempFile = tempFileManager.createManagedTempFile("ai-workflow");
file.transferTo(tempFile.getPath());
final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename());
resources.add(
new FileSystemResource(tempFile.getFile()) {
@Override
public String getFilename() {
return originalName;
}
});
resources.add(toResource(file));
}
return resources;
}
app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java
-78
View File
@@ -30,11 +30,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.pdf.parser.PageImageLocator;
import stirling.software.SPDF.pdf.parser.PdfIngester;
import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage;
import stirling.software.SPDF.pdf.parser.PdfModels.RawLine;
import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment;
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.PdfUtils;
@@ -50,7 +46,6 @@ import stirling.software.proprietary.model.api.ai.FolioType;
public class PdfContentExtractor {
private final TabulaTableParser tabulaTableParser;
private final PdfIngester pdfIngester;
private static final int MAX_CHARACTERS_PER_PAGE = 4_000;
@@ -196,8 +191,6 @@ public class PdfContentExtractor {
case PAGE_TEXT, FULL_TEXT ->
Optional.<PdfContentResult>ofNullable(
extractText(lf, fileReq, remainingPages, remainingCharacters));
case PAGE_LAYOUT ->
Optional.<PdfContentResult>ofNullable(extractPageLayout(lf, remainingPages));
default -> {
log.warn(
"Content type {} not yet implemented, skipping for {}",
@@ -222,35 +215,6 @@ public class PdfContentExtractor {
return extracted.isEmpty() ? null : buildExtractedFileText(lf.fileName(), extracted);
}
private PageLayoutFileResult extractPageLayout(LoadedFile lf, int maxPages) throws IOException {
List<ParsedPage> parsedPages = pdfIngester.parse(lf.document(), maxPages);
List<LayoutPage> pages = new ArrayList<>();
for (ParsedPage pp : parsedPages) {
if (pp.layoutLines().isEmpty()) continue;
List<LayoutLine> lines = new ArrayList<>();
for (RawLine rawLine : pp.layoutLines()) {
List<LayoutFragment> fragments = new ArrayList<>();
for (TextFragment tf : rawLine.fragments()) {
fragments.add(
new LayoutFragment(
tf.text(),
tf.bounds().x(),
tf.bounds().y(),
tf.bounds().width(),
tf.fontSize(),
tf.bold()));
}
lines.add(new LayoutLine(rawLine.bounds().y(), fragments));
}
pages.add(new LayoutPage(pp.pageNumber(), lines));
}
if (pages.isEmpty()) return null;
PageLayoutFileResult result = new PageLayoutFileResult();
result.setFileName(lf.fileName());
result.setPages(pages);
return result;
}
private WorkflowArtifact buildArtifact(ArtifactKind kind, List<PdfContentResult> results) {
return switch (kind) {
case EXTRACTED_TEXT -> {
@@ -258,11 +222,6 @@ public class PdfContentExtractor {
artifact.setFiles(results.stream().map(ExtractedFileText.class::cast).toList());
yield artifact;
}
case PAGE_LAYOUT -> {
PageLayoutArtifact artifact = new PageLayoutArtifact();
artifact.setFiles(results.stream().map(PageLayoutFileResult.class::cast).toList());
yield artifact;
}
case TOOL_REPORT ->
throw new IllegalArgumentException(
"TOOL_REPORT artifacts are not produced by PdfContentExtractor");
@@ -569,7 +528,6 @@ public class PdfContentExtractor {
*/
enum ArtifactKind {
EXTRACTED_TEXT("extracted_text"),
PAGE_LAYOUT("page_layout"),
TOOL_REPORT("tool_report");
private final String value;
@@ -633,40 +591,4 @@ public class PdfContentExtractor {
this.report = report;
}
}
// Serialization contract with the Python engine — see PageLayoutArtifactContractTest.
/** One text fragment with its bounding-box geometry and font properties. */
record LayoutFragment(
String text, float x, float y, float width, float fontSize, boolean bold) {}
/** A visual line on the page: y-coordinate and all fragments on that line. */
record LayoutLine(float y, List<LayoutFragment> fragments) {}
/** All layout lines for a single page. */
record LayoutPage(int pageNumber, List<LayoutLine> lines) {}
/** Page layout data for one file, as a PdfContentResult. */
@Data
static final class PageLayoutFileResult implements PdfContentResult {
private String fileName;
private List<LayoutPage> pages = new ArrayList<>();
@Override
public ArtifactKind getArtifactKind() {
return ArtifactKind.PAGE_LAYOUT;
}
@Override
public int pagesConsumed() {
return pages.size();
}
}
/** Artifact carrying full spatial page layout for all input files. */
@Data
static final class PageLayoutArtifact implements WorkflowArtifact {
private final ArtifactKind kind = ArtifactKind.PAGE_LAYOUT;
private List<PageLayoutFileResult> files = new ArrayList<>();
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/config/FolderAccessGuardTest.java
+99
View File
@@ -0,0 +1,99 @@
package stirling.software.proprietary.policy.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Path;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.core.env.StandardEnvironment;
import stirling.software.common.configuration.InstallationPathConfig;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.Policy;
/**
* Tests for {@link FolderAccessGuard}: folder access is fail-closed, confined to the configured
* allowed roots, never reaches Stirling's own config directory, and is off entirely under SaaS.
*/
class FolderAccessGuardTest {
@TempDir Path tempDir;
private FolderAccessGuard guard(List<String> allowedRoots, String... activeProfiles) {
ApplicationProperties properties = new ApplicationProperties();
properties.getPolicies().setAllowedFolderRoots(allowedRoots);
StandardEnvironment environment = new StandardEnvironment();
environment.setActiveProfiles(activeProfiles);
return new FolderAccessGuard(properties, environment);
}
@Test
void permitsAndNormalisesADirectoryWithinAnAllowedRoot() {
FolderAccessGuard guard = guard(List.of(tempDir.toString()));
Path within = tempDir.resolve("inbox");
assertEquals(within.toAbsolutePath().normalize(), guard.requirePermitted(within));
}
@Test
void rejectsADirectoryOutsideEveryAllowedRoot() {
FolderAccessGuard guard = guard(List.of(tempDir.toString()));
assertThrows(
IllegalArgumentException.class,
() -> guard.requirePermitted(tempDir.resolveSibling("elsewhere")));
}
@Test
void rejectsTraversalThatWalksOutOfAnAllowedRoot() {
FolderAccessGuard guard = guard(List.of(tempDir.toString()));
assertThrows(
IllegalArgumentException.class,
() -> guard.requirePermitted(tempDir.resolve("..").resolve("escaped")));
}
@Test
void rejectsEverythingWhenNoRootsAreConfigured() {
FolderAccessGuard guard = guard(List.of());
assertThrows(IllegalArgumentException.class, () -> guard.requirePermitted(tempDir));
}
@Test
void rejectsTheStirlingConfigDirectoryEvenWhenItWouldBeInsideAnAllowedRoot() {
Path configDir =
Path.of(InstallationPathConfig.getConfigPath()).toAbsolutePath().normalize();
// Allow the config dir's parent, so only the protected-path rule can reject it.
FolderAccessGuard guard = guard(List.of(configDir.getParent().toString()));
assertThrows(
IllegalArgumentException.class,
() -> guard.requirePermitted(configDir.resolve("settings.yml")));
}
@Test
void refusesAllFolderAccessUnderTheSaasProfile() {
FolderAccessGuard guard = guard(List.of(tempDir.toString()), "saas");
assertThrows(IllegalArgumentException.class, () -> guard.requirePermitted(tempDir));
}
@Test
void usesFolderAccessDetectsFolderSourcesAndOutputs() {
FolderAccessGuard guard = guard(List.of(tempDir.toString()));
assertTrue(
guard.usesFolderAccess(
policy(List.of(InputSpec.folder("/in")), OutputSpec.inline())));
assertTrue(guard.usesFolderAccess(policy(List.of(), OutputSpec.folder("/out"))));
assertFalse(guard.usesFolderAccess(policy(List.of(), OutputSpec.inline())));
}
private static Policy policy(List<InputSpec> sources, OutputSpec output) {
return new Policy("p1", "p", "owner", true, null, sources, List.of(), output);
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java
+2 -3
View File
@@ -50,7 +50,6 @@ import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.output.InlineOutputSink;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
@@ -93,7 +92,7 @@ class PolicyEngineTest {
toolMetadataService,
tempFileManager,
JsonMapper.builder().build());
registry = new PolicyRunRegistry(30);
registry = new PolicyRunRegistry(new ApplicationProperties());
InlineOutputSink sink = new InlineOutputSink(fileStorage);
engine =
new PolicyEngine(
@@ -190,7 +189,7 @@ class PolicyEngineTest {
"rotate",
"owner",
true,
TriggerConfig.manual(),
null,
List.of(new PipelineStep(ROTATE, Map.of())),
OutputSpec.inline());
app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java
+2 -1
View File
@@ -12,6 +12,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.model.PipelineDefinition;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.WaitState;
@@ -25,7 +26,7 @@ class PolicyRunRegistryTest {
@BeforeEach
void setUp() {
registry = new PolicyRunRegistry(30);
registry = new PolicyRunRegistry(new ApplicationProperties());
}
@AfterEach
app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java
+159
View File
@@ -0,0 +1,159 @@
package stirling.software.proprietary.policy.engine;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.input.ResolvedInput;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
/**
* Tests for {@link PolicyRunner}: the one place that turns a policy's sources into runs. Verifies
* it pulls every source, runs one job per unit of work, feeds each unit's completion hook the run
* outcome, and that a generator (no sources) still runs once.
*/
@ExtendWith(MockitoExtension.class)
class PolicyRunnerTest {
@Mock private PolicyEngine policyEngine;
@Mock private InputSource folderSource;
private PolicyRunner runner;
@BeforeEach
void setUp() {
runner = new PolicyRunner(policyEngine, List.of(folderSource));
}
@Test
void runsOnceWithNoFilesWhenThePolicyHasNoSources() {
Policy policy = policy(List.of());
when(policyEngine.runPolicy(eq(policy), any(), any()))
.thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
runner.run(policy);
ArgumentCaptor<PolicyInputs> inputs = ArgumentCaptor.forClass(PolicyInputs.class);
verify(policyEngine).runPolicy(eq(policy), inputs.capture(), any());
assertTrue(inputs.getValue().primary().isEmpty());
}
@Test
void pullsEverySourceAndRunsOnePerUnitOfWork() throws Exception {
InputSpec spec = InputSpec.folder("/in");
Policy policy = policy(List.of(spec));
when(folderSource.supports(spec)).thenReturn(true);
when(folderSource.resolve(spec))
.thenReturn(
List.of(
ResolvedInput.of(PolicyInputs.of(List.of())),
ResolvedInput.of(PolicyInputs.of(List.of()))));
when(policyEngine.runPolicy(any(), any(), any()))
.thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
runner.run(policy);
verify(policyEngine, times(2)).runPolicy(eq(policy), any(), any());
}
@Test
void feedsEachUnitsCompletionHookTheRunOutcome() throws Exception {
InputSpec spec = InputSpec.folder("/in");
Policy policy = policy(List.of(spec));
AtomicBoolean outcome = new AtomicBoolean(false);
ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set);
when(folderSource.supports(spec)).thenReturn(true);
when(folderSource.resolve(spec)).thenReturn(List.of(unit));
CompletableFuture<PolicyRun> completion = new CompletableFuture<>();
when(policyEngine.runPolicy(any(), any(), any()))
.thenReturn(new PolicyRunHandle("r", completion));
runner.run(policy);
PolicyRun run = mock(PolicyRun.class);
when(run.getStatus()).thenReturn(PolicyRunStatus.COMPLETED);
completion.complete(run);
assertTrue(outcome.get());
}
@Test
void reportsFailureToTheCompletionHookWhenTheRunDoesNotComplete() throws Exception {
InputSpec spec = InputSpec.folder("/in");
Policy policy = policy(List.of(spec));
AtomicBoolean outcome = new AtomicBoolean(true);
ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set);
when(folderSource.supports(spec)).thenReturn(true);
when(folderSource.resolve(spec)).thenReturn(List.of(unit));
CompletableFuture<PolicyRun> completion = new CompletableFuture<>();
when(policyEngine.runPolicy(any(), any(), any()))
.thenReturn(new PolicyRunHandle("r", completion));
runner.run(policy);
completion.completeExceptionally(new RuntimeException("boom"));
assertFalse(outcome.get());
}
@Test
void skipsSourcesWithNoMatchingBean() {
InputSpec spec = new InputSpec("s3", Map.of());
Policy policy = policy(List.of(spec));
when(folderSource.supports(spec)).thenReturn(false);
runner.run(policy);
verifyNoInteractions(policyEngine);
}
@Test
void runWithSuppliedInputsBypassesSources() {
Policy policy = policy(List.of(InputSpec.folder("/in")));
PolicyInputs inputs = PolicyInputs.of(List.of());
PolicyRunHandle handle = new PolicyRunHandle("r", new CompletableFuture<>());
when(policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP))
.thenReturn(handle);
assertSame(handle, runner.runWith(policy, inputs, PolicyProgressListener.NOOP));
verifyNoInteractions(folderSource);
}
private static Policy policy(List<InputSpec> sources) {
return new Policy(
"p1",
"p",
"owner",
true,
null,
sources,
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyValidatorTest.java
+114
View File
@@ -0,0 +1,114 @@
package stirling.software.proprietary.policy.engine;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.output.PolicyOutputSink;
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
/** Tests for {@link PolicyValidator}: routes each facet to its handler and surfaces failures. */
@ExtendWith(MockitoExtension.class)
class PolicyValidatorTest {
@Mock private PolicyTrigger trigger;
@Mock private InputSource inputSource;
@Mock private PolicyOutputSink outputSink;
private PolicyValidator validator;
@BeforeEach
void setUp() {
validator =
new PolicyValidator(List.of(trigger), List.of(inputSource), List.of(outputSink));
}
@Test
void delegatesEachFacetToItsHandler() {
when(trigger.type()).thenReturn("schedule");
when(inputSource.supports(any())).thenReturn(true);
when(outputSink.supports(any())).thenReturn(true);
Policy policy = policy("schedule");
validator.validate(policy);
verify(trigger).validate(policy);
verify(inputSource).validate(policy.sources().get(0));
verify(outputSink).validate(policy.output());
}
@Test
void skipsTriggerValidationForAManualOnlyPolicy() {
when(inputSource.supports(any())).thenReturn(true);
when(outputSink.supports(any())).thenReturn(true);
validator.validate(manualOnly());
verify(trigger, never()).validate(any());
}
@Test
void surfacesAnInvalidConfigFromAHandler() {
when(trigger.type()).thenReturn("schedule");
doThrow(new IllegalArgumentException("invalid schedule")).when(trigger).validate(any());
IllegalArgumentException ex =
assertThrows(
IllegalArgumentException.class,
() -> validator.validate(policy("schedule")));
assertTrue(ex.getMessage().contains("schedule"));
}
@Test
void rejectsAnUnknownTriggerType() {
when(trigger.type()).thenReturn("schedule");
IllegalArgumentException ex =
assertThrows(
IllegalArgumentException.class,
() -> validator.validate(policy("mystery")));
assertTrue(ex.getMessage().contains("unknown trigger type"));
}
private static Policy policy(String triggerType) {
return new Policy(
"p1",
"p",
"owner",
true,
new TriggerConfig(triggerType, Map.of()),
List.of(InputSpec.folder("/in")),
List.of(),
OutputSpec.inline());
}
private static Policy manualOnly() {
return new Policy(
"p1",
"p",
"owner",
true,
null,
List.of(InputSpec.folder("/in")),
List.of(),
OutputSpec.inline());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java
+137
View File
@@ -0,0 +1,137 @@
package stirling.software.proprietary.policy.input;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.core.env.StandardEnvironment;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.util.FileReadinessChecker;
import stirling.software.proprietary.policy.config.FolderAccessGuard;
import stirling.software.proprietary.policy.model.InputSpec;
/** Tests for {@link FolderInputSource}: consume (claim + route) and snapshot (read-only) modes. */
@ExtendWith(MockitoExtension.class)
class FolderInputSourceTest {
@Mock private FileReadinessChecker readinessChecker;
@TempDir Path tempDir;
private FolderInputSource source;
@BeforeEach
void setUp() {
ApplicationProperties properties = new ApplicationProperties();
properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString()));
FolderAccessGuard guard = new FolderAccessGuard(properties, new StandardEnvironment());
source = new FolderInputSource(readinessChecker, guard);
// Lenient: the missing-dir / nonexistent-dir cases return before any readiness check.
lenient().when(readinessChecker.isReady(any())).thenReturn(true);
}
@Test
void consumeClaimsFilesAndRoutesToDoneOnSuccess() throws IOException {
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
Files.writeString(inputDir.resolve("doc.pdf"), "data");
List<ResolvedInput> work = source.resolve(InputSpec.folder(inputDir.toString()));
assertEquals(1, work.size());
assertEquals(1, work.get(0).inputs().primary().size());
// Claimed out of the input dir.
assertFalse(Files.exists(inputDir.resolve("doc.pdf")));
assertTrue(
Files.exists(
inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf")));
work.get(0).onComplete().accept(true);
assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("done").resolve("doc.pdf")));
assertFalse(
Files.exists(
inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf")));
}
@Test
void consumeRoutesToErrorOnFailure() throws IOException {
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
Files.writeString(inputDir.resolve("doc.pdf"), "data");
List<ResolvedInput> work = source.resolve(InputSpec.folder(inputDir.toString()));
work.get(0).onComplete().accept(false);
assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("error").resolve("doc.pdf")));
}
@Test
void snapshotReadsWithoutClaiming() throws IOException {
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
Files.writeString(inputDir.resolve("doc.pdf"), "data");
List<ResolvedInput> work =
source.resolve(
new InputSpec(
"folder",
Map.of("directory", inputDir.toString(), "mode", "snapshot")));
assertEquals(1, work.size());
// Not moved, and completing the run is a no-op.
assertTrue(Files.exists(inputDir.resolve("doc.pdf")));
work.get(0).onComplete().accept(true);
assertTrue(Files.exists(inputDir.resolve("doc.pdf")));
}
@Test
void missingDirectoryOptionFails() {
assertThrows(
IllegalArgumentException.class,
() -> source.resolve(new InputSpec("folder", Map.of())));
}
@Test
void nonexistentDirectoryYieldsNoWork() throws IOException {
List<ResolvedInput> work =
source.resolve(InputSpec.folder(tempDir.resolve("nope").toString()));
assertTrue(work.isEmpty());
}
@Test
void validateRejectsMissingDirectory() {
assertThrows(
IllegalArgumentException.class,
() -> source.validate(new InputSpec("folder", Map.of())));
}
@Test
void rejectsADirectoryOutsideTheAllowedRoots() {
Path outside = tempDir.resolveSibling("not-allowed");
assertThrows(
IllegalArgumentException.class,
() -> source.resolve(InputSpec.folder(outside.toString())));
assertThrows(
IllegalArgumentException.class,
() -> source.validate(InputSpec.folder(outside.toString())));
}
@Test
void watchTargetsIsTheConfiguredDirectory() {
Path inputDir = tempDir.resolve("in");
assertEquals(List.of(inputDir), source.watchTargets(InputSpec.folder(inputDir.toString())));
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java
+105
View File
@@ -0,0 +1,105 @@
package stirling.software.proprietary.policy.output;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.core.env.StandardEnvironment;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.job.ResultFile;
import stirling.software.proprietary.policy.config.FolderAccessGuard;
import stirling.software.proprietary.policy.model.OutputSpec;
/** Tests for {@link FolderOutputSink}: outputs are written to the configured directory on disk. */
class FolderOutputSinkTest {
@TempDir Path tempDir;
private FolderOutputSink sink;
@BeforeEach
void setUp() {
ApplicationProperties properties = new ApplicationProperties();
properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString()));
sink = new FolderOutputSink(new FolderAccessGuard(properties, new StandardEnvironment()));
}
@Test
void writesEachOutputToTheDirectory() throws IOException {
Path out = tempDir.resolve("out");
List<Resource> outputs = List.of(named("a.pdf", "aaa"), named("b.pdf", "bb"));
List<ResultFile> results =
sink.deliver("run-1", outputs, OutputSpec.folder(out.toString()));
assertEquals(2, results.size());
assertTrue(Files.exists(out.resolve("a.pdf")));
assertEquals("aaa", Files.readString(out.resolve("a.pdf")));
assertEquals("bb", Files.readString(out.resolve("b.pdf")));
}
@Test
void collidingNamesGetAUniqueSuffix() throws IOException {
Path out = tempDir.resolve("out");
List<Resource> outputs = List.of(named("a.pdf", "first"), named("a.pdf", "second"));
sink.deliver("run-1", outputs, OutputSpec.folder(out.toString()));
assertTrue(Files.exists(out.resolve("a.pdf")));
assertTrue(Files.exists(out.resolve("a (1).pdf")));
}
@Test
void missingDirectoryOptionIsRejected() {
OutputSpec noDir = new OutputSpec("folder", Map.of());
assertThrows(IllegalArgumentException.class, () -> sink.validate(noDir));
assertThrows(
IllegalArgumentException.class,
() -> sink.deliver("run-1", List.of(named("a.pdf", "x")), noDir));
}
@Test
void aDirectoryOutsideTheAllowedRootsIsRejected() {
OutputSpec outside = OutputSpec.folder(tempDir.resolveSibling("not-allowed").toString());
assertThrows(IllegalArgumentException.class, () -> sink.validate(outside));
assertThrows(
IllegalArgumentException.class,
() -> sink.deliver("run-1", List.of(named("a.pdf", "x")), outside));
}
@Test
void filenamesWithPathTraversalAreConfinedToTheDirectory() throws IOException {
Path out = tempDir.resolve("out");
List<Resource> outputs =
List.of(named("../escape.pdf", "x"), named("nested/deep.pdf", "y"));
sink.deliver("run-1", outputs, OutputSpec.folder(out.toString()));
// Each name is reduced to its bare form inside the target dir; nothing escapes.
assertTrue(Files.exists(out.resolve("escape.pdf")));
assertTrue(Files.exists(out.resolve("deep.pdf")));
assertFalse(Files.exists(tempDir.resolve("escape.pdf")));
}
private static ByteArrayResource named(String filename, String content) {
return new ByteArrayResource(content.getBytes()) {
@Override
public String getFilename() {
return filename;
}
};
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java
+13 -10
View File
@@ -28,7 +28,7 @@ class InProcessPolicyStoreTest {
@Test
void savedPolicyGetsAnIdAndIsRetrievable() {
Policy saved = store.save(policy(null, "compress", "manual", true));
Policy saved = store.save(policy(null, "compress", null, true));
assertNotNull(saved.id());
assertFalse(saved.id().isBlank());
@@ -37,7 +37,7 @@ class InProcessPolicyStoreTest {
@Test
void savingWithAnExistingIdUpdatesInPlace() {
Policy created = store.save(policy(null, "before", "manual", true));
Policy created = store.save(policy(null, "before", null, true));
store.save(
new Policy(
@@ -45,7 +45,7 @@ class InProcessPolicyStoreTest {
"after",
"owner",
true,
TriggerConfig.manual(),
null,
List.of(),
OutputSpec.inline()));
@@ -55,19 +55,20 @@ class InProcessPolicyStoreTest {
@Test
void findByTriggerTypeReturnsOnlyEnabledMatches() {
store.save(policy(null, "watch", "folder", true));
store.save(policy(null, "watch-disabled", "folder", false));
store.save(policy(null, "nightly", "schedule", true));
store.save(policy(null, "nightly-disabled", "schedule", false));
store.save(policy(null, "hooked", "webhook", true));
store.save(policy(null, "on-demand", null, true)); // manual-only: no trigger
List<Policy> folder = store.findByTriggerType("folder");
List<Policy> scheduled = store.findByTriggerType("schedule");
assertEquals(1, folder.size());
assertEquals("watch", folder.get(0).name());
assertEquals(1, scheduled.size());
assertEquals("nightly", scheduled.get(0).name());
}
@Test
void deleteRemovesThePolicy() {
Policy saved = store.save(policy(null, "p", "manual", true));
Policy saved = store.save(policy(null, "p", null, true));
assertTrue(store.delete(saved.id()));
assertTrue(store.get(saved.id()).isEmpty());
@@ -75,12 +76,14 @@ class InProcessPolicyStoreTest {
}
private static Policy policy(String id, String name, String triggerType, boolean enabled) {
TriggerConfig trigger =
triggerType == null ? null : new TriggerConfig(triggerType, Map.of());
return new Policy(
id,
name,
"owner",
enabled,
new TriggerConfig(triggerType, Map.of()),
trigger,
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java
+11 -9
View File
@@ -18,6 +18,7 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
@@ -53,7 +54,8 @@ class JpaPolicyStoreTest {
"compress incoming",
"alice",
true,
new TriggerConfig("folder", Map.of("path", "/in")),
new TriggerConfig("schedule", Map.of()),
List.of(InputSpec.folder("/in")),
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline()));
@@ -62,7 +64,7 @@ class JpaPolicyStoreTest {
verify(repository).save(captor.capture());
PolicyEntity entity = captor.getValue();
assertEquals(saved.id(), entity.getId());
assertEquals("folder", entity.getTriggerType());
assertEquals("schedule", entity.getTriggerType());
assertTrue(entity.isEnabled());
// The stored JSON round-trips back to an equal policy.
assertEquals(saved, objectMapper.readValue(entity.getPolicyJson(), Policy.class));
@@ -76,7 +78,7 @@ class JpaPolicyStoreTest {
"rotate",
"alice",
true,
TriggerConfig.manual(),
null, // manual-only: no automatic trigger
List.of(
new PipelineStep(
"/api/v1/general/rotate-pdf", Map.of("angle", 90))),
@@ -94,16 +96,16 @@ class JpaPolicyStoreTest {
"watch",
"alice",
true,
new TriggerConfig("folder", Map.of()),
new TriggerConfig("schedule", Map.of()),
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
when(repository.findByTriggerTypeAndEnabledTrue("folder"))
when(repository.findByTriggerTypeAndEnabledTrue("schedule"))
.thenReturn(List.of(entityFor(policy)));
List<Policy> folder = store.findByTriggerType("folder");
List<Policy> scheduled = store.findByTriggerType("schedule");
assertEquals(1, folder.size());
assertEquals("p1", folder.get(0).id());
assertEquals(1, scheduled.size());
assertEquals("p1", scheduled.get(0).id());
}
@Test
@@ -122,7 +124,7 @@ class JpaPolicyStoreTest {
entity.setName(policy.name());
entity.setOwner(policy.owner());
entity.setEnabled(policy.enabled());
entity.setTriggerType(policy.trigger().type());
entity.setTriggerType(policy.trigger() == null ? null : policy.trigger().type());
entity.setPolicyJson(objectMapper.writeValueAsString(policy));
return entity;
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java
+178
View File
@@ -0,0 +1,178 @@
package stirling.software.proprietary.policy.trigger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.WatchService;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.store.PolicyStore;
/**
* Tests for {@link FolderWatchTrigger}'s dispatch logic via the package-visible {@code
* runForChangedDirs}/{@code runAll}, plus its cross-facet validation. The OS watch loop and
* scheduled reconcile are thin glue around these and are not exercised here (a real {@code
* WatchService} is timing-dependent), mirroring how {@link ScheduleTriggerTest} drives {@code
* sweep} directly. The folder source is stubbed to mirror {@code FolderInputSource.watchTargets}.
*/
@ExtendWith(MockitoExtension.class)
class FolderWatchTriggerTest {
@Mock private PolicyStore policyStore;
@Mock private PolicyRunner policyRunner;
@Mock private InputSource folderSource;
@TempDir Path tempDir;
private FolderWatchTrigger trigger;
@BeforeEach
void setUp() {
trigger =
new FolderWatchTrigger(
policyStore,
policyRunner,
List.of(folderSource),
new ApplicationProperties());
lenient().when(folderSource.supports(any())).thenReturn(true);
lenient()
.when(folderSource.watchTargets(any()))
.thenAnswer(
invocation -> {
InputSpec spec = invocation.getArgument(0);
Object dir = spec.options().get("directory");
if (dir == null) {
throw new IllegalArgumentException(
"folder input requires a 'directory' option");
}
return List.of(Path.of(dir.toString()));
});
}
@Test
void validateRejectsPolicyWithNoWatchableSource() {
assertThrows(
IllegalArgumentException.class,
() -> trigger.validate(folderWatch("p1", List.of())));
}
@Test
void validateAcceptsPolicyWithAFolderSource() {
trigger.validate(folderWatch("p1", List.of(InputSpec.folder("/in"))));
}
@Test
void runsOnlyPoliciesDrawingFromTheChangedDirectory() {
Policy a = folderWatch("a", List.of(InputSpec.folder("/in/a")));
Policy b = folderWatch("b", List.of(InputSpec.folder("/in/b")));
when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b));
trigger.runForChangedDirs(Set.of(normalized("/in/a")));
verify(policyRunner).run(a);
verify(policyRunner, never()).run(b);
}
@Test
void skipsAMisconfiguredPolicyButStillRunsTheOthers() {
Policy bad = folderWatch("bad", List.of(new InputSpec("folder", Map.of())));
Policy good = folderWatch("good", List.of(InputSpec.folder("/in/a")));
when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(bad, good));
trigger.runForChangedDirs(Set.of(normalized("/in/a")));
verify(policyRunner).run(good);
verify(policyRunner, never()).run(bad);
}
@Test
void anEmptyChangeSetDoesNothing() {
trigger.runForChangedDirs(Set.of());
verifyNoInteractions(policyStore, policyRunner);
}
@Test
void reconcileRunsEveryFolderWatchPolicyAsASafetyNet() {
Policy a = folderWatch("a", List.of(InputSpec.folder("/in/a")));
Policy b = folderWatch("b", List.of(InputSpec.folder("/in/b")));
when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b));
trigger.runAll();
verify(policyRunner).run(a);
verify(policyRunner).run(b);
}
@Test
void syncRegistrationsWatchesExistingDirsAndCancelsRemovedOnes() throws Exception {
Path dirA = Files.createDirectories(tempDir.resolve("a"));
Path dirB = Files.createDirectories(tempDir.resolve("b"));
Path missing = tempDir.resolve("missing"); // never created on disk
Policy a = folderWatch("a", List.of(InputSpec.folder(dirA.toString())));
Policy b = folderWatch("b", List.of(InputSpec.folder(dirB.toString())));
Policy m = folderWatch("m", List.of(InputSpec.folder(missing.toString())));
WatchService service = FileSystems.getDefault().newWatchService();
try {
trigger.watchService = service;
when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b, m));
trigger.syncRegistrations();
// Existing dirs are watched; the non-existent one is skipped.
assertEquals(
Set.of(normalized(dirA.toString()), normalized(dirB.toString())),
trigger.watchedDirs());
// b's policy is removed: its registration is cancelled, a remains.
when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a));
trigger.syncRegistrations();
assertEquals(Set.of(normalized(dirA.toString())), trigger.watchedDirs());
} finally {
service.close();
}
}
private static Path normalized(String dir) {
return Path.of(dir).toAbsolutePath().normalize();
}
private static Policy folderWatch(String id, List<InputSpec> sources) {
return new Policy(
id,
"watcher",
"owner",
true,
new TriggerConfig("folder-watch", Map.of()),
sources,
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManagerTest.java
+51
View File
@@ -0,0 +1,51 @@
package stirling.software.proprietary.policy.trigger;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.verify;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
/**
* Tests for {@link PolicyTriggerManager}: starts/stops every trigger, tolerating individual
* failures.
*/
@ExtendWith(MockitoExtension.class)
class PolicyTriggerManagerTest {
@Mock private PolicyTrigger triggerA;
@Mock private PolicyTrigger triggerB;
@Test
void startsAndStopsAllTriggers() {
PolicyTriggerManager manager = new PolicyTriggerManager(List.of(triggerA, triggerB));
assertFalse(manager.isRunning());
manager.start();
verify(triggerA).start();
verify(triggerB).start();
assertTrue(manager.isRunning());
manager.stop();
verify(triggerA).stop();
verify(triggerB).stop();
assertFalse(manager.isRunning());
}
@Test
void oneTriggerFailingToStartDoesNotBlockTheOthers() {
doThrow(new RuntimeException("boom")).when(triggerA).start();
PolicyTriggerManager manager = new PolicyTriggerManager(List.of(triggerA, triggerB));
manager.start();
verify(triggerB).start();
assertTrue(manager.isRunning());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java
+147
View File
@@ -0,0 +1,147 @@
package stirling.software.proprietary.policy.trigger;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.time.DayOfWeek;
import java.time.Instant;
import java.time.LocalTime;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.Schedule;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.store.PolicyStore;
import tools.jackson.databind.json.JsonMapper;
/**
* Tests for {@link ScheduleTrigger}'s due-firing logic via the package-visible {@code
* sweep(Instant)}. The trigger only decides when a policy is due; pulling sources and starting runs
* is the {@link PolicyRunner}'s job, so these assert it delegates to the runner. Schedules default
* to UTC, so explicit UTC instants make these deterministic.
*/
@ExtendWith(MockitoExtension.class)
class ScheduleTriggerTest {
@Mock private PolicyStore policyStore;
@Mock private PolicyRunner policyRunner;
private ScheduleTrigger trigger;
@BeforeEach
void setUp() {
trigger =
new ScheduleTrigger(
policyStore,
policyRunner,
JsonMapper.builder().build(),
new ApplicationProperties());
}
@Test
void firesOncePerScheduleWhenItComesDue() {
Policy policy = scheduled("p1", new Schedule.Every(1, Schedule.Unit.MINUTES));
when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy));
Instant t0 = Instant.parse("2026-06-05T10:00:30Z");
trigger.sweep(t0); // first sight: baseline, must not fire immediately
verify(policyRunner, never()).run(any());
trigger.sweep(t0.plusSeconds(120)); // the one-minute mark has passed
verify(policyRunner, times(1)).run(eq(policy));
}
@Test
void doesNotFireBeforeTheNextScheduledTime() {
Policy policy = scheduled("p1", new Schedule.Daily(LocalTime.of(3, 0))); // 03:00 UTC daily
when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy));
Instant t0 = Instant.parse("2026-06-05T10:00:00Z");
trigger.sweep(t0);
trigger.sweep(t0.plusSeconds(60)); // next 03:00 is far away
verify(policyRunner, never()).run(any());
}
@Test
void firesWeeklyOnAChosenDay() {
// 2026-06-05 is a Friday; the next Monday 09:00 is the soonest firing.
Policy policy =
scheduled("p1", new Schedule.Weekly(Set.of(DayOfWeek.MONDAY), LocalTime.of(9, 0)));
when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy));
Instant friday = Instant.parse("2026-06-05T10:00:00Z");
trigger.sweep(friday); // baseline
trigger.sweep(Instant.parse("2026-06-08T09:00:00Z")); // Monday 09:00
verify(policyRunner, times(1)).run(eq(policy));
}
@Test
void skipsPoliciesWithAnInvalidSchedule() {
Policy policy = scheduledWithRawOptions("p1", Map.of()); // no schedule
when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy));
trigger.sweep(Instant.parse("2026-06-05T10:00:00Z"));
verify(policyRunner, never()).run(any());
}
@Test
void validateRejectsMissingSchedule() {
assertThrows(
IllegalArgumentException.class,
() -> trigger.validate(scheduledWithRawOptions("p1", Map.of())));
}
@Test
void validateRejectsAnInvalidSchedule() {
Map<String, Object> options =
Map.of("schedule", Map.of("type", "every", "count", -5, "unit", "MINUTES"));
assertThrows(
IllegalArgumentException.class,
() -> trigger.validate(scheduledWithRawOptions("p1", options)));
}
@Test
void validateAcceptsAValidScheduleAndZone() {
Map<String, Object> options = new LinkedHashMap<>();
options.put("schedule", new Schedule.Daily(LocalTime.of(2, 0)));
options.put("zone", "Europe/London");
trigger.validate(scheduledWithRawOptions("p1", options));
}
private static Policy scheduled(String id, Schedule schedule) {
return scheduledWithRawOptions(id, Map.of("schedule", schedule));
}
private static Policy scheduledWithRawOptions(String id, Map<String, Object> options) {
return new Policy(
id,
"nightly",
"owner",
true,
new TriggerConfig("schedule", options),
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java
+29
View File
@@ -439,6 +439,35 @@ class AiWorkflowServiceTest {
verify(internalApiClient, never()).post(anyString(), any());
}
@Test
void convertMarkdownRunsDeterministicConversionAndReturnsMdFile() throws IOException {
MockMultipartFile input = pdf("multi-column-test_lorem.pdf", "pdf-bytes");
when(fileIdStrategy.idFor(any())).thenReturn("doc-1");
stubOrchestrator(
"""
{
"outcome":"convert_markdown",
"reason":"PDF to Markdown requested.",
"filesToIngest":[{"id":"doc-1","name":"multi-column-test_lorem.pdf"}]
}
""");
when(toolMetadataService.shouldUnpackZipResponse("/api/v1/convert/pdf/markdown"))
.thenReturn(false);
stubEndpoint(
"/api/v1/convert/pdf/markdown",
pdfResource("# Title", "multi-column-test_lorem.md"));
AtomicInteger ids = stubFileStorage();
AiWorkflowResponse result = service.orchestrate(requestFor(input, "convert to markdown"));
assertEquals(AiWorkflowOutcome.COMPLETED, result.getOutcome());
assertEquals(1, result.getResultFiles().size());
// Extension changes (pdf -> md), so the converter's response filename wins.
assertEquals("multi-column-test_lorem.md", result.getResultFiles().get(0).getFileName());
assertEquals(1, ids.get());
verify(internalApiClient, times(1)).post(eq("/api/v1/convert/pdf/markdown"), any());
}
@Test
void toolCallWithoutEndpointFallsBackToCannotContinue() throws IOException {
MockMultipartFile input = pdf("input.pdf", "bytes");
app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java
-66
View File
@@ -1,66 +0,0 @@
package stirling.software.proprietary.service;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import org.junit.jupiter.api.Test;
import stirling.software.proprietary.service.PdfContentExtractor.LayoutFragment;
import stirling.software.proprietary.service.PdfContentExtractor.LayoutLine;
import stirling.software.proprietary.service.PdfContentExtractor.LayoutPage;
import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutArtifact;
import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutFileResult;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.json.JsonMapper;
/**
* Contract test: verifies that {@link PageLayoutArtifact} serializes to the JSON field names that
* the Python engine expects in {@code engine/src/stirling/contracts/pdf_to_markdown.py}.
*
* <p>The companion Python test in {@code tests/test_pdf_to_markdown.py} deserializes the same JSON
* literal and asserts field values. If either side renames a field, one of these tests fails.
*/
class PageLayoutArtifactContractTest {
static final String CONTRACT_JSON =
"""
{"kind":"page_layout","files":[{"fileName":"test.pdf","pages":[{"pageNumber":1,"lines":[{"y":10.0,"fragments":[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}""";
@Test
void pageLayoutArtifact_serialisesToExpectedJson() throws Exception {
LayoutFragment fragment = new LayoutFragment("Hello", 1.0f, 2.0f, 30.0f, 12.0f, true);
LayoutLine line = new LayoutLine(10.0f, List.of(fragment));
LayoutPage page = new LayoutPage(1, List.of(line));
PageLayoutFileResult fileResult = new PageLayoutFileResult();
fileResult.setFileName("test.pdf");
fileResult.setPages(List.of(page));
PageLayoutArtifact artifact = new PageLayoutArtifact();
artifact.setFiles(List.of(fileResult));
JsonNode json = new JsonMapper().valueToTree(artifact);
assertEquals("page_layout", json.get("kind").asText());
JsonNode file = json.get("files").get(0);
assertEquals("test.pdf", file.get("fileName").asText());
JsonNode pg = file.get("pages").get(0);
assertEquals(1, pg.get("pageNumber").asInt());
JsonNode ln = pg.get("lines").get(0);
assertEquals(10.0, ln.get("y").asDouble(), 0.001);
JsonNode frag = ln.get("fragments").get(0);
assertEquals("Hello", frag.get("text").asText());
assertEquals(1.0, frag.get("x").asDouble(), 0.001);
assertEquals(2.0, frag.get("y").asDouble(), 0.001);
assertEquals(30.0, frag.get("width").asDouble(), 0.001);
assertEquals(12.0, frag.get("fontSize").asDouble(), 0.001);
assertTrue(frag.get("bold").asBoolean());
}
}