arsvendg/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/arsvendg/Stirling-PDF.git synced 2026-07-14 10:34:06 +02:00
Code Issues Actions 4 Packages Projects Releases 1 Wiki Activity

Merge branch 'main' into SaaS

Browse Source
This commit is contained in:
Anthony Stirling
2026-06-10 14:58:44 +01:00
committed by GitHub
parent be0db3fd8a 611468b972
commit 9a1804ce04
93 changed files with 6398 additions and 2421 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.taskfiles/engine.yml
+15 -2
View File
@@ -1,5 +1,12 @@
version: '3'
vars:
# Engine-specific names to avoid overriding the root Taskfile's FIND_FREE_PORT_*
# vars (Task merges included-file vars into the global scope).
# Paths are relative to the engine/ include dir.
ENGINE_FIND_FREE_PORT_SH: "bash ../scripts/find-free-port.sh"
ENGINE_FIND_FREE_PORT_PS: "powershell -NoProfile -File ../scripts/find-free-port.ps1"
tasks:
install:
desc: "Install engine dependencies"
@@ -29,7 +36,10 @@ tasks:
ignore_error: true
dir: src
vars:
PORT: '{{.PORT | default "5001"}}'
# When PORT is provided (e.g. from dev:all), use it directly.
# When running standalone, probe for a free port starting at 5001.
PORT:
sh: '{{if .PORT}}echo {{.PORT}}{{else if eq OS "windows"}}{{.ENGINE_FIND_FREE_PORT_PS}} 5001{{else}}{{.ENGINE_FIND_FREE_PORT_SH}} 5001{{end}}'
env:
PYTHONUNBUFFERED: "1"
cmds:
@@ -41,7 +51,10 @@ tasks:
ignore_error: true
dir: src
vars:
PORT: '{{.PORT | default "5001"}}'
# When PORT is provided (e.g. from dev:all), use it directly.
# When running standalone, probe for a free port starting at 5001.
PORT:
sh: '{{if .PORT}}echo {{.PORT}}{{else if eq OS "windows"}}{{.ENGINE_FIND_FREE_PORT_PS}} 5001{{else}}{{.ENGINE_FIND_FREE_PORT_SH}} 5001{{end}}'
env:
PYTHONUNBUFFERED: "1"
cmds:
app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java
-73
View File
@@ -1,73 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link
* LineAlignmentTableParser}. The first parser returning a result above {@link
* #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page.
*/
@Service
@Primary
@RequiredArgsConstructor
@Slf4j
public class CompositeTableParser implements TableParser {
/** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */
static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f;
private final TabulaTableParser tabulaParser;
private final LineAlignmentTableParser lineAlignmentParser;
@Override
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
// Step 1: Tabula lattice mode (ruled/bordered tables).
List<TableFragment> latticeResults = filterConfident(tabulaParser.parse(document, rawPage));
if (!latticeResults.isEmpty()) {
log.debug(
"Page {}: using Tabula lattice ({} table(s))",
rawPage.pageNumber(),
latticeResults.size());
return latticeResults;
}
// Step 2: Tabula stream mode (borderless/whitespace-delimited tables).
// parseStream is not on the TableParser interface — this intentionally couples to the
// concrete TabulaTableParser since stream mode is a Tabula-specific concept.
List<TableFragment> streamResults =
filterConfident(tabulaParser.parseStream(document, rawPage));
if (!streamResults.isEmpty()) {
log.debug(
"Page {}: using Tabula stream ({} table(s))",
rawPage.pageNumber(),
streamResults.size());
return streamResults;
}
// Step 3: Geometry-based line-alignment fallback.
List<TableFragment> lineResults = lineAlignmentParser.parse(document, rawPage);
if (!lineResults.isEmpty()) {
log.debug(
"Page {}: using LineAlignment ({} table(s))",
rawPage.pageNumber(),
lineResults.size());
return lineResults;
}
return List.of();
}
private List<TableFragment> filterConfident(List<TableFragment> tables) {
return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList();
}
}
app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java
-528
View File
@@ -1,528 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
import java.util.regex.Pattern;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
/**
* Fallback {@link TableParser} for borderless financial tables using text geometry.
*
* <p>Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge
* positions, groups vertically proximate anchor lines into table candidates, then scores each group
* on column consistency and anchor density (confidence ceiling 0.85).
*/
@Service
@Slf4j
public class LineAlignmentTableParser implements TableParser {
/** Width in points of each column position bucket. */
static final float COLUMN_BUCKET_PT = 5f;
/** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */
private static final int COLUMN_MATCH_BUCKETS = 2;
/** Maximum gap (as a multiple of modal line spacing) before splitting a group. */
private static final float MAX_GAP_FACTOR = 2.5f;
/** Minimum anchor rows (numeric-heavy) to form a valid table. */
static final int MIN_TABLE_ROWS = 3;
/** Minimum confirmed column positions to form a valid table. */
static final int MIN_COLUMNS = 2;
/**
* Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A
* rows).
*/
private static final double COLUMN_MIN_FREQUENCY = 0.40;
/**
* Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency,
* percent, nil dashes.
*/
private static final Pattern NUMERIC =
Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$");
/**
* Lines within this y-distance are merged into one row (restores rows split by LineBuilder's
* column-gap logic).
*/
static final float ROW_MERGE_TOLERANCE_PT = 2f;
// ── public API ───────────────────────────────────────────────────────────────────────────────
@Override
public List<TableFragment> parse(PDDocument document, RawPage rawPage) throws IOException {
List<RawLine> lines = rawPage.lines();
if (lines.size() < MIN_TABLE_ROWS) return List.of();
float modalSpacing = computeModalSpacing(lines);
List<TokenizedLine> tokenized =
mergeCoincidentLines(lines.stream().map(this::tokenize).toList());
List<TokenizedLine> anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList();
if (anchors.size() < MIN_TABLE_ROWS) return List.of();
List<Float> columnGrid = buildColumnGrid(anchors);
if (columnGrid.size() < MIN_COLUMNS) {
log.debug(
"Page {}: LineAlignment — fewer than {} confirmed columns, skipping",
rawPage.pageNumber(),
MIN_COLUMNS);
return List.of();
}
List<List<TokenizedLine>> groups = groupRows(tokenized, columnGrid, modalSpacing);
List<TableFragment> results = new ArrayList<>();
for (int i = 0; i < groups.size(); i++) {
buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i)
.ifPresent(results::add);
}
log.debug(
"Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)",
rawPage.pageNumber(),
results.size(),
anchors.size(),
columnGrid.size());
return results;
}
// ── coincident-line merging ──────────────────────────────────────────────────────────────────
/**
* Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves
* split by LineBuilder.
*/
List<TokenizedLine> mergeCoincidentLines(List<TokenizedLine> tokenized) {
if (tokenized.size() < 2) return tokenized;
List<TokenizedLine> result = new ArrayList<>();
int i = 0;
while (i < tokenized.size()) {
float baseY = tokenized.get(i).line().bounds().y();
int j = i + 1;
while (j < tokenized.size()
&& Math.abs(tokenized.get(j).line().bounds().y() - baseY)
<= ROW_MERGE_TOLERANCE_PT) {
j++;
}
if (j == i + 1) {
result.add(tokenized.get(i));
} else {
result.add(mergeGroup(tokenized.subList(i, j)));
}
i = j;
}
return result;
}
private TokenizedLine mergeGroup(List<TokenizedLine> group) {
List<TextFragment> mergedFragments =
group.stream()
.flatMap(tl -> tl.line().fragments().stream())
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
.toList();
Bounds mergedBounds =
group.stream()
.map(tl -> tl.line().bounds())
.reduce(Bounds::merge)
.orElse(group.get(0).line().bounds());
RawLine mergedLine =
new RawLine(
group.get(0).line().lineId(),
mergedFragments,
mergedBounds,
group.get(0).line().pageNumber());
return tokenize(mergedLine);
}
// ── tokenisation ─────────────────────────────────────────────────────────────────────────────
/**
* Splits fragments into word-level tokens; x-positions are estimated linearly within each
* fragment.
*/
TokenizedLine tokenize(RawLine line) {
List<LineToken> tokens = new ArrayList<>();
for (TextFragment frag : line.fragments()) {
tokens.addAll(tokensFromFragment(frag));
}
List<LineToken> numeric = tokens.stream().filter(LineToken::numeric).toList();
return new TokenizedLine(line, tokens, numeric);
}
private List<LineToken> tokensFromFragment(TextFragment frag) {
String raw = frag.text();
if (raw == null || raw.isBlank()) return List.of();
float fragX = frag.bounds().x();
float fragWidth = frag.bounds().width();
int rawLen = raw.length();
List<LineToken> result = new ArrayList<>();
int offset = 0;
for (String part : raw.split("\\s+")) {
if (part.isEmpty()) {
offset++;
continue;
}
int idx = raw.indexOf(part, offset);
if (idx < 0) idx = offset;
float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX;
float tokenRight =
rawLen > 0
? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth
: fragX + fragWidth;
result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches()));
offset = idx + part.length();
}
return result;
}
// ── column grid ──────────────────────────────────────────────────────────────────────────────
/**
* Returns confirmed column right-edge positions — those appearing on ≥ {@value
* #COLUMN_MIN_FREQUENCY} × N anchor lines.
*/
private List<Float> buildColumnGrid(List<TokenizedLine> anchors) {
// bucket → set of line indices that contributed a numeric token to that bucket
Map<Integer, List<Integer>> bucketLines = new HashMap<>();
for (int i = 0; i < anchors.size(); i++) {
for (LineToken t : anchors.get(i).numeric()) {
int bucket = bucket(t.right());
bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i);
}
}
int minHits =
Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY));
// Confirmed buckets → average right-edge for that bucket
TreeMap<Integer, Float> confirmed = new TreeMap<>();
for (Map.Entry<Integer, List<Integer>> entry : bucketLines.entrySet()) {
// Count distinct lines
long distinctLines = entry.getValue().stream().distinct().count();
if (distinctLines >= minHits) {
double avg =
entry.getValue().stream()
.distinct() // weight each line equally regardless of token count
.mapToDouble(
lineIdx ->
avgRightEdgeForBucket(
anchors, lineIdx, entry.getKey()))
.average()
.orElse(entry.getKey() * (double) COLUMN_BUCKET_PT);
confirmed.put(entry.getKey(), (float) avg);
}
}
return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right)
}
/**
* Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code
* targetBucket}, falling back to the bucket's nominal centre when no tokens match.
*/
private double avgRightEdgeForBucket(
List<TokenizedLine> anchors, int lineIdx, int targetBucket) {
return anchors.get(lineIdx).numeric().stream()
.filter(t -> bucket(t.right()) == targetBucket)
.mapToDouble(LineToken::right)
.average()
.orElse(targetBucket * (double) COLUMN_BUCKET_PT);
}
// ── grouping ─────────────────────────────────────────────────────────────────────────────────
/**
* Groups anchor lines into table candidates, including adjacent label rows; a gap &gt;
* MAX_GAP_FACTOR × modal spacing splits groups.
*/
private List<List<TokenizedLine>> groupRows(
List<TokenizedLine> all, List<Float> columnGrid, float modalSpacing) {
float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f;
List<List<TokenizedLine>> groups = new ArrayList<>();
List<TokenizedLine> current = new ArrayList<>();
for (int i = 0; i < all.size(); i++) {
TokenizedLine tl = all.get(i);
boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid);
if (current.isEmpty()) {
if (fits) current.add(tl);
continue;
}
float gap =
tl.line().bounds().y()
- current.get(current.size() - 1).line().bounds().bottom();
if (gap > maxGap) {
groups.add(current);
current = new ArrayList<>();
if (fits) current.add(tl);
continue;
}
if (fits) {
current.add(tl);
} else if (!tl.line().text().isBlank()) {
// Include non-anchor lines (labels) only if they have text and are within
// proximity.
current.add(tl);
}
}
if (!current.isEmpty()) groups.add(current);
return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList();
}
private boolean hasEnoughAnchorRows(List<TokenizedLine> group, List<Float> columnGrid) {
return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count()
>= MIN_TABLE_ROWS;
}
/** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */
private boolean matchesGrid(TokenizedLine tl, List<Float> columnGrid) {
if (tl.numeric().isEmpty()) return false;
long matches =
tl.numeric().stream()
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
.count();
return (double) matches / tl.numeric().size() >= 0.60;
}
private boolean hasInconsistentColumnMatch(TokenizedLine tl, List<Float> columnGrid) {
if (tl.numeric().isEmpty()) return false;
long hits =
tl.numeric().stream()
.filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0)
.count();
return (double) hits / tl.numeric().size() < 0.60;
}
// ── fragment assembly ────────────────────────────────────────────────────────────────────────
private Optional<TableFragment> buildFragment(
List<TokenizedLine> group, List<Float> columnGrid, int pageNumber, int tableIndex) {
long anchorCount =
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
if (anchorCount < MIN_TABLE_ROWS) return Optional.empty();
List<String> warnings = new ArrayList<>();
List<List<String>> rawRows = new ArrayList<>();
List<TableRow> rows = new ArrayList<>();
for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) {
TokenizedLine tl = group.get(rowIdx);
List<String> rawRow = buildRawRow(tl, columnGrid);
rawRows.add(Collections.unmodifiableList(rawRow));
rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid));
}
// Column count = 1 label column + confirmed numeric columns
int colCount = columnGrid.size() + 1;
Bounds bounds = computeGroupBounds(group);
float confidence = computeConfidence(group, columnGrid, warnings);
return Optional.of(
new TableFragment(
"tbl-la-p" + pageNumber + "-" + tableIndex,
pageNumber,
bounds,
List.of(),
Collections.unmodifiableList(rows),
Collections.unmodifiableList(rawRows),
colCount,
confidence,
Collections.unmodifiableList(warnings),
null));
}
/**
* Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values.
*/
private List<String> buildRawRow(TokenizedLine tl, List<Float> columnGrid) {
String[] cells = new String[columnGrid.size() + 1];
Arrays.fill(cells, "");
// Separate label tokens (those not landing in any confirmed column) from column tokens.
List<String> labelParts = new ArrayList<>();
for (LineToken token : tl.all()) {
int col = nearestColumnIndex(token.right(), columnGrid);
if (col >= 0 && token.numeric()) {
int cellIdx = col + 1;
cells[cellIdx] =
cells[cellIdx].isEmpty()
? token.text()
: cells[cellIdx] + " " + token.text();
} else {
labelParts.add(token.text());
}
}
cells[0] = String.join(" ", labelParts).trim();
return Arrays.asList(cells);
}
private TableRow buildTableRow(
int rowIdx, TokenizedLine tl, List<String> rawRow, List<Float> columnGrid) {
List<TableCell> cells = new ArrayList<>(rawRow.size());
// Label cell: use the line's full bounds as an approximation.
cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds()));
for (int col = 0; col < columnGrid.size(); col++) {
String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : "";
float right = columnGrid.get(col);
float left = col > 0 ? columnGrid.get(col - 1) : right - 50f;
Bounds cellBounds =
new Bounds(
left,
tl.line().bounds().y(),
right - left,
tl.line().bounds().height());
cells.add(TableCell.of(col + 1, text, cellBounds));
}
return new TableRow(rowIdx, Collections.unmodifiableList(cells));
}
// ── confidence scoring ───────────────────────────────────────────────────────────────────────
/**
* Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at
* 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if
* &gt;30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors.
*/
private float computeConfidence(
List<TokenizedLine> group, List<Float> columnGrid, List<String> warnings) {
float score = 0.70f;
long anchorCount =
group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count();
long totalRows = group.size();
// More columns
int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2);
score += extraCols * 0.05f;
// More anchor rows
if (anchorCount >= 5) score += 0.05f;
if (anchorCount >= 8) score += 0.05f;
// Inconsistent column matching
long inconsistent =
group.stream()
.filter(TokenizedLine::isAnchor)
.filter(tl -> hasInconsistentColumnMatch(tl, columnGrid))
.count();
if (inconsistent > anchorCount * 0.30) {
score -= 0.15f;
warnings.add(
"Column match inconsistent on "
+ inconsistent
+ "/"
+ anchorCount
+ " anchor rows");
}
// Label-heavy
long nonAnchor = totalRows - anchorCount;
if (nonAnchor > anchorCount) {
score -= 0.10f;
warnings.add(
"Non-anchor rows ("
+ nonAnchor
+ ") outnumber anchor rows ("
+ anchorCount
+ ")");
}
return Math.max(0f, Math.min(0.85f, score));
}
// ── utility ──────────────────────────────────────────────────────────────────────────────────
/**
* Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value
* #COLUMN_MATCH_BUCKETS} buckets.
*/
private int nearestColumnIndex(float rightEdge, List<Float> grid) {
int nearest = -1;
float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f;
for (int i = 0; i < grid.size(); i++) {
float dist = Math.abs(rightEdge - grid.get(i));
if (dist < minDist) {
minDist = dist;
nearest = i;
}
}
return nearest;
}
private Bounds computeGroupBounds(List<TokenizedLine> group) {
return group.stream()
.map(tl -> tl.line().bounds())
.reduce(Bounds::merge)
.orElse(new Bounds(0, 0, 0, 0));
}
/** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */
private float computeModalSpacing(List<RawLine> lines) {
if (lines.size() < 2) return 0f;
Map<Float, Long> freq = new HashMap<>();
for (int i = 1; i < lines.size(); i++) {
float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom();
if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum);
}
return freq.entrySet().stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse(0f);
}
private static int bucket(float x) {
return Math.round(x / COLUMN_BUCKET_PT);
}
// ── private data types ───────────────────────────────────────────────────────────────────────
/** A word-level token with an approximate right-edge x-position. */
record LineToken(String text, float x, float right, boolean numeric) {}
/** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */
record TokenizedLine(RawLine line, List<LineToken> all, List<LineToken> numeric) {
boolean isAnchor() {
return numeric.size() >= 2;
}
}
}
app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java
-139
View File
@@ -1,139 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import org.springframework.stereotype.Service;
import lombok.extern.slf4j.Slf4j;
/**
* Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity.
*
* <p>Fragments are on the same line when their baselines are within a font-size-derived tolerance.
* A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code
* max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text.
*/
@Service
@Slf4j
public class LineBuilder {
/** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */
private static final float BASELINE_TOLERANCE_FACTOR = 0.5f;
/** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */
private static final float MIN_BASELINE_TOLERANCE = 2f;
/**
* Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below
* two-column gutters.
*/
static final float COLUMN_GAP_RATIO = 0.10f;
/** Floor for the column-gap threshold so narrow pages don't over-split lines. */
static final float COLUMN_GAP_MIN_PT = 40f;
public List<RawLine> build(List<TextFragment> fragments, int pageNumber) {
if (fragments.isEmpty()) return List.of();
float effectiveWidth = inferEffectiveWidth(fragments);
float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT);
log.debug(
"LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt",
pageNumber,
effectiveWidth,
columnGapThreshold);
// Sort top-to-bottom first, then left-to-right within the same baseline band.
List<TextFragment> sorted =
fragments.stream()
.sorted(
Comparator.comparingDouble(TextFragment::baseline)
.thenComparingDouble(f -> f.bounds().x()))
.toList();
List<List<TextFragment>> groups = groupByBaseline(sorted, columnGapThreshold);
List<RawLine> lines = new ArrayList<>(groups.size());
for (int i = 0; i < groups.size(); i++) {
List<TextFragment> group =
groups.get(i).stream()
.sorted(Comparator.comparingDouble(f -> f.bounds().x()))
.toList();
Bounds lineBounds =
group.stream()
.map(TextFragment::bounds)
.reduce(Bounds::merge)
.orElse(new Bounds(0, 0, 0, 0));
lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber));
}
return lines;
}
private List<List<TextFragment>> groupByBaseline(
List<TextFragment> sorted, float columnGapThreshold) {
List<List<TextFragment>> groups = new ArrayList<>();
List<TextFragment> current = new ArrayList<>();
float currentBaseline = Float.NaN;
for (TextFragment fragment : sorted) {
if (current.isEmpty()) {
current.add(fragment);
currentBaseline = fragment.baseline();
continue;
}
float maxFontSize =
Math.max(
fragment.fontSize(),
(float)
current.stream()
.mapToDouble(TextFragment::fontSize)
.max()
.orElse(0));
float tolerance =
Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE);
boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance;
boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold);
if (sameBaseline && !columnGap) {
current.add(fragment);
// Anchor to the weighted mean baseline so long lines stay stable.
currentBaseline =
(currentBaseline * (current.size() - 1) + fragment.baseline())
/ current.size();
} else {
groups.add(current);
current = new ArrayList<>();
current.add(fragment);
currentBaseline = fragment.baseline();
}
}
if (!current.isEmpty()) groups.add(current);
return groups;
}
/**
* True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code
* threshold}.
*/
private static boolean hasColumnGap(
TextFragment next, List<TextFragment> group, float threshold) {
float lastRight = group.get(group.size() - 1).bounds().right();
return next.bounds().x() - lastRight > threshold;
}
/** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */
private static float inferEffectiveWidth(List<TextFragment> fragments) {
double maxRight =
fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0);
return (float) maxRight * 1.10f;
}
}
app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java
-79
View File
@@ -1,79 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} →
* {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the
* {@link PDDocument} lifecycle.
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class PdfIngester {
private final LineBuilder lineBuilder;
private final TableParser tableParser;
public List<ParsedPage> parse(PDDocument document) throws IOException {
return parse(document, document.getNumberOfPages());
}
public List<ParsedPage> parse(PDDocument document, int maxPages) throws IOException {
int pageCount = Math.min(document.getNumberOfPages(), maxPages);
List<ParsedPage> pages = new ArrayList<>(pageCount);
long fragmentsMs = 0;
long tablesMs = 0;
long t0 = System.currentTimeMillis();
for (int p = 1; p <= pageCount; p++) {
long ft = System.currentTimeMillis();
List<TextFragment> fragments = extractFragments(document, p);
fragmentsMs += System.currentTimeMillis() - ft;
PDPage page = document.getPage(p - 1);
PDRectangle mediaBox = page.getMediaBox();
List<RawLine> lines = lineBuilder.build(fragments, p);
RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines);
long tt = System.currentTimeMillis();
List<TableFragment> tables = tableParser.parse(document, rawPage);
tablesMs += System.currentTimeMillis() - tt;
log.debug(
"Page {}: {} fragments → {} lines, {} table(s)",
p,
fragments.size(),
lines.size(),
tables.size());
pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines));
}
log.info(
"[timing] parse pages={} total={}ms fragments={}ms tables={}ms",
pageCount,
System.currentTimeMillis() - t0,
fragmentsMs,
tablesMs);
return pages;
}
private List<TextFragment> extractFragments(PDDocument document, int pageNumber)
throws IOException {
WordExtractingStripper stripper = new WordExtractingStripper(pageNumber);
stripper.getText(document);
return stripper.getFragments();
}
}
app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java
-113
View File
@@ -1,113 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.font.PDFont;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.TextPosition;
/**
* Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata.
*
* <p>Overrides {@link #writeString} to split each content-stream string into word-level {@link
* TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in
* PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline.
*/
class WordExtractingStripper extends PDFTextStripper {
private final int targetPage;
private final List<TextFragment> fragments = new ArrayList<>();
private int fragmentIndex = 0;
WordExtractingStripper(int pageNumber) throws IOException {
this.targetPage = pageNumber;
setStartPage(pageNumber);
setEndPage(pageNumber);
setSortByPosition(true);
}
@Override
protected void startPage(PDPage page) throws IOException {
super.startPage(page);
fragments.clear();
fragmentIndex = 0;
}
@Override
protected void writeString(String text, List<TextPosition> textPositions) throws IOException {
if (text == null || text.isBlank()) return;
// Fast path: no whitespace → emit one fragment (most financial PDFs have each
// number as its own string operation, so this is the common case).
if (text.indexOf(' ') < 0) {
emitFragment(text, textPositions);
return;
}
// Per-word splitting requires 1:1 text-char to TextPosition correspondence.
// Fall back to one fragment when sizes differ (ligatures, encoding edge cases).
if (textPositions.size() != text.length()) {
emitFragment(text, textPositions);
return;
}
// Emit one TextFragment per whitespace-delimited word with accurate per-word bounds.
int start = 0;
for (int i = 0; i <= text.length(); i++) {
if (i == text.length() || text.charAt(i) == ' ') {
if (start < i) {
emitFragment(text.substring(start, i), textPositions.subList(start, i));
}
start = i + 1;
}
}
}
private void emitFragment(String text, List<TextPosition> positions) {
if (positions.isEmpty()) return;
float minX = Float.MAX_VALUE;
float minY = Float.MAX_VALUE;
float maxRight = -Float.MAX_VALUE;
float maxBaseline = -Float.MAX_VALUE;
TextPosition first = null;
for (TextPosition tp : positions) {
if (tp == null) continue;
if (first == null) first = tp;
float x = tp.getX();
// getY() is the baseline; top of character = getY() - getHeight().
float top = tp.getY() - tp.getHeight();
float right = x + tp.getWidth();
float baseline = tp.getY();
minX = Math.min(minX, x);
minY = Math.min(minY, top);
maxRight = Math.max(maxRight, right);
maxBaseline = Math.max(maxBaseline, baseline);
}
if (first == null) return;
PDFont font = first.getFont();
String fontName = font != null ? font.getName() : "";
boolean bold = fontName != null && fontName.toLowerCase().contains("bold");
// getHeight() gives the rendered glyph height, which is the most reliable visual size.
float fontSize = first.getHeight();
Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY);
String id = "tf-p" + targetPage + "-" + fragmentIndex++;
fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold));
}
List<TextFragment> getFragments() {
return Collections.unmodifiableList(fragments);
}
}
app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java
+40
View File
@@ -80,6 +80,7 @@ public class ApplicationProperties {
private Mcp mcp = new Mcp();
private InternalApi internalApi = new InternalApi();
private Cluster cluster = new Cluster();
private Policies policies = new Policies();
@Bean
public PropertySource<?> dynamicYamlPropertySource(ConfigurableEnvironment environment)
@@ -203,6 +204,45 @@ public class ApplicationProperties {
}
}
@Data
public static class Policies {
/**
* Absolute directories that policy folder input sources and output sinks may read from or
* write to. Empty (the default) disables folder access entirely, so a policy can never be
* pointed at an arbitrary server path. Stirling's own config directory is always
* off-limits, and folder access is always disabled in SaaS mode regardless of this list.
*/
private List<String> allowedFolderRoots = new java.util.ArrayList<>();
/** How often (seconds) the schedule trigger checks for policies whose schedule is due. */
private long scheduleSweepSeconds = 60;
/**
* How often (seconds) the folder-watch trigger reconciles its watch registrations and
* re-runs every folder-watch policy as a safety net for filesystem events that were missed
* (NFS, bind mounts, inotify-queue overflow).
*/
private long watchReconcileSeconds = 300;
/**
* How long (milliseconds) the folder-watch trigger keeps draining filesystem events after
* the first, so a burst from a single file copy coalesces into one run instead of many.
*/
private long watchQuietPeriodMs = 500;
/**
* SSE emitter timeout (milliseconds) for streamed runs; generous for long multi-step runs.
*/
private long streamTimeoutMs = 1800000;
/**
* How long (minutes) a finished run's in-memory state is retained before eviction,
* mirroring the job-result expiry so rich run state does not outlive the process. Active
* and paused runs are kept regardless of age.
*/
private int runExpiryMinutes = 30;
}
@Data
public static class PdfEditor {
private Cache cache = new Cache();
app/common/src/main/java/stirling/software/common/pdf/HeadingDetector.java
+191
View File
@@ -0,0 +1,191 @@
package stirling.software.common.pdf;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import stirling.software.jpdfium.text.PageText;
import stirling.software.jpdfium.text.TextChar;
import stirling.software.jpdfium.text.TextLine;
import stirling.software.jpdfium.text.TextWord;
final class HeadingDetector {
private HeadingDetector() {}
/** A heading is at most this many words; longer lines are treated as body text. */
private static final int MAX_HEADING_WORDS = 12;
/**
* Returns the Markdown heading prefix for a line. The decision combines several signals, never
* text matching, so a plain line that merely shares text with a heading is never promoted:
*
* <ul>
* <li><b>Size</b> — dominant glyph font size vs. the document body median (primary signal).
* Some PDFs encode visual size in the text matrix, so every glyph reports ~1.0; for those
* the line height is used as the proxy instead.
* <li><b>Brevity</b> — headings are short labels; a line over {@value #MAX_HEADING_WORDS}
* words is body text regardless of size.
* <li><b>Not a sentence</b> — a line ending in {@code . ! ?} reads as prose, not a heading.
* </ul>
*
* <p>Boldness is deliberately <em>not</em> a heading signal — a bold-but-not-larger line is
* emphasis, not a heading (see {@link #isBoldLabel}); promoting it to {@code #}/{@code ##} is
* the main source of false-positive headings.
*
* <ul>
* <li>size &gt; baseline * 1.4 → {@code "# "}
* <li>size &gt; baseline * 1.2 → {@code "## "}
* <li>otherwise → {@code ""}
* </ul>
*/
static String headingPrefix(TextLine line, float medianBodySize, float medianBodyHeight) {
String text = line.text().strip();
if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) {
return "";
}
float dominant = dominantFontSize(line);
float value;
float baseline;
if (dominant > 2f && medianBodySize > 2f) {
value = dominant;
baseline = medianBodySize;
} else {
value = line.height();
baseline = medianBodyHeight;
}
if (baseline <= 0f) {
return "";
}
float ratio = value / baseline;
if (ratio > 1.4f) {
return "# ";
}
if (ratio > 1.2f) {
return "## ";
}
return "";
}
/**
* True when a line should be emphasised as bold (rendered {@code **like this**}) rather than
* promoted to a heading: it is bold, short, and not a full sentence. Used for bold labels that
* are not large enough to be headings.
*/
static boolean isBoldLabel(TextLine line) {
String text = line.text().strip();
if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) {
return false;
}
return isBold(line);
}
private static int wordCount(String text) {
return text.split("\\s+").length;
}
private static boolean endsLikeSentence(String text) {
char last = text.charAt(text.length() - 1);
return last == '.' || last == '!' || last == '?';
}
/** True when the line's dominant font is bold, inferred from PostScript font names. */
private static boolean isBold(TextLine line) {
Map<String, Integer> counts = new HashMap<>();
for (TextWord word : line.words()) {
for (TextChar ch : word.chars()) {
if (ch.isWhitespace() || ch.isNewline()) {
continue;
}
String name = ch.fontName();
if (name != null && !name.isBlank()) {
counts.merge(name, 1, Integer::sum);
}
}
}
String dominantFont = "";
int max = -1;
for (Map.Entry<String, Integer> e : counts.entrySet()) {
if (e.getValue() > max) {
max = e.getValue();
dominantFont = e.getKey();
}
}
String lower = dominantFont.toLowerCase(java.util.Locale.ROOT);
return lower.contains("bold")
|| lower.contains("black")
|| lower.contains("heavy")
|| lower.contains("semibold");
}
/** Computes the median glyph font size across all pages. */
static float medianFontSize(List<PageText> allPages) {
List<Float> sizes = new ArrayList<>();
for (PageText page : allPages) {
for (TextChar ch : page.chars()) {
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
sizes.add(ch.fontSize());
}
}
}
return median(sizes, 12f);
}
/** Computes the median TextLine height across all pages. Used when font size is degenerate. */
static float medianLineHeight(List<PageText> allPages) {
List<Float> heights = new ArrayList<>();
for (PageText page : allPages) {
for (TextLine line : page.lines()) {
if (line.height() > 0f && !line.text().isBlank()) {
heights.add(line.height());
}
}
}
return median(heights, 12f);
}
private static float median(List<Float> values, float fallback) {
if (values.isEmpty()) {
return fallback;
}
Collections.sort(values);
int mid = values.size() / 2;
if (values.size() % 2 == 0) {
return (values.get(mid - 1) + values.get(mid)) / 2f;
}
return values.get(mid);
}
/**
* Returns the font size that appears most often (by character count) in the given line. Ties
* are broken in favour of the larger size.
*/
private static float dominantFontSize(TextLine line) {
Map<Float, Integer> counts = new HashMap<>();
for (TextWord word : line.words()) {
for (TextChar ch : word.chars()) {
if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) {
counts.merge(ch.fontSize(), 1, Integer::sum);
}
}
}
if (counts.isEmpty()) {
return 0f;
}
float dominant = 0f;
int maxCount = -1;
for (Map.Entry<Float, Integer> entry : counts.entrySet()) {
int count = entry.getValue();
float size = entry.getKey();
if (count > maxCount || (count == maxCount && size > dominant)) {
maxCount = count;
dominant = size;
}
}
return dominant;
}
}
app/common/src/main/java/stirling/software/common/pdf/PdfMarkdownConverter.java
+1043
View File
File diff suppressed because it is too large Load Diff
app/common/src/main/java/stirling/software/common/pdf/TableRenderer.java
+82
View File
@@ -0,0 +1,82 @@
package stirling.software.common.pdf;
import stirling.software.jpdfium.text.Table;
final class TableRenderer {
private TableRenderer() {}
/** Renders a Table as a GitHub-Flavoured Markdown table string. */
static String render(Table table) {
if (table.rowCount() == 0) {
return "";
}
String[][] grid = table.asGrid();
if (table.rowCount() < 2) {
// No separator row possible — return plain lines
StringBuilder sb = new StringBuilder();
for (int c = 0; c < grid[0].length; c++) {
if (c > 0) sb.append('\n');
sb.append(escape(grid[0][c].trim()));
}
return sb.toString();
}
int cols = grid[0].length;
// Compute column widths: max(3, max content length across all rows)
int[] widths = new int[cols];
for (int c = 0; c < cols; c++) {
widths[c] = 3;
}
for (String[] row : grid) {
for (int c = 0; c < cols; c++) {
String cell = c < row.length ? row[c].trim() : "";
widths[c] = Math.max(widths[c], escape(cell).length());
}
}
StringBuilder sb = new StringBuilder();
// Header row
sb.append(buildRow(grid[0], widths, cols));
sb.append('\n');
// Separator row
sb.append('|');
for (int c = 0; c < cols; c++) {
sb.append('-').append("-".repeat(widths[c])).append('-').append('|');
}
sb.append('\n');
// Data rows
for (int r = 1; r < grid.length; r++) {
sb.append(buildRow(grid[r], widths, cols));
if (r < grid.length - 1) {
sb.append('\n');
}
}
return sb.toString();
}
private static String buildRow(String[] row, int[] widths, int cols) {
StringBuilder sb = new StringBuilder();
sb.append('|');
for (int c = 0; c < cols; c++) {
String cell = c < row.length ? escape(row[c].trim()) : "";
sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|');
}
return sb.toString();
}
private static String escape(String cell) {
return cell.replace("|", "\\|");
}
private static String padRight(String s, int width) {
if (s.length() >= width) return s;
return s + " ".repeat(width - s.length());
}
}
app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java
-153
View File
@@ -1,153 +0,0 @@
package stirling.software.SPDF.pdf.parser;
import static org.assertj.core.api.Assertions.assertThat;
import static stirling.software.SPDF.pdf.parser.PdfModels.*;
import java.util.List;
import org.junit.jupiter.api.Test;
/**
* Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and
* column-grid construction.
*/
class LineAlignmentTableParserTest {
private final LineAlignmentTableParser parser = new LineAlignmentTableParser();
// ── mergeCoincidentLines ─────────────────────────────────────────────────────────────────────
@Test
void mergeCoincidentLines_singleLine_unchanged() {
var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue")));
assertThat(parser.mergeCoincidentLines(lines)).hasSize(1);
}
@Test
void mergeCoincidentLines_distinctYLines_unchanged() {
// Two lines at different y positions — must NOT be merged.
var lines =
List.of(
tokenized(rawLine(10f, 100f, "Revenue")),
tokenized(rawLine(10f, 115f, "Cost")));
assertThat(parser.mergeCoincidentLines(lines)).hasSize(2);
}
@Test
void mergeCoincidentLines_sameY_merged() {
// Simulates a financial-table row split by LineBuilder at the column gap:
// label fragment at x=72 → "Revenue"
// value fragment at x=350 → "1,234"
// Both have y=100. After merge they should form one TokenizedLine.
var label = rawLine(72f, 100f, "Revenue");
var value = rawLine(350f, 100f, "1,234");
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
assertThat(merged).hasSize(1);
// The merged line should contain tokens from both halves.
var tokens = merged.get(0).all();
assertThat(tokens.stream().map(t -> t.text()).toList())
.containsExactlyInAnyOrder("Revenue", "1,234");
}
@Test
void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() {
var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114
var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value)));
var bounds = merged.get(0).line().bounds();
assertThat(bounds.x()).isEqualTo(72f);
assertThat(bounds.right()).isEqualTo(380f);
}
@Test
void mergeCoincidentLines_withinTolerance_merged() {
// Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge.
var a = rawLine(10f, 100.0f, "Alpha");
var b = rawLine(200f, 101.5f, "99");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
assertThat(merged).hasSize(1);
}
@Test
void mergeCoincidentLines_beyondTolerance_notMerged() {
// Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge.
var a = rawLine(10f, 100.0f, "Alpha");
var b = rawLine(200f, 103.0f, "99");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b)));
assertThat(merged).hasSize(2);
}
@Test
void mergeCoincidentLines_threeCoincident_allMerged() {
// Three fragments at the same y (e.g. wide financial table with two value columns).
var a = rawLine(72f, 100f, "Revenue");
var b = rawLine(300f, 100f, "1,234");
var c = rawLine(400f, 100f, "5,678");
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
assertThat(merged).hasSize(1);
assertThat(merged.get(0).all()).hasSize(3);
}
@Test
void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() {
var a = rawLine(72f, 100f, "Revenue");
var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a
var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate
var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c)));
assertThat(merged).hasSize(2);
}
@Test
void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() {
// After merging, the combined line should be an anchor (≥2 numeric tokens).
// "Revenue" alone → not an anchor. "1,234 567" alone → anchor.
// Merged → anchor with at least 2 numerics.
var label = rawLine(72f, 100f, "Revenue");
var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f);
var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values)));
assertThat(merged).hasSize(1);
assertThat(merged.get(0).isAnchor()).isTrue();
}
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
/** Creates a RawLine with a single TextFragment of the given text at the given position. */
private static RawLine rawLine(float x, float y, String text) {
float width = text.length() * 6f; // ~6pt per char — rough but consistent
float height = 12f;
Bounds bounds = new Bounds(x, y, width, height);
TextFragment fragment =
new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false);
return new RawLine("ln-test", List.of(fragment), bounds, 1);
}
/**
* Creates a RawLine with two TextFragments representing two words separated by a small gap.
* Used to simulate a values-only line with multiple numeric tokens.
*/
private static RawLine rawLineMultiWord(
float x, float y, String word1, float w1, String word2, float w2) {
float height = 12f;
Bounds b1 = new Bounds(x, y, w1, height);
Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height);
TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false);
TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false);
Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height);
return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1);
}
/** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */
private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) {
return parser.tokenize(line);
}
}
app/common/src/test/java/stirling/software/common/pdf/PdfMarkdownConverterTest.java
+269
View File
@@ -0,0 +1,269 @@
package stirling.software.common.pdf;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import stirling.software.jpdfium.PdfDocument;
import stirling.software.jpdfium.text.TextLine;
import stirling.software.jpdfium.text.TextWord;
/**
* Accuracy and robustness tests for {@link PdfMarkdownConverter}, comparing conversion output
* against hand-authored golden Markdown for a set of owned/synthetic fixtures.
*
* <p>The {@link #gatedFixtures()} set is enforced in CI: those fixtures currently convert within
* the accuracy threshold and guard against regressions. Fixtures still being iterated on live in
* {@link #wipFixtures()} under a {@link Disabled} test so the goldens stay in the tree without
* breaking the build. Enable the WIP test locally to see per-fixture scores while working on the
* converter.
*/
class PdfMarkdownConverterTest {
/** Accuracy threshold: output must share at least this fraction of content with the golden. */
private static final double THRESHOLD = 0.95;
@TempDir Path tmp;
/** Fixtures that meet the accuracy threshold today and therefore gate CI. */
static Stream<Arguments> gatedFixtures() {
return Stream.of(
Arguments.of("multi-column-test_lorem.pdf", "multi-column-test_lorem.md"),
Arguments.of("bordered-table-test_widget.pdf", "bordered-table-test_widget.md"),
Arguments.of("many-tables-test_stress.pdf", "many-tables-test_stress.md"));
}
/** Fixtures still below the threshold; tracked here, enable locally to iterate. */
static Stream<Arguments> wipFixtures() {
return Stream.of(
Arguments.of(
"wrapped-cell-test_expense-report.pdf",
"wrapped-cell-test_expense-report.md"));
}
@ParameterizedTest(name = "{0}")
@MethodSource("gatedFixtures")
void convertMatchesGoldenMarkdown(String pdfName, String mdName) throws IOException {
assertConversionMatchesGolden(pdfName, mdName);
}
@Disabled("WIP fixtures below the accuracy threshold; enable locally to iterate")
@ParameterizedTest(name = "{0}")
@MethodSource("wipFixtures")
void convertMatchesGoldenMarkdownWip(String pdfName, String mdName) throws IOException {
assertConversionMatchesGolden(pdfName, mdName);
}
/**
* Degenerate/extreme geometry must not crash the converter. A crafted or malformed PDF can
* position text anywhere via a text matrix, so a row's words can span from near the origin to a
* coordinate beyond {@link Integer#MAX_VALUE}. The old column-detection code sized an {@code
* int[]} straight from {@code (int) Math.ceil(maxX) - lo}, which either allocated a multi-GB
* array (OutOfMemoryError) or overflowed to a negative length (NegativeArraySizeException) —
* taking down the request thread. Detection must instead bail out and return no columns.
*/
@Test
void columnDetectionSurvivesDegenerateGeometry() {
// x ≈ 2.5e9 is past Integer.MAX_VALUE; combined with a near-origin word it yields an
// implausible span that the pre-fix code turned into a fatal array allocation.
List<TextLine> rows = new ArrayList<>();
for (int r = 0; r < 4; r++) {
float y = 400f - r * 12f;
TextWord near = new TextWord(List.of(), 50f, y, 30f, 10f);
TextWord far = new TextWord(List.of(), 2_500_000_000f, y, 30f, 10f);
rows.add(new TextLine(List.of(near, far), 50f, y, 2_499_999_980f, 10f));
}
List<float[]> columns =
assertDoesNotThrow(() -> PdfMarkdownConverter.findColumnRangesFromLines(rows));
assertTrue(
columns.isEmpty(),
"implausible page span should disable column detection, not allocate from it");
}
private void assertConversionMatchesGolden(String pdfName, String mdName) throws IOException {
Path pdfPath = tmp.resolve(pdfName);
try (InputStream in =
getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + pdfName)) {
if (in == null) {
fail("Fixture not found on classpath: /pdf-ingestion-fixtures/" + pdfName);
}
Files.copy(in, pdfPath);
}
String actual;
try (PdfDocument doc = PdfDocument.open(pdfPath)) {
actual = new PdfMarkdownConverter().convert(doc);
}
String expected;
try (InputStream in = getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + mdName)) {
if (in == null) {
fail("Golden file not found on classpath: /pdf-ingestion-fixtures/" + mdName);
}
expected = new String(in.readAllBytes(), StandardCharsets.UTF_8);
}
// Image placeholders are not scored: their body text is a TODO ("ideally, add the info
// available about the image...") rather than real content, so comparing it would penalise
// output for matching a placeholder we intend to replace. Drop those lines from both sides.
expected = stripImagePlaceholders(expected);
actual = stripImagePlaceholders(actual);
double similarity = similarity(expected, actual);
if (similarity < THRESHOLD) {
fail(
String.format(
"Markdown output differs from golden file '%s' by %.1f%% (threshold %.0f%%):%n%s",
mdName,
(1.0 - similarity) * 100,
(1.0 - THRESHOLD) * 100,
unifiedDiff(expected, actual)));
}
}
/** Substring identifying an image-placeholder line, which is excluded from scoring. */
private static final String IMAGE_PLACEHOLDER_MARKER = "Image intentionally redacted";
/**
* Removes non-content lines from the comparison: image placeholders (TODO text we intend to
* replace) and GFM table separator rows (the {@code |---|---|} divider, whose exact dash count
* is cosmetic — any run of three or more dashes is valid Markdown).
*/
private static String stripImagePlaceholders(String md) {
StringBuilder sb = new StringBuilder();
for (String line : md.split("\n", -1)) {
if (line.contains(IMAGE_PLACEHOLDER_MARKER)
|| line.strip().startsWith("<image redacted")
|| isTableSeparatorRow(line)) {
continue;
}
if (sb.length() > 0) {
sb.append('\n');
}
sb.append(line);
}
return sb.toString();
}
/** True for a GFM table separator row, e.g. {@code |---|:--:|---|} (only |, -, :, space). */
private static boolean isTableSeparatorRow(String line) {
String t = line.strip();
if (!t.contains("-")) {
return false;
}
return t.chars().allMatch(c -> c == '|' || c == '-' || c == ':' || c == ' ');
}
/**
* Character-level similarity: proportion of expected characters that appear in the LCS. O(n*m)
* but golden files are small enough that this is fine.
*/
private static double similarity(String expected, String actual) {
if (expected.isEmpty() && actual.isEmpty()) return 1.0;
if (expected.isEmpty() || actual.isEmpty()) return 0.0;
// Strip all whitespace for a content-focused comparison
String e = expected.replaceAll("\\s+", " ").strip();
String a = actual.replaceAll("\\s+", " ").strip();
int lcs = lcsLength(e, a);
return (double) lcs / Math.max(e.length(), a.length());
}
private static int lcsLength(String a, String b) {
// Use two-row DP to keep memory reasonable
int m = a.length(), n = b.length();
int[] prev = new int[n + 1];
int[] curr = new int[n + 1];
for (int i = 1; i <= m; i++) {
for (int j = 1; j <= n; j++) {
if (a.charAt(i - 1) == b.charAt(j - 1)) {
curr[j] = prev[j - 1] + 1;
} else {
curr[j] = Math.max(curr[j - 1], prev[j]);
}
}
int[] tmp = prev;
prev = curr;
curr = tmp;
java.util.Arrays.fill(curr, 0);
}
return prev[n];
}
private static String unifiedDiff(String expected, String actual) {
String[] expectedLines = expected.split("\n", -1);
String[] actualLines = actual.split("\n", -1);
List<String> diff = new ArrayList<>();
diff.add("--- expected");
diff.add("+++ actual");
int maxLines = Math.max(expectedLines.length, actualLines.length);
int context = 3;
boolean inHunk = false;
int hunkStart = -1;
List<String> hunkLines = new ArrayList<>();
for (int i = 0; i < maxLines; i++) {
String exp = i < expectedLines.length ? expectedLines[i] : null;
String act = i < actualLines.length ? actualLines[i] : null;
boolean changed = exp == null || act == null || !exp.equals(act);
if (changed) {
if (!inHunk) {
inHunk = true;
hunkStart = Math.max(0, i - context);
// add context lines before change
for (int c = hunkStart; c < i; c++) {
hunkLines.add(" " + (c < expectedLines.length ? expectedLines[c] : ""));
}
}
if (exp != null) hunkLines.add("-" + exp);
if (act != null) hunkLines.add("+" + act);
} else {
if (inHunk) {
hunkLines.add(" " + exp);
// check if we're far enough past the last change to close the hunk
boolean moreChanges = false;
for (int j = i + 1; j < Math.min(i + context, maxLines); j++) {
String e2 = j < expectedLines.length ? expectedLines[j] : null;
String a2 = j < actualLines.length ? actualLines[j] : null;
if (e2 == null || a2 == null || !e2.equals(a2)) {
moreChanges = true;
break;
}
}
if (!moreChanges && (i - hunkStart) >= context) {
diff.add("@@ -" + (hunkStart + 1) + " @@");
diff.addAll(hunkLines);
hunkLines.clear();
inHunk = false;
}
}
}
}
if (inHunk && !hunkLines.isEmpty()) {
diff.add("@@ -" + (hunkStart + 1) + " @@");
diff.addAll(hunkLines);
}
return String.join("\n", diff);
}
}
app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.md
+10
View File
@@ -0,0 +1,10 @@
# Widget Inventory Report
This report lists current stock levels for each warehouse.
| Region | Units | Status |
|---|---|---|
| North | 1200 | OK |
| South | 950 | Low |
| East | 1430 | OK |
| West | 875 | Low |
app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.pdf
+74
View File
@@ -0,0 +1,74 @@
%PDF-1.4
%“Œ‹ž ReportLab Generated PDF document (opensource)
1 0 obj
<<
/F1 2 0 R /F2 3 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
5 0 obj
<<
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
>>
endobj
6 0 obj
<<
/Author (\(anonymous\)) /CreationDate (D:20260603003133+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603003133+01'00') /Producer (ReportLab PDF Library - \(opensource\))
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
>>
endobj
7 0 obj
<<
/Count 1 /Kids [ 4 0 R ] /Type /Pages
>>
endobj
8 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 500
>>
stream
Gas1[9i&Y\%))C:pc(6t3;pQ8%0T@tD+-Nf,MP8j(><jDruNO^VsZ'm=ue7^Ia=;l(`h-+jUU3'Knh#YjH-MEIFj6r!oqf+FjMh;(rCq>R4tMCHL4NI*\1+UNiI'V9NC%VeJKn/YI0J];XQt&X83?=ihrg<*Mcn1n!1nWcDaQPe\P"9gnJuHl(jf]JQgZ[,&^uobI4QF',k"*^S)3c;)GMWC(T'=",ErnS#U=YCUN0&q4+*KmK1Zd*NI\GQDiZUG7;PTja8lulb"\PWWO#WcfI[ZB:6s*3g$be%?JH<b[c(?4J1!2SLDmbmkh8.c&CU)B?Xu,cp$<hP@=fLc>(n`oaEJ[XE'%QW=HE04M<,;ERm[MS=uYF=nN3jG'f@#?O48Ia,6Y-3m&tTWVq1?DeiBkp.Ug*;lVZX`Z=P.eklHhNV;!R_?QOuoeJ<0%7idG7GM8boU$^>N.N,2^;25]0Z8M<<]XMCct>noC'Qfb?`*[Mo+,F9#>t~>endstream
endobj
xref
0 9
0000000000 65535 f
0000000061 00000 n
0000000102 00000 n
0000000209 00000 n
0000000321 00000 n
0000000514 00000 n
0000000582 00000 n
0000000862 00000 n
0000000921 00000 n
trailer
<<
/ID
[<ee376a59c53e3f2af87024f65fd4222c><ee376a59c53e3f2af87024f65fd4222c>]
% ReportLab generated PDF document -- digest (opensource)
/Info 6 0 R
/Root 5 0 R
/Size 9
>>
startxref
1511
%%EOF
app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.md
+222
View File
@@ -0,0 +1,222 @@
Intro paragraph for section 1.
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
# Section 2 Heading
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
| golf | 301 | india |
## Section 3 Heading
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
| golf | 301 | india | 23 |
| juliet | 401 | lima | 33 |
Intro paragraph for section 4.
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
| golf | 301 | india | 23 | kilo |
| juliet | 401 | lima | 33 | november |
| mike | 501 | oscar | 43 | alpha |
# Section 5 Heading
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
| golf | 301 |
| juliet | 401 |
| mike | 501 |
| papa | 601 |
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
# Section 7 Heading
Intro paragraph for section 7.
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
| golf | 301 | india | 23 |
## Section 8 Heading
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
| golf | 301 | india | 23 | kilo |
| juliet | 401 | lima | 33 | november |
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
| golf | 301 |
| juliet | 401 |
| mike | 501 |
# Section 10 Heading
Intro paragraph for section 10.
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
| golf | 301 | india |
| juliet | 401 | lima |
| mike | 501 | oscar |
| papa | 601 | bravo |
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
# Section 12 Heading
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
| golf | 301 | india | 23 | kilo |
## Section 13 Heading
Intro paragraph for section 13.
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
| golf | 301 |
| juliet | 401 |
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
| golf | 301 | india |
| juliet | 401 | lima |
| mike | 501 | oscar |
# Section 15 Heading
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
| golf | 301 | india | 23 |
| juliet | 401 | lima | 33 |
| mike | 501 | oscar | 43 |
| papa | 601 | bravo | 53 |
Intro paragraph for section 16.
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
# Section 17 Heading
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
| golf | 301 |
## Section 18 Heading
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
| golf | 301 | india |
| juliet | 401 | lima |
Intro paragraph for section 19.
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
| golf | 301 | india | 23 |
| juliet | 401 | lima | 33 |
| mike | 501 | oscar | 43 |
# Section 20 Heading
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
| golf | 301 | india | 23 | kilo |
| juliet | 401 | lima | 33 | november |
| mike | 501 | oscar | 43 | alpha |
| papa | 601 | bravo | 53 | delta |
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
# Section 22 Heading
Intro paragraph for section 22.
| Name | Qty | Price |
|---|---|---|
| alpha | 101 | charlie |
| delta | 201 | foxtrot |
| golf | 301 | india |
## Section 23 Heading
| Name | Qty | Price | Region |
|---|---|---|---|
| alpha | 101 | charlie | 3 |
| delta | 201 | foxtrot | 13 |
| golf | 301 | india | 23 |
| juliet | 401 | lima | 33 |
| Name | Qty | Price | Region | Status |
|---|---|---|---|---|
| alpha | 101 | charlie | 3 | echo |
| delta | 201 | foxtrot | 13 | hotel |
| golf | 301 | india | 23 | kilo |
| juliet | 401 | lima | 33 | november |
| mike | 501 | oscar | 43 | alpha |
# Section 25 Heading
Intro paragraph for section 25.
| Name | Qty |
|---|---|
| alpha | 101 |
| delta | 201 |
| golf | 301 |
| juliet | 401 |
| mike | 501 |
| papa | 601 |
app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.pdf
+169
View File
@@ -0,0 +1,169 @@
%PDF-1.4
%“Œ‹ž ReportLab Generated PDF document (opensource)
1 0 obj
<<
/F1 2 0 R /F2 3 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/Contents 13 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
5 0 obj
<<
/Contents 14 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
6 0 obj
<<
/Contents 15 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
7 0 obj
<<
/Contents 16 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
8 0 obj
<<
/Contents 17 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
9 0 obj
<<
/Contents 18 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
10 0 obj
<<
/PageMode /UseNone /Pages 12 0 R /Type /Catalog
>>
endobj
11 0 obj
<<
/Author (\(anonymous\)) /CreationDate (D:20260603005358+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603005358+01'00') /Producer (ReportLab PDF Library - \(opensource\))
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
>>
endobj
12 0 obj
<<
/Count 6 /Kids [ 4 0 R 5 0 R 6 0 R 7 0 R 8 0 R 9 0 R ] /Type /Pages
>>
endobj
13 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1007
>>
stream
GauHKgN)$k&:O:Sn<io8&IUqd[51N1e*/]GIR%m`>DZmeMUK]*(BdhV"2[X,j0;*\%_E*0HXJRc`<Z.PlPcH-)*5ON#Qr4-#h+U(ll5egXa[FW7^<l-+Mko1pU57W'%??d'1Q-,]\qWS80;JOp0HFGa1T[+D>q%4S#/VM`O[Rh4i5T.Phi,$]aT$05!q:PBuCQU/.p2]@1koN*Tb*K9k5G\ht+Dr\K=+8\NZ"alMaOEo**@OK:8-.O1X3-?Gg`@m3%,ti3'">T-&c=M&Wuu?cDbGDp.gO<KniopM)A\M"ajn9N]6p@tefHlX67:iW3SE/[O\LV1TKe.i9es,%kWY+1-s,ft.u27]:^>F0!r6&&CLC$tM?fIR"M37["/*k9@YkpKKSS@Np"1/4#R]`I^(g*1Sc,9L3Qt6N(T@F`A>oBGgL-!r#Y4)R\G`D,iVtFeJUE9u4iuUQ?D%C&SB4kkp.>D5tii>nDKJ"Y07jANhOb$R(_$=U7Stjs)-/KZ(6IBm`6<Z:IO1/g<Q6n^@fdK*]SO#g&Wj4B)g1,#O?30",57V_ONl0p`%uQN]'EAZV$475Z"'DApH2)Sg6UQ75XC?4^'<&6&d?YC`OF9F3eE;WIMt@a?'q)c\3oWQ_BD]d`4=O\,'aSUFV4HsE7)O:lPOi6Ht1,[dTm6JqQM6es[?9G4VC(YieF*)uM6[SZT#[TN2,BIir^1;/Ku"E1laH[\"M+F'8=n0!;?_#hHd!hkpV0u\Ij\4VC-km.[F&JhGmbc`6@I1=mf>u<3;i.Bh4+MFJ"H:.XWUQX6%LU(sg4Tt$_":5`p.2,gZkpUfdg,Hd(qR1)9\ltF2^8b5,,XbY%VfhD52O7A.c.u]dhTc5t%0<0L5E38`Bq+i;"%J7kcc#@i)@okdN-qiaA"33DdPgPrUs7;j%+47_cnGN@&ug9/dqGOAHA4f.N*&guHspf?E;GIc-Wt>:<(m1AmcS_Zc2VlEI'_S_>@#!MF^m1$$<mB1`gt\X~>endstream
endobj
14 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 987
>>
stream
GatU3gN&c;&:O:SkV;H,eI!J\f?X"LSPIZ'"4Y.F#oAAY?N.Y?8Iu:s)eV;,aNG1Lea>G$!MSc\rU6m7jF'AOI09V1,^TSTi$A+f4sZVQ%2^>uj$40\AI>WbZo9q!NL^CS9P3E9X.:XEMb$a/X"qH9hN6fY,]D!OEOVT,722%RJnVqK4f'[4d3(/hbJWJRs,>>28jk5A:nCl/%FlC#LntsAC5cE$q`QO_@Pk%Lp>7V23")NNrFkcXfuoI=SJJ-`g)_+K\@,SQ@8ORGg(_(B.[u[WXD%Q8@I";9d(]M62K_)Q+-<\kHY5,)o99%BB2bcQVkd:+MZVH=$Z`S@BhH]-XfPANk@qBN.:Jc'?.Kn\p7(aPI<Cb@g:XRZ9Km+RD-e6S\N*kQMA?P5>QOI(du7U]WDrNNPW%d"8P_j^Y8d%G<t1\CJ_5-m+>,'JJLXrV.UD!ah19&9b#*fgR2o730<9)QX)X0"EK6u;mBJs3/fl\d&K$B3bXI5R>F*E,^\%+b?0o8s$o.CuX4uDAT_?G0(p4N3La,8qfi'j?e`e88KrZ8*NDgc:62'icCb],4B]Z"SO&e/BRC*C$2PmbSKAjc\$FLD8?&qHVH3G4p834/77n[%()-p/JDRBapcO2b\,3dW%#U9u!ilSkd%2'rr&g[u".1O]W%0J<@#(ptUD;%0:^.^ls:.#.V6NgR[`2\RUW!k$-*GZIf3DO<K8np=hCA>BiB_Mu"=LVGlO\1Z]0pF(@UaiVQ=dh?h5kfQ_gj>$jHW$8TCt:C.jVU98ne.XoiMtoV;q8XhnlOE;3a]=dGJ8KL&+Khj"&P4Q=P-:nHsDprY"B4+#A3dF,<W=NC>;b&gf(sT\Ts*iH)f!KhfHV;\mMSHej.9.XrBG41_<:b)MG;-Y(:&nCKS0F]r&CGl>EfMc0td\0GHScl3Lr;?OUoo6619Qdr)S8S);]7L5rmWW&(5t4Dil"USDhhY/Y=!=!X:85**$:~>endstream
endobj
15 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 976
>>
stream
GatU3gN&c;'R]XVkgAc"70atdYFXp#3h7VV#7/=-%N$"G?N.YO&iX#cVsS_l_5i8)eum:1-)(7`mXJ7LnhqY0hGZal,We=q^e"$M]Lu;_=4AoIN$QOjOF.@i*Sg!_rLL=-'?kk;m;M&R`$D>'HCL7<qMHOgP.\NgCm:Zf5tLAgarh6YCKLONS%?!`3T6F7YKNJ(dBFh(U1"e7D6YdC:71/`?cjR"QrKtsGg*:'=Ck>]$A,-S`H1I/1T]4\B3=gQ9:I4j9f]+af*Z7*GH5H+;j73Z-T#DAcgpO<SFuA['l7@MF*6c^<pB]o84rG$&I@Cs3`KIG,N/bG0f<k/M@gUKb"%\W-lZ#AU+i=gUKkm46K-b>W(/'b?XF9+X+'-Dqi"+1U7?6=ZRJNBSJ6ojDTa+]o4RF^26C+i0s(9G2o;$@RNbs\(f75],L(SVpI0M!'7JVhZ"$t/%kb1+)F\p54/@jpVJF2+)7^F=gVcp8(h*m%F#'p9c%9Ep9?b7g^F5>(>I3t^d_"f'aue%^E\X4kN:g\P[;sC=g3h,nD/n9*`94uoM?sBCL<kr$AOqh:,Y&c90,npSDXN*bEfY(k^0LZ_\I@Y+L2nn64-qsuX5D'=o'B2"8rCsRJ?h/-4Ur6khb;ph%X"8-mXTUU5-!Uh$iVuRi1.=>4;XilfMPT4]&FC.U%DI(c6uf3*XPJP2MRdq@Ag3uYF().nqLFAE2Nih'(^p\5FnSNa>n+cI?U65A61_N1<9ZXH4Xr)Fgq(6md3H9[MQ3Q^%aq>E?XM:O:RKGjc+QV2MB!.I.17QpSt]X[C*6ap.`#'o;8F(ebZ.VMM6k`BkZB\I,2MNoX]J"k]Qd";,@($aWX&29q"6k;>Let)@]QHm:i/lf[DB_&U@ROq-0MLQp;@j+]?$"E)brXVun52AVQo$"[a7@!?!LT*>$&:5X_.R;9)'$l-S.>WjLQIVTo.JU%(H\,-*pl_kP9~>endstream
endobj
16 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1024
>>
stream
GauHKflEQ9'Rf^WgrHc4<#\/Qm7c-rFIIq+TFSD%Z#L'6o(Nl&^_!k0ds=.%-s&pt#i0QBKE91*<^/MJJCcfoHA_e;aL?\&_BAj]Dt;HW$6(8Ql\%O7-5%uT8<qrc9!'@hZ!'`C3a&iu\ZDZi+5lF*E@>0ZIL#\l)(+[ABOTNKmq!2F,S:.F93Z6QIu[nf\Q[qCE`Y-=.8k$67iCR!7=El+50a@f&b^6'Yg'mqlcRQ-5j:%[K@'5!A_m0L)&VlW1T50^X)"3Ma3,U1P8Di$>uPZ)Zj_igBc0lm]WE0#e>*M53(Yh6\9jHYh6B+3bM\b[Q9j'O_9:!:W0<K\bX;'ZV[%iHrL=:rH.Wh5O-II8-Rh3..Sd^\dNItUUd<nli^`$=A@?\WW4/7Wnp>$Ijm,#(bKn_D?UYMGJ7LLqC0\[$l<8U*O5^GnN;!N6YB'?d[d8DAPt^>+E:r0<2$'VL/TtuNg;C@5iM#%KdS-GU->0]`/20eLXVW#qnjsU.P:Xj&=enVE[mpqDCn2!qDRuQhI/#cZ-#m6`U9'SI4"9\<WSWoVPaf@]<NRl?FrTPOd:V_=>"r:4p.\W>IK_$#C,EkU#uO1g6*HoUK':XiJjtQfI'Sdc)B3J1eqgGJ+l?"$<J+^?VHgiq^90uD"@6ar+/_WhKG+8t!Slo>//&C$^f'TA!2:K!0pd;^Zif)c[:f!C/8hNKc7+'+1WbOFKr9T7c_G$s*O634O2Zs:"iN>j,pWU+:kEQ>5YbF=$27*59iJE<glrr=EAMssR&N"lVT1:*a,OcI=8\"6/>A//7(KsL\pUG.G[+JB-r8t;Sih+(W.A'8Q*LZK*I9WjCLjEt>lT%2RFJZg0Ps:5c,HdK#tSNI:#Rp!]O$Ic=T%K0[._E_Fcjn>OcM,iSAf9iF7,,+Qe:@o'Y4o_:54t*l+TAg<i_F?ke9:\G<n<b*t@"_`_;08ft@[9;qtKC#H0Z6cJ+d)MdidY-OFV1K"=!=K=^?hK-Q.s+?\[I98%IXRiAI.!:AVTf`~>endstream
endobj
17 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1075
>>
stream
GauHKgN&c;&:O:SkV;H,6&Q<FCM0)BVUK5trr=4p-3<-<P.#jI+b[BoQ`]$,GaorC,Bfp>Z[5)3-pL]0bpl':D91fR,o"F49.1/bfmFt32ljt6eO[j7!K]\nZ:^QQASu[l@3no(6*HL'@3I!Qfi7%,.N\.>A91O)D\n5*o@JWSOI@ng5p/"_I`g*Y$j0+5PWaso'25ltcglFd,QIaO'dPO0Yr\h5/,=m\PiP:P+Gg0&N;_5iLN$]BD.#VL+h,tSe\fL&,;+2fg&%\pAJiV#eVhq@ro%%/`[(&8n29YiYd>bDF^Bg[AVf/n>[FE(au"ZAe&>%Q[7/n3-QhsPXuPbp#<$d"UR8u-nBnr!Q$Wjd1;9WN/,KG8])Ca5rrDrI'Ue'*I0C%pn+f`Hd"4sB_&2*)S$pX=F/6"DC"TU`bMtd/m7u]?]=J5L[SNBuE:6Ri,NmW7.*qUp)97Tt+a84b\os`Ti8/uRD<mg2;n&q)23L#\f:[QJk+sYYhIC_O\TNS/;3<qfN@GE^L#jl^i&)"7_/%Cb;-M;;ooT&`R;C&\WTZ4"\YIcTZDa;0CKTX+?@ra\94$H:Q<l)r*NHFJWjMr7[tCBb^PX,G]H!fs0g3,5^'AZ%<4Zqp(JCd+;^lkoDJ%q(Bb84V=:PCgOtM#Rr-=oHW&DmO?/!mYrV;g]MR?14cGuH_H*]73!34(()[A.)c<V3:@I>lhrl*Xrn(a0\,%4%*k^rL\k3%6ako`&A3MoN^CXV77D!Qg<S_OEMIh5(T*nFt'&4]_38'Q<s*V7!;A%[email protected]/(oCu;?^7kb[#R/-P*GF@KHDm6E*dLMg/4Q$W(<Ta[fVpGB5td>%=j9Ge+0B@dZDHELb2fDU(,iiXXRl^*U,U#Fld!R72UIpK<KU`cn&R/_eYS//1PE/$d7p:t-BP62fL<3/QQ4%=NoD?Tlf:A,GpJc<]*uYaL6fSAqm"-E.#%h]idd9$6\DZKKp8Vd[>GWu#-DXi653a?U$fqCs18gt(5/#U%!u,gkMK;>OT_/='S:kn,iD@u]<8-rlTRu'+\7L$U7-#!5-5\WQn(n:q@-p24u!~>endstream
endobj
18 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 590
>>
stream
GasbXgMWc?&;KY!MRgt!"n`^Bf\9$c*`Q-Sb6tfd8P(q-dT1eng=QV-!OG*\kiWo2AGBdKLqI^%h,XNB)-gDk+GFVBa?g*a%:!J\97Vc8-jH8>8P>0S@Wr)o*"Hu<6qPp`EF:M3'l4@S\c2]`&[J%dO@5p$<(([H6:SlB;FS8L`&pO%6Y\V"/O7F]E%L@d.%>L@"4bK_XXs&";n_.W^Nr847HH,X@$.pC"4bYC?9RH,dN?h8<BTH9l89:QMq:es\ePUpFRc4?hV'h3b4`sE_b4r->a+8/!?Q=D\F41Nc@'B`';4XgMeh!;U23C+>AbMQD-o--lJ/":m#(Yt,X5(`1KL,GTMBLlr6=-1mi#k.-Ou\\T4$Pun2dEU(\?$&;1@T4dm^t!KuOD-U@N_AMr"&uVpsGm,+8I7B*f!%9.o4cC1<Nr^$d5!dH-#UIr,n6Z;JUFYWCTVA\,or>a[CZ12(hd>1*0bU`k2-MXo1[Gor#kmXGIM'#R49X#NSAOpdf'0ilUH4:M(^Snc3;m/+><UlJ1jSN&ZuoQ68F8`c/Y-e0[!aljuioC5fR7-VUAX`%g<0P5J7"CFlQJT1~>endstream
endobj
xref
0 19
0000000000 65535 f
0000000061 00000 n
0000000102 00000 n
0000000209 00000 n
0000000321 00000 n
0000000516 00000 n
0000000711 00000 n
0000000906 00000 n
0000001101 00000 n
0000001296 00000 n
0000001491 00000 n
0000001561 00000 n
0000001842 00000 n
0000001932 00000 n
0000003031 00000 n
0000004109 00000 n
0000005176 00000 n
0000006292 00000 n
0000007459 00000 n
trailer
<<
/ID
[<ba877f458edd6b06b69a7e843c67586d><ba877f458edd6b06b69a7e843c67586d>]
% ReportLab generated PDF document -- digest (opensource)
/Info 11 0 R
/Root 10 0 R
/Size 19
>>
startxref
8140
%%EOF
app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.md
+25
View File
@@ -0,0 +1,25 @@
# Lorem Ipsum in Two Columns
## 1. Origins
Lorem ipsum dolor sit amet consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
## 2. Structure
Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
## 3. Usage
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
## 4. Variations
Excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum.
## 5. Typography
Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam various turpis et commodo pharetra est.
## 6. Conclusion
Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum.
app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.pdf
+74
View File
@@ -0,0 +1,74 @@
%PDF-1.4
%“Œ‹ž ReportLab Generated PDF document (opensource)
1 0 obj
<<
/F1 2 0 R /F2 3 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
>>
endobj
4 0 obj
<<
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
5 0 obj
<<
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
>>
endobj
6 0 obj
<<
/Author (\(anonymous\)) /CreationDate (D:20260603021636+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603021636+01'00') /Producer (ReportLab PDF Library - \(opensource\))
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
>>
endobj
7 0 obj
<<
/Count 1 /Kids [ 4 0 R ] /Type /Pages
>>
endobj
8 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 861
>>
stream
Gat=i>Ak00&;B$5/*8Q/Z)fmNp(^2^OK'MC[dT6#Oq.&b^4c4;1No6>+D%TC.n+fice+l9R0[L%'(o<DJBpB+r$jsRcu23$RjI].E7rS*BB"3t\qRKaS-SYt[_<-]),Cp@'$i;6dQuJ#Q*+lGY?X<phgEWPa?op]_1g@kf_>[email protected]:T=#%ZIIl18hB7Zf<piVi-V+;]ct^jS<=Z4l:IOd[,o"%q.hK[[k]r1!^oX'%44os7T9[m%YrbZ3KU/;aR0$cV$;D._".OT8)jeEBKU'l<A`'aD&R'L"Bo]Q)R+=cKOEld8s#M0ZYZN4[lRm1*F6^d9-Gfl5>I(hVU+?`]UK%;aabAH:q>9NUGQq^.=u])Wt-ETI))C'a[FE@elj!RSrf2Q'F>URAO.C,!DneTPqrj#4e2kb9%1"4qfZ)"#&^0j9nHQ?9nF!j7mVPP5\*Uq'_jMVS]9%`kQB\8*AF_bpr/hGj;HCUOSQU-%5:6S79Ud\b!*tPbr_'pCr$Ea#(FYP31NFhSX.-("1M:$cgH#hX8L(2]R3Q>'BYHCS%pI!;=WdJp,'ii[`QPZ_9mcd\baZ2U(_;c\-p,8EoIEpQ*lstL>]LE;C#\dLnT2R:)BM-fTc['3_He[U,k'!Bo".uERd>SkhRj^J+koSIrZ_dEf_5L'/1h.`+DTK(R:P,WH)h5\se=SZ"L/5b8b..,e/E\o+4YQ+*im^C>AERG/TieEK\)#>U@HXnJ,H0A9-MqhhkDp8%.6Lr,OrK*lih;B<-opZ8%EU?,$r^jmCDAQ`-0/-8/`[p]7Fm%0f:E&S*FV)DX2>#q\bRqA=^_`43#8EA$u%8r6F5`rc9>K@q>E4q^*~>endstream
endobj
xref
0 9
0000000000 65535 f
0000000061 00000 n
0000000102 00000 n
0000000209 00000 n
0000000321 00000 n
0000000514 00000 n
0000000582 00000 n
0000000862 00000 n
0000000921 00000 n
trailer
<<
/ID
[<21a9fbd0a0991a91b6e6e2db0856056e><21a9fbd0a0991a91b6e6e2db0856056e>]
% ReportLab generated PDF document -- digest (opensource)
/Info 6 0 R
/Root 5 0 R
/Size 9
>>
startxref
1872
%%EOF
app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.md
+62
View File
@@ -0,0 +1,62 @@
# Employee Expense Report
Reimbursement Request
EMP-1047
**Report Header**
| Employee Name | Michael Tran |
|---|---|
| Employee ID | EMP-1047 |
| Department | Client Services |
| Report Date | January 20th, 2026 |
| Reporting Period | January 5th–16th, 2026 |
| Manager Approver | Laura Simmons |
**Company Information**
| Company | Summit Consulting Partners |
|---|---|
| Company Address | 88 Riverside Plaza, Suite 1400, New York, NY 10069 |
| Accounting Department Email | expenses@example.com |
**Trip Purpose**
The trip was undertaken for client onsite meetings with Atlantic Energy Solutions in Boston, MA.
**Expense Details**
| Description | Amount | Date | Category |
|---|---|---|---|
| Flight (NYC to Boston roundtrip) | $325.40 | January 5th, 2026 | Airline ticket |
| Hotel (3 nights at Harborview Hotel) | $822.75 | January 5th–8th, 2026 | Lodging |
| Taxi from airport to hotel | $48.00 | January 5th, 2026 | Ground transportation |
| Client dinner (3 attendees) | $186.20 | January 6th, 2026 | Meals |
| Parking at JFK Airport | $72.00 | January 5th–8th, 2026 | Parking |
| Breakfast (per diem not used) | $18.50 | January 7th, 2026 | Meals |
| Description | Amount | Date | Category |
|---|---|---|---|
| Uber to client office | $22.10 | January 7th, 2026 | Ground transportation |
| Printing + presentation materials | $46.90 | January 8th, 2026 | Materials |
| Lunch with client | $39.75 | January 8th, 2026 | Meals |
| Office supplies (notebooks, pens) | $27.60 | January 10th, 2026 | Supplies |
| Mileage reimbursement (client visit in NJ, 42 miles @ $0.67/mile) | $28.14 | January 14th, 2026 | Mileage |
| Team lunch meeting (internal) | $64.30 | January 15th, 2026 | Meals |
Total Expenses $1,701.64
Reimbursement Method
Reimbursement method Direct deposit
Notes
All receipts are attached. Expenses are business-related and comply with company travel policy.
**Approval**
Michael Tran, Employee
Laura Simmons, Manager
app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.pdf
BIN
View File
Binary file not shown.
app/core/src/main/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdown.java
+27 -5
View File
@@ -1,11 +1,13 @@
package stirling.software.SPDF.model.api.converters;
import org.springframework.core.io.Resource;
import java.nio.charset.StandardCharsets;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.multipart.MultipartFile;
import io.github.pixee.security.Filenames;
import io.swagger.v3.oas.annotations.Operation;
import lombok.RequiredArgsConstructor;
@@ -15,8 +17,11 @@ import stirling.software.common.annotations.AutoJobPostMapping;
import stirling.software.common.annotations.api.ConvertApi;
import stirling.software.common.enumeration.ResourceWeight;
import stirling.software.common.model.api.PDFFile;
import stirling.software.common.util.PDFToFile;
import stirling.software.common.pdf.PdfMarkdownConverter;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
import stirling.software.jpdfium.PdfDocument;
@ConvertApi
@RequiredArgsConstructor
@@ -33,10 +38,27 @@ public class ConvertPDFToMarkdown {
summary = "Convert PDF to Markdown",
description =
"This endpoint converts a PDF file to Markdown format. Input:PDF Output:Markdown Type:SISO")
public ResponseEntity<Resource> processPdfToMarkdown(@ModelAttribute PDFFile file)
public ResponseEntity<byte[]> processPdfToMarkdown(@ModelAttribute PDFFile file)
throws Exception {
MultipartFile inputFile = file.getFileInput();
PDFToFile pdfToFile = new PDFToFile(tempFileManager);
return pdfToFile.processPdfToMarkdown(inputFile);
String originalName = Filenames.toSimpleFileName(inputFile.getOriginalFilename());
String baseName =
originalName.contains(".")
? originalName.substring(0, originalName.lastIndexOf('.'))
: originalName;
String markdown;
try (TempFile tempInput = new TempFile(tempFileManager, ".pdf")) {
inputFile.transferTo(tempInput.getFile());
try (PdfDocument doc = PdfDocument.open(tempInput.getPath())) {
markdown = new PdfMarkdownConverter().convert(doc);
}
}
return WebResponseUtils.bytesToWebResponse(
markdown.getBytes(StandardCharsets.UTF_8),
baseName + ".md",
MediaType.valueOf("text/markdown"));
}
}
app/core/src/main/resources/settings.yml.template
+12
View File
@@ -364,6 +364,18 @@ aiEngine:
url: http://localhost:5001 # URL of the Python AI engine
timeoutSeconds: 120 # Timeout in seconds for AI engine requests
policies:
# Folder automations can read from and write to the directories you allow here, so treat this as a
# security boundary. Leave allowedFolderRoots empty (default) to disable folder sources/outputs
# entirely; list absolute directories to permit folder access only within them. Stirling's own
# config directory is always off-limits, and folder access is always disabled in SaaS mode.
allowedFolderRoots: [] # e.g. ["/data/inbox", "/data/outbox"]
scheduleSweepSeconds: 60 # How often (seconds) scheduled policies are checked for being due
watchReconcileSeconds: 300 # How often (seconds) folder-watch re-syncs watches and re-runs as a safety net for missed events
watchQuietPeriodMs: 500 # How long (ms) folder-watch coalesces a burst of file events into a single run
streamTimeoutMs: 1800000 # SSE timeout (ms) for live run-progress streams
runExpiryMinutes: 30 # How long (minutes) a finished run's in-memory state is kept before eviction
# Model Context Protocol (MCP) server. Exposes Stirling's PDF tools (grouped by namespace)
# plus the AI agents to MCP clients (Inspector, Claude Desktop, custom). OAuth-protected.
# Disabled by default - enable explicitly per deployment after configuring mcp.auth.
app/core/src/test/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdownTest.java
+48 -46
View File
@@ -1,16 +1,17 @@
package stirling.software.SPDF.model.api.converters;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.MockedConstruction;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
@@ -21,9 +22,10 @@ import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.multipart.MultipartFile;
import stirling.software.common.util.PDFToFile;
import stirling.software.common.pdf.PdfMarkdownConverter;
import stirling.software.common.util.TempFile;
import stirling.software.jpdfium.PdfDocument;
class ConvertPDFToMarkdownTest {
@@ -47,68 +49,68 @@ class ConvertPDFToMarkdownTest {
@Test
void pdfToMarkdownReturnsMarkdownBytes() throws Exception {
byte[] md = "# heading\n\ncontent\n".getBytes(StandardCharsets.UTF_8);
String expectedMd = "# heading\n\ncontent\n";
try (MockedConstruction<PDFToFile> construction =
Mockito.mockConstruction(
PDFToFile.class,
(mock, ctx) -> {
when(mock.processPdfToMarkdown(any(MultipartFile.class)))
.thenAnswer(
inv ->
ResponseEntity.ok()
.header("Content-Type", "text/markdown")
.body(new ByteArrayResource(md)));
})) {
File tmpFile = File.createTempFile("test", ".pdf");
tmpFile.deleteOnExit();
MockMvc mvc = mockMvc();
try (MockedConstruction<TempFile> tempMock =
Mockito.mockConstruction(
TempFile.class,
(mock, ctx) -> {
when(mock.getFile()).thenReturn(tmpFile);
when(mock.getPath()).thenReturn(tmpFile.toPath());
});
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
MockedConstruction<PdfMarkdownConverter> converterMock =
Mockito.mockConstruction(
PdfMarkdownConverter.class,
(mock, ctx) -> when(mock.convert(any())).thenReturn(expectedMd))) {
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
MockMultipartFile file =
new MockMultipartFile(
"fileInput", // must match the field name in PDFFile
"input.pdf",
"application/pdf",
new byte[] {1, 2, 3});
"fileInput", "input.pdf", "application/pdf", new byte[] {1, 2, 3});
// ResponseEntity<Resource> is written synchronously on the request thread,
// so there is no async dispatch to wait for (unlike the old StreamingResponseBody
// path).
mvc.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
mockMvc()
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
.andExpect(status().isOk())
.andExpect(header().string("Content-Type", "text/markdown"))
.andExpect(content().bytes(md));
// Verify that exactly one instance was created
assert construction.constructed().size() == 1;
// And that the uploaded file was passed to processPdfToMarkdown()
PDFToFile created = construction.constructed().get(0);
ArgumentCaptor<MultipartFile> captor = ArgumentCaptor.forClass(MultipartFile.class);
verify(created, times(1)).processPdfToMarkdown(captor.capture());
MultipartFile passed = captor.getValue();
// Minimal plausibility checks
assertEquals("input.pdf", passed.getOriginalFilename());
assertEquals("application/pdf", passed.getContentType());
}
}
@Test
void pdfToMarkdownWhenServiceThrowsReturns500() throws Exception {
try (MockedConstruction<PDFToFile> ignored =
Mockito.mockConstruction(
PDFToFile.class,
(mock, ctx) -> {
when(mock.processPdfToMarkdown(any(MultipartFile.class)))
.thenThrow(new RuntimeException("boom"));
})) {
File tmpFile = File.createTempFile("test", ".pdf");
tmpFile.deleteOnExit();
MockMvc mvc = mockMvc();
try (MockedConstruction<TempFile> tempMock =
Mockito.mockConstruction(
TempFile.class,
(mock, ctx) -> {
when(mock.getFile()).thenReturn(tmpFile);
when(mock.getPath()).thenReturn(tmpFile.toPath());
});
MockedStatic<PdfDocument> docStatic = Mockito.mockStatic(PdfDocument.class);
MockedConstruction<PdfMarkdownConverter> converterMock =
Mockito.mockConstruction(
PdfMarkdownConverter.class,
(mock, ctx) ->
when(mock.convert(any()))
.thenThrow(new RuntimeException("boom")))) {
PdfDocument mockDoc = Mockito.mock(PdfDocument.class);
docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc);
MockMultipartFile file =
new MockMultipartFile(
"fileInput", "x.pdf", "application/pdf", new byte[] {0x01});
mvc.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
mockMvc()
.perform(multipart("/api/v1/convert/pdf/markdown").file(file))
.andExpect(status().isInternalServerError());
}
}
app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java
+2 -1
View File
@@ -21,7 +21,8 @@ public enum AiWorkflowOutcome {
COMPLETED("completed"),
UNSUPPORTED_CAPABILITY("unsupported_capability"),
CANNOT_CONTINUE("cannot_continue"),
GENERATE_FILE("generate_file");
GENERATE_FILE("generate_file"),
CONVERT_MARKDOWN("convert_markdown");
private final String value;
app/proprietary/src/main/java/stirling/software/proprietary/policy/config/FolderAccessGuard.java
+106
View File
@@ -0,0 +1,106 @@
package stirling.software.proprietary.policy.config;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
import stirling.software.common.configuration.InstallationPathConfig;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.model.Policy;
/**
* The single authority on which filesystem locations a policy may read from or write to. Folder
* sources and sinks take a configured directory, so without this a user who can save a policy could
* point one at Stirling's own config/secrets directory and exfiltrate (or overwrite) it. Every
* folder source and sink runs its directory through {@link #requirePermitted(Path)} at save time
* and again at run time.
*
* <p>Enforced fail-closed, in order:
*
* <ul>
* <li><b>Disabled in SaaS</b> - folder access is never allowed when the {@code saas} profile is
* active; a tenant must not reach the host filesystem at all.
* <li><b>Protected paths</b> - Stirling's own config directory (settings, database, keys,
* backups) is always rejected, even if an allowed root were misconfigured to contain it.
* <li><b>Allowlist</b> - the directory must resolve within one of {@code
* policies.allowedFolderRoots}; with none configured, all folder access is refused.
* </ul>
*
* <p>Paths are compared after normalisation, so {@code ..} segments cannot walk out of an allowed
* root. (Symlink escape is not defended here; an operator who configures an allowed root containing
* a symlink to a sensitive location is trusted.)
*/
@Component
public class FolderAccessGuard {
public static final String FOLDER_TYPE = "folder";
private final boolean saasActive;
private final List<Path> allowedRoots;
private final List<Path> protectedRoots;
public FolderAccessGuard(ApplicationProperties applicationProperties, Environment environment) {
this.saasActive = Arrays.asList(environment.getActiveProfiles()).contains("saas");
this.allowedRoots =
normalizeAll(applicationProperties.getPolicies().getAllowedFolderRoots());
this.protectedRoots = List.of(normalize(Path.of(InstallationPathConfig.getConfigPath())));
}
/**
* Check that {@code dir} is a permitted folder location, returning its normalised absolute
* form.
*
* @throws IllegalArgumentException if folder access is disabled (SaaS or no roots configured),
* the path is inside a protected directory, or it falls outside every allowed root
*/
public Path requirePermitted(Path dir) {
if (saasActive) {
throw new IllegalArgumentException(
"folder sources and outputs are not available in SaaS mode");
}
Path normalized = normalize(dir);
for (Path protectedRoot : protectedRoots) {
if (normalized.startsWith(protectedRoot)) {
throw new IllegalArgumentException(
"folder may not point inside a protected Stirling directory");
}
}
if (allowedRoots.isEmpty()) {
throw new IllegalArgumentException(
"folder access is disabled; set policies.allowedFolderRoots to permit it");
}
boolean within = allowedRoots.stream().anyMatch(normalized::startsWith);
if (!within) {
throw new IllegalArgumentException(
"folder '" + normalized + "' is outside the allowed folder roots");
}
return normalized;
}
/** Whether this policy reads from or writes to a folder, and so is subject to these rules. */
public boolean usesFolderAccess(Policy policy) {
boolean readsFolder =
policy.sources().stream().anyMatch(spec -> FOLDER_TYPE.equals(spec.type()));
boolean writesFolder =
policy.output() != null && FOLDER_TYPE.equals(policy.output().type());
return readsFolder || writesFolder;
}
private static List<Path> normalizeAll(List<String> roots) {
List<Path> result = new ArrayList<>();
for (String root : roots) {
if (root != null && !root.isBlank()) {
result.add(normalize(Path.of(root)));
}
}
return result;
}
private static Path normalize(Path path) {
return path.toAbsolutePath().normalize();
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java
+44 -12
View File
@@ -6,7 +6,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus;
@@ -34,11 +33,16 @@ import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.job.JobResponse;
import stirling.software.common.service.UserServiceInterface;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.proprietary.policy.config.FolderAccessGuard;
import stirling.software.proprietary.policy.engine.PolicyRunHandle;
import stirling.software.proprietary.policy.engine.PolicyRunRegistry;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.engine.PolicyValidator;
import stirling.software.proprietary.policy.model.PipelineDefinition;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
@@ -47,7 +51,6 @@ import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.model.PolicyRunView;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
import stirling.software.proprietary.policy.store.PolicyStore;
import stirling.software.proprietary.policy.trigger.ManualTrigger;
import stirling.software.proprietary.security.config.PremiumEndpoint;
import tools.jackson.core.JacksonException;
@@ -71,16 +74,16 @@ import tools.jackson.databind.ObjectMapper;
@Tag(name = "Policies", description = "Run tool pipelines on the backend")
public class PolicyController {
private final ManualTrigger manualTrigger;
private final PolicyRunner policyRunner;
private final PolicyRunRegistry runRegistry;
private final PolicyStore policyStore;
private final PolicyValidator policyValidator;
private final FolderAccessGuard folderAccessGuard;
private final UserServiceInterface userService;
private final ApplicationProperties applicationProperties;
private final ObjectMapper objectMapper;
private final TempFileManager tempFileManager;
/** SSE emitter timeout, generous enough for long multi-step runs on large files. */
@Value("${stirling.policies.streamTimeoutMs:1800000}")
private long streamTimeoutMs;
@PostMapping(value = "/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@Operation(
summary = "Run a tool pipeline",
@@ -95,7 +98,8 @@ public class PolicyController {
throws IOException {
PipelineDefinition definition = parseDefinition(json);
PolicyInputs inputs = collectInputs(request);
String runId = manualTrigger.fire(definition, inputs, PolicyProgressListener.NOOP).runId();
String runId =
policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP).runId();
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
}
@@ -112,10 +116,11 @@ public class PolicyController {
PipelineDefinition definition = parseDefinition(json);
PolicyInputs inputs = collectInputs(request);
SseEmitter emitter = new SseEmitter(streamTimeoutMs);
SseEmitter emitter =
new SseEmitter(applicationProperties.getPolicies().getStreamTimeoutMs());
emitter.onError(e -> log.warn("Policy run SSE emitter error", e));
PolicyRunHandle handle = manualTrigger.fire(definition, inputs, streamListener(emitter));
PolicyRunHandle handle = policyRunner.runAdHoc(definition, inputs, streamListener(emitter));
// Close the stream with a terminal event once the run finishes. whenComplete runs on the
// engine's worker thread after the run is done, so this never races the step events.
handle.completion()
@@ -155,7 +160,34 @@ public class PolicyController {
"Stores a policy (trigger config + steps + output + metadata). A blank id is"
+ " assigned; returns the stored policy with its id.")
public ResponseEntity<Policy> savePolicy(@RequestBody String json) {
return ResponseEntity.ok(policyStore.save(parsePolicy(json)));
Policy policy = parsePolicy(json);
requireAuthorizedForFolderAccess(policy);
try {
policyValidator.validate(policy);
} catch (IllegalArgumentException e) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
}
return ResponseEntity.ok(policyStore.save(policy));
}
/**
* A policy that reads from or writes to a server folder grants whoever saves it access to that
* path, so restrict it to administrators on multi-user deployments. Single-user deployments
* (login disabled, e.g. desktop) trust the local operator. The {@link FolderAccessGuard} still
* enforces SaaS-off and the path allowlist during validation regardless of who saves.
*/
private void requireAuthorizedForFolderAccess(Policy policy) {
if (!folderAccessGuard.usesFolderAccess(policy)) {
return;
}
if (!applicationProperties.getSecurity().isEnableLogin()) {
return;
}
if (!userService.isCurrentUserAdmin()) {
throw new ResponseStatusException(
HttpStatus.FORBIDDEN,
"Folder sources and outputs may only be configured by an administrator");
}
}
@GetMapping
@@ -199,7 +231,7 @@ public class PolicyController {
new ResponseStatusException(
HttpStatus.NOT_FOUND, "No policy: " + policyId));
PolicyInputs inputs = collectInputs(request);
String runId = manualTrigger.run(policy, inputs, PolicyProgressListener.NOOP).runId();
String runId = policyRunner.runWith(policy, inputs, PolicyProgressListener.NOOP).runId();
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java
+6 -6
View File
@@ -9,13 +9,13 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import jakarta.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.model.PolicyRun;
/**
@@ -24,9 +24,9 @@ import stirling.software.proprietary.policy.model.PolicyRun;
* TaskManager}.
*
* <p>Finished runs are evicted on a fixed interval once they age past {@code
* stirling.policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a
* run's rich in-memory state does not outlive the process. Only terminal runs are evicted; active
* and paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not
* policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a run's
* rich in-memory state does not outlive the process. Only terminal runs are evicted; active and
* paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not
* touched here: a run shares its runId with a {@code TaskManager} job, which owns file-lifecycle
* cleanup, so eviction only frees this map's entry.
*/
@@ -41,8 +41,8 @@ public class PolicyRunRegistry {
Executors.newSingleThreadScheduledExecutor(
Thread.ofVirtual().name("policy-run-cleanup-", 0).factory());
public PolicyRunRegistry(
@Value("${stirling.policies.runExpiryMinutes:30}") int runExpiryMinutes) {
public PolicyRunRegistry(ApplicationProperties applicationProperties) {
int runExpiryMinutes = applicationProperties.getPolicies().getRunExpiryMinutes();
this.runExpiry = Duration.ofMinutes(runExpiryMinutes);
cleanupExecutor.scheduleAtFixedRate(this::evictExpiredRuns, 10, 10, TimeUnit.MINUTES);
log.debug(
app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java
+116
View File
@@ -0,0 +1,116 @@
package stirling.software.proprietary.policy.engine;
import java.io.IOException;
import java.util.List;
import java.util.function.Consumer;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.input.ResolvedInput;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.PipelineDefinition;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
/**
* Runs policies, and is the one place that knows how to turn a policy's configured {@link InputSpec
* sources} into actual runs. Triggers (schedule, and future webhook/folder-watch) decide
* <em>when</em> to run and call {@link #run(Policy)}; they never touch sources themselves. The
* controller uses the supplied-input and ad-hoc entry points for on-demand work.
*
* <p>This is the seam that keeps triggers and sources independent: a trigger depends on the runner,
* the runner depends on the {@link InputSource} beans, and a source depends on neither - it just
* yields {@link ResolvedInput units of work}, each carrying its own completion hook.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class PolicyRunner {
private final PolicyEngine policyEngine;
private final List<InputSource> inputSources;
/**
* Run a policy by pulling from every source it configures: each source yields zero or more
* units of work, and each unit becomes its own run so one failure does not affect the others. A
* policy with no sources runs once with no input files (a generator pipeline). Used by
* automatic triggers.
*/
public void run(Policy policy) {
List<InputSpec> sources = policy.sources();
if (sources.isEmpty()) {
startRun(policy, PolicyInputs.of(List.of()), unused -> {});
return;
}
for (InputSpec spec : sources) {
pullAndRun(policy, spec);
}
}
/**
* Run a stored policy on files supplied directly by the caller (e.g. a manual run with
* uploads), bypassing its configured sources. Returns the run handle so callers can stream
* progress.
*/
public PolicyRunHandle runWith(
Policy policy, PolicyInputs inputs, PolicyProgressListener listener) {
return policyEngine.runPolicy(policy, inputs, listener);
}
/** Run an ad-hoc pipeline with no stored policy (AI/Automate one-offs). */
public PolicyRunHandle runAdHoc(
PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) {
return policyEngine.submit(definition, inputs, listener);
}
private void pullAndRun(Policy policy, InputSpec spec) {
InputSource source = sourceFor(spec);
if (source == null) {
log.warn(
"No input source for type '{}' (policy {}); skipping",
spec.type(),
policy.id());
return;
}
List<ResolvedInput> work;
try {
work = source.resolve(spec);
} catch (IOException | RuntimeException e) {
log.warn(
"Failed to resolve source '{}' for policy {}: {}",
spec.type(),
policy.id(),
e.getMessage());
return;
}
for (ResolvedInput unit : work) {
startRun(policy, unit.inputs(), unit.onComplete());
}
}
private void startRun(Policy policy, PolicyInputs inputs, Consumer<Boolean> onComplete) {
log.info("Running policy {} ({})", policy.id(), policy.name());
PolicyRunHandle handle =
policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP);
handle.completion()
.whenComplete((run, throwable) -> onComplete.accept(succeeded(run, throwable)));
}
private static boolean succeeded(PolicyRun run, Throwable throwable) {
return throwable == null && run != null && run.getStatus() == PolicyRunStatus.COMPLETED;
}
private InputSource sourceFor(InputSpec spec) {
return inputSources.stream()
.filter(source -> source.supports(spec))
.findFirst()
.orElse(null);
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyValidator.java
+74
View File
@@ -0,0 +1,74 @@
package stirling.software.proprietary.policy.engine;
import java.util.List;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.output.PolicyOutputSink;
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
/**
* Validates a policy's trigger, sources, and output configuration by delegating each facet to the
* bean that handles its type. Called when a policy is saved so a misconfigured schedule, missing
* folder directory, or unknown type fails fast instead of silently misbehaving at run time.
*
* <p>The trigger is optional (a {@code null} trigger is a manual-only policy and needs no
* validation); every configured source is validated.
*/
@Service
@RequiredArgsConstructor
public class PolicyValidator {
private final List<PolicyTrigger> triggers;
private final List<InputSource> inputSources;
private final List<PolicyOutputSink> outputSinks;
/**
* @throws IllegalArgumentException if any facet's type is unknown or its configuration is
* invalid
*/
public void validate(Policy policy) {
if (policy.trigger() != null) {
triggerFor(policy.trigger()).validate(policy);
}
for (InputSpec source : policy.sources()) {
inputSourceFor(source).validate(source);
}
outputSinkFor(policy.output()).validate(policy.output());
}
private PolicyTrigger triggerFor(TriggerConfig config) {
return triggers.stream()
.filter(trigger -> trigger.type().equals(config.type()))
.findFirst()
.orElseThrow(
() ->
new IllegalArgumentException(
"unknown trigger type: " + config.type()));
}
private InputSource inputSourceFor(InputSpec spec) {
return inputSources.stream()
.filter(source -> source.supports(spec))
.findFirst()
.orElseThrow(
() ->
new IllegalArgumentException(
"unknown input source type: " + spec.type()));
}
private PolicyOutputSink outputSinkFor(OutputSpec spec) {
return outputSinks.stream()
.filter(sink -> sink.supports(spec))
.findFirst()
.orElseThrow(
() -> new IllegalArgumentException("unknown output type: " + spec.type()));
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java
+186
View File
@@ -0,0 +1,186 @@
package stirling.software.proprietary.policy.input;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.util.FileReadinessChecker;
import stirling.software.proprietary.policy.config.FolderAccessGuard;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.PolicyInputs;
/**
* Reads input files from a directory. Each ready file becomes its own unit of work (one run per
* file) so a failure on one file does not affect the others.
*
* <p>Two modes via the {@code mode} option:
*
* <ul>
* <li>{@code "consume"} (default) - claim each file by moving it into {@code
* .stirling/processing}, then route it to {@code .stirling/done} or {@code .stirling/error}
* when its run finishes. Each file is processed once; right for "process new arrivals" (and
* the basis of watched folders).
* <li>{@code "snapshot"} - read the directory's current files without moving them; every run sees
* the full set again. Right for "always regenerate from a fixed input set".
* </ul>
*
* Readiness is checked first (via {@link FileReadinessChecker}) so files mid-write are skipped.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class FolderInputSource implements InputSource {
private static final String TYPE = FolderAccessGuard.FOLDER_TYPE;
// Bookkeeping lives under one hidden namespace dir so the watched folder stays tidy.
private static final String WORK_SUBDIR = ".stirling";
private static final String PROCESSING_SUBDIR = "processing";
private static final String DONE_SUBDIR = "done";
private static final String ERROR_SUBDIR = "error";
private final FileReadinessChecker readinessChecker;
private final FolderAccessGuard accessGuard;
@Override
public String type() {
return TYPE;
}
@Override
public boolean supports(InputSpec spec) {
return spec != null && TYPE.equals(spec.type());
}
@Override
public void validate(InputSpec spec) {
accessGuard.requirePermitted(FolderConfig.from(spec.options()).directory());
}
@Override
public List<Path> watchTargets(InputSpec spec) {
return List.of(FolderConfig.from(spec.options()).directory());
}
@Override
public List<ResolvedInput> resolve(InputSpec spec) throws IOException {
FolderConfig config = FolderConfig.from(spec.options());
Path inputDir = accessGuard.requirePermitted(config.directory());
if (!Files.isDirectory(inputDir)) {
log.debug("Folder input dir does not exist: {}", inputDir);
return List.of();
}
List<Path> ready = new ArrayList<>();
try (Stream<Path> entries = Files.list(inputDir)) {
entries.filter(Files::isRegularFile)
.filter(readinessChecker::isReady)
.forEach(ready::add);
}
List<ResolvedInput> work = new ArrayList<>();
for (Path file : ready) {
if (config.snapshot()) {
work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file)))));
} else {
Path claimed = claim(inputDir, file);
if (claimed == null) {
continue; // another sweep/process grabbed it
}
work.add(
new ResolvedInput(
PolicyInputs.of(List.of(fileResource(claimed))),
success -> route(inputDir, claimed, success)));
}
}
return work;
}
private Path claim(Path inputDir, Path file) {
try {
Path processingDir = workDir(inputDir, PROCESSING_SUBDIR);
Files.createDirectories(processingDir);
Path claimed = uniqueTarget(processingDir, file.getFileName().toString());
Files.move(file, claimed, StandardCopyOption.ATOMIC_MOVE);
return claimed;
} catch (IOException e) {
log.debug("Could not claim {}: {}", file, e.getMessage());
return null;
}
}
private void route(Path inputDir, Path claimed, boolean success) {
String subdir = success ? DONE_SUBDIR : ERROR_SUBDIR;
try {
Path destDir = workDir(inputDir, subdir);
Files.createDirectories(destDir);
Files.move(
claimed,
uniqueTarget(destDir, claimed.getFileName().toString()),
StandardCopyOption.ATOMIC_MOVE);
} catch (IOException e) {
log.warn(
"Could not move processed input {} to {}: {}", claimed, subdir, e.getMessage());
}
}
/** A bookkeeping subdirectory under the watched folder's {@code .stirling} namespace. */
private static Path workDir(Path inputDir, String subdir) {
return inputDir.resolve(WORK_SUBDIR).resolve(subdir);
}
private static Resource fileResource(Path path) {
String name = path.getFileName().toString();
return new FileSystemResource(path.toFile()) {
@Override
public String getFilename() {
return name;
}
};
}
private static Path uniqueTarget(Path dir, String filename) {
Path candidate = dir.resolve(filename);
if (!Files.exists(candidate)) {
return candidate;
}
int dot = filename.lastIndexOf('.');
String base = dot < 0 ? filename : filename.substring(0, dot);
String ext = dot < 0 ? "" : filename.substring(dot);
for (int n = 1; ; n++) {
Path next = dir.resolve(base + " (" + n + ")" + ext);
if (!Files.exists(next)) {
return next;
}
}
}
/** The typed, validated form of a folder source's options: the directory and dedup mode. */
record FolderConfig(Path directory, boolean snapshot) {
private static final String DIRECTORY_OPTION = "directory";
private static final String MODE_OPTION = "mode";
private static final String MODE_SNAPSHOT = "snapshot";
static FolderConfig from(Map<String, Object> options) {
Object directory = options.get(DIRECTORY_OPTION);
if (directory == null || directory.toString().isBlank()) {
throw new IllegalArgumentException("folder input requires a 'directory' option");
}
Object mode = options.get(MODE_OPTION);
boolean snapshot = mode != null && MODE_SNAPSHOT.equals(mode.toString());
return new FolderConfig(Path.of(directory.toString()), snapshot);
}
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java
+49
View File
@@ -0,0 +1,49 @@
package stirling.software.proprietary.policy.input;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import stirling.software.proprietary.policy.model.InputSpec;
/**
* Resolves one of a policy's {@link InputSpec sources} into the files to run on - answering
* <em>where</em> a run's files come from, independent of <em>when</em> it runs. The counterpart of
* {@code PolicyOutputSink}: implementations are beans selected by {@link #supports(InputSpec)}, so
* a new source kind (folder, S3) is just a new bean.
*
* <p>Driven by the {@code PolicyRunner}, which a trigger calls when a policy is due; a source is
* passive and knows nothing about what triggered the run. A manual run may instead supply files
* directly and bypass sources entirely.
*/
public interface InputSource {
/** Stable identifier for this source, matching {@code InputSpec.type()} (e.g. "folder"). */
String type();
/** Whether this source can handle the given spec. */
boolean supports(InputSpec spec);
/**
* Check that an input spec is usable, throwing {@link IllegalArgumentException} if not. Called
* when a policy is saved so misconfiguration fails fast rather than at run time.
*/
default void validate(InputSpec spec) {}
/**
* Resolve the spec into zero or more units of work, each carrying the files for one run and a
* completion hook. Returning an empty list means there is nothing to run right now.
*/
List<ResolvedInput> resolve(InputSpec spec) throws IOException;
/**
* The local filesystem directories this source draws from, if any, for triggers that want to
* react to changes there (the folder-watch trigger) rather than poll. Advisory only: it merely
* tells a trigger <em>where</em> to watch; resolving the spec into files is still this source's
* job via {@link #resolve}. Non-filesystem sources (S3, ...) return an empty list and so are
* simply not watchable. Default: nothing to watch.
*/
default List<Path> watchTargets(InputSpec spec) {
return List.of();
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolvedInput.java
+23
View File
@@ -0,0 +1,23 @@
package stirling.software.proprietary.policy.input;
import java.util.function.Consumer;
import stirling.software.proprietary.policy.model.PolicyInputs;
/**
* One unit of work produced by an {@link InputSource}: the files to run plus a completion callback
* invoked with the run's success once it finishes (e.g. a folder source routes the input to {@code
* .stirling/done} or {@code .stirling/error}). A source may return several of these (e.g. one per
* file).
*/
public record ResolvedInput(PolicyInputs inputs, Consumer<Boolean> onComplete) {
public ResolvedInput {
onComplete = onComplete == null ? success -> {} : onComplete;
}
/** A unit of work with no completion side effect. */
public static ResolvedInput of(PolicyInputs inputs) {
return new ResolvedInput(inputs, success -> {});
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/InputSpec.java
+26
View File
@@ -0,0 +1,26 @@
package stirling.software.proprietary.policy.model;
import java.util.Map;
/**
* One source a policy's input files come from. {@code type} selects an {@code InputSource}
* ("folder", and in future "s3", ...); {@code options} carries source-specific configuration (a
* directory, a bucket, a dedup mode, ...).
*
* <p>A policy holds a list of these; a run pulls from every one. An empty list means the policy
* runs with no input files (a generator pipeline, or files supplied directly to a manual run).
*
* <p>Data-driven and parallel to {@link OutputSpec}/{@link TriggerConfig}: a new source kind is a
* new {@code type} handled by a new {@code InputSource} bean.
*/
public record InputSpec(String type, Map<String, Object> options) {
public InputSpec {
options = options == null ? Map.of() : options;
}
/** Read input files from a directory on disk. */
public static InputSpec folder(String directory) {
return new InputSpec("folder", Map.of("directory", directory));
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java
+5
View File
@@ -18,4 +18,9 @@ public record OutputSpec(String type, Map<String, Object> options) {
public static OutputSpec inline() {
return new OutputSpec("inline", Map.of());
}
/** Write outputs to a directory on disk. */
public static OutputSpec folder(String directory) {
return new OutputSpec("folder", Map.of("directory", directory));
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java
+22 -11
View File
@@ -3,18 +3,16 @@ package stirling.software.proprietary.policy.model;
import java.util.List;
/**
* A stored, owned automation: how it is triggered, the ordered tool steps to run, and where its
* output goes, plus identity and metadata.
* A stored automation: an ordered chain of tool steps, the sources its input files come from, and
* an output destination for the results.
*
* <p>This is the central object of the feature. Everything that runs a chain of tools is a use of a
* Policy: a watched folder is a Policy with a folder {@link TriggerConfig} and a folder {@link
* OutputSpec}; a scheduled job is a Policy with a schedule trigger; manual/Automate/AI runs execute
* a Policy (or an ad-hoc {@link PipelineDefinition}) on demand. The engine itself only ever
* executes the {@link PipelineDefinition} this exposes via {@link #toDefinition()} - it is
* trigger-agnostic.
* <p>Every policy can always be run on demand (manually). It may additionally carry one automatic
* {@link TriggerConfig} - usually a schedule - that fires it without a person asking; a {@code
* null} trigger means manual-only. A trigger decides <em>when</em> a run happens and a {@link
* InputSpec source} decides <em>where</em> its files come from; the two are independent, and a run
* pulls from every configured source.
*
* <p>{@code enabled} gates automatic triggering (a disabled policy is not picked up by its
* trigger); it does not block an explicit manual run.
* <p>This is the feature's central configuration object - what a user defines and the engine runs.
*/
public record Policy(
String id,
@@ -22,15 +20,28 @@ public record Policy(
String owner,
boolean enabled,
TriggerConfig trigger,
List<InputSpec> sources,
List<PipelineStep> steps,
OutputSpec output) {
public Policy {
trigger = trigger == null ? TriggerConfig.manual() : trigger;
sources = sources == null ? List.of() : List.copyOf(sources);
steps = steps == null ? List.of() : steps;
output = output == null ? OutputSpec.inline() : output;
}
/** A policy with no configured sources (a generator, or files supplied directly to a run). */
public Policy(
String id,
String name,
String owner,
boolean enabled,
TriggerConfig trigger,
List<PipelineStep> steps,
OutputSpec output) {
this(id, name, owner, enabled, trigger, List.of(), steps, output);
}
/** The engine-level, trigger-agnostic view of this policy's pipeline. */
public PipelineDefinition toDefinition() {
return new PipelineDefinition(name, steps, output);
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Schedule.java
+141
View File
@@ -0,0 +1,141 @@
package stirling.software.proprietary.policy.model;
import java.time.DayOfWeek;
import java.time.LocalTime;
import java.time.ZonedDateTime;
import java.util.EnumSet;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* When a scheduled policy should fire, expressed as intent ("every day at 02:00") rather than a
* cron string. A frontend builds one of these from a friendly picker; the schedule trigger asks it
* for the next firing after a given moment.
*
* <p>Sealed so every cadence is an explicit, self-validating shape and adding a new one forces a
* new subtype rather than another string convention to learn. The {@code type} discriminator stays
* on the wire so the frontend can switch on it without knowing the Java hierarchy.
*
* <p>Wall-clock kinds ({@link Daily}, {@link Weekly}, {@link Monthly}) are evaluated in the zone of
* the {@code after} argument; {@link Every} is a fixed offset and ignores wall-clock time.
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
@JsonSubTypes({
@JsonSubTypes.Type(value = Schedule.Every.class, name = "every"),
@JsonSubTypes.Type(value = Schedule.Daily.class, name = "daily"),
@JsonSubTypes.Type(value = Schedule.Weekly.class, name = "weekly"),
@JsonSubTypes.Type(value = Schedule.Monthly.class, name = "monthly"),
})
@JsonIgnoreProperties(ignoreUnknown = true)
public sealed interface Schedule {
/** The next firing strictly after {@code after}, evaluated in {@code after}'s zone. */
ZonedDateTime nextAfter(ZonedDateTime after);
/** The granularities a fixed-interval schedule can repeat on. */
enum Unit {
MINUTES,
HOURS,
DAYS
}
/**
* A fixed cadence repeating from when the policy was last seen: "every 15 minutes", "every 6
* hours". Time of day is irrelevant.
*/
record Every(long count, Unit unit) implements Schedule {
public Every {
if (count <= 0) {
throw new IllegalArgumentException("'every' schedule needs a positive count");
}
if (unit == null) {
throw new IllegalArgumentException("'every' schedule needs a unit");
}
}
@Override
public ZonedDateTime nextAfter(ZonedDateTime after) {
return switch (unit) {
case MINUTES -> after.plusMinutes(count);
case HOURS -> after.plusHours(count);
case DAYS -> after.plusDays(count);
};
}
}
/** Once a day at a wall-clock time: "every day at 02:00". */
record Daily(LocalTime at) implements Schedule {
public Daily {
requireTime(at);
}
@Override
public ZonedDateTime nextAfter(ZonedDateTime after) {
ZonedDateTime today = after.with(at);
return today.isAfter(after) ? today : today.plusDays(1);
}
}
/** On chosen weekdays at a wall-clock time: "every Monday and Thursday at 09:00". */
record Weekly(Set<DayOfWeek> days, LocalTime at) implements Schedule {
public Weekly {
if (days == null || days.isEmpty()) {
throw new IllegalArgumentException("'weekly' schedule needs at least one day");
}
requireTime(at);
days = EnumSet.copyOf(days);
}
@Override
public ZonedDateTime nextAfter(ZonedDateTime after) {
// The soonest of the next 7 days that lands on a chosen weekday, at the configured
// time.
for (int i = 0; i <= 7; i++) {
ZonedDateTime candidate = after.plusDays(i).with(at);
if (candidate.isAfter(after) && days.contains(candidate.getDayOfWeek())) {
return candidate;
}
}
throw new IllegalStateException("unreachable: a chosen weekday recurs within 8 days");
}
}
/**
* On a day of the month at a wall-clock time: "the 1st at 00:00". Months too short for the
* chosen day (e.g. the 31st in February) are skipped, not clamped.
*/
record Monthly(int dayOfMonth, LocalTime at) implements Schedule {
public Monthly {
if (dayOfMonth < 1 || dayOfMonth > 31) {
throw new IllegalArgumentException("'monthly' day-of-month must be 1-31");
}
requireTime(at);
}
@Override
public ZonedDateTime nextAfter(ZonedDateTime after) {
ZonedDateTime firstOfMonth = after.withDayOfMonth(1).with(at);
// Scan forward a few years' worth of months to skip ones without the chosen day.
for (int i = 0; i < 48; i++) {
ZonedDateTime month = firstOfMonth.plusMonths(i);
if (month.toLocalDate().lengthOfMonth() >= dayOfMonth) {
ZonedDateTime fire = month.withDayOfMonth(dayOfMonth);
if (fire.isAfter(after)) {
return fire;
}
}
}
throw new IllegalStateException(
"unreachable: a month with the chosen day recurs yearly");
}
}
private static void requireTime(LocalTime at) {
if (at == null) {
throw new IllegalArgumentException("schedule needs a time of day ('at')");
}
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java
+9 -11
View File
@@ -3,23 +3,21 @@ package stirling.software.proprietary.policy.model;
import java.util.Map;
/**
* How a {@link Policy} is automatically triggered. {@code type} selects a trigger kind ("manual",
* "folder", "schedule", "s3"); {@code options} carries type-specific configuration (a folder path,
* a cron expression, a bucket, ...).
* A {@link Policy}'s optional automatic trigger - what fires it without a person asking. {@code
* type} selects a trigger kind ("schedule", and in future "webhook", "folder-watch", ...); {@code
* options} carries type-specific configuration (a {@link Schedule}, a webhook secret, ...).
*
* <p>Manual running is <em>not</em> a trigger kind: every policy can always be run on demand, so a
* policy with no automatic trigger simply has a {@code null} {@code TriggerConfig}. A trigger
* answers only "when"; where a run's files come from is a separate concern owned by the policy's
* {@link InputSpec sources}.
*
* <p>Data-driven and parallel to {@link OutputSpec}: new trigger kinds are new {@code type} values
* handled by a new trigger bean, with no change to the model. {@code "manual"} means there is no
* automatic trigger - the policy is only ever run on demand.
* handled by a new trigger bean, with no change to the model.
*/
public record TriggerConfig(String type, Map<String, Object> options) {
public TriggerConfig {
type = type == null || type.isBlank() ? "manual" : type;
options = options == null ? Map.of() : options;
}
/** No automatic trigger; the policy is run on demand only. */
public static TriggerConfig manual() {
return new TriggerConfig("manual", Map.of());
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java
+130
View File
@@ -0,0 +1,130 @@
package stirling.software.proprietary.policy.output;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import org.apache.commons.io.FilenameUtils;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
import org.springframework.http.MediaTypeFactory;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.job.ResultFile;
import stirling.software.proprietary.policy.config.FolderAccessGuard;
import stirling.software.proprietary.policy.model.OutputSpec;
/**
* Writes a run's output files to a directory on disk. The destination is the {@code directory}
* option of the {@link OutputSpec}.
*
* <p>Files are streamed to disk (so large outputs are not buffered) and given unique names to avoid
* clobbering existing files. The returned {@link ResultFile}s describe what was written (path +
* size); they carry a synthetic id because the deliverable is the file on disk, not a {@code
* FileStorage} entry, so folder outputs are not downloadable via {@code /files/{id}}.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class FolderOutputSink implements PolicyOutputSink {
static final String TYPE = FolderAccessGuard.FOLDER_TYPE;
static final String DIRECTORY_OPTION = "directory";
private final FolderAccessGuard accessGuard;
@Override
public String type() {
return TYPE;
}
@Override
public boolean supports(OutputSpec spec) {
return spec != null && TYPE.equals(spec.type());
}
@Override
public void validate(OutputSpec spec) {
accessGuard.requirePermitted(directoryOf(spec));
}
@Override
public List<ResultFile> deliver(String runId, List<Resource> outputs, OutputSpec spec)
throws IOException {
Path targetDir = accessGuard.requirePermitted(directoryOf(spec));
Files.createDirectories(targetDir);
List<ResultFile> results = new ArrayList<>();
for (int i = 0; i < outputs.size(); i++) {
Resource resource = outputs.get(i);
String name = safeName(resource.getFilename(), i);
Path target = uniqueTarget(targetDir, name);
try (InputStream is = resource.getInputStream()) {
Files.copy(is, target);
}
long size = Files.size(target);
String contentType =
MediaTypeFactory.getMediaType(name)
.orElse(MediaType.APPLICATION_OCTET_STREAM)
.toString();
results.add(
ResultFile.builder()
.fileId(UUID.randomUUID().toString())
.fileName(target.toString())
.contentType(contentType)
.fileSize(size)
.build());
log.debug("Wrote policy run {} output to {}", runId, target);
}
return results;
}
private static Path directoryOf(OutputSpec spec) {
Object directory = spec.options().get(DIRECTORY_OPTION);
if (directory == null || directory.toString().isBlank()) {
throw new IllegalArgumentException(
"folder output requires a '" + DIRECTORY_OPTION + "' option");
}
return Path.of(directory.toString());
}
/**
* The resource's filename reduced to a bare, traversal-free name: any directory component or
* "../" is stripped so a crafted output name cannot escape {@code targetDir}. Falls back to a
* synthetic name when the filename is absent or reduces to nothing usable.
*/
private static String safeName(String filename, int index) {
if (filename == null || filename.isBlank()) {
return "output-" + index;
}
String name = FilenameUtils.getName(filename);
if (name.isBlank() || ".".equals(name) || "..".equals(name)) {
return "output-" + index;
}
return name;
}
/** Resolve a non-colliding path in {@code dir}, appending " (n)" before the extension. */
private static Path uniqueTarget(Path dir, String filename) {
Path candidate = dir.resolve(filename);
if (!Files.exists(candidate)) {
return candidate;
}
String base = FilenameUtils.getBaseName(filename);
String ext = FilenameUtils.getExtension(filename);
String suffix = ext.isEmpty() ? "" : "." + ext;
for (int n = 1; ; n++) {
Path next = dir.resolve(base + " (" + n + ")" + suffix);
if (!Files.exists(next)) {
return next;
}
}
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java
+6
View File
@@ -23,6 +23,12 @@ public interface PolicyOutputSink {
/** Whether this sink can handle the given output spec. */
boolean supports(OutputSpec spec);
/**
* Check that an output spec is usable, throwing {@link IllegalArgumentException} if not. Called
* when a policy is saved so misconfiguration fails fast rather than at run time.
*/
default void validate(OutputSpec spec) {}
/**
* Persist/deliver the output files and return their descriptors.
*
app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java
+2
View File
@@ -30,6 +30,7 @@ public class InProcessPolicyStore implements PolicyStore {
policy.owner(),
policy.enabled(),
policy.trigger(),
policy.sources(),
policy.steps(),
policy.output());
policies.put(id, stored);
@@ -50,6 +51,7 @@ public class InProcessPolicyStore implements PolicyStore {
public List<Policy> findByTriggerType(String triggerType) {
return policies.values().stream()
.filter(Policy::enabled)
.filter(policy -> policy.trigger() != null)
.filter(policy -> triggerType.equals(policy.trigger().type()))
.toList();
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java
+2 -1
View File
@@ -37,6 +37,7 @@ public class JpaPolicyStore implements PolicyStore {
policy.owner(),
policy.enabled(),
policy.trigger(),
policy.sources(),
policy.steps(),
policy.output());
@@ -45,7 +46,7 @@ public class JpaPolicyStore implements PolicyStore {
entity.setName(stored.name());
entity.setOwner(stored.owner());
entity.setEnabled(stored.enabled());
entity.setTriggerType(stored.trigger().type());
entity.setTriggerType(stored.trigger() == null ? null : stored.trigger().type());
entity.setPolicyJson(objectMapper.writeValueAsString(stored));
repository.save(entity);
return stored;
app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java
+302
View File
@@ -0,0 +1,302 @@
package stirling.software.proprietary.policy.trigger;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import java.io.IOException;
import java.nio.file.ClosedWatchServiceException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.store.PolicyStore;
/**
* Fires policies the moment a file lands in one of their folder sources, instead of polling on a
* timer. The trigger only reads that location; turning it into files is still the source's job.
*
* <p>The watcher is a latency optimisation, not a source of truth, so this pairs an event watch
* with a low-frequency <b>reconcile</b> sweep ({@code watchReconcileSeconds}). The reconcile both
* (a) re-syncs which directories are watched as policies are created/edited/deleted and folders
* appear on disk, and (b) runs every folder-watch policy once, catching files that pre-dated the
* watch, events lost to inotify-queue overflow, and changes on filesystems that do not deliver
* events at all (NFS, many container bind mounts). Both paths just call {@link PolicyRunner#run};
* the {@link InputSource} does the claiming, so a redundant run finds nothing to claim and is
* harmless.
*
* <p>Like {@link ScheduleTrigger}, watch state is per-node and in memory; this assumes a single
* node and rebuilds its registrations on restart from the {@link PolicyStore}.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class FolderWatchTrigger implements PolicyTrigger {
private static final String TYPE = "folder-watch";
private final PolicyStore policyStore;
private final PolicyRunner policyRunner;
private final List<InputSource> inputSources;
private final ApplicationProperties applicationProperties;
private final Map<Path, WatchKey> keysByDir = new ConcurrentHashMap<>();
private final Map<WatchKey, Path> dirByKey = new ConcurrentHashMap<>();
private volatile boolean running;
// Package-visible (not private) so tests can drive syncRegistrations() against a real service.
volatile WatchService watchService;
private volatile ScheduledExecutorService reconciler;
@Override
public String type() {
return TYPE;
}
@Override
public void validate(Policy policy) {
if (watchDirsOf(policy).isEmpty()) {
throw new IllegalArgumentException(
"folder-watch trigger requires at least one watchable (folder) input source");
}
}
@Override
public synchronized void start() {
if (watchService != null) {
return;
}
try {
watchService = FileSystems.getDefault().newWatchService();
} catch (IOException e) {
log.error("Could not start folder-watch trigger: {}", e.getMessage(), e);
return;
}
running = true;
Thread.ofVirtual().name("policy-folder-watch").start(this::watchLoop);
long reconcileSeconds = applicationProperties.getPolicies().getWatchReconcileSeconds();
reconciler =
Executors.newSingleThreadScheduledExecutor(
Thread.ofVirtual().name("policy-folder-reconcile-", 0).factory());
// First reconcile runs immediately so pre-existing files are picked up at startup.
reconciler.scheduleAtFixedRate(this::safeReconcile, 0, reconcileSeconds, TimeUnit.SECONDS);
log.info("Folder-watch trigger started (reconcile every {}s)", reconcileSeconds);
}
@Override
public synchronized void stop() {
running = false;
if (reconciler != null) {
reconciler.shutdownNow();
reconciler = null;
}
if (watchService != null) {
try {
watchService.close(); // wakes the watch loop with ClosedWatchServiceException
} catch (IOException e) {
log.debug("Error closing folder watch service: {}", e.getMessage());
}
watchService = null;
}
keysByDir.clear();
dirByKey.clear();
}
private void watchLoop() {
// Capture the service once: stop() may null the field, and a local avoids racing that to an
// NPE (close() still wakes take()/poll() on this same instance).
WatchService watcher = watchService;
if (watcher == null) {
return;
}
while (running) {
WatchKey first;
try {
first = watcher.take();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
} catch (ClosedWatchServiceException e) {
return;
}
runForChangedDirs(drainBurst(watcher, first));
}
}
/**
* Collect the directories touched by {@code first} and any further events that arrive within
* the quiet period, so a burst of file-system events becomes a single set of affected
* directories. The event kinds are irrelevant: any event on a watched dir just means "go look".
*/
private Set<Path> drainBurst(WatchService watcher, WatchKey first) {
long quietPeriodMs = applicationProperties.getPolicies().getWatchQuietPeriodMs();
Set<Path> changed = new HashSet<>();
WatchKey key = first;
while (key != null) {
key.pollEvents();
Path dir = dirByKey.get(key);
if (dir != null) {
changed.add(dir);
}
key.reset();
try {
key = watcher.poll(quietPeriodMs, TimeUnit.MILLISECONDS);
} catch (ClosedWatchServiceException | InterruptedException e) {
break;
}
}
return changed;
}
/** Run every folder-watch policy that draws from one of the changed directories. */
void runForChangedDirs(Set<Path> changedDirs) {
if (changedDirs.isEmpty()) {
return;
}
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
List<Path> dirs;
try {
dirs = watchDirsOf(policy);
} catch (RuntimeException e) {
log.warn(
"Folder-watch policy {} is misconfigured: {}", policy.id(), e.getMessage());
continue;
}
if (dirs.stream().anyMatch(changedDirs::contains)) {
log.debug("Folder-watch policy {} ({}) saw activity", policy.id(), policy.name());
policyRunner.run(policy);
}
}
}
private void safeReconcile() {
try {
syncRegistrations();
runAll();
} catch (RuntimeException e) {
log.error("Folder-watch reconcile failed: {}", e.getMessage(), e);
}
}
/** The reconcile safety net: run every folder-watch policy regardless of watch events. */
void runAll() {
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
try {
policyRunner.run(policy);
} catch (RuntimeException e) {
log.warn(
"Folder-watch reconcile run failed for policy {}: {}",
policy.id(),
e.getMessage());
}
}
}
/**
* Bring the set of watched directories in line with the current folder-watch policies: register
* newly required directories that exist on disk, and cancel ones no longer wanted.
*/
synchronized void syncRegistrations() {
if (watchService == null) {
return;
}
Set<Path> desired = desiredDirs();
keysByDir
.entrySet()
.removeIf(
entry -> {
if (desired.contains(entry.getKey())) {
return false;
}
entry.getValue().cancel();
dirByKey.remove(entry.getValue());
return true;
});
for (Path dir : desired) {
if (keysByDir.containsKey(dir)) {
continue;
}
try {
WatchKey key = dir.register(watchService, ENTRY_CREATE, ENTRY_MODIFY);
keysByDir.put(dir, key);
dirByKey.put(key, dir);
log.info("Watching {} for folder-watch policies", dir);
} catch (IOException | RuntimeException e) {
log.warn("Could not watch {}: {}", dir, e.getMessage());
}
}
}
/** The directories currently registered with the watch service. Visible for tests. */
Set<Path> watchedDirs() {
return Set.copyOf(keysByDir.keySet());
}
/** Every existing directory any current folder-watch policy wants watched. */
private Set<Path> desiredDirs() {
Set<Path> dirs = new HashSet<>();
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
try {
for (Path dir : watchDirsOf(policy)) {
if (Files.isDirectory(dir)) {
dirs.add(dir);
}
}
} catch (RuntimeException e) {
log.warn(
"Folder-watch policy {} is misconfigured: {}", policy.id(), e.getMessage());
}
}
return dirs;
}
/**
* The normalised, absolute directories this policy's sources expose to watch. Normalisation
* makes registration keys and event-time matching comparable regardless of how the path was
* configured.
*/
private List<Path> watchDirsOf(Policy policy) {
List<Path> dirs = new ArrayList<>();
for (InputSpec spec : policy.sources()) {
InputSource source = sourceFor(spec);
if (source == null) {
continue;
}
for (Path dir : source.watchTargets(spec)) {
dirs.add(dir.toAbsolutePath().normalize());
}
}
return dirs;
}
private InputSource sourceFor(InputSpec spec) {
return inputSources.stream()
.filter(source -> source.supports(spec))
.findFirst()
.orElse(null);
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java
-41
View File
@@ -1,41 +0,0 @@
package stirling.software.proprietary.policy.trigger;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import stirling.software.proprietary.policy.engine.PolicyEngine;
import stirling.software.proprietary.policy.engine.PolicyRunHandle;
import stirling.software.proprietary.policy.model.PipelineDefinition;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
/**
* Runs policies on demand, in response to a request (the {@code PolicyController} endpoints, an AI,
* or another automation). It is the request-driven trigger: no background lifecycle, it just
* forwards to the engine. Any policy can be run manually regardless of its configured trigger type.
*/
@Service
@RequiredArgsConstructor
public class ManualTrigger implements PolicyTrigger {
private final PolicyEngine policyEngine;
@Override
public String type() {
return "manual";
}
/** Run a stored policy immediately and return its run handle. */
public PolicyRunHandle run(
Policy policy, PolicyInputs inputs, PolicyProgressListener listener) {
return policyEngine.runPolicy(policy, inputs, listener);
}
/** Run an ad-hoc pipeline (no stored policy), e.g. for AI or Automate one-offs. */
public PolicyRunHandle fire(
PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) {
return policyEngine.submit(definition, inputs, listener);
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java
+23 -9
View File
@@ -1,22 +1,36 @@
package stirling.software.proprietary.policy.trigger;
import stirling.software.proprietary.policy.model.Policy;
/**
* Activates policies of one trigger type. A trigger owns a {@link #type()} (matching {@code
* TriggerConfig.type()}); when its condition fires it runs the relevant {@code Policy} through the
* {@code PolicyEngine}.
* An automatic trigger: the thing that decides <em>when</em> a policy runs without a person asking.
* A trigger owns a {@link #type()} (matching {@code TriggerConfig.type()}); when its condition
* fires it hands the policy to the {@code PolicyRunner}, which pulls the policy's sources and
* starts the runs. A trigger never resolves sources itself.
*
* <p>Background triggers (folder watcher, schedule) are driven by configuration: on {@link
* #start()} they begin watching/scheduling for the policies returned by {@code
* PolicyStore.findByTriggerType(type())}, and stop on {@link #stop()}. Request-driven triggers
* (manual) have no background lifecycle and run a policy directly in response to a call. New
* trigger kinds are new beans of this type; the engine and the {@code Policy} model do not change.
* <p>Triggers are background, configuration-driven beans (schedule, and in future webhook or
* folder-watch): on {@link #start()} they begin watching/scheduling for the policies returned by
* {@code PolicyStore.findByTriggerType(type())}, and stop on {@link #stop()}. New trigger kinds are
* new beans of this type; the runner and the {@code Policy} model do not change.
*
* <p>Manual running is not a trigger - every policy can always be run on demand via the {@code
* PolicyRunner} regardless of whether it has a trigger.
*/
public interface PolicyTrigger {
/** Stable identifier for this trigger kind, matching {@code TriggerConfig.type()}. */
String type();
/** Begin activating policies of this type (e.g. start a folder watcher). No-op for manual. */
/**
* Check that this trigger is usable for the given policy, throwing {@link
* IllegalArgumentException} if not. Called when a policy is saved so misconfiguration fails
* fast rather than at fire time. Receives the whole {@link Policy} (not just its {@code
* TriggerConfig}) so a trigger whose firing depends on the policy's sources (folder-watch) can
* assert that relationship; most triggers only inspect {@code policy.trigger()}.
*/
default void validate(Policy policy) {}
/** Begin activating policies of this type (e.g. start the schedule sweep). */
default void start() {}
/** Stop activating and release any resources. */
app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManager.java
+56
View File
@@ -0,0 +1,56 @@
package stirling.software.proprietary.policy.trigger;
import java.util.List;
import org.springframework.context.SmartLifecycle;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Starts and stops every {@link PolicyTrigger} with the application lifecycle. Background triggers
* (schedule, and future folder/S3) begin watching on {@link #start()} and release resources on
* {@link #stop()}; request-driven triggers (manual) are no-ops.
*
* <p>This is the single activation point for triggers - a new background trigger only has to be a
* {@link PolicyTrigger} bean.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class PolicyTriggerManager implements SmartLifecycle {
private final List<PolicyTrigger> triggers;
private volatile boolean running;
@Override
public void start() {
for (PolicyTrigger trigger : triggers) {
try {
trigger.start();
} catch (RuntimeException e) {
log.error("Failed to start trigger '{}': {}", trigger.type(), e.getMessage(), e);
}
}
running = true;
}
@Override
public void stop() {
for (PolicyTrigger trigger : triggers) {
try {
trigger.stop();
} catch (RuntimeException e) {
log.error("Failed to stop trigger '{}': {}", trigger.type(), e.getMessage(), e);
}
}
running = false;
}
@Override
public boolean isRunning() {
return running;
}
}
app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java
+158
View File
@@ -0,0 +1,158 @@
package stirling.software.proprietary.policy.trigger;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.springframework.stereotype.Service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.Schedule;
import stirling.software.proprietary.policy.store.PolicyStore;
import tools.jackson.databind.ObjectMapper;
/**
* Fires policies on a {@link Schedule}. On {@link #start()} it sweeps on a fixed interval; each
* sweep finds the enabled "schedule" policies and runs any whose next firing has come due since it
* last fired.
*
* <p>The trigger only decides <em>when</em>: once a policy is due it hands it to the {@link
* PolicyRunner}, which pulls from the policy's configured sources and starts the runs. The trigger
* knows nothing about folders, buckets, or how many runs a sweep produces.
*
* <p>Caveat: last-fire times are tracked <b>in memory</b>, so this assumes a single node and resets
* on restart; cluster-wide coordination (leader election) is a follow-up.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class ScheduleTrigger implements PolicyTrigger {
private static final String TYPE = "schedule";
private final PolicyStore policyStore;
private final PolicyRunner policyRunner;
private final ObjectMapper objectMapper;
private final ApplicationProperties applicationProperties;
private final Map<String, Instant> lastFiredByPolicy = new ConcurrentHashMap<>();
private volatile ScheduledExecutorService scheduler;
@Override
public String type() {
return TYPE;
}
@Override
public void validate(Policy policy) {
ScheduleConfig.from(objectMapper, policy.trigger().options());
}
@Override
public synchronized void start() {
if (scheduler != null) {
return;
}
long sweepSeconds = applicationProperties.getPolicies().getScheduleSweepSeconds();
scheduler =
Executors.newSingleThreadScheduledExecutor(
Thread.ofVirtual().name("policy-schedule-", 0).factory());
scheduler.scheduleAtFixedRate(
this::safeSweep, sweepSeconds, sweepSeconds, TimeUnit.SECONDS);
log.info("Schedule trigger started (sweep every {}s)", sweepSeconds);
}
@Override
public synchronized void stop() {
if (scheduler != null) {
scheduler.shutdownNow();
scheduler = null;
}
}
private void safeSweep() {
try {
sweep(Instant.now());
} catch (RuntimeException e) {
log.error("Schedule sweep failed: {}", e.getMessage(), e);
}
}
/** Fire every scheduled policy that is due as of {@code now}. Package-visible for testing. */
void sweep(Instant now) {
for (Policy policy : policyStore.findByTriggerType(TYPE)) {
ScheduleConfig config;
try {
config = ScheduleConfig.from(objectMapper, policy.trigger().options());
} catch (IllegalArgumentException e) {
log.warn("Scheduled policy {} is misconfigured: {}", policy.id(), e.getMessage());
continue;
}
// First time we see a policy, baseline its last-fire to now so it does not fire
// immediately; subsequent sweeps fire it once its next firing has passed.
Instant last = lastFiredByPolicy.computeIfAbsent(policy.id(), id -> now);
ZonedDateTime next = config.schedule().nextAfter(last.atZone(config.zone()));
if (!next.toInstant().isAfter(now)) {
lastFiredByPolicy.put(policy.id(), now);
log.info("Scheduled policy {} ({}) is due", policy.id(), policy.name());
policyRunner.run(policy);
}
}
}
/**
* The typed, validated form of a schedule trigger's options: the {@link Schedule} and the zone
* its wall-clock kinds are evaluated in (default UTC). Construction fails for a missing/invalid
* schedule or zone.
*/
record ScheduleConfig(Schedule schedule, ZoneId zone) {
private static final String SCHEDULE_OPTION = "schedule";
private static final String ZONE_OPTION = "zone";
static ScheduleConfig from(ObjectMapper mapper, Map<String, Object> options) {
Object scheduleNode = options.get(SCHEDULE_OPTION);
if (scheduleNode == null) {
throw new IllegalArgumentException("schedule trigger requires a 'schedule'");
}
Schedule schedule;
try {
schedule = mapper.convertValue(scheduleNode, Schedule.class);
} catch (RuntimeException e) {
throw new IllegalArgumentException("invalid schedule: " + rootMessage(e), e);
}
ZoneId zone = ZoneOffset.UTC;
Object zoneNode = options.get(ZONE_OPTION);
if (zoneNode != null && !zoneNode.toString().isBlank()) {
try {
zone = ZoneId.of(zoneNode.toString());
} catch (RuntimeException e) {
throw new IllegalArgumentException("invalid zone '" + zoneNode + "'");
}
}
return new ScheduleConfig(schedule, zone);
}
private static String rootMessage(Throwable t) {
Throwable cause = t;
while (cause.getCause() != null) {
cause = cause.getCause();
}
return cause.getMessage();
}
}
}
app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java
+74 -10
View File
@@ -66,6 +66,7 @@ import tools.jackson.databind.ObjectMapper;
public class AiWorkflowService {
private static final String DOCUMENTS_ENDPOINT = "/api/v1/documents";
private static final String PDF_TO_MARKDOWN_ENDPOINT = "/api/v1/convert/pdf/markdown";
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final AiEngineClient aiEngineClient;
@@ -194,6 +195,7 @@ public class AiWorkflowService {
return switch (response.getOutcome()) {
case NEED_CONTENT -> onNeedContent(response, filesById, request, listener);
case NEED_INGEST -> onNeedIngest(response, filesById, request, listener);
case CONVERT_MARKDOWN -> onConvertMarkdown(response, filesById, listener);
case TOOL_CALL -> onToolCall(response, filesById, listener);
case PLAN -> onPlan(response, filesById, request, listener);
case ANSWER -> onAnswer(response, filesById, request, listener);
@@ -330,6 +332,77 @@ public class AiWorkflowService {
return new WorkflowState.Pending(nextRequest);
}
/**
* Deterministically convert each requested PDF to Markdown via the {@code
* /convert/pdf/markdown} endpoint (backed by {@code PdfMarkdownConverter}) and return the
* {@code .md} file(s) as a completed result. No AI resume — the conversion output is the final
* answer.
*/
private WorkflowState onConvertMarkdown(
AiWorkflowResponse response,
Map<String, MultipartFile> filesById,
ProgressListener listener) {
List<AiFile> filesToConvert = response.getFilesToIngest();
if (filesToConvert == null || filesToConvert.isEmpty()) {
return new WorkflowState.Terminal(
cannotContinue(
"AI engine requested markdown conversion without listing any files."));
}
try {
List<Resource> resultFiles = new ArrayList<>();
List<String> inputNames = new ArrayList<>();
for (int i = 0; i < filesToConvert.size(); i++) {
AiFile file = filesToConvert.get(i);
MultipartFile multipartFile = filesById.get(file.getId());
if (multipartFile == null) {
return new WorkflowState.Terminal(
cannotContinue(
"AI engine requested markdown conversion for unknown file: "
+ file.getName()));
}
listener.onProgress(
AiWorkflowProgressEvent.executingTool(
PDF_TO_MARKDOWN_ENDPOINT, i + 1, filesToConvert.size()));
Resource input = toResource(multipartFile);
PipelineDefinition definition =
new PipelineDefinition(
"convert-markdown",
List.of(new PipelineStep(PDF_TO_MARKDOWN_ENDPOINT, Map.of())),
null);
PolicyExecutionResult result =
policyExecutor.execute(
definition,
PolicyInputs.of(List.of(input)),
PolicyProgressListener.NOOP);
resultFiles.addAll(result.files());
inputNames.add(multipartFile.getOriginalFilename());
}
return new WorkflowState.Terminal(
buildCompletedResponse(null, resultFiles, inputNames, null));
} catch (InternalApiTimeoutException e) {
log.error("PDF to Markdown conversion timed out: {}", e.getMessage());
return new WorkflowState.Terminal(
cannotContinue(toolTimeoutMessage(PDF_TO_MARKDOWN_ENDPOINT, e)));
} catch (Exception e) {
log.error("Failed to convert PDF to Markdown: {}", e.getMessage(), e);
return new WorkflowState.Terminal(
cannotContinue(toolFailureMessage(PDF_TO_MARKDOWN_ENDPOINT, e)));
}
}
private Resource toResource(MultipartFile file) throws IOException {
TempFile tempFile = tempFileManager.createManagedTempFile("ai-workflow");
file.transferTo(tempFile.getPath());
final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename());
return new FileSystemResource(tempFile.getFile()) {
@Override
public String getFilename() {
return originalName;
}
};
}
private void ingestFile(AiFile file, MultipartFile multipartFile) throws IOException {
List<AiPageText> pages = new ArrayList<>();
try (PDDocument document = pdfDocumentFactory.load(multipartFile, true)) {
@@ -551,16 +624,7 @@ public class AiWorkflowService {
private List<Resource> toResources(Map<String, MultipartFile> filesById) throws IOException {
List<Resource> resources = new ArrayList<>();
for (MultipartFile file : filesById.values()) {
TempFile tempFile = tempFileManager.createManagedTempFile("ai-workflow");
file.transferTo(tempFile.getPath());
final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename());
resources.add(
new FileSystemResource(tempFile.getFile()) {
@Override
public String getFilename() {
return originalName;
}
});
resources.add(toResource(file));
}
return resources;
}
app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java
-78
View File
@@ -30,11 +30,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.SPDF.pdf.parser.PageImageLocator;
import stirling.software.SPDF.pdf.parser.PdfIngester;
import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage;
import stirling.software.SPDF.pdf.parser.PdfModels.RawLine;
import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment;
import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment;
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
import stirling.software.common.util.ExceptionUtils;
import stirling.software.common.util.PdfUtils;
@@ -50,7 +46,6 @@ import stirling.software.proprietary.model.api.ai.FolioType;
public class PdfContentExtractor {
private final TabulaTableParser tabulaTableParser;
private final PdfIngester pdfIngester;
private static final int MAX_CHARACTERS_PER_PAGE = 4_000;
@@ -196,8 +191,6 @@ public class PdfContentExtractor {
case PAGE_TEXT, FULL_TEXT ->
Optional.<PdfContentResult>ofNullable(
extractText(lf, fileReq, remainingPages, remainingCharacters));
case PAGE_LAYOUT ->
Optional.<PdfContentResult>ofNullable(extractPageLayout(lf, remainingPages));
default -> {
log.warn(
"Content type {} not yet implemented, skipping for {}",
@@ -222,35 +215,6 @@ public class PdfContentExtractor {
return extracted.isEmpty() ? null : buildExtractedFileText(lf.fileName(), extracted);
}
private PageLayoutFileResult extractPageLayout(LoadedFile lf, int maxPages) throws IOException {
List<ParsedPage> parsedPages = pdfIngester.parse(lf.document(), maxPages);
List<LayoutPage> pages = new ArrayList<>();
for (ParsedPage pp : parsedPages) {
if (pp.layoutLines().isEmpty()) continue;
List<LayoutLine> lines = new ArrayList<>();
for (RawLine rawLine : pp.layoutLines()) {
List<LayoutFragment> fragments = new ArrayList<>();
for (TextFragment tf : rawLine.fragments()) {
fragments.add(
new LayoutFragment(
tf.text(),
tf.bounds().x(),
tf.bounds().y(),
tf.bounds().width(),
tf.fontSize(),
tf.bold()));
}
lines.add(new LayoutLine(rawLine.bounds().y(), fragments));
}
pages.add(new LayoutPage(pp.pageNumber(), lines));
}
if (pages.isEmpty()) return null;
PageLayoutFileResult result = new PageLayoutFileResult();
result.setFileName(lf.fileName());
result.setPages(pages);
return result;
}
private WorkflowArtifact buildArtifact(ArtifactKind kind, List<PdfContentResult> results) {
return switch (kind) {
case EXTRACTED_TEXT -> {
@@ -258,11 +222,6 @@ public class PdfContentExtractor {
artifact.setFiles(results.stream().map(ExtractedFileText.class::cast).toList());
yield artifact;
}
case PAGE_LAYOUT -> {
PageLayoutArtifact artifact = new PageLayoutArtifact();
artifact.setFiles(results.stream().map(PageLayoutFileResult.class::cast).toList());
yield artifact;
}
case TOOL_REPORT ->
throw new IllegalArgumentException(
"TOOL_REPORT artifacts are not produced by PdfContentExtractor");
@@ -569,7 +528,6 @@ public class PdfContentExtractor {
*/
enum ArtifactKind {
EXTRACTED_TEXT("extracted_text"),
PAGE_LAYOUT("page_layout"),
TOOL_REPORT("tool_report");
private final String value;
@@ -633,40 +591,4 @@ public class PdfContentExtractor {
this.report = report;
}
}
// Serialization contract with the Python engine — see PageLayoutArtifactContractTest.
/** One text fragment with its bounding-box geometry and font properties. */
record LayoutFragment(
String text, float x, float y, float width, float fontSize, boolean bold) {}
/** A visual line on the page: y-coordinate and all fragments on that line. */
record LayoutLine(float y, List<LayoutFragment> fragments) {}
/** All layout lines for a single page. */
record LayoutPage(int pageNumber, List<LayoutLine> lines) {}
/** Page layout data for one file, as a PdfContentResult. */
@Data
static final class PageLayoutFileResult implements PdfContentResult {
private String fileName;
private List<LayoutPage> pages = new ArrayList<>();
@Override
public ArtifactKind getArtifactKind() {
return ArtifactKind.PAGE_LAYOUT;
}
@Override
public int pagesConsumed() {
return pages.size();
}
}
/** Artifact carrying full spatial page layout for all input files. */
@Data
static final class PageLayoutArtifact implements WorkflowArtifact {
private final ArtifactKind kind = ArtifactKind.PAGE_LAYOUT;
private List<PageLayoutFileResult> files = new ArrayList<>();
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/config/FolderAccessGuardTest.java
+99
View File
@@ -0,0 +1,99 @@
package stirling.software.proprietary.policy.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Path;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.core.env.StandardEnvironment;
import stirling.software.common.configuration.InstallationPathConfig;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.Policy;
/**
* Tests for {@link FolderAccessGuard}: folder access is fail-closed, confined to the configured
* allowed roots, never reaches Stirling's own config directory, and is off entirely under SaaS.
*/
class FolderAccessGuardTest {
@TempDir Path tempDir;
private FolderAccessGuard guard(List<String> allowedRoots, String... activeProfiles) {
ApplicationProperties properties = new ApplicationProperties();
properties.getPolicies().setAllowedFolderRoots(allowedRoots);
StandardEnvironment environment = new StandardEnvironment();
environment.setActiveProfiles(activeProfiles);
return new FolderAccessGuard(properties, environment);
}
@Test
void permitsAndNormalisesADirectoryWithinAnAllowedRoot() {
FolderAccessGuard guard = guard(List.of(tempDir.toString()));
Path within = tempDir.resolve("inbox");
assertEquals(within.toAbsolutePath().normalize(), guard.requirePermitted(within));
}
@Test
void rejectsADirectoryOutsideEveryAllowedRoot() {
FolderAccessGuard guard = guard(List.of(tempDir.toString()));
assertThrows(
IllegalArgumentException.class,
() -> guard.requirePermitted(tempDir.resolveSibling("elsewhere")));
}
@Test
void rejectsTraversalThatWalksOutOfAnAllowedRoot() {
FolderAccessGuard guard = guard(List.of(tempDir.toString()));
assertThrows(
IllegalArgumentException.class,
() -> guard.requirePermitted(tempDir.resolve("..").resolve("escaped")));
}
@Test
void rejectsEverythingWhenNoRootsAreConfigured() {
FolderAccessGuard guard = guard(List.of());
assertThrows(IllegalArgumentException.class, () -> guard.requirePermitted(tempDir));
}
@Test
void rejectsTheStirlingConfigDirectoryEvenWhenItWouldBeInsideAnAllowedRoot() {
Path configDir =
Path.of(InstallationPathConfig.getConfigPath()).toAbsolutePath().normalize();
// Allow the config dir's parent, so only the protected-path rule can reject it.
FolderAccessGuard guard = guard(List.of(configDir.getParent().toString()));
assertThrows(
IllegalArgumentException.class,
() -> guard.requirePermitted(configDir.resolve("settings.yml")));
}
@Test
void refusesAllFolderAccessUnderTheSaasProfile() {
FolderAccessGuard guard = guard(List.of(tempDir.toString()), "saas");
assertThrows(IllegalArgumentException.class, () -> guard.requirePermitted(tempDir));
}
@Test
void usesFolderAccessDetectsFolderSourcesAndOutputs() {
FolderAccessGuard guard = guard(List.of(tempDir.toString()));
assertTrue(
guard.usesFolderAccess(
policy(List.of(InputSpec.folder("/in")), OutputSpec.inline())));
assertTrue(guard.usesFolderAccess(policy(List.of(), OutputSpec.folder("/out"))));
assertFalse(guard.usesFolderAccess(policy(List.of(), OutputSpec.inline())));
}
private static Policy policy(List<InputSpec> sources, OutputSpec output) {
return new Policy("p1", "p", "owner", true, null, sources, List.of(), output);
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java
+2 -3
View File
@@ -50,7 +50,6 @@ import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.output.InlineOutputSink;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
@@ -93,7 +92,7 @@ class PolicyEngineTest {
toolMetadataService,
tempFileManager,
JsonMapper.builder().build());
registry = new PolicyRunRegistry(30);
registry = new PolicyRunRegistry(new ApplicationProperties());
InlineOutputSink sink = new InlineOutputSink(fileStorage);
engine =
new PolicyEngine(
@@ -190,7 +189,7 @@ class PolicyEngineTest {
"rotate",
"owner",
true,
TriggerConfig.manual(),
null,
List.of(new PipelineStep(ROTATE, Map.of())),
OutputSpec.inline());
app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java
+2 -1
View File
@@ -12,6 +12,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.model.PipelineDefinition;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.WaitState;
@@ -25,7 +26,7 @@ class PolicyRunRegistryTest {
@BeforeEach
void setUp() {
registry = new PolicyRunRegistry(30);
registry = new PolicyRunRegistry(new ApplicationProperties());
}
@AfterEach
app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java
+159
View File
@@ -0,0 +1,159 @@
package stirling.software.proprietary.policy.engine;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.input.ResolvedInput;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.PolicyInputs;
import stirling.software.proprietary.policy.model.PolicyRun;
import stirling.software.proprietary.policy.model.PolicyRunStatus;
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
/**
* Tests for {@link PolicyRunner}: the one place that turns a policy's sources into runs. Verifies
* it pulls every source, runs one job per unit of work, feeds each unit's completion hook the run
* outcome, and that a generator (no sources) still runs once.
*/
@ExtendWith(MockitoExtension.class)
class PolicyRunnerTest {
@Mock private PolicyEngine policyEngine;
@Mock private InputSource folderSource;
private PolicyRunner runner;
@BeforeEach
void setUp() {
runner = new PolicyRunner(policyEngine, List.of(folderSource));
}
@Test
void runsOnceWithNoFilesWhenThePolicyHasNoSources() {
Policy policy = policy(List.of());
when(policyEngine.runPolicy(eq(policy), any(), any()))
.thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
runner.run(policy);
ArgumentCaptor<PolicyInputs> inputs = ArgumentCaptor.forClass(PolicyInputs.class);
verify(policyEngine).runPolicy(eq(policy), inputs.capture(), any());
assertTrue(inputs.getValue().primary().isEmpty());
}
@Test
void pullsEverySourceAndRunsOnePerUnitOfWork() throws Exception {
InputSpec spec = InputSpec.folder("/in");
Policy policy = policy(List.of(spec));
when(folderSource.supports(spec)).thenReturn(true);
when(folderSource.resolve(spec))
.thenReturn(
List.of(
ResolvedInput.of(PolicyInputs.of(List.of())),
ResolvedInput.of(PolicyInputs.of(List.of()))));
when(policyEngine.runPolicy(any(), any(), any()))
.thenReturn(new PolicyRunHandle("r", new CompletableFuture<>()));
runner.run(policy);
verify(policyEngine, times(2)).runPolicy(eq(policy), any(), any());
}
@Test
void feedsEachUnitsCompletionHookTheRunOutcome() throws Exception {
InputSpec spec = InputSpec.folder("/in");
Policy policy = policy(List.of(spec));
AtomicBoolean outcome = new AtomicBoolean(false);
ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set);
when(folderSource.supports(spec)).thenReturn(true);
when(folderSource.resolve(spec)).thenReturn(List.of(unit));
CompletableFuture<PolicyRun> completion = new CompletableFuture<>();
when(policyEngine.runPolicy(any(), any(), any()))
.thenReturn(new PolicyRunHandle("r", completion));
runner.run(policy);
PolicyRun run = mock(PolicyRun.class);
when(run.getStatus()).thenReturn(PolicyRunStatus.COMPLETED);
completion.complete(run);
assertTrue(outcome.get());
}
@Test
void reportsFailureToTheCompletionHookWhenTheRunDoesNotComplete() throws Exception {
InputSpec spec = InputSpec.folder("/in");
Policy policy = policy(List.of(spec));
AtomicBoolean outcome = new AtomicBoolean(true);
ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set);
when(folderSource.supports(spec)).thenReturn(true);
when(folderSource.resolve(spec)).thenReturn(List.of(unit));
CompletableFuture<PolicyRun> completion = new CompletableFuture<>();
when(policyEngine.runPolicy(any(), any(), any()))
.thenReturn(new PolicyRunHandle("r", completion));
runner.run(policy);
completion.completeExceptionally(new RuntimeException("boom"));
assertFalse(outcome.get());
}
@Test
void skipsSourcesWithNoMatchingBean() {
InputSpec spec = new InputSpec("s3", Map.of());
Policy policy = policy(List.of(spec));
when(folderSource.supports(spec)).thenReturn(false);
runner.run(policy);
verifyNoInteractions(policyEngine);
}
@Test
void runWithSuppliedInputsBypassesSources() {
Policy policy = policy(List.of(InputSpec.folder("/in")));
PolicyInputs inputs = PolicyInputs.of(List.of());
PolicyRunHandle handle = new PolicyRunHandle("r", new CompletableFuture<>());
when(policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP))
.thenReturn(handle);
assertSame(handle, runner.runWith(policy, inputs, PolicyProgressListener.NOOP));
verifyNoInteractions(folderSource);
}
private static Policy policy(List<InputSpec> sources) {
return new Policy(
"p1",
"p",
"owner",
true,
null,
sources,
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyValidatorTest.java
+114
View File
@@ -0,0 +1,114 @@
package stirling.software.proprietary.policy.engine;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.output.PolicyOutputSink;
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
/** Tests for {@link PolicyValidator}: routes each facet to its handler and surfaces failures. */
@ExtendWith(MockitoExtension.class)
class PolicyValidatorTest {
@Mock private PolicyTrigger trigger;
@Mock private InputSource inputSource;
@Mock private PolicyOutputSink outputSink;
private PolicyValidator validator;
@BeforeEach
void setUp() {
validator =
new PolicyValidator(List.of(trigger), List.of(inputSource), List.of(outputSink));
}
@Test
void delegatesEachFacetToItsHandler() {
when(trigger.type()).thenReturn("schedule");
when(inputSource.supports(any())).thenReturn(true);
when(outputSink.supports(any())).thenReturn(true);
Policy policy = policy("schedule");
validator.validate(policy);
verify(trigger).validate(policy);
verify(inputSource).validate(policy.sources().get(0));
verify(outputSink).validate(policy.output());
}
@Test
void skipsTriggerValidationForAManualOnlyPolicy() {
when(inputSource.supports(any())).thenReturn(true);
when(outputSink.supports(any())).thenReturn(true);
validator.validate(manualOnly());
verify(trigger, never()).validate(any());
}
@Test
void surfacesAnInvalidConfigFromAHandler() {
when(trigger.type()).thenReturn("schedule");
doThrow(new IllegalArgumentException("invalid schedule")).when(trigger).validate(any());
IllegalArgumentException ex =
assertThrows(
IllegalArgumentException.class,
() -> validator.validate(policy("schedule")));
assertTrue(ex.getMessage().contains("schedule"));
}
@Test
void rejectsAnUnknownTriggerType() {
when(trigger.type()).thenReturn("schedule");
IllegalArgumentException ex =
assertThrows(
IllegalArgumentException.class,
() -> validator.validate(policy("mystery")));
assertTrue(ex.getMessage().contains("unknown trigger type"));
}
private static Policy policy(String triggerType) {
return new Policy(
"p1",
"p",
"owner",
true,
new TriggerConfig(triggerType, Map.of()),
List.of(InputSpec.folder("/in")),
List.of(),
OutputSpec.inline());
}
private static Policy manualOnly() {
return new Policy(
"p1",
"p",
"owner",
true,
null,
List.of(InputSpec.folder("/in")),
List.of(),
OutputSpec.inline());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java
+137
View File
@@ -0,0 +1,137 @@
package stirling.software.proprietary.policy.input;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.core.env.StandardEnvironment;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.util.FileReadinessChecker;
import stirling.software.proprietary.policy.config.FolderAccessGuard;
import stirling.software.proprietary.policy.model.InputSpec;
/** Tests for {@link FolderInputSource}: consume (claim + route) and snapshot (read-only) modes. */
@ExtendWith(MockitoExtension.class)
class FolderInputSourceTest {
@Mock private FileReadinessChecker readinessChecker;
@TempDir Path tempDir;
private FolderInputSource source;
@BeforeEach
void setUp() {
ApplicationProperties properties = new ApplicationProperties();
properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString()));
FolderAccessGuard guard = new FolderAccessGuard(properties, new StandardEnvironment());
source = new FolderInputSource(readinessChecker, guard);
// Lenient: the missing-dir / nonexistent-dir cases return before any readiness check.
lenient().when(readinessChecker.isReady(any())).thenReturn(true);
}
@Test
void consumeClaimsFilesAndRoutesToDoneOnSuccess() throws IOException {
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
Files.writeString(inputDir.resolve("doc.pdf"), "data");
List<ResolvedInput> work = source.resolve(InputSpec.folder(inputDir.toString()));
assertEquals(1, work.size());
assertEquals(1, work.get(0).inputs().primary().size());
// Claimed out of the input dir.
assertFalse(Files.exists(inputDir.resolve("doc.pdf")));
assertTrue(
Files.exists(
inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf")));
work.get(0).onComplete().accept(true);
assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("done").resolve("doc.pdf")));
assertFalse(
Files.exists(
inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf")));
}
@Test
void consumeRoutesToErrorOnFailure() throws IOException {
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
Files.writeString(inputDir.resolve("doc.pdf"), "data");
List<ResolvedInput> work = source.resolve(InputSpec.folder(inputDir.toString()));
work.get(0).onComplete().accept(false);
assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("error").resolve("doc.pdf")));
}
@Test
void snapshotReadsWithoutClaiming() throws IOException {
Path inputDir = Files.createDirectories(tempDir.resolve("in"));
Files.writeString(inputDir.resolve("doc.pdf"), "data");
List<ResolvedInput> work =
source.resolve(
new InputSpec(
"folder",
Map.of("directory", inputDir.toString(), "mode", "snapshot")));
assertEquals(1, work.size());
// Not moved, and completing the run is a no-op.
assertTrue(Files.exists(inputDir.resolve("doc.pdf")));
work.get(0).onComplete().accept(true);
assertTrue(Files.exists(inputDir.resolve("doc.pdf")));
}
@Test
void missingDirectoryOptionFails() {
assertThrows(
IllegalArgumentException.class,
() -> source.resolve(new InputSpec("folder", Map.of())));
}
@Test
void nonexistentDirectoryYieldsNoWork() throws IOException {
List<ResolvedInput> work =
source.resolve(InputSpec.folder(tempDir.resolve("nope").toString()));
assertTrue(work.isEmpty());
}
@Test
void validateRejectsMissingDirectory() {
assertThrows(
IllegalArgumentException.class,
() -> source.validate(new InputSpec("folder", Map.of())));
}
@Test
void rejectsADirectoryOutsideTheAllowedRoots() {
Path outside = tempDir.resolveSibling("not-allowed");
assertThrows(
IllegalArgumentException.class,
() -> source.resolve(InputSpec.folder(outside.toString())));
assertThrows(
IllegalArgumentException.class,
() -> source.validate(InputSpec.folder(outside.toString())));
}
@Test
void watchTargetsIsTheConfiguredDirectory() {
Path inputDir = tempDir.resolve("in");
assertEquals(List.of(inputDir), source.watchTargets(InputSpec.folder(inputDir.toString())));
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java
+105
View File
@@ -0,0 +1,105 @@
package stirling.software.proprietary.policy.output;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.core.env.StandardEnvironment;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.common.model.job.ResultFile;
import stirling.software.proprietary.policy.config.FolderAccessGuard;
import stirling.software.proprietary.policy.model.OutputSpec;
/** Tests for {@link FolderOutputSink}: outputs are written to the configured directory on disk. */
class FolderOutputSinkTest {
@TempDir Path tempDir;
private FolderOutputSink sink;
@BeforeEach
void setUp() {
ApplicationProperties properties = new ApplicationProperties();
properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString()));
sink = new FolderOutputSink(new FolderAccessGuard(properties, new StandardEnvironment()));
}
@Test
void writesEachOutputToTheDirectory() throws IOException {
Path out = tempDir.resolve("out");
List<Resource> outputs = List.of(named("a.pdf", "aaa"), named("b.pdf", "bb"));
List<ResultFile> results =
sink.deliver("run-1", outputs, OutputSpec.folder(out.toString()));
assertEquals(2, results.size());
assertTrue(Files.exists(out.resolve("a.pdf")));
assertEquals("aaa", Files.readString(out.resolve("a.pdf")));
assertEquals("bb", Files.readString(out.resolve("b.pdf")));
}
@Test
void collidingNamesGetAUniqueSuffix() throws IOException {
Path out = tempDir.resolve("out");
List<Resource> outputs = List.of(named("a.pdf", "first"), named("a.pdf", "second"));
sink.deliver("run-1", outputs, OutputSpec.folder(out.toString()));
assertTrue(Files.exists(out.resolve("a.pdf")));
assertTrue(Files.exists(out.resolve("a (1).pdf")));
}
@Test
void missingDirectoryOptionIsRejected() {
OutputSpec noDir = new OutputSpec("folder", Map.of());
assertThrows(IllegalArgumentException.class, () -> sink.validate(noDir));
assertThrows(
IllegalArgumentException.class,
() -> sink.deliver("run-1", List.of(named("a.pdf", "x")), noDir));
}
@Test
void aDirectoryOutsideTheAllowedRootsIsRejected() {
OutputSpec outside = OutputSpec.folder(tempDir.resolveSibling("not-allowed").toString());
assertThrows(IllegalArgumentException.class, () -> sink.validate(outside));
assertThrows(
IllegalArgumentException.class,
() -> sink.deliver("run-1", List.of(named("a.pdf", "x")), outside));
}
@Test
void filenamesWithPathTraversalAreConfinedToTheDirectory() throws IOException {
Path out = tempDir.resolve("out");
List<Resource> outputs =
List.of(named("../escape.pdf", "x"), named("nested/deep.pdf", "y"));
sink.deliver("run-1", outputs, OutputSpec.folder(out.toString()));
// Each name is reduced to its bare form inside the target dir; nothing escapes.
assertTrue(Files.exists(out.resolve("escape.pdf")));
assertTrue(Files.exists(out.resolve("deep.pdf")));
assertFalse(Files.exists(tempDir.resolve("escape.pdf")));
}
private static ByteArrayResource named(String filename, String content) {
return new ByteArrayResource(content.getBytes()) {
@Override
public String getFilename() {
return filename;
}
};
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java
+13 -10
View File
@@ -28,7 +28,7 @@ class InProcessPolicyStoreTest {
@Test
void savedPolicyGetsAnIdAndIsRetrievable() {
Policy saved = store.save(policy(null, "compress", "manual", true));
Policy saved = store.save(policy(null, "compress", null, true));
assertNotNull(saved.id());
assertFalse(saved.id().isBlank());
@@ -37,7 +37,7 @@ class InProcessPolicyStoreTest {
@Test
void savingWithAnExistingIdUpdatesInPlace() {
Policy created = store.save(policy(null, "before", "manual", true));
Policy created = store.save(policy(null, "before", null, true));
store.save(
new Policy(
@@ -45,7 +45,7 @@ class InProcessPolicyStoreTest {
"after",
"owner",
true,
TriggerConfig.manual(),
null,
List.of(),
OutputSpec.inline()));
@@ -55,19 +55,20 @@ class InProcessPolicyStoreTest {
@Test
void findByTriggerTypeReturnsOnlyEnabledMatches() {
store.save(policy(null, "watch", "folder", true));
store.save(policy(null, "watch-disabled", "folder", false));
store.save(policy(null, "nightly", "schedule", true));
store.save(policy(null, "nightly-disabled", "schedule", false));
store.save(policy(null, "hooked", "webhook", true));
store.save(policy(null, "on-demand", null, true)); // manual-only: no trigger
List<Policy> folder = store.findByTriggerType("folder");
List<Policy> scheduled = store.findByTriggerType("schedule");
assertEquals(1, folder.size());
assertEquals("watch", folder.get(0).name());
assertEquals(1, scheduled.size());
assertEquals("nightly", scheduled.get(0).name());
}
@Test
void deleteRemovesThePolicy() {
Policy saved = store.save(policy(null, "p", "manual", true));
Policy saved = store.save(policy(null, "p", null, true));
assertTrue(store.delete(saved.id()));
assertTrue(store.get(saved.id()).isEmpty());
@@ -75,12 +76,14 @@ class InProcessPolicyStoreTest {
}
private static Policy policy(String id, String name, String triggerType, boolean enabled) {
TriggerConfig trigger =
triggerType == null ? null : new TriggerConfig(triggerType, Map.of());
return new Policy(
id,
name,
"owner",
enabled,
new TriggerConfig(triggerType, Map.of()),
trigger,
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java
+11 -9
View File
@@ -18,6 +18,7 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
@@ -53,7 +54,8 @@ class JpaPolicyStoreTest {
"compress incoming",
"alice",
true,
new TriggerConfig("folder", Map.of("path", "/in")),
new TriggerConfig("schedule", Map.of()),
List.of(InputSpec.folder("/in")),
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline()));
@@ -62,7 +64,7 @@ class JpaPolicyStoreTest {
verify(repository).save(captor.capture());
PolicyEntity entity = captor.getValue();
assertEquals(saved.id(), entity.getId());
assertEquals("folder", entity.getTriggerType());
assertEquals("schedule", entity.getTriggerType());
assertTrue(entity.isEnabled());
// The stored JSON round-trips back to an equal policy.
assertEquals(saved, objectMapper.readValue(entity.getPolicyJson(), Policy.class));
@@ -76,7 +78,7 @@ class JpaPolicyStoreTest {
"rotate",
"alice",
true,
TriggerConfig.manual(),
null, // manual-only: no automatic trigger
List.of(
new PipelineStep(
"/api/v1/general/rotate-pdf", Map.of("angle", 90))),
@@ -94,16 +96,16 @@ class JpaPolicyStoreTest {
"watch",
"alice",
true,
new TriggerConfig("folder", Map.of()),
new TriggerConfig("schedule", Map.of()),
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
when(repository.findByTriggerTypeAndEnabledTrue("folder"))
when(repository.findByTriggerTypeAndEnabledTrue("schedule"))
.thenReturn(List.of(entityFor(policy)));
List<Policy> folder = store.findByTriggerType("folder");
List<Policy> scheduled = store.findByTriggerType("schedule");
assertEquals(1, folder.size());
assertEquals("p1", folder.get(0).id());
assertEquals(1, scheduled.size());
assertEquals("p1", scheduled.get(0).id());
}
@Test
@@ -122,7 +124,7 @@ class JpaPolicyStoreTest {
entity.setName(policy.name());
entity.setOwner(policy.owner());
entity.setEnabled(policy.enabled());
entity.setTriggerType(policy.trigger().type());
entity.setTriggerType(policy.trigger() == null ? null : policy.trigger().type());
entity.setPolicyJson(objectMapper.writeValueAsString(policy));
return entity;
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java
+178
View File
@@ -0,0 +1,178 @@
package stirling.software.proprietary.policy.trigger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.WatchService;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.input.InputSource;
import stirling.software.proprietary.policy.model.InputSpec;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.store.PolicyStore;
/**
* Tests for {@link FolderWatchTrigger}'s dispatch logic via the package-visible {@code
* runForChangedDirs}/{@code runAll}, plus its cross-facet validation. The OS watch loop and
* scheduled reconcile are thin glue around these and are not exercised here (a real {@code
* WatchService} is timing-dependent), mirroring how {@link ScheduleTriggerTest} drives {@code
* sweep} directly. The folder source is stubbed to mirror {@code FolderInputSource.watchTargets}.
*/
@ExtendWith(MockitoExtension.class)
class FolderWatchTriggerTest {
@Mock private PolicyStore policyStore;
@Mock private PolicyRunner policyRunner;
@Mock private InputSource folderSource;
@TempDir Path tempDir;
private FolderWatchTrigger trigger;
@BeforeEach
void setUp() {
trigger =
new FolderWatchTrigger(
policyStore,
policyRunner,
List.of(folderSource),
new ApplicationProperties());
lenient().when(folderSource.supports(any())).thenReturn(true);
lenient()
.when(folderSource.watchTargets(any()))
.thenAnswer(
invocation -> {
InputSpec spec = invocation.getArgument(0);
Object dir = spec.options().get("directory");
if (dir == null) {
throw new IllegalArgumentException(
"folder input requires a 'directory' option");
}
return List.of(Path.of(dir.toString()));
});
}
@Test
void validateRejectsPolicyWithNoWatchableSource() {
assertThrows(
IllegalArgumentException.class,
() -> trigger.validate(folderWatch("p1", List.of())));
}
@Test
void validateAcceptsPolicyWithAFolderSource() {
trigger.validate(folderWatch("p1", List.of(InputSpec.folder("/in"))));
}
@Test
void runsOnlyPoliciesDrawingFromTheChangedDirectory() {
Policy a = folderWatch("a", List.of(InputSpec.folder("/in/a")));
Policy b = folderWatch("b", List.of(InputSpec.folder("/in/b")));
when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b));
trigger.runForChangedDirs(Set.of(normalized("/in/a")));
verify(policyRunner).run(a);
verify(policyRunner, never()).run(b);
}
@Test
void skipsAMisconfiguredPolicyButStillRunsTheOthers() {
Policy bad = folderWatch("bad", List.of(new InputSpec("folder", Map.of())));
Policy good = folderWatch("good", List.of(InputSpec.folder("/in/a")));
when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(bad, good));
trigger.runForChangedDirs(Set.of(normalized("/in/a")));
verify(policyRunner).run(good);
verify(policyRunner, never()).run(bad);
}
@Test
void anEmptyChangeSetDoesNothing() {
trigger.runForChangedDirs(Set.of());
verifyNoInteractions(policyStore, policyRunner);
}
@Test
void reconcileRunsEveryFolderWatchPolicyAsASafetyNet() {
Policy a = folderWatch("a", List.of(InputSpec.folder("/in/a")));
Policy b = folderWatch("b", List.of(InputSpec.folder("/in/b")));
when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b));
trigger.runAll();
verify(policyRunner).run(a);
verify(policyRunner).run(b);
}
@Test
void syncRegistrationsWatchesExistingDirsAndCancelsRemovedOnes() throws Exception {
Path dirA = Files.createDirectories(tempDir.resolve("a"));
Path dirB = Files.createDirectories(tempDir.resolve("b"));
Path missing = tempDir.resolve("missing"); // never created on disk
Policy a = folderWatch("a", List.of(InputSpec.folder(dirA.toString())));
Policy b = folderWatch("b", List.of(InputSpec.folder(dirB.toString())));
Policy m = folderWatch("m", List.of(InputSpec.folder(missing.toString())));
WatchService service = FileSystems.getDefault().newWatchService();
try {
trigger.watchService = service;
when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b, m));
trigger.syncRegistrations();
// Existing dirs are watched; the non-existent one is skipped.
assertEquals(
Set.of(normalized(dirA.toString()), normalized(dirB.toString())),
trigger.watchedDirs());
// b's policy is removed: its registration is cancelled, a remains.
when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a));
trigger.syncRegistrations();
assertEquals(Set.of(normalized(dirA.toString())), trigger.watchedDirs());
} finally {
service.close();
}
}
private static Path normalized(String dir) {
return Path.of(dir).toAbsolutePath().normalize();
}
private static Policy folderWatch(String id, List<InputSpec> sources) {
return new Policy(
id,
"watcher",
"owner",
true,
new TriggerConfig("folder-watch", Map.of()),
sources,
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManagerTest.java
+51
View File
@@ -0,0 +1,51 @@
package stirling.software.proprietary.policy.trigger;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.verify;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
/**
* Tests for {@link PolicyTriggerManager}: starts/stops every trigger, tolerating individual
* failures.
*/
@ExtendWith(MockitoExtension.class)
class PolicyTriggerManagerTest {
@Mock private PolicyTrigger triggerA;
@Mock private PolicyTrigger triggerB;
@Test
void startsAndStopsAllTriggers() {
PolicyTriggerManager manager = new PolicyTriggerManager(List.of(triggerA, triggerB));
assertFalse(manager.isRunning());
manager.start();
verify(triggerA).start();
verify(triggerB).start();
assertTrue(manager.isRunning());
manager.stop();
verify(triggerA).stop();
verify(triggerB).stop();
assertFalse(manager.isRunning());
}
@Test
void oneTriggerFailingToStartDoesNotBlockTheOthers() {
doThrow(new RuntimeException("boom")).when(triggerA).start();
PolicyTriggerManager manager = new PolicyTriggerManager(List.of(triggerA, triggerB));
manager.start();
verify(triggerB).start();
assertTrue(manager.isRunning());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java
+147
View File
@@ -0,0 +1,147 @@
package stirling.software.proprietary.policy.trigger;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.time.DayOfWeek;
import java.time.Instant;
import java.time.LocalTime;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import stirling.software.common.model.ApplicationProperties;
import stirling.software.proprietary.policy.engine.PolicyRunner;
import stirling.software.proprietary.policy.model.OutputSpec;
import stirling.software.proprietary.policy.model.PipelineStep;
import stirling.software.proprietary.policy.model.Policy;
import stirling.software.proprietary.policy.model.Schedule;
import stirling.software.proprietary.policy.model.TriggerConfig;
import stirling.software.proprietary.policy.store.PolicyStore;
import tools.jackson.databind.json.JsonMapper;
/**
* Tests for {@link ScheduleTrigger}'s due-firing logic via the package-visible {@code
* sweep(Instant)}. The trigger only decides when a policy is due; pulling sources and starting runs
* is the {@link PolicyRunner}'s job, so these assert it delegates to the runner. Schedules default
* to UTC, so explicit UTC instants make these deterministic.
*/
@ExtendWith(MockitoExtension.class)
class ScheduleTriggerTest {
@Mock private PolicyStore policyStore;
@Mock private PolicyRunner policyRunner;
private ScheduleTrigger trigger;
@BeforeEach
void setUp() {
trigger =
new ScheduleTrigger(
policyStore,
policyRunner,
JsonMapper.builder().build(),
new ApplicationProperties());
}
@Test
void firesOncePerScheduleWhenItComesDue() {
Policy policy = scheduled("p1", new Schedule.Every(1, Schedule.Unit.MINUTES));
when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy));
Instant t0 = Instant.parse("2026-06-05T10:00:30Z");
trigger.sweep(t0); // first sight: baseline, must not fire immediately
verify(policyRunner, never()).run(any());
trigger.sweep(t0.plusSeconds(120)); // the one-minute mark has passed
verify(policyRunner, times(1)).run(eq(policy));
}
@Test
void doesNotFireBeforeTheNextScheduledTime() {
Policy policy = scheduled("p1", new Schedule.Daily(LocalTime.of(3, 0))); // 03:00 UTC daily
when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy));
Instant t0 = Instant.parse("2026-06-05T10:00:00Z");
trigger.sweep(t0);
trigger.sweep(t0.plusSeconds(60)); // next 03:00 is far away
verify(policyRunner, never()).run(any());
}
@Test
void firesWeeklyOnAChosenDay() {
// 2026-06-05 is a Friday; the next Monday 09:00 is the soonest firing.
Policy policy =
scheduled("p1", new Schedule.Weekly(Set.of(DayOfWeek.MONDAY), LocalTime.of(9, 0)));
when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy));
Instant friday = Instant.parse("2026-06-05T10:00:00Z");
trigger.sweep(friday); // baseline
trigger.sweep(Instant.parse("2026-06-08T09:00:00Z")); // Monday 09:00
verify(policyRunner, times(1)).run(eq(policy));
}
@Test
void skipsPoliciesWithAnInvalidSchedule() {
Policy policy = scheduledWithRawOptions("p1", Map.of()); // no schedule
when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy));
trigger.sweep(Instant.parse("2026-06-05T10:00:00Z"));
verify(policyRunner, never()).run(any());
}
@Test
void validateRejectsMissingSchedule() {
assertThrows(
IllegalArgumentException.class,
() -> trigger.validate(scheduledWithRawOptions("p1", Map.of())));
}
@Test
void validateRejectsAnInvalidSchedule() {
Map<String, Object> options =
Map.of("schedule", Map.of("type", "every", "count", -5, "unit", "MINUTES"));
assertThrows(
IllegalArgumentException.class,
() -> trigger.validate(scheduledWithRawOptions("p1", options)));
}
@Test
void validateAcceptsAValidScheduleAndZone() {
Map<String, Object> options = new LinkedHashMap<>();
options.put("schedule", new Schedule.Daily(LocalTime.of(2, 0)));
options.put("zone", "Europe/London");
trigger.validate(scheduledWithRawOptions("p1", options));
}
private static Policy scheduled(String id, Schedule schedule) {
return scheduledWithRawOptions(id, Map.of("schedule", schedule));
}
private static Policy scheduledWithRawOptions(String id, Map<String, Object> options) {
return new Policy(
id,
"nightly",
"owner",
true,
new TriggerConfig("schedule", options),
List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())),
OutputSpec.inline());
}
}
app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java
+29
View File
@@ -439,6 +439,35 @@ class AiWorkflowServiceTest {
verify(internalApiClient, never()).post(anyString(), any());
}
@Test
void convertMarkdownRunsDeterministicConversionAndReturnsMdFile() throws IOException {
MockMultipartFile input = pdf("multi-column-test_lorem.pdf", "pdf-bytes");
when(fileIdStrategy.idFor(any())).thenReturn("doc-1");
stubOrchestrator(
"""
{
"outcome":"convert_markdown",
"reason":"PDF to Markdown requested.",
"filesToIngest":[{"id":"doc-1","name":"multi-column-test_lorem.pdf"}]
}
""");
when(toolMetadataService.shouldUnpackZipResponse("/api/v1/convert/pdf/markdown"))
.thenReturn(false);
stubEndpoint(
"/api/v1/convert/pdf/markdown",
pdfResource("# Title", "multi-column-test_lorem.md"));
AtomicInteger ids = stubFileStorage();
AiWorkflowResponse result = service.orchestrate(requestFor(input, "convert to markdown"));
assertEquals(AiWorkflowOutcome.COMPLETED, result.getOutcome());
assertEquals(1, result.getResultFiles().size());
// Extension changes (pdf -> md), so the converter's response filename wins.
assertEquals("multi-column-test_lorem.md", result.getResultFiles().get(0).getFileName());
assertEquals(1, ids.get());
verify(internalApiClient, times(1)).post(eq("/api/v1/convert/pdf/markdown"), any());
}
@Test
void toolCallWithoutEndpointFallsBackToCannotContinue() throws IOException {
MockMultipartFile input = pdf("input.pdf", "bytes");
app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java
-66
View File
@@ -1,66 +0,0 @@
package stirling.software.proprietary.service;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import org.junit.jupiter.api.Test;
import stirling.software.proprietary.service.PdfContentExtractor.LayoutFragment;
import stirling.software.proprietary.service.PdfContentExtractor.LayoutLine;
import stirling.software.proprietary.service.PdfContentExtractor.LayoutPage;
import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutArtifact;
import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutFileResult;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.json.JsonMapper;
/**
* Contract test: verifies that {@link PageLayoutArtifact} serializes to the JSON field names that
* the Python engine expects in {@code engine/src/stirling/contracts/pdf_to_markdown.py}.
*
* <p>The companion Python test in {@code tests/test_pdf_to_markdown.py} deserializes the same JSON
* literal and asserts field values. If either side renames a field, one of these tests fails.
*/
class PageLayoutArtifactContractTest {
static final String CONTRACT_JSON =
"""
{"kind":"page_layout","files":[{"fileName":"test.pdf","pages":[{"pageNumber":1,"lines":[{"y":10.0,"fragments":[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}""";
@Test
void pageLayoutArtifact_serialisesToExpectedJson() throws Exception {
LayoutFragment fragment = new LayoutFragment("Hello", 1.0f, 2.0f, 30.0f, 12.0f, true);
LayoutLine line = new LayoutLine(10.0f, List.of(fragment));
LayoutPage page = new LayoutPage(1, List.of(line));
PageLayoutFileResult fileResult = new PageLayoutFileResult();
fileResult.setFileName("test.pdf");
fileResult.setPages(List.of(page));
PageLayoutArtifact artifact = new PageLayoutArtifact();
artifact.setFiles(List.of(fileResult));
JsonNode json = new JsonMapper().valueToTree(artifact);
assertEquals("page_layout", json.get("kind").asText());
JsonNode file = json.get("files").get(0);
assertEquals("test.pdf", file.get("fileName").asText());
JsonNode pg = file.get("pages").get(0);
assertEquals(1, pg.get("pageNumber").asInt());
JsonNode ln = pg.get("lines").get(0);
assertEquals(10.0, ln.get("y").asDouble(), 0.001);
JsonNode frag = ln.get("fragments").get(0);
assertEquals("Hello", frag.get("text").asText());
assertEquals(1.0, frag.get("x").asDouble(), 0.001);
assertEquals(2.0, frag.get("y").asDouble(), 0.001);
assertEquals(30.0, frag.get("width").asDouble(), 0.001);
assertEquals(12.0, frag.get("fontSize").asDouble(), 0.001);
assertTrue(frag.get("bold").asBoolean());
}
}
engine/src/stirling/agents/__init__.py
-2
View File
@@ -5,7 +5,6 @@ from .orchestrator import OrchestratorAgent
from .pdf_edit import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection
from .pdf_questions import PdfQuestionAgent
from .pdf_review import PdfReviewAgent
from .pdf_to_markdown import PdfToMarkdownAgent
from .user_spec import UserSpecAgent
__all__ = [
@@ -16,6 +15,5 @@ __all__ = [
"PdfEditPlanSelection",
"PdfQuestionAgent",
"PdfReviewAgent",
"PdfToMarkdownAgent",
"UserSpecAgent",
]
engine/src/stirling/agents/orchestrator.py
+15 -20
View File
@@ -11,14 +11,13 @@ from pydantic_ai.tools import RunContext
from stirling.agents.pdf_edit import PdfEditAgent
from stirling.agents.pdf_questions import PdfQuestionAgent
from stirling.agents.pdf_review import PdfReviewAgent
from stirling.agents.pdf_to_markdown import PdfToMarkdownAgent
from stirling.agents.user_spec import UserSpecAgent
from stirling.contracts import (
AgentDraftWorkflowResponse,
ConvertMarkdownResponse,
ExtractedTextArtifact,
OrchestratorRequest,
OrchestratorResponse,
PageLayoutArtifact,
PdfEditResponse,
PdfQuestionOrchestrateResponse,
PdfReviewOrchestrateResponse,
@@ -27,7 +26,6 @@ from stirling.contracts import (
format_conversation_history,
format_file_names,
)
from stirling.contracts.pdf_to_markdown import PdfToMarkdownOrchestrateResponse
from stirling.services import AppRuntime
logger = logging.getLogger(__name__)
@@ -72,9 +70,11 @@ class OrchestratorAgent:
),
),
ToolOutput(
self.delegate_pdf_to_markdown,
name="delegate_pdf_to_markdown",
description=("Delegate requests to reconstruct a PDF as a Markdown document."),
self.delegate_pdf_ingest,
name="delegate_pdf_ingest",
description=(
"Delegate requests to convert a PDF to Markdown or extract its content as readable text."
),
),
ToolOutput(
self.unsupported_capability,
@@ -92,8 +92,8 @@ class OrchestratorAgent:
"Use delegate_pdf_review when the user wants the PDF returned with review"
" comments attached — anything like 'review this', 'annotate with comments',"
" 'leave feedback on the PDF'. "
"Use delegate_pdf_to_markdown for any request to convert a PDF to Markdown "
"or reconstruct its content as readable text. "
"Use delegate_pdf_ingest for any request to convert a PDF to Markdown "
"or extract its content as readable text. "
"Use unsupported_capability when the user asks about the assistant itself "
"or when none of the other outputs fit; supply a helpful message."
),
@@ -133,13 +133,12 @@ class OrchestratorAgent:
return await self._run_pdf_edit(request)
case SupportedCapability.AGENT_DRAFT:
return await self._run_agent_draft(request)
case SupportedCapability.PDF_TO_MARKDOWN:
return await self._run_pdf_to_markdown(request)
case (
SupportedCapability.ORCHESTRATE
| SupportedCapability.AGENT_REVISE
| SupportedCapability.AGENT_NEXT_ACTION
| SupportedCapability.MATH_AUDITOR_AGENT
| SupportedCapability.PDF_TO_MARKDOWN
):
raise ValueError(f"Cannot resume orchestrator with capability: {capability}")
case _ as unreachable:
@@ -163,11 +162,12 @@ class OrchestratorAgent:
async def _run_agent_draft(self, request: OrchestratorRequest) -> AgentDraftWorkflowResponse:
return await UserSpecAgent(self.runtime).orchestrate(request)
async def delegate_pdf_to_markdown(self, ctx: RunContext[OrchestratorDeps]) -> PdfToMarkdownOrchestrateResponse:
return await self._run_pdf_to_markdown(ctx.deps.request)
async def _run_pdf_to_markdown(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse:
return await PdfToMarkdownAgent(self.runtime).orchestrate(request)
async def delegate_pdf_ingest(self, ctx: RunContext[OrchestratorDeps]) -> ConvertMarkdownResponse:
request = ctx.deps.request
return ConvertMarkdownResponse(
reason="PDF to Markdown requested — Java converts deterministically.",
files_to_ingest=request.files,
)
async def delegate_pdf_review(self, ctx: RunContext[OrchestratorDeps]) -> PdfReviewOrchestrateResponse:
return await self._run_pdf_review(ctx.deps.request)
@@ -204,10 +204,5 @@ class OrchestratorAgent:
file_names = [f.file_name for f in artifact.files]
descriptions.append(f"- extracted_text: {total_pages} pages from {file_names}")
continue
if isinstance(artifact, PageLayoutArtifact):
total_pages = sum(len(f.pages) for f in artifact.files)
file_names = [f.file_name for f in artifact.files]
descriptions.append(f"- page_layout: {total_pages} pages from {file_names}")
continue
descriptions.append("- unknown artifact")
return "\n".join(descriptions)
engine/src/stirling/agents/pdf_to_markdown/__init__.py
-3
View File
@@ -1,3 +0,0 @@
from .agent import PdfToMarkdownAgent
__all__ = ["PdfToMarkdownAgent"]
engine/src/stirling/agents/pdf_to_markdown/agent.py
-435
View File
@@ -1,435 +0,0 @@
"""PDF to Markdown Agent.
Converts a parsed PDF document into a single clean Markdown document, preserving
headings, paragraphs, and tables in reading order.
"""
from __future__ import annotations
import asyncio
import logging
import re
import time
from pydantic import BaseModel, Field
from pydantic_ai import Agent
from pydantic_ai.output import NativeOutput
from stirling.contracts import (
EditCannotDoResponse,
GenerateFileResponse,
NeedContentFileRequest,
NeedContentResponse,
OrchestratorRequest,
PdfContentType,
SupportedCapability,
format_conversation_history,
)
from stirling.contracts.pdf_to_markdown import (
PageLayout,
PageLayoutArtifact,
PdfToMarkdownCannotDoResponse,
PdfToMarkdownOrchestrateResponse,
PdfToMarkdownRequest,
PdfToMarkdownResponse,
PdfToMarkdownSuccessResponse,
)
from stirling.services import AppRuntime
logger = logging.getLogger(__name__)
# Warn when output tokens are close to the typical model output limit (~8192 for most
# configurations). The actual limit is model-specific; this threshold catches likely truncation.
_OUTPUT_TOKEN_TRUNCATION_THRESHOLD = 7500
# Chunking limits — keep each LLM call to a manageable payload size.
# Fragment count is the primary driver of JSON payload size (each fragment carries x/y/width/
# fontSize/bold metadata beyond its text). Page cap prevents low-text pages accumulating.
_MAX_CHUNK_FRAGMENTS = 1_000
_MAX_CHUNK_PAGES = 10
# Max concurrent LLM calls — limits API rate pressure on large documents.
_MAX_PARALLEL_CHUNKS = 3
# ── LLM output model ────────────────────────────────────────────────────────────────────────────
class _ReconstructionOutput(BaseModel):
markdown: str = Field(description="Full document reconstructed as clean Markdown.")
# ── Agent ────────────────────────────────────────────────────────────────────────────────────────
class PdfToMarkdownAgent:
def __init__(self, runtime: AppRuntime) -> None:
self.runtime = runtime
self._sem = asyncio.Semaphore(_MAX_PARALLEL_CHUNKS)
self._reconstruct_agent = Agent(
model=runtime.smart_model,
output_type=NativeOutput(_ReconstructionOutput),
system_prompt=(
"You reconstruct PDF pages into clean Markdown from spatial fragment data.\n"
"Input: PAGE LAYOUT — per-fragment x/y/font data for structural analysis.\n\n"
"COLUMN DETECTION (for tables in page_layout):\n"
"- Look at the x-positions of fragments across 3+ consecutive lines.\n"
"- If fragments cluster at the same x-positions across multiple lines, those are table columns.\n"
"- Each distinct x-cluster is one column."
" Name them from the header row (the first line in the cluster).\n"
"- Do NOT merge values from different x-columns into one cell.\n\n"
"ROW DETECTION:\n"
"- Each unique y-coordinate (or group within 3pt) is one table row.\n"
"- Every line of layout data is its own row — do not merge rows.\n"
"- If a column has no fragment on a given y-row, that cell is empty.\n\n"
"TABLE RENDERING:\n"
"- Render as: | col1 | col2 | col3 |\n"
" | --- | --- | --- |\n"
" | val | val | val |\n"
"- One source row = one table row. Never collapse multiple rows into one.\n"
"- Preserve numeric values exactly (no rounding, no formatting changes).\n"
"- Bold cells: wrap with ** in the Markdown cell.\n"
"- CRITICAL: the separator row `| --- | --- |` appears EXACTLY ONCE per table, immediately\n"
" after the header row. NEVER put `| --- |` after a data row or between data rows.\n"
" NEVER put a blank line inside a table. All rows (header + data) must be consecutive.\n"
"- Do NOT produce a header-only table followed by a second table with the data rows.\n"
" One logical table = one markdown table block, with header, one separator, then all data.\n\n"
"GROUP HEADERS (label-only rows inside a table):\n"
"- A row is a group header when: the first column has text AND every numeric column is empty.\n"
"- Do NOT render group headers as table rows with empty cells.\n"
"- Break the table, emit the label as **bold text** on its own line,"
" then start a new table for the rows that follow.\n"
"- Example labels: 'Policy functions', 'Non-current assets'.\n\n"
"TOTAL AND SUBTOTAL ROWS:\n"
"- Detect rows whose first cell contains (case-insensitive):"
" total, subtotal, surplus, balance, net, sum.\n"
"- These rows have numeric content — they are NOT group headers.\n"
"- Render the entire row in bold: | **Total income** | **1,234** | **5,678** |\n"
"- Keep total rows attached to the group they summarise.\n\n"
"MULTI-LEVEL TABLES (year or period as a row label):\n"
"- Detect when a row contains only a single label (a year like '2010' or period like 'Q1 2023')"
" with no numeric content, followed by repeated metric rows.\n"
"- Do NOT render the year as a table row.\n"
"- Normalise: add 'Year' as the first column, 'Metric' as the second,"
" and repeat the year value on each metric row.\n\n"
"PROSE REGIONS:\n"
"- Lines where x-positions vary across lines (not repeating columns) are prose.\n"
"- Merge lines at the same x-level into paragraphs. Separate indented lines.\n\n"
"HEADINGS:\n"
"- A line is a heading when it is bold OR font_size ≥2pt above body.\n"
" CRITICAL EXCEPTION: a bold fragment is a TABLE HEADER CELL, not a document heading, when\n"
" the same y-row in page_layout contains other fragments at different x-positions.\n"
" Only classify a bold line as a document heading when it is the SOLE fragment on its y-row.\n"
" Example: 'Non-current assets' at y=120 with '2010'@x=350, '2009'@x=420, '2008'@x=490\n"
" → this is a table header row, NOT a heading. Render it as the first cell of the table.\n"
"- Use ## for section headings, ### for sub-headings. Use # only for the document title.\n\n"
"ORDERING:\n"
"- Process content top-to-bottom as it appears on the page.\n"
"- Interleave prose blocks and table blocks in page order.\n"
"- Do not move text that appears before a table to after it, or vice versa.\n\n"
"FIDELITY:\n"
"- Do NOT invent, summarise, or omit any content.\n"
"- Do NOT add commentary, metadata, or JSON — output Markdown only."
),
model_settings={
**runtime.smart_model_settings,
"temperature": 0.0,
"max_tokens": _OUTPUT_TOKEN_TRUNCATION_THRESHOLD,
},
)
async def orchestrate(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse:
"""Entry point for the orchestrator delegate.
First turn: requests PAGE_LAYOUT extraction from Java via NeedContentResponse.
Resume turn: runs the LLM reconstruction and returns a write-file plan step.
"""
layout_artifact = next(
(a for a in request.artifacts if isinstance(a, PageLayoutArtifact)),
None,
)
if layout_artifact is None:
return NeedContentResponse(
resume_with=SupportedCapability.PDF_TO_MARKDOWN,
reason="Page layout data is required to reconstruct the document.",
files=[
NeedContentFileRequest(file=f, content_types=[PdfContentType.PAGE_LAYOUT]) for f in request.files
],
max_pages=self.runtime.settings.max_pages,
max_characters=self.runtime.settings.max_characters,
)
page_layout = [page for entry in layout_artifact.files for page in entry.pages]
file_names = [f.name for f in request.files]
result = await self.handle(
PdfToMarkdownRequest(
user_message=request.user_message,
file_names=file_names,
conversation_history=request.conversation_history,
page_layout=page_layout,
)
)
if isinstance(result, PdfToMarkdownCannotDoResponse):
return EditCannotDoResponse(reason=result.reason)
base = file_names[0].rsplit(".", 1)[0] if file_names else "document"
return GenerateFileResponse(
content=result.markdown,
filename=f"{base}-reconstruction.md",
summary="Reconstructed the document as a Markdown file.",
)
async def handle(self, request: PdfToMarkdownRequest) -> PdfToMarkdownResponse:
total_fragments = sum(len(line.fragments) for page in request.page_layout for line in page.lines)
logger.info(
"[pdf-to-markdown] received layout-pages=%d fragments=%d",
len(request.page_layout),
total_fragments,
)
if not request.page_layout:
logger.warning("[pdf-to-markdown] no content extracted from document; returning cannot_do")
return PdfToMarkdownCannotDoResponse(
reason=(
"No content was extracted from the document. "
"The file may be a scanned image PDF with no readable text. "
"Try running OCR on the document first."
)
)
chunks = _build_page_chunks(request.page_layout)
logger.info("[pdf-to-markdown] chunks=%d (max %d in parallel)", len(chunks), _MAX_PARALLEL_CHUNKS)
if len(chunks) == 1:
return await self._reconstruct_chunk(request, chunks[0], chunk_num=1, total_chunks=1)
total = len(chunks)
results = await asyncio.gather(
*(
self._reconstruct_chunk(request, chunk, chunk_num=i + 1, total_chunks=total)
for i, chunk in enumerate(chunks)
)
)
markdown_parts: list[str] = []
for result in results:
if isinstance(result, PdfToMarkdownSuccessResponse) and result.markdown:
markdown_parts.append(result.markdown)
elif isinstance(result, PdfToMarkdownCannotDoResponse):
logger.warning("[pdf-to-markdown] chunk dropped: %s", result.reason)
if not markdown_parts:
return PdfToMarkdownCannotDoResponse(reason="The document could not be reconstructed. All chunks failed.")
logger.info("[pdf-to-markdown] assembly: %d/%d chunks produced output", len(markdown_parts), len(chunks))
return PdfToMarkdownSuccessResponse(markdown="\n\n".join(markdown_parts))
async def _reconstruct_chunk(
self,
request: PdfToMarkdownRequest,
pages: list[PageLayout],
chunk_num: int,
total_chunks: int,
) -> PdfToMarkdownResponse:
chunk_request = PdfToMarkdownRequest(
user_message=request.user_message,
file_names=request.file_names,
conversation_history=request.conversation_history,
page_layout=pages,
)
try:
async with self._sem:
return await self._reconstruct_document(chunk_request, chunk_num, total_chunks)
except Exception as e:
logger.error("[pdf-to-markdown] chunk %d/%d failed: %s", chunk_num, total_chunks, e, exc_info=True)
return PdfToMarkdownCannotDoResponse(
reason="The document could not be reconstructed. The AI model failed to process it."
)
async def _reconstruct_document(
self, request: PdfToMarkdownRequest, chunk_num: int = 1, total_chunks: int = 1
) -> PdfToMarkdownSuccessResponse:
content = _build_reconstruction_prompt(request)
logger.info("[timing] chunk %d/%d llm-call prompt-chars=%d", chunk_num, total_chunks, len(content))
t0 = time.monotonic()
result = await self._reconstruct_agent.run([content])
llm_ms = int((time.monotonic() - t0) * 1000)
output: _ReconstructionOutput = result.output
usage = result.usage()
logger.info(
"[timing] chunk %d/%d llm-done ms=%d input-tokens=%s output-tokens=%s markdown-chars=%d",
chunk_num,
total_chunks,
llm_ms,
usage.input_tokens,
usage.output_tokens,
len(output.markdown),
)
if usage.output_tokens and usage.output_tokens >= _OUTPUT_TOKEN_TRUNCATION_THRESHOLD:
logger.warning(
"[timing] chunk %d/%d output likely truncated (output-tokens=%d)",
chunk_num,
total_chunks,
usage.output_tokens,
)
markdown = _remove_extra_separators(_fix_markdown_tables(_merge_orphaned_table_rows(output.markdown)))
return PdfToMarkdownSuccessResponse(markdown=markdown)
# ── Chunking ────────────────────────────────────────────────────────────────────────────────────
def _build_page_chunks(pages: list[PageLayout]) -> list[list[PageLayout]]:
chunks: list[list[PageLayout]] = []
current: list[PageLayout] = []
current_fragments = 0
for page in pages:
page_fragments = sum(len(line.fragments) for line in page.lines)
fragment_full = current and current_fragments + page_fragments > _MAX_CHUNK_FRAGMENTS
page_full = len(current) >= _MAX_CHUNK_PAGES
if fragment_full or page_full:
chunks.append(current)
current = []
current_fragments = 0
current.append(page)
current_fragments += page_fragments
if current:
chunks.append(current)
return chunks
# ── Prompt builders (module-level, no state) ────────────────────────────────────────────────────
def _build_reconstruction_prompt(request: PdfToMarkdownRequest) -> str:
history = format_conversation_history(request.conversation_history)
file_names = ", ".join(request.file_names) if request.file_names else "Unknown files"
layout_section = _format_layout(request.page_layout)
return (
f"Files: {file_names}\n\n"
f"User request: {request.user_message}\n\n"
f"Conversation history:\n{history}\n\n"
"PAGE LAYOUT (structural source — x/y fragment positions):\n"
"Each line is: y=NNN | text@(x,y) fs=N text@(x,y) fs=N ...\n"
"- y=NNN is the vertical position (row). Lines close in y are the same visual row.\n"
"- x=NNN is the horizontal position (column). Consistent x across rows = a column.\n"
"- fs=N is font size. Larger = likely a heading.\n"
"- **bold** markers indicate bold text.\n\n"
f"{layout_section}"
)
# ── LLM output post-processing ──────────────────────────────────────────────────────────────────
def _fix_markdown_tables(markdown: str) -> str:
"""Remove blank lines between table rows produced by the LLM."""
lines = markdown.split("\n")
result: list[str] = []
i = 0
while i < len(lines):
result.append(lines[i])
if lines[i].strip().startswith("|"):
j = i + 1
while j < len(lines) and lines[j].strip() == "":
j += 1
if j < len(lines) and lines[j].strip().startswith("|"):
i = j
continue
i += 1
return "\n".join(result)
_SEP_CELL = re.compile(r"^:?-+:?$")
def _is_sep_row(line: str) -> bool:
"""Return True when a pipe row is a Markdown table separator (| --- | --- |)."""
stripped = line.strip()
if not stripped.startswith("|"):
return False
cells = [c.strip() for c in stripped.split("|") if c.strip()]
return bool(cells) and all(_SEP_CELL.match(c) for c in cells)
def _merge_orphaned_table_rows(markdown: str) -> str:
"""Merge pipe-row blocks that lack a separator into the preceding table.
When the LLM incorrectly breaks a table (e.g. on a false group-header), it emits
orphaned pipe rows with no header or separator. These are invalid markdown and get
merged back into the preceding table, discarding the intervening non-table content.
"""
lines = markdown.split("\n")
segments: list[tuple[str, list[str]]] = []
i = 0
while i < len(lines):
if lines[i].strip().startswith("|"):
block: list[str] = []
while i < len(lines) and lines[i].strip().startswith("|"):
block.append(lines[i])
i += 1
has_sep = any(_is_sep_row(row) for row in block)
segments.append(("table" if has_sep else "orphan", block))
else:
block = []
while i < len(lines) and not lines[i].strip().startswith("|"):
block.append(lines[i])
i += 1
segments.append(("prose", block))
result: list[tuple[str, list[str]]] = []
last_table_idx: int | None = None
for seg_type, seg_lines in segments:
if seg_type == "orphan":
if last_table_idx is not None:
result = result[: last_table_idx + 1]
result[-1] = ("table", result[-1][1] + seg_lines)
else:
result.append((seg_type, seg_lines))
else:
if seg_type == "table":
last_table_idx = len(result)
result.append((seg_type, seg_lines))
return "\n".join(line for _, seg_lines in result for line in seg_lines)
def _remove_extra_separators(markdown: str) -> str:
"""Within each contiguous table block, keep only the first separator row."""
lines = markdown.split("\n")
result: list[str] = []
seen_sep = False
for line in lines:
if not line.strip().startswith("|"):
seen_sep = False
result.append(line)
continue
if _is_sep_row(line):
if seen_sep:
continue
seen_sep = True
result.append(line)
return "\n".join(result)
# ── Formatting helpers (module-level, no state) ──────────────────────────────────────────────────
def _format_layout(pages: list[PageLayout]) -> str:
if not pages:
return "None"
parts: list[str] = []
for page in pages:
line_strs: list[str] = []
for line in page.lines:
frags = " ".join(
f"{'**' if f.bold else ''}{f.text}{'**' if f.bold else ''}@({f.x:.0f},{f.y:.0f}) fs={f.font_size:.0f}"
for f in line.fragments
)
line_strs.append(f"y={line.y:.0f} | {frags}")
parts.append(f"--- Page {page.page_number} ---\n" + "\n".join(line_strs))
return "\n\n".join(parts)
engine/src/stirling/contracts/__init__.py
+2 -20
View File
@@ -13,6 +13,7 @@ from .common import (
AiFile,
ArtifactKind,
ConversationMessage,
ConvertMarkdownResponse,
ExtractedFileText,
GenerateFileResponse,
MathAuditorToolReportArtifact,
@@ -96,17 +97,6 @@ from .pdf_questions import (
PdfQuestionTerminalResponse,
)
from .pdf_review import PdfReviewOrchestrateResponse
from .pdf_to_markdown import (
LayoutFragment,
LayoutLine,
PageLayout,
PageLayoutArtifact,
PageLayoutFileEntry,
PdfToMarkdownCannotDoResponse,
PdfToMarkdownOrchestrateResponse,
PdfToMarkdownRequest,
PdfToMarkdownResponse,
)
from .progress import (
ProgressEvent,
WholeDocCompressionRound,
@@ -139,10 +129,6 @@ __all__ = [
"ConversationMessage",
"DeleteDocumentResponse",
"PurgeOwnerResponse",
"PdfToMarkdownCannotDoResponse",
"PdfToMarkdownOrchestrateResponse",
"PdfToMarkdownRequest",
"PdfToMarkdownResponse",
"Discrepancy",
"DiscrepancyKind",
"EditCannotDoResponse",
@@ -166,15 +152,11 @@ __all__ = [
"NeedContentFileRequest",
"NeedContentResponse",
"NeedIngestResponse",
"ConvertMarkdownResponse",
"NextExecutionAction",
"OrchestratorRequest",
"OrchestratorResponse",
"LayoutFragment",
"LayoutLine",
"Page",
"PageLayout",
"PageLayoutArtifact",
"PageLayoutFileEntry",
"PageRange",
"PageText",
"PdfCommentInstruction",
engine/src/stirling/contracts/common.py
+14
View File
@@ -62,6 +62,7 @@ class WorkflowOutcome(StrEnum):
CANNOT_CONTINUE = "cannot_continue"
UNSUPPORTED_CAPABILITY = "unsupported_capability"
GENERATE_FILE = "generate_file"
CONVERT_MARKDOWN = "convert_markdown"
class ArtifactKind(StrEnum):
@@ -183,6 +184,19 @@ class NeedIngestResponse(ApiModel):
content_types: list[PdfContentType] = Field(default_factory=list)
class ConvertMarkdownResponse(ApiModel):
"""Terminal signal: convert the listed files to Markdown deterministically.
This is a deterministic, non-AI conversion. Java runs the PDF→Markdown converter
(``PdfMarkdownConverter``) on each file and returns the resulting ``.md`` file(s) as a
completed result. There is no resume turn — the conversion output is the final answer.
"""
outcome: Literal[WorkflowOutcome.CONVERT_MARKDOWN] = WorkflowOutcome.CONVERT_MARKDOWN
reason: str
files_to_ingest: list[AiFile]
class ToolOperationStep(ApiModel):
kind: Literal[StepKind.TOOL] = StepKind.TOOL
tool: AnyToolId
engine/src/stirling/contracts/orchestrator.py
+3 -2
View File
@@ -11,6 +11,7 @@ from .common import (
AiFile,
ArtifactKind,
ConversationMessage,
ConvertMarkdownResponse,
ExtractedFileText,
GenerateFileResponse,
NeedContentResponse,
@@ -23,7 +24,6 @@ from .common import (
from .execution import NextExecutionAction
from .pdf_edit import PdfEditTerminalResponse
from .pdf_questions import PdfQuestionTerminalResponse
from .pdf_to_markdown import PageLayoutArtifact
class ExtractedTextArtifact(ApiModel):
@@ -32,7 +32,7 @@ class ExtractedTextArtifact(ApiModel):
WorkflowArtifact = Annotated[
ExtractedTextArtifact | PageLayoutArtifact | ToolReportArtifact,
ExtractedTextArtifact | ToolReportArtifact,
Field(discriminator="kind"),
]
@@ -61,6 +61,7 @@ type OrchestratorResponse = Annotated[
| GenerateFileResponse
| NeedContentResponse
| NeedIngestResponse
| ConvertMarkdownResponse
| AgentDraftResponse
| NextExecutionAction
| UnsupportedCapabilityResponse,
engine/src/stirling/contracts/pdf_to_markdown.py
-105
View File
@@ -1,105 +0,0 @@
"""Contracts for the PDF to Markdown Agent.
The agent accepts a parsed document and returns a single Markdown document that
faithfully reconstructs the PDF content — headings, paragraphs, and tables in
reading order, using page_layout as the primary source of truth for structure.
Java extracts page layout via PdfIngester and returns it as a PageLayoutArtifact
through the orchestrator resume_with pattern.
"""
from __future__ import annotations
from typing import Annotated, Literal
from pydantic import Field
from stirling.models import ApiModel
from .common import ArtifactKind, ConversationMessage, GenerateFileResponse, NeedContentResponse
from .pdf_edit import EditCannotDoResponse
# ── Input: layout models (mirror Java's RawLine / TextFragment geometry) ────────────────────────
class LayoutFragment(ApiModel):
"""One text fragment with its bounding-box geometry and font properties."""
text: str
x: float
y: float
width: float
font_size: float
bold: bool
class LayoutLine(ApiModel):
"""A visual line on the page: one y-coordinate and all fragments on that line."""
y: float
fragments: list[LayoutFragment]
class PageLayout(ApiModel):
"""All layout lines for a single page, in top-to-bottom order."""
page_number: int
lines: list[LayoutLine]
# ── Artifact: page layout (produced by Java, consumed by orchestrate()) ──────────────────────────
class PageLayoutFileEntry(ApiModel):
"""Page layout data for one file, as extracted by Java's PdfIngester."""
file_name: str
pages: list[PageLayout] = Field(default_factory=list)
class PageLayoutArtifact(ApiModel):
"""Artifact carrying full spatial page layout for all input files."""
kind: Literal[ArtifactKind.PAGE_LAYOUT] = ArtifactKind.PAGE_LAYOUT
files: list[PageLayoutFileEntry] = Field(default_factory=list)
# ── Input: full request ──────────────────────────────────────────────────────────────────────────
class PdfToMarkdownRequest(ApiModel):
"""Request sent by Java after PdfIngester has parsed the document.
page_layout: per-fragment positional data from the original (y-sorted) line order.
Each fragment carries its x/y position, width, font size, and bold flag.
This is the primary source of truth for column detection and heading hierarchy.
"""
user_message: str
file_names: list[str] = Field(default_factory=list)
conversation_history: list[ConversationMessage] = Field(default_factory=list)
page_layout: list[PageLayout] = Field(default_factory=list)
# ── Output: response variants ────────────────────────────────────────────────────────────────────
class PdfToMarkdownSuccessResponse(ApiModel):
outcome: Literal["document_reconstructed"] = "document_reconstructed"
markdown: str
class PdfToMarkdownCannotDoResponse(ApiModel):
outcome: Literal["cannot_do"] = "cannot_do"
reason: str
type PdfToMarkdownResponse = Annotated[
PdfToMarkdownSuccessResponse | PdfToMarkdownCannotDoResponse,
Field(discriminator="outcome"),
]
type PdfToMarkdownOrchestrateResponse = Annotated[
GenerateFileResponse | EditCannotDoResponse | NeedContentResponse,
Field(discriminator="outcome"),
]
engine/tests/test_pdf_to_markdown.py
-138
View File
@@ -1,138 +0,0 @@
"""Tests for PDF to Markdown agent.
Two cases:
1. Narrative-only page: request validates and routes to reconstruction.
2. Mixed text + table page: layout with table region validates correctly.
"""
from __future__ import annotations
from stirling.contracts.pdf_to_markdown import (
LayoutFragment,
LayoutLine,
PageLayout,
PageLayoutArtifact,
PdfToMarkdownRequest,
PdfToMarkdownSuccessResponse,
)
def _frag(text: str, x: float, y: float, font_size: float = 10.0, bold: bool = False) -> LayoutFragment:
return LayoutFragment(text=text, x=x, y=y, width=float(len(text) * 6), font_size=font_size, bold=bold)
def _line(y: float, *frags: LayoutFragment) -> LayoutLine:
return LayoutLine(y=y, fragments=list(frags))
# ── Test 1: Narrative-only reconstruction ────────────────────────────────────────────────────────
# ── Contract test: Java serialization ↔ Python deserialization ──────────────────────────────────
# This JSON is also asserted field-by-field in PageLayoutArtifactContractTest.java.
# If either side renames a field, one of these tests fails.
_CONTRACT_JSON = (
'{"kind":"page_layout","files":[{"fileName":"test.pdf","pages":'
'[{"pageNumber":1,"lines":[{"y":10.0,"fragments":'
'[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}'
)
def test_page_layout_artifact_deserialises_java_json() -> None:
artifact = PageLayoutArtifact.model_validate_json(_CONTRACT_JSON)
assert artifact.kind == "page_layout"
assert artifact.files[0].file_name == "test.pdf"
page = artifact.files[0].pages[0]
assert page.page_number == 1
line = page.lines[0]
assert line.y == 10.0
frag = line.fragments[0]
assert frag.text == "Hello"
assert frag.x == 1.0
assert frag.y == 2.0
assert frag.width == 30.0
assert frag.font_size == 12.0
assert frag.bold is True
def test_narrative_reconstruction_request_validates() -> None:
"""A prose-only page with no tables produces a valid PdfToMarkdownRequest."""
layout = PageLayout(
page_number=1,
lines=[
_line(72.0, _frag("Annual Report 2023", x=72.0, y=72.0, font_size=18.0, bold=True)),
_line(100.0, _frag("Our revenue grew significantly", x=72.0, y=100.0)),
_line(114.0, _frag("during the fiscal year ended", x=72.0, y=114.0)),
_line(128.0, _frag("December 31, 2023.", x=72.0, y=128.0)),
],
)
request = PdfToMarkdownRequest(
user_message="reconstruct this document",
page_layout=[layout],
)
assert len(request.page_layout) == 1
assert len(request.page_layout[0].lines) == 4
assert request.page_layout[0].lines[0].fragments[0].bold is True
assert request.page_layout[0].lines[0].fragments[0].font_size == 18.0
def test_narrative_reconstruction_response_validates() -> None:
"""PdfToMarkdownSuccessResponse accepts markdown and returns document_reconstructed outcome."""
response = PdfToMarkdownSuccessResponse(
markdown="# Annual Report 2023\n\nOur revenue grew significantly during the fiscal year.",
)
assert response.outcome == "document_reconstructed"
assert response.markdown.startswith("#")
# ── Test 2: Mixed text + table reconstruction ─────────────────────────────────────────────────────
def test_mixed_page_layout_validates() -> None:
"""A page with both prose lines and a table region produces a valid request."""
layout = PageLayout(
page_number=1,
lines=[
# Prose heading
_line(50.0, _frag("Projects in Development", x=72.0, y=50.0, font_size=14.0, bold=True)),
# Table header row
_line(
80.0,
_frag("Project Name", x=72.0, y=80.0, bold=True),
_frag("Location", x=200.0, y=80.0, bold=True),
_frag("Size (MW)", x=290.0, y=80.0, bold=True),
),
# Table data rows
_line(
95.0,
_frag("Chaplin Wind 1", x=72.0, y=95.0),
_frag("Saskatchewan", x=200.0, y=95.0),
_frag("177", x=290.0, y=95.0),
),
_line(
110.0,
_frag("Amherst Island 2", x=72.0, y=110.0),
_frag("Ontario", x=200.0, y=110.0),
_frag("75", x=290.0, y=110.0),
),
# Prose after table
_line(140.0, _frag("Notes:", x=72.0, y=140.0, bold=True)),
_line(154.0, _frag("1 PPA signed", x=85.0, y=154.0)),
],
)
request = PdfToMarkdownRequest(
user_message="markdown",
page_layout=[layout],
)
assert len(request.page_layout[0].lines) == 6
# Header line has 3 fragments at distinct x-positions (column detection)
header_line = request.page_layout[0].lines[1]
xs = [f.x for f in header_line.fragments]
assert xs == [72.0, 200.0, 290.0]
# Data rows have matching x-positions
data_row = request.page_layout[0].lines[2]
assert [f.x for f in data_row.fragments] == [72.0, 200.0, 290.0]
frontend/editor/public/locales/en-GB/translation.toml
+43 -23
View File
@@ -2700,6 +2700,13 @@ whole_doc_compression_round = "Consolidating notes..."
whole_doc_read_done = "Finished reading the document..."
whole_doc_read_started = "Reading the document..."
whole_doc_slice_done = "Reading the document... ({{percent}}% complete)"
ranForSeconds = "Ran for {{count}} seconds"
ranForSeconds_one = "Ran for 1 second"
ranForSeconds_other = "Ran for {{count}} seconds"
ranForMinutes = "Ran for {{count}} minutes"
ranForMinutes_one = "Ran for 1 minute"
ranForMinutes_other = "Ran for {{count}} minutes"
ranForMinutesSeconds = "Ran for {{minutes}} min, {{seconds}} sec"
[chat.quickActions]
browseYourFiles = "Browse your files"
@@ -3042,6 +3049,37 @@ impact = "Any applications or services currently using these keys will stop work
title = "Refresh API Keys"
warning = "⚠️ Warning: This action will generate new API keys and make your previous keys invalid."
[config.mcp]
description = "Model Context Protocol (MCP) lets AI assistants like Claude use your Stirling PDF tools directly. Connect a client once and your assistant can convert, edit, secure and process documents on your behalf."
navLabel = "MCP Server"
tip = "Every action your assistant runs is performed as your account and counts towards your usage, just like using Stirling PDF directly."
title = "MCP Server"
viewApiKeys = "View API keys"
[config.mcp.copy]
configLabel = "Config"
copied = "Copied"
copy = "Copy"
endpointLabel = "Endpoint URL"
tooltip = "Copy {{label}}"
tooltipCopied = "{{label}} copied"
[config.mcp.endpoint]
label = "Your MCP endpoint"
[config.mcp.setup]
addTo = "Add to"
hint = "Pick your client, paste the snippet into the file shown, then restart it. You'll sign in with your Stirling account on first use - no keys to copy."
title = "Connect your AI assistant"
[config.mcp.tools]
ai = "AI"
convert = "Convert"
misc = "Misc"
pages = "Pages"
security = "Security"
title = "What your assistant can do"
[config.overview]
description = "Current application settings and configuration details."
error = "Error"
@@ -3203,7 +3241,6 @@ creditsRemaining = "{{current}} of {{total}} remaining"
creditsThisMonth = "Monthly credits"
current = "Current Plan"
customPricing = "Custom"
customWatchedFolders = "Custom Watched Folders"
dedicatedSupportSlas = "Dedicated support & SLAs"
enterpriseSubscription = "Enterprise"
everythingInCredits = "Everything in Credits, plus:"
@@ -3511,7 +3548,6 @@ addFiles = "Add Files"
[fileManager]
active = "Active"
addToWatchedFolder = "Add to Watched Folder"
addToUpload = "Add to Upload"
changesNotUploaded = "Changes not uploaded"
clearAll = "Clear All"
@@ -4471,7 +4507,7 @@ alreadyHaveAccount = "Already have an account?"
choosePassword = "Choose a password"
confirmPassword = "Confirm password"
confirmPasswordPlaceholder = "Re-enter your password"
createAccount = "Create Account"
createAccount = "Create account"
creating = "Creating Account..."
email = "Email address"
emailPlaceholder = "Enter your email address"
@@ -4554,11 +4590,10 @@ sending = "Sending…"
sendMagicLink = "Send Magic Link"
sendResetLink = "Send reset link"
sessionExpired = "Your session has expired. Please sign in again."
createAccount = "Create an account"
signin = "Sign in"
signInAnonymously = "Sign Up as a Guest"
signingIn = "Signing in..."
signInWith = "Sign in with"
subtitle = "Sign back in to Stirling PDF"
title = "Sign in"
unexpectedError = "Unexpected error: {{message}}"
updatePassword = "Update password"
@@ -5774,7 +5809,6 @@ label = "SMTP Username"
[quickAccess]
access = "Access"
watchedFolders = "Watched Folders"
accessAddPerson = "Add another person"
accessBack = "Back"
accessCopyLink = "Copy link"
@@ -6865,6 +6899,7 @@ createFailed = "Failed to create signing request"
[signup]
accountCreatedSuccessfully = "Account created successfully! You can now sign in."
alreadyHaveAccount = "I already have an account"
checkEmailConfirmation = "Check your email for a confirmation link to complete your registration."
confirmPassword = "Confirm password"
confirmPasswordPlaceholder = "Confirm password"
@@ -6885,10 +6920,11 @@ passwordsDoNotMatch = "Passwords do not match"
passwordTooShort = "Password must be at least 6 characters long"
pleaseFillAllFields = "Please fill in all fields"
signUp = "Sign Up"
signUpWith = "Sign up with"
skip = "Skip"
subtitle = "Join Stirling PDF to get started"
title = "Create an account"
unexpectedError = "Unexpected error: {{message}}"
useEmailInstead = "Use Email Instead"
[sizes]
large = "Large"
@@ -8162,20 +8198,14 @@ newFolder = "New folder"
noFolders = "No watched folders yet"
sidebarTitle = "Watched Folders"
title = "Watched Folders"
sidebarFiles = "My Files"
[watchedFolders.modal]
automation = "Automation"
automationSaved = "Automation saved"
createFolder = "Create Folder"
stepsSaved = "Steps saved — click Create Folder to finish"
automationRequired = "Add at least one configured step before saving."
color = "Accent color"
createTitle = "New Watched Folder"
description = "Description"
descriptionPlaceholder = "What does this folder do?"
editTitle = "Edit Watched Folder"
icon = "Icon"
name = "Folder name"
namePlaceholder = "My Watched Folder"
nameRequired = "Folder name is required"
@@ -8183,17 +8213,13 @@ nameTooLong = "Folder name must be 50 characters or less"
saveChanges = "Save Changes"
saveFailed = "Failed to save folder. Please try again."
automationNameFallback = "Watched Folder automation"
retryLabel = "Auto-retry"
maxRetries = "Max retries"
maxRetriesDesc = "0 to disable"
retryDelay = "Delay (minutes)"
outputLabel = "Output"
outputModeVersion = "Replace original?"
outputModeVersionDesc = "Output becomes a new version of the input file rather than a separate file"
outputModeNewDesc = "Output is saved as a new separate file"
outputName = "Output filename prefix"
outputNameSuffix = "Suffix"
outputNamePlaceholderVersion = "Same as original"
sectionFolder = "Folder"
sectionSourceOutput = "Source & Output"
sectionSteps = "Steps"
@@ -8220,14 +8246,10 @@ positionSuffix = "Suffix"
[watchedFolders.home]
create = "Create your first Watched Folder"
dropHere = "Drop to process"
editFolder = "Edit folder"
empty = "No watched folders yet"
file = "file"
files = "files"
openFolder = "Open folder"
title = "Watched Folders"
subtitle = "Folders that automatically process PDFs with your configured pipeline"
emptyTitle = "Automate your PDF workflows"
emptyDesc = "Set up a Watched Folder once. Drop PDFs in and they're automatically compressed, OCR'd, split, merged — whatever your pipeline does."
addAnother = "Add another Watched Folder"
@@ -8235,7 +8257,6 @@ addAnotherDesc = "Automatically process files with a new pipeline"
resume = "Resume"
pause = "Pause"
deleteFolder = "Delete folder"
noSteps = "No automation steps configured"
[watchedFolders.card]
edit = "Edit folder"
@@ -8318,7 +8339,6 @@ previewLoadFailed = "Could not load file preview."
back = "Back"
dismiss = "Dismiss"
retry = "Retry"
view = "View"
download = "Download"
downloadInput = "Download input"
downloadOutput = "Download output"
frontend/editor/public/modern-logo/LoginDarkModeHeader.svg
+5
View File
File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 7.8 KiB

frontend/editor/public/modern-logo/LoginLightModeHeader.svg
+6
View File
File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 7.8 KiB

frontend/editor/src/core/components/chat/ChatContext.tsx
+1
View File
@@ -10,6 +10,7 @@ export function useChat() {
isOpen: false,
isLoading: false,
progress: null,
progressLog: [] as never[],
toggleOpen: () => {},
setOpen: (_open: boolean) => {},
sendMessage: async (_content: string) => {},
frontend/editor/src/core/tests/live/authentication-login.spec.ts
+1 -6
View File
@@ -19,12 +19,7 @@ test.describe("1. Authentication and Login", () => {
.first(),
).toBeVisible();
// Step 3: Confirm the heading for "Sign In" / "Login" is visible
await expect(
page.getByRole("heading", { name: /sign in|login|masuk/i }),
).toBeVisible();
// Step 4: Confirm a "Username" text input field is present and empty
// Step 3: Confirm a "Username" text input field is present and empty
const usernameInput = page.locator("#email");
await expect(usernameInput).toBeVisible();
await expect(usernameInput).toHaveValue("");
frontend/editor/src/proprietary/components/chat/ChatContext.tsx
+66 -12
View File
@@ -37,6 +37,14 @@ export interface ChatMessage {
* turns that answered without running any tool.
*/
toolsUsed?: string[];
/**
* Full ordered progress log captured during the AI turn that produced this message.
* Only set on assistant messages; used to render the "Ran for X seconds" collapsed
* history dropdown above the response.
*/
progressLog?: AiWorkflowProgress[];
/** Wall-clock duration of the AI turn in milliseconds. Only set on assistant messages. */
durationMs?: number;
}
export enum AiWorkflowPhase {
@@ -178,12 +186,20 @@ interface ChatState {
isOpen: boolean;
isLoading: boolean;
progress: AiWorkflowProgress | null;
/** Ordered log of every progress event in the current request. UI shows the last N entries. */
progressLog: AiWorkflowProgress[];
}
/**
* Maximum number of progress steps retained in the live buffer.
*/
export const PROGRESS_LOG_MAX = 4;
type ChatAction =
| { type: "ADD_MESSAGE"; message: ChatMessage }
| { type: "SET_LOADING"; loading: boolean }
| { type: "SET_PROGRESS"; progress: AiWorkflowProgress | null }
| { type: "APPEND_PROGRESS"; progress: AiWorkflowProgress }
| { type: "TOGGLE_OPEN" }
| { type: "SET_OPEN"; open: boolean }
| { type: "CLEAR" };
@@ -193,15 +209,40 @@ function chatReducer(state: ChatState, action: ChatAction): ChatState {
case "ADD_MESSAGE":
return { ...state, messages: [...state.messages, action.message] };
case "SET_LOADING":
return { ...state, isLoading: action.loading };
// Reset the log on both start (true) and end (false) of a request.
return {
...state,
isLoading: action.loading,
progress: action.loading ? state.progress : null,
progressLog: [],
};
case "SET_PROGRESS":
return { ...state, progress: action.progress };
case "APPEND_PROGRESS":
// Cap the live buffer so each append copies at most PROGRESS_LOG_MAX elements
return {
...state,
progress: action.progress,
progressLog:
state.progressLog.length < PROGRESS_LOG_MAX
? [...state.progressLog, action.progress]
: [
...state.progressLog.slice(1 - PROGRESS_LOG_MAX),
action.progress,
],
};
case "TOGGLE_OPEN":
return { ...state, isOpen: !state.isOpen };
case "SET_OPEN":
return { ...state, isOpen: action.open };
case "CLEAR":
return { ...state, messages: [], isLoading: false, progress: null };
return {
...state,
messages: [],
isLoading: false,
progress: null,
progressLog: [],
};
}
}
@@ -325,6 +366,8 @@ interface ChatContextValue {
isOpen: boolean;
isLoading: boolean;
progress: AiWorkflowProgress | null;
/** Ordered log of every progress event for the current in-flight request. */
progressLog: AiWorkflowProgress[];
toggleOpen: () => void;
setOpen: (open: boolean) => void;
sendMessage: (content: string) => Promise<void>;
@@ -339,6 +382,7 @@ const initialState: ChatState = {
isOpen: false,
isLoading: false,
progress: null,
progressLog: [],
};
export function ChatProvider({ children }: { children: ReactNode }) {
@@ -440,6 +484,11 @@ export function ChatProvider({ children }: { children: ReactNode }) {
abortRef.current = controller;
const priorMessages = messagesRef.current;
const startTime = Date.now();
// Mirror every progress event locally so we can attach the full log to
// the assistant message when the result arrives — without needing a ref
// into the reducer state.
const progressLogLocal: AiWorkflowProgress[] = [];
const userMessage: ChatMessage = {
id: generateId(),
@@ -503,16 +552,15 @@ export function ChatProvider({ children }: { children: ReactNode }) {
) {
toolsUsed.push(data.tool);
}
dispatch({
type: "SET_PROGRESS",
progress: {
phase: data.phase as AiWorkflowPhase,
tool: data.tool,
stepIndex: data.stepIndex,
stepCount: data.stepCount,
engineDetail: data.engineDetail,
},
});
const progressItem: AiWorkflowProgress = {
phase: data.phase as AiWorkflowPhase,
tool: data.tool,
stepIndex: data.stepIndex,
stepCount: data.stepCount,
engineDetail: data.engineDetail,
};
progressLogLocal.push(progressItem);
dispatch({ type: "APPEND_PROGRESS", progress: progressItem });
},
onResult: (data) => {
receivedResult = true;
@@ -526,6 +574,11 @@ export function ChatProvider({ children }: { children: ReactNode }) {
content: replyContent,
timestamp: Date.now(),
toolsUsed: toolsUsed.length > 0 ? toolsUsed : undefined,
progressLog:
progressLogLocal.length > 0
? [...progressLogLocal]
: undefined,
durationMs: Date.now() - startTime,
},
});
if (data.fileId || data.resultFiles?.length) {
@@ -599,6 +652,7 @@ export function ChatProvider({ children }: { children: ReactNode }) {
isOpen: state.isOpen,
isLoading: state.isLoading,
progress: state.progress,
progressLog: state.progressLog,
toggleOpen,
setOpen,
sendMessage,
frontend/editor/src/proprietary/components/chat/ChatPanel.css
+183 -1
View File
@@ -396,7 +396,182 @@
margin-bottom: 0;
}
/* Thinking indicator */
/* ─── Progress step log ─────────────────────────────────────────────────── */
/*
* Shown while the AI is working. Each step slides in from below as events
* arrive; the active (last) step is full-opacity with a pulsing icon, and
* earlier completed steps are dimmed. Only the most-recent N steps are
* rendered — there is no scrollable list.
*/
.chat-progress-log {
display: flex;
flex-direction: column;
padding: 0.15rem 0.5rem 0.35rem;
/* No gap — the connector line div provides the inter-step spacing. */
}
.chat-progress-step {
display: flex;
align-items: flex-start;
gap: 0.65rem;
/*
* Use color rather than opacity for past-step dimming so the connector
* line (a sibling element inside __left, not a text node) is NOT affected
* and stays clearly visible. currentColor flows into SVG icon fills;
* color inherits into the label span automatically.
*/
color: var(--text-muted);
transition: color 250ms ease-out;
animation: chat-step-enter 300ms cubic-bezier(0.22, 1, 0.36, 1) both;
}
/*
* Active step: white text on dark mode so currentColor feeds directly into
* the StirlingLogoAnimated SVG paths and the label at the same time.
* Light-mode falls back to the default text colour so it's not invisible.
*/
/* Active step: icon and label are coloured independently so the logo can be
blue while the text uses the normal text colour. */
.chat-progress-step--active .chat-progress-step__icon {
color: var(--mantine-color-blue-filled);
}
.chat-progress-step--active .chat-progress-step__label {
color: var(--mantine-color-text);
}
[data-mantine-color-scheme="dark"]
.chat-progress-step--active
.chat-progress-step__label {
color: #ffffff;
}
.chat-progress-step__left {
display: flex;
flex-direction: column;
align-items: center;
flex-shrink: 0;
width: 20px;
}
/* Icon cell — fixed height so the connector line aligns cleanly. */
.chat-progress-step__icon {
width: 20px;
height: 24px;
display: flex;
align-items: center;
justify-content: center;
overflow: hidden;
flex-shrink: 0;
/* Inherits color from .chat-progress-step so currentColor flows into icons. */
}
/*
* Connector line between steps. Not a child of the label so it is unaffected
* by the step's color; it sits on its own and reads clearly against the
* toolbar background.
*/
.chat-progress-step__line {
width: 1px;
height: 18px;
background: var(--border-subtle);
flex-shrink: 0;
margin-top: 2px;
}
/* Active step label — slightly larger, inherits the active color. */
.chat-progress-step__label {
font-size: 0.875rem;
padding-top: 3px;
line-height: 1.35;
}
/* Past step labels — a touch smaller so the active row reads as the primary. */
.chat-progress-step:not(.chat-progress-step--active)
.chat-progress-step__label {
font-size: 0.8rem;
padding-top: 4px;
}
/*
* Wrapper that scales a registry tool icon (LocalIcon, typically 1.5rem/24px)
* down to fit the 20px icon column. Transform does not affect layout, so the
* parent overflow:hidden clips the un-scaled footprint cleanly.
*/
.chat-step-icon-scaled {
display: inline-flex;
align-items: center;
justify-content: center;
transform: scale(0.75);
transform-origin: center;
flex-shrink: 0;
}
/* Entry: steps slide up from a few pixels below their final position. */
@keyframes chat-step-enter {
from {
transform: translateY(7px);
}
to {
transform: translateY(0);
}
}
/* ─── Completed progress log dropdown ──────────────────────────────────── */
/*
* Shown above each completed assistant turn. The toggle displays a small
* "Ran for X seconds" label; expanding reveals the full ordered step list
* for that turn in a compact, read-only format.
*/
.chat-completed-log {
margin-bottom: 0.55rem;
}
.chat-completed-log__toggle {
display: inline-flex;
align-items: center;
padding: 0.1rem 0.25rem 0.1rem 0.1rem;
border-radius: 0.35rem;
color: var(--text-muted);
transition: background 120ms ease-out;
}
.chat-completed-log__toggle:hover {
background: var(--mantine-color-default-hover);
}
.chat-completed-log__steps {
display: flex;
flex-direction: column;
padding: 0.35rem 0.4rem 0.2rem 0.25rem;
margin-top: 0.2rem;
}
/* In the completed log every step is a past step — no active highlighting,
no entry animation (the content is static once opened). */
.chat-completed-log__steps .chat-progress-step {
animation: none;
}
/* Tighten the icon and connector dimensions for the denser historical view. */
.chat-completed-log__steps .chat-progress-step__icon {
height: 20px;
}
.chat-completed-log__steps .chat-progress-step__line {
height: 14px;
}
.chat-completed-log__steps .chat-progress-step__label {
font-size: 0.775rem;
padding-top: 2px;
}
/* Legacy thinking indicator (kept in case any other code references it). */
.chat-thinking {
display: flex;
align-items: center;
@@ -405,6 +580,13 @@
color: var(--mantine-color-blue-filled);
}
@media (prefers-reduced-motion: reduce) {
.chat-progress-step {
animation: none;
transition: none;
}
}
/* ─── Stirling logo thinking animation ─────────────────────────────────── */
.stirling-thinking__path-right {
frontend/editor/src/proprietary/components/chat/ChatPanel.tsx
+249 -51
View File
@@ -4,6 +4,7 @@ import {
useEffect,
useState,
type KeyboardEvent,
type ReactNode,
} from "react";
import { renderMarkdown } from "@app/components/viewer/nonpdf/MarkdownRenderer";
import { TFunction } from "i18next";
@@ -13,7 +14,6 @@ import {
Box,
Collapse,
Group,
List,
Menu,
Paper,
ScrollArea,
@@ -23,7 +23,10 @@ import {
UnstyledButton,
} from "@mantine/core";
import ArrowUpwardIcon from "@mui/icons-material/ArrowUpward";
import ArticleOutlinedIcon from "@mui/icons-material/ArticleOutlined";
import BuildOutlinedIcon from "@mui/icons-material/BuildOutlined";
import CloseIcon from "@mui/icons-material/Close";
import CloudOutlinedIcon from "@mui/icons-material/CloudOutlined";
import ContentCopyIcon from "@mui/icons-material/ContentCopy";
import DeleteSweepIcon from "@mui/icons-material/DeleteSweep";
import ExpandLessIcon from "@mui/icons-material/ExpandLess";
@@ -33,14 +36,15 @@ import {
useChat,
AiWorkflowPhase,
ChatRole,
PROGRESS_LOG_MAX,
isKnownEngineProgressDetail,
type AiWorkflowProgress,
type AnyEngineProgressDetail,
} from "@app/components/chat/ChatContext";
import { formatRelativeTime } from "@app/utils/timeUtils";
import { useTranslatedToolCatalog } from "@app/data/useTranslatedToolRegistry";
import { StirlingLogoOutline } from "@app/components/agents/StirlingLogoOutline";
import { StirlingLogoAnimated } from "@app/components/agents/StirlingLogoAnimated";
import { StirlingLogoOutline } from "@app/components/agents/StirlingLogoOutline";
import { ChatQuickActions } from "@app/components/chat/ChatQuickActions";
import "@app/components/chat/ChatPanel.css";
@@ -68,6 +72,27 @@ function useToolNameResolver(): ToolNameResolver {
}, [allTools]);
}
/** Resolver mapping a tool endpoint path to its registry icon ReactNode. */
type ToolIconResolver = (endpoint: string) => ReactNode | null;
/**
* Look up a tool's icon ReactNode from the tool catalog, keyed by API endpoint path.
* Returns null when the endpoint is not found (use a generic fallback icon in that case).
*/
function useToolIconResolver(): ToolIconResolver {
const { allTools } = useTranslatedToolCatalog();
return useMemo(() => {
const iconByEndpoint = new Map<string, ReactNode>();
Object.values(allTools).forEach((tool) => {
const endpoint = tool.operationConfig?.endpoint;
if (typeof endpoint === "string") {
iconByEndpoint.set(endpoint, tool.icon);
}
});
return (endpoint: string) => iconByEndpoint.get(endpoint) ?? null;
}, [allTools]);
}
function formatProgress(
progress: AiWorkflowProgress,
t: TranslateFn,
@@ -125,31 +150,152 @@ function formatEngineProgress(
}
}
function ToolsUsedBlock({
tools,
resolveToolName,
/**
* Choose an icon for a progress step.
*
* The active (current) step always shows the animated Stirling logo so it reads
* as the "live" indicator. Past steps get a phase-specific icon so the trail
* is scannable at a glance.
*/
function progressStepIcon(
progress: AiWorkflowProgress,
resolveToolIcon: ToolIconResolver,
isActive: boolean,
): ReactNode {
if (isActive) {
return <StirlingLogoAnimated size={18} />;
}
if (progress.phase === AiWorkflowPhase.EXECUTING_TOOL) {
const registryIcon = progress.tool ? resolveToolIcon(progress.tool) : null;
if (registryIcon) {
return <span className="chat-step-icon-scaled">{registryIcon}</span>;
}
return <BuildOutlinedIcon sx={{ fontSize: 17 }} />;
}
if (
progress.phase === AiWorkflowPhase.EXTRACTING_CONTENT ||
progress.phase === AiWorkflowPhase.ENGINE_PROGRESS
) {
return <ArticleOutlinedIcon sx={{ fontSize: 17 }} />;
}
return <CloudOutlinedIcon sx={{ fontSize: 17 }} />;
}
/**
* Animated step-by-step progress log shown while the AI is working.
* Displays the last {@link PROGRESS_LOG_VISIBLE} steps from the live event stream,
* with the active (most recent) step highlighted and older steps dimmed.
*/
function ProgressLogDisplay({
progressLog,
t,
resolveToolName,
resolveToolIcon,
}: {
tools: string[];
resolveToolName: ToolNameResolver;
progressLog: AiWorkflowProgress[];
t: TranslateFn;
resolveToolName: ToolNameResolver;
resolveToolIcon: ToolIconResolver;
}) {
// Placeholder shown before the first SSE event arrives.
if (progressLog.length === 0) {
return (
<div className="chat-progress-log">
<div className="chat-progress-step chat-progress-step--active">
<div className="chat-progress-step__left">
<div className="chat-progress-step__icon">
<StirlingLogoAnimated size={18} />
</div>
</div>
<span className="chat-progress-step__label">
{t("chat.progress.thinking")}
</span>
</div>
</div>
);
}
// Chronological order: oldest at top, newest (active) at bottom.
// The reducer already caps progressLog at PROGRESS_LOG_MAX entries, so this
// slice is effectively a no-op but kept for defensive correctness.
const visibleSteps = progressLog.slice(-PROGRESS_LOG_MAX);
const startIndex = progressLog.length - visibleSteps.length;
return (
<div className="chat-progress-log">
{visibleSteps.map((step, i) => {
// Stable key based on absolute position in the full log — React reuses
// existing DOM elements and only mounts (and animates) new ones.
const globalIndex = startIndex + i;
const isActive = i === visibleSteps.length - 1; // last = newest = bottom
// Connector runs below every step except the active one at the bottom.
const showConnector = i < visibleSteps.length - 1;
const label = formatProgress(step, t, resolveToolName);
return (
<div
key={globalIndex}
className={`chat-progress-step${isActive ? " chat-progress-step--active" : ""}`}
>
<div className="chat-progress-step__left">
<div className="chat-progress-step__icon">
{progressStepIcon(step, resolveToolIcon, isActive)}
</div>
{showConnector && <div className="chat-progress-step__line" />}
</div>
<span className="chat-progress-step__label">{label}</span>
</div>
);
})}
</div>
);
}
function formatDuration(ms: number, t: TranslateFn): string {
const totalSeconds = Math.max(1, Math.round(ms / 1000));
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
if (minutes === 0) {
return t("chat.progress.ranForSeconds", { count: totalSeconds });
}
if (seconds === 0) {
return t("chat.progress.ranForMinutes", { count: minutes });
}
return t("chat.progress.ranForMinutesSeconds", { minutes, seconds });
}
/**
* Collapsed "Ran for X seconds" dropdown that appears above each completed
* assistant turn. Expands to show the full ordered progress log for that turn.
*/
function CompletedProgressLogDropdown({
progressLog,
durationMs,
t,
resolveToolName,
resolveToolIcon,
}: {
progressLog: AiWorkflowProgress[];
durationMs: number;
t: TranslateFn;
resolveToolName: ToolNameResolver;
resolveToolIcon: ToolIconResolver;
}) {
const [expanded, setExpanded] = useState(false);
const names = tools.map(
(endpoint) => resolveToolName(endpoint) ?? t("chat.toolsUsed.unknownTool"),
);
const label = t("chat.toolsUsed.summary", { count: tools.length });
const label = formatDuration(durationMs, t);
return (
<Box mt={6}>
<div className="chat-completed-log">
<UnstyledButton
className="chat-completed-log__toggle"
onClick={() => setExpanded((v) => !v)}
aria-expanded={expanded}
>
<Group gap={4} wrap="nowrap">
<Group gap={4} wrap="nowrap" align="center">
{expanded ? (
<ExpandLessIcon sx={{ fontSize: 14 }} />
<ExpandLessIcon sx={{ fontSize: 12 }} />
) : (
<ExpandMoreIcon sx={{ fontSize: 14 }} />
<ExpandMoreIcon sx={{ fontSize: 12 }} />
)}
<Text size="xs" c="dimmed">
{label}
@@ -157,19 +303,30 @@ function ToolsUsedBlock({
</Group>
</UnstyledButton>
<Collapse in={expanded}>
<List
type="ordered"
size="xs"
mt={4}
pl="lg"
styles={{ itemWrapper: { lineHeight: 1.4 } }}
>
{names.map((name, i) => (
<List.Item key={i}>{name}</List.Item>
))}
</List>
<div className="chat-completed-log__steps">
{progressLog.map((step, i) => {
const showConnector = i < progressLog.length - 1;
const stepLabel = formatProgress(step, t, resolveToolName);
return (
<div
key={i}
className="chat-progress-step chat-progress-step--done"
>
<div className="chat-progress-step__left">
<div className="chat-progress-step__icon">
{progressStepIcon(step, resolveToolIcon, false)}
</div>
{showConnector && (
<div className="chat-progress-step__line" />
)}
</div>
<span className="chat-progress-step__label">{stepLabel}</span>
</div>
);
})}
</div>
</Collapse>
</Box>
</div>
);
}
@@ -177,15 +334,19 @@ function ChatMessageBubble({
role,
content,
timestamp,
toolsUsed,
progressLog,
durationMs,
resolveToolName,
resolveToolIcon,
t,
}: {
role: ChatRole;
content: string;
timestamp: number;
toolsUsed?: string[];
progressLog?: AiWorkflowProgress[];
durationMs?: number;
resolveToolName: ToolNameResolver;
resolveToolIcon: ToolIconResolver;
t: TranslateFn;
}) {
const [copied, setCopied] = useState(false);
@@ -231,16 +392,18 @@ function ChatMessageBubble({
return (
<div className="chat-message chat-message-assistant">
<div className="chat-bubble-assistant">
{progressLog && progressLog.length > 0 && durationMs != null && (
<CompletedProgressLogDropdown
progressLog={progressLog}
durationMs={durationMs}
t={t}
resolveToolName={resolveToolName}
resolveToolIcon={resolveToolIcon}
/>
)}
<Text size="sm" component="div">
{renderMarkdown(content)}
</Text>
{toolsUsed && toolsUsed.length > 0 && (
<ToolsUsedBlock
tools={toolsUsed}
resolveToolName={resolveToolName}
t={t}
/>
)}
{actions}
</div>
</div>
@@ -256,20 +419,55 @@ export interface ChatPanelProps {
export function ChatPanel({ onBack, backLabel }: ChatPanelProps) {
const { t } = useTranslation();
const { messages, isLoading, progress, sendMessage, clearChat } = useChat();
const { messages, isLoading, progressLog, sendMessage, clearChat } =
useChat();
const resolveToolName = useToolNameResolver();
const resolveToolIcon = useToolIconResolver();
const [input, setInput] = useState("");
const scrollRef = useRef<HTMLDivElement>(null);
const inputRef = useRef<HTMLTextAreaElement>(null);
// Tracks whether the user manually scrolled away from the bottom.
// A ref (not state) so scroll events don't cause re-renders.
const userScrolledUp = useRef(false);
// Jump to the bottom on first render so existing conversations open at the
// most recent message rather than the top.
useEffect(() => {
if (scrollRef.current) {
scrollRef.current.scrollTo({
top: scrollRef.current.scrollHeight,
behavior: "smooth",
requestAnimationFrame(() => {
const el = scrollRef.current;
if (el) el.scrollTop = el.scrollHeight;
});
}, []);
// Attach a passive scroll listener to track whether the user has scrolled
// away from the bottom (breaks auto-scroll) or returned to it (re-latches).
useEffect(() => {
const el = scrollRef.current;
if (!el) return;
const onScroll = () => {
const distFromBottom = el.scrollHeight - el.scrollTop - el.clientHeight;
userScrolledUp.current = distFromBottom > 50;
};
el.addEventListener("scroll", onScroll, { passive: true });
return () => el.removeEventListener("scroll", onScroll);
}, []);
// Scroll to the bottom when messages arrive or live progress steps update,
// unless the user has scrolled up (they're reading history).
// Scrolling back to the bottom resets the ref, so the next update re-latches.
//
// RAF defers the scroll until after the browser has laid out the new nodes,
// so scrollHeight is correct. Direct scrollTop assignment avoids the
// smooth-scroll interruption problem that occurs when SSE events arrive
// faster than a smooth animation can complete.
useEffect(() => {
if (!userScrolledUp.current) {
requestAnimationFrame(() => {
const el = scrollRef.current;
if (el) el.scrollTop = el.scrollHeight;
});
}
}, [messages]);
}, [messages, progressLog]);
useEffect(() => {
inputRef.current?.focus();
@@ -343,21 +541,21 @@ export function ChatPanel({ onBack, backLabel }: ChatPanelProps) {
role={msg.role}
content={msg.content}
timestamp={msg.timestamp}
toolsUsed={msg.toolsUsed}
progressLog={msg.progressLog}
durationMs={msg.durationMs}
resolveToolName={resolveToolName}
resolveToolIcon={resolveToolIcon}
t={t}
/>
))}
{isLoading && (
<div className="chat-message chat-message-assistant">
<div className="chat-thinking">
<StirlingLogoAnimated size={20} />
<Text size="sm" c="dimmed">
{progress
? formatProgress(progress, t, resolveToolName)
: t("chat.progress.thinking")}
</Text>
</div>
<ProgressLogDisplay
progressLog={progressLog}
t={t}
resolveToolName={resolveToolName}
resolveToolIcon={resolveToolIcon}
/>
</div>
)}
</Stack>
frontend/editor/src/proprietary/routes/Login.tsx
+25 -9
View File
@@ -17,12 +17,11 @@ import { useDocumentMeta } from "@app/hooks/useDocumentMeta";
import AuthLayout from "@app/routes/authShared/AuthLayout";
import { useBackendProbe } from "@app/hooks/useBackendProbe";
import apiClient from "@app/services/apiClient";
import { BASE_PATH } from "@app/constants/app";
import { BASE_PATH, withBasePath } from "@app/constants/app";
import { type OAuthProvider } from "@app/auth/oauthTypes";
import { updateSupportedLanguages } from "@app/i18n";
// Import login components
import LoginHeader from "@app/routes/login/LoginHeader";
import ErrorMessage from "@app/routes/login/ErrorMessage";
import EmailPasswordForm from "@app/routes/login/EmailPasswordForm";
import OAuthButtons, {
@@ -74,7 +73,6 @@ export default function Login() {
const autoLoginErrorRecorded = useRef(false);
const isUserPassAllowed = loginMethod === "all" || loginMethod === "normal";
const isSsoOnlyMode = loginMethod !== "all" && loginMethod !== "normal";
const isSingleSsoOnly = !isUserPassAllowed && enabledProviders.length === 1;
const AUTO_LOGIN_ATTEMPTS_KEY = "stirling_sso_auto_login_attempts";
const AUTO_LOGIN_ERRORS_KEY = "stirling_sso_auto_login_errors";
@@ -461,7 +459,6 @@ export default function Login() {
// If backend isn't ready yet, show a lightweight status screen instead of the form
if (backendProbe.status !== "up" && !loginDisabled) {
const backendTitle = t("backendStartup.notFoundTitle", "Backend not found");
const handleRetry = async () => {
const result = await backendProbe.probe();
if (result.status === "up") {
@@ -471,7 +468,18 @@ export default function Login() {
};
return (
<AuthLayout>
<LoginHeader title={backendTitle} />
<div className="auth-logo-block">
<img
src={withBasePath("/modern-logo/LoginLightModeHeader.svg")}
alt="Stirling PDF"
className="auth-logo-header auth-logo-header--light"
/>
<img
src={withBasePath("/modern-logo/LoginDarkModeHeader.svg")}
alt="Stirling PDF"
className="auth-logo-header auth-logo-header--dark"
/>
</div>
<div
className="auth-section"
style={{
@@ -555,10 +563,18 @@ export default function Login() {
return (
<AuthLayout>
<LoginHeader
title={isSingleSsoOnly ? "" : t("login.login") || "Sign in"}
centerOnly={isSingleSsoOnly}
/>
<div className="auth-logo-block">
<img
src={withBasePath("/modern-logo/LoginLightModeHeader.svg")}
alt="Stirling PDF"
className="auth-logo-header auth-logo-header--light"
/>
<img
src={withBasePath("/modern-logo/LoginDarkModeHeader.svg")}
alt="Stirling PDF"
className="auth-logo-header auth-logo-header--dark"
/>
</div>
{/* Success message */}
{successMessage && (
frontend/editor/src/proprietary/routes/Signup.tsx
+13 -6
View File
@@ -5,10 +5,9 @@ import { useDocumentMeta } from "@app/hooks/useDocumentMeta";
import { useAuth } from "@app/auth/UseSession";
import AuthLayout from "@app/routes/authShared/AuthLayout";
import "@app/routes/authShared/auth.css";
import { BASE_PATH } from "@app/constants/app";
import { BASE_PATH, withBasePath } from "@app/constants/app";
// Import signup components
import LoginHeader from "@app/routes/login/LoginHeader";
import ErrorMessage from "@app/routes/login/ErrorMessage";
import DividerWithText from "@app/components/shared/DividerWithText";
import SignupForm from "@app/routes/signup/SignupForm";
@@ -92,10 +91,18 @@ export default function Signup() {
return (
<AuthLayout>
<LoginHeader
title={t("signup.title", "Create an account")}
subtitle={t("signup.subtitle", "Join Stirling PDF")}
/>
<div className="auth-logo-block">
<img
src={withBasePath("/modern-logo/LoginLightModeHeader.svg")}
alt="Stirling PDF"
className="auth-logo-header auth-logo-header--light"
/>
<img
src={withBasePath("/modern-logo/LoginDarkModeHeader.svg")}
alt="Stirling PDF"
className="auth-logo-header auth-logo-header--dark"
/>
</div>
<ErrorMessage error={error} />
frontend/editor/src/proprietary/routes/authShared/auth.css
+87
View File
@@ -615,3 +615,90 @@
background-color: #af3434 !important;
opacity: 0.6 !important;
}
/* ── Logo block ─────────────────────────────────────────────────────── */
.auth-logo-block {
display: flex;
flex-direction: column;
align-items: center;
margin-bottom: 2rem;
margin-top: 0.5rem;
animation: authFadeUp 0.4s ease both;
}
.auth-logo-header {
height: 8rem;
width: auto;
}
.auth-logo-header--dark {
display: none;
}
/* ── Page entrance animation ────────────────────────────────────────── */
@keyframes authFadeUp {
from {
opacity: 0;
transform: translateY(12px);
}
to {
opacity: 1;
transform: translateY(0);
}
}
/* ── Expandable trigger button state ────────────────────────────────── */
.auth-expandable-trigger {
transition:
background-color 180ms ease,
box-shadow 180ms ease,
border-color 180ms ease;
}
.auth-expandable-trigger--active {
border-color: #af3434 !important;
box-shadow: 0 0 0 3px rgba(175, 52, 52, 0.12);
}
/* ── Animated expand/collapse via grid-template-rows ───────────────── */
.auth-expand-grid {
display: grid;
grid-template-rows: 0fr;
transition:
grid-template-rows 280ms cubic-bezier(0.4, 0, 0.2, 1),
opacity 220ms ease;
opacity: 0;
}
.auth-expand-grid--open {
grid-template-rows: 1fr;
opacity: 1;
}
.auth-expand-inner {
overflow: hidden;
min-height: 0;
}
/* ── Icon+label group — keeps icons vertically aligned across buttons ── */
.oauth-btn-group {
display: inline-flex;
align-items: center;
min-width: 12.5rem;
}
.oauth-btn-label {
flex: 1;
text-align: center;
}
/* ── Icon helpers inside oauth-button-fullwidth ─────────────────────── */
.auth-at-icon {
font-size: 1.25rem;
line-height: 1;
margin-right: 0.5rem;
display: inline-block;
width: 1.75rem;
text-align: center;
flex-shrink: 0;
}
frontend/editor/src/saas/components/shared/config/configSections/McpSection.tsx
+262
View File
@@ -0,0 +1,262 @@
import { useMemo } from "react";
import { useTranslation } from "react-i18next";
import {
Stack,
Paper,
Text,
Group,
Alert,
Code,
Badge,
Button,
CopyButton,
Tabs,
Tooltip,
ThemeIcon,
} from "@mantine/core";
import LocalIcon from "@app/components/shared/LocalIcon";
import { useAppConfig } from "@app/contexts/AppConfigContext";
import { openAppSettings } from "@app/utils/appSettings";
/** Strip a single trailing slash so we can safely append paths. */
function trimTrailingSlash(url: string): string {
return url.endsWith("/") ? url.slice(0, -1) : url;
}
/** A small copy-to-clipboard button that sits inline next to a URL/snippet. */
function CopyInline({ value, label }: { value: string; label: string }) {
const { t } = useTranslation();
return (
<CopyButton value={value} timeout={1500}>
{({ copied, copy }) => (
<Tooltip
label={
copied
? t("config.mcp.copy.tooltipCopied", "{{label}} copied", {
label,
})
: t("config.mcp.copy.tooltip", "Copy {{label}}", { label })
}
withArrow
>
<Button
size="compact-xs"
variant="subtle"
color={copied ? "teal" : "gray"}
onClick={copy}
leftSection={
<LocalIcon
icon={copied ? "check-rounded" : "content-copy-rounded"}
width={14}
height={14}
/>
}
>
{copied
? t("config.mcp.copy.copied", "Copied")
: t("config.mcp.copy.copy", "Copy")}
</Button>
</Tooltip>
)}
</CopyButton>
);
}
// SaaS MCP guide: always shown, explains how to point an AI assistant at the
// OAuth-protected /mcp endpoint with per-client config.
export default function McpSection() {
const { t } = useTranslation();
const { config } = useAppConfig();
const baseUrl = useMemo(() => {
const raw =
config?.baseUrl ||
(typeof window !== "undefined" ? window.location.origin : "");
return trimTrailingSlash(raw);
}, [config?.baseUrl]);
const mcpUrl = `${baseUrl}/mcp`;
// Per-client connection snippets pointing at this deployment's /mcp endpoint.
const clients = useMemo(
() => [
{
value: "claude",
label: "Claude Desktop",
file: "claude_desktop_config.json",
config: JSON.stringify(
{ mcpServers: { "stirling-pdf": { type: "http", url: mcpUrl } } },
null,
2,
),
},
{
value: "codex",
label: "Codex CLI",
file: "~/.codex/config.toml",
config: `[mcp_servers.stirling-pdf]\nurl = "${mcpUrl}"`,
},
{
value: "vscode",
label: "VS Code",
file: ".vscode/mcp.json",
config: JSON.stringify(
{ servers: { "stirling-pdf": { type: "http", url: mcpUrl } } },
null,
2,
),
},
],
[mcpUrl],
);
const tools: { icon: string; key: string; fallback: string }[] = [
{
icon: "sync-alt-rounded",
key: "config.mcp.tools.convert",
fallback: "Convert",
},
{
icon: "description-rounded",
key: "config.mcp.tools.pages",
fallback: "Pages",
},
{ icon: "lock", key: "config.mcp.tools.security", fallback: "Security" },
{
icon: "construction-rounded",
key: "config.mcp.tools.misc",
fallback: "Misc",
},
{ icon: "smart-toy-rounded", key: "config.mcp.tools.ai", fallback: "AI" },
];
return (
<div className="settings-section-container">
<Stack gap="md" className="settings-section-content">
<div>
<Group gap="sm" align="center">
<ThemeIcon variant="light" size="lg" radius="md">
<LocalIcon icon="smart-toy-rounded" width={22} height={22} />
</ThemeIcon>
<Text fw={600} size="lg">
{t("config.mcp.title", "MCP Server")}
</Text>
</Group>
<Text size="sm" c="dimmed" mt={6}>
{t(
"config.mcp.description",
"Model Context Protocol (MCP) lets AI assistants like Claude use your Stirling PDF tools directly. Connect a client once and your assistant can convert, edit, secure and process documents on your behalf.",
)}
</Text>
</div>
{/* Endpoint */}
<Paper withBorder p="sm" radius="md">
<Group gap="xs" wrap="nowrap" align="center">
<Stack gap={2} style={{ flex: 1, minWidth: 0 }}>
<Text fw={500} size="sm">
{t("config.mcp.endpoint.label", "Your MCP endpoint")}
</Text>
<Code style={{ overflowX: "auto" }}>{mcpUrl}</Code>
</Stack>
<CopyInline
value={mcpUrl}
label={t("config.mcp.copy.endpointLabel", "Endpoint URL")}
/>
</Group>
</Paper>
{/* Per-client setup */}
<Paper withBorder p="sm" radius="md">
<Stack gap="xs">
<Text fw={500} size="sm">
{t("config.mcp.setup.title", "Connect your AI assistant")}
</Text>
<Text size="xs" c="dimmed">
{t(
"config.mcp.setup.hint",
"Pick your client, paste the snippet into the file shown, then restart it. You'll sign in with your Stirling account on first use - no keys to copy.",
)}
</Text>
<Tabs defaultValue="claude" variant="pills" radius="md" mt={4}>
<Tabs.List>
{clients.map((c) => (
<Tabs.Tab key={c.value} value={c.value}>
{c.label}
</Tabs.Tab>
))}
</Tabs.List>
{clients.map((c) => (
<Tabs.Panel key={c.value} value={c.value} pt="sm">
<Stack gap="xs">
<Group justify="space-between" align="center" wrap="nowrap">
<Text size="xs" c="dimmed">
{t("config.mcp.setup.addTo", "Add to")}{" "}
<Code>{c.file}</Code>
</Text>
<CopyInline
value={c.config}
label={t("config.mcp.copy.configLabel", "Config")}
/>
</Group>
<Code block>{c.config}</Code>
</Stack>
</Tabs.Panel>
))}
</Tabs>
</Stack>
</Paper>
{/* Available tools */}
<div>
<Text fw={500} size="sm" mb="xs">
{t("config.mcp.tools.title", "What your assistant can do")}
</Text>
<Group gap="xs">
{tools.map((tool) => (
<Badge
key={tool.key}
variant="light"
color="gray"
radius="sm"
size="lg"
leftSection={
<LocalIcon icon={tool.icon} width={13} height={13} />
}
>
{t(tool.key, tool.fallback)}
</Badge>
))}
</Group>
</div>
{/* Tip / cross-link */}
<Alert
variant="light"
color="blue"
icon={<LocalIcon icon="info-rounded" width="1rem" height="1rem" />}
>
<Group justify="space-between" align="center" wrap="nowrap" gap="sm">
<Text size="sm">
{t(
"config.mcp.tip",
"Every action your assistant runs is performed as your account and counts towards your usage, just like using Stirling PDF directly.",
)}
</Text>
<Button
size="xs"
variant="light"
style={{ flexShrink: 0 }}
leftSection={
<LocalIcon icon="key-rounded" width={14} height={14} />
}
onClick={() => openAppSettings("api-keys")}
>
{t("config.mcp.viewApiKeys", "View API keys")}
</Button>
</Group>
</Alert>
</Stack>
</div>
);
}
frontend/editor/src/saas/components/shared/config/saasConfigNavSections.tsx
+40
View File
@@ -9,6 +9,7 @@ import GeneralSection from "@app/components/shared/config/configSections/General
import PasswordSecurity from "@app/components/shared/config/configSections/PasswordSecurity";
import ApiKeys from "@app/components/shared/config/configSections/ApiKeys";
import Plan from "@app/components/shared/config/configSections/Plan";
import McpSection from "@app/components/shared/config/configSections/McpSection";
type OverviewComponent = React.ComponentType<{ onLogoutClick: () => void }>;
@@ -108,6 +109,44 @@ function appendBillingSection(
];
}
// Add an "MCP Server" tab in the Developer section. Always shown in SaaS;
// purely informational, so it appears for anonymous users too.
function appendMcpSection(
sections: ConfigNavSection[],
t: TFunction<"translation", undefined>,
): ConfigNavSection[] {
const hasMcp = sections.some((section) =>
section.items.some((item) => item.key === "mcp"),
);
if (hasMcp) {
return sections;
}
const mcpItem = {
key: "mcp" as const,
label: t("config.mcp.navLabel", "MCP Server"),
icon: "smart-toy-rounded",
component: <McpSection />,
};
const developerIndex = sections.findIndex((section) =>
section.items.some(
(item) => item.key === "developer" || item.key === "api-keys",
),
);
if (developerIndex === -1) {
return [...sections, { title: "Developer", items: [mcpItem] }];
}
return sections.map((section, index) =>
index === developerIndex
? { ...section, items: [...section.items, mcpItem] }
: section,
);
}
export function createSaasConfigNavSections(
Overview: OverviewComponent,
onLogoutClick: () => void,
@@ -151,6 +190,7 @@ export function createSaasConfigNavSections(
sections = ensurePreferencesSection(sections);
sections = appendDeveloperSection(sections);
sections = appendMcpSection(sections, t);
if (!isAnonymous) {
sections = appendBillingSection(sections, t);
frontend/editor/src/saas/components/shared/config/types.ts
+5 -4
View File
@@ -1,8 +1,9 @@
import { VALID_NAV_KEYS as CORE_NAV_KEYS } from "@core/components/shared/config/types";
// SaaS adds an "overview" account section. All other keys (including ones
// SaaS doesn't render today) come from core - subtracting them here would
// just break the type union without affecting runtime nav.
export const VALID_NAV_KEYS = [...CORE_NAV_KEYS, "overview"] as const;
// SaaS adds an "overview" account section and an "mcp" integrations tab. All
// other keys (including ones SaaS doesn't render today) come from core -
// subtracting them here would just break the type union without affecting
// runtime nav.
export const VALID_NAV_KEYS = [...CORE_NAV_KEYS, "overview", "mcp"] as const;
export type NavKey = (typeof VALID_NAV_KEYS)[number];
frontend/editor/src/saas/routes/Login.tsx
+196 -104
View File
@@ -7,17 +7,18 @@ import { useDocumentMeta } from "@app/hooks/useDocumentMeta";
import AuthLayout from "@app/routes/authShared/AuthLayout";
import "@app/routes/authShared/auth.css";
import "@app/routes/authShared/saas-auth.css";
import GuestSignInButton from "@app/routes/authShared/GuestSignInButton";
import {
absoluteWithBasePath,
getBaseUrl,
withBasePath,
} from "@app/constants/app";
import LinkRoundedIcon from "@mui/icons-material/LinkRounded";
// Import login components
import LoginHeader from "@app/routes/login/LoginHeader";
import ErrorMessage from "@app/routes/login/ErrorMessage";
import EmailPasswordForm from "@app/routes/login/EmailPasswordForm";
import MagicLinkForm from "@app/routes/login/MagicLinkForm";
import OAuthButtons from "@app/routes/login/OAuthButtons";
import DividerWithText from "@app/components/shared/DividerWithText";
import LoggedInState from "@app/routes/login/LoggedInState";
import { absoluteWithBasePath, getBaseUrl } from "@app/constants/app";
export default function Login() {
const navigate = useNavigate();
@@ -25,11 +26,13 @@ export default function Login() {
const { t } = useTranslation();
const [isSigningIn, setIsSigningIn] = useState(false);
const [error, setError] = useState<string | null>(null);
const [showMagicLink, setShowMagicLink] = useState(false);
const [showMagicLinkForm, setShowMagicLinkForm] = useState(false);
const [showEmailForm, setShowEmailForm] = useState(false);
const [email, setEmail] = useState("");
const [password, setPassword] = useState("");
const [magicLinkEmail, setMagicLinkEmail] = useState("");
const [magicLinkSent, setMagicLinkSent] = useState(false);
// Prefill email from query param (e.g. after password reset)
useEffect(() => {
try {
@@ -37,6 +40,7 @@ export default function Login() {
const emailFromQuery = url.searchParams.get("email");
if (emailFromQuery) {
setEmail(emailFromQuery);
setShowEmailForm(true);
}
} catch (_) {
// ignore
@@ -174,12 +178,9 @@ export default function Login() {
setError(error.message);
} else {
setError(null);
alert(t("login.magicLinkSent", { email: magicLinkEmail }));
setMagicLinkEmail("");
setShowMagicLink(false);
setMagicLinkSent(true);
}
} catch (err) {
console.error("[Login] Unexpected error:", err);
setError(
t("login.unexpectedError", {
message: err instanceof Error ? err.message : "Unknown error",
@@ -190,10 +191,6 @@ export default function Login() {
}
};
const handleForgotPassword = () => {
navigate("/auth/reset");
};
const handleAnonymousSignIn = async () => {
try {
setIsSigningIn(true);
@@ -227,115 +224,210 @@ export default function Login() {
}
};
const toggleEmailForm = () => {
setShowEmailForm((v) => !v);
setShowMagicLinkForm(false);
setMagicLinkSent(false);
};
const toggleMagicLink = () => {
setShowMagicLinkForm((v) => !v);
setShowEmailForm(false);
setMagicLinkSent(false);
};
return (
<AuthLayout isEmailFormExpanded={showEmailForm}>
<LoginHeader
title={t("login.login")}
subtitle={t("login.subtitle", "Sign back in to Stirling PDF")}
/>
<AuthLayout isEmailFormExpanded={showEmailForm || showMagicLinkForm}>
{/* Centered logo */}
<div className="auth-logo-block">
<img
src={withBasePath("/modern-logo/LoginLightModeHeader.svg")}
alt="Stirling PDF"
className="auth-logo-header auth-logo-header--light"
/>
<img
src={withBasePath("/modern-logo/LoginDarkModeHeader.svg")}
alt="Stirling PDF"
className="auth-logo-header auth-logo-header--dark"
/>
</div>
<ErrorMessage error={error} />
{/* OAuth first */}
<OAuthButtons
onProviderClick={signInWithProvider}
isSubmitting={isSigningIn}
layout="fullwidth"
/>
{/* OAuth + magic link group — single flex column so gap is uniform */}
<div
style={{
display: "flex",
flexDirection: "column",
gap: "0.75rem",
marginBottom: "2.5rem",
}}
>
<OAuthButtons
onProviderClick={signInWithProvider}
isSubmitting={isSigningIn}
layout="fullwidth"
labelPrefix={`${t("login.signInWith", "Sign in with")} `}
/>
{/* Divider between OAuth and Email */}
<DividerWithText
text={t("signup.or", "or")}
respondsToDarkMode={false}
opacity={0.4}
/>
{/* Magic link button + its expandable form as one unit */}
<div>
<button
type="button"
disabled={isSigningIn}
onClick={toggleMagicLink}
className={`oauth-button-fullwidth auth-expandable-trigger ${showMagicLinkForm ? "auth-expandable-trigger--active" : ""}`}
>
<span className="oauth-btn-group">
<LinkRoundedIcon
style={{
width: "1.75rem",
height: "1.75rem",
marginRight: "0.5rem",
flexShrink: 0,
}}
/>
<span className="oauth-btn-label">
{t("login.useMagicLink", "Use magic link")}
</span>
</span>
</button>
{/* Sign in with email button (primary color to match signup CTA) */}
<div className="auth-section">
<div
className={`auth-expand-grid ${showMagicLinkForm ? "auth-expand-grid--open" : ""}`}
>
<div className="auth-expand-inner">
<div style={{ paddingTop: "0.25rem" }}>
{magicLinkSent ? (
<p
style={{
fontSize: "0.875rem",
color: "#059669",
margin: 0,
}}
>
{t("login.magicLinkSent", { email: magicLinkEmail })}
</p>
) : (
<div className="auth-magic-row">
<input
type="email"
placeholder={t(
"login.enterEmailForMagicLink",
"Enter your email",
)}
value={magicLinkEmail}
onChange={(e) => setMagicLinkEmail(e.target.value)}
onKeyDown={(e) =>
e.key === "Enter" &&
!isSigningIn &&
signInWithMagicLink()
}
className="auth-input"
/>
<button
onClick={signInWithMagicLink}
disabled={isSigningIn || !magicLinkEmail}
className="auth-magic-button"
>
{isSigningIn
? t("login.sending")
: t("login.sendMagicLink")}
</button>
</div>
)}
</div>
</div>
</div>
</div>
</div>
{/* Email & Password button */}
<button
type="button"
disabled={isSigningIn}
onClick={toggleEmailForm}
className={`oauth-button-fullwidth auth-expandable-trigger ${showEmailForm ? "auth-expandable-trigger--active" : ""}`}
style={{ marginBottom: "0.75rem" }}
>
<span className="oauth-btn-group">
<span className="auth-at-icon">@</span>
<span className="oauth-btn-label">{`${t("login.signInWith", "Sign in with")} email`}</span>
</span>
</button>
{/* Email form — animated expand */}
<div
className={`auth-expand-grid ${showEmailForm ? "auth-expand-grid--open" : ""}`}
>
<div className="auth-expand-inner">
<div style={{ paddingBottom: "0.5rem" }}>
<EmailPasswordForm
email={email}
password={password}
setEmail={setEmail}
setPassword={setPassword}
onSubmit={signInWithEmail}
isSubmitting={isSigningIn}
submitButtonText={
isSigningIn ? t("login.loggingIn") : t("login.login")
}
/>
<button
type="button"
onClick={() => navigate("/auth/reset")}
className="auth-link-black"
style={{ fontSize: "0.8125rem", marginTop: "0.25rem" }}
>
{t("login.forgotPassword", "Forgot your password?")}
</button>
</div>
</div>
</div>
{/* Skip */}
<div style={{ textAlign: "center", margin: "1rem 0" }}>
<button
type="button"
onClick={() => setShowEmailForm((v) => !v)}
onClick={handleAnonymousSignIn}
disabled={isSigningIn}
className="w-full px-4 py-[0.75rem] rounded-[0.625rem] text-base font-semibold mb-2 cursor-pointer border-0 disabled:opacity-50 disabled:cursor-not-allowed auth-cta-button"
style={{
background: "none",
border: "none",
cursor: "pointer",
fontSize: "1rem",
fontWeight: 700,
color: "#000000",
}}
>
{t("login.useEmailInstead", "Sign in with email")}
{isSigningIn
? t("login.signingIn", "Signing in...")
: `${t("signup.skip", "Skip")} →`}
</button>
</div>
{showEmailForm && (
<EmailPasswordForm
email={email}
password={password}
setEmail={setEmail}
setPassword={setPassword}
onSubmit={signInWithEmail}
isSubmitting={isSigningIn}
submitButtonText={
isSigningIn ? t("login.loggingIn") : t("login.login")
}
/>
)}
{showEmailForm && (
<div className="auth-section-sm">
<button
type="button"
onClick={handleForgotPassword}
className="auth-link-black"
>
{t("login.forgotPassword", "Forgot your password?")}
</button>
</div>
)}
{/* Divider then Guest */}
<DividerWithText
text={t("signup.or", "or")}
respondsToDarkMode={false}
opacity={0.4}
/>
<GuestSignInButton
onClick={handleAnonymousSignIn}
disabled={isSigningIn}
label={
isSigningIn
? t("login.signingIn", "Signing in...")
: t("login.signInAnonymously", "Sign in as a Guest")
}
/>
<div className="auth-bottom-row">
<button
type="button"
onClick={() => setShowMagicLink(true)}
className="auth-link-black"
>
{t("login.useMagicLink", "Sign in with magic link")}
</button>
{/* Bottom */}
<div
style={{
textAlign: "center",
marginTop: "auto",
paddingTop: "1rem",
}}
>
<button
type="button"
onClick={() => navigate("/signup")}
className="auth-link-black"
style={{
background: "none",
border: "none",
cursor: "pointer",
fontSize: "0.875rem",
color: "#9ca3af",
}}
>
{t("signup.signUp", "Sign up")}
{t("login.createAccount", "Create an account")}
</button>
</div>
{/* Magic link form renders on demand */}
{showMagicLink && (
<div style={{ marginTop: "0.5rem" }}>
<MagicLinkForm
showMagicLink={showMagicLink}
magicLinkEmail={magicLinkEmail}
setMagicLinkEmail={setMagicLinkEmail}
setShowMagicLink={setShowMagicLink}
onSubmit={signInWithMagicLink}
isSubmitting={isSigningIn}
/>
</div>
)}
</AuthLayout>
);
}
frontend/editor/src/saas/routes/Signup.tsx
+84 -59
View File
@@ -4,18 +4,15 @@ import { signInAnonymously } from "@app/auth/supabase";
import { useAuth } from "@app/auth/UseSession";
import { useTranslation } from "@app/hooks/useTranslation";
import { useDocumentMeta } from "@app/hooks/useDocumentMeta";
import { getBaseUrl } from "@app/constants/app";
import { getBaseUrl, withBasePath } from "@app/constants/app";
import AuthLayout from "@app/routes/authShared/AuthLayout";
import "@app/routes/authShared/auth.css";
import "@app/routes/authShared/saas-auth.css";
import GuestSignInButton from "@app/routes/authShared/GuestSignInButton";
import { alert } from "@app/components/toast";
// Import signup components
import LoginHeader from "@app/routes/login/LoginHeader";
import ErrorMessage from "@app/routes/login/ErrorMessage";
import OAuthButtons from "@app/routes/login/OAuthButtons";
import DividerWithText from "@app/components/shared/DividerWithText";
import SignupForm from "@app/routes/signup/SignupForm";
import {
useSignupFormValidation,
@@ -184,84 +181,112 @@ export default function Signup() {
return (
<AuthLayout isEmailFormExpanded={showEmailForm}>
<LoginHeader title={t("signup.title")} subtitle={t("signup.subtitle")} />
{/* Centered logo */}
<div className="auth-logo-block">
<img
src={withBasePath("/modern-logo/LoginLightModeHeader.svg")}
alt="Stirling PDF"
className="auth-logo-header auth-logo-header--light"
/>
<img
src={withBasePath("/modern-logo/LoginDarkModeHeader.svg")}
alt="Stirling PDF"
className="auth-logo-header auth-logo-header--dark"
/>
</div>
<ErrorMessage error={error} />
{/* OAuth first */}
<div style={{ marginBottom: "0.5rem" }}>
{/* OAuth providers */}
<div>
<OAuthButtons
onProviderClick={handleProviderSignIn}
isSubmitting={isSigningUp}
layout="fullwidth"
labelPrefix={`${t("signup.signUpWith", "Sign up with")} `}
/>
</div>
{/* Divider between OAuth and Email */}
<div style={{ margin: "0.5rem 0" }}>
<DividerWithText
text={t("signup.or", "or")}
respondsToDarkMode={false}
opacity={0.4}
/>
{/* Email & Password button */}
<button
type="button"
disabled={isSigningUp}
onClick={() => setShowEmailForm((v) => !v)}
className={`oauth-button-fullwidth auth-expandable-trigger ${showEmailForm ? "auth-expandable-trigger--active" : ""}`}
style={{ marginTop: "2.5rem", marginBottom: "0.75rem" }}
>
<span className="oauth-btn-group">
<span className="auth-at-icon">@</span>
<span className="oauth-btn-label">{`${t("signup.signUpWith", "Sign up with")} email`}</span>
</span>
</button>
{/* Email form — animated expand */}
<div
className={`auth-expand-grid ${showEmailForm ? "auth-expand-grid--open" : ""}`}
>
<div className="auth-expand-inner">
<div style={{ paddingBottom: "0.5rem" }}>
<SignupForm
name={name}
email={email}
password={password}
confirmPassword={confirmPassword}
agree={agree}
setName={setName}
setEmail={setEmail}
setPassword={setPassword}
setConfirmPassword={setConfirmPassword}
setAgree={setAgree}
onSubmit={handleSignUp}
isSubmitting={isSigningUp}
fieldErrors={fieldErrors}
/>
</div>
</div>
</div>
{/* Use Email Instead button (toggles email form) */}
<div className="auth-section">
{/* Skip */}
<div style={{ textAlign: "center", margin: "1rem 0" }}>
<button
type="button"
onClick={handleAnonymousSignIn}
disabled={isSigningUp}
onClick={() => setShowEmailForm((v) => !v)}
className="w-full px-4 py-[0.75rem] rounded-[0.625rem] text-base font-semibold mb-2 cursor-pointer border-0 disabled:opacity-50 disabled:cursor-not-allowed auth-cta-button"
style={{
background: "none",
border: "none",
cursor: "pointer",
fontSize: "1rem",
fontWeight: 700,
color: "#000000",
}}
>
{t("signup.useEmailInstead", "Use Email Instead")}
{isSigningUp
? t("login.signingIn", "Signing in...")
: `${t("signup.skip", "Skip")} →`}
</button>
</div>
{showEmailForm && (
<SignupForm
name={name}
email={email}
password={password}
confirmPassword={confirmPassword}
agree={agree}
setName={setName}
setEmail={setEmail}
setPassword={setPassword}
setConfirmPassword={setConfirmPassword}
setAgree={setAgree}
onSubmit={handleSignUp}
isSubmitting={isSigningUp}
fieldErrors={fieldErrors}
/>
)}
<div className="auth-section-sm">
<DividerWithText
text={t("signup.or", "or")}
respondsToDarkMode={false}
opacity={0.4}
/>
</div>
<GuestSignInButton
onClick={handleAnonymousSignIn}
disabled={isSigningUp}
label={
isSigningUp
? t("login.signingIn", "Signing in...")
: t("login.signInAnonymously", "Sign in as a Guest")
}
/>
{/* Bottom row */}
<div className="auth-bottom-right">
{/* Bottom */}
<div
style={{
textAlign: "center",
marginTop: "auto",
paddingTop: "1rem",
}}
>
<button
type="button"
onClick={() => navigate("/login")}
className="auth-link-black"
style={{
background: "none",
border: "none",
cursor: "pointer",
fontSize: "0.875rem",
color: "#9ca3af",
}}
>
{t("login.logIn", "Log In")}
{t("signup.alreadyHaveAccount", "I already have an account")}
</button>
</div>
</AuthLayout>
frontend/editor/src/saas/routes/authShared/saas-auth.css
+97 -1
View File
@@ -14,7 +14,7 @@
justify-content: center;
padding: 0.75rem 1rem;
border: 1px solid #d1d5db;
border-radius: 0.625rem;
border-radius: 100px;
background-color: #ffffff;
font-size: 1rem;
font-weight: 600;
@@ -22,6 +22,10 @@
cursor: pointer;
gap: 0.5rem;
box-shadow: 0 0.125rem 0.375rem rgba(0, 0, 0, 0.04);
transition:
background-color 150ms ease,
box-shadow 150ms ease,
border-color 150ms ease;
}
.oauth-button-fullwidth:disabled {
@@ -29,6 +33,11 @@
opacity: 0.6;
}
.oauth-button-fullwidth:hover:not(:disabled) {
background-color: #fafafa;
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.08);
}
.auth-dropdown-wrapper {
position: relative;
}
@@ -72,3 +81,90 @@
color: #9c2f30;
border: 2px solid currentColor;
}
/* ── Logo block ─────────────────────────────────────────────────────── */
.auth-logo-block {
display: flex;
flex-direction: column;
align-items: center;
margin-bottom: 2rem;
margin-top: 0.5rem;
animation: authFadeUp 0.4s ease both;
}
.auth-logo-header {
height: 8rem;
width: auto;
}
.auth-logo-header--dark {
display: none;
}
/* ── Page entrance animation ────────────────────────────────────────── */
@keyframes authFadeUp {
from {
opacity: 0;
transform: translateY(12px);
}
to {
opacity: 1;
transform: translateY(0);
}
}
/* ── Expandable trigger button state ────────────────────────────────── */
.auth-expandable-trigger {
transition:
background-color 180ms ease,
box-shadow 180ms ease,
border-color 180ms ease;
}
.auth-expandable-trigger--active {
border-color: #af3434 !important;
box-shadow: 0 0 0 3px rgba(175, 52, 52, 0.12);
}
/* ── Animated expand/collapse via grid-template-rows ───────────────── */
.auth-expand-grid {
display: grid;
grid-template-rows: 0fr;
transition:
grid-template-rows 280ms cubic-bezier(0.4, 0, 0.2, 1),
opacity 220ms ease;
opacity: 0;
}
.auth-expand-grid--open {
grid-template-rows: 1fr;
opacity: 1;
}
.auth-expand-inner {
overflow: hidden;
min-height: 0;
}
/* ── Icon+label group — keeps icons vertically aligned across buttons ── */
.oauth-btn-group {
display: inline-flex;
align-items: center;
min-width: 12.5rem;
}
.oauth-btn-label {
flex: 1;
text-align: center;
}
/* ── Icon helpers inside oauth-button-fullwidth ─────────────────────── */
.auth-at-icon {
font-size: 1.25rem;
line-height: 1;
margin-right: 0.5rem;
display: inline-block;
width: 1.75rem;
text-align: center;
flex-shrink: 0;
}
frontend/editor/src/saas/routes/login/OAuthButtons.tsx
+14 -6
View File
@@ -17,6 +17,7 @@ interface OAuthButtonsProps {
isSubmitting: boolean;
layout?: "vertical" | "grid" | "icons" | "fullwidth";
enabledProviders?: string[]; // List of enabled provider IDs from backend
labelPrefix?: string;
}
export default function OAuthButtons({
@@ -24,6 +25,7 @@ export default function OAuthButtons({
isSubmitting,
layout = "vertical",
enabledProviders: _enabledProviders = [],
labelPrefix = "",
}: OAuthButtonsProps) {
const { t } = useTranslation();
@@ -92,12 +94,18 @@ export default function OAuthButtons({
className="oauth-button-fullwidth"
title={p.label}
>
<img
src={withBasePath(`/Login/${p.file}`)}
alt={p.label}
className={`oauth-icon-medium ${p.isDisabled ? "opacity-20" : ""}`}
/>
{p.label}
<span className="oauth-btn-group">
<img
src={withBasePath(`/Login/${p.file}`)}
alt={p.label}
className={`oauth-icon-medium ${p.isDisabled ? "opacity-20" : ""}`}
style={{ marginRight: "0.5rem", flexShrink: 0 }}
/>
<span className="oauth-btn-label">
{labelPrefix}
{p.label}
</span>
</span>
</button>
))}
</div>
frontend/editor/src/saas/services/apiClient.test.ts
+9
View File
@@ -13,6 +13,15 @@ vi.mock("@app/auth/supabase", () => ({
},
}));
// Stub apiClient's heavy UI/util deps so importing it stays cheap. None are
// exercised here (toast, plan settings and handleHttpError paths aren't hit),
// and re-importing the real toast graph per test made these tests time out.
vi.mock("@app/components/toast", () => ({ alert: vi.fn() }));
vi.mock("@app/utils/appSettings", () => ({ openPlanSettings: vi.fn() }));
vi.mock("@app/services/httpErrorHandler", () => ({
handleHttpError: vi.fn().mockResolvedValue(undefined),
}));
describe("apiClient", () => {
beforeEach(() => {
vi.clearAllMocks();
frontend/editor/src/saas/styles/saas-theme.css
+2 -2
View File
@@ -35,7 +35,7 @@
--auth-label-text: #2b3230;
--auth-button-bg: #af3434;
--auth-button-text: #ffffff;
--auth-magic-button-bg: #8b5cf6;
--auth-magic-button-bg: #af3434;
--auth-magic-button-text: #ffffff;
/* Light-only auth colors (no dark mode equivalents) used for login/signup */
@@ -45,7 +45,7 @@
--auth-label-text-light-only: #2b3230;
--auth-button-bg-light-only: #af3434;
--auth-button-text-light-only: #ffffff;
--auth-magic-button-bg-light-only: #8b5cf6;
--auth-magic-button-bg-light-only: #af3434;
--auth-magic-button-text-light-only: #ffffff;
--auth-bg-color-light-only: #ffffff;
--auth-card-bg-light-only: #ffffff;
testing/cucumber/features/external.feature
+1 -2
View File
@@ -233,8 +233,7 @@ Feature: API Validation
When I send the API request to the endpoint "/api/v1/convert/pdf/markdown"
Then the response status code should be 200
And the response file should have size greater than 100
And the response file should have extension ".zip"
And the response ZIP should contain 4 files
And the response file should have extension ".md"
@positive @pdftocsv
testing/test.sh
+3 -3
View File
@@ -911,7 +911,7 @@ main() {
CUCUMBER_JUNIT_DIR="$PROJECT_ROOT/testing/cucumber/junit"
mkdir -p "$CUCUMBER_JUNIT_DIR"
cd "testing/cucumber"
start_test_timer "Stirling-PDF-Regression"
start_test_timer "Stirling-PDF-Regression $CONTAINER_NAME"
# Snapshot docker log line count before behave so we can extract only behave-window logs
DOCKER_LOG_BEFORE=$(docker logs "$CONTAINER_NAME" 2>&1 | wc -l)
@@ -999,7 +999,7 @@ main() {
# Save docker logs from the behave window to a dedicated file
local cucumber_log="$REPORT_DIR/cucumber-docker-context.log"
docker logs "$CONTAINER_NAME" 2>&1 | tail -n +"$((DOCKER_LOG_BEFORE + 1))" > "$cucumber_log" 2>/dev/null || true
test_failure_logs["Stirling-PDF-Regression"]="$cucumber_log"
test_failure_logs["Stirling-PDF-Regression $CONTAINER_NAME"]="$cucumber_log"
gha_group "Docker logs during behave run: $CONTAINER_NAME"
tail -100 "$cucumber_log"
@@ -1012,7 +1012,7 @@ main() {
capture_file_list "$CONTAINER_NAME" "$AFTER_FILE"
compare_file_lists "$BEFORE_FILE" "$AFTER_FILE" "$DIFF_FILE" "$CONTAINER_NAME"
fi
stop_test_timer "Stirling-PDF-Regression"
stop_test_timer "Stirling-PDF-Regression $CONTAINER_NAME"
fi
# `down` with the override removes the agent bind-mount cleanly. The
# SIGTERM that `down` sends is what triggers dumponexit=true in the