diff --git a/.taskfiles/engine.yml b/.taskfiles/engine.yml index 5c9d6a8e6..1834495b1 100644 --- a/.taskfiles/engine.yml +++ b/.taskfiles/engine.yml @@ -1,5 +1,12 @@ version: '3' +vars: + # Engine-specific names to avoid overriding the root Taskfile's FIND_FREE_PORT_* + # vars (Task merges included-file vars into the global scope). + # Paths are relative to the engine/ include dir. + ENGINE_FIND_FREE_PORT_SH: "bash ../scripts/find-free-port.sh" + ENGINE_FIND_FREE_PORT_PS: "powershell -NoProfile -File ../scripts/find-free-port.ps1" + tasks: install: desc: "Install engine dependencies" @@ -29,7 +36,10 @@ tasks: ignore_error: true dir: src vars: - PORT: '{{.PORT | default "5001"}}' + # When PORT is provided (e.g. from dev:all), use it directly. + # When running standalone, probe for a free port starting at 5001. + PORT: + sh: '{{if .PORT}}echo {{.PORT}}{{else if eq OS "windows"}}{{.ENGINE_FIND_FREE_PORT_PS}} 5001{{else}}{{.ENGINE_FIND_FREE_PORT_SH}} 5001{{end}}' env: PYTHONUNBUFFERED: "1" cmds: @@ -41,7 +51,10 @@ tasks: ignore_error: true dir: src vars: - PORT: '{{.PORT | default "5001"}}' + # When PORT is provided (e.g. from dev:all), use it directly. + # When running standalone, probe for a free port starting at 5001. + PORT: + sh: '{{if .PORT}}echo {{.PORT}}{{else if eq OS "windows"}}{{.ENGINE_FIND_FREE_PORT_PS}} 5001{{else}}{{.ENGINE_FIND_FREE_PORT_SH}} 5001{{end}}' env: PYTHONUNBUFFERED: "1" cmds: diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java deleted file mode 100644 index 429f180f3..000000000 --- a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java +++ /dev/null @@ -1,73 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.io.IOException; -import java.util.List; - -import org.apache.pdfbox.pdmodel.PDDocument; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Service; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** - * Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link - * LineAlignmentTableParser}. The first parser returning a result above {@link - * #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page. - */ -@Service -@Primary -@RequiredArgsConstructor -@Slf4j -public class CompositeTableParser implements TableParser { - - /** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */ - static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f; - - private final TabulaTableParser tabulaParser; - private final LineAlignmentTableParser lineAlignmentParser; - - @Override - public List parse(PDDocument document, RawPage rawPage) throws IOException { - // Step 1: Tabula lattice mode (ruled/bordered tables). - List latticeResults = filterConfident(tabulaParser.parse(document, rawPage)); - if (!latticeResults.isEmpty()) { - log.debug( - "Page {}: using Tabula lattice ({} table(s))", - rawPage.pageNumber(), - latticeResults.size()); - return latticeResults; - } - - // Step 2: Tabula stream mode (borderless/whitespace-delimited tables). - // parseStream is not on the TableParser interface — this intentionally couples to the - // concrete TabulaTableParser since stream mode is a Tabula-specific concept. - List streamResults = - filterConfident(tabulaParser.parseStream(document, rawPage)); - if (!streamResults.isEmpty()) { - log.debug( - "Page {}: using Tabula stream ({} table(s))", - rawPage.pageNumber(), - streamResults.size()); - return streamResults; - } - - // Step 3: Geometry-based line-alignment fallback. - List lineResults = lineAlignmentParser.parse(document, rawPage); - if (!lineResults.isEmpty()) { - log.debug( - "Page {}: using LineAlignment ({} table(s))", - rawPage.pageNumber(), - lineResults.size()); - return lineResults; - } - - return List.of(); - } - - private List filterConfident(List tables) { - return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList(); - } -} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java deleted file mode 100644 index b2d8de516..000000000 --- a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java +++ /dev/null @@ -1,528 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.regex.Pattern; - -import org.apache.pdfbox.pdmodel.PDDocument; -import org.springframework.stereotype.Service; - -import lombok.extern.slf4j.Slf4j; - -/** - * Fallback {@link TableParser} for borderless financial tables using text geometry. - * - *

Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge - * positions, groups vertically proximate anchor lines into table candidates, then scores each group - * on column consistency and anchor density (confidence ceiling 0.85). - */ -@Service -@Slf4j -public class LineAlignmentTableParser implements TableParser { - - /** Width in points of each column position bucket. */ - static final float COLUMN_BUCKET_PT = 5f; - - /** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */ - private static final int COLUMN_MATCH_BUCKETS = 2; - - /** Maximum gap (as a multiple of modal line spacing) before splitting a group. */ - private static final float MAX_GAP_FACTOR = 2.5f; - - /** Minimum anchor rows (numeric-heavy) to form a valid table. */ - static final int MIN_TABLE_ROWS = 3; - - /** Minimum confirmed column positions to form a valid table. */ - static final int MIN_COLUMNS = 2; - - /** - * Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A - * rows). - */ - private static final double COLUMN_MIN_FREQUENCY = 0.40; - - /** - * Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency, - * percent, nil dashes. - */ - private static final Pattern NUMERIC = - Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$"); - - /** - * Lines within this y-distance are merged into one row (restores rows split by LineBuilder's - * column-gap logic). - */ - static final float ROW_MERGE_TOLERANCE_PT = 2f; - - // ── public API ─────────────────────────────────────────────────────────────────────────────── - - @Override - public List parse(PDDocument document, RawPage rawPage) throws IOException { - List lines = rawPage.lines(); - if (lines.size() < MIN_TABLE_ROWS) return List.of(); - - float modalSpacing = computeModalSpacing(lines); - List tokenized = - mergeCoincidentLines(lines.stream().map(this::tokenize).toList()); - - List anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList(); - - if (anchors.size() < MIN_TABLE_ROWS) return List.of(); - - List columnGrid = buildColumnGrid(anchors); - if (columnGrid.size() < MIN_COLUMNS) { - log.debug( - "Page {}: LineAlignment — fewer than {} confirmed columns, skipping", - rawPage.pageNumber(), - MIN_COLUMNS); - return List.of(); - } - - List> groups = groupRows(tokenized, columnGrid, modalSpacing); - - List results = new ArrayList<>(); - for (int i = 0; i < groups.size(); i++) { - buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i) - .ifPresent(results::add); - } - - log.debug( - "Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)", - rawPage.pageNumber(), - results.size(), - anchors.size(), - columnGrid.size()); - return results; - } - - // ── coincident-line merging ────────────────────────────────────────────────────────────────── - - /** - * Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves - * split by LineBuilder. - */ - List mergeCoincidentLines(List tokenized) { - if (tokenized.size() < 2) return tokenized; - - List result = new ArrayList<>(); - int i = 0; - - while (i < tokenized.size()) { - float baseY = tokenized.get(i).line().bounds().y(); - int j = i + 1; - while (j < tokenized.size() - && Math.abs(tokenized.get(j).line().bounds().y() - baseY) - <= ROW_MERGE_TOLERANCE_PT) { - j++; - } - - if (j == i + 1) { - result.add(tokenized.get(i)); - } else { - result.add(mergeGroup(tokenized.subList(i, j))); - } - i = j; - } - - return result; - } - - private TokenizedLine mergeGroup(List group) { - List mergedFragments = - group.stream() - .flatMap(tl -> tl.line().fragments().stream()) - .sorted(Comparator.comparingDouble(f -> f.bounds().x())) - .toList(); - - Bounds mergedBounds = - group.stream() - .map(tl -> tl.line().bounds()) - .reduce(Bounds::merge) - .orElse(group.get(0).line().bounds()); - - RawLine mergedLine = - new RawLine( - group.get(0).line().lineId(), - mergedFragments, - mergedBounds, - group.get(0).line().pageNumber()); - - return tokenize(mergedLine); - } - - // ── tokenisation ───────────────────────────────────────────────────────────────────────────── - - /** - * Splits fragments into word-level tokens; x-positions are estimated linearly within each - * fragment. - */ - TokenizedLine tokenize(RawLine line) { - List tokens = new ArrayList<>(); - for (TextFragment frag : line.fragments()) { - tokens.addAll(tokensFromFragment(frag)); - } - List numeric = tokens.stream().filter(LineToken::numeric).toList(); - return new TokenizedLine(line, tokens, numeric); - } - - private List tokensFromFragment(TextFragment frag) { - String raw = frag.text(); - if (raw == null || raw.isBlank()) return List.of(); - - float fragX = frag.bounds().x(); - float fragWidth = frag.bounds().width(); - int rawLen = raw.length(); - - List result = new ArrayList<>(); - int offset = 0; - for (String part : raw.split("\\s+")) { - if (part.isEmpty()) { - offset++; - continue; - } - int idx = raw.indexOf(part, offset); - if (idx < 0) idx = offset; - - float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX; - float tokenRight = - rawLen > 0 - ? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth - : fragX + fragWidth; - - result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches())); - offset = idx + part.length(); - } - return result; - } - - // ── column grid ────────────────────────────────────────────────────────────────────────────── - - /** - * Returns confirmed column right-edge positions — those appearing on ≥ {@value - * #COLUMN_MIN_FREQUENCY} × N anchor lines. - */ - private List buildColumnGrid(List anchors) { - // bucket → set of line indices that contributed a numeric token to that bucket - Map> bucketLines = new HashMap<>(); - for (int i = 0; i < anchors.size(); i++) { - for (LineToken t : anchors.get(i).numeric()) { - int bucket = bucket(t.right()); - bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i); - } - } - - int minHits = - Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY)); - - // Confirmed buckets → average right-edge for that bucket - TreeMap confirmed = new TreeMap<>(); - for (Map.Entry> entry : bucketLines.entrySet()) { - // Count distinct lines - long distinctLines = entry.getValue().stream().distinct().count(); - if (distinctLines >= minHits) { - double avg = - entry.getValue().stream() - .distinct() // weight each line equally regardless of token count - .mapToDouble( - lineIdx -> - avgRightEdgeForBucket( - anchors, lineIdx, entry.getKey())) - .average() - .orElse(entry.getKey() * (double) COLUMN_BUCKET_PT); - confirmed.put(entry.getKey(), (float) avg); - } - } - - return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right) - } - - /** - * Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code - * targetBucket}, falling back to the bucket's nominal centre when no tokens match. - */ - private double avgRightEdgeForBucket( - List anchors, int lineIdx, int targetBucket) { - return anchors.get(lineIdx).numeric().stream() - .filter(t -> bucket(t.right()) == targetBucket) - .mapToDouble(LineToken::right) - .average() - .orElse(targetBucket * (double) COLUMN_BUCKET_PT); - } - - // ── grouping ───────────────────────────────────────────────────────────────────────────────── - - /** - * Groups anchor lines into table candidates, including adjacent label rows; a gap > - * MAX_GAP_FACTOR × modal spacing splits groups. - */ - private List> groupRows( - List all, List columnGrid, float modalSpacing) { - float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f; - - List> groups = new ArrayList<>(); - List current = new ArrayList<>(); - - for (int i = 0; i < all.size(); i++) { - TokenizedLine tl = all.get(i); - boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid); - - if (current.isEmpty()) { - if (fits) current.add(tl); - continue; - } - - float gap = - tl.line().bounds().y() - - current.get(current.size() - 1).line().bounds().bottom(); - - if (gap > maxGap) { - groups.add(current); - current = new ArrayList<>(); - if (fits) current.add(tl); - continue; - } - - if (fits) { - current.add(tl); - } else if (!tl.line().text().isBlank()) { - // Include non-anchor lines (labels) only if they have text and are within - // proximity. - current.add(tl); - } - } - - if (!current.isEmpty()) groups.add(current); - - return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList(); - } - - private boolean hasEnoughAnchorRows(List group, List columnGrid) { - return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count() - >= MIN_TABLE_ROWS; - } - - /** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */ - private boolean matchesGrid(TokenizedLine tl, List columnGrid) { - if (tl.numeric().isEmpty()) return false; - long matches = - tl.numeric().stream() - .filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0) - .count(); - return (double) matches / tl.numeric().size() >= 0.60; - } - - private boolean hasInconsistentColumnMatch(TokenizedLine tl, List columnGrid) { - if (tl.numeric().isEmpty()) return false; - long hits = - tl.numeric().stream() - .filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0) - .count(); - return (double) hits / tl.numeric().size() < 0.60; - } - - // ── fragment assembly ──────────────────────────────────────────────────────────────────────── - - private Optional buildFragment( - List group, List columnGrid, int pageNumber, int tableIndex) { - - long anchorCount = - group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count(); - if (anchorCount < MIN_TABLE_ROWS) return Optional.empty(); - - List warnings = new ArrayList<>(); - List> rawRows = new ArrayList<>(); - List rows = new ArrayList<>(); - - for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) { - TokenizedLine tl = group.get(rowIdx); - List rawRow = buildRawRow(tl, columnGrid); - rawRows.add(Collections.unmodifiableList(rawRow)); - rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid)); - } - - // Column count = 1 label column + confirmed numeric columns - int colCount = columnGrid.size() + 1; - Bounds bounds = computeGroupBounds(group); - float confidence = computeConfidence(group, columnGrid, warnings); - - return Optional.of( - new TableFragment( - "tbl-la-p" + pageNumber + "-" + tableIndex, - pageNumber, - bounds, - List.of(), - Collections.unmodifiableList(rows), - Collections.unmodifiableList(rawRows), - colCount, - confidence, - Collections.unmodifiableList(warnings), - null)); - } - - /** - * Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values. - */ - private List buildRawRow(TokenizedLine tl, List columnGrid) { - String[] cells = new String[columnGrid.size() + 1]; - Arrays.fill(cells, ""); - - // Separate label tokens (those not landing in any confirmed column) from column tokens. - List labelParts = new ArrayList<>(); - for (LineToken token : tl.all()) { - int col = nearestColumnIndex(token.right(), columnGrid); - if (col >= 0 && token.numeric()) { - int cellIdx = col + 1; - cells[cellIdx] = - cells[cellIdx].isEmpty() - ? token.text() - : cells[cellIdx] + " " + token.text(); - } else { - labelParts.add(token.text()); - } - } - cells[0] = String.join(" ", labelParts).trim(); - return Arrays.asList(cells); - } - - private TableRow buildTableRow( - int rowIdx, TokenizedLine tl, List rawRow, List columnGrid) { - List cells = new ArrayList<>(rawRow.size()); - - // Label cell: use the line's full bounds as an approximation. - cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds())); - - for (int col = 0; col < columnGrid.size(); col++) { - String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : ""; - float right = columnGrid.get(col); - float left = col > 0 ? columnGrid.get(col - 1) : right - 50f; - Bounds cellBounds = - new Bounds( - left, - tl.line().bounds().y(), - right - left, - tl.line().bounds().height()); - cells.add(TableCell.of(col + 1, text, cellBounds)); - } - return new TableRow(rowIdx, Collections.unmodifiableList(cells)); - } - - // ── confidence scoring ─────────────────────────────────────────────────────────────────────── - - /** - * Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at - * 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if - * >30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors. - */ - private float computeConfidence( - List group, List columnGrid, List warnings) { - float score = 0.70f; - - long anchorCount = - group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count(); - long totalRows = group.size(); - - // More columns - int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2); - score += extraCols * 0.05f; - - // More anchor rows - if (anchorCount >= 5) score += 0.05f; - if (anchorCount >= 8) score += 0.05f; - - // Inconsistent column matching - long inconsistent = - group.stream() - .filter(TokenizedLine::isAnchor) - .filter(tl -> hasInconsistentColumnMatch(tl, columnGrid)) - .count(); - if (inconsistent > anchorCount * 0.30) { - score -= 0.15f; - warnings.add( - "Column match inconsistent on " - + inconsistent - + "/" - + anchorCount - + " anchor rows"); - } - - // Label-heavy - long nonAnchor = totalRows - anchorCount; - if (nonAnchor > anchorCount) { - score -= 0.10f; - warnings.add( - "Non-anchor rows (" - + nonAnchor - + ") outnumber anchor rows (" - + anchorCount - + ")"); - } - - return Math.max(0f, Math.min(0.85f, score)); - } - - // ── utility ────────────────────────────────────────────────────────────────────────────────── - - /** - * Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value - * #COLUMN_MATCH_BUCKETS} buckets. - */ - private int nearestColumnIndex(float rightEdge, List grid) { - int nearest = -1; - float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f; - for (int i = 0; i < grid.size(); i++) { - float dist = Math.abs(rightEdge - grid.get(i)); - if (dist < minDist) { - minDist = dist; - nearest = i; - } - } - return nearest; - } - - private Bounds computeGroupBounds(List group) { - return group.stream() - .map(tl -> tl.line().bounds()) - .reduce(Bounds::merge) - .orElse(new Bounds(0, 0, 0, 0)); - } - - /** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */ - private float computeModalSpacing(List lines) { - if (lines.size() < 2) return 0f; - Map freq = new HashMap<>(); - for (int i = 1; i < lines.size(); i++) { - float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom(); - if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum); - } - return freq.entrySet().stream() - .max(Map.Entry.comparingByValue()) - .map(Map.Entry::getKey) - .orElse(0f); - } - - private static int bucket(float x) { - return Math.round(x / COLUMN_BUCKET_PT); - } - - // ── private data types ─────────────────────────────────────────────────────────────────────── - - /** A word-level token with an approximate right-edge x-position. */ - record LineToken(String text, float x, float right, boolean numeric) {} - - /** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */ - record TokenizedLine(RawLine line, List all, List numeric) { - boolean isAnchor() { - return numeric.size() >= 2; - } - } -} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java deleted file mode 100644 index 6831f6d73..000000000 --- a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java +++ /dev/null @@ -1,139 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - -import org.springframework.stereotype.Service; - -import lombok.extern.slf4j.Slf4j; - -/** - * Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity. - * - *

Fragments are on the same line when their baselines are within a font-size-derived tolerance. - * A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code - * max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text. - */ -@Service -@Slf4j -public class LineBuilder { - - /** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */ - private static final float BASELINE_TOLERANCE_FACTOR = 0.5f; - - /** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */ - private static final float MIN_BASELINE_TOLERANCE = 2f; - - /** - * Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below - * two-column gutters. - */ - static final float COLUMN_GAP_RATIO = 0.10f; - - /** Floor for the column-gap threshold so narrow pages don't over-split lines. */ - static final float COLUMN_GAP_MIN_PT = 40f; - - public List build(List fragments, int pageNumber) { - if (fragments.isEmpty()) return List.of(); - - float effectiveWidth = inferEffectiveWidth(fragments); - float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT); - log.debug( - "LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt", - pageNumber, - effectiveWidth, - columnGapThreshold); - - // Sort top-to-bottom first, then left-to-right within the same baseline band. - List sorted = - fragments.stream() - .sorted( - Comparator.comparingDouble(TextFragment::baseline) - .thenComparingDouble(f -> f.bounds().x())) - .toList(); - - List> groups = groupByBaseline(sorted, columnGapThreshold); - - List lines = new ArrayList<>(groups.size()); - for (int i = 0; i < groups.size(); i++) { - List group = - groups.get(i).stream() - .sorted(Comparator.comparingDouble(f -> f.bounds().x())) - .toList(); - - Bounds lineBounds = - group.stream() - .map(TextFragment::bounds) - .reduce(Bounds::merge) - .orElse(new Bounds(0, 0, 0, 0)); - - lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber)); - } - return lines; - } - - private List> groupByBaseline( - List sorted, float columnGapThreshold) { - List> groups = new ArrayList<>(); - List current = new ArrayList<>(); - float currentBaseline = Float.NaN; - - for (TextFragment fragment : sorted) { - if (current.isEmpty()) { - current.add(fragment); - currentBaseline = fragment.baseline(); - continue; - } - - float maxFontSize = - Math.max( - fragment.fontSize(), - (float) - current.stream() - .mapToDouble(TextFragment::fontSize) - .max() - .orElse(0)); - float tolerance = - Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE); - - boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance; - boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold); - - if (sameBaseline && !columnGap) { - current.add(fragment); - // Anchor to the weighted mean baseline so long lines stay stable. - currentBaseline = - (currentBaseline * (current.size() - 1) + fragment.baseline()) - / current.size(); - } else { - groups.add(current); - current = new ArrayList<>(); - current.add(fragment); - currentBaseline = fragment.baseline(); - } - } - - if (!current.isEmpty()) groups.add(current); - return groups; - } - - /** - * True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code - * threshold}. - */ - private static boolean hasColumnGap( - TextFragment next, List group, float threshold) { - float lastRight = group.get(group.size() - 1).bounds().right(); - return next.bounds().x() - lastRight > threshold; - } - - /** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */ - private static float inferEffectiveWidth(List fragments) { - double maxRight = - fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0); - return (float) maxRight * 1.10f; - } -} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java deleted file mode 100644 index a7dc9c282..000000000 --- a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java +++ /dev/null @@ -1,79 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.common.PDRectangle; -import org.springframework.stereotype.Service; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** - * Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} → - * {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the - * {@link PDDocument} lifecycle. - */ -@Service -@RequiredArgsConstructor -@Slf4j -public class PdfIngester { - - private final LineBuilder lineBuilder; - private final TableParser tableParser; - - public List parse(PDDocument document) throws IOException { - return parse(document, document.getNumberOfPages()); - } - - public List parse(PDDocument document, int maxPages) throws IOException { - int pageCount = Math.min(document.getNumberOfPages(), maxPages); - List pages = new ArrayList<>(pageCount); - long fragmentsMs = 0; - long tablesMs = 0; - long t0 = System.currentTimeMillis(); - - for (int p = 1; p <= pageCount; p++) { - long ft = System.currentTimeMillis(); - List fragments = extractFragments(document, p); - fragmentsMs += System.currentTimeMillis() - ft; - - PDPage page = document.getPage(p - 1); - PDRectangle mediaBox = page.getMediaBox(); - List lines = lineBuilder.build(fragments, p); - RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines); - - long tt = System.currentTimeMillis(); - List tables = tableParser.parse(document, rawPage); - tablesMs += System.currentTimeMillis() - tt; - - log.debug( - "Page {}: {} fragments → {} lines, {} table(s)", - p, - fragments.size(), - lines.size(), - tables.size()); - pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines)); - } - - log.info( - "[timing] parse pages={} total={}ms fragments={}ms tables={}ms", - pageCount, - System.currentTimeMillis() - t0, - fragmentsMs, - tablesMs); - return pages; - } - - private List extractFragments(PDDocument document, int pageNumber) - throws IOException { - WordExtractingStripper stripper = new WordExtractingStripper(pageNumber); - stripper.getText(document); - return stripper.getFragments(); - } -} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java deleted file mode 100644 index 52ab9d9a1..000000000 --- a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java +++ /dev/null @@ -1,113 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.font.PDFont; -import org.apache.pdfbox.text.PDFTextStripper; -import org.apache.pdfbox.text.TextPosition; - -/** - * Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata. - * - *

Overrides {@link #writeString} to split each content-stream string into word-level {@link - * TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in - * PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline. - */ -class WordExtractingStripper extends PDFTextStripper { - - private final int targetPage; - private final List fragments = new ArrayList<>(); - private int fragmentIndex = 0; - - WordExtractingStripper(int pageNumber) throws IOException { - this.targetPage = pageNumber; - setStartPage(pageNumber); - setEndPage(pageNumber); - setSortByPosition(true); - } - - @Override - protected void startPage(PDPage page) throws IOException { - super.startPage(page); - fragments.clear(); - fragmentIndex = 0; - } - - @Override - protected void writeString(String text, List textPositions) throws IOException { - if (text == null || text.isBlank()) return; - - // Fast path: no whitespace → emit one fragment (most financial PDFs have each - // number as its own string operation, so this is the common case). - if (text.indexOf(' ') < 0) { - emitFragment(text, textPositions); - return; - } - - // Per-word splitting requires 1:1 text-char to TextPosition correspondence. - // Fall back to one fragment when sizes differ (ligatures, encoding edge cases). - if (textPositions.size() != text.length()) { - emitFragment(text, textPositions); - return; - } - - // Emit one TextFragment per whitespace-delimited word with accurate per-word bounds. - int start = 0; - for (int i = 0; i <= text.length(); i++) { - if (i == text.length() || text.charAt(i) == ' ') { - if (start < i) { - emitFragment(text.substring(start, i), textPositions.subList(start, i)); - } - start = i + 1; - } - } - } - - private void emitFragment(String text, List positions) { - if (positions.isEmpty()) return; - - float minX = Float.MAX_VALUE; - float minY = Float.MAX_VALUE; - float maxRight = -Float.MAX_VALUE; - float maxBaseline = -Float.MAX_VALUE; - TextPosition first = null; - - for (TextPosition tp : positions) { - if (tp == null) continue; - if (first == null) first = tp; - - float x = tp.getX(); - // getY() is the baseline; top of character = getY() - getHeight(). - float top = tp.getY() - tp.getHeight(); - float right = x + tp.getWidth(); - float baseline = tp.getY(); - - minX = Math.min(minX, x); - minY = Math.min(minY, top); - maxRight = Math.max(maxRight, right); - maxBaseline = Math.max(maxBaseline, baseline); - } - - if (first == null) return; - - PDFont font = first.getFont(); - String fontName = font != null ? font.getName() : ""; - boolean bold = fontName != null && fontName.toLowerCase().contains("bold"); - // getHeight() gives the rendered glyph height, which is the most reliable visual size. - float fontSize = first.getHeight(); - - Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY); - String id = "tf-p" + targetPage + "-" + fragmentIndex++; - fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold)); - } - - List getFragments() { - return Collections.unmodifiableList(fragments); - } -} diff --git a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java index 00e26ec64..067be54eb 100644 --- a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java +++ b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java @@ -80,6 +80,7 @@ public class ApplicationProperties { private Mcp mcp = new Mcp(); private InternalApi internalApi = new InternalApi(); private Cluster cluster = new Cluster(); + private Policies policies = new Policies(); @Bean public PropertySource dynamicYamlPropertySource(ConfigurableEnvironment environment) @@ -203,6 +204,45 @@ public class ApplicationProperties { } } + @Data + public static class Policies { + /** + * Absolute directories that policy folder input sources and output sinks may read from or + * write to. Empty (the default) disables folder access entirely, so a policy can never be + * pointed at an arbitrary server path. Stirling's own config directory is always + * off-limits, and folder access is always disabled in SaaS mode regardless of this list. + */ + private List allowedFolderRoots = new java.util.ArrayList<>(); + + /** How often (seconds) the schedule trigger checks for policies whose schedule is due. */ + private long scheduleSweepSeconds = 60; + + /** + * How often (seconds) the folder-watch trigger reconciles its watch registrations and + * re-runs every folder-watch policy as a safety net for filesystem events that were missed + * (NFS, bind mounts, inotify-queue overflow). + */ + private long watchReconcileSeconds = 300; + + /** + * How long (milliseconds) the folder-watch trigger keeps draining filesystem events after + * the first, so a burst from a single file copy coalesces into one run instead of many. + */ + private long watchQuietPeriodMs = 500; + + /** + * SSE emitter timeout (milliseconds) for streamed runs; generous for long multi-step runs. + */ + private long streamTimeoutMs = 1800000; + + /** + * How long (minutes) a finished run's in-memory state is retained before eviction, + * mirroring the job-result expiry so rich run state does not outlive the process. Active + * and paused runs are kept regardless of age. + */ + private int runExpiryMinutes = 30; + } + @Data public static class PdfEditor { private Cache cache = new Cache(); diff --git a/app/common/src/main/java/stirling/software/common/pdf/HeadingDetector.java b/app/common/src/main/java/stirling/software/common/pdf/HeadingDetector.java new file mode 100644 index 000000000..0937cef64 --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/pdf/HeadingDetector.java @@ -0,0 +1,191 @@ +package stirling.software.common.pdf; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import stirling.software.jpdfium.text.PageText; +import stirling.software.jpdfium.text.TextChar; +import stirling.software.jpdfium.text.TextLine; +import stirling.software.jpdfium.text.TextWord; + +final class HeadingDetector { + + private HeadingDetector() {} + + /** A heading is at most this many words; longer lines are treated as body text. */ + private static final int MAX_HEADING_WORDS = 12; + + /** + * Returns the Markdown heading prefix for a line. The decision combines several signals, never + * text matching, so a plain line that merely shares text with a heading is never promoted: + * + *

    + *
  • Size — dominant glyph font size vs. the document body median (primary signal). + * Some PDFs encode visual size in the text matrix, so every glyph reports ~1.0; for those + * the line height is used as the proxy instead. + *
  • Brevity — headings are short labels; a line over {@value #MAX_HEADING_WORDS} + * words is body text regardless of size. + *
  • Not a sentence — a line ending in {@code . ! ?} reads as prose, not a heading. + *
+ * + *

Boldness is deliberately not a heading signal — a bold-but-not-larger line is + * emphasis, not a heading (see {@link #isBoldLabel}); promoting it to {@code #}/{@code ##} is + * the main source of false-positive headings. + * + *

    + *
  • size > baseline * 1.4 → {@code "# "} + *
  • size > baseline * 1.2 → {@code "## "} + *
  • otherwise → {@code ""} + *
+ */ + static String headingPrefix(TextLine line, float medianBodySize, float medianBodyHeight) { + String text = line.text().strip(); + if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) { + return ""; + } + + float dominant = dominantFontSize(line); + float value; + float baseline; + if (dominant > 2f && medianBodySize > 2f) { + value = dominant; + baseline = medianBodySize; + } else { + value = line.height(); + baseline = medianBodyHeight; + } + if (baseline <= 0f) { + return ""; + } + + float ratio = value / baseline; + if (ratio > 1.4f) { + return "# "; + } + if (ratio > 1.2f) { + return "## "; + } + return ""; + } + + /** + * True when a line should be emphasised as bold (rendered {@code **like this**}) rather than + * promoted to a heading: it is bold, short, and not a full sentence. Used for bold labels that + * are not large enough to be headings. + */ + static boolean isBoldLabel(TextLine line) { + String text = line.text().strip(); + if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) { + return false; + } + return isBold(line); + } + + private static int wordCount(String text) { + return text.split("\\s+").length; + } + + private static boolean endsLikeSentence(String text) { + char last = text.charAt(text.length() - 1); + return last == '.' || last == '!' || last == '?'; + } + + /** True when the line's dominant font is bold, inferred from PostScript font names. */ + private static boolean isBold(TextLine line) { + Map counts = new HashMap<>(); + for (TextWord word : line.words()) { + for (TextChar ch : word.chars()) { + if (ch.isWhitespace() || ch.isNewline()) { + continue; + } + String name = ch.fontName(); + if (name != null && !name.isBlank()) { + counts.merge(name, 1, Integer::sum); + } + } + } + String dominantFont = ""; + int max = -1; + for (Map.Entry e : counts.entrySet()) { + if (e.getValue() > max) { + max = e.getValue(); + dominantFont = e.getKey(); + } + } + String lower = dominantFont.toLowerCase(java.util.Locale.ROOT); + return lower.contains("bold") + || lower.contains("black") + || lower.contains("heavy") + || lower.contains("semibold"); + } + + /** Computes the median glyph font size across all pages. */ + static float medianFontSize(List allPages) { + List sizes = new ArrayList<>(); + for (PageText page : allPages) { + for (TextChar ch : page.chars()) { + if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) { + sizes.add(ch.fontSize()); + } + } + } + return median(sizes, 12f); + } + + /** Computes the median TextLine height across all pages. Used when font size is degenerate. */ + static float medianLineHeight(List allPages) { + List heights = new ArrayList<>(); + for (PageText page : allPages) { + for (TextLine line : page.lines()) { + if (line.height() > 0f && !line.text().isBlank()) { + heights.add(line.height()); + } + } + } + return median(heights, 12f); + } + + private static float median(List values, float fallback) { + if (values.isEmpty()) { + return fallback; + } + Collections.sort(values); + int mid = values.size() / 2; + if (values.size() % 2 == 0) { + return (values.get(mid - 1) + values.get(mid)) / 2f; + } + return values.get(mid); + } + + /** + * Returns the font size that appears most often (by character count) in the given line. Ties + * are broken in favour of the larger size. + */ + private static float dominantFontSize(TextLine line) { + Map counts = new HashMap<>(); + for (TextWord word : line.words()) { + for (TextChar ch : word.chars()) { + if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) { + counts.merge(ch.fontSize(), 1, Integer::sum); + } + } + } + if (counts.isEmpty()) { + return 0f; + } + float dominant = 0f; + int maxCount = -1; + for (Map.Entry entry : counts.entrySet()) { + int count = entry.getValue(); + float size = entry.getKey(); + if (count > maxCount || (count == maxCount && size > dominant)) { + maxCount = count; + dominant = size; + } + } + return dominant; + } +} diff --git a/app/common/src/main/java/stirling/software/common/pdf/PdfMarkdownConverter.java b/app/common/src/main/java/stirling/software/common/pdf/PdfMarkdownConverter.java new file mode 100644 index 000000000..c19468b5e --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/pdf/PdfMarkdownConverter.java @@ -0,0 +1,1043 @@ +package stirling.software.common.pdf; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import stirling.software.jpdfium.PdfDocument; +import stirling.software.jpdfium.PdfPage; +import stirling.software.jpdfium.doc.ExtractedImage; +import stirling.software.jpdfium.doc.PdfImageExtractor; +import stirling.software.jpdfium.model.Rect; +import stirling.software.jpdfium.text.PageText; +import stirling.software.jpdfium.text.PdfTableExtractor; +import stirling.software.jpdfium.text.PdfTextExtractor; +import stirling.software.jpdfium.text.Table; +import stirling.software.jpdfium.text.TextLine; +import stirling.software.jpdfium.text.TextWord; + +/** + * Converts a PDF to Markdown using a TextLine-driven body pipeline. + * + *

Body text is rebuilt from {@link PdfTextExtractor} {@link TextLine}s. TextLines group words + * faithfully and keep paragraph order, so the only pre-processing needed is stitching narrow + * standalone glyph fragments (apostrophes, quotes, asterisks, superscript footnote markers, + * bullets) back into the line they belong to. Column layout and tables are derived from line/word + * geometry directly. + */ +public class PdfMarkdownConverter { + + private static final Pattern SOFT_HYPHEN = Pattern.compile("(\\w+)-\\n([a-z])"); + + /** Width below which a TextLine is treated as a stray glyph fragment to be stitched. */ + private static final float GLYPH_WIDTH = 7.5f; + + public String convert(PdfDocument doc) throws IOException { + List allPageText = PdfTextExtractor.extractAll(doc); + float medianSize = HeadingDetector.medianFontSize(allPageText); + float medianHeight = HeadingDetector.medianLineHeight(allPageText); + + int pageCount = doc.pageCount(); + // Elements are either rendered text (String) or a structured TableBlock. Tables stay + // structured until after the page loop so a table split across a page break can be stitched + // back together before rendering. + List output = new ArrayList<>(); + // Header text of a table that ended the previous page, used to spot a continuation whose + // header repeats at the top of the current page. Null when the previous page did not end in + // a table. + String prevPageTrailingTableHeader = null; + + for (int pageIndex = 0; pageIndex < pageCount; pageIndex++) { + List rawLines = + pageIndex < allPageText.size() ? allPageText.get(pageIndex).lines() : List.of(); + + // Stitch stray glyph fragments (apostrophes, asterisks, superscripts, bullets) into + // their host lines so paragraph assembly sees faithful, complete lines. + List lines = stitchGlyphs(rawLines); + if (lines.isEmpty()) { + emitImages(doc, pageIndex, output); + prevPageTrailingTableHeader = null; + continue; + } + + // Sort top-to-bottom (PDF y=0 is the bottom of the page). + lines.sort(Comparator.comparingDouble((Line l) -> l.y).reversed()); + + // Multi-column guard: only genuine two-column prose should be split. A table's column + // gutters must NOT be mistaken for a page-layout gutter, so this looks at whether row + // lines span the gutter (table) or stay within one side (two-column prose). + // A table that ran to the bottom of the previous page and repeats its header at the top + // of this page is a continuation, not a new two-column layout. Detecting the repeated + // header keeps this page out of the two-column path so the continuation is rebuilt as a + // table and stitched back onto the previous block. + final String continuationHeader = prevPageTrailingTableHeader; + boolean tableContinuation = + continuationHeader != null + && lines.stream() + .anyMatch( + l -> normaliseSpace(l.text).equals(continuationHeader)); + + boolean twoColumn = !tableContinuation && detectsTwoColumns(lines); + + // Tables are detected from text/word geometry (the word-grid detector), which handles + // both ruled and borderless tables and places cells by column alignment. The native + // ruled-line extractor is not used: it both mis-renders cells and double-emits rows. + Set tableRowTexts = new HashSet<>(); + List blocks = twoColumn ? List.of() : findTableBlocks(lines); + Set tableLines = new HashSet<>(); + for (TableBlock b : blocks) { + for (List row : b.rows()) { + for (Line l : row) { + tableLines.add(l); + tableRowTexts.add(repairHyphens(l.text).strip()); + } + } + } + + List pageItems = new ArrayList<>(); + if (twoColumn) { + for (List col : splitIntoColumns(lines)) { + List paras = new ArrayList<>(); + assembleParagraphs(col, medianSize, medianHeight, paras, tableRowTexts); + pageItems.addAll(paras); + } + } else { + // Interleave tables with surrounding text by vertical position. Each block sits in + // its own slot; non-table lines fall into the slot for their y (text above a block, + // between blocks, or below the last). This keeps multiple tables on one page + // separate and in reading order. + List> segments = new ArrayList<>(); + for (int s = 0; s <= blocks.size(); s++) { + segments.add(new ArrayList<>()); + } + for (Line l : lines) { + if (tableLines.contains(l)) { + continue; + } + int slot = 0; + for (TableBlock b : blocks) { + if (b.bottom() > l.y) { + slot++; + } + } + segments.get(slot).add(l); + } + for (int s = 0; s <= blocks.size(); s++) { + List paras = new ArrayList<>(); + assembleParagraphs( + segments.get(s), medianSize, medianHeight, paras, tableRowTexts); + pageItems.addAll(paras); + if (s < blocks.size()) { + pageItems.add(blocks.get(s)); + } + } + } + + emitImages(doc, pageIndex, pageItems); + + if (pageItems.isEmpty()) { + continue; + } + + mergeAcrossPageBoundary(output, pageItems); + output.addAll(pageItems); + prevPageTrailingTableHeader = trailingTableHeader(pageItems); + } + + // Stitch tables split across page breaks, then render every element to Markdown. + List stitched = stitchTables(output); + List rendered = new ArrayList<>(); + for (Object e : stitched) { + rendered.add(e instanceof TableBlock tb ? tb.render() : (String) e); + } + return String.join("\n\n", rendered); + } + + // --- Glyph stitching --------------------------------------------------- + + /** A mutable assembled line: text plus geometry used for ordering and heading detection. */ + private static final class Line { + String text; + float x; + float y; + float width; + float height; + final TextLine source; + + Line(TextLine src) { + this.source = src; + this.text = src.text(); + this.x = src.x(); + this.y = src.y(); + this.width = src.width(); + this.height = src.height(); + } + } + + /** + * Merges narrow glyph fragments (width < {@link #GLYPH_WIDTH}) into the line they belong to. + * + *
    + *
  • A glyph between a left fragment that ends near it and a right fragment that starts near + * it (both on the same baseline) is inserted inline: {@code aren} + {@code '} + {@code t} + * → {@code aren't}. + *
  • A glyph immediately right of a line's end is appended (e.g. superscript footnote marker + * after a number). + *
  • A glyph immediately left of a line's start is prepended (e.g. footnote marker before + * its text). + *
+ */ + private static List stitchGlyphs(List raw) { + List hosts = new ArrayList<>(); + List glyphs = new ArrayList<>(); + for (TextLine l : raw) { + String t = l.text().strip(); + if (t.isEmpty()) { + continue; + } + if (l.width() < GLYPH_WIDTH && t.length() <= 2) { + glyphs.add(l); + } else { + hosts.add(l); + } + } + + List lines = hosts.stream().map(Line::new).collect(Collectors.toList()); + + for (TextLine g : glyphs) { + String gt = g.text().strip(); + if (isBulletGlyph(gt)) { + attachBullet(g, gt, lines); + } else { + attachInlineGlyph(g, gt, lines); + } + } + return lines; + } + + private static boolean isBulletGlyph(String gt) { + return "•".equals(gt) || "▪".equals(gt) || "◦".equals(gt); + } + + /** + * Attaches a bullet glyph to the body line it introduces: the closest line that begins to the + * right of the bullet at roughly the same height or just below it. + */ + private static void attachBullet(TextLine g, String gt, List lines) { + Line best = null; + float bestScore = Float.MAX_VALUE; + for (Line h : lines) { + if (h.x < g.x() - 2f) { + continue; + } + float dy = g.y() - h.y; + if (dy < -4f || dy > 28f) { + continue; + } + float score = Math.abs(dy) + (h.x - g.x()) * 0.2f; + if (score < bestScore) { + bestScore = score; + best = h; + } + } + if (best != null && !best.text.startsWith("•")) { + best.text = "• " + best.text; + best.x = g.x(); + } else { + lines.add(new Line(g)); + } + } + + /** + * Stitches a narrow inline glyph (apostrophe, quote, asterisk, superscript marker) into the + * line it belongs to: inline between two same-baseline fragments, appended to the line that + * ends at it, or prepended to the line that starts at it. + */ + private static void attachInlineGlyph(TextLine g, String gt, List lines) { + Line left = null; + Line right = null; + float lb = 7f; + float rb = 7f; + for (Line h : lines) { + boolean sameBaseline = g.y() >= h.y - 4f && g.y() <= h.y + h.height + 5f; + if (!sameBaseline) { + continue; + } + float rightEdge = h.x + h.width; + float dxLeft = Math.abs(rightEdge - g.x()); + if (dxLeft < lb) { + lb = dxLeft; + left = h; + } + float dxRight = Math.abs(h.x - g.x()); + if (dxRight < rb) { + rb = dxRight; + right = h; + } + } + + if (left != null && right != null && left != right && Math.abs(left.y - right.y) < 6f) { + left.text = left.text + gt + right.text; + left.width = (right.x + right.width) - left.x; + lines.remove(right); + } else if (left != null) { + left.text = left.text + gt; + left.width = Math.max(left.width, g.x() + g.width() - left.x); + } else if (right != null) { + right.text = gt + right.text; + right.x = g.x(); + } else { + lines.add(new Line(g)); + } + } + + // --- Column detection (guard only) ------------------------------------- + + /** + * Returns true when the page is a genuine two-column layout. Uses line/word geometry: body + * blocks (ignoring narrow glyph blocks) and requires a wide horizontal gutter populated on both + * sides, so single apostrophe glyphs cannot create a false second column. + */ + private static boolean detectsTwoColumns(List lines) { + if (lines.size() < 8) { + return false; + } + float minX = Float.MAX_VALUE; + float maxX = -Float.MAX_VALUE; + for (Line l : lines) { + minX = Math.min(minX, l.x); + maxX = Math.max(maxX, l.x + l.width); + } + if (maxX - minX < 200f) { + return false; + } + + // Scan candidate gutter positions across the central band (35%-65% of width) and pick the + // one crossed by the fewest lines. Two-column prose has a gutter that only a handful of + // full-width lines (title, section headings) cross; a table's rows all span the full width, + // so every candidate gutter is crossed by most lines. + float centreLo = minX + (maxX - minX) * 0.35f; + float centreHi = minX + (maxX - minX) * 0.65f; + int bestCrossing = Integer.MAX_VALUE; + int bestLeft = 0; + int bestRight = 0; + for (float gutter = centreLo; gutter <= centreHi; gutter += 2f) { + int crossing = 0; + int leftOnly = 0; + int rightOnly = 0; + for (Line l : lines) { + float lx = l.x; + float rx = l.x + l.width; + if (lx < gutter - 5f && rx > gutter + 5f) { + crossing++; + } else if (rx <= gutter) { + leftOnly++; + } else { + rightOnly++; + } + } + if (crossing < bestCrossing) { + bestCrossing = crossing; + bestLeft = leftOnly; + bestRight = rightOnly; + } + } + + return bestLeft >= 4 && bestRight >= 4 && bestCrossing <= (int) (lines.size() * 0.25f); + } + + private static List> splitIntoColumns(List lines) { + List xs = + lines.stream() + .filter(l -> l.width >= 40f) + .map(l -> l.x) + .sorted() + .collect(Collectors.toList()); + if (xs.isEmpty()) { + return List.of(lines); + } + float minX = xs.get(0); + float maxX = xs.get(xs.size() - 1); + float splitAt = (minX + maxX) / 2f; + float biggestGap = 0; + for (int i = 1; i < xs.size(); i++) { + float gap = xs.get(i) - xs.get(i - 1); + if (gap > biggestGap) { + biggestGap = gap; + splitAt = (xs.get(i - 1) + xs.get(i)) / 2f; + } + } + List left = new ArrayList<>(); + List right = new ArrayList<>(); + for (Line l : lines) { + if (l.x < splitAt) { + left.add(l); + } else { + right.add(l); + } + } + if (left.isEmpty()) { + return List.of(right); + } + if (right.isEmpty()) { + return List.of(left); + } + return List.of(left, right); + } + + // --- Paragraph assembly ------------------------------------------------ + + private static void assembleParagraphs( + List lines, + float medianSize, + float medianHeight, + List out, + Set tableRowTexts) { + StringBuilder para = new StringBuilder(); + float prevBottomY = Float.MAX_VALUE; + float prevHeight = 0f; + + for (Line line : lines) { + String text = repairHyphens(line.text).strip(); + if (text.isEmpty()) { + continue; + } + if (tableRowTexts.contains(text)) { + continue; + } + + float blockTop = line.y + line.height; + float gap = prevBottomY - blockTop; + boolean paragraphBreak = prevHeight > 0f && gap > prevHeight * 0.8f; + + String prefix = HeadingDetector.headingPrefix(line.source, medianSize, medianHeight); + boolean isHeading = !prefix.isEmpty(); + boolean isBullet = startsWithBullet(text); + + if (isHeading) { + flushParagraph(para, out); + out.add(prefix + escapeMarkdown(text)); + } else if (isBullet) { + flushParagraph(para, out); + out.add(escapeMarkdown(text)); + } else if (HeadingDetector.isBoldLabel(line.source)) { + // Bold but not large enough to be a heading → emphasise as bold, don't promote. + flushParagraph(para, out); + out.add("**" + escapeMarkdown(text) + "**"); + } else if (paragraphBreak) { + flushParagraph(para, out); + para.append(text); + } else { + if (!para.isEmpty()) { + char fc = text.charAt(0); + boolean noSpace = fc == '\'' || fc == '’' || fc == '‘' || fc == '"'; + if (!noSpace) { + para.append(' '); + } + } + para.append(text); + } + + prevBottomY = line.y; + prevHeight = line.height; + } + flushParagraph(para, out); + } + + private static boolean startsWithBullet(String text) { + return text.startsWith("•") || text.startsWith("▪") || text.startsWith("◦"); + } + + // --- Word-grid table detection ----------------------------------------- + + /** + * A detected table. Each row is a list of source lines: usually one, but more when a cell wraps + * onto extra lines (those continuation lines are absorbed into the row they belong to). + */ + private record TableBlock(List> rows, float top, float bottom) { + String render() { + return buildTableFromRows(rows); + } + } + + /** + * Detects table blocks on a page. Anchor rows (lines with table-like column gaps) are grouped + * into vertically-contiguous runs separated by large vertical gaps, so multiple separate tables + * on one page stay separate. Non-anchor lines that fall within a run's vertical span are + * treated as wrapped-cell continuations and absorbed into the nearest anchor row above them. + */ + private static List findTableBlocks(List lines) { + List cands = + lines.stream() + .filter(l -> isTableCandidate(l.source)) + .sorted(Comparator.comparingDouble((Line l) -> l.y).reversed()) + .collect(Collectors.toList()); + if (cands.size() < 2) { + return List.of(); + } + + List gaps = new ArrayList<>(); + for (int i = 1; i < cands.size(); i++) { + gaps.add(cands.get(i - 1).y - cands.get(i).y); + } + List sorted = new ArrayList<>(gaps); + sorted.sort(Comparator.naturalOrder()); + float medianGap = sorted.get(sorted.size() / 2); + float splitThreshold = Math.max(medianGap * 2.5f, medianGap + 6f); + + List> anchorGroups = new ArrayList<>(); + List current = new ArrayList<>(); + current.add(cands.get(0)); + for (int i = 1; i < cands.size(); i++) { + float gap = cands.get(i - 1).y - cands.get(i).y; + if (gap > splitThreshold) { + anchorGroups.add(current); + current = new ArrayList<>(); + } + current.add(cands.get(i)); + } + anchorGroups.add(current); + + List nonCandidates = + lines.stream() + .filter(l -> !isTableCandidate(l.source)) + .collect(Collectors.toList()); + + List blocks = new ArrayList<>(); + for (List anchors : anchorGroups) { + if (anchors.size() < 2) { + continue; + } + float top = anchors.get(0).y; + float bottom = anchors.get(anchors.size() - 1).y; + + // Each anchor seeds a row; absorb wrapped continuation lines (non-anchors within the + // run's vertical span, with a little slack below the last row) into the anchor above. + List> rows = new ArrayList<>(); + for (Line a : anchors) { + List row = new ArrayList<>(); + row.add(a); + rows.add(row); + } + for (Line nc : nonCandidates) { + if (nc.y > top || nc.y < bottom - medianGap) { + continue; + } + int owner = 0; + float bestDelta = Float.MAX_VALUE; + for (int i = 0; i < anchors.size(); i++) { + float delta = anchors.get(i).y - nc.y; // positive when anchor is above nc + if (delta >= -1f && delta < bestDelta) { + bestDelta = delta; + owner = i; + } + } + rows.get(owner).add(nc); + } + + if (buildTableFromRows(rows).isBlank()) { + continue; + } + blocks.add(new TableBlock(rows, top, bottom)); + } + return blocks; + } + + private static String buildTableFromRows(List> rowGroups) { + // Detect columns by vertical-whitespace projection across all lines, rather than a 1-D gap + // threshold on pooled word x's. Pooled-gap detection is fragile when numbers are + // right-aligned (a 10-digit value starts well left of a 7-digit one) or when sparse cells + // sit in their own x-band. Projection asks "which x-bands are occupied across many rows", + // which is stable under those conditions. + List flat = rowGroups.stream().flatMap(List::stream).collect(Collectors.toList()); + List columns = findColumnRanges(flat); + if (columns.size() < 2 || columns.size() > 15) { + return ""; + } + + float[] centers = new float[columns.size()]; + for (int i = 0; i < columns.size(); i++) { + centers[i] = (columns.get(i)[0] + columns.get(i)[1]) / 2f; + } + + int cols = centers.length; + List rows = new ArrayList<>(); + for (List rowLines : rowGroups) { + String[] row = new String[cols]; + for (int i = 0; i < cols; i++) { + row[i] = ""; + } + // Top line first so a wrapped cell's words stay in reading order within the cell. + rowLines.sort(Comparator.comparingDouble((Line l) -> l.y).reversed()); + for (Line line : rowLines) { + for (TextWord word : line.source.words()) { + String wt = word.text().strip(); + if (wt.isEmpty()) { + continue; + } + int col = nearestColumn(word.x() + word.width() / 2f, centers); + row[col] = row[col].isEmpty() ? wt : row[col] + " " + wt; + } + } + rows.add(row); + } + + // Guard against false positives while tolerating uneven rows (sparse cells, merged/spanning + // headers). The columns already come from cross-row whitespace alignment, so a stable grid + // exists. Additionally require: at least one "anchor" row that nearly fills the grid (so + // the + // column count is real, not an artefact), and that most rows are genuinely multi-column. + int anchorWidth = Math.max(2, Math.round(cols * 0.6f)); + long anchorRows = rows.stream().filter(r -> filledCells(r) >= anchorWidth).count(); + long multiColumnRows = rows.stream().filter(r -> filledCells(r) >= 2).count(); + if (anchorRows < 1 || multiColumnRows < 2 || multiColumnRows < rows.size() * 0.5) { + return ""; + } + return renderGfm(rows, cols); + } + + /** + * Visible for testing: column detection depends only on word geometry, so tests can drive it + * from synthetic {@link TextLine}s to exercise degenerate-coordinate handling (the crash path + * an extreme text matrix can produce) without needing a binary PDF fixture. + */ + static List findColumnRangesFromLines(List rows) { + return findColumnRanges(rows.stream().map(Line::new).collect(Collectors.toList())); + } + + /** + * Finds column x-ranges by vertical-whitespace projection. Each row contributes coverage for + * the x-bands its words occupy; a column is a contiguous band covered by a sufficient fraction + * of rows, and the gaps between such bands are the gutters. + */ + private static List findColumnRanges(List rows) { + float minX = Float.MAX_VALUE; + float maxX = -Float.MAX_VALUE; + for (Line l : rows) { + for (TextWord w : l.source.words()) { + minX = Math.min(minX, w.x()); + maxX = Math.max(maxX, w.x() + w.width()); + } + } + // Real pages are under ~2000pt wide; anything larger is a malformed/crafted coordinate + // that would allocate a multi-GB array or produce a negative span on overflow. + if (maxX <= minX || (maxX - minX) > 2000f) { + return List.of(); + } + + int lo = (int) Math.floor(minX); + int span = Math.min((int) Math.ceil(maxX) - lo + 1, 2001); + int[] coverage = new int[span]; + for (Line l : rows) { + boolean[] covered = new boolean[span]; + for (TextWord w : l.source.words()) { + int a = Math.max(0, (int) Math.floor(w.x()) - lo); + int b = Math.min(span, (int) Math.ceil(w.x() + w.width()) - lo); + for (int x = a; x < b; x++) { + covered[x] = true; + } + } + for (int x = 0; x < span; x++) { + if (covered[x]) { + coverage[x]++; + } + } + } + + // A column band must be occupied by at least this many rows; below it is gutter. + int support = Math.max(2, Math.round(rows.size() * 0.35f)); + List columns = new ArrayList<>(); + int start = -1; + for (int x = 0; x < span; x++) { + boolean isColumn = coverage[x] >= support; + if (isColumn && start < 0) { + start = x; + } else if (!isColumn && start >= 0) { + columns.add(new float[] {lo + start, lo + x}); + start = -1; + } + } + if (start >= 0) { + columns.add(new float[] {(float) (lo + start), (float) (lo + span)}); + } + + // Merge bands separated by only a narrow gutter. A real column separator is several + // characters wide; the gaps *inside* a multi-word cell (ordinary word spacing) are about + // one character. Without this, a cell like "January 20th, 2026" — whose words align + // vertically across every row — would be split into three spurious columns. + float charWidth = averageCharWidth(rows); + float minGutter = Math.max(10f, charWidth * 2.5f); + List merged = new ArrayList<>(); + for (float[] band : columns) { + if (!merged.isEmpty() && band[0] - merged.get(merged.size() - 1)[1] < minGutter) { + merged.get(merged.size() - 1)[1] = band[1]; + } else { + merged.add(new float[] {band[0], band[1]}); + } + } + return merged; + } + + private static float averageCharWidth(List rows) { + double totalWidth = 0; + int totalChars = 0; + for (Line l : rows) { + for (TextWord w : l.source.words()) { + totalWidth += w.width(); + totalChars += Math.max(1, w.text().strip().length()); + } + } + return totalChars == 0 ? 6f : (float) (totalWidth / totalChars); + } + + private static int nearestColumn(float x, float[] centers) { + int best = 0; + float bestDist = Float.MAX_VALUE; + for (int i = 0; i < centers.length; i++) { + float d = Math.abs(x - centers[i]); + if (d < bestDist) { + bestDist = d; + best = i; + } + } + return best; + } + + private static int filledCells(String[] row) { + int count = 0; + for (String cell : row) { + if (!cell.isEmpty()) { + count++; + } + } + return count; + } + + private static String renderGfm(List rows, int cols) { + if (rows.isEmpty()) { + return ""; + } + int[] widths = new int[cols]; + for (int c = 0; c < cols; c++) { + widths[c] = 3; + } + for (String[] row : rows) { + for (int c = 0; c < cols; c++) { + if (c < row.length) { + widths[c] = Math.max(widths[c], escapeCell(row[c]).length()); + } + } + } + StringBuilder sb = new StringBuilder(); + sb.append(buildGfmRow(rows.get(0), widths, cols)).append('\n'); + sb.append('|'); + for (int c = 0; c < cols; c++) { + sb.append('-').append("-".repeat(widths[c])).append('-').append('|'); + } + for (int r = 1; r < rows.size(); r++) { + sb.append('\n').append(buildGfmRow(rows.get(r), widths, cols)); + } + return sb.toString(); + } + + /** + * A line looks like a table row if it has at least two words separated by a gap far wider than + * normal inter-word spacing. The threshold is derived from the line's own character width + * rather than a document font size, because some PDFs report a unit (matrix-scaled) font size + * that makes absolute thresholds meaningless. (Two-word rows are allowed so two-column tables + * are detected; spurious matches are filtered later by block contiguity and column + * consistency.) + */ + private static boolean isTableCandidate(TextLine line) { + List words = line.words(); + if (words.size() < 2) { + return false; + } + double totalWidth = 0; + int totalChars = 0; + for (TextWord w : words) { + totalWidth += w.width(); + totalChars += Math.max(1, w.text().strip().length()); + } + float charWidth = (float) (totalWidth / Math.max(1, totalChars)); + // A deliberate cell gap is several blank characters wide; ordinary word spaces are ~a third + // of a character. Floor at 8pt so tiny fonts still need a real gap. + float cellGap = Math.max(8f, charWidth * 3f); + for (int i = 1; i < words.size(); i++) { + TextWord prev = words.get(i - 1); + float gap = words.get(i).x() - (prev.x() + prev.width()); + if (gap >= cellGap) { + return true; + } + } + return false; + } + + private static String buildGfmRow(String[] row, int[] widths, int cols) { + StringBuilder sb = new StringBuilder().append('|'); + for (int c = 0; c < cols; c++) { + String cell = c < row.length ? escapeCell(row[c]) : ""; + sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|'); + } + return sb.toString(); + } + + private static String escapeCell(String cell) { + // Cell content is inline context: escape inline markdown (including the column delimiter) + // but not leading block markers, which have no meaning inside a table cell. + return escapeMarkdownInline(cell); + } + + /** + * Escapes Markdown control characters in body text extracted from the PDF so that literal + * characters (e.g. a line that reads {@code # Heading} or {@code [label](url)}, or an embedded + * {@code }) are emitted as text rather than being reinterpreted as structure or raw HTML. + * Applied to all body text — headings, paragraphs, bold labels, bullets — before emission. + * + *

The generated Markdown should still be treated as untrusted content by any downstream + * renderer: this hardens fidelity and is defence-in-depth, not a substitute for safe rendering. + */ + private static String escapeMarkdown(String text) { + if (text.isEmpty()) { + return text; + } + String inline = escapeMarkdownInline(text); + return escapeLeadingBlockMarker(inline, text); + } + + /** Escapes inline-significant Markdown characters anywhere in the string. */ + private static String escapeMarkdownInline(String text) { + StringBuilder sb = new StringBuilder(text.length() + 8); + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + switch (c) { + case '\\', '`', '*', '_', '[', ']', '<', '>', '|', '~' -> sb.append('\\').append(c); + default -> sb.append(c); + } + } + return sb.toString(); + } + + /** + * Escapes block-level markers that are only significant at the start of a line: ATX headings + * ({@code #}), unordered list / thematic break markers ({@code -}, {@code +}), and ordered list + * markers ({@code 1.} / {@code 1)}). {@code original} carries the unescaped leading characters, + * none of which are altered by inline escaping, so positions line up with {@code escaped}. + */ + private static String escapeLeadingBlockMarker(String escaped, String original) { + char c0 = original.charAt(0); + if (c0 == '#' || c0 == '-' || c0 == '+') { + return "\\" + escaped; + } + int i = 0; + while (i < original.length() && Character.isDigit(original.charAt(i))) { + i++; + } + if (i > 0 && i < original.length()) { + char delim = original.charAt(i); + if (delim == '.' || delim == ')') { + return escaped.substring(0, i) + "\\" + escaped.substring(i); + } + } + return escaped; + } + + private static String padRight(String s, int width) { + return s.length() >= width ? s : s + " ".repeat(width - s.length()); + } + + // --- Page-level emission helpers --------------------------------------- + + private static void emitImages(PdfDocument doc, int pageIndex, List pageItems) + throws IOException { + try (PdfPage page = doc.page(pageIndex)) { + List images = + PdfImageExtractor.extract(page.rawDocHandle(), page.rawHandle(), pageIndex); + for (ExtractedImage img : images) { + pageItems.add(describeImage(img)); + } + } + } + + /** + * Builds an image placeholder annotated with whatever metadata JPDFium exposes: pixel + * dimensions, on-page placement (points), effective DPI, encoded format, colour space and bit + * depth. Missing fields are simply omitted so the line stays valid for any image. + */ + private static String describeImage(ExtractedImage img) { + List parts = new ArrayList<>(); + if (img.width() > 0 && img.height() > 0) { + parts.add(img.width() + "x" + img.height() + "px"); + } + Rect b = img.bounds(); + if (b != null && b.width() > 0 && b.height() > 0) { + parts.add(String.format("%.0fx%.0fpt", b.width(), b.height())); + if (img.width() > 0) { + float dpiX = img.width() / (b.width() / 72f); + float dpiY = img.height() / (b.height() / 72f); + if (Float.isFinite(dpiX) && dpiX > 0) { + parts.add(String.format("~%.0fdpi", (dpiX + dpiY) / 2f)); + } + } + } + String ext = img.suggestedExtension(); + if (ext != null && !ext.isBlank()) { + parts.add(ext.replaceFirst("^\\.", "").toUpperCase(java.util.Locale.ROOT)); + } + if (img.colorSpace() != null) { + parts.add(img.colorSpace().toString()); + } + if (img.bitsPerPixel() > 0) { + parts.add(img.bitsPerPixel() + "bpp"); + } + + StringBuilder sb = new StringBuilder("'); + return sb.toString(); + } + + private static void mergeAcrossPageBoundary(List output, List pageItems) { + if (output.isEmpty() || pageItems.isEmpty()) { + return; + } + // Only merge a sentence continuation between two text paragraphs, never into/out of a + // table. + if (!(output.get(output.size() - 1) instanceof String last) + || !(pageItems.get(0) instanceof String first)) { + return; + } + if (!first.isEmpty() + && Character.isLowerCase(first.charAt(0)) + && !endsWithSentencePunctuation(last)) { + output.set(output.size() - 1, last + " " + first); + pageItems.remove(0); + } + } + + /** + * Joins tables split across a page break. Two consecutive {@link TableBlock}s (no text between + * them — i.e. one ended a page and the next began the following page) are merged when their + * column layouts match; a repeated header row on the continuation is dropped. + */ + private static List stitchTables(List elements) { + List out = new ArrayList<>(); + for (Object e : elements) { + if (e instanceof TableBlock tb + && !out.isEmpty() + && out.get(out.size() - 1) instanceof TableBlock prev + && columnsMatch(flatten(prev.rows()), flatten(tb.rows()))) { + List> merged = new ArrayList<>(prev.rows()); + List> tail = tb.rows(); + if (!tail.isEmpty() + && !prev.rows().isEmpty() + && rowText(tail.get(0)).equals(rowText(prev.rows().get(0)))) { + tail = tail.subList(1, tail.size()); + } + merged.addAll(tail); + out.set(out.size() - 1, new TableBlock(merged, prev.top(), tb.bottom())); + } else { + out.add(e); + } + } + return out; + } + + private static String normaliseSpace(String s) { + return s.strip().replaceAll("\\s+", " "); + } + + private static List flatten(List> rows) { + return rows.stream().flatMap(List::stream).collect(Collectors.toList()); + } + + /** Whitespace-normalised text of a row's lines (top to bottom), for header de-duplication. */ + /** + * Header text of a table at the very bottom of a page, or null if the page does not end in one. + * Trailing image placeholders are skipped; any other text after a table means it did not run to + * the page bottom and so is not a continuation candidate. + */ + private static String trailingTableHeader(List pageItems) { + for (int i = pageItems.size() - 1; i >= 0; i--) { + Object e = pageItems.get(i); + if (e instanceof String s && s.strip().startsWith(" row) { + List ordered = new ArrayList<>(row); + ordered.sort(Comparator.comparingDouble((Line l) -> l.y).reversed()); + StringBuilder sb = new StringBuilder(); + for (Line l : ordered) { + if (sb.length() > 0) { + sb.append(' '); + } + sb.append(l.text); + } + return normaliseSpace(sb.toString()); + } + + /** True when two table blocks have the same number of columns at near-identical x-centres. */ + private static boolean columnsMatch(List a, List b) { + List ca = findColumnRanges(a); + List cb = findColumnRanges(b); + if (ca.size() < 2 || ca.size() != cb.size()) { + return false; + } + for (int i = 0; i < ca.size(); i++) { + float centreA = (ca.get(i)[0] + ca.get(i)[1]) / 2f; + float centreB = (cb.get(i)[0] + cb.get(i)[1]) / 2f; + if (Math.abs(centreA - centreB) > 15f) { + return false; + } + } + return true; + } + + private static void flushParagraph(StringBuilder para, List out) { + if (!para.isEmpty()) { + out.add(escapeMarkdown(para.toString())); + para.setLength(0); + } + } + + private static String repairHyphens(String text) { + return SOFT_HYPHEN.matcher(text).replaceAll("$1$2"); + } + + private static boolean endsWithSentencePunctuation(String s) { + if (s.isEmpty()) { + return false; + } + char last = s.charAt(s.length() - 1); + return last == '.' || last == '?' || last == '!' || last == ':'; + } + + // --- Methods used by other components / tests -------------------------- + + List extractAllPageText(PdfDocument doc) throws IOException { + return PdfTextExtractor.extractAll(doc); + } + + List extractTables(PdfDocument doc, int pageIndex) throws IOException { + return PdfTableExtractor.extract(doc, pageIndex); + } + + List renderTables(List
tables) { + return tables.stream().map(TableRenderer::render).toList(); + } +} diff --git a/app/common/src/main/java/stirling/software/common/pdf/TableRenderer.java b/app/common/src/main/java/stirling/software/common/pdf/TableRenderer.java new file mode 100644 index 000000000..3f468699f --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/pdf/TableRenderer.java @@ -0,0 +1,82 @@ +package stirling.software.common.pdf; + +import stirling.software.jpdfium.text.Table; + +final class TableRenderer { + private TableRenderer() {} + + /** Renders a Table as a GitHub-Flavoured Markdown table string. */ + static String render(Table table) { + if (table.rowCount() == 0) { + return ""; + } + + String[][] grid = table.asGrid(); + + if (table.rowCount() < 2) { + // No separator row possible — return plain lines + StringBuilder sb = new StringBuilder(); + for (int c = 0; c < grid[0].length; c++) { + if (c > 0) sb.append('\n'); + sb.append(escape(grid[0][c].trim())); + } + return sb.toString(); + } + + int cols = grid[0].length; + + // Compute column widths: max(3, max content length across all rows) + int[] widths = new int[cols]; + for (int c = 0; c < cols; c++) { + widths[c] = 3; + } + for (String[] row : grid) { + for (int c = 0; c < cols; c++) { + String cell = c < row.length ? row[c].trim() : ""; + widths[c] = Math.max(widths[c], escape(cell).length()); + } + } + + StringBuilder sb = new StringBuilder(); + + // Header row + sb.append(buildRow(grid[0], widths, cols)); + sb.append('\n'); + + // Separator row + sb.append('|'); + for (int c = 0; c < cols; c++) { + sb.append('-').append("-".repeat(widths[c])).append('-').append('|'); + } + sb.append('\n'); + + // Data rows + for (int r = 1; r < grid.length; r++) { + sb.append(buildRow(grid[r], widths, cols)); + if (r < grid.length - 1) { + sb.append('\n'); + } + } + + return sb.toString(); + } + + private static String buildRow(String[] row, int[] widths, int cols) { + StringBuilder sb = new StringBuilder(); + sb.append('|'); + for (int c = 0; c < cols; c++) { + String cell = c < row.length ? escape(row[c].trim()) : ""; + sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|'); + } + return sb.toString(); + } + + private static String escape(String cell) { + return cell.replace("|", "\\|"); + } + + private static String padRight(String s, int width) { + if (s.length() >= width) return s; + return s + " ".repeat(width - s.length()); + } +} diff --git a/app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java b/app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java deleted file mode 100644 index fbbf5af9c..000000000 --- a/app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java +++ /dev/null @@ -1,153 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static org.assertj.core.api.Assertions.assertThat; -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.util.List; - -import org.junit.jupiter.api.Test; - -/** - * Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and - * column-grid construction. - */ -class LineAlignmentTableParserTest { - - private final LineAlignmentTableParser parser = new LineAlignmentTableParser(); - - // ── mergeCoincidentLines ───────────────────────────────────────────────────────────────────── - - @Test - void mergeCoincidentLines_singleLine_unchanged() { - var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue"))); - assertThat(parser.mergeCoincidentLines(lines)).hasSize(1); - } - - @Test - void mergeCoincidentLines_distinctYLines_unchanged() { - // Two lines at different y positions — must NOT be merged. - var lines = - List.of( - tokenized(rawLine(10f, 100f, "Revenue")), - tokenized(rawLine(10f, 115f, "Cost"))); - assertThat(parser.mergeCoincidentLines(lines)).hasSize(2); - } - - @Test - void mergeCoincidentLines_sameY_merged() { - // Simulates a financial-table row split by LineBuilder at the column gap: - // label fragment at x=72 → "Revenue" - // value fragment at x=350 → "1,234" - // Both have y=100. After merge they should form one TokenizedLine. - var label = rawLine(72f, 100f, "Revenue"); - var value = rawLine(350f, 100f, "1,234"); - - var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value))); - - assertThat(merged).hasSize(1); - // The merged line should contain tokens from both halves. - var tokens = merged.get(0).all(); - assertThat(tokens.stream().map(t -> t.text()).toList()) - .containsExactlyInAnyOrder("Revenue", "1,234"); - } - - @Test - void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() { - var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114 - var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380 - - var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value))); - - var bounds = merged.get(0).line().bounds(); - assertThat(bounds.x()).isEqualTo(72f); - assertThat(bounds.right()).isEqualTo(380f); - } - - @Test - void mergeCoincidentLines_withinTolerance_merged() { - // Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge. - var a = rawLine(10f, 100.0f, "Alpha"); - var b = rawLine(200f, 101.5f, "99"); - - var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b))); - assertThat(merged).hasSize(1); - } - - @Test - void mergeCoincidentLines_beyondTolerance_notMerged() { - // Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge. - var a = rawLine(10f, 100.0f, "Alpha"); - var b = rawLine(200f, 103.0f, "99"); - - var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b))); - assertThat(merged).hasSize(2); - } - - @Test - void mergeCoincidentLines_threeCoincident_allMerged() { - // Three fragments at the same y (e.g. wide financial table with two value columns). - var a = rawLine(72f, 100f, "Revenue"); - var b = rawLine(300f, 100f, "1,234"); - var c = rawLine(400f, 100f, "5,678"); - - var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c))); - assertThat(merged).hasSize(1); - assertThat(merged.get(0).all()).hasSize(3); - } - - @Test - void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() { - var a = rawLine(72f, 100f, "Revenue"); - var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a - var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate - - var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c))); - assertThat(merged).hasSize(2); - } - - @Test - void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() { - // After merging, the combined line should be an anchor (≥2 numeric tokens). - // "Revenue" alone → not an anchor. "1,234 567" alone → anchor. - // Merged → anchor with at least 2 numerics. - var label = rawLine(72f, 100f, "Revenue"); - var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f); - - var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values))); - - assertThat(merged).hasSize(1); - assertThat(merged.get(0).isAnchor()).isTrue(); - } - - // ── helpers ────────────────────────────────────────────────────────────────────────────────── - - /** Creates a RawLine with a single TextFragment of the given text at the given position. */ - private static RawLine rawLine(float x, float y, String text) { - float width = text.length() * 6f; // ~6pt per char — rough but consistent - float height = 12f; - Bounds bounds = new Bounds(x, y, width, height); - TextFragment fragment = - new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false); - return new RawLine("ln-test", List.of(fragment), bounds, 1); - } - - /** - * Creates a RawLine with two TextFragments representing two words separated by a small gap. - * Used to simulate a values-only line with multiple numeric tokens. - */ - private static RawLine rawLineMultiWord( - float x, float y, String word1, float w1, String word2, float w2) { - float height = 12f; - Bounds b1 = new Bounds(x, y, w1, height); - Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height); - TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false); - TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false); - Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height); - return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1); - } - - /** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */ - private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) { - return parser.tokenize(line); - } -} diff --git a/app/common/src/test/java/stirling/software/common/pdf/PdfMarkdownConverterTest.java b/app/common/src/test/java/stirling/software/common/pdf/PdfMarkdownConverterTest.java new file mode 100644 index 000000000..b3c104da8 --- /dev/null +++ b/app/common/src/test/java/stirling/software/common/pdf/PdfMarkdownConverterTest.java @@ -0,0 +1,269 @@ +package stirling.software.common.pdf; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import stirling.software.jpdfium.PdfDocument; +import stirling.software.jpdfium.text.TextLine; +import stirling.software.jpdfium.text.TextWord; + +/** + * Accuracy and robustness tests for {@link PdfMarkdownConverter}, comparing conversion output + * against hand-authored golden Markdown for a set of owned/synthetic fixtures. + * + *

The {@link #gatedFixtures()} set is enforced in CI: those fixtures currently convert within + * the accuracy threshold and guard against regressions. Fixtures still being iterated on live in + * {@link #wipFixtures()} under a {@link Disabled} test so the goldens stay in the tree without + * breaking the build. Enable the WIP test locally to see per-fixture scores while working on the + * converter. + */ +class PdfMarkdownConverterTest { + + /** Accuracy threshold: output must share at least this fraction of content with the golden. */ + private static final double THRESHOLD = 0.95; + + @TempDir Path tmp; + + /** Fixtures that meet the accuracy threshold today and therefore gate CI. */ + static Stream gatedFixtures() { + return Stream.of( + Arguments.of("multi-column-test_lorem.pdf", "multi-column-test_lorem.md"), + Arguments.of("bordered-table-test_widget.pdf", "bordered-table-test_widget.md"), + Arguments.of("many-tables-test_stress.pdf", "many-tables-test_stress.md")); + } + + /** Fixtures still below the threshold; tracked here, enable locally to iterate. */ + static Stream wipFixtures() { + return Stream.of( + Arguments.of( + "wrapped-cell-test_expense-report.pdf", + "wrapped-cell-test_expense-report.md")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("gatedFixtures") + void convertMatchesGoldenMarkdown(String pdfName, String mdName) throws IOException { + assertConversionMatchesGolden(pdfName, mdName); + } + + @Disabled("WIP fixtures below the accuracy threshold; enable locally to iterate") + @ParameterizedTest(name = "{0}") + @MethodSource("wipFixtures") + void convertMatchesGoldenMarkdownWip(String pdfName, String mdName) throws IOException { + assertConversionMatchesGolden(pdfName, mdName); + } + + /** + * Degenerate/extreme geometry must not crash the converter. A crafted or malformed PDF can + * position text anywhere via a text matrix, so a row's words can span from near the origin to a + * coordinate beyond {@link Integer#MAX_VALUE}. The old column-detection code sized an {@code + * int[]} straight from {@code (int) Math.ceil(maxX) - lo}, which either allocated a multi-GB + * array (OutOfMemoryError) or overflowed to a negative length (NegativeArraySizeException) — + * taking down the request thread. Detection must instead bail out and return no columns. + */ + @Test + void columnDetectionSurvivesDegenerateGeometry() { + // x ≈ 2.5e9 is past Integer.MAX_VALUE; combined with a near-origin word it yields an + // implausible span that the pre-fix code turned into a fatal array allocation. + List rows = new ArrayList<>(); + for (int r = 0; r < 4; r++) { + float y = 400f - r * 12f; + TextWord near = new TextWord(List.of(), 50f, y, 30f, 10f); + TextWord far = new TextWord(List.of(), 2_500_000_000f, y, 30f, 10f); + rows.add(new TextLine(List.of(near, far), 50f, y, 2_499_999_980f, 10f)); + } + + List columns = + assertDoesNotThrow(() -> PdfMarkdownConverter.findColumnRangesFromLines(rows)); + assertTrue( + columns.isEmpty(), + "implausible page span should disable column detection, not allocate from it"); + } + + private void assertConversionMatchesGolden(String pdfName, String mdName) throws IOException { + Path pdfPath = tmp.resolve(pdfName); + try (InputStream in = + getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + pdfName)) { + if (in == null) { + fail("Fixture not found on classpath: /pdf-ingestion-fixtures/" + pdfName); + } + Files.copy(in, pdfPath); + } + + String actual; + try (PdfDocument doc = PdfDocument.open(pdfPath)) { + actual = new PdfMarkdownConverter().convert(doc); + } + + String expected; + try (InputStream in = getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + mdName)) { + if (in == null) { + fail("Golden file not found on classpath: /pdf-ingestion-fixtures/" + mdName); + } + expected = new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + + // Image placeholders are not scored: their body text is a TODO ("ideally, add the info + // available about the image...") rather than real content, so comparing it would penalise + // output for matching a placeholder we intend to replace. Drop those lines from both sides. + expected = stripImagePlaceholders(expected); + actual = stripImagePlaceholders(actual); + + double similarity = similarity(expected, actual); + if (similarity < THRESHOLD) { + fail( + String.format( + "Markdown output differs from golden file '%s' by %.1f%% (threshold %.0f%%):%n%s", + mdName, + (1.0 - similarity) * 100, + (1.0 - THRESHOLD) * 100, + unifiedDiff(expected, actual))); + } + } + + /** Substring identifying an image-placeholder line, which is excluded from scoring. */ + private static final String IMAGE_PLACEHOLDER_MARKER = "Image intentionally redacted"; + + /** + * Removes non-content lines from the comparison: image placeholders (TODO text we intend to + * replace) and GFM table separator rows (the {@code |---|---|} divider, whose exact dash count + * is cosmetic — any run of three or more dashes is valid Markdown). + */ + private static String stripImagePlaceholders(String md) { + StringBuilder sb = new StringBuilder(); + for (String line : md.split("\n", -1)) { + if (line.contains(IMAGE_PLACEHOLDER_MARKER) + || line.strip().startsWith(" 0) { + sb.append('\n'); + } + sb.append(line); + } + return sb.toString(); + } + + /** True for a GFM table separator row, e.g. {@code |---|:--:|---|} (only |, -, :, space). */ + private static boolean isTableSeparatorRow(String line) { + String t = line.strip(); + if (!t.contains("-")) { + return false; + } + return t.chars().allMatch(c -> c == '|' || c == '-' || c == ':' || c == ' '); + } + + /** + * Character-level similarity: proportion of expected characters that appear in the LCS. O(n*m) + * but golden files are small enough that this is fine. + */ + private static double similarity(String expected, String actual) { + if (expected.isEmpty() && actual.isEmpty()) return 1.0; + if (expected.isEmpty() || actual.isEmpty()) return 0.0; + // Strip all whitespace for a content-focused comparison + String e = expected.replaceAll("\\s+", " ").strip(); + String a = actual.replaceAll("\\s+", " ").strip(); + int lcs = lcsLength(e, a); + return (double) lcs / Math.max(e.length(), a.length()); + } + + private static int lcsLength(String a, String b) { + // Use two-row DP to keep memory reasonable + int m = a.length(), n = b.length(); + int[] prev = new int[n + 1]; + int[] curr = new int[n + 1]; + for (int i = 1; i <= m; i++) { + for (int j = 1; j <= n; j++) { + if (a.charAt(i - 1) == b.charAt(j - 1)) { + curr[j] = prev[j - 1] + 1; + } else { + curr[j] = Math.max(curr[j - 1], prev[j]); + } + } + int[] tmp = prev; + prev = curr; + curr = tmp; + java.util.Arrays.fill(curr, 0); + } + return prev[n]; + } + + private static String unifiedDiff(String expected, String actual) { + String[] expectedLines = expected.split("\n", -1); + String[] actualLines = actual.split("\n", -1); + + List diff = new ArrayList<>(); + diff.add("--- expected"); + diff.add("+++ actual"); + + int maxLines = Math.max(expectedLines.length, actualLines.length); + int context = 3; + boolean inHunk = false; + int hunkStart = -1; + List hunkLines = new ArrayList<>(); + + for (int i = 0; i < maxLines; i++) { + String exp = i < expectedLines.length ? expectedLines[i] : null; + String act = i < actualLines.length ? actualLines[i] : null; + + boolean changed = exp == null || act == null || !exp.equals(act); + if (changed) { + if (!inHunk) { + inHunk = true; + hunkStart = Math.max(0, i - context); + // add context lines before change + for (int c = hunkStart; c < i; c++) { + hunkLines.add(" " + (c < expectedLines.length ? expectedLines[c] : "")); + } + } + if (exp != null) hunkLines.add("-" + exp); + if (act != null) hunkLines.add("+" + act); + } else { + if (inHunk) { + hunkLines.add(" " + exp); + // check if we're far enough past the last change to close the hunk + boolean moreChanges = false; + for (int j = i + 1; j < Math.min(i + context, maxLines); j++) { + String e2 = j < expectedLines.length ? expectedLines[j] : null; + String a2 = j < actualLines.length ? actualLines[j] : null; + if (e2 == null || a2 == null || !e2.equals(a2)) { + moreChanges = true; + break; + } + } + if (!moreChanges && (i - hunkStart) >= context) { + diff.add("@@ -" + (hunkStart + 1) + " @@"); + diff.addAll(hunkLines); + hunkLines.clear(); + inHunk = false; + } + } + } + } + + if (inHunk && !hunkLines.isEmpty()) { + diff.add("@@ -" + (hunkStart + 1) + " @@"); + diff.addAll(hunkLines); + } + + return String.join("\n", diff); + } +} diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.md b/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.md new file mode 100644 index 000000000..4b590e4b6 --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.md @@ -0,0 +1,10 @@ +# Widget Inventory Report + +This report lists current stock levels for each warehouse. + +| Region | Units | Status | +|---|---|---| +| North | 1200 | OK | +| South | 950 | Low | +| East | 1430 | OK | +| West | 875 | Low | diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.pdf b/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.pdf new file mode 100644 index 000000000..8da041e28 --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.pdf @@ -0,0 +1,74 @@ +%PDF-1.4 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20260603003133+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603003133+01'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 500 +>> +stream +Gas1[9i&Y\%))C:pc(6t3;pQ8%0T@tD+-Nf,MP8j(>R4tMCHL4NI*\1+UNiI'V9NC%VeJKn/YI0J];XQt&X83?=ihrg<*Mcn1n!1nWcDaQPe\P"9gnJuHl(jf]JQgZ[,&^uobI4QF',k"*^S)3c;)GMWC(T'=",ErnS#U=YCUN0&q4+*KmK1Zd*NI\GQDiZUG7;PTja8lulb"\PWWO#WcfI[ZB:6s*3g$be%?JH(n`oaEJ[XE'%QW=HE04M<,;ERm[MS=uYF=nN3jG'f@#?O48Ia,6Y-3m&tTWVq1?DeiBkp.Ug*;lVZX`Z=P.eklHhNV;!R_?QOuoeJ<0%7idG7GM8boU$^>N.N,2^;25]0Z8M<<]XMCct>noC'Qfb?`*[Mo+,F9#>t~>endstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000514 00000 n +0000000582 00000 n +0000000862 00000 n +0000000921 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1511 +%%EOF diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.md b/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.md new file mode 100644 index 000000000..4b456165d --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.md @@ -0,0 +1,222 @@ +Intro paragraph for section 1. + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | + +# Section 2 Heading + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | +| golf | 301 | india | + +## Section 3 Heading + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | +| golf | 301 | india | 23 | +| juliet | 401 | lima | 33 | + +Intro paragraph for section 4. + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | +| golf | 301 | india | 23 | kilo | +| juliet | 401 | lima | 33 | november | +| mike | 501 | oscar | 43 | alpha | + +# Section 5 Heading + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | +| golf | 301 | +| juliet | 401 | +| mike | 501 | +| papa | 601 | + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | + +# Section 7 Heading + +Intro paragraph for section 7. + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | +| golf | 301 | india | 23 | + +## Section 8 Heading + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | +| golf | 301 | india | 23 | kilo | +| juliet | 401 | lima | 33 | november | + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | +| golf | 301 | +| juliet | 401 | +| mike | 501 | + +# Section 10 Heading + +Intro paragraph for section 10. + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | +| golf | 301 | india | +| juliet | 401 | lima | +| mike | 501 | oscar | +| papa | 601 | bravo | + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | + +# Section 12 Heading + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | +| golf | 301 | india | 23 | kilo | + +## Section 13 Heading + +Intro paragraph for section 13. + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | +| golf | 301 | +| juliet | 401 | + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | +| golf | 301 | india | +| juliet | 401 | lima | +| mike | 501 | oscar | + +# Section 15 Heading + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | +| golf | 301 | india | 23 | +| juliet | 401 | lima | 33 | +| mike | 501 | oscar | 43 | +| papa | 601 | bravo | 53 | + +Intro paragraph for section 16. + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | + +# Section 17 Heading + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | +| golf | 301 | + +## Section 18 Heading + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | +| golf | 301 | india | +| juliet | 401 | lima | + +Intro paragraph for section 19. + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | +| golf | 301 | india | 23 | +| juliet | 401 | lima | 33 | +| mike | 501 | oscar | 43 | + +# Section 20 Heading + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | +| golf | 301 | india | 23 | kilo | +| juliet | 401 | lima | 33 | november | +| mike | 501 | oscar | 43 | alpha | +| papa | 601 | bravo | 53 | delta | + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | + +# Section 22 Heading + +Intro paragraph for section 22. + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | +| golf | 301 | india | + +## Section 23 Heading + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | +| golf | 301 | india | 23 | +| juliet | 401 | lima | 33 | + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | +| golf | 301 | india | 23 | kilo | +| juliet | 401 | lima | 33 | november | +| mike | 501 | oscar | 43 | alpha | + +# Section 25 Heading + +Intro paragraph for section 25. + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | +| golf | 301 | +| juliet | 401 | +| mike | 501 | +| papa | 601 | diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.pdf b/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.pdf new file mode 100644 index 000000000..f12925cda --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.pdf @@ -0,0 +1,169 @@ +%PDF-1.4 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 13 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/Contents 14 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +6 0 obj +<< +/Contents 15 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +7 0 obj +<< +/Contents 16 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +8 0 obj +<< +/Contents 17 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +9 0 obj +<< +/Contents 18 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +10 0 obj +<< +/PageMode /UseNone /Pages 12 0 R /Type /Catalog +>> +endobj +11 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20260603005358+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603005358+01'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +12 0 obj +<< +/Count 6 /Kids [ 4 0 R 5 0 R 6 0 R 7 0 R 8 0 R 9 0 R ] /Type /Pages +>> +endobj +13 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 1007 +>> +stream +GauHKgN)$k&:O:SnDZmeMUK]*(BdhV"2[X,j0;*\%_E*0HXJRc`q%4S#/VM`O[Rh4i5T.Phi,$]aT$05!q:PBuCQU/.p2]@1koN*Tb*K9k5G\ht+Dr\K=+8\NZ"alMaOEo**@OK:8-.O1X3-?Gg`@m3%,ti3'">T-&c=M&Wuu?cDbGDp.gOF0!r6&&CLC$tM?fIR"M37["/*k9@YkpKKSS@Np"1/4#R]`I^(g*1Sc,9L3Qt6N(T@F`A>oBGgL-!r#Y4)R\G`D,iVtFeJUE9u4iuUQ?D%C&SB4kkp.>D5tii>nDKJ"Y07jANhOb$R(_$=U7Stjs)-/KZ(6IBm`6u<3;i.Bh4+MFJ"H:.XWUQX6%LU(sg4Tt$_":5`p.2,gZkpUfdg,Hd(qR1)9\ltF2^8b5,,XbY%VfhD52O7A.c.u]dhTc5t%0<0L5E38`Bq+i;"%J7kcc#@i)@okdN-qiaA"33DdPgPrUs7;j%+47_cnGN@&ug9/dqGOAHA4f.N*&guHspf?E;GIc-Wt>:<(m1AmcS_Zc2VlEI'_S_>@#!MF^m1$$endstream +endobj +14 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 987 +>> +stream +GatU3gN&c;&:O:SkV;H,eI!J\f?X"LSPIZ'"4Y.F#oAAY?N.Y?8Iu:s)eV;,aNG1Lea>G$!MSc\rU6m7jF'AOI09V1,^TSTi$A+f4sZVQ%2^>uj$40\AI>WbZo9q!NL^CS9P3E9X.:XEMb$a/X"qH9hN6fY,]D!OEOVT,722%RJnVqK4f'[4d3(/hbJWJRs,>>28jk5A:nCl/%FlC#LntsAC5cE$q`QO_@Pk%Lp>7V23")NNrFkcXfuoI=SJJ-`g)_+K\@,SQ@8ORGg(_(B.[u[WXD%Q8@I";9d(]M62K_)Q+-<\kHY5,)o99%BB2bcQVkd:+MZVH=$Z`S@BhH]-XfPANk@qBN.:Jc'?.Kn\p7(aPIQOI(du7U]WDrNNPW%d"8P_j^Y8d%G,'JJLXrV.UD!ah19&9b#*fgR2o730<9)QX)X0"EK6u;mBJs3/fl\d&K$B3bXI5R>F*E,^\%+b?0o8s$o.CuX4uDAT_?G0(p4N3La,8qfi'j?e`e88KrZ8*NDgc:62'icCb],4B]Z"SO&e/BRC*C$2PmbSKAjc\$FLD8?&qHVH3G4p834/77n[%()-p/JDRBapcO2b\,3dW%#U9u!ilSkd%2'rr&g[u".1O]W%0J<@#(ptUD;%0:^.^ls:.#.V6NgR[`2\RUW!k$-*GZIf3DOBiB_Mu"=LVGlO\1Z]0pF(@UaiVQ=dh?h5kfQ_gj>$jHW$8TCt:C.jVU98ne.XoiMtoV;q8XhnlOE;3a]=dGJ8KL&+Khj"&P4Q=P-:nHsDprY"B4+#A3dF,;b&gf(sT\Ts*iH)f!KhfHV;\mMSHej.9.XrBG41_<:b)MG;-Y(:&nCKS0F]r&CGl>EfMc0td\0GHScl3Lr;?OUoo6619Qdr)S8S);]7L5rmWW&(5t4Dil"USDhhY/Y=!=!X:85**$:~>endstream +endobj +15 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 976 +>> +stream +GatU3gN&c;'R]XVkgAc"70atdYFXp#3h7VV#7/=-%N$"G?N.YO&iX#cVsS_l_5i8)eum:1-)(7`mXJ7LnhqY0hGZal,We=q^e"$M]Lu;_=4AoIN$QOjOF.@i*Sg!_rLL=-'?kk;m;M&R`$D>'HCL7]$A,-S`H1I/1T]4\B3=gQ9:I4j9f]+af*Z7*GH5H+;j73Z-T#DAcgpOW(/'b?XF9+X+'-Dqi"+1U7?6=ZRJNBSJ6ojDTa+]o4RF^26C+i0s(9G2o;$@RNbs\(f75],L(SVpI0M!'7JVhZ"$t/%kb1+)F\p54/@jpVJF2+)7^F=gVcp8(h*m%F#'p9c%9Ep9?b7g^F5>(>I3t^d_"f'aue%^E\X4kN:g\P[;sC=g3h,nD/n9*`94uoM?sBCL4;XilfMPT4]&FC.U%DI(c6uf3*XPJP2MRdq@Ag3uYF().nqLFAE2Nih'(^p\5FnSNa>n+cI?U65A61_N1<9ZXH4Xr)Fgq(6md3H9[MQ3Q^%aq>E?XM:O:RKGjc+QV2MB!.I.17QpSt]X[C*6ap.`#'o;8F(ebZ.VMM6k`BkZB\I,2MNoX]J"k]Qd";,@($aWX&29q"6k;>Let)@]QHm:i/lf[DB_&U@ROq-0MLQp;@j+]?$"E)brXVun52AVQo$"[a7@!?!LT*>$&:5X_.R;9)'$l-S.>WjLQIVTo.JU%(H\,-*pl_kP9~>endstream +endobj +16 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 1024 +>> +stream +GauHKflEQ9'Rf^WgrHc4<#\/Qm7c-rFIIq+TFSD%Z#L'6o(Nl&^_!k0ds=.%-s&pt#i0QBKE91*<^/MJJCcfoHA_e;aL?\&_BAj]Dt;HW$6(8Ql\%O7-5%uT80ZIL#\l)(+[ABOTNKmq!2F,S:.F93Z6QIu[nf\Q[qCE`Y-=.8k$67iCR!7=El+50a@f&b^6'Yg'mqlcRQ-5j:%[K@'5!A_m0L)&VlW1T50^X)"3Ma3,U1P8Di$>uPZ)Zj_igBc0lm]WE0#e>*M53(Yh6\9jHYh6B+3bM\b[Q9j'O_9:!:W0$Ijm,#(bKn_D?UYMGJ7LLqC0\[$l<8U*O5^GnN;!N6YB'?d[d8DAPt^>+E:r0<2$'VL/TtuNg;C@5iM#%KdS-GU->0]`/20eLXVW#qnjsU.P:Xj&=enVE[mpqDCn2!qDRuQhI/#cZ-#m6`U9'SI4"9\"r:4p.\W>IK_$#C,EkU#uO1g6*HoUK':XiJjtQfI'Sdc)B3J1eqgGJ+l?"$//&C$^f'TA!2:K!0pd;^Zif)c[:f!C/8hNKc7+'+1WbOFKr9T7c_G$s*O634O2Zs:"iN>j,pWU+:kEQ>5YbF=$27*59iJEA//7(KsL\pUG.G[+JB-r8t;Sih+(W.A'8Q*LZK*I9WjCLjEt>lT%2RFJZg0Ps:5c,HdK#tSNI:#Rp!]O$Ic=T%K0[._E_Fcjn>OcM,iSAf9iF7,,+Qe:@o'Y4o_:54t*l+TAgendstream +endobj +17 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 1075 +>> +stream +GauHKgN&c;&:O:SkV;H,6&QZ[5)3-pL]0bpl':D91fR,o"F49.1/bfmFt32ljt6eO[j7!K]\nZ:^QQASu[l@3no(6*HL'@3I!Qfi7%,.N\.>A91O)D\n5*o@JWSOI@ng5p/"_I`g*Y$j0+5PWaso'25ltcglFd,QIaO'dPO0Yr\h5/,=m\PiP:P+Gg0&N;_5iLN$]BD.#VL+h,tSe\fL&,;+2fg&%\pAJiV#eVhq@ro%%/`[(&8n29YiYd>bDF^Bg[AVf/n>[FE(au"ZAe&>%Q[7/n3-QhsPXuPbp#<$d"UR8u-nBnr!Q$Wjd1;9WN/,KG8])Ca5rrDrI'Ue'*I0C%pn+f`Hd"4sB_&2*)S$pX=F/6"DC"TU`bMtd/m7u]?]=J5L[SNBuE:6Ri,NmW7.*qUp)97Tt+a84b\os`Ti8/uRDlhrl*Xrn(a0\,%4%*k^rL\k3%6ako`&A3MoN^CXV77D!Qg%=j9Ge+0B@dZDHELb2fDU(,iiXXRl^*U,U#Fld!R72UIpKGWu#-DXi653a?U$fqCs18gt(5/#U%!u,gkMK;>OT_/='S:kn,iD@u]<8-rlTRu'+\7L$U7-#!5-5\WQn(n:q@-p24u!~>endstream +endobj +18 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 590 +>> +stream +GasbXgMWc?&;KY!MRgt!"n`^Bf\9$c*`Q-Sb6tfd8P(q-dT1eng=QV-!OG*\kiWo2AGBdKLqI^%h,XNB)-gDk+GFVBa?g*a%:!J\97Vc8-jH8>8P>0S@Wr)o*"Hu<6qPp`EF:M3'l4@S\c2]`&[J%dO@5p$<(([H6:SlB;FS8L`&pO%6Y\V"/O7F]E%L@d.%>L@"4bK_XXs&";n_.W^Nr847HH,X@$.pC"4bYC?9RH,dN?h8a+8/!?Q=D\F41Nc@'B`';4XgMeh!;U23C+>AbMQD-o--lJ/":m#(Yt,X5(`1KL,GTMBLlr6=-1mi#k.-Ou\\T4$Pun2dEU(\?$&;1@T4dm^t!KuOD-U@N_AMr"&uVpsGm,+8I7B*f!%9.o4cC1a[CZ12(hd>1*0bU`k2-MXo1[Gor#kmXGIM'#R49X#NSAOpdf'0ilUH4:M(^Snc3;m/+>endstream +endobj +xref +0 19 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000516 00000 n +0000000711 00000 n +0000000906 00000 n +0000001101 00000 n +0000001296 00000 n +0000001491 00000 n +0000001561 00000 n +0000001842 00000 n +0000001932 00000 n +0000003031 00000 n +0000004109 00000 n +0000005176 00000 n +0000006292 00000 n +0000007459 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 11 0 R +/Root 10 0 R +/Size 19 +>> +startxref +8140 +%%EOF diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.md b/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.md new file mode 100644 index 000000000..5c35de111 --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.md @@ -0,0 +1,25 @@ +# Lorem Ipsum in Two Columns + +## 1. Origins + +Lorem ipsum dolor sit amet consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. + +## 2. Structure + +Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. + +## 3. Usage + +Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. + +## 4. Variations + +Excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum. + +## 5. Typography + +Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam various turpis et commodo pharetra est. + +## 6. Conclusion + +Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.pdf b/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.pdf new file mode 100644 index 000000000..36dc3a1a6 --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.pdf @@ -0,0 +1,74 @@ +%PDF-1.4 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20260603021636+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603021636+01'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 861 +>> +stream +Gat=i>Ak00&;B$5/*8Q/Z)fmNp(^2^OK'MC[dT6#Oq.&b^4c4;1No6>+D%TC.n+fice+l9R0[L%'(osQ?dQP@AUa.A:T=#%ZIIl18hB7ZfI(hVU+?`]UK%;aabAH:q>9NUGQq^.=u])Wt-ETI))C'a[FE@elj!RSrf2Q'F>URAO.C,!DneTPqrj#4e2kb9%1"4qfZ)"#&^0j9nHQ?9nF!j7mVPP5\*Uq'_jMVS]9%`kQB\8*AF_bpr/hGj;HCUOSQU-%5:6S79Ud\b!*tPbr_'pCr$Ea#(FYP31NFhSX.-("1M:$cgH#hX8L(2]R3Q>'BYHCS%pI!;=WdJp,'ii[`QPZ_9mcd\baZ2U(_;c\-p,8EoIEpQ*lstL>]LE;C#\dLnT2R:)BM-fTc['3_He[U,k'!Bo".uERd>SkhRj^J+koSIrZ_dEf_5L'/1h.`+DTK(R:P,WH)h5\se=SZ"L/5b8b..,e/E\o+4YQ+*im^C>AERG/TieEK\)#>U@HXnJ,H0A9-MqhhkDp8%.6Lr,OrK*lih;B<-opZ8%EU?,$r^jmCDAQ`-0/-8/`[p]7Fm%0f:E&S*FV)DX2>#q\bRqA=^_`43#8EA$u%8r6F5`rc9>K@q>E4q^*~>endstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000514 00000 n +0000000582 00000 n +0000000862 00000 n +0000000921 00000 n +trailer +<< +/ID +[<21a9fbd0a0991a91b6e6e2db0856056e><21a9fbd0a0991a91b6e6e2db0856056e>] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1872 +%%EOF diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.md b/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.md new file mode 100644 index 000000000..8008a3b97 --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.md @@ -0,0 +1,62 @@ +# Employee Expense Report + +Reimbursement Request + +EMP-1047 + +**Report Header** + +| Employee Name | Michael Tran | +|---|---| +| Employee ID | EMP-1047 | +| Department | Client Services | +| Report Date | January 20th, 2026 | +| Reporting Period | January 5th–16th, 2026 | +| Manager Approver | Laura Simmons | + +**Company Information** + +| Company | Summit Consulting Partners | +|---|---| +| Company Address | 88 Riverside Plaza, Suite 1400, New York, NY 10069 | +| Accounting Department Email | expenses@example.com | + +**Trip Purpose** + +The trip was undertaken for client onsite meetings with Atlantic Energy Solutions in Boston, MA. + +**Expense Details** + +| Description | Amount | Date | Category | +|---|---|---|---| +| Flight (NYC to Boston roundtrip) | $325.40 | January 5th, 2026 | Airline ticket | +| Hotel (3 nights at Harborview Hotel) | $822.75 | January 5th–8th, 2026 | Lodging | +| Taxi from airport to hotel | $48.00 | January 5th, 2026 | Ground transportation | +| Client dinner (3 attendees) | $186.20 | January 6th, 2026 | Meals | +| Parking at JFK Airport | $72.00 | January 5th–8th, 2026 | Parking | +| Breakfast (per diem not used) | $18.50 | January 7th, 2026 | Meals | + +| Description | Amount | Date | Category | +|---|---|---|---| +| Uber to client office | $22.10 | January 7th, 2026 | Ground transportation | +| Printing + presentation materials | $46.90 | January 8th, 2026 | Materials | +| Lunch with client | $39.75 | January 8th, 2026 | Meals | +| Office supplies (notebooks, pens) | $27.60 | January 10th, 2026 | Supplies | +| Mileage reimbursement (client visit in NJ, 42 miles @ $0.67/mile) | $28.14 | January 14th, 2026 | Mileage | +| Team lunch meeting (internal) | $64.30 | January 15th, 2026 | Meals | + +Total Expenses $1,701.64 + +Reimbursement Method + +Reimbursement method Direct deposit + +Notes + +All receipts are attached. Expenses are business-related and comply with company travel policy. + +**Approval** + +Michael Tran, Employee + +Laura Simmons, Manager diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.pdf b/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.pdf new file mode 100644 index 000000000..95a0b2e07 Binary files /dev/null and b/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.pdf differ diff --git a/app/core/src/main/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdown.java b/app/core/src/main/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdown.java index ce5a61078..42ebd51ab 100644 --- a/app/core/src/main/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdown.java +++ b/app/core/src/main/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdown.java @@ -1,11 +1,13 @@ package stirling.software.SPDF.model.api.converters; -import org.springframework.core.io.Resource; +import java.nio.charset.StandardCharsets; + import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.multipart.MultipartFile; +import io.github.pixee.security.Filenames; import io.swagger.v3.oas.annotations.Operation; import lombok.RequiredArgsConstructor; @@ -15,8 +17,11 @@ import stirling.software.common.annotations.AutoJobPostMapping; import stirling.software.common.annotations.api.ConvertApi; import stirling.software.common.enumeration.ResourceWeight; import stirling.software.common.model.api.PDFFile; -import stirling.software.common.util.PDFToFile; +import stirling.software.common.pdf.PdfMarkdownConverter; +import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.WebResponseUtils; +import stirling.software.jpdfium.PdfDocument; @ConvertApi @RequiredArgsConstructor @@ -33,10 +38,27 @@ public class ConvertPDFToMarkdown { summary = "Convert PDF to Markdown", description = "This endpoint converts a PDF file to Markdown format. Input:PDF Output:Markdown Type:SISO") - public ResponseEntity processPdfToMarkdown(@ModelAttribute PDFFile file) + public ResponseEntity processPdfToMarkdown(@ModelAttribute PDFFile file) throws Exception { MultipartFile inputFile = file.getFileInput(); - PDFToFile pdfToFile = new PDFToFile(tempFileManager); - return pdfToFile.processPdfToMarkdown(inputFile); + + String originalName = Filenames.toSimpleFileName(inputFile.getOriginalFilename()); + String baseName = + originalName.contains(".") + ? originalName.substring(0, originalName.lastIndexOf('.')) + : originalName; + + String markdown; + try (TempFile tempInput = new TempFile(tempFileManager, ".pdf")) { + inputFile.transferTo(tempInput.getFile()); + try (PdfDocument doc = PdfDocument.open(tempInput.getPath())) { + markdown = new PdfMarkdownConverter().convert(doc); + } + } + + return WebResponseUtils.bytesToWebResponse( + markdown.getBytes(StandardCharsets.UTF_8), + baseName + ".md", + MediaType.valueOf("text/markdown")); } } diff --git a/app/core/src/main/resources/settings.yml.template b/app/core/src/main/resources/settings.yml.template index d4ecb35f7..413f39706 100644 --- a/app/core/src/main/resources/settings.yml.template +++ b/app/core/src/main/resources/settings.yml.template @@ -364,6 +364,18 @@ aiEngine: url: http://localhost:5001 # URL of the Python AI engine timeoutSeconds: 120 # Timeout in seconds for AI engine requests +policies: + # Folder automations can read from and write to the directories you allow here, so treat this as a + # security boundary. Leave allowedFolderRoots empty (default) to disable folder sources/outputs + # entirely; list absolute directories to permit folder access only within them. Stirling's own + # config directory is always off-limits, and folder access is always disabled in SaaS mode. + allowedFolderRoots: [] # e.g. ["/data/inbox", "/data/outbox"] + scheduleSweepSeconds: 60 # How often (seconds) scheduled policies are checked for being due + watchReconcileSeconds: 300 # How often (seconds) folder-watch re-syncs watches and re-runs as a safety net for missed events + watchQuietPeriodMs: 500 # How long (ms) folder-watch coalesces a burst of file events into a single run + streamTimeoutMs: 1800000 # SSE timeout (ms) for live run-progress streams + runExpiryMinutes: 30 # How long (minutes) a finished run's in-memory state is kept before eviction + # Model Context Protocol (MCP) server. Exposes Stirling's PDF tools (grouped by namespace) # plus the AI agents to MCP clients (Inspector, Claude Desktop, custom). OAuth-protected. # Disabled by default - enable explicitly per deployment after configuring mcp.auth. diff --git a/app/core/src/test/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdownTest.java b/app/core/src/test/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdownTest.java index b63e58b52..3bd6b7fad 100644 --- a/app/core/src/test/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdownTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdownTest.java @@ -1,16 +1,17 @@ package stirling.software.SPDF.model.api.converters; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; +import java.io.File; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.Resource; @@ -21,9 +22,10 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; -import org.springframework.web.multipart.MultipartFile; -import stirling.software.common.util.PDFToFile; +import stirling.software.common.pdf.PdfMarkdownConverter; +import stirling.software.common.util.TempFile; +import stirling.software.jpdfium.PdfDocument; class ConvertPDFToMarkdownTest { @@ -47,68 +49,68 @@ class ConvertPDFToMarkdownTest { @Test void pdfToMarkdownReturnsMarkdownBytes() throws Exception { byte[] md = "# heading\n\ncontent\n".getBytes(StandardCharsets.UTF_8); + String expectedMd = "# heading\n\ncontent\n"; - try (MockedConstruction construction = - Mockito.mockConstruction( - PDFToFile.class, - (mock, ctx) -> { - when(mock.processPdfToMarkdown(any(MultipartFile.class))) - .thenAnswer( - inv -> - ResponseEntity.ok() - .header("Content-Type", "text/markdown") - .body(new ByteArrayResource(md))); - })) { + File tmpFile = File.createTempFile("test", ".pdf"); + tmpFile.deleteOnExit(); - MockMvc mvc = mockMvc(); + try (MockedConstruction tempMock = + Mockito.mockConstruction( + TempFile.class, + (mock, ctx) -> { + when(mock.getFile()).thenReturn(tmpFile); + when(mock.getPath()).thenReturn(tmpFile.toPath()); + }); + MockedStatic docStatic = Mockito.mockStatic(PdfDocument.class); + MockedConstruction converterMock = + Mockito.mockConstruction( + PdfMarkdownConverter.class, + (mock, ctx) -> when(mock.convert(any())).thenReturn(expectedMd))) { + + PdfDocument mockDoc = Mockito.mock(PdfDocument.class); + docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc); MockMultipartFile file = new MockMultipartFile( - "fileInput", // must match the field name in PDFFile - "input.pdf", - "application/pdf", - new byte[] {1, 2, 3}); + "fileInput", "input.pdf", "application/pdf", new byte[] {1, 2, 3}); - // ResponseEntity is written synchronously on the request thread, - // so there is no async dispatch to wait for (unlike the old StreamingResponseBody - // path). - mvc.perform(multipart("/api/v1/convert/pdf/markdown").file(file)) + mockMvc() + .perform(multipart("/api/v1/convert/pdf/markdown").file(file)) .andExpect(status().isOk()) .andExpect(header().string("Content-Type", "text/markdown")) .andExpect(content().bytes(md)); - - // Verify that exactly one instance was created - assert construction.constructed().size() == 1; - - // And that the uploaded file was passed to processPdfToMarkdown() - PDFToFile created = construction.constructed().get(0); - ArgumentCaptor captor = ArgumentCaptor.forClass(MultipartFile.class); - verify(created, times(1)).processPdfToMarkdown(captor.capture()); - MultipartFile passed = captor.getValue(); - - // Minimal plausibility checks - assertEquals("input.pdf", passed.getOriginalFilename()); - assertEquals("application/pdf", passed.getContentType()); } } @Test void pdfToMarkdownWhenServiceThrowsReturns500() throws Exception { - try (MockedConstruction ignored = - Mockito.mockConstruction( - PDFToFile.class, - (mock, ctx) -> { - when(mock.processPdfToMarkdown(any(MultipartFile.class))) - .thenThrow(new RuntimeException("boom")); - })) { + File tmpFile = File.createTempFile("test", ".pdf"); + tmpFile.deleteOnExit(); - MockMvc mvc = mockMvc(); + try (MockedConstruction tempMock = + Mockito.mockConstruction( + TempFile.class, + (mock, ctx) -> { + when(mock.getFile()).thenReturn(tmpFile); + when(mock.getPath()).thenReturn(tmpFile.toPath()); + }); + MockedStatic docStatic = Mockito.mockStatic(PdfDocument.class); + MockedConstruction converterMock = + Mockito.mockConstruction( + PdfMarkdownConverter.class, + (mock, ctx) -> + when(mock.convert(any())) + .thenThrow(new RuntimeException("boom")))) { + + PdfDocument mockDoc = Mockito.mock(PdfDocument.class); + docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc); MockMultipartFile file = new MockMultipartFile( "fileInput", "x.pdf", "application/pdf", new byte[] {0x01}); - mvc.perform(multipart("/api/v1/convert/pdf/markdown").file(file)) + mockMvc() + .perform(multipart("/api/v1/convert/pdf/markdown").file(file)) .andExpect(status().isInternalServerError()); } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java index a7239e8f9..2bed56f0f 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java @@ -21,7 +21,8 @@ public enum AiWorkflowOutcome { COMPLETED("completed"), UNSUPPORTED_CAPABILITY("unsupported_capability"), CANNOT_CONTINUE("cannot_continue"), - GENERATE_FILE("generate_file"); + GENERATE_FILE("generate_file"), + CONVERT_MARKDOWN("convert_markdown"); private final String value; diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/FolderAccessGuard.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/FolderAccessGuard.java new file mode 100644 index 000000000..3abf919f1 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/FolderAccessGuard.java @@ -0,0 +1,106 @@ +package stirling.software.proprietary.policy.config; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +import stirling.software.common.configuration.InstallationPathConfig; +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.model.Policy; + +/** + * The single authority on which filesystem locations a policy may read from or write to. Folder + * sources and sinks take a configured directory, so without this a user who can save a policy could + * point one at Stirling's own config/secrets directory and exfiltrate (or overwrite) it. Every + * folder source and sink runs its directory through {@link #requirePermitted(Path)} at save time + * and again at run time. + * + *

Enforced fail-closed, in order: + * + *

    + *
  • Disabled in SaaS - folder access is never allowed when the {@code saas} profile is + * active; a tenant must not reach the host filesystem at all. + *
  • Protected paths - Stirling's own config directory (settings, database, keys, + * backups) is always rejected, even if an allowed root were misconfigured to contain it. + *
  • Allowlist - the directory must resolve within one of {@code + * policies.allowedFolderRoots}; with none configured, all folder access is refused. + *
+ * + *

Paths are compared after normalisation, so {@code ..} segments cannot walk out of an allowed + * root. (Symlink escape is not defended here; an operator who configures an allowed root containing + * a symlink to a sensitive location is trusted.) + */ +@Component +public class FolderAccessGuard { + + public static final String FOLDER_TYPE = "folder"; + + private final boolean saasActive; + private final List allowedRoots; + private final List protectedRoots; + + public FolderAccessGuard(ApplicationProperties applicationProperties, Environment environment) { + this.saasActive = Arrays.asList(environment.getActiveProfiles()).contains("saas"); + this.allowedRoots = + normalizeAll(applicationProperties.getPolicies().getAllowedFolderRoots()); + this.protectedRoots = List.of(normalize(Path.of(InstallationPathConfig.getConfigPath()))); + } + + /** + * Check that {@code dir} is a permitted folder location, returning its normalised absolute + * form. + * + * @throws IllegalArgumentException if folder access is disabled (SaaS or no roots configured), + * the path is inside a protected directory, or it falls outside every allowed root + */ + public Path requirePermitted(Path dir) { + if (saasActive) { + throw new IllegalArgumentException( + "folder sources and outputs are not available in SaaS mode"); + } + Path normalized = normalize(dir); + for (Path protectedRoot : protectedRoots) { + if (normalized.startsWith(protectedRoot)) { + throw new IllegalArgumentException( + "folder may not point inside a protected Stirling directory"); + } + } + if (allowedRoots.isEmpty()) { + throw new IllegalArgumentException( + "folder access is disabled; set policies.allowedFolderRoots to permit it"); + } + boolean within = allowedRoots.stream().anyMatch(normalized::startsWith); + if (!within) { + throw new IllegalArgumentException( + "folder '" + normalized + "' is outside the allowed folder roots"); + } + return normalized; + } + + /** Whether this policy reads from or writes to a folder, and so is subject to these rules. */ + public boolean usesFolderAccess(Policy policy) { + boolean readsFolder = + policy.sources().stream().anyMatch(spec -> FOLDER_TYPE.equals(spec.type())); + boolean writesFolder = + policy.output() != null && FOLDER_TYPE.equals(policy.output().type()); + return readsFolder || writesFolder; + } + + private static List normalizeAll(List roots) { + List result = new ArrayList<>(); + for (String root : roots) { + if (root != null && !root.isBlank()) { + result.add(normalize(Path.of(root))); + } + } + return result; + } + + private static Path normalize(Path path) { + return path.toAbsolutePath().normalize(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java index 0a94eef1f..df65fbd99 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -6,7 +6,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; import org.springframework.http.HttpStatus; @@ -34,11 +33,16 @@ import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.job.JobResponse; +import stirling.software.common.service.UserServiceInterface; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; +import stirling.software.proprietary.policy.config.FolderAccessGuard; import stirling.software.proprietary.policy.engine.PolicyRunHandle; import stirling.software.proprietary.policy.engine.PolicyRunRegistry; +import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.engine.PolicyValidator; import stirling.software.proprietary.policy.model.PipelineDefinition; import stirling.software.proprietary.policy.model.Policy; import stirling.software.proprietary.policy.model.PolicyInputs; @@ -47,7 +51,6 @@ import stirling.software.proprietary.policy.model.PolicyRunStatus; import stirling.software.proprietary.policy.model.PolicyRunView; import stirling.software.proprietary.policy.progress.PolicyProgressListener; import stirling.software.proprietary.policy.store.PolicyStore; -import stirling.software.proprietary.policy.trigger.ManualTrigger; import stirling.software.proprietary.security.config.PremiumEndpoint; import tools.jackson.core.JacksonException; @@ -71,16 +74,16 @@ import tools.jackson.databind.ObjectMapper; @Tag(name = "Policies", description = "Run tool pipelines on the backend") public class PolicyController { - private final ManualTrigger manualTrigger; + private final PolicyRunner policyRunner; private final PolicyRunRegistry runRegistry; private final PolicyStore policyStore; + private final PolicyValidator policyValidator; + private final FolderAccessGuard folderAccessGuard; + private final UserServiceInterface userService; + private final ApplicationProperties applicationProperties; private final ObjectMapper objectMapper; private final TempFileManager tempFileManager; - /** SSE emitter timeout, generous enough for long multi-step runs on large files. */ - @Value("${stirling.policies.streamTimeoutMs:1800000}") - private long streamTimeoutMs; - @PostMapping(value = "/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) @Operation( summary = "Run a tool pipeline", @@ -95,7 +98,8 @@ public class PolicyController { throws IOException { PipelineDefinition definition = parseDefinition(json); PolicyInputs inputs = collectInputs(request); - String runId = manualTrigger.fire(definition, inputs, PolicyProgressListener.NOOP).runId(); + String runId = + policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP).runId(); return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null)); } @@ -112,10 +116,11 @@ public class PolicyController { PipelineDefinition definition = parseDefinition(json); PolicyInputs inputs = collectInputs(request); - SseEmitter emitter = new SseEmitter(streamTimeoutMs); + SseEmitter emitter = + new SseEmitter(applicationProperties.getPolicies().getStreamTimeoutMs()); emitter.onError(e -> log.warn("Policy run SSE emitter error", e)); - PolicyRunHandle handle = manualTrigger.fire(definition, inputs, streamListener(emitter)); + PolicyRunHandle handle = policyRunner.runAdHoc(definition, inputs, streamListener(emitter)); // Close the stream with a terminal event once the run finishes. whenComplete runs on the // engine's worker thread after the run is done, so this never races the step events. handle.completion() @@ -155,7 +160,34 @@ public class PolicyController { "Stores a policy (trigger config + steps + output + metadata). A blank id is" + " assigned; returns the stored policy with its id.") public ResponseEntity savePolicy(@RequestBody String json) { - return ResponseEntity.ok(policyStore.save(parsePolicy(json))); + Policy policy = parsePolicy(json); + requireAuthorizedForFolderAccess(policy); + try { + policyValidator.validate(policy); + } catch (IllegalArgumentException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); + } + return ResponseEntity.ok(policyStore.save(policy)); + } + + /** + * A policy that reads from or writes to a server folder grants whoever saves it access to that + * path, so restrict it to administrators on multi-user deployments. Single-user deployments + * (login disabled, e.g. desktop) trust the local operator. The {@link FolderAccessGuard} still + * enforces SaaS-off and the path allowlist during validation regardless of who saves. + */ + private void requireAuthorizedForFolderAccess(Policy policy) { + if (!folderAccessGuard.usesFolderAccess(policy)) { + return; + } + if (!applicationProperties.getSecurity().isEnableLogin()) { + return; + } + if (!userService.isCurrentUserAdmin()) { + throw new ResponseStatusException( + HttpStatus.FORBIDDEN, + "Folder sources and outputs may only be configured by an administrator"); + } } @GetMapping @@ -199,7 +231,7 @@ public class PolicyController { new ResponseStatusException( HttpStatus.NOT_FOUND, "No policy: " + policyId)); PolicyInputs inputs = collectInputs(request); - String runId = manualTrigger.run(policy, inputs, PolicyProgressListener.NOOP).runId(); + String runId = policyRunner.runWith(policy, inputs, PolicyProgressListener.NOOP).runId(); return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null)); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java index 0b613726d..acf5a3168 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java @@ -9,13 +9,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; +import stirling.software.common.model.ApplicationProperties; import stirling.software.proprietary.policy.model.PolicyRun; /** @@ -24,9 +24,9 @@ import stirling.software.proprietary.policy.model.PolicyRun; * TaskManager}. * *

Finished runs are evicted on a fixed interval once they age past {@code - * stirling.policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a - * run's rich in-memory state does not outlive the process. Only terminal runs are evicted; active - * and paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not + * policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a run's + * rich in-memory state does not outlive the process. Only terminal runs are evicted; active and + * paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not * touched here: a run shares its runId with a {@code TaskManager} job, which owns file-lifecycle * cleanup, so eviction only frees this map's entry. */ @@ -41,8 +41,8 @@ public class PolicyRunRegistry { Executors.newSingleThreadScheduledExecutor( Thread.ofVirtual().name("policy-run-cleanup-", 0).factory()); - public PolicyRunRegistry( - @Value("${stirling.policies.runExpiryMinutes:30}") int runExpiryMinutes) { + public PolicyRunRegistry(ApplicationProperties applicationProperties) { + int runExpiryMinutes = applicationProperties.getPolicies().getRunExpiryMinutes(); this.runExpiry = Duration.ofMinutes(runExpiryMinutes); cleanupExecutor.scheduleAtFixedRate(this::evictExpiredRuns, 10, 10, TimeUnit.MINUTES); log.debug( diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java new file mode 100644 index 000000000..8685bf020 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java @@ -0,0 +1,116 @@ +package stirling.software.proprietary.policy.engine; + +import java.io.IOException; +import java.util.List; +import java.util.function.Consumer; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.PolicyRunStatus; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +/** + * Runs policies, and is the one place that knows how to turn a policy's configured {@link InputSpec + * sources} into actual runs. Triggers (schedule, and future webhook/folder-watch) decide + * when to run and call {@link #run(Policy)}; they never touch sources themselves. The + * controller uses the supplied-input and ad-hoc entry points for on-demand work. + * + *

This is the seam that keeps triggers and sources independent: a trigger depends on the runner, + * the runner depends on the {@link InputSource} beans, and a source depends on neither - it just + * yields {@link ResolvedInput units of work}, each carrying its own completion hook. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class PolicyRunner { + + private final PolicyEngine policyEngine; + private final List inputSources; + + /** + * Run a policy by pulling from every source it configures: each source yields zero or more + * units of work, and each unit becomes its own run so one failure does not affect the others. A + * policy with no sources runs once with no input files (a generator pipeline). Used by + * automatic triggers. + */ + public void run(Policy policy) { + List sources = policy.sources(); + if (sources.isEmpty()) { + startRun(policy, PolicyInputs.of(List.of()), unused -> {}); + return; + } + for (InputSpec spec : sources) { + pullAndRun(policy, spec); + } + } + + /** + * Run a stored policy on files supplied directly by the caller (e.g. a manual run with + * uploads), bypassing its configured sources. Returns the run handle so callers can stream + * progress. + */ + public PolicyRunHandle runWith( + Policy policy, PolicyInputs inputs, PolicyProgressListener listener) { + return policyEngine.runPolicy(policy, inputs, listener); + } + + /** Run an ad-hoc pipeline with no stored policy (AI/Automate one-offs). */ + public PolicyRunHandle runAdHoc( + PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) { + return policyEngine.submit(definition, inputs, listener); + } + + private void pullAndRun(Policy policy, InputSpec spec) { + InputSource source = sourceFor(spec); + if (source == null) { + log.warn( + "No input source for type '{}' (policy {}); skipping", + spec.type(), + policy.id()); + return; + } + List work; + try { + work = source.resolve(spec); + } catch (IOException | RuntimeException e) { + log.warn( + "Failed to resolve source '{}' for policy {}: {}", + spec.type(), + policy.id(), + e.getMessage()); + return; + } + for (ResolvedInput unit : work) { + startRun(policy, unit.inputs(), unit.onComplete()); + } + } + + private void startRun(Policy policy, PolicyInputs inputs, Consumer onComplete) { + log.info("Running policy {} ({})", policy.id(), policy.name()); + PolicyRunHandle handle = + policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP); + handle.completion() + .whenComplete((run, throwable) -> onComplete.accept(succeeded(run, throwable))); + } + + private static boolean succeeded(PolicyRun run, Throwable throwable) { + return throwable == null && run != null && run.getStatus() == PolicyRunStatus.COMPLETED; + } + + private InputSource sourceFor(InputSpec spec) { + return inputSources.stream() + .filter(source -> source.supports(spec)) + .findFirst() + .orElse(null); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyValidator.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyValidator.java new file mode 100644 index 000000000..2aa4d98be --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyValidator.java @@ -0,0 +1,74 @@ +package stirling.software.proprietary.policy.engine; + +import java.util.List; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; + +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.TriggerConfig; +import stirling.software.proprietary.policy.output.PolicyOutputSink; +import stirling.software.proprietary.policy.trigger.PolicyTrigger; + +/** + * Validates a policy's trigger, sources, and output configuration by delegating each facet to the + * bean that handles its type. Called when a policy is saved so a misconfigured schedule, missing + * folder directory, or unknown type fails fast instead of silently misbehaving at run time. + * + *

The trigger is optional (a {@code null} trigger is a manual-only policy and needs no + * validation); every configured source is validated. + */ +@Service +@RequiredArgsConstructor +public class PolicyValidator { + + private final List triggers; + private final List inputSources; + private final List outputSinks; + + /** + * @throws IllegalArgumentException if any facet's type is unknown or its configuration is + * invalid + */ + public void validate(Policy policy) { + if (policy.trigger() != null) { + triggerFor(policy.trigger()).validate(policy); + } + for (InputSpec source : policy.sources()) { + inputSourceFor(source).validate(source); + } + outputSinkFor(policy.output()).validate(policy.output()); + } + + private PolicyTrigger triggerFor(TriggerConfig config) { + return triggers.stream() + .filter(trigger -> trigger.type().equals(config.type())) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "unknown trigger type: " + config.type())); + } + + private InputSource inputSourceFor(InputSpec spec) { + return inputSources.stream() + .filter(source -> source.supports(spec)) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "unknown input source type: " + spec.type())); + } + + private PolicyOutputSink outputSinkFor(OutputSpec spec) { + return outputSinks.stream() + .filter(sink -> sink.supports(spec)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException("unknown output type: " + spec.type())); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java new file mode 100644 index 000000000..d67820275 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java @@ -0,0 +1,186 @@ +package stirling.software.proprietary.policy.input; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.util.FileReadinessChecker; +import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.PolicyInputs; + +/** + * Reads input files from a directory. Each ready file becomes its own unit of work (one run per + * file) so a failure on one file does not affect the others. + * + *

Two modes via the {@code mode} option: + * + *

    + *
  • {@code "consume"} (default) - claim each file by moving it into {@code + * .stirling/processing}, then route it to {@code .stirling/done} or {@code .stirling/error} + * when its run finishes. Each file is processed once; right for "process new arrivals" (and + * the basis of watched folders). + *
  • {@code "snapshot"} - read the directory's current files without moving them; every run sees + * the full set again. Right for "always regenerate from a fixed input set". + *
+ * + * Readiness is checked first (via {@link FileReadinessChecker}) so files mid-write are skipped. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class FolderInputSource implements InputSource { + + private static final String TYPE = FolderAccessGuard.FOLDER_TYPE; + // Bookkeeping lives under one hidden namespace dir so the watched folder stays tidy. + private static final String WORK_SUBDIR = ".stirling"; + private static final String PROCESSING_SUBDIR = "processing"; + private static final String DONE_SUBDIR = "done"; + private static final String ERROR_SUBDIR = "error"; + + private final FileReadinessChecker readinessChecker; + private final FolderAccessGuard accessGuard; + + @Override + public String type() { + return TYPE; + } + + @Override + public boolean supports(InputSpec spec) { + return spec != null && TYPE.equals(spec.type()); + } + + @Override + public void validate(InputSpec spec) { + accessGuard.requirePermitted(FolderConfig.from(spec.options()).directory()); + } + + @Override + public List watchTargets(InputSpec spec) { + return List.of(FolderConfig.from(spec.options()).directory()); + } + + @Override + public List resolve(InputSpec spec) throws IOException { + FolderConfig config = FolderConfig.from(spec.options()); + Path inputDir = accessGuard.requirePermitted(config.directory()); + if (!Files.isDirectory(inputDir)) { + log.debug("Folder input dir does not exist: {}", inputDir); + return List.of(); + } + + List ready = new ArrayList<>(); + try (Stream entries = Files.list(inputDir)) { + entries.filter(Files::isRegularFile) + .filter(readinessChecker::isReady) + .forEach(ready::add); + } + + List work = new ArrayList<>(); + for (Path file : ready) { + if (config.snapshot()) { + work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file))))); + } else { + Path claimed = claim(inputDir, file); + if (claimed == null) { + continue; // another sweep/process grabbed it + } + work.add( + new ResolvedInput( + PolicyInputs.of(List.of(fileResource(claimed))), + success -> route(inputDir, claimed, success))); + } + } + return work; + } + + private Path claim(Path inputDir, Path file) { + try { + Path processingDir = workDir(inputDir, PROCESSING_SUBDIR); + Files.createDirectories(processingDir); + Path claimed = uniqueTarget(processingDir, file.getFileName().toString()); + Files.move(file, claimed, StandardCopyOption.ATOMIC_MOVE); + return claimed; + } catch (IOException e) { + log.debug("Could not claim {}: {}", file, e.getMessage()); + return null; + } + } + + private void route(Path inputDir, Path claimed, boolean success) { + String subdir = success ? DONE_SUBDIR : ERROR_SUBDIR; + try { + Path destDir = workDir(inputDir, subdir); + Files.createDirectories(destDir); + Files.move( + claimed, + uniqueTarget(destDir, claimed.getFileName().toString()), + StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + log.warn( + "Could not move processed input {} to {}: {}", claimed, subdir, e.getMessage()); + } + } + + /** A bookkeeping subdirectory under the watched folder's {@code .stirling} namespace. */ + private static Path workDir(Path inputDir, String subdir) { + return inputDir.resolve(WORK_SUBDIR).resolve(subdir); + } + + private static Resource fileResource(Path path) { + String name = path.getFileName().toString(); + return new FileSystemResource(path.toFile()) { + @Override + public String getFilename() { + return name; + } + }; + } + + private static Path uniqueTarget(Path dir, String filename) { + Path candidate = dir.resolve(filename); + if (!Files.exists(candidate)) { + return candidate; + } + int dot = filename.lastIndexOf('.'); + String base = dot < 0 ? filename : filename.substring(0, dot); + String ext = dot < 0 ? "" : filename.substring(dot); + for (int n = 1; ; n++) { + Path next = dir.resolve(base + " (" + n + ")" + ext); + if (!Files.exists(next)) { + return next; + } + } + } + + /** The typed, validated form of a folder source's options: the directory and dedup mode. */ + record FolderConfig(Path directory, boolean snapshot) { + + private static final String DIRECTORY_OPTION = "directory"; + private static final String MODE_OPTION = "mode"; + private static final String MODE_SNAPSHOT = "snapshot"; + + static FolderConfig from(Map options) { + Object directory = options.get(DIRECTORY_OPTION); + if (directory == null || directory.toString().isBlank()) { + throw new IllegalArgumentException("folder input requires a 'directory' option"); + } + Object mode = options.get(MODE_OPTION); + boolean snapshot = mode != null && MODE_SNAPSHOT.equals(mode.toString()); + return new FolderConfig(Path.of(directory.toString()), snapshot); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java new file mode 100644 index 000000000..a6f116d02 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java @@ -0,0 +1,49 @@ +package stirling.software.proprietary.policy.input; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +import stirling.software.proprietary.policy.model.InputSpec; + +/** + * Resolves one of a policy's {@link InputSpec sources} into the files to run on - answering + * where a run's files come from, independent of when it runs. The counterpart of + * {@code PolicyOutputSink}: implementations are beans selected by {@link #supports(InputSpec)}, so + * a new source kind (folder, S3) is just a new bean. + * + *

Driven by the {@code PolicyRunner}, which a trigger calls when a policy is due; a source is + * passive and knows nothing about what triggered the run. A manual run may instead supply files + * directly and bypass sources entirely. + */ +public interface InputSource { + + /** Stable identifier for this source, matching {@code InputSpec.type()} (e.g. "folder"). */ + String type(); + + /** Whether this source can handle the given spec. */ + boolean supports(InputSpec spec); + + /** + * Check that an input spec is usable, throwing {@link IllegalArgumentException} if not. Called + * when a policy is saved so misconfiguration fails fast rather than at run time. + */ + default void validate(InputSpec spec) {} + + /** + * Resolve the spec into zero or more units of work, each carrying the files for one run and a + * completion hook. Returning an empty list means there is nothing to run right now. + */ + List resolve(InputSpec spec) throws IOException; + + /** + * The local filesystem directories this source draws from, if any, for triggers that want to + * react to changes there (the folder-watch trigger) rather than poll. Advisory only: it merely + * tells a trigger where to watch; resolving the spec into files is still this source's + * job via {@link #resolve}. Non-filesystem sources (S3, ...) return an empty list and so are + * simply not watchable. Default: nothing to watch. + */ + default List watchTargets(InputSpec spec) { + return List.of(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolvedInput.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolvedInput.java new file mode 100644 index 000000000..285413436 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolvedInput.java @@ -0,0 +1,23 @@ +package stirling.software.proprietary.policy.input; + +import java.util.function.Consumer; + +import stirling.software.proprietary.policy.model.PolicyInputs; + +/** + * One unit of work produced by an {@link InputSource}: the files to run plus a completion callback + * invoked with the run's success once it finishes (e.g. a folder source routes the input to {@code + * .stirling/done} or {@code .stirling/error}). A source may return several of these (e.g. one per + * file). + */ +public record ResolvedInput(PolicyInputs inputs, Consumer onComplete) { + + public ResolvedInput { + onComplete = onComplete == null ? success -> {} : onComplete; + } + + /** A unit of work with no completion side effect. */ + public static ResolvedInput of(PolicyInputs inputs) { + return new ResolvedInput(inputs, success -> {}); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/InputSpec.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/InputSpec.java new file mode 100644 index 000000000..d6816c60f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/InputSpec.java @@ -0,0 +1,26 @@ +package stirling.software.proprietary.policy.model; + +import java.util.Map; + +/** + * One source a policy's input files come from. {@code type} selects an {@code InputSource} + * ("folder", and in future "s3", ...); {@code options} carries source-specific configuration (a + * directory, a bucket, a dedup mode, ...). + * + *

A policy holds a list of these; a run pulls from every one. An empty list means the policy + * runs with no input files (a generator pipeline, or files supplied directly to a manual run). + * + *

Data-driven and parallel to {@link OutputSpec}/{@link TriggerConfig}: a new source kind is a + * new {@code type} handled by a new {@code InputSource} bean. + */ +public record InputSpec(String type, Map options) { + + public InputSpec { + options = options == null ? Map.of() : options; + } + + /** Read input files from a directory on disk. */ + public static InputSpec folder(String directory) { + return new InputSpec("folder", Map.of("directory", directory)); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java index 4759810bc..3caab8afc 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java @@ -18,4 +18,9 @@ public record OutputSpec(String type, Map options) { public static OutputSpec inline() { return new OutputSpec("inline", Map.of()); } + + /** Write outputs to a directory on disk. */ + public static OutputSpec folder(String directory) { + return new OutputSpec("folder", Map.of("directory", directory)); + } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java index 3a2acaf0c..14fd35b63 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java @@ -3,18 +3,16 @@ package stirling.software.proprietary.policy.model; import java.util.List; /** - * A stored, owned automation: how it is triggered, the ordered tool steps to run, and where its - * output goes, plus identity and metadata. + * A stored automation: an ordered chain of tool steps, the sources its input files come from, and + * an output destination for the results. * - *

This is the central object of the feature. Everything that runs a chain of tools is a use of a - * Policy: a watched folder is a Policy with a folder {@link TriggerConfig} and a folder {@link - * OutputSpec}; a scheduled job is a Policy with a schedule trigger; manual/Automate/AI runs execute - * a Policy (or an ad-hoc {@link PipelineDefinition}) on demand. The engine itself only ever - * executes the {@link PipelineDefinition} this exposes via {@link #toDefinition()} - it is - * trigger-agnostic. + *

Every policy can always be run on demand (manually). It may additionally carry one automatic + * {@link TriggerConfig} - usually a schedule - that fires it without a person asking; a {@code + * null} trigger means manual-only. A trigger decides when a run happens and a {@link + * InputSpec source} decides where its files come from; the two are independent, and a run + * pulls from every configured source. * - *

{@code enabled} gates automatic triggering (a disabled policy is not picked up by its - * trigger); it does not block an explicit manual run. + *

This is the feature's central configuration object - what a user defines and the engine runs. */ public record Policy( String id, @@ -22,15 +20,28 @@ public record Policy( String owner, boolean enabled, TriggerConfig trigger, + List sources, List steps, OutputSpec output) { public Policy { - trigger = trigger == null ? TriggerConfig.manual() : trigger; + sources = sources == null ? List.of() : List.copyOf(sources); steps = steps == null ? List.of() : steps; output = output == null ? OutputSpec.inline() : output; } + /** A policy with no configured sources (a generator, or files supplied directly to a run). */ + public Policy( + String id, + String name, + String owner, + boolean enabled, + TriggerConfig trigger, + List steps, + OutputSpec output) { + this(id, name, owner, enabled, trigger, List.of(), steps, output); + } + /** The engine-level, trigger-agnostic view of this policy's pipeline. */ public PipelineDefinition toDefinition() { return new PipelineDefinition(name, steps, output); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Schedule.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Schedule.java new file mode 100644 index 000000000..148534636 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Schedule.java @@ -0,0 +1,141 @@ +package stirling.software.proprietary.policy.model; + +import java.time.DayOfWeek; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.EnumSet; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +/** + * When a scheduled policy should fire, expressed as intent ("every day at 02:00") rather than a + * cron string. A frontend builds one of these from a friendly picker; the schedule trigger asks it + * for the next firing after a given moment. + * + *

Sealed so every cadence is an explicit, self-validating shape and adding a new one forces a + * new subtype rather than another string convention to learn. The {@code type} discriminator stays + * on the wire so the frontend can switch on it without knowing the Java hierarchy. + * + *

Wall-clock kinds ({@link Daily}, {@link Weekly}, {@link Monthly}) are evaluated in the zone of + * the {@code after} argument; {@link Every} is a fixed offset and ignores wall-clock time. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = Schedule.Every.class, name = "every"), + @JsonSubTypes.Type(value = Schedule.Daily.class, name = "daily"), + @JsonSubTypes.Type(value = Schedule.Weekly.class, name = "weekly"), + @JsonSubTypes.Type(value = Schedule.Monthly.class, name = "monthly"), +}) +@JsonIgnoreProperties(ignoreUnknown = true) +public sealed interface Schedule { + + /** The next firing strictly after {@code after}, evaluated in {@code after}'s zone. */ + ZonedDateTime nextAfter(ZonedDateTime after); + + /** The granularities a fixed-interval schedule can repeat on. */ + enum Unit { + MINUTES, + HOURS, + DAYS + } + + /** + * A fixed cadence repeating from when the policy was last seen: "every 15 minutes", "every 6 + * hours". Time of day is irrelevant. + */ + record Every(long count, Unit unit) implements Schedule { + public Every { + if (count <= 0) { + throw new IllegalArgumentException("'every' schedule needs a positive count"); + } + if (unit == null) { + throw new IllegalArgumentException("'every' schedule needs a unit"); + } + } + + @Override + public ZonedDateTime nextAfter(ZonedDateTime after) { + return switch (unit) { + case MINUTES -> after.plusMinutes(count); + case HOURS -> after.plusHours(count); + case DAYS -> after.plusDays(count); + }; + } + } + + /** Once a day at a wall-clock time: "every day at 02:00". */ + record Daily(LocalTime at) implements Schedule { + public Daily { + requireTime(at); + } + + @Override + public ZonedDateTime nextAfter(ZonedDateTime after) { + ZonedDateTime today = after.with(at); + return today.isAfter(after) ? today : today.plusDays(1); + } + } + + /** On chosen weekdays at a wall-clock time: "every Monday and Thursday at 09:00". */ + record Weekly(Set days, LocalTime at) implements Schedule { + public Weekly { + if (days == null || days.isEmpty()) { + throw new IllegalArgumentException("'weekly' schedule needs at least one day"); + } + requireTime(at); + days = EnumSet.copyOf(days); + } + + @Override + public ZonedDateTime nextAfter(ZonedDateTime after) { + // The soonest of the next 7 days that lands on a chosen weekday, at the configured + // time. + for (int i = 0; i <= 7; i++) { + ZonedDateTime candidate = after.plusDays(i).with(at); + if (candidate.isAfter(after) && days.contains(candidate.getDayOfWeek())) { + return candidate; + } + } + throw new IllegalStateException("unreachable: a chosen weekday recurs within 8 days"); + } + } + + /** + * On a day of the month at a wall-clock time: "the 1st at 00:00". Months too short for the + * chosen day (e.g. the 31st in February) are skipped, not clamped. + */ + record Monthly(int dayOfMonth, LocalTime at) implements Schedule { + public Monthly { + if (dayOfMonth < 1 || dayOfMonth > 31) { + throw new IllegalArgumentException("'monthly' day-of-month must be 1-31"); + } + requireTime(at); + } + + @Override + public ZonedDateTime nextAfter(ZonedDateTime after) { + ZonedDateTime firstOfMonth = after.withDayOfMonth(1).with(at); + // Scan forward a few years' worth of months to skip ones without the chosen day. + for (int i = 0; i < 48; i++) { + ZonedDateTime month = firstOfMonth.plusMonths(i); + if (month.toLocalDate().lengthOfMonth() >= dayOfMonth) { + ZonedDateTime fire = month.withDayOfMonth(dayOfMonth); + if (fire.isAfter(after)) { + return fire; + } + } + } + throw new IllegalStateException( + "unreachable: a month with the chosen day recurs yearly"); + } + } + + private static void requireTime(LocalTime at) { + if (at == null) { + throw new IllegalArgumentException("schedule needs a time of day ('at')"); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java index 848c48e4f..e18347dc3 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java @@ -3,23 +3,21 @@ package stirling.software.proprietary.policy.model; import java.util.Map; /** - * How a {@link Policy} is automatically triggered. {@code type} selects a trigger kind ("manual", - * "folder", "schedule", "s3"); {@code options} carries type-specific configuration (a folder path, - * a cron expression, a bucket, ...). + * A {@link Policy}'s optional automatic trigger - what fires it without a person asking. {@code + * type} selects a trigger kind ("schedule", and in future "webhook", "folder-watch", ...); {@code + * options} carries type-specific configuration (a {@link Schedule}, a webhook secret, ...). + * + *

Manual running is not a trigger kind: every policy can always be run on demand, so a + * policy with no automatic trigger simply has a {@code null} {@code TriggerConfig}. A trigger + * answers only "when"; where a run's files come from is a separate concern owned by the policy's + * {@link InputSpec sources}. * *

Data-driven and parallel to {@link OutputSpec}: new trigger kinds are new {@code type} values - * handled by a new trigger bean, with no change to the model. {@code "manual"} means there is no - * automatic trigger - the policy is only ever run on demand. + * handled by a new trigger bean, with no change to the model. */ public record TriggerConfig(String type, Map options) { public TriggerConfig { - type = type == null || type.isBlank() ? "manual" : type; options = options == null ? Map.of() : options; } - - /** No automatic trigger; the policy is run on demand only. */ - public static TriggerConfig manual() { - return new TriggerConfig("manual", Map.of()); - } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java new file mode 100644 index 000000000..6ecec3c93 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java @@ -0,0 +1,130 @@ +package stirling.software.proprietary.policy.output; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import org.apache.commons.io.FilenameUtils; +import org.springframework.core.io.Resource; +import org.springframework.http.MediaType; +import org.springframework.http.MediaTypeFactory; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.job.ResultFile; +import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.model.OutputSpec; + +/** + * Writes a run's output files to a directory on disk. The destination is the {@code directory} + * option of the {@link OutputSpec}. + * + *

Files are streamed to disk (so large outputs are not buffered) and given unique names to avoid + * clobbering existing files. The returned {@link ResultFile}s describe what was written (path + + * size); they carry a synthetic id because the deliverable is the file on disk, not a {@code + * FileStorage} entry, so folder outputs are not downloadable via {@code /files/{id}}. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class FolderOutputSink implements PolicyOutputSink { + + static final String TYPE = FolderAccessGuard.FOLDER_TYPE; + static final String DIRECTORY_OPTION = "directory"; + + private final FolderAccessGuard accessGuard; + + @Override + public String type() { + return TYPE; + } + + @Override + public boolean supports(OutputSpec spec) { + return spec != null && TYPE.equals(spec.type()); + } + + @Override + public void validate(OutputSpec spec) { + accessGuard.requirePermitted(directoryOf(spec)); + } + + @Override + public List deliver(String runId, List outputs, OutputSpec spec) + throws IOException { + Path targetDir = accessGuard.requirePermitted(directoryOf(spec)); + Files.createDirectories(targetDir); + + List results = new ArrayList<>(); + for (int i = 0; i < outputs.size(); i++) { + Resource resource = outputs.get(i); + String name = safeName(resource.getFilename(), i); + Path target = uniqueTarget(targetDir, name); + try (InputStream is = resource.getInputStream()) { + Files.copy(is, target); + } + long size = Files.size(target); + String contentType = + MediaTypeFactory.getMediaType(name) + .orElse(MediaType.APPLICATION_OCTET_STREAM) + .toString(); + results.add( + ResultFile.builder() + .fileId(UUID.randomUUID().toString()) + .fileName(target.toString()) + .contentType(contentType) + .fileSize(size) + .build()); + log.debug("Wrote policy run {} output to {}", runId, target); + } + return results; + } + + private static Path directoryOf(OutputSpec spec) { + Object directory = spec.options().get(DIRECTORY_OPTION); + if (directory == null || directory.toString().isBlank()) { + throw new IllegalArgumentException( + "folder output requires a '" + DIRECTORY_OPTION + "' option"); + } + return Path.of(directory.toString()); + } + + /** + * The resource's filename reduced to a bare, traversal-free name: any directory component or + * "../" is stripped so a crafted output name cannot escape {@code targetDir}. Falls back to a + * synthetic name when the filename is absent or reduces to nothing usable. + */ + private static String safeName(String filename, int index) { + if (filename == null || filename.isBlank()) { + return "output-" + index; + } + String name = FilenameUtils.getName(filename); + if (name.isBlank() || ".".equals(name) || "..".equals(name)) { + return "output-" + index; + } + return name; + } + + /** Resolve a non-colliding path in {@code dir}, appending " (n)" before the extension. */ + private static Path uniqueTarget(Path dir, String filename) { + Path candidate = dir.resolve(filename); + if (!Files.exists(candidate)) { + return candidate; + } + String base = FilenameUtils.getBaseName(filename); + String ext = FilenameUtils.getExtension(filename); + String suffix = ext.isEmpty() ? "" : "." + ext; + for (int n = 1; ; n++) { + Path next = dir.resolve(base + " (" + n + ")" + suffix); + if (!Files.exists(next)) { + return next; + } + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java index f6bf478e7..c98b55ad6 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java @@ -23,6 +23,12 @@ public interface PolicyOutputSink { /** Whether this sink can handle the given output spec. */ boolean supports(OutputSpec spec); + /** + * Check that an output spec is usable, throwing {@link IllegalArgumentException} if not. Called + * when a policy is saved so misconfiguration fails fast rather than at run time. + */ + default void validate(OutputSpec spec) {} + /** * Persist/deliver the output files and return their descriptors. * diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java index df8fc9339..2ec94d1eb 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java @@ -30,6 +30,7 @@ public class InProcessPolicyStore implements PolicyStore { policy.owner(), policy.enabled(), policy.trigger(), + policy.sources(), policy.steps(), policy.output()); policies.put(id, stored); @@ -50,6 +51,7 @@ public class InProcessPolicyStore implements PolicyStore { public List findByTriggerType(String triggerType) { return policies.values().stream() .filter(Policy::enabled) + .filter(policy -> policy.trigger() != null) .filter(policy -> triggerType.equals(policy.trigger().type())) .toList(); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java index 977e8c9c1..e4e05ea4c 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java @@ -37,6 +37,7 @@ public class JpaPolicyStore implements PolicyStore { policy.owner(), policy.enabled(), policy.trigger(), + policy.sources(), policy.steps(), policy.output()); @@ -45,7 +46,7 @@ public class JpaPolicyStore implements PolicyStore { entity.setName(stored.name()); entity.setOwner(stored.owner()); entity.setEnabled(stored.enabled()); - entity.setTriggerType(stored.trigger().type()); + entity.setTriggerType(stored.trigger() == null ? null : stored.trigger().type()); entity.setPolicyJson(objectMapper.writeValueAsString(stored)); repository.save(entity); return stored; diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java new file mode 100644 index 000000000..dbff801b3 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java @@ -0,0 +1,302 @@ +package stirling.software.proprietary.policy.trigger; + +import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; + +import java.io.IOException; +import java.nio.file.ClosedWatchServiceException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.store.PolicyStore; + +/** + * Fires policies the moment a file lands in one of their folder sources, instead of polling on a + * timer. The trigger only reads that location; turning it into files is still the source's job. + * + *

The watcher is a latency optimisation, not a source of truth, so this pairs an event watch + * with a low-frequency reconcile sweep ({@code watchReconcileSeconds}). The reconcile both + * (a) re-syncs which directories are watched as policies are created/edited/deleted and folders + * appear on disk, and (b) runs every folder-watch policy once, catching files that pre-dated the + * watch, events lost to inotify-queue overflow, and changes on filesystems that do not deliver + * events at all (NFS, many container bind mounts). Both paths just call {@link PolicyRunner#run}; + * the {@link InputSource} does the claiming, so a redundant run finds nothing to claim and is + * harmless. + * + *

Like {@link ScheduleTrigger}, watch state is per-node and in memory; this assumes a single + * node and rebuilds its registrations on restart from the {@link PolicyStore}. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class FolderWatchTrigger implements PolicyTrigger { + + private static final String TYPE = "folder-watch"; + + private final PolicyStore policyStore; + private final PolicyRunner policyRunner; + private final List inputSources; + private final ApplicationProperties applicationProperties; + + private final Map keysByDir = new ConcurrentHashMap<>(); + private final Map dirByKey = new ConcurrentHashMap<>(); + + private volatile boolean running; + + // Package-visible (not private) so tests can drive syncRegistrations() against a real service. + volatile WatchService watchService; + + private volatile ScheduledExecutorService reconciler; + + @Override + public String type() { + return TYPE; + } + + @Override + public void validate(Policy policy) { + if (watchDirsOf(policy).isEmpty()) { + throw new IllegalArgumentException( + "folder-watch trigger requires at least one watchable (folder) input source"); + } + } + + @Override + public synchronized void start() { + if (watchService != null) { + return; + } + try { + watchService = FileSystems.getDefault().newWatchService(); + } catch (IOException e) { + log.error("Could not start folder-watch trigger: {}", e.getMessage(), e); + return; + } + running = true; + Thread.ofVirtual().name("policy-folder-watch").start(this::watchLoop); + long reconcileSeconds = applicationProperties.getPolicies().getWatchReconcileSeconds(); + reconciler = + Executors.newSingleThreadScheduledExecutor( + Thread.ofVirtual().name("policy-folder-reconcile-", 0).factory()); + // First reconcile runs immediately so pre-existing files are picked up at startup. + reconciler.scheduleAtFixedRate(this::safeReconcile, 0, reconcileSeconds, TimeUnit.SECONDS); + log.info("Folder-watch trigger started (reconcile every {}s)", reconcileSeconds); + } + + @Override + public synchronized void stop() { + running = false; + if (reconciler != null) { + reconciler.shutdownNow(); + reconciler = null; + } + if (watchService != null) { + try { + watchService.close(); // wakes the watch loop with ClosedWatchServiceException + } catch (IOException e) { + log.debug("Error closing folder watch service: {}", e.getMessage()); + } + watchService = null; + } + keysByDir.clear(); + dirByKey.clear(); + } + + private void watchLoop() { + // Capture the service once: stop() may null the field, and a local avoids racing that to an + // NPE (close() still wakes take()/poll() on this same instance). + WatchService watcher = watchService; + if (watcher == null) { + return; + } + while (running) { + WatchKey first; + try { + first = watcher.take(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } catch (ClosedWatchServiceException e) { + return; + } + runForChangedDirs(drainBurst(watcher, first)); + } + } + + /** + * Collect the directories touched by {@code first} and any further events that arrive within + * the quiet period, so a burst of file-system events becomes a single set of affected + * directories. The event kinds are irrelevant: any event on a watched dir just means "go look". + */ + private Set drainBurst(WatchService watcher, WatchKey first) { + long quietPeriodMs = applicationProperties.getPolicies().getWatchQuietPeriodMs(); + Set changed = new HashSet<>(); + WatchKey key = first; + while (key != null) { + key.pollEvents(); + Path dir = dirByKey.get(key); + if (dir != null) { + changed.add(dir); + } + key.reset(); + try { + key = watcher.poll(quietPeriodMs, TimeUnit.MILLISECONDS); + } catch (ClosedWatchServiceException | InterruptedException e) { + break; + } + } + return changed; + } + + /** Run every folder-watch policy that draws from one of the changed directories. */ + void runForChangedDirs(Set changedDirs) { + if (changedDirs.isEmpty()) { + return; + } + for (Policy policy : policyStore.findByTriggerType(TYPE)) { + List dirs; + try { + dirs = watchDirsOf(policy); + } catch (RuntimeException e) { + log.warn( + "Folder-watch policy {} is misconfigured: {}", policy.id(), e.getMessage()); + continue; + } + if (dirs.stream().anyMatch(changedDirs::contains)) { + log.debug("Folder-watch policy {} ({}) saw activity", policy.id(), policy.name()); + policyRunner.run(policy); + } + } + } + + private void safeReconcile() { + try { + syncRegistrations(); + runAll(); + } catch (RuntimeException e) { + log.error("Folder-watch reconcile failed: {}", e.getMessage(), e); + } + } + + /** The reconcile safety net: run every folder-watch policy regardless of watch events. */ + void runAll() { + for (Policy policy : policyStore.findByTriggerType(TYPE)) { + try { + policyRunner.run(policy); + } catch (RuntimeException e) { + log.warn( + "Folder-watch reconcile run failed for policy {}: {}", + policy.id(), + e.getMessage()); + } + } + } + + /** + * Bring the set of watched directories in line with the current folder-watch policies: register + * newly required directories that exist on disk, and cancel ones no longer wanted. + */ + synchronized void syncRegistrations() { + if (watchService == null) { + return; + } + Set desired = desiredDirs(); + + keysByDir + .entrySet() + .removeIf( + entry -> { + if (desired.contains(entry.getKey())) { + return false; + } + entry.getValue().cancel(); + dirByKey.remove(entry.getValue()); + return true; + }); + + for (Path dir : desired) { + if (keysByDir.containsKey(dir)) { + continue; + } + try { + WatchKey key = dir.register(watchService, ENTRY_CREATE, ENTRY_MODIFY); + keysByDir.put(dir, key); + dirByKey.put(key, dir); + log.info("Watching {} for folder-watch policies", dir); + } catch (IOException | RuntimeException e) { + log.warn("Could not watch {}: {}", dir, e.getMessage()); + } + } + } + + /** The directories currently registered with the watch service. Visible for tests. */ + Set watchedDirs() { + return Set.copyOf(keysByDir.keySet()); + } + + /** Every existing directory any current folder-watch policy wants watched. */ + private Set desiredDirs() { + Set dirs = new HashSet<>(); + for (Policy policy : policyStore.findByTriggerType(TYPE)) { + try { + for (Path dir : watchDirsOf(policy)) { + if (Files.isDirectory(dir)) { + dirs.add(dir); + } + } + } catch (RuntimeException e) { + log.warn( + "Folder-watch policy {} is misconfigured: {}", policy.id(), e.getMessage()); + } + } + return dirs; + } + + /** + * The normalised, absolute directories this policy's sources expose to watch. Normalisation + * makes registration keys and event-time matching comparable regardless of how the path was + * configured. + */ + private List watchDirsOf(Policy policy) { + List dirs = new ArrayList<>(); + for (InputSpec spec : policy.sources()) { + InputSource source = sourceFor(spec); + if (source == null) { + continue; + } + for (Path dir : source.watchTargets(spec)) { + dirs.add(dir.toAbsolutePath().normalize()); + } + } + return dirs; + } + + private InputSource sourceFor(InputSpec spec) { + return inputSources.stream() + .filter(source -> source.supports(spec)) + .findFirst() + .orElse(null); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java deleted file mode 100644 index 2ae24671a..000000000 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java +++ /dev/null @@ -1,41 +0,0 @@ -package stirling.software.proprietary.policy.trigger; - -import org.springframework.stereotype.Service; - -import lombok.RequiredArgsConstructor; - -import stirling.software.proprietary.policy.engine.PolicyEngine; -import stirling.software.proprietary.policy.engine.PolicyRunHandle; -import stirling.software.proprietary.policy.model.PipelineDefinition; -import stirling.software.proprietary.policy.model.Policy; -import stirling.software.proprietary.policy.model.PolicyInputs; -import stirling.software.proprietary.policy.progress.PolicyProgressListener; - -/** - * Runs policies on demand, in response to a request (the {@code PolicyController} endpoints, an AI, - * or another automation). It is the request-driven trigger: no background lifecycle, it just - * forwards to the engine. Any policy can be run manually regardless of its configured trigger type. - */ -@Service -@RequiredArgsConstructor -public class ManualTrigger implements PolicyTrigger { - - private final PolicyEngine policyEngine; - - @Override - public String type() { - return "manual"; - } - - /** Run a stored policy immediately and return its run handle. */ - public PolicyRunHandle run( - Policy policy, PolicyInputs inputs, PolicyProgressListener listener) { - return policyEngine.runPolicy(policy, inputs, listener); - } - - /** Run an ad-hoc pipeline (no stored policy), e.g. for AI or Automate one-offs. */ - public PolicyRunHandle fire( - PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) { - return policyEngine.submit(definition, inputs, listener); - } -} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java index c63e5a1da..1e3718d7f 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java @@ -1,22 +1,36 @@ package stirling.software.proprietary.policy.trigger; +import stirling.software.proprietary.policy.model.Policy; + /** - * Activates policies of one trigger type. A trigger owns a {@link #type()} (matching {@code - * TriggerConfig.type()}); when its condition fires it runs the relevant {@code Policy} through the - * {@code PolicyEngine}. + * An automatic trigger: the thing that decides when a policy runs without a person asking. + * A trigger owns a {@link #type()} (matching {@code TriggerConfig.type()}); when its condition + * fires it hands the policy to the {@code PolicyRunner}, which pulls the policy's sources and + * starts the runs. A trigger never resolves sources itself. * - *

Background triggers (folder watcher, schedule) are driven by configuration: on {@link - * #start()} they begin watching/scheduling for the policies returned by {@code - * PolicyStore.findByTriggerType(type())}, and stop on {@link #stop()}. Request-driven triggers - * (manual) have no background lifecycle and run a policy directly in response to a call. New - * trigger kinds are new beans of this type; the engine and the {@code Policy} model do not change. + *

Triggers are background, configuration-driven beans (schedule, and in future webhook or + * folder-watch): on {@link #start()} they begin watching/scheduling for the policies returned by + * {@code PolicyStore.findByTriggerType(type())}, and stop on {@link #stop()}. New trigger kinds are + * new beans of this type; the runner and the {@code Policy} model do not change. + * + *

Manual running is not a trigger - every policy can always be run on demand via the {@code + * PolicyRunner} regardless of whether it has a trigger. */ public interface PolicyTrigger { /** Stable identifier for this trigger kind, matching {@code TriggerConfig.type()}. */ String type(); - /** Begin activating policies of this type (e.g. start a folder watcher). No-op for manual. */ + /** + * Check that this trigger is usable for the given policy, throwing {@link + * IllegalArgumentException} if not. Called when a policy is saved so misconfiguration fails + * fast rather than at fire time. Receives the whole {@link Policy} (not just its {@code + * TriggerConfig}) so a trigger whose firing depends on the policy's sources (folder-watch) can + * assert that relationship; most triggers only inspect {@code policy.trigger()}. + */ + default void validate(Policy policy) {} + + /** Begin activating policies of this type (e.g. start the schedule sweep). */ default void start() {} /** Stop activating and release any resources. */ diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManager.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManager.java new file mode 100644 index 000000000..1e87dcd4d --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManager.java @@ -0,0 +1,56 @@ +package stirling.software.proprietary.policy.trigger; + +import java.util.List; + +import org.springframework.context.SmartLifecycle; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * Starts and stops every {@link PolicyTrigger} with the application lifecycle. Background triggers + * (schedule, and future folder/S3) begin watching on {@link #start()} and release resources on + * {@link #stop()}; request-driven triggers (manual) are no-ops. + * + *

This is the single activation point for triggers - a new background trigger only has to be a + * {@link PolicyTrigger} bean. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class PolicyTriggerManager implements SmartLifecycle { + + private final List triggers; + + private volatile boolean running; + + @Override + public void start() { + for (PolicyTrigger trigger : triggers) { + try { + trigger.start(); + } catch (RuntimeException e) { + log.error("Failed to start trigger '{}': {}", trigger.type(), e.getMessage(), e); + } + } + running = true; + } + + @Override + public void stop() { + for (PolicyTrigger trigger : triggers) { + try { + trigger.stop(); + } catch (RuntimeException e) { + log.error("Failed to stop trigger '{}': {}", trigger.type(), e.getMessage(), e); + } + } + running = false; + } + + @Override + public boolean isRunning() { + return running; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java new file mode 100644 index 000000000..f138710d7 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java @@ -0,0 +1,158 @@ +package stirling.software.proprietary.policy.trigger; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.Schedule; +import stirling.software.proprietary.policy.store.PolicyStore; + +import tools.jackson.databind.ObjectMapper; + +/** + * Fires policies on a {@link Schedule}. On {@link #start()} it sweeps on a fixed interval; each + * sweep finds the enabled "schedule" policies and runs any whose next firing has come due since it + * last fired. + * + *

The trigger only decides when: once a policy is due it hands it to the {@link + * PolicyRunner}, which pulls from the policy's configured sources and starts the runs. The trigger + * knows nothing about folders, buckets, or how many runs a sweep produces. + * + *

Caveat: last-fire times are tracked in memory, so this assumes a single node and resets + * on restart; cluster-wide coordination (leader election) is a follow-up. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class ScheduleTrigger implements PolicyTrigger { + + private static final String TYPE = "schedule"; + + private final PolicyStore policyStore; + private final PolicyRunner policyRunner; + private final ObjectMapper objectMapper; + private final ApplicationProperties applicationProperties; + + private final Map lastFiredByPolicy = new ConcurrentHashMap<>(); + private volatile ScheduledExecutorService scheduler; + + @Override + public String type() { + return TYPE; + } + + @Override + public void validate(Policy policy) { + ScheduleConfig.from(objectMapper, policy.trigger().options()); + } + + @Override + public synchronized void start() { + if (scheduler != null) { + return; + } + long sweepSeconds = applicationProperties.getPolicies().getScheduleSweepSeconds(); + scheduler = + Executors.newSingleThreadScheduledExecutor( + Thread.ofVirtual().name("policy-schedule-", 0).factory()); + scheduler.scheduleAtFixedRate( + this::safeSweep, sweepSeconds, sweepSeconds, TimeUnit.SECONDS); + log.info("Schedule trigger started (sweep every {}s)", sweepSeconds); + } + + @Override + public synchronized void stop() { + if (scheduler != null) { + scheduler.shutdownNow(); + scheduler = null; + } + } + + private void safeSweep() { + try { + sweep(Instant.now()); + } catch (RuntimeException e) { + log.error("Schedule sweep failed: {}", e.getMessage(), e); + } + } + + /** Fire every scheduled policy that is due as of {@code now}. Package-visible for testing. */ + void sweep(Instant now) { + for (Policy policy : policyStore.findByTriggerType(TYPE)) { + ScheduleConfig config; + try { + config = ScheduleConfig.from(objectMapper, policy.trigger().options()); + } catch (IllegalArgumentException e) { + log.warn("Scheduled policy {} is misconfigured: {}", policy.id(), e.getMessage()); + continue; + } + + // First time we see a policy, baseline its last-fire to now so it does not fire + // immediately; subsequent sweeps fire it once its next firing has passed. + Instant last = lastFiredByPolicy.computeIfAbsent(policy.id(), id -> now); + ZonedDateTime next = config.schedule().nextAfter(last.atZone(config.zone())); + if (!next.toInstant().isAfter(now)) { + lastFiredByPolicy.put(policy.id(), now); + log.info("Scheduled policy {} ({}) is due", policy.id(), policy.name()); + policyRunner.run(policy); + } + } + } + + /** + * The typed, validated form of a schedule trigger's options: the {@link Schedule} and the zone + * its wall-clock kinds are evaluated in (default UTC). Construction fails for a missing/invalid + * schedule or zone. + */ + record ScheduleConfig(Schedule schedule, ZoneId zone) { + + private static final String SCHEDULE_OPTION = "schedule"; + private static final String ZONE_OPTION = "zone"; + + static ScheduleConfig from(ObjectMapper mapper, Map options) { + Object scheduleNode = options.get(SCHEDULE_OPTION); + if (scheduleNode == null) { + throw new IllegalArgumentException("schedule trigger requires a 'schedule'"); + } + Schedule schedule; + try { + schedule = mapper.convertValue(scheduleNode, Schedule.class); + } catch (RuntimeException e) { + throw new IllegalArgumentException("invalid schedule: " + rootMessage(e), e); + } + + ZoneId zone = ZoneOffset.UTC; + Object zoneNode = options.get(ZONE_OPTION); + if (zoneNode != null && !zoneNode.toString().isBlank()) { + try { + zone = ZoneId.of(zoneNode.toString()); + } catch (RuntimeException e) { + throw new IllegalArgumentException("invalid zone '" + zoneNode + "'"); + } + } + return new ScheduleConfig(schedule, zone); + } + + private static String rootMessage(Throwable t) { + Throwable cause = t; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + return cause.getMessage(); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java index 5683d634c..8f82a3658 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java @@ -66,6 +66,7 @@ import tools.jackson.databind.ObjectMapper; public class AiWorkflowService { private static final String DOCUMENTS_ENDPOINT = "/api/v1/documents"; + private static final String PDF_TO_MARKDOWN_ENDPOINT = "/api/v1/convert/pdf/markdown"; private final CustomPDFDocumentFactory pdfDocumentFactory; private final AiEngineClient aiEngineClient; @@ -194,6 +195,7 @@ public class AiWorkflowService { return switch (response.getOutcome()) { case NEED_CONTENT -> onNeedContent(response, filesById, request, listener); case NEED_INGEST -> onNeedIngest(response, filesById, request, listener); + case CONVERT_MARKDOWN -> onConvertMarkdown(response, filesById, listener); case TOOL_CALL -> onToolCall(response, filesById, listener); case PLAN -> onPlan(response, filesById, request, listener); case ANSWER -> onAnswer(response, filesById, request, listener); @@ -330,6 +332,77 @@ public class AiWorkflowService { return new WorkflowState.Pending(nextRequest); } + /** + * Deterministically convert each requested PDF to Markdown via the {@code + * /convert/pdf/markdown} endpoint (backed by {@code PdfMarkdownConverter}) and return the + * {@code .md} file(s) as a completed result. No AI resume — the conversion output is the final + * answer. + */ + private WorkflowState onConvertMarkdown( + AiWorkflowResponse response, + Map filesById, + ProgressListener listener) { + List filesToConvert = response.getFilesToIngest(); + if (filesToConvert == null || filesToConvert.isEmpty()) { + return new WorkflowState.Terminal( + cannotContinue( + "AI engine requested markdown conversion without listing any files.")); + } + + try { + List resultFiles = new ArrayList<>(); + List inputNames = new ArrayList<>(); + for (int i = 0; i < filesToConvert.size(); i++) { + AiFile file = filesToConvert.get(i); + MultipartFile multipartFile = filesById.get(file.getId()); + if (multipartFile == null) { + return new WorkflowState.Terminal( + cannotContinue( + "AI engine requested markdown conversion for unknown file: " + + file.getName())); + } + listener.onProgress( + AiWorkflowProgressEvent.executingTool( + PDF_TO_MARKDOWN_ENDPOINT, i + 1, filesToConvert.size())); + Resource input = toResource(multipartFile); + PipelineDefinition definition = + new PipelineDefinition( + "convert-markdown", + List.of(new PipelineStep(PDF_TO_MARKDOWN_ENDPOINT, Map.of())), + null); + PolicyExecutionResult result = + policyExecutor.execute( + definition, + PolicyInputs.of(List.of(input)), + PolicyProgressListener.NOOP); + resultFiles.addAll(result.files()); + inputNames.add(multipartFile.getOriginalFilename()); + } + return new WorkflowState.Terminal( + buildCompletedResponse(null, resultFiles, inputNames, null)); + } catch (InternalApiTimeoutException e) { + log.error("PDF to Markdown conversion timed out: {}", e.getMessage()); + return new WorkflowState.Terminal( + cannotContinue(toolTimeoutMessage(PDF_TO_MARKDOWN_ENDPOINT, e))); + } catch (Exception e) { + log.error("Failed to convert PDF to Markdown: {}", e.getMessage(), e); + return new WorkflowState.Terminal( + cannotContinue(toolFailureMessage(PDF_TO_MARKDOWN_ENDPOINT, e))); + } + } + + private Resource toResource(MultipartFile file) throws IOException { + TempFile tempFile = tempFileManager.createManagedTempFile("ai-workflow"); + file.transferTo(tempFile.getPath()); + final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename()); + return new FileSystemResource(tempFile.getFile()) { + @Override + public String getFilename() { + return originalName; + } + }; + } + private void ingestFile(AiFile file, MultipartFile multipartFile) throws IOException { List pages = new ArrayList<>(); try (PDDocument document = pdfDocumentFactory.load(multipartFile, true)) { @@ -551,16 +624,7 @@ public class AiWorkflowService { private List toResources(Map filesById) throws IOException { List resources = new ArrayList<>(); for (MultipartFile file : filesById.values()) { - TempFile tempFile = tempFileManager.createManagedTempFile("ai-workflow"); - file.transferTo(tempFile.getPath()); - final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename()); - resources.add( - new FileSystemResource(tempFile.getFile()) { - @Override - public String getFilename() { - return originalName; - } - }); + resources.add(toResource(file)); } return resources; } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java index c06007f31..9dccb91f3 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java @@ -30,11 +30,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import stirling.software.SPDF.pdf.parser.PageImageLocator; -import stirling.software.SPDF.pdf.parser.PdfIngester; -import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage; -import stirling.software.SPDF.pdf.parser.PdfModels.RawLine; import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment; -import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment; import stirling.software.SPDF.pdf.parser.TabulaTableParser; import stirling.software.common.util.ExceptionUtils; import stirling.software.common.util.PdfUtils; @@ -50,7 +46,6 @@ import stirling.software.proprietary.model.api.ai.FolioType; public class PdfContentExtractor { private final TabulaTableParser tabulaTableParser; - private final PdfIngester pdfIngester; private static final int MAX_CHARACTERS_PER_PAGE = 4_000; @@ -196,8 +191,6 @@ public class PdfContentExtractor { case PAGE_TEXT, FULL_TEXT -> Optional.ofNullable( extractText(lf, fileReq, remainingPages, remainingCharacters)); - case PAGE_LAYOUT -> - Optional.ofNullable(extractPageLayout(lf, remainingPages)); default -> { log.warn( "Content type {} not yet implemented, skipping for {}", @@ -222,35 +215,6 @@ public class PdfContentExtractor { return extracted.isEmpty() ? null : buildExtractedFileText(lf.fileName(), extracted); } - private PageLayoutFileResult extractPageLayout(LoadedFile lf, int maxPages) throws IOException { - List parsedPages = pdfIngester.parse(lf.document(), maxPages); - List pages = new ArrayList<>(); - for (ParsedPage pp : parsedPages) { - if (pp.layoutLines().isEmpty()) continue; - List lines = new ArrayList<>(); - for (RawLine rawLine : pp.layoutLines()) { - List fragments = new ArrayList<>(); - for (TextFragment tf : rawLine.fragments()) { - fragments.add( - new LayoutFragment( - tf.text(), - tf.bounds().x(), - tf.bounds().y(), - tf.bounds().width(), - tf.fontSize(), - tf.bold())); - } - lines.add(new LayoutLine(rawLine.bounds().y(), fragments)); - } - pages.add(new LayoutPage(pp.pageNumber(), lines)); - } - if (pages.isEmpty()) return null; - PageLayoutFileResult result = new PageLayoutFileResult(); - result.setFileName(lf.fileName()); - result.setPages(pages); - return result; - } - private WorkflowArtifact buildArtifact(ArtifactKind kind, List results) { return switch (kind) { case EXTRACTED_TEXT -> { @@ -258,11 +222,6 @@ public class PdfContentExtractor { artifact.setFiles(results.stream().map(ExtractedFileText.class::cast).toList()); yield artifact; } - case PAGE_LAYOUT -> { - PageLayoutArtifact artifact = new PageLayoutArtifact(); - artifact.setFiles(results.stream().map(PageLayoutFileResult.class::cast).toList()); - yield artifact; - } case TOOL_REPORT -> throw new IllegalArgumentException( "TOOL_REPORT artifacts are not produced by PdfContentExtractor"); @@ -569,7 +528,6 @@ public class PdfContentExtractor { */ enum ArtifactKind { EXTRACTED_TEXT("extracted_text"), - PAGE_LAYOUT("page_layout"), TOOL_REPORT("tool_report"); private final String value; @@ -633,40 +591,4 @@ public class PdfContentExtractor { this.report = report; } } - - // Serialization contract with the Python engine — see PageLayoutArtifactContractTest. - - /** One text fragment with its bounding-box geometry and font properties. */ - record LayoutFragment( - String text, float x, float y, float width, float fontSize, boolean bold) {} - - /** A visual line on the page: y-coordinate and all fragments on that line. */ - record LayoutLine(float y, List fragments) {} - - /** All layout lines for a single page. */ - record LayoutPage(int pageNumber, List lines) {} - - /** Page layout data for one file, as a PdfContentResult. */ - @Data - static final class PageLayoutFileResult implements PdfContentResult { - private String fileName; - private List pages = new ArrayList<>(); - - @Override - public ArtifactKind getArtifactKind() { - return ArtifactKind.PAGE_LAYOUT; - } - - @Override - public int pagesConsumed() { - return pages.size(); - } - } - - /** Artifact carrying full spatial page layout for all input files. */ - @Data - static final class PageLayoutArtifact implements WorkflowArtifact { - private final ArtifactKind kind = ArtifactKind.PAGE_LAYOUT; - private List files = new ArrayList<>(); - } } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/config/FolderAccessGuardTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/config/FolderAccessGuardTest.java new file mode 100644 index 000000000..ea33319a8 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/config/FolderAccessGuardTest.java @@ -0,0 +1,99 @@ +package stirling.software.proprietary.policy.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.core.env.StandardEnvironment; + +import stirling.software.common.configuration.InstallationPathConfig; +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.Policy; + +/** + * Tests for {@link FolderAccessGuard}: folder access is fail-closed, confined to the configured + * allowed roots, never reaches Stirling's own config directory, and is off entirely under SaaS. + */ +class FolderAccessGuardTest { + + @TempDir Path tempDir; + + private FolderAccessGuard guard(List allowedRoots, String... activeProfiles) { + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowedFolderRoots(allowedRoots); + StandardEnvironment environment = new StandardEnvironment(); + environment.setActiveProfiles(activeProfiles); + return new FolderAccessGuard(properties, environment); + } + + @Test + void permitsAndNormalisesADirectoryWithinAnAllowedRoot() { + FolderAccessGuard guard = guard(List.of(tempDir.toString())); + Path within = tempDir.resolve("inbox"); + + assertEquals(within.toAbsolutePath().normalize(), guard.requirePermitted(within)); + } + + @Test + void rejectsADirectoryOutsideEveryAllowedRoot() { + FolderAccessGuard guard = guard(List.of(tempDir.toString())); + assertThrows( + IllegalArgumentException.class, + () -> guard.requirePermitted(tempDir.resolveSibling("elsewhere"))); + } + + @Test + void rejectsTraversalThatWalksOutOfAnAllowedRoot() { + FolderAccessGuard guard = guard(List.of(tempDir.toString())); + assertThrows( + IllegalArgumentException.class, + () -> guard.requirePermitted(tempDir.resolve("..").resolve("escaped"))); + } + + @Test + void rejectsEverythingWhenNoRootsAreConfigured() { + FolderAccessGuard guard = guard(List.of()); + assertThrows(IllegalArgumentException.class, () -> guard.requirePermitted(tempDir)); + } + + @Test + void rejectsTheStirlingConfigDirectoryEvenWhenItWouldBeInsideAnAllowedRoot() { + Path configDir = + Path.of(InstallationPathConfig.getConfigPath()).toAbsolutePath().normalize(); + // Allow the config dir's parent, so only the protected-path rule can reject it. + FolderAccessGuard guard = guard(List.of(configDir.getParent().toString())); + + assertThrows( + IllegalArgumentException.class, + () -> guard.requirePermitted(configDir.resolve("settings.yml"))); + } + + @Test + void refusesAllFolderAccessUnderTheSaasProfile() { + FolderAccessGuard guard = guard(List.of(tempDir.toString()), "saas"); + assertThrows(IllegalArgumentException.class, () -> guard.requirePermitted(tempDir)); + } + + @Test + void usesFolderAccessDetectsFolderSourcesAndOutputs() { + FolderAccessGuard guard = guard(List.of(tempDir.toString())); + + assertTrue( + guard.usesFolderAccess( + policy(List.of(InputSpec.folder("/in")), OutputSpec.inline()))); + assertTrue(guard.usesFolderAccess(policy(List.of(), OutputSpec.folder("/out")))); + assertFalse(guard.usesFolderAccess(policy(List.of(), OutputSpec.inline()))); + } + + private static Policy policy(List sources, OutputSpec output) { + return new Policy("p1", "p", "owner", true, null, sources, List.of(), output); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java index bf6442b42..e47916dbe 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java @@ -50,7 +50,6 @@ import stirling.software.proprietary.policy.model.Policy; import stirling.software.proprietary.policy.model.PolicyInputs; import stirling.software.proprietary.policy.model.PolicyRun; import stirling.software.proprietary.policy.model.PolicyRunStatus; -import stirling.software.proprietary.policy.model.TriggerConfig; import stirling.software.proprietary.policy.output.InlineOutputSink; import stirling.software.proprietary.policy.progress.PolicyProgressListener; @@ -93,7 +92,7 @@ class PolicyEngineTest { toolMetadataService, tempFileManager, JsonMapper.builder().build()); - registry = new PolicyRunRegistry(30); + registry = new PolicyRunRegistry(new ApplicationProperties()); InlineOutputSink sink = new InlineOutputSink(fileStorage); engine = new PolicyEngine( @@ -190,7 +189,7 @@ class PolicyEngineTest { "rotate", "owner", true, - TriggerConfig.manual(), + null, List.of(new PipelineStep(ROTATE, Map.of())), OutputSpec.inline()); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java index 823d53edc..51645e71b 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java @@ -12,6 +12,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import stirling.software.common.model.ApplicationProperties; import stirling.software.proprietary.policy.model.PipelineDefinition; import stirling.software.proprietary.policy.model.PolicyRun; import stirling.software.proprietary.policy.model.WaitState; @@ -25,7 +26,7 @@ class PolicyRunRegistryTest { @BeforeEach void setUp() { - registry = new PolicyRunRegistry(30); + registry = new PolicyRunRegistry(new ApplicationProperties()); } @AfterEach diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java new file mode 100644 index 000000000..2cefb1b27 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java @@ -0,0 +1,159 @@ +package stirling.software.proprietary.policy.engine; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.PolicyRunStatus; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +/** + * Tests for {@link PolicyRunner}: the one place that turns a policy's sources into runs. Verifies + * it pulls every source, runs one job per unit of work, feeds each unit's completion hook the run + * outcome, and that a generator (no sources) still runs once. + */ +@ExtendWith(MockitoExtension.class) +class PolicyRunnerTest { + + @Mock private PolicyEngine policyEngine; + @Mock private InputSource folderSource; + + private PolicyRunner runner; + + @BeforeEach + void setUp() { + runner = new PolicyRunner(policyEngine, List.of(folderSource)); + } + + @Test + void runsOnceWithNoFilesWhenThePolicyHasNoSources() { + Policy policy = policy(List.of()); + when(policyEngine.runPolicy(eq(policy), any(), any())) + .thenReturn(new PolicyRunHandle("r", new CompletableFuture<>())); + + runner.run(policy); + + ArgumentCaptor inputs = ArgumentCaptor.forClass(PolicyInputs.class); + verify(policyEngine).runPolicy(eq(policy), inputs.capture(), any()); + assertTrue(inputs.getValue().primary().isEmpty()); + } + + @Test + void pullsEverySourceAndRunsOnePerUnitOfWork() throws Exception { + InputSpec spec = InputSpec.folder("/in"); + Policy policy = policy(List.of(spec)); + when(folderSource.supports(spec)).thenReturn(true); + when(folderSource.resolve(spec)) + .thenReturn( + List.of( + ResolvedInput.of(PolicyInputs.of(List.of())), + ResolvedInput.of(PolicyInputs.of(List.of())))); + when(policyEngine.runPolicy(any(), any(), any())) + .thenReturn(new PolicyRunHandle("r", new CompletableFuture<>())); + + runner.run(policy); + + verify(policyEngine, times(2)).runPolicy(eq(policy), any(), any()); + } + + @Test + void feedsEachUnitsCompletionHookTheRunOutcome() throws Exception { + InputSpec spec = InputSpec.folder("/in"); + Policy policy = policy(List.of(spec)); + AtomicBoolean outcome = new AtomicBoolean(false); + ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set); + when(folderSource.supports(spec)).thenReturn(true); + when(folderSource.resolve(spec)).thenReturn(List.of(unit)); + CompletableFuture completion = new CompletableFuture<>(); + when(policyEngine.runPolicy(any(), any(), any())) + .thenReturn(new PolicyRunHandle("r", completion)); + + runner.run(policy); + + PolicyRun run = mock(PolicyRun.class); + when(run.getStatus()).thenReturn(PolicyRunStatus.COMPLETED); + completion.complete(run); + + assertTrue(outcome.get()); + } + + @Test + void reportsFailureToTheCompletionHookWhenTheRunDoesNotComplete() throws Exception { + InputSpec spec = InputSpec.folder("/in"); + Policy policy = policy(List.of(spec)); + AtomicBoolean outcome = new AtomicBoolean(true); + ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set); + when(folderSource.supports(spec)).thenReturn(true); + when(folderSource.resolve(spec)).thenReturn(List.of(unit)); + CompletableFuture completion = new CompletableFuture<>(); + when(policyEngine.runPolicy(any(), any(), any())) + .thenReturn(new PolicyRunHandle("r", completion)); + + runner.run(policy); + completion.completeExceptionally(new RuntimeException("boom")); + + assertFalse(outcome.get()); + } + + @Test + void skipsSourcesWithNoMatchingBean() { + InputSpec spec = new InputSpec("s3", Map.of()); + Policy policy = policy(List.of(spec)); + when(folderSource.supports(spec)).thenReturn(false); + + runner.run(policy); + + verifyNoInteractions(policyEngine); + } + + @Test + void runWithSuppliedInputsBypassesSources() { + Policy policy = policy(List.of(InputSpec.folder("/in"))); + PolicyInputs inputs = PolicyInputs.of(List.of()); + PolicyRunHandle handle = new PolicyRunHandle("r", new CompletableFuture<>()); + when(policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP)) + .thenReturn(handle); + + assertSame(handle, runner.runWith(policy, inputs, PolicyProgressListener.NOOP)); + verifyNoInteractions(folderSource); + } + + private static Policy policy(List sources) { + return new Policy( + "p1", + "p", + "owner", + true, + null, + sources, + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyValidatorTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyValidatorTest.java new file mode 100644 index 000000000..edaa58651 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyValidatorTest.java @@ -0,0 +1,114 @@ +package stirling.software.proprietary.policy.engine; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.TriggerConfig; +import stirling.software.proprietary.policy.output.PolicyOutputSink; +import stirling.software.proprietary.policy.trigger.PolicyTrigger; + +/** Tests for {@link PolicyValidator}: routes each facet to its handler and surfaces failures. */ +@ExtendWith(MockitoExtension.class) +class PolicyValidatorTest { + + @Mock private PolicyTrigger trigger; + @Mock private InputSource inputSource; + @Mock private PolicyOutputSink outputSink; + + private PolicyValidator validator; + + @BeforeEach + void setUp() { + validator = + new PolicyValidator(List.of(trigger), List.of(inputSource), List.of(outputSink)); + } + + @Test + void delegatesEachFacetToItsHandler() { + when(trigger.type()).thenReturn("schedule"); + when(inputSource.supports(any())).thenReturn(true); + when(outputSink.supports(any())).thenReturn(true); + Policy policy = policy("schedule"); + + validator.validate(policy); + + verify(trigger).validate(policy); + verify(inputSource).validate(policy.sources().get(0)); + verify(outputSink).validate(policy.output()); + } + + @Test + void skipsTriggerValidationForAManualOnlyPolicy() { + when(inputSource.supports(any())).thenReturn(true); + when(outputSink.supports(any())).thenReturn(true); + + validator.validate(manualOnly()); + + verify(trigger, never()).validate(any()); + } + + @Test + void surfacesAnInvalidConfigFromAHandler() { + when(trigger.type()).thenReturn("schedule"); + doThrow(new IllegalArgumentException("invalid schedule")).when(trigger).validate(any()); + + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> validator.validate(policy("schedule"))); + assertTrue(ex.getMessage().contains("schedule")); + } + + @Test + void rejectsAnUnknownTriggerType() { + when(trigger.type()).thenReturn("schedule"); + + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> validator.validate(policy("mystery"))); + assertTrue(ex.getMessage().contains("unknown trigger type")); + } + + private static Policy policy(String triggerType) { + return new Policy( + "p1", + "p", + "owner", + true, + new TriggerConfig(triggerType, Map.of()), + List.of(InputSpec.folder("/in")), + List.of(), + OutputSpec.inline()); + } + + private static Policy manualOnly() { + return new Policy( + "p1", + "p", + "owner", + true, + null, + List.of(InputSpec.folder("/in")), + List.of(), + OutputSpec.inline()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java new file mode 100644 index 000000000..3e9d34ccf --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java @@ -0,0 +1,137 @@ +package stirling.software.proprietary.policy.input; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.core.env.StandardEnvironment; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.util.FileReadinessChecker; +import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.model.InputSpec; + +/** Tests for {@link FolderInputSource}: consume (claim + route) and snapshot (read-only) modes. */ +@ExtendWith(MockitoExtension.class) +class FolderInputSourceTest { + + @Mock private FileReadinessChecker readinessChecker; + + @TempDir Path tempDir; + + private FolderInputSource source; + + @BeforeEach + void setUp() { + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString())); + FolderAccessGuard guard = new FolderAccessGuard(properties, new StandardEnvironment()); + source = new FolderInputSource(readinessChecker, guard); + // Lenient: the missing-dir / nonexistent-dir cases return before any readiness check. + lenient().when(readinessChecker.isReady(any())).thenReturn(true); + } + + @Test + void consumeClaimsFilesAndRoutesToDoneOnSuccess() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Files.writeString(inputDir.resolve("doc.pdf"), "data"); + + List work = source.resolve(InputSpec.folder(inputDir.toString())); + + assertEquals(1, work.size()); + assertEquals(1, work.get(0).inputs().primary().size()); + // Claimed out of the input dir. + assertFalse(Files.exists(inputDir.resolve("doc.pdf"))); + assertTrue( + Files.exists( + inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf"))); + + work.get(0).onComplete().accept(true); + assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("done").resolve("doc.pdf"))); + assertFalse( + Files.exists( + inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf"))); + } + + @Test + void consumeRoutesToErrorOnFailure() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Files.writeString(inputDir.resolve("doc.pdf"), "data"); + + List work = source.resolve(InputSpec.folder(inputDir.toString())); + work.get(0).onComplete().accept(false); + + assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("error").resolve("doc.pdf"))); + } + + @Test + void snapshotReadsWithoutClaiming() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Files.writeString(inputDir.resolve("doc.pdf"), "data"); + + List work = + source.resolve( + new InputSpec( + "folder", + Map.of("directory", inputDir.toString(), "mode", "snapshot"))); + + assertEquals(1, work.size()); + // Not moved, and completing the run is a no-op. + assertTrue(Files.exists(inputDir.resolve("doc.pdf"))); + work.get(0).onComplete().accept(true); + assertTrue(Files.exists(inputDir.resolve("doc.pdf"))); + } + + @Test + void missingDirectoryOptionFails() { + assertThrows( + IllegalArgumentException.class, + () -> source.resolve(new InputSpec("folder", Map.of()))); + } + + @Test + void nonexistentDirectoryYieldsNoWork() throws IOException { + List work = + source.resolve(InputSpec.folder(tempDir.resolve("nope").toString())); + assertTrue(work.isEmpty()); + } + + @Test + void validateRejectsMissingDirectory() { + assertThrows( + IllegalArgumentException.class, + () -> source.validate(new InputSpec("folder", Map.of()))); + } + + @Test + void rejectsADirectoryOutsideTheAllowedRoots() { + Path outside = tempDir.resolveSibling("not-allowed"); + assertThrows( + IllegalArgumentException.class, + () -> source.resolve(InputSpec.folder(outside.toString()))); + assertThrows( + IllegalArgumentException.class, + () -> source.validate(InputSpec.folder(outside.toString()))); + } + + @Test + void watchTargetsIsTheConfiguredDirectory() { + Path inputDir = tempDir.resolve("in"); + assertEquals(List.of(inputDir), source.watchTargets(InputSpec.folder(inputDir.toString()))); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java new file mode 100644 index 000000000..0d714162d --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java @@ -0,0 +1,105 @@ +package stirling.software.proprietary.policy.output; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.model.job.ResultFile; +import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.model.OutputSpec; + +/** Tests for {@link FolderOutputSink}: outputs are written to the configured directory on disk. */ +class FolderOutputSinkTest { + + @TempDir Path tempDir; + + private FolderOutputSink sink; + + @BeforeEach + void setUp() { + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString())); + sink = new FolderOutputSink(new FolderAccessGuard(properties, new StandardEnvironment())); + } + + @Test + void writesEachOutputToTheDirectory() throws IOException { + Path out = tempDir.resolve("out"); + List outputs = List.of(named("a.pdf", "aaa"), named("b.pdf", "bb")); + + List results = + sink.deliver("run-1", outputs, OutputSpec.folder(out.toString())); + + assertEquals(2, results.size()); + assertTrue(Files.exists(out.resolve("a.pdf"))); + assertEquals("aaa", Files.readString(out.resolve("a.pdf"))); + assertEquals("bb", Files.readString(out.resolve("b.pdf"))); + } + + @Test + void collidingNamesGetAUniqueSuffix() throws IOException { + Path out = tempDir.resolve("out"); + List outputs = List.of(named("a.pdf", "first"), named("a.pdf", "second")); + + sink.deliver("run-1", outputs, OutputSpec.folder(out.toString())); + + assertTrue(Files.exists(out.resolve("a.pdf"))); + assertTrue(Files.exists(out.resolve("a (1).pdf"))); + } + + @Test + void missingDirectoryOptionIsRejected() { + OutputSpec noDir = new OutputSpec("folder", Map.of()); + assertThrows(IllegalArgumentException.class, () -> sink.validate(noDir)); + assertThrows( + IllegalArgumentException.class, + () -> sink.deliver("run-1", List.of(named("a.pdf", "x")), noDir)); + } + + @Test + void aDirectoryOutsideTheAllowedRootsIsRejected() { + OutputSpec outside = OutputSpec.folder(tempDir.resolveSibling("not-allowed").toString()); + assertThrows(IllegalArgumentException.class, () -> sink.validate(outside)); + assertThrows( + IllegalArgumentException.class, + () -> sink.deliver("run-1", List.of(named("a.pdf", "x")), outside)); + } + + @Test + void filenamesWithPathTraversalAreConfinedToTheDirectory() throws IOException { + Path out = tempDir.resolve("out"); + List outputs = + List.of(named("../escape.pdf", "x"), named("nested/deep.pdf", "y")); + + sink.deliver("run-1", outputs, OutputSpec.folder(out.toString())); + + // Each name is reduced to its bare form inside the target dir; nothing escapes. + assertTrue(Files.exists(out.resolve("escape.pdf"))); + assertTrue(Files.exists(out.resolve("deep.pdf"))); + assertFalse(Files.exists(tempDir.resolve("escape.pdf"))); + } + + private static ByteArrayResource named(String filename, String content) { + return new ByteArrayResource(content.getBytes()) { + @Override + public String getFilename() { + return filename; + } + }; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java index 2e9701918..60f14be20 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java @@ -28,7 +28,7 @@ class InProcessPolicyStoreTest { @Test void savedPolicyGetsAnIdAndIsRetrievable() { - Policy saved = store.save(policy(null, "compress", "manual", true)); + Policy saved = store.save(policy(null, "compress", null, true)); assertNotNull(saved.id()); assertFalse(saved.id().isBlank()); @@ -37,7 +37,7 @@ class InProcessPolicyStoreTest { @Test void savingWithAnExistingIdUpdatesInPlace() { - Policy created = store.save(policy(null, "before", "manual", true)); + Policy created = store.save(policy(null, "before", null, true)); store.save( new Policy( @@ -45,7 +45,7 @@ class InProcessPolicyStoreTest { "after", "owner", true, - TriggerConfig.manual(), + null, List.of(), OutputSpec.inline())); @@ -55,19 +55,20 @@ class InProcessPolicyStoreTest { @Test void findByTriggerTypeReturnsOnlyEnabledMatches() { - store.save(policy(null, "watch", "folder", true)); - store.save(policy(null, "watch-disabled", "folder", false)); store.save(policy(null, "nightly", "schedule", true)); + store.save(policy(null, "nightly-disabled", "schedule", false)); + store.save(policy(null, "hooked", "webhook", true)); + store.save(policy(null, "on-demand", null, true)); // manual-only: no trigger - List folder = store.findByTriggerType("folder"); + List scheduled = store.findByTriggerType("schedule"); - assertEquals(1, folder.size()); - assertEquals("watch", folder.get(0).name()); + assertEquals(1, scheduled.size()); + assertEquals("nightly", scheduled.get(0).name()); } @Test void deleteRemovesThePolicy() { - Policy saved = store.save(policy(null, "p", "manual", true)); + Policy saved = store.save(policy(null, "p", null, true)); assertTrue(store.delete(saved.id())); assertTrue(store.get(saved.id()).isEmpty()); @@ -75,12 +76,14 @@ class InProcessPolicyStoreTest { } private static Policy policy(String id, String name, String triggerType, boolean enabled) { + TriggerConfig trigger = + triggerType == null ? null : new TriggerConfig(triggerType, Map.of()); return new Policy( id, name, "owner", enabled, - new TriggerConfig(triggerType, Map.of()), + trigger, List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), OutputSpec.inline()); } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java index 2a8212e2d..2bdbc08a7 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java @@ -18,6 +18,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import stirling.software.proprietary.policy.model.InputSpec; import stirling.software.proprietary.policy.model.OutputSpec; import stirling.software.proprietary.policy.model.PipelineStep; import stirling.software.proprietary.policy.model.Policy; @@ -53,7 +54,8 @@ class JpaPolicyStoreTest { "compress incoming", "alice", true, - new TriggerConfig("folder", Map.of("path", "/in")), + new TriggerConfig("schedule", Map.of()), + List.of(InputSpec.folder("/in")), List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), OutputSpec.inline())); @@ -62,7 +64,7 @@ class JpaPolicyStoreTest { verify(repository).save(captor.capture()); PolicyEntity entity = captor.getValue(); assertEquals(saved.id(), entity.getId()); - assertEquals("folder", entity.getTriggerType()); + assertEquals("schedule", entity.getTriggerType()); assertTrue(entity.isEnabled()); // The stored JSON round-trips back to an equal policy. assertEquals(saved, objectMapper.readValue(entity.getPolicyJson(), Policy.class)); @@ -76,7 +78,7 @@ class JpaPolicyStoreTest { "rotate", "alice", true, - TriggerConfig.manual(), + null, // manual-only: no automatic trigger List.of( new PipelineStep( "/api/v1/general/rotate-pdf", Map.of("angle", 90))), @@ -94,16 +96,16 @@ class JpaPolicyStoreTest { "watch", "alice", true, - new TriggerConfig("folder", Map.of()), + new TriggerConfig("schedule", Map.of()), List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), OutputSpec.inline()); - when(repository.findByTriggerTypeAndEnabledTrue("folder")) + when(repository.findByTriggerTypeAndEnabledTrue("schedule")) .thenReturn(List.of(entityFor(policy))); - List folder = store.findByTriggerType("folder"); + List scheduled = store.findByTriggerType("schedule"); - assertEquals(1, folder.size()); - assertEquals("p1", folder.get(0).id()); + assertEquals(1, scheduled.size()); + assertEquals("p1", scheduled.get(0).id()); } @Test @@ -122,7 +124,7 @@ class JpaPolicyStoreTest { entity.setName(policy.name()); entity.setOwner(policy.owner()); entity.setEnabled(policy.enabled()); - entity.setTriggerType(policy.trigger().type()); + entity.setTriggerType(policy.trigger() == null ? null : policy.trigger().type()); entity.setPolicyJson(objectMapper.writeValueAsString(policy)); return entity; } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java new file mode 100644 index 000000000..9b0b9f1d2 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java @@ -0,0 +1,178 @@ +package stirling.software.proprietary.policy.trigger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.WatchService; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.TriggerConfig; +import stirling.software.proprietary.policy.store.PolicyStore; + +/** + * Tests for {@link FolderWatchTrigger}'s dispatch logic via the package-visible {@code + * runForChangedDirs}/{@code runAll}, plus its cross-facet validation. The OS watch loop and + * scheduled reconcile are thin glue around these and are not exercised here (a real {@code + * WatchService} is timing-dependent), mirroring how {@link ScheduleTriggerTest} drives {@code + * sweep} directly. The folder source is stubbed to mirror {@code FolderInputSource.watchTargets}. + */ +@ExtendWith(MockitoExtension.class) +class FolderWatchTriggerTest { + + @Mock private PolicyStore policyStore; + @Mock private PolicyRunner policyRunner; + @Mock private InputSource folderSource; + + @TempDir Path tempDir; + + private FolderWatchTrigger trigger; + + @BeforeEach + void setUp() { + trigger = + new FolderWatchTrigger( + policyStore, + policyRunner, + List.of(folderSource), + new ApplicationProperties()); + lenient().when(folderSource.supports(any())).thenReturn(true); + lenient() + .when(folderSource.watchTargets(any())) + .thenAnswer( + invocation -> { + InputSpec spec = invocation.getArgument(0); + Object dir = spec.options().get("directory"); + if (dir == null) { + throw new IllegalArgumentException( + "folder input requires a 'directory' option"); + } + return List.of(Path.of(dir.toString())); + }); + } + + @Test + void validateRejectsPolicyWithNoWatchableSource() { + assertThrows( + IllegalArgumentException.class, + () -> trigger.validate(folderWatch("p1", List.of()))); + } + + @Test + void validateAcceptsPolicyWithAFolderSource() { + trigger.validate(folderWatch("p1", List.of(InputSpec.folder("/in")))); + } + + @Test + void runsOnlyPoliciesDrawingFromTheChangedDirectory() { + Policy a = folderWatch("a", List.of(InputSpec.folder("/in/a"))); + Policy b = folderWatch("b", List.of(InputSpec.folder("/in/b"))); + when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b)); + + trigger.runForChangedDirs(Set.of(normalized("/in/a"))); + + verify(policyRunner).run(a); + verify(policyRunner, never()).run(b); + } + + @Test + void skipsAMisconfiguredPolicyButStillRunsTheOthers() { + Policy bad = folderWatch("bad", List.of(new InputSpec("folder", Map.of()))); + Policy good = folderWatch("good", List.of(InputSpec.folder("/in/a"))); + when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(bad, good)); + + trigger.runForChangedDirs(Set.of(normalized("/in/a"))); + + verify(policyRunner).run(good); + verify(policyRunner, never()).run(bad); + } + + @Test + void anEmptyChangeSetDoesNothing() { + trigger.runForChangedDirs(Set.of()); + + verifyNoInteractions(policyStore, policyRunner); + } + + @Test + void reconcileRunsEveryFolderWatchPolicyAsASafetyNet() { + Policy a = folderWatch("a", List.of(InputSpec.folder("/in/a"))); + Policy b = folderWatch("b", List.of(InputSpec.folder("/in/b"))); + when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b)); + + trigger.runAll(); + + verify(policyRunner).run(a); + verify(policyRunner).run(b); + } + + @Test + void syncRegistrationsWatchesExistingDirsAndCancelsRemovedOnes() throws Exception { + Path dirA = Files.createDirectories(tempDir.resolve("a")); + Path dirB = Files.createDirectories(tempDir.resolve("b")); + Path missing = tempDir.resolve("missing"); // never created on disk + + Policy a = folderWatch("a", List.of(InputSpec.folder(dirA.toString()))); + Policy b = folderWatch("b", List.of(InputSpec.folder(dirB.toString()))); + Policy m = folderWatch("m", List.of(InputSpec.folder(missing.toString()))); + + WatchService service = FileSystems.getDefault().newWatchService(); + try { + trigger.watchService = service; + + when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b, m)); + trigger.syncRegistrations(); + // Existing dirs are watched; the non-existent one is skipped. + assertEquals( + Set.of(normalized(dirA.toString()), normalized(dirB.toString())), + trigger.watchedDirs()); + + // b's policy is removed: its registration is cancelled, a remains. + when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a)); + trigger.syncRegistrations(); + assertEquals(Set.of(normalized(dirA.toString())), trigger.watchedDirs()); + } finally { + service.close(); + } + } + + private static Path normalized(String dir) { + return Path.of(dir).toAbsolutePath().normalize(); + } + + private static Policy folderWatch(String id, List sources) { + return new Policy( + id, + "watcher", + "owner", + true, + new TriggerConfig("folder-watch", Map.of()), + sources, + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManagerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManagerTest.java new file mode 100644 index 000000000..41504424a --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManagerTest.java @@ -0,0 +1,51 @@ +package stirling.software.proprietary.policy.trigger; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Tests for {@link PolicyTriggerManager}: starts/stops every trigger, tolerating individual + * failures. + */ +@ExtendWith(MockitoExtension.class) +class PolicyTriggerManagerTest { + + @Mock private PolicyTrigger triggerA; + @Mock private PolicyTrigger triggerB; + + @Test + void startsAndStopsAllTriggers() { + PolicyTriggerManager manager = new PolicyTriggerManager(List.of(triggerA, triggerB)); + assertFalse(manager.isRunning()); + + manager.start(); + verify(triggerA).start(); + verify(triggerB).start(); + assertTrue(manager.isRunning()); + + manager.stop(); + verify(triggerA).stop(); + verify(triggerB).stop(); + assertFalse(manager.isRunning()); + } + + @Test + void oneTriggerFailingToStartDoesNotBlockTheOthers() { + doThrow(new RuntimeException("boom")).when(triggerA).start(); + PolicyTriggerManager manager = new PolicyTriggerManager(List.of(triggerA, triggerB)); + + manager.start(); + + verify(triggerB).start(); + assertTrue(manager.isRunning()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java new file mode 100644 index 000000000..889c50ee4 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java @@ -0,0 +1,147 @@ +package stirling.software.proprietary.policy.trigger; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.DayOfWeek; +import java.time.Instant; +import java.time.LocalTime; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.Schedule; +import stirling.software.proprietary.policy.model.TriggerConfig; +import stirling.software.proprietary.policy.store.PolicyStore; + +import tools.jackson.databind.json.JsonMapper; + +/** + * Tests for {@link ScheduleTrigger}'s due-firing logic via the package-visible {@code + * sweep(Instant)}. The trigger only decides when a policy is due; pulling sources and starting runs + * is the {@link PolicyRunner}'s job, so these assert it delegates to the runner. Schedules default + * to UTC, so explicit UTC instants make these deterministic. + */ +@ExtendWith(MockitoExtension.class) +class ScheduleTriggerTest { + + @Mock private PolicyStore policyStore; + @Mock private PolicyRunner policyRunner; + + private ScheduleTrigger trigger; + + @BeforeEach + void setUp() { + trigger = + new ScheduleTrigger( + policyStore, + policyRunner, + JsonMapper.builder().build(), + new ApplicationProperties()); + } + + @Test + void firesOncePerScheduleWhenItComesDue() { + Policy policy = scheduled("p1", new Schedule.Every(1, Schedule.Unit.MINUTES)); + when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy)); + + Instant t0 = Instant.parse("2026-06-05T10:00:30Z"); + trigger.sweep(t0); // first sight: baseline, must not fire immediately + verify(policyRunner, never()).run(any()); + + trigger.sweep(t0.plusSeconds(120)); // the one-minute mark has passed + verify(policyRunner, times(1)).run(eq(policy)); + } + + @Test + void doesNotFireBeforeTheNextScheduledTime() { + Policy policy = scheduled("p1", new Schedule.Daily(LocalTime.of(3, 0))); // 03:00 UTC daily + when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy)); + + Instant t0 = Instant.parse("2026-06-05T10:00:00Z"); + trigger.sweep(t0); + trigger.sweep(t0.plusSeconds(60)); // next 03:00 is far away + + verify(policyRunner, never()).run(any()); + } + + @Test + void firesWeeklyOnAChosenDay() { + // 2026-06-05 is a Friday; the next Monday 09:00 is the soonest firing. + Policy policy = + scheduled("p1", new Schedule.Weekly(Set.of(DayOfWeek.MONDAY), LocalTime.of(9, 0))); + when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy)); + + Instant friday = Instant.parse("2026-06-05T10:00:00Z"); + trigger.sweep(friday); // baseline + trigger.sweep(Instant.parse("2026-06-08T09:00:00Z")); // Monday 09:00 + + verify(policyRunner, times(1)).run(eq(policy)); + } + + @Test + void skipsPoliciesWithAnInvalidSchedule() { + Policy policy = scheduledWithRawOptions("p1", Map.of()); // no schedule + when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy)); + + trigger.sweep(Instant.parse("2026-06-05T10:00:00Z")); + + verify(policyRunner, never()).run(any()); + } + + @Test + void validateRejectsMissingSchedule() { + assertThrows( + IllegalArgumentException.class, + () -> trigger.validate(scheduledWithRawOptions("p1", Map.of()))); + } + + @Test + void validateRejectsAnInvalidSchedule() { + Map options = + Map.of("schedule", Map.of("type", "every", "count", -5, "unit", "MINUTES")); + assertThrows( + IllegalArgumentException.class, + () -> trigger.validate(scheduledWithRawOptions("p1", options))); + } + + @Test + void validateAcceptsAValidScheduleAndZone() { + Map options = new LinkedHashMap<>(); + options.put("schedule", new Schedule.Daily(LocalTime.of(2, 0))); + options.put("zone", "Europe/London"); + trigger.validate(scheduledWithRawOptions("p1", options)); + } + + private static Policy scheduled(String id, Schedule schedule) { + return scheduledWithRawOptions(id, Map.of("schedule", schedule)); + } + + private static Policy scheduledWithRawOptions(String id, Map options) { + return new Policy( + id, + "nightly", + "owner", + true, + new TriggerConfig("schedule", options), + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java index 81d1ccdfa..9e611cf32 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java @@ -439,6 +439,35 @@ class AiWorkflowServiceTest { verify(internalApiClient, never()).post(anyString(), any()); } + @Test + void convertMarkdownRunsDeterministicConversionAndReturnsMdFile() throws IOException { + MockMultipartFile input = pdf("multi-column-test_lorem.pdf", "pdf-bytes"); + when(fileIdStrategy.idFor(any())).thenReturn("doc-1"); + stubOrchestrator( + """ + { + "outcome":"convert_markdown", + "reason":"PDF to Markdown requested.", + "filesToIngest":[{"id":"doc-1","name":"multi-column-test_lorem.pdf"}] + } + """); + when(toolMetadataService.shouldUnpackZipResponse("/api/v1/convert/pdf/markdown")) + .thenReturn(false); + stubEndpoint( + "/api/v1/convert/pdf/markdown", + pdfResource("# Title", "multi-column-test_lorem.md")); + AtomicInteger ids = stubFileStorage(); + + AiWorkflowResponse result = service.orchestrate(requestFor(input, "convert to markdown")); + + assertEquals(AiWorkflowOutcome.COMPLETED, result.getOutcome()); + assertEquals(1, result.getResultFiles().size()); + // Extension changes (pdf -> md), so the converter's response filename wins. + assertEquals("multi-column-test_lorem.md", result.getResultFiles().get(0).getFileName()); + assertEquals(1, ids.get()); + verify(internalApiClient, times(1)).post(eq("/api/v1/convert/pdf/markdown"), any()); + } + @Test void toolCallWithoutEndpointFallsBackToCannotContinue() throws IOException { MockMultipartFile input = pdf("input.pdf", "bytes"); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java deleted file mode 100644 index ae853b2e6..000000000 --- a/app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java +++ /dev/null @@ -1,66 +0,0 @@ -package stirling.software.proprietary.service; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; - -import org.junit.jupiter.api.Test; - -import stirling.software.proprietary.service.PdfContentExtractor.LayoutFragment; -import stirling.software.proprietary.service.PdfContentExtractor.LayoutLine; -import stirling.software.proprietary.service.PdfContentExtractor.LayoutPage; -import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutArtifact; -import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutFileResult; - -import tools.jackson.databind.JsonNode; -import tools.jackson.databind.json.JsonMapper; - -/** - * Contract test: verifies that {@link PageLayoutArtifact} serializes to the JSON field names that - * the Python engine expects in {@code engine/src/stirling/contracts/pdf_to_markdown.py}. - * - *

The companion Python test in {@code tests/test_pdf_to_markdown.py} deserializes the same JSON - * literal and asserts field values. If either side renames a field, one of these tests fails. - */ -class PageLayoutArtifactContractTest { - - static final String CONTRACT_JSON = - """ - {"kind":"page_layout","files":[{"fileName":"test.pdf","pages":[{"pageNumber":1,"lines":[{"y":10.0,"fragments":[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}"""; - - @Test - void pageLayoutArtifact_serialisesToExpectedJson() throws Exception { - LayoutFragment fragment = new LayoutFragment("Hello", 1.0f, 2.0f, 30.0f, 12.0f, true); - LayoutLine line = new LayoutLine(10.0f, List.of(fragment)); - LayoutPage page = new LayoutPage(1, List.of(line)); - - PageLayoutFileResult fileResult = new PageLayoutFileResult(); - fileResult.setFileName("test.pdf"); - fileResult.setPages(List.of(page)); - - PageLayoutArtifact artifact = new PageLayoutArtifact(); - artifact.setFiles(List.of(fileResult)); - - JsonNode json = new JsonMapper().valueToTree(artifact); - - assertEquals("page_layout", json.get("kind").asText()); - - JsonNode file = json.get("files").get(0); - assertEquals("test.pdf", file.get("fileName").asText()); - - JsonNode pg = file.get("pages").get(0); - assertEquals(1, pg.get("pageNumber").asInt()); - - JsonNode ln = pg.get("lines").get(0); - assertEquals(10.0, ln.get("y").asDouble(), 0.001); - - JsonNode frag = ln.get("fragments").get(0); - assertEquals("Hello", frag.get("text").asText()); - assertEquals(1.0, frag.get("x").asDouble(), 0.001); - assertEquals(2.0, frag.get("y").asDouble(), 0.001); - assertEquals(30.0, frag.get("width").asDouble(), 0.001); - assertEquals(12.0, frag.get("fontSize").asDouble(), 0.001); - assertTrue(frag.get("bold").asBoolean()); - } -} diff --git a/engine/src/stirling/agents/__init__.py b/engine/src/stirling/agents/__init__.py index cddd0275c..5410ac098 100644 --- a/engine/src/stirling/agents/__init__.py +++ b/engine/src/stirling/agents/__init__.py @@ -5,7 +5,6 @@ from .orchestrator import OrchestratorAgent from .pdf_edit import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection from .pdf_questions import PdfQuestionAgent from .pdf_review import PdfReviewAgent -from .pdf_to_markdown import PdfToMarkdownAgent from .user_spec import UserSpecAgent __all__ = [ @@ -16,6 +15,5 @@ __all__ = [ "PdfEditPlanSelection", "PdfQuestionAgent", "PdfReviewAgent", - "PdfToMarkdownAgent", "UserSpecAgent", ] diff --git a/engine/src/stirling/agents/orchestrator.py b/engine/src/stirling/agents/orchestrator.py index 4dbf0b65a..d2a0b4a19 100644 --- a/engine/src/stirling/agents/orchestrator.py +++ b/engine/src/stirling/agents/orchestrator.py @@ -11,14 +11,13 @@ from pydantic_ai.tools import RunContext from stirling.agents.pdf_edit import PdfEditAgent from stirling.agents.pdf_questions import PdfQuestionAgent from stirling.agents.pdf_review import PdfReviewAgent -from stirling.agents.pdf_to_markdown import PdfToMarkdownAgent from stirling.agents.user_spec import UserSpecAgent from stirling.contracts import ( AgentDraftWorkflowResponse, + ConvertMarkdownResponse, ExtractedTextArtifact, OrchestratorRequest, OrchestratorResponse, - PageLayoutArtifact, PdfEditResponse, PdfQuestionOrchestrateResponse, PdfReviewOrchestrateResponse, @@ -27,7 +26,6 @@ from stirling.contracts import ( format_conversation_history, format_file_names, ) -from stirling.contracts.pdf_to_markdown import PdfToMarkdownOrchestrateResponse from stirling.services import AppRuntime logger = logging.getLogger(__name__) @@ -72,9 +70,11 @@ class OrchestratorAgent: ), ), ToolOutput( - self.delegate_pdf_to_markdown, - name="delegate_pdf_to_markdown", - description=("Delegate requests to reconstruct a PDF as a Markdown document."), + self.delegate_pdf_ingest, + name="delegate_pdf_ingest", + description=( + "Delegate requests to convert a PDF to Markdown or extract its content as readable text." + ), ), ToolOutput( self.unsupported_capability, @@ -92,8 +92,8 @@ class OrchestratorAgent: "Use delegate_pdf_review when the user wants the PDF returned with review" " comments attached — anything like 'review this', 'annotate with comments'," " 'leave feedback on the PDF'. " - "Use delegate_pdf_to_markdown for any request to convert a PDF to Markdown " - "or reconstruct its content as readable text. " + "Use delegate_pdf_ingest for any request to convert a PDF to Markdown " + "or extract its content as readable text. " "Use unsupported_capability when the user asks about the assistant itself " "or when none of the other outputs fit; supply a helpful message." ), @@ -133,13 +133,12 @@ class OrchestratorAgent: return await self._run_pdf_edit(request) case SupportedCapability.AGENT_DRAFT: return await self._run_agent_draft(request) - case SupportedCapability.PDF_TO_MARKDOWN: - return await self._run_pdf_to_markdown(request) case ( SupportedCapability.ORCHESTRATE | SupportedCapability.AGENT_REVISE | SupportedCapability.AGENT_NEXT_ACTION | SupportedCapability.MATH_AUDITOR_AGENT + | SupportedCapability.PDF_TO_MARKDOWN ): raise ValueError(f"Cannot resume orchestrator with capability: {capability}") case _ as unreachable: @@ -163,11 +162,12 @@ class OrchestratorAgent: async def _run_agent_draft(self, request: OrchestratorRequest) -> AgentDraftWorkflowResponse: return await UserSpecAgent(self.runtime).orchestrate(request) - async def delegate_pdf_to_markdown(self, ctx: RunContext[OrchestratorDeps]) -> PdfToMarkdownOrchestrateResponse: - return await self._run_pdf_to_markdown(ctx.deps.request) - - async def _run_pdf_to_markdown(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse: - return await PdfToMarkdownAgent(self.runtime).orchestrate(request) + async def delegate_pdf_ingest(self, ctx: RunContext[OrchestratorDeps]) -> ConvertMarkdownResponse: + request = ctx.deps.request + return ConvertMarkdownResponse( + reason="PDF to Markdown requested — Java converts deterministically.", + files_to_ingest=request.files, + ) async def delegate_pdf_review(self, ctx: RunContext[OrchestratorDeps]) -> PdfReviewOrchestrateResponse: return await self._run_pdf_review(ctx.deps.request) @@ -204,10 +204,5 @@ class OrchestratorAgent: file_names = [f.file_name for f in artifact.files] descriptions.append(f"- extracted_text: {total_pages} pages from {file_names}") continue - if isinstance(artifact, PageLayoutArtifact): - total_pages = sum(len(f.pages) for f in artifact.files) - file_names = [f.file_name for f in artifact.files] - descriptions.append(f"- page_layout: {total_pages} pages from {file_names}") - continue descriptions.append("- unknown artifact") return "\n".join(descriptions) diff --git a/engine/src/stirling/agents/pdf_to_markdown/__init__.py b/engine/src/stirling/agents/pdf_to_markdown/__init__.py deleted file mode 100644 index d35ae05c7..000000000 --- a/engine/src/stirling/agents/pdf_to_markdown/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .agent import PdfToMarkdownAgent - -__all__ = ["PdfToMarkdownAgent"] diff --git a/engine/src/stirling/agents/pdf_to_markdown/agent.py b/engine/src/stirling/agents/pdf_to_markdown/agent.py deleted file mode 100644 index 8c0d7d8ee..000000000 --- a/engine/src/stirling/agents/pdf_to_markdown/agent.py +++ /dev/null @@ -1,435 +0,0 @@ -"""PDF to Markdown Agent. - -Converts a parsed PDF document into a single clean Markdown document, preserving -headings, paragraphs, and tables in reading order. -""" - -from __future__ import annotations - -import asyncio -import logging -import re -import time - -from pydantic import BaseModel, Field -from pydantic_ai import Agent -from pydantic_ai.output import NativeOutput - -from stirling.contracts import ( - EditCannotDoResponse, - GenerateFileResponse, - NeedContentFileRequest, - NeedContentResponse, - OrchestratorRequest, - PdfContentType, - SupportedCapability, - format_conversation_history, -) -from stirling.contracts.pdf_to_markdown import ( - PageLayout, - PageLayoutArtifact, - PdfToMarkdownCannotDoResponse, - PdfToMarkdownOrchestrateResponse, - PdfToMarkdownRequest, - PdfToMarkdownResponse, - PdfToMarkdownSuccessResponse, -) -from stirling.services import AppRuntime - -logger = logging.getLogger(__name__) - - -# Warn when output tokens are close to the typical model output limit (~8192 for most -# configurations). The actual limit is model-specific; this threshold catches likely truncation. -_OUTPUT_TOKEN_TRUNCATION_THRESHOLD = 7500 - -# Chunking limits — keep each LLM call to a manageable payload size. -# Fragment count is the primary driver of JSON payload size (each fragment carries x/y/width/ -# fontSize/bold metadata beyond its text). Page cap prevents low-text pages accumulating. -_MAX_CHUNK_FRAGMENTS = 1_000 -_MAX_CHUNK_PAGES = 10 - -# Max concurrent LLM calls — limits API rate pressure on large documents. -_MAX_PARALLEL_CHUNKS = 3 - -# ── LLM output model ──────────────────────────────────────────────────────────────────────────── - - -class _ReconstructionOutput(BaseModel): - markdown: str = Field(description="Full document reconstructed as clean Markdown.") - - -# ── Agent ──────────────────────────────────────────────────────────────────────────────────────── - - -class PdfToMarkdownAgent: - def __init__(self, runtime: AppRuntime) -> None: - self.runtime = runtime - self._sem = asyncio.Semaphore(_MAX_PARALLEL_CHUNKS) - self._reconstruct_agent = Agent( - model=runtime.smart_model, - output_type=NativeOutput(_ReconstructionOutput), - system_prompt=( - "You reconstruct PDF pages into clean Markdown from spatial fragment data.\n" - "Input: PAGE LAYOUT — per-fragment x/y/font data for structural analysis.\n\n" - "COLUMN DETECTION (for tables in page_layout):\n" - "- Look at the x-positions of fragments across 3+ consecutive lines.\n" - "- If fragments cluster at the same x-positions across multiple lines, those are table columns.\n" - "- Each distinct x-cluster is one column." - " Name them from the header row (the first line in the cluster).\n" - "- Do NOT merge values from different x-columns into one cell.\n\n" - "ROW DETECTION:\n" - "- Each unique y-coordinate (or group within 3pt) is one table row.\n" - "- Every line of layout data is its own row — do not merge rows.\n" - "- If a column has no fragment on a given y-row, that cell is empty.\n\n" - "TABLE RENDERING:\n" - "- Render as: | col1 | col2 | col3 |\n" - " | --- | --- | --- |\n" - " | val | val | val |\n" - "- One source row = one table row. Never collapse multiple rows into one.\n" - "- Preserve numeric values exactly (no rounding, no formatting changes).\n" - "- Bold cells: wrap with ** in the Markdown cell.\n" - "- CRITICAL: the separator row `| --- | --- |` appears EXACTLY ONCE per table, immediately\n" - " after the header row. NEVER put `| --- |` after a data row or between data rows.\n" - " NEVER put a blank line inside a table. All rows (header + data) must be consecutive.\n" - "- Do NOT produce a header-only table followed by a second table with the data rows.\n" - " One logical table = one markdown table block, with header, one separator, then all data.\n\n" - "GROUP HEADERS (label-only rows inside a table):\n" - "- A row is a group header when: the first column has text AND every numeric column is empty.\n" - "- Do NOT render group headers as table rows with empty cells.\n" - "- Break the table, emit the label as **bold text** on its own line," - " then start a new table for the rows that follow.\n" - "- Example labels: 'Policy functions', 'Non-current assets'.\n\n" - "TOTAL AND SUBTOTAL ROWS:\n" - "- Detect rows whose first cell contains (case-insensitive):" - " total, subtotal, surplus, balance, net, sum.\n" - "- These rows have numeric content — they are NOT group headers.\n" - "- Render the entire row in bold: | **Total income** | **1,234** | **5,678** |\n" - "- Keep total rows attached to the group they summarise.\n\n" - "MULTI-LEVEL TABLES (year or period as a row label):\n" - "- Detect when a row contains only a single label (a year like '2010' or period like 'Q1 2023')" - " with no numeric content, followed by repeated metric rows.\n" - "- Do NOT render the year as a table row.\n" - "- Normalise: add 'Year' as the first column, 'Metric' as the second," - " and repeat the year value on each metric row.\n\n" - "PROSE REGIONS:\n" - "- Lines where x-positions vary across lines (not repeating columns) are prose.\n" - "- Merge lines at the same x-level into paragraphs. Separate indented lines.\n\n" - "HEADINGS:\n" - "- A line is a heading when it is bold OR font_size ≥2pt above body.\n" - " CRITICAL EXCEPTION: a bold fragment is a TABLE HEADER CELL, not a document heading, when\n" - " the same y-row in page_layout contains other fragments at different x-positions.\n" - " Only classify a bold line as a document heading when it is the SOLE fragment on its y-row.\n" - " Example: 'Non-current assets' at y=120 with '2010'@x=350, '2009'@x=420, '2008'@x=490\n" - " → this is a table header row, NOT a heading. Render it as the first cell of the table.\n" - "- Use ## for section headings, ### for sub-headings. Use # only for the document title.\n\n" - "ORDERING:\n" - "- Process content top-to-bottom as it appears on the page.\n" - "- Interleave prose blocks and table blocks in page order.\n" - "- Do not move text that appears before a table to after it, or vice versa.\n\n" - "FIDELITY:\n" - "- Do NOT invent, summarise, or omit any content.\n" - "- Do NOT add commentary, metadata, or JSON — output Markdown only." - ), - model_settings={ - **runtime.smart_model_settings, - "temperature": 0.0, - "max_tokens": _OUTPUT_TOKEN_TRUNCATION_THRESHOLD, - }, - ) - - async def orchestrate(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse: - """Entry point for the orchestrator delegate. - - First turn: requests PAGE_LAYOUT extraction from Java via NeedContentResponse. - Resume turn: runs the LLM reconstruction and returns a write-file plan step. - """ - layout_artifact = next( - (a for a in request.artifacts if isinstance(a, PageLayoutArtifact)), - None, - ) - if layout_artifact is None: - return NeedContentResponse( - resume_with=SupportedCapability.PDF_TO_MARKDOWN, - reason="Page layout data is required to reconstruct the document.", - files=[ - NeedContentFileRequest(file=f, content_types=[PdfContentType.PAGE_LAYOUT]) for f in request.files - ], - max_pages=self.runtime.settings.max_pages, - max_characters=self.runtime.settings.max_characters, - ) - - page_layout = [page for entry in layout_artifact.files for page in entry.pages] - file_names = [f.name for f in request.files] - result = await self.handle( - PdfToMarkdownRequest( - user_message=request.user_message, - file_names=file_names, - conversation_history=request.conversation_history, - page_layout=page_layout, - ) - ) - if isinstance(result, PdfToMarkdownCannotDoResponse): - return EditCannotDoResponse(reason=result.reason) - - base = file_names[0].rsplit(".", 1)[0] if file_names else "document" - return GenerateFileResponse( - content=result.markdown, - filename=f"{base}-reconstruction.md", - summary="Reconstructed the document as a Markdown file.", - ) - - async def handle(self, request: PdfToMarkdownRequest) -> PdfToMarkdownResponse: - total_fragments = sum(len(line.fragments) for page in request.page_layout for line in page.lines) - logger.info( - "[pdf-to-markdown] received layout-pages=%d fragments=%d", - len(request.page_layout), - total_fragments, - ) - - if not request.page_layout: - logger.warning("[pdf-to-markdown] no content extracted from document; returning cannot_do") - return PdfToMarkdownCannotDoResponse( - reason=( - "No content was extracted from the document. " - "The file may be a scanned image PDF with no readable text. " - "Try running OCR on the document first." - ) - ) - - chunks = _build_page_chunks(request.page_layout) - logger.info("[pdf-to-markdown] chunks=%d (max %d in parallel)", len(chunks), _MAX_PARALLEL_CHUNKS) - - if len(chunks) == 1: - return await self._reconstruct_chunk(request, chunks[0], chunk_num=1, total_chunks=1) - - total = len(chunks) - results = await asyncio.gather( - *( - self._reconstruct_chunk(request, chunk, chunk_num=i + 1, total_chunks=total) - for i, chunk in enumerate(chunks) - ) - ) - - markdown_parts: list[str] = [] - for result in results: - if isinstance(result, PdfToMarkdownSuccessResponse) and result.markdown: - markdown_parts.append(result.markdown) - elif isinstance(result, PdfToMarkdownCannotDoResponse): - logger.warning("[pdf-to-markdown] chunk dropped: %s", result.reason) - - if not markdown_parts: - return PdfToMarkdownCannotDoResponse(reason="The document could not be reconstructed. All chunks failed.") - - logger.info("[pdf-to-markdown] assembly: %d/%d chunks produced output", len(markdown_parts), len(chunks)) - return PdfToMarkdownSuccessResponse(markdown="\n\n".join(markdown_parts)) - - async def _reconstruct_chunk( - self, - request: PdfToMarkdownRequest, - pages: list[PageLayout], - chunk_num: int, - total_chunks: int, - ) -> PdfToMarkdownResponse: - chunk_request = PdfToMarkdownRequest( - user_message=request.user_message, - file_names=request.file_names, - conversation_history=request.conversation_history, - page_layout=pages, - ) - try: - async with self._sem: - return await self._reconstruct_document(chunk_request, chunk_num, total_chunks) - except Exception as e: - logger.error("[pdf-to-markdown] chunk %d/%d failed: %s", chunk_num, total_chunks, e, exc_info=True) - return PdfToMarkdownCannotDoResponse( - reason="The document could not be reconstructed. The AI model failed to process it." - ) - - async def _reconstruct_document( - self, request: PdfToMarkdownRequest, chunk_num: int = 1, total_chunks: int = 1 - ) -> PdfToMarkdownSuccessResponse: - content = _build_reconstruction_prompt(request) - logger.info("[timing] chunk %d/%d llm-call prompt-chars=%d", chunk_num, total_chunks, len(content)) - t0 = time.monotonic() - result = await self._reconstruct_agent.run([content]) - llm_ms = int((time.monotonic() - t0) * 1000) - output: _ReconstructionOutput = result.output - usage = result.usage() - logger.info( - "[timing] chunk %d/%d llm-done ms=%d input-tokens=%s output-tokens=%s markdown-chars=%d", - chunk_num, - total_chunks, - llm_ms, - usage.input_tokens, - usage.output_tokens, - len(output.markdown), - ) - if usage.output_tokens and usage.output_tokens >= _OUTPUT_TOKEN_TRUNCATION_THRESHOLD: - logger.warning( - "[timing] chunk %d/%d output likely truncated (output-tokens=%d)", - chunk_num, - total_chunks, - usage.output_tokens, - ) - markdown = _remove_extra_separators(_fix_markdown_tables(_merge_orphaned_table_rows(output.markdown))) - return PdfToMarkdownSuccessResponse(markdown=markdown) - - -# ── Chunking ──────────────────────────────────────────────────────────────────────────────────── - - -def _build_page_chunks(pages: list[PageLayout]) -> list[list[PageLayout]]: - chunks: list[list[PageLayout]] = [] - current: list[PageLayout] = [] - current_fragments = 0 - for page in pages: - page_fragments = sum(len(line.fragments) for line in page.lines) - fragment_full = current and current_fragments + page_fragments > _MAX_CHUNK_FRAGMENTS - page_full = len(current) >= _MAX_CHUNK_PAGES - if fragment_full or page_full: - chunks.append(current) - current = [] - current_fragments = 0 - current.append(page) - current_fragments += page_fragments - if current: - chunks.append(current) - return chunks - - -# ── Prompt builders (module-level, no state) ──────────────────────────────────────────────────── - - -def _build_reconstruction_prompt(request: PdfToMarkdownRequest) -> str: - history = format_conversation_history(request.conversation_history) - file_names = ", ".join(request.file_names) if request.file_names else "Unknown files" - layout_section = _format_layout(request.page_layout) - - return ( - f"Files: {file_names}\n\n" - f"User request: {request.user_message}\n\n" - f"Conversation history:\n{history}\n\n" - "PAGE LAYOUT (structural source — x/y fragment positions):\n" - "Each line is: y=NNN | text@(x,y) fs=N text@(x,y) fs=N ...\n" - "- y=NNN is the vertical position (row). Lines close in y are the same visual row.\n" - "- x=NNN is the horizontal position (column). Consistent x across rows = a column.\n" - "- fs=N is font size. Larger = likely a heading.\n" - "- **bold** markers indicate bold text.\n\n" - f"{layout_section}" - ) - - -# ── LLM output post-processing ────────────────────────────────────────────────────────────────── - - -def _fix_markdown_tables(markdown: str) -> str: - """Remove blank lines between table rows produced by the LLM.""" - lines = markdown.split("\n") - result: list[str] = [] - i = 0 - while i < len(lines): - result.append(lines[i]) - if lines[i].strip().startswith("|"): - j = i + 1 - while j < len(lines) and lines[j].strip() == "": - j += 1 - if j < len(lines) and lines[j].strip().startswith("|"): - i = j - continue - i += 1 - return "\n".join(result) - - -_SEP_CELL = re.compile(r"^:?-+:?$") - - -def _is_sep_row(line: str) -> bool: - """Return True when a pipe row is a Markdown table separator (| --- | --- |).""" - stripped = line.strip() - if not stripped.startswith("|"): - return False - cells = [c.strip() for c in stripped.split("|") if c.strip()] - return bool(cells) and all(_SEP_CELL.match(c) for c in cells) - - -def _merge_orphaned_table_rows(markdown: str) -> str: - """Merge pipe-row blocks that lack a separator into the preceding table. - - When the LLM incorrectly breaks a table (e.g. on a false group-header), it emits - orphaned pipe rows with no header or separator. These are invalid markdown and get - merged back into the preceding table, discarding the intervening non-table content. - """ - lines = markdown.split("\n") - - segments: list[tuple[str, list[str]]] = [] - i = 0 - while i < len(lines): - if lines[i].strip().startswith("|"): - block: list[str] = [] - while i < len(lines) and lines[i].strip().startswith("|"): - block.append(lines[i]) - i += 1 - has_sep = any(_is_sep_row(row) for row in block) - segments.append(("table" if has_sep else "orphan", block)) - else: - block = [] - while i < len(lines) and not lines[i].strip().startswith("|"): - block.append(lines[i]) - i += 1 - segments.append(("prose", block)) - - result: list[tuple[str, list[str]]] = [] - last_table_idx: int | None = None - for seg_type, seg_lines in segments: - if seg_type == "orphan": - if last_table_idx is not None: - result = result[: last_table_idx + 1] - result[-1] = ("table", result[-1][1] + seg_lines) - else: - result.append((seg_type, seg_lines)) - else: - if seg_type == "table": - last_table_idx = len(result) - result.append((seg_type, seg_lines)) - - return "\n".join(line for _, seg_lines in result for line in seg_lines) - - -def _remove_extra_separators(markdown: str) -> str: - """Within each contiguous table block, keep only the first separator row.""" - lines = markdown.split("\n") - result: list[str] = [] - seen_sep = False - - for line in lines: - if not line.strip().startswith("|"): - seen_sep = False - result.append(line) - continue - if _is_sep_row(line): - if seen_sep: - continue - seen_sep = True - result.append(line) - - return "\n".join(result) - - -# ── Formatting helpers (module-level, no state) ────────────────────────────────────────────────── - - -def _format_layout(pages: list[PageLayout]) -> str: - if not pages: - return "None" - parts: list[str] = [] - for page in pages: - line_strs: list[str] = [] - for line in page.lines: - frags = " ".join( - f"{'**' if f.bold else ''}{f.text}{'**' if f.bold else ''}@({f.x:.0f},{f.y:.0f}) fs={f.font_size:.0f}" - for f in line.fragments - ) - line_strs.append(f"y={line.y:.0f} | {frags}") - parts.append(f"--- Page {page.page_number} ---\n" + "\n".join(line_strs)) - return "\n\n".join(parts) diff --git a/engine/src/stirling/contracts/__init__.py b/engine/src/stirling/contracts/__init__.py index 696749d7d..4bc4febcf 100644 --- a/engine/src/stirling/contracts/__init__.py +++ b/engine/src/stirling/contracts/__init__.py @@ -13,6 +13,7 @@ from .common import ( AiFile, ArtifactKind, ConversationMessage, + ConvertMarkdownResponse, ExtractedFileText, GenerateFileResponse, MathAuditorToolReportArtifact, @@ -96,17 +97,6 @@ from .pdf_questions import ( PdfQuestionTerminalResponse, ) from .pdf_review import PdfReviewOrchestrateResponse -from .pdf_to_markdown import ( - LayoutFragment, - LayoutLine, - PageLayout, - PageLayoutArtifact, - PageLayoutFileEntry, - PdfToMarkdownCannotDoResponse, - PdfToMarkdownOrchestrateResponse, - PdfToMarkdownRequest, - PdfToMarkdownResponse, -) from .progress import ( ProgressEvent, WholeDocCompressionRound, @@ -139,10 +129,6 @@ __all__ = [ "ConversationMessage", "DeleteDocumentResponse", "PurgeOwnerResponse", - "PdfToMarkdownCannotDoResponse", - "PdfToMarkdownOrchestrateResponse", - "PdfToMarkdownRequest", - "PdfToMarkdownResponse", "Discrepancy", "DiscrepancyKind", "EditCannotDoResponse", @@ -166,15 +152,11 @@ __all__ = [ "NeedContentFileRequest", "NeedContentResponse", "NeedIngestResponse", + "ConvertMarkdownResponse", "NextExecutionAction", "OrchestratorRequest", "OrchestratorResponse", - "LayoutFragment", - "LayoutLine", "Page", - "PageLayout", - "PageLayoutArtifact", - "PageLayoutFileEntry", "PageRange", "PageText", "PdfCommentInstruction", diff --git a/engine/src/stirling/contracts/common.py b/engine/src/stirling/contracts/common.py index 05103b1a4..b8030c58b 100644 --- a/engine/src/stirling/contracts/common.py +++ b/engine/src/stirling/contracts/common.py @@ -62,6 +62,7 @@ class WorkflowOutcome(StrEnum): CANNOT_CONTINUE = "cannot_continue" UNSUPPORTED_CAPABILITY = "unsupported_capability" GENERATE_FILE = "generate_file" + CONVERT_MARKDOWN = "convert_markdown" class ArtifactKind(StrEnum): @@ -183,6 +184,19 @@ class NeedIngestResponse(ApiModel): content_types: list[PdfContentType] = Field(default_factory=list) +class ConvertMarkdownResponse(ApiModel): + """Terminal signal: convert the listed files to Markdown deterministically. + + This is a deterministic, non-AI conversion. Java runs the PDF→Markdown converter + (``PdfMarkdownConverter``) on each file and returns the resulting ``.md`` file(s) as a + completed result. There is no resume turn — the conversion output is the final answer. + """ + + outcome: Literal[WorkflowOutcome.CONVERT_MARKDOWN] = WorkflowOutcome.CONVERT_MARKDOWN + reason: str + files_to_ingest: list[AiFile] + + class ToolOperationStep(ApiModel): kind: Literal[StepKind.TOOL] = StepKind.TOOL tool: AnyToolId diff --git a/engine/src/stirling/contracts/orchestrator.py b/engine/src/stirling/contracts/orchestrator.py index 1bf0f6eb3..8b916ccaf 100644 --- a/engine/src/stirling/contracts/orchestrator.py +++ b/engine/src/stirling/contracts/orchestrator.py @@ -11,6 +11,7 @@ from .common import ( AiFile, ArtifactKind, ConversationMessage, + ConvertMarkdownResponse, ExtractedFileText, GenerateFileResponse, NeedContentResponse, @@ -23,7 +24,6 @@ from .common import ( from .execution import NextExecutionAction from .pdf_edit import PdfEditTerminalResponse from .pdf_questions import PdfQuestionTerminalResponse -from .pdf_to_markdown import PageLayoutArtifact class ExtractedTextArtifact(ApiModel): @@ -32,7 +32,7 @@ class ExtractedTextArtifact(ApiModel): WorkflowArtifact = Annotated[ - ExtractedTextArtifact | PageLayoutArtifact | ToolReportArtifact, + ExtractedTextArtifact | ToolReportArtifact, Field(discriminator="kind"), ] @@ -61,6 +61,7 @@ type OrchestratorResponse = Annotated[ | GenerateFileResponse | NeedContentResponse | NeedIngestResponse + | ConvertMarkdownResponse | AgentDraftResponse | NextExecutionAction | UnsupportedCapabilityResponse, diff --git a/engine/src/stirling/contracts/pdf_to_markdown.py b/engine/src/stirling/contracts/pdf_to_markdown.py deleted file mode 100644 index 4d272e6e2..000000000 --- a/engine/src/stirling/contracts/pdf_to_markdown.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Contracts for the PDF to Markdown Agent. - -The agent accepts a parsed document and returns a single Markdown document that -faithfully reconstructs the PDF content — headings, paragraphs, and tables in -reading order, using page_layout as the primary source of truth for structure. - -Java extracts page layout via PdfIngester and returns it as a PageLayoutArtifact -through the orchestrator resume_with pattern. -""" - -from __future__ import annotations - -from typing import Annotated, Literal - -from pydantic import Field - -from stirling.models import ApiModel - -from .common import ArtifactKind, ConversationMessage, GenerateFileResponse, NeedContentResponse -from .pdf_edit import EditCannotDoResponse - -# ── Input: layout models (mirror Java's RawLine / TextFragment geometry) ──────────────────────── - - -class LayoutFragment(ApiModel): - """One text fragment with its bounding-box geometry and font properties.""" - - text: str - x: float - y: float - width: float - font_size: float - bold: bool - - -class LayoutLine(ApiModel): - """A visual line on the page: one y-coordinate and all fragments on that line.""" - - y: float - fragments: list[LayoutFragment] - - -class PageLayout(ApiModel): - """All layout lines for a single page, in top-to-bottom order.""" - - page_number: int - lines: list[LayoutLine] - - -# ── Artifact: page layout (produced by Java, consumed by orchestrate()) ────────────────────────── - - -class PageLayoutFileEntry(ApiModel): - """Page layout data for one file, as extracted by Java's PdfIngester.""" - - file_name: str - pages: list[PageLayout] = Field(default_factory=list) - - -class PageLayoutArtifact(ApiModel): - """Artifact carrying full spatial page layout for all input files.""" - - kind: Literal[ArtifactKind.PAGE_LAYOUT] = ArtifactKind.PAGE_LAYOUT - files: list[PageLayoutFileEntry] = Field(default_factory=list) - - -# ── Input: full request ────────────────────────────────────────────────────────────────────────── - - -class PdfToMarkdownRequest(ApiModel): - """Request sent by Java after PdfIngester has parsed the document. - - page_layout: per-fragment positional data from the original (y-sorted) line order. - Each fragment carries its x/y position, width, font size, and bold flag. - This is the primary source of truth for column detection and heading hierarchy. - """ - - user_message: str - file_names: list[str] = Field(default_factory=list) - conversation_history: list[ConversationMessage] = Field(default_factory=list) - page_layout: list[PageLayout] = Field(default_factory=list) - - -# ── Output: response variants ──────────────────────────────────────────────────────────────────── - - -class PdfToMarkdownSuccessResponse(ApiModel): - outcome: Literal["document_reconstructed"] = "document_reconstructed" - markdown: str - - -class PdfToMarkdownCannotDoResponse(ApiModel): - outcome: Literal["cannot_do"] = "cannot_do" - reason: str - - -type PdfToMarkdownResponse = Annotated[ - PdfToMarkdownSuccessResponse | PdfToMarkdownCannotDoResponse, - Field(discriminator="outcome"), -] - -type PdfToMarkdownOrchestrateResponse = Annotated[ - GenerateFileResponse | EditCannotDoResponse | NeedContentResponse, - Field(discriminator="outcome"), -] diff --git a/engine/tests/test_pdf_to_markdown.py b/engine/tests/test_pdf_to_markdown.py deleted file mode 100644 index 32870a945..000000000 --- a/engine/tests/test_pdf_to_markdown.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Tests for PDF to Markdown agent. - -Two cases: -1. Narrative-only page: request validates and routes to reconstruction. -2. Mixed text + table page: layout with table region validates correctly. -""" - -from __future__ import annotations - -from stirling.contracts.pdf_to_markdown import ( - LayoutFragment, - LayoutLine, - PageLayout, - PageLayoutArtifact, - PdfToMarkdownRequest, - PdfToMarkdownSuccessResponse, -) - - -def _frag(text: str, x: float, y: float, font_size: float = 10.0, bold: bool = False) -> LayoutFragment: - return LayoutFragment(text=text, x=x, y=y, width=float(len(text) * 6), font_size=font_size, bold=bold) - - -def _line(y: float, *frags: LayoutFragment) -> LayoutLine: - return LayoutLine(y=y, fragments=list(frags)) - - -# ── Test 1: Narrative-only reconstruction ──────────────────────────────────────────────────────── - - -# ── Contract test: Java serialization ↔ Python deserialization ────────────────────────────────── -# This JSON is also asserted field-by-field in PageLayoutArtifactContractTest.java. -# If either side renames a field, one of these tests fails. -_CONTRACT_JSON = ( - '{"kind":"page_layout","files":[{"fileName":"test.pdf","pages":' - '[{"pageNumber":1,"lines":[{"y":10.0,"fragments":' - '[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}' -) - - -def test_page_layout_artifact_deserialises_java_json() -> None: - artifact = PageLayoutArtifact.model_validate_json(_CONTRACT_JSON) - - assert artifact.kind == "page_layout" - assert artifact.files[0].file_name == "test.pdf" - page = artifact.files[0].pages[0] - assert page.page_number == 1 - line = page.lines[0] - assert line.y == 10.0 - frag = line.fragments[0] - assert frag.text == "Hello" - assert frag.x == 1.0 - assert frag.y == 2.0 - assert frag.width == 30.0 - assert frag.font_size == 12.0 - assert frag.bold is True - - -def test_narrative_reconstruction_request_validates() -> None: - """A prose-only page with no tables produces a valid PdfToMarkdownRequest.""" - layout = PageLayout( - page_number=1, - lines=[ - _line(72.0, _frag("Annual Report 2023", x=72.0, y=72.0, font_size=18.0, bold=True)), - _line(100.0, _frag("Our revenue grew significantly", x=72.0, y=100.0)), - _line(114.0, _frag("during the fiscal year ended", x=72.0, y=114.0)), - _line(128.0, _frag("December 31, 2023.", x=72.0, y=128.0)), - ], - ) - request = PdfToMarkdownRequest( - user_message="reconstruct this document", - page_layout=[layout], - ) - - assert len(request.page_layout) == 1 - assert len(request.page_layout[0].lines) == 4 - assert request.page_layout[0].lines[0].fragments[0].bold is True - assert request.page_layout[0].lines[0].fragments[0].font_size == 18.0 - - -def test_narrative_reconstruction_response_validates() -> None: - """PdfToMarkdownSuccessResponse accepts markdown and returns document_reconstructed outcome.""" - response = PdfToMarkdownSuccessResponse( - markdown="# Annual Report 2023\n\nOur revenue grew significantly during the fiscal year.", - ) - - assert response.outcome == "document_reconstructed" - assert response.markdown.startswith("#") - - -# ── Test 2: Mixed text + table reconstruction ───────────────────────────────────────────────────── - - -def test_mixed_page_layout_validates() -> None: - """A page with both prose lines and a table region produces a valid request.""" - layout = PageLayout( - page_number=1, - lines=[ - # Prose heading - _line(50.0, _frag("Projects in Development", x=72.0, y=50.0, font_size=14.0, bold=True)), - # Table header row - _line( - 80.0, - _frag("Project Name", x=72.0, y=80.0, bold=True), - _frag("Location", x=200.0, y=80.0, bold=True), - _frag("Size (MW)", x=290.0, y=80.0, bold=True), - ), - # Table data rows - _line( - 95.0, - _frag("Chaplin Wind 1", x=72.0, y=95.0), - _frag("Saskatchewan", x=200.0, y=95.0), - _frag("177", x=290.0, y=95.0), - ), - _line( - 110.0, - _frag("Amherst Island 2", x=72.0, y=110.0), - _frag("Ontario", x=200.0, y=110.0), - _frag("75", x=290.0, y=110.0), - ), - # Prose after table - _line(140.0, _frag("Notes:", x=72.0, y=140.0, bold=True)), - _line(154.0, _frag("1 PPA signed", x=85.0, y=154.0)), - ], - ) - request = PdfToMarkdownRequest( - user_message="markdown", - page_layout=[layout], - ) - - assert len(request.page_layout[0].lines) == 6 - # Header line has 3 fragments at distinct x-positions (column detection) - header_line = request.page_layout[0].lines[1] - xs = [f.x for f in header_line.fragments] - assert xs == [72.0, 200.0, 290.0] - # Data rows have matching x-positions - data_row = request.page_layout[0].lines[2] - assert [f.x for f in data_row.fragments] == [72.0, 200.0, 290.0] diff --git a/frontend/editor/public/locales/en-GB/translation.toml b/frontend/editor/public/locales/en-GB/translation.toml index 39db57af9..b3f5109ce 100644 --- a/frontend/editor/public/locales/en-GB/translation.toml +++ b/frontend/editor/public/locales/en-GB/translation.toml @@ -2700,6 +2700,13 @@ whole_doc_compression_round = "Consolidating notes..." whole_doc_read_done = "Finished reading the document..." whole_doc_read_started = "Reading the document..." whole_doc_slice_done = "Reading the document... ({{percent}}% complete)" +ranForSeconds = "Ran for {{count}} seconds" +ranForSeconds_one = "Ran for 1 second" +ranForSeconds_other = "Ran for {{count}} seconds" +ranForMinutes = "Ran for {{count}} minutes" +ranForMinutes_one = "Ran for 1 minute" +ranForMinutes_other = "Ran for {{count}} minutes" +ranForMinutesSeconds = "Ran for {{minutes}} min, {{seconds}} sec" [chat.quickActions] browseYourFiles = "Browse your files" @@ -3042,6 +3049,37 @@ impact = "Any applications or services currently using these keys will stop work title = "Refresh API Keys" warning = "⚠️ Warning: This action will generate new API keys and make your previous keys invalid." +[config.mcp] +description = "Model Context Protocol (MCP) lets AI assistants like Claude use your Stirling PDF tools directly. Connect a client once and your assistant can convert, edit, secure and process documents on your behalf." +navLabel = "MCP Server" +tip = "Every action your assistant runs is performed as your account and counts towards your usage, just like using Stirling PDF directly." +title = "MCP Server" +viewApiKeys = "View API keys" + +[config.mcp.copy] +configLabel = "Config" +copied = "Copied" +copy = "Copy" +endpointLabel = "Endpoint URL" +tooltip = "Copy {{label}}" +tooltipCopied = "{{label}} copied" + +[config.mcp.endpoint] +label = "Your MCP endpoint" + +[config.mcp.setup] +addTo = "Add to" +hint = "Pick your client, paste the snippet into the file shown, then restart it. You'll sign in with your Stirling account on first use - no keys to copy." +title = "Connect your AI assistant" + +[config.mcp.tools] +ai = "AI" +convert = "Convert" +misc = "Misc" +pages = "Pages" +security = "Security" +title = "What your assistant can do" + [config.overview] description = "Current application settings and configuration details." error = "Error" @@ -3203,7 +3241,6 @@ creditsRemaining = "{{current}} of {{total}} remaining" creditsThisMonth = "Monthly credits" current = "Current Plan" customPricing = "Custom" -customWatchedFolders = "Custom Watched Folders" dedicatedSupportSlas = "Dedicated support & SLAs" enterpriseSubscription = "Enterprise" everythingInCredits = "Everything in Credits, plus:" @@ -3511,7 +3548,6 @@ addFiles = "Add Files" [fileManager] active = "Active" -addToWatchedFolder = "Add to Watched Folder" addToUpload = "Add to Upload" changesNotUploaded = "Changes not uploaded" clearAll = "Clear All" @@ -4471,7 +4507,7 @@ alreadyHaveAccount = "Already have an account?" choosePassword = "Choose a password" confirmPassword = "Confirm password" confirmPasswordPlaceholder = "Re-enter your password" -createAccount = "Create Account" +createAccount = "Create account" creating = "Creating Account..." email = "Email address" emailPlaceholder = "Enter your email address" @@ -4554,11 +4590,10 @@ sending = "Sending…" sendMagicLink = "Send Magic Link" sendResetLink = "Send reset link" sessionExpired = "Your session has expired. Please sign in again." +createAccount = "Create an account" signin = "Sign in" -signInAnonymously = "Sign Up as a Guest" signingIn = "Signing in..." signInWith = "Sign in with" -subtitle = "Sign back in to Stirling PDF" title = "Sign in" unexpectedError = "Unexpected error: {{message}}" updatePassword = "Update password" @@ -5774,7 +5809,6 @@ label = "SMTP Username" [quickAccess] access = "Access" -watchedFolders = "Watched Folders" accessAddPerson = "Add another person" accessBack = "Back" accessCopyLink = "Copy link" @@ -6865,6 +6899,7 @@ createFailed = "Failed to create signing request" [signup] accountCreatedSuccessfully = "Account created successfully! You can now sign in." +alreadyHaveAccount = "I already have an account" checkEmailConfirmation = "Check your email for a confirmation link to complete your registration." confirmPassword = "Confirm password" confirmPasswordPlaceholder = "Confirm password" @@ -6885,10 +6920,11 @@ passwordsDoNotMatch = "Passwords do not match" passwordTooShort = "Password must be at least 6 characters long" pleaseFillAllFields = "Please fill in all fields" signUp = "Sign Up" +signUpWith = "Sign up with" +skip = "Skip" subtitle = "Join Stirling PDF to get started" title = "Create an account" unexpectedError = "Unexpected error: {{message}}" -useEmailInstead = "Use Email Instead" [sizes] large = "Large" @@ -8162,20 +8198,14 @@ newFolder = "New folder" noFolders = "No watched folders yet" sidebarTitle = "Watched Folders" title = "Watched Folders" -sidebarFiles = "My Files" [watchedFolders.modal] automation = "Automation" -automationSaved = "Automation saved" createFolder = "Create Folder" -stepsSaved = "Steps saved — click Create Folder to finish" automationRequired = "Add at least one configured step before saving." color = "Accent color" createTitle = "New Watched Folder" -description = "Description" -descriptionPlaceholder = "What does this folder do?" editTitle = "Edit Watched Folder" -icon = "Icon" name = "Folder name" namePlaceholder = "My Watched Folder" nameRequired = "Folder name is required" @@ -8183,17 +8213,13 @@ nameTooLong = "Folder name must be 50 characters or less" saveChanges = "Save Changes" saveFailed = "Failed to save folder. Please try again." automationNameFallback = "Watched Folder automation" -retryLabel = "Auto-retry" maxRetries = "Max retries" -maxRetriesDesc = "0 to disable" retryDelay = "Delay (minutes)" -outputLabel = "Output" outputModeVersion = "Replace original?" outputModeVersionDesc = "Output becomes a new version of the input file rather than a separate file" outputModeNewDesc = "Output is saved as a new separate file" outputName = "Output filename prefix" outputNameSuffix = "Suffix" -outputNamePlaceholderVersion = "Same as original" sectionFolder = "Folder" sectionSourceOutput = "Source & Output" sectionSteps = "Steps" @@ -8220,14 +8246,10 @@ positionSuffix = "Suffix" [watchedFolders.home] create = "Create your first Watched Folder" -dropHere = "Drop to process" editFolder = "Edit folder" empty = "No watched folders yet" file = "file" files = "files" -openFolder = "Open folder" -title = "Watched Folders" -subtitle = "Folders that automatically process PDFs with your configured pipeline" emptyTitle = "Automate your PDF workflows" emptyDesc = "Set up a Watched Folder once. Drop PDFs in and they're automatically compressed, OCR'd, split, merged — whatever your pipeline does." addAnother = "Add another Watched Folder" @@ -8235,7 +8257,6 @@ addAnotherDesc = "Automatically process files with a new pipeline" resume = "Resume" pause = "Pause" deleteFolder = "Delete folder" -noSteps = "No automation steps configured" [watchedFolders.card] edit = "Edit folder" @@ -8318,7 +8339,6 @@ previewLoadFailed = "Could not load file preview." back = "Back" dismiss = "Dismiss" retry = "Retry" -view = "View" download = "Download" downloadInput = "Download input" downloadOutput = "Download output" diff --git a/frontend/editor/public/modern-logo/LoginDarkModeHeader.svg b/frontend/editor/public/modern-logo/LoginDarkModeHeader.svg new file mode 100644 index 000000000..59d84449f --- /dev/null +++ b/frontend/editor/public/modern-logo/LoginDarkModeHeader.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/frontend/editor/public/modern-logo/LoginLightModeHeader.svg b/frontend/editor/public/modern-logo/LoginLightModeHeader.svg new file mode 100644 index 000000000..6b7c6fa8c --- /dev/null +++ b/frontend/editor/public/modern-logo/LoginLightModeHeader.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/frontend/editor/src/core/components/chat/ChatContext.tsx b/frontend/editor/src/core/components/chat/ChatContext.tsx index 3ef7276fb..621fafd6e 100644 --- a/frontend/editor/src/core/components/chat/ChatContext.tsx +++ b/frontend/editor/src/core/components/chat/ChatContext.tsx @@ -10,6 +10,7 @@ export function useChat() { isOpen: false, isLoading: false, progress: null, + progressLog: [] as never[], toggleOpen: () => {}, setOpen: (_open: boolean) => {}, sendMessage: async (_content: string) => {}, diff --git a/frontend/editor/src/core/tests/live/authentication-login.spec.ts b/frontend/editor/src/core/tests/live/authentication-login.spec.ts index 310dd5e6f..a8a6a75cf 100644 --- a/frontend/editor/src/core/tests/live/authentication-login.spec.ts +++ b/frontend/editor/src/core/tests/live/authentication-login.spec.ts @@ -19,12 +19,7 @@ test.describe("1. Authentication and Login", () => { .first(), ).toBeVisible(); - // Step 3: Confirm the heading for "Sign In" / "Login" is visible - await expect( - page.getByRole("heading", { name: /sign in|login|masuk/i }), - ).toBeVisible(); - - // Step 4: Confirm a "Username" text input field is present and empty + // Step 3: Confirm a "Username" text input field is present and empty const usernameInput = page.locator("#email"); await expect(usernameInput).toBeVisible(); await expect(usernameInput).toHaveValue(""); diff --git a/frontend/editor/src/proprietary/components/chat/ChatContext.tsx b/frontend/editor/src/proprietary/components/chat/ChatContext.tsx index 2e3caf33d..0c9cb314b 100644 --- a/frontend/editor/src/proprietary/components/chat/ChatContext.tsx +++ b/frontend/editor/src/proprietary/components/chat/ChatContext.tsx @@ -37,6 +37,14 @@ export interface ChatMessage { * turns that answered without running any tool. */ toolsUsed?: string[]; + /** + * Full ordered progress log captured during the AI turn that produced this message. + * Only set on assistant messages; used to render the "Ran for X seconds" collapsed + * history dropdown above the response. + */ + progressLog?: AiWorkflowProgress[]; + /** Wall-clock duration of the AI turn in milliseconds. Only set on assistant messages. */ + durationMs?: number; } export enum AiWorkflowPhase { @@ -178,12 +186,20 @@ interface ChatState { isOpen: boolean; isLoading: boolean; progress: AiWorkflowProgress | null; + /** Ordered log of every progress event in the current request. UI shows the last N entries. */ + progressLog: AiWorkflowProgress[]; } +/** + * Maximum number of progress steps retained in the live buffer. + */ +export const PROGRESS_LOG_MAX = 4; + type ChatAction = | { type: "ADD_MESSAGE"; message: ChatMessage } | { type: "SET_LOADING"; loading: boolean } | { type: "SET_PROGRESS"; progress: AiWorkflowProgress | null } + | { type: "APPEND_PROGRESS"; progress: AiWorkflowProgress } | { type: "TOGGLE_OPEN" } | { type: "SET_OPEN"; open: boolean } | { type: "CLEAR" }; @@ -193,15 +209,40 @@ function chatReducer(state: ChatState, action: ChatAction): ChatState { case "ADD_MESSAGE": return { ...state, messages: [...state.messages, action.message] }; case "SET_LOADING": - return { ...state, isLoading: action.loading }; + // Reset the log on both start (true) and end (false) of a request. + return { + ...state, + isLoading: action.loading, + progress: action.loading ? state.progress : null, + progressLog: [], + }; case "SET_PROGRESS": return { ...state, progress: action.progress }; + case "APPEND_PROGRESS": + // Cap the live buffer so each append copies at most PROGRESS_LOG_MAX elements + return { + ...state, + progress: action.progress, + progressLog: + state.progressLog.length < PROGRESS_LOG_MAX + ? [...state.progressLog, action.progress] + : [ + ...state.progressLog.slice(1 - PROGRESS_LOG_MAX), + action.progress, + ], + }; case "TOGGLE_OPEN": return { ...state, isOpen: !state.isOpen }; case "SET_OPEN": return { ...state, isOpen: action.open }; case "CLEAR": - return { ...state, messages: [], isLoading: false, progress: null }; + return { + ...state, + messages: [], + isLoading: false, + progress: null, + progressLog: [], + }; } } @@ -325,6 +366,8 @@ interface ChatContextValue { isOpen: boolean; isLoading: boolean; progress: AiWorkflowProgress | null; + /** Ordered log of every progress event for the current in-flight request. */ + progressLog: AiWorkflowProgress[]; toggleOpen: () => void; setOpen: (open: boolean) => void; sendMessage: (content: string) => Promise; @@ -339,6 +382,7 @@ const initialState: ChatState = { isOpen: false, isLoading: false, progress: null, + progressLog: [], }; export function ChatProvider({ children }: { children: ReactNode }) { @@ -440,6 +484,11 @@ export function ChatProvider({ children }: { children: ReactNode }) { abortRef.current = controller; const priorMessages = messagesRef.current; + const startTime = Date.now(); + // Mirror every progress event locally so we can attach the full log to + // the assistant message when the result arrives — without needing a ref + // into the reducer state. + const progressLogLocal: AiWorkflowProgress[] = []; const userMessage: ChatMessage = { id: generateId(), @@ -503,16 +552,15 @@ export function ChatProvider({ children }: { children: ReactNode }) { ) { toolsUsed.push(data.tool); } - dispatch({ - type: "SET_PROGRESS", - progress: { - phase: data.phase as AiWorkflowPhase, - tool: data.tool, - stepIndex: data.stepIndex, - stepCount: data.stepCount, - engineDetail: data.engineDetail, - }, - }); + const progressItem: AiWorkflowProgress = { + phase: data.phase as AiWorkflowPhase, + tool: data.tool, + stepIndex: data.stepIndex, + stepCount: data.stepCount, + engineDetail: data.engineDetail, + }; + progressLogLocal.push(progressItem); + dispatch({ type: "APPEND_PROGRESS", progress: progressItem }); }, onResult: (data) => { receivedResult = true; @@ -526,6 +574,11 @@ export function ChatProvider({ children }: { children: ReactNode }) { content: replyContent, timestamp: Date.now(), toolsUsed: toolsUsed.length > 0 ? toolsUsed : undefined, + progressLog: + progressLogLocal.length > 0 + ? [...progressLogLocal] + : undefined, + durationMs: Date.now() - startTime, }, }); if (data.fileId || data.resultFiles?.length) { @@ -599,6 +652,7 @@ export function ChatProvider({ children }: { children: ReactNode }) { isOpen: state.isOpen, isLoading: state.isLoading, progress: state.progress, + progressLog: state.progressLog, toggleOpen, setOpen, sendMessage, diff --git a/frontend/editor/src/proprietary/components/chat/ChatPanel.css b/frontend/editor/src/proprietary/components/chat/ChatPanel.css index 0176172ce..62402108f 100644 --- a/frontend/editor/src/proprietary/components/chat/ChatPanel.css +++ b/frontend/editor/src/proprietary/components/chat/ChatPanel.css @@ -396,7 +396,182 @@ margin-bottom: 0; } -/* Thinking indicator */ +/* ─── Progress step log ─────────────────────────────────────────────────── */ + +/* + * Shown while the AI is working. Each step slides in from below as events + * arrive; the active (last) step is full-opacity with a pulsing icon, and + * earlier completed steps are dimmed. Only the most-recent N steps are + * rendered — there is no scrollable list. + */ + +.chat-progress-log { + display: flex; + flex-direction: column; + padding: 0.15rem 0.5rem 0.35rem; + /* No gap — the connector line div provides the inter-step spacing. */ +} + +.chat-progress-step { + display: flex; + align-items: flex-start; + gap: 0.65rem; + /* + * Use color rather than opacity for past-step dimming so the connector + * line (a sibling element inside __left, not a text node) is NOT affected + * and stays clearly visible. currentColor flows into SVG icon fills; + * color inherits into the label span automatically. + */ + color: var(--text-muted); + transition: color 250ms ease-out; + animation: chat-step-enter 300ms cubic-bezier(0.22, 1, 0.36, 1) both; +} + +/* + * Active step: white text on dark mode so currentColor feeds directly into + * the StirlingLogoAnimated SVG paths and the label at the same time. + * Light-mode falls back to the default text colour so it's not invisible. + */ +/* Active step: icon and label are coloured independently so the logo can be + blue while the text uses the normal text colour. */ +.chat-progress-step--active .chat-progress-step__icon { + color: var(--mantine-color-blue-filled); +} + +.chat-progress-step--active .chat-progress-step__label { + color: var(--mantine-color-text); +} + +[data-mantine-color-scheme="dark"] + .chat-progress-step--active + .chat-progress-step__label { + color: #ffffff; +} + +.chat-progress-step__left { + display: flex; + flex-direction: column; + align-items: center; + flex-shrink: 0; + width: 20px; +} + +/* Icon cell — fixed height so the connector line aligns cleanly. */ +.chat-progress-step__icon { + width: 20px; + height: 24px; + display: flex; + align-items: center; + justify-content: center; + overflow: hidden; + flex-shrink: 0; + /* Inherits color from .chat-progress-step so currentColor flows into icons. */ +} + +/* + * Connector line between steps. Not a child of the label so it is unaffected + * by the step's color; it sits on its own and reads clearly against the + * toolbar background. + */ +.chat-progress-step__line { + width: 1px; + height: 18px; + background: var(--border-subtle); + flex-shrink: 0; + margin-top: 2px; +} + +/* Active step label — slightly larger, inherits the active color. */ +.chat-progress-step__label { + font-size: 0.875rem; + padding-top: 3px; + line-height: 1.35; +} + +/* Past step labels — a touch smaller so the active row reads as the primary. */ +.chat-progress-step:not(.chat-progress-step--active) + .chat-progress-step__label { + font-size: 0.8rem; + padding-top: 4px; +} + +/* + * Wrapper that scales a registry tool icon (LocalIcon, typically 1.5rem/24px) + * down to fit the 20px icon column. Transform does not affect layout, so the + * parent overflow:hidden clips the un-scaled footprint cleanly. + */ +.chat-step-icon-scaled { + display: inline-flex; + align-items: center; + justify-content: center; + transform: scale(0.75); + transform-origin: center; + flex-shrink: 0; +} + +/* Entry: steps slide up from a few pixels below their final position. */ +@keyframes chat-step-enter { + from { + transform: translateY(7px); + } + to { + transform: translateY(0); + } +} + +/* ─── Completed progress log dropdown ──────────────────────────────────── */ + +/* + * Shown above each completed assistant turn. The toggle displays a small + * "Ran for X seconds" label; expanding reveals the full ordered step list + * for that turn in a compact, read-only format. + */ + +.chat-completed-log { + margin-bottom: 0.55rem; +} + +.chat-completed-log__toggle { + display: inline-flex; + align-items: center; + padding: 0.1rem 0.25rem 0.1rem 0.1rem; + border-radius: 0.35rem; + color: var(--text-muted); + transition: background 120ms ease-out; +} + +.chat-completed-log__toggle:hover { + background: var(--mantine-color-default-hover); +} + +.chat-completed-log__steps { + display: flex; + flex-direction: column; + padding: 0.35rem 0.4rem 0.2rem 0.25rem; + margin-top: 0.2rem; +} + +/* In the completed log every step is a past step — no active highlighting, + no entry animation (the content is static once opened). */ +.chat-completed-log__steps .chat-progress-step { + animation: none; +} + +/* Tighten the icon and connector dimensions for the denser historical view. */ +.chat-completed-log__steps .chat-progress-step__icon { + height: 20px; +} + +.chat-completed-log__steps .chat-progress-step__line { + height: 14px; +} + +.chat-completed-log__steps .chat-progress-step__label { + font-size: 0.775rem; + padding-top: 2px; +} + +/* Legacy thinking indicator (kept in case any other code references it). */ .chat-thinking { display: flex; align-items: center; @@ -405,6 +580,13 @@ color: var(--mantine-color-blue-filled); } +@media (prefers-reduced-motion: reduce) { + .chat-progress-step { + animation: none; + transition: none; + } +} + /* ─── Stirling logo thinking animation ─────────────────────────────────── */ .stirling-thinking__path-right { diff --git a/frontend/editor/src/proprietary/components/chat/ChatPanel.tsx b/frontend/editor/src/proprietary/components/chat/ChatPanel.tsx index 7baf5d0a5..ff0fdc63b 100644 --- a/frontend/editor/src/proprietary/components/chat/ChatPanel.tsx +++ b/frontend/editor/src/proprietary/components/chat/ChatPanel.tsx @@ -4,6 +4,7 @@ import { useEffect, useState, type KeyboardEvent, + type ReactNode, } from "react"; import { renderMarkdown } from "@app/components/viewer/nonpdf/MarkdownRenderer"; import { TFunction } from "i18next"; @@ -13,7 +14,6 @@ import { Box, Collapse, Group, - List, Menu, Paper, ScrollArea, @@ -23,7 +23,10 @@ import { UnstyledButton, } from "@mantine/core"; import ArrowUpwardIcon from "@mui/icons-material/ArrowUpward"; +import ArticleOutlinedIcon from "@mui/icons-material/ArticleOutlined"; +import BuildOutlinedIcon from "@mui/icons-material/BuildOutlined"; import CloseIcon from "@mui/icons-material/Close"; +import CloudOutlinedIcon from "@mui/icons-material/CloudOutlined"; import ContentCopyIcon from "@mui/icons-material/ContentCopy"; import DeleteSweepIcon from "@mui/icons-material/DeleteSweep"; import ExpandLessIcon from "@mui/icons-material/ExpandLess"; @@ -33,14 +36,15 @@ import { useChat, AiWorkflowPhase, ChatRole, + PROGRESS_LOG_MAX, isKnownEngineProgressDetail, type AiWorkflowProgress, type AnyEngineProgressDetail, } from "@app/components/chat/ChatContext"; import { formatRelativeTime } from "@app/utils/timeUtils"; import { useTranslatedToolCatalog } from "@app/data/useTranslatedToolRegistry"; -import { StirlingLogoOutline } from "@app/components/agents/StirlingLogoOutline"; import { StirlingLogoAnimated } from "@app/components/agents/StirlingLogoAnimated"; +import { StirlingLogoOutline } from "@app/components/agents/StirlingLogoOutline"; import { ChatQuickActions } from "@app/components/chat/ChatQuickActions"; import "@app/components/chat/ChatPanel.css"; @@ -68,6 +72,27 @@ function useToolNameResolver(): ToolNameResolver { }, [allTools]); } +/** Resolver mapping a tool endpoint path to its registry icon ReactNode. */ +type ToolIconResolver = (endpoint: string) => ReactNode | null; + +/** + * Look up a tool's icon ReactNode from the tool catalog, keyed by API endpoint path. + * Returns null when the endpoint is not found (use a generic fallback icon in that case). + */ +function useToolIconResolver(): ToolIconResolver { + const { allTools } = useTranslatedToolCatalog(); + return useMemo(() => { + const iconByEndpoint = new Map(); + Object.values(allTools).forEach((tool) => { + const endpoint = tool.operationConfig?.endpoint; + if (typeof endpoint === "string") { + iconByEndpoint.set(endpoint, tool.icon); + } + }); + return (endpoint: string) => iconByEndpoint.get(endpoint) ?? null; + }, [allTools]); +} + function formatProgress( progress: AiWorkflowProgress, t: TranslateFn, @@ -125,31 +150,152 @@ function formatEngineProgress( } } -function ToolsUsedBlock({ - tools, - resolveToolName, +/** + * Choose an icon for a progress step. + * + * The active (current) step always shows the animated Stirling logo so it reads + * as the "live" indicator. Past steps get a phase-specific icon so the trail + * is scannable at a glance. + */ +function progressStepIcon( + progress: AiWorkflowProgress, + resolveToolIcon: ToolIconResolver, + isActive: boolean, +): ReactNode { + if (isActive) { + return ; + } + if (progress.phase === AiWorkflowPhase.EXECUTING_TOOL) { + const registryIcon = progress.tool ? resolveToolIcon(progress.tool) : null; + if (registryIcon) { + return {registryIcon}; + } + return ; + } + if ( + progress.phase === AiWorkflowPhase.EXTRACTING_CONTENT || + progress.phase === AiWorkflowPhase.ENGINE_PROGRESS + ) { + return ; + } + return ; +} + +/** + * Animated step-by-step progress log shown while the AI is working. + * Displays the last {@link PROGRESS_LOG_VISIBLE} steps from the live event stream, + * with the active (most recent) step highlighted and older steps dimmed. + */ +function ProgressLogDisplay({ + progressLog, t, + resolveToolName, + resolveToolIcon, }: { - tools: string[]; - resolveToolName: ToolNameResolver; + progressLog: AiWorkflowProgress[]; t: TranslateFn; + resolveToolName: ToolNameResolver; + resolveToolIcon: ToolIconResolver; +}) { + // Placeholder shown before the first SSE event arrives. + if (progressLog.length === 0) { + return ( +

+
+
+
+ +
+
+ + {t("chat.progress.thinking")} + +
+
+ ); + } + + // Chronological order: oldest at top, newest (active) at bottom. + // The reducer already caps progressLog at PROGRESS_LOG_MAX entries, so this + // slice is effectively a no-op but kept for defensive correctness. + const visibleSteps = progressLog.slice(-PROGRESS_LOG_MAX); + const startIndex = progressLog.length - visibleSteps.length; + + return ( +
+ {visibleSteps.map((step, i) => { + // Stable key based on absolute position in the full log — React reuses + // existing DOM elements and only mounts (and animates) new ones. + const globalIndex = startIndex + i; + const isActive = i === visibleSteps.length - 1; // last = newest = bottom + // Connector runs below every step except the active one at the bottom. + const showConnector = i < visibleSteps.length - 1; + const label = formatProgress(step, t, resolveToolName); + return ( +
+
+
+ {progressStepIcon(step, resolveToolIcon, isActive)} +
+ {showConnector &&
} +
+ {label} +
+ ); + })} +
+ ); +} + +function formatDuration(ms: number, t: TranslateFn): string { + const totalSeconds = Math.max(1, Math.round(ms / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + + if (minutes === 0) { + return t("chat.progress.ranForSeconds", { count: totalSeconds }); + } + if (seconds === 0) { + return t("chat.progress.ranForMinutes", { count: minutes }); + } + return t("chat.progress.ranForMinutesSeconds", { minutes, seconds }); +} + +/** + * Collapsed "Ran for X seconds" dropdown that appears above each completed + * assistant turn. Expands to show the full ordered progress log for that turn. + */ +function CompletedProgressLogDropdown({ + progressLog, + durationMs, + t, + resolveToolName, + resolveToolIcon, +}: { + progressLog: AiWorkflowProgress[]; + durationMs: number; + t: TranslateFn; + resolveToolName: ToolNameResolver; + resolveToolIcon: ToolIconResolver; }) { const [expanded, setExpanded] = useState(false); - const names = tools.map( - (endpoint) => resolveToolName(endpoint) ?? t("chat.toolsUsed.unknownTool"), - ); - const label = t("chat.toolsUsed.summary", { count: tools.length }); + const label = formatDuration(durationMs, t); + return ( - +
setExpanded((v) => !v)} aria-expanded={expanded} > - + {expanded ? ( - + ) : ( - + )} {label} @@ -157,19 +303,30 @@ function ToolsUsedBlock({ - - {names.map((name, i) => ( - {name} - ))} - +
+ {progressLog.map((step, i) => { + const showConnector = i < progressLog.length - 1; + const stepLabel = formatProgress(step, t, resolveToolName); + return ( +
+
+
+ {progressStepIcon(step, resolveToolIcon, false)} +
+ {showConnector && ( +
+ )} +
+ {stepLabel} +
+ ); + })} +
- +
); } @@ -177,15 +334,19 @@ function ChatMessageBubble({ role, content, timestamp, - toolsUsed, + progressLog, + durationMs, resolveToolName, + resolveToolIcon, t, }: { role: ChatRole; content: string; timestamp: number; - toolsUsed?: string[]; + progressLog?: AiWorkflowProgress[]; + durationMs?: number; resolveToolName: ToolNameResolver; + resolveToolIcon: ToolIconResolver; t: TranslateFn; }) { const [copied, setCopied] = useState(false); @@ -231,16 +392,18 @@ function ChatMessageBubble({ return (
+ {progressLog && progressLog.length > 0 && durationMs != null && ( + + )} {renderMarkdown(content)} - {toolsUsed && toolsUsed.length > 0 && ( - - )} {actions}
@@ -256,20 +419,55 @@ export interface ChatPanelProps { export function ChatPanel({ onBack, backLabel }: ChatPanelProps) { const { t } = useTranslation(); - const { messages, isLoading, progress, sendMessage, clearChat } = useChat(); + const { messages, isLoading, progressLog, sendMessage, clearChat } = + useChat(); const resolveToolName = useToolNameResolver(); + const resolveToolIcon = useToolIconResolver(); const [input, setInput] = useState(""); const scrollRef = useRef(null); const inputRef = useRef(null); + // Tracks whether the user manually scrolled away from the bottom. + // A ref (not state) so scroll events don't cause re-renders. + const userScrolledUp = useRef(false); + // Jump to the bottom on first render so existing conversations open at the + // most recent message rather than the top. useEffect(() => { - if (scrollRef.current) { - scrollRef.current.scrollTo({ - top: scrollRef.current.scrollHeight, - behavior: "smooth", + requestAnimationFrame(() => { + const el = scrollRef.current; + if (el) el.scrollTop = el.scrollHeight; + }); + }, []); + + // Attach a passive scroll listener to track whether the user has scrolled + // away from the bottom (breaks auto-scroll) or returned to it (re-latches). + useEffect(() => { + const el = scrollRef.current; + if (!el) return; + const onScroll = () => { + const distFromBottom = el.scrollHeight - el.scrollTop - el.clientHeight; + userScrolledUp.current = distFromBottom > 50; + }; + el.addEventListener("scroll", onScroll, { passive: true }); + return () => el.removeEventListener("scroll", onScroll); + }, []); + + // Scroll to the bottom when messages arrive or live progress steps update, + // unless the user has scrolled up (they're reading history). + // Scrolling back to the bottom resets the ref, so the next update re-latches. + // + // RAF defers the scroll until after the browser has laid out the new nodes, + // so scrollHeight is correct. Direct scrollTop assignment avoids the + // smooth-scroll interruption problem that occurs when SSE events arrive + // faster than a smooth animation can complete. + useEffect(() => { + if (!userScrolledUp.current) { + requestAnimationFrame(() => { + const el = scrollRef.current; + if (el) el.scrollTop = el.scrollHeight; }); } - }, [messages]); + }, [messages, progressLog]); useEffect(() => { inputRef.current?.focus(); @@ -343,21 +541,21 @@ export function ChatPanel({ onBack, backLabel }: ChatPanelProps) { role={msg.role} content={msg.content} timestamp={msg.timestamp} - toolsUsed={msg.toolsUsed} + progressLog={msg.progressLog} + durationMs={msg.durationMs} resolveToolName={resolveToolName} + resolveToolIcon={resolveToolIcon} t={t} /> ))} {isLoading && (
-
- - - {progress - ? formatProgress(progress, t, resolveToolName) - : t("chat.progress.thinking")} - -
+
)} diff --git a/frontend/editor/src/proprietary/routes/Login.tsx b/frontend/editor/src/proprietary/routes/Login.tsx index 85c42fce8..c334bef8d 100644 --- a/frontend/editor/src/proprietary/routes/Login.tsx +++ b/frontend/editor/src/proprietary/routes/Login.tsx @@ -17,12 +17,11 @@ import { useDocumentMeta } from "@app/hooks/useDocumentMeta"; import AuthLayout from "@app/routes/authShared/AuthLayout"; import { useBackendProbe } from "@app/hooks/useBackendProbe"; import apiClient from "@app/services/apiClient"; -import { BASE_PATH } from "@app/constants/app"; +import { BASE_PATH, withBasePath } from "@app/constants/app"; import { type OAuthProvider } from "@app/auth/oauthTypes"; import { updateSupportedLanguages } from "@app/i18n"; // Import login components -import LoginHeader from "@app/routes/login/LoginHeader"; import ErrorMessage from "@app/routes/login/ErrorMessage"; import EmailPasswordForm from "@app/routes/login/EmailPasswordForm"; import OAuthButtons, { @@ -74,7 +73,6 @@ export default function Login() { const autoLoginErrorRecorded = useRef(false); const isUserPassAllowed = loginMethod === "all" || loginMethod === "normal"; const isSsoOnlyMode = loginMethod !== "all" && loginMethod !== "normal"; - const isSingleSsoOnly = !isUserPassAllowed && enabledProviders.length === 1; const AUTO_LOGIN_ATTEMPTS_KEY = "stirling_sso_auto_login_attempts"; const AUTO_LOGIN_ERRORS_KEY = "stirling_sso_auto_login_errors"; @@ -461,7 +459,6 @@ export default function Login() { // If backend isn't ready yet, show a lightweight status screen instead of the form if (backendProbe.status !== "up" && !loginDisabled) { - const backendTitle = t("backendStartup.notFoundTitle", "Backend not found"); const handleRetry = async () => { const result = await backendProbe.probe(); if (result.status === "up") { @@ -471,7 +468,18 @@ export default function Login() { }; return ( - +
+ Stirling PDF + Stirling PDF +
- +
+ Stirling PDF + Stirling PDF +
{/* Success message */} {successMessage && ( diff --git a/frontend/editor/src/proprietary/routes/Signup.tsx b/frontend/editor/src/proprietary/routes/Signup.tsx index 28676a085..9773106c4 100644 --- a/frontend/editor/src/proprietary/routes/Signup.tsx +++ b/frontend/editor/src/proprietary/routes/Signup.tsx @@ -5,10 +5,9 @@ import { useDocumentMeta } from "@app/hooks/useDocumentMeta"; import { useAuth } from "@app/auth/UseSession"; import AuthLayout from "@app/routes/authShared/AuthLayout"; import "@app/routes/authShared/auth.css"; -import { BASE_PATH } from "@app/constants/app"; +import { BASE_PATH, withBasePath } from "@app/constants/app"; // Import signup components -import LoginHeader from "@app/routes/login/LoginHeader"; import ErrorMessage from "@app/routes/login/ErrorMessage"; import DividerWithText from "@app/components/shared/DividerWithText"; import SignupForm from "@app/routes/signup/SignupForm"; @@ -92,10 +91,18 @@ export default function Signup() { return ( - +
+ Stirling PDF + Stirling PDF +
diff --git a/frontend/editor/src/proprietary/routes/authShared/auth.css b/frontend/editor/src/proprietary/routes/authShared/auth.css index e0abf4e7a..080c28092 100644 --- a/frontend/editor/src/proprietary/routes/authShared/auth.css +++ b/frontend/editor/src/proprietary/routes/authShared/auth.css @@ -615,3 +615,90 @@ background-color: #af3434 !important; opacity: 0.6 !important; } + +/* ── Logo block ─────────────────────────────────────────────────────── */ +.auth-logo-block { + display: flex; + flex-direction: column; + align-items: center; + margin-bottom: 2rem; + margin-top: 0.5rem; + animation: authFadeUp 0.4s ease both; +} + +.auth-logo-header { + height: 8rem; + width: auto; +} + +.auth-logo-header--dark { + display: none; +} + +/* ── Page entrance animation ────────────────────────────────────────── */ +@keyframes authFadeUp { + from { + opacity: 0; + transform: translateY(12px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +/* ── Expandable trigger button state ────────────────────────────────── */ +.auth-expandable-trigger { + transition: + background-color 180ms ease, + box-shadow 180ms ease, + border-color 180ms ease; +} + +.auth-expandable-trigger--active { + border-color: #af3434 !important; + box-shadow: 0 0 0 3px rgba(175, 52, 52, 0.12); +} + +/* ── Animated expand/collapse via grid-template-rows ───────────────── */ +.auth-expand-grid { + display: grid; + grid-template-rows: 0fr; + transition: + grid-template-rows 280ms cubic-bezier(0.4, 0, 0.2, 1), + opacity 220ms ease; + opacity: 0; +} + +.auth-expand-grid--open { + grid-template-rows: 1fr; + opacity: 1; +} + +.auth-expand-inner { + overflow: hidden; + min-height: 0; +} + +/* ── Icon+label group — keeps icons vertically aligned across buttons ── */ +.oauth-btn-group { + display: inline-flex; + align-items: center; + min-width: 12.5rem; +} + +.oauth-btn-label { + flex: 1; + text-align: center; +} + +/* ── Icon helpers inside oauth-button-fullwidth ─────────────────────── */ +.auth-at-icon { + font-size: 1.25rem; + line-height: 1; + margin-right: 0.5rem; + display: inline-block; + width: 1.75rem; + text-align: center; + flex-shrink: 0; +} diff --git a/frontend/editor/src/saas/components/shared/config/configSections/McpSection.tsx b/frontend/editor/src/saas/components/shared/config/configSections/McpSection.tsx new file mode 100644 index 000000000..8313fdd12 --- /dev/null +++ b/frontend/editor/src/saas/components/shared/config/configSections/McpSection.tsx @@ -0,0 +1,262 @@ +import { useMemo } from "react"; +import { useTranslation } from "react-i18next"; +import { + Stack, + Paper, + Text, + Group, + Alert, + Code, + Badge, + Button, + CopyButton, + Tabs, + Tooltip, + ThemeIcon, +} from "@mantine/core"; +import LocalIcon from "@app/components/shared/LocalIcon"; +import { useAppConfig } from "@app/contexts/AppConfigContext"; +import { openAppSettings } from "@app/utils/appSettings"; + +/** Strip a single trailing slash so we can safely append paths. */ +function trimTrailingSlash(url: string): string { + return url.endsWith("/") ? url.slice(0, -1) : url; +} + +/** A small copy-to-clipboard button that sits inline next to a URL/snippet. */ +function CopyInline({ value, label }: { value: string; label: string }) { + const { t } = useTranslation(); + return ( + + {({ copied, copy }) => ( + + + + )} + + ); +} + +// SaaS MCP guide: always shown, explains how to point an AI assistant at the +// OAuth-protected /mcp endpoint with per-client config. +export default function McpSection() { + const { t } = useTranslation(); + const { config } = useAppConfig(); + + const baseUrl = useMemo(() => { + const raw = + config?.baseUrl || + (typeof window !== "undefined" ? window.location.origin : ""); + return trimTrailingSlash(raw); + }, [config?.baseUrl]); + + const mcpUrl = `${baseUrl}/mcp`; + + // Per-client connection snippets pointing at this deployment's /mcp endpoint. + const clients = useMemo( + () => [ + { + value: "claude", + label: "Claude Desktop", + file: "claude_desktop_config.json", + config: JSON.stringify( + { mcpServers: { "stirling-pdf": { type: "http", url: mcpUrl } } }, + null, + 2, + ), + }, + { + value: "codex", + label: "Codex CLI", + file: "~/.codex/config.toml", + config: `[mcp_servers.stirling-pdf]\nurl = "${mcpUrl}"`, + }, + { + value: "vscode", + label: "VS Code", + file: ".vscode/mcp.json", + config: JSON.stringify( + { servers: { "stirling-pdf": { type: "http", url: mcpUrl } } }, + null, + 2, + ), + }, + ], + [mcpUrl], + ); + + const tools: { icon: string; key: string; fallback: string }[] = [ + { + icon: "sync-alt-rounded", + key: "config.mcp.tools.convert", + fallback: "Convert", + }, + { + icon: "description-rounded", + key: "config.mcp.tools.pages", + fallback: "Pages", + }, + { icon: "lock", key: "config.mcp.tools.security", fallback: "Security" }, + { + icon: "construction-rounded", + key: "config.mcp.tools.misc", + fallback: "Misc", + }, + { icon: "smart-toy-rounded", key: "config.mcp.tools.ai", fallback: "AI" }, + ]; + + return ( +
+ +
+ + + + + + {t("config.mcp.title", "MCP Server")} + + + + {t( + "config.mcp.description", + "Model Context Protocol (MCP) lets AI assistants like Claude use your Stirling PDF tools directly. Connect a client once and your assistant can convert, edit, secure and process documents on your behalf.", + )} + +
+ + {/* Endpoint */} + + + + + {t("config.mcp.endpoint.label", "Your MCP endpoint")} + + {mcpUrl} + + + + + + {/* Per-client setup */} + + + + {t("config.mcp.setup.title", "Connect your AI assistant")} + + + {t( + "config.mcp.setup.hint", + "Pick your client, paste the snippet into the file shown, then restart it. You'll sign in with your Stirling account on first use - no keys to copy.", + )} + + + + {clients.map((c) => ( + + {c.label} + + ))} + + {clients.map((c) => ( + + + + + {t("config.mcp.setup.addTo", "Add to")}{" "} + {c.file} + + + + {c.config} + + + ))} + + + + + {/* Available tools */} +
+ + {t("config.mcp.tools.title", "What your assistant can do")} + + + {tools.map((tool) => ( + + } + > + {t(tool.key, tool.fallback)} + + ))} + +
+ + {/* Tip / cross-link */} + } + > + + + {t( + "config.mcp.tip", + "Every action your assistant runs is performed as your account and counts towards your usage, just like using Stirling PDF directly.", + )} + + + + + +
+ ); +} diff --git a/frontend/editor/src/saas/components/shared/config/saasConfigNavSections.tsx b/frontend/editor/src/saas/components/shared/config/saasConfigNavSections.tsx index 5dd293451..27979ff5a 100644 --- a/frontend/editor/src/saas/components/shared/config/saasConfigNavSections.tsx +++ b/frontend/editor/src/saas/components/shared/config/saasConfigNavSections.tsx @@ -9,6 +9,7 @@ import GeneralSection from "@app/components/shared/config/configSections/General import PasswordSecurity from "@app/components/shared/config/configSections/PasswordSecurity"; import ApiKeys from "@app/components/shared/config/configSections/ApiKeys"; import Plan from "@app/components/shared/config/configSections/Plan"; +import McpSection from "@app/components/shared/config/configSections/McpSection"; type OverviewComponent = React.ComponentType<{ onLogoutClick: () => void }>; @@ -108,6 +109,44 @@ function appendBillingSection( ]; } +// Add an "MCP Server" tab in the Developer section. Always shown in SaaS; +// purely informational, so it appears for anonymous users too. +function appendMcpSection( + sections: ConfigNavSection[], + t: TFunction<"translation", undefined>, +): ConfigNavSection[] { + const hasMcp = sections.some((section) => + section.items.some((item) => item.key === "mcp"), + ); + + if (hasMcp) { + return sections; + } + + const mcpItem = { + key: "mcp" as const, + label: t("config.mcp.navLabel", "MCP Server"), + icon: "smart-toy-rounded", + component: , + }; + + const developerIndex = sections.findIndex((section) => + section.items.some( + (item) => item.key === "developer" || item.key === "api-keys", + ), + ); + + if (developerIndex === -1) { + return [...sections, { title: "Developer", items: [mcpItem] }]; + } + + return sections.map((section, index) => + index === developerIndex + ? { ...section, items: [...section.items, mcpItem] } + : section, + ); +} + export function createSaasConfigNavSections( Overview: OverviewComponent, onLogoutClick: () => void, @@ -151,6 +190,7 @@ export function createSaasConfigNavSections( sections = ensurePreferencesSection(sections); sections = appendDeveloperSection(sections); + sections = appendMcpSection(sections, t); if (!isAnonymous) { sections = appendBillingSection(sections, t); diff --git a/frontend/editor/src/saas/components/shared/config/types.ts b/frontend/editor/src/saas/components/shared/config/types.ts index e39944212..f80bd1ac1 100644 --- a/frontend/editor/src/saas/components/shared/config/types.ts +++ b/frontend/editor/src/saas/components/shared/config/types.ts @@ -1,8 +1,9 @@ import { VALID_NAV_KEYS as CORE_NAV_KEYS } from "@core/components/shared/config/types"; -// SaaS adds an "overview" account section. All other keys (including ones -// SaaS doesn't render today) come from core - subtracting them here would -// just break the type union without affecting runtime nav. -export const VALID_NAV_KEYS = [...CORE_NAV_KEYS, "overview"] as const; +// SaaS adds an "overview" account section and an "mcp" integrations tab. All +// other keys (including ones SaaS doesn't render today) come from core - +// subtracting them here would just break the type union without affecting +// runtime nav. +export const VALID_NAV_KEYS = [...CORE_NAV_KEYS, "overview", "mcp"] as const; export type NavKey = (typeof VALID_NAV_KEYS)[number]; diff --git a/frontend/editor/src/saas/routes/Login.tsx b/frontend/editor/src/saas/routes/Login.tsx index 848792147..4be5ed9b5 100644 --- a/frontend/editor/src/saas/routes/Login.tsx +++ b/frontend/editor/src/saas/routes/Login.tsx @@ -7,17 +7,18 @@ import { useDocumentMeta } from "@app/hooks/useDocumentMeta"; import AuthLayout from "@app/routes/authShared/AuthLayout"; import "@app/routes/authShared/auth.css"; import "@app/routes/authShared/saas-auth.css"; -import GuestSignInButton from "@app/routes/authShared/GuestSignInButton"; +import { + absoluteWithBasePath, + getBaseUrl, + withBasePath, +} from "@app/constants/app"; +import LinkRoundedIcon from "@mui/icons-material/LinkRounded"; // Import login components -import LoginHeader from "@app/routes/login/LoginHeader"; import ErrorMessage from "@app/routes/login/ErrorMessage"; import EmailPasswordForm from "@app/routes/login/EmailPasswordForm"; -import MagicLinkForm from "@app/routes/login/MagicLinkForm"; import OAuthButtons from "@app/routes/login/OAuthButtons"; -import DividerWithText from "@app/components/shared/DividerWithText"; import LoggedInState from "@app/routes/login/LoggedInState"; -import { absoluteWithBasePath, getBaseUrl } from "@app/constants/app"; export default function Login() { const navigate = useNavigate(); @@ -25,11 +26,13 @@ export default function Login() { const { t } = useTranslation(); const [isSigningIn, setIsSigningIn] = useState(false); const [error, setError] = useState(null); - const [showMagicLink, setShowMagicLink] = useState(false); + const [showMagicLinkForm, setShowMagicLinkForm] = useState(false); const [showEmailForm, setShowEmailForm] = useState(false); const [email, setEmail] = useState(""); const [password, setPassword] = useState(""); const [magicLinkEmail, setMagicLinkEmail] = useState(""); + const [magicLinkSent, setMagicLinkSent] = useState(false); + // Prefill email from query param (e.g. after password reset) useEffect(() => { try { @@ -37,6 +40,7 @@ export default function Login() { const emailFromQuery = url.searchParams.get("email"); if (emailFromQuery) { setEmail(emailFromQuery); + setShowEmailForm(true); } } catch (_) { // ignore @@ -174,12 +178,9 @@ export default function Login() { setError(error.message); } else { setError(null); - alert(t("login.magicLinkSent", { email: magicLinkEmail })); - setMagicLinkEmail(""); - setShowMagicLink(false); + setMagicLinkSent(true); } } catch (err) { - console.error("[Login] Unexpected error:", err); setError( t("login.unexpectedError", { message: err instanceof Error ? err.message : "Unknown error", @@ -190,10 +191,6 @@ export default function Login() { } }; - const handleForgotPassword = () => { - navigate("/auth/reset"); - }; - const handleAnonymousSignIn = async () => { try { setIsSigningIn(true); @@ -227,115 +224,210 @@ export default function Login() { } }; + const toggleEmailForm = () => { + setShowEmailForm((v) => !v); + setShowMagicLinkForm(false); + setMagicLinkSent(false); + }; + + const toggleMagicLink = () => { + setShowMagicLinkForm((v) => !v); + setShowEmailForm(false); + setMagicLinkSent(false); + }; + return ( - - + + {/* Centered logo */} +
+ Stirling PDF + Stirling PDF +
- {/* OAuth first */} - + {/* OAuth + magic link group — single flex column so gap is uniform */} +
+ - {/* Divider between OAuth and Email */} - + {/* Magic link button + its expandable form as one unit */} +
+ - {/* Sign in with email button (primary color to match signup CTA) */} -
+
+
+
+ {magicLinkSent ? ( +

+ {t("login.magicLinkSent", { email: magicLinkEmail })} +

+ ) : ( +
+ setMagicLinkEmail(e.target.value)} + onKeyDown={(e) => + e.key === "Enter" && + !isSigningIn && + signInWithMagicLink() + } + className="auth-input" + /> + +
+ )} +
+
+
+
+
+ + {/* Email & Password button */} + + + {/* Email form — animated expand */} +
+
+
+ + +
+
+
+ + {/* Skip */} +
- {showEmailForm && ( - - )} - - {showEmailForm && ( -
- -
- )} - - {/* Divider then Guest */} - - - - -
- - + {/* Bottom */} +
- - {/* Magic link form renders on demand */} - {showMagicLink && ( -
- -
- )} ); } diff --git a/frontend/editor/src/saas/routes/Signup.tsx b/frontend/editor/src/saas/routes/Signup.tsx index d772cbe37..724bf455a 100644 --- a/frontend/editor/src/saas/routes/Signup.tsx +++ b/frontend/editor/src/saas/routes/Signup.tsx @@ -4,18 +4,15 @@ import { signInAnonymously } from "@app/auth/supabase"; import { useAuth } from "@app/auth/UseSession"; import { useTranslation } from "@app/hooks/useTranslation"; import { useDocumentMeta } from "@app/hooks/useDocumentMeta"; -import { getBaseUrl } from "@app/constants/app"; +import { getBaseUrl, withBasePath } from "@app/constants/app"; import AuthLayout from "@app/routes/authShared/AuthLayout"; import "@app/routes/authShared/auth.css"; import "@app/routes/authShared/saas-auth.css"; -import GuestSignInButton from "@app/routes/authShared/GuestSignInButton"; import { alert } from "@app/components/toast"; // Import signup components -import LoginHeader from "@app/routes/login/LoginHeader"; import ErrorMessage from "@app/routes/login/ErrorMessage"; import OAuthButtons from "@app/routes/login/OAuthButtons"; -import DividerWithText from "@app/components/shared/DividerWithText"; import SignupForm from "@app/routes/signup/SignupForm"; import { useSignupFormValidation, @@ -184,84 +181,112 @@ export default function Signup() { return ( - + {/* Centered logo */} +
+ Stirling PDF + Stirling PDF +
- {/* OAuth first */} -
+ {/* OAuth providers */} +
- {/* Divider between OAuth and Email */} -
- + {/* Email & Password button */} + + + {/* Email form — animated expand */} +
+
+
+ +
+
- {/* Use Email Instead button (toggles email form) */} -
+ {/* Skip */} +
- {showEmailForm && ( - - )} - -
- -
- - - - {/* Bottom row */} -
+ {/* Bottom */} +
diff --git a/frontend/editor/src/saas/routes/authShared/saas-auth.css b/frontend/editor/src/saas/routes/authShared/saas-auth.css index 5073a24c9..9a8943fc2 100644 --- a/frontend/editor/src/saas/routes/authShared/saas-auth.css +++ b/frontend/editor/src/saas/routes/authShared/saas-auth.css @@ -14,7 +14,7 @@ justify-content: center; padding: 0.75rem 1rem; border: 1px solid #d1d5db; - border-radius: 0.625rem; + border-radius: 100px; background-color: #ffffff; font-size: 1rem; font-weight: 600; @@ -22,6 +22,10 @@ cursor: pointer; gap: 0.5rem; box-shadow: 0 0.125rem 0.375rem rgba(0, 0, 0, 0.04); + transition: + background-color 150ms ease, + box-shadow 150ms ease, + border-color 150ms ease; } .oauth-button-fullwidth:disabled { @@ -29,6 +33,11 @@ opacity: 0.6; } +.oauth-button-fullwidth:hover:not(:disabled) { + background-color: #fafafa; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.08); +} + .auth-dropdown-wrapper { position: relative; } @@ -72,3 +81,90 @@ color: #9c2f30; border: 2px solid currentColor; } + +/* ── Logo block ─────────────────────────────────────────────────────── */ +.auth-logo-block { + display: flex; + flex-direction: column; + align-items: center; + margin-bottom: 2rem; + margin-top: 0.5rem; + animation: authFadeUp 0.4s ease both; +} + +.auth-logo-header { + height: 8rem; + width: auto; +} + +.auth-logo-header--dark { + display: none; +} + +/* ── Page entrance animation ────────────────────────────────────────── */ +@keyframes authFadeUp { + from { + opacity: 0; + transform: translateY(12px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +/* ── Expandable trigger button state ────────────────────────────────── */ +.auth-expandable-trigger { + transition: + background-color 180ms ease, + box-shadow 180ms ease, + border-color 180ms ease; +} + +.auth-expandable-trigger--active { + border-color: #af3434 !important; + box-shadow: 0 0 0 3px rgba(175, 52, 52, 0.12); +} + +/* ── Animated expand/collapse via grid-template-rows ───────────────── */ +.auth-expand-grid { + display: grid; + grid-template-rows: 0fr; + transition: + grid-template-rows 280ms cubic-bezier(0.4, 0, 0.2, 1), + opacity 220ms ease; + opacity: 0; +} + +.auth-expand-grid--open { + grid-template-rows: 1fr; + opacity: 1; +} + +.auth-expand-inner { + overflow: hidden; + min-height: 0; +} + +/* ── Icon+label group — keeps icons vertically aligned across buttons ── */ +.oauth-btn-group { + display: inline-flex; + align-items: center; + min-width: 12.5rem; +} + +.oauth-btn-label { + flex: 1; + text-align: center; +} + +/* ── Icon helpers inside oauth-button-fullwidth ─────────────────────── */ +.auth-at-icon { + font-size: 1.25rem; + line-height: 1; + margin-right: 0.5rem; + display: inline-block; + width: 1.75rem; + text-align: center; + flex-shrink: 0; +} diff --git a/frontend/editor/src/saas/routes/login/OAuthButtons.tsx b/frontend/editor/src/saas/routes/login/OAuthButtons.tsx index e016c125e..717f96b9a 100644 --- a/frontend/editor/src/saas/routes/login/OAuthButtons.tsx +++ b/frontend/editor/src/saas/routes/login/OAuthButtons.tsx @@ -17,6 +17,7 @@ interface OAuthButtonsProps { isSubmitting: boolean; layout?: "vertical" | "grid" | "icons" | "fullwidth"; enabledProviders?: string[]; // List of enabled provider IDs from backend + labelPrefix?: string; } export default function OAuthButtons({ @@ -24,6 +25,7 @@ export default function OAuthButtons({ isSubmitting, layout = "vertical", enabledProviders: _enabledProviders = [], + labelPrefix = "", }: OAuthButtonsProps) { const { t } = useTranslation(); @@ -92,12 +94,18 @@ export default function OAuthButtons({ className="oauth-button-fullwidth" title={p.label} > - {p.label} - {p.label} + + {p.label} + + {labelPrefix} + {p.label} + + ))}
diff --git a/frontend/editor/src/saas/services/apiClient.test.ts b/frontend/editor/src/saas/services/apiClient.test.ts index 56480bb7c..45646bf8d 100644 --- a/frontend/editor/src/saas/services/apiClient.test.ts +++ b/frontend/editor/src/saas/services/apiClient.test.ts @@ -13,6 +13,15 @@ vi.mock("@app/auth/supabase", () => ({ }, })); +// Stub apiClient's heavy UI/util deps so importing it stays cheap. None are +// exercised here (toast, plan settings and handleHttpError paths aren't hit), +// and re-importing the real toast graph per test made these tests time out. +vi.mock("@app/components/toast", () => ({ alert: vi.fn() })); +vi.mock("@app/utils/appSettings", () => ({ openPlanSettings: vi.fn() })); +vi.mock("@app/services/httpErrorHandler", () => ({ + handleHttpError: vi.fn().mockResolvedValue(undefined), +})); + describe("apiClient", () => { beforeEach(() => { vi.clearAllMocks(); diff --git a/frontend/editor/src/saas/styles/saas-theme.css b/frontend/editor/src/saas/styles/saas-theme.css index 9d670c98d..2aad69c54 100644 --- a/frontend/editor/src/saas/styles/saas-theme.css +++ b/frontend/editor/src/saas/styles/saas-theme.css @@ -35,7 +35,7 @@ --auth-label-text: #2b3230; --auth-button-bg: #af3434; --auth-button-text: #ffffff; - --auth-magic-button-bg: #8b5cf6; + --auth-magic-button-bg: #af3434; --auth-magic-button-text: #ffffff; /* Light-only auth colors (no dark mode equivalents) used for login/signup */ @@ -45,7 +45,7 @@ --auth-label-text-light-only: #2b3230; --auth-button-bg-light-only: #af3434; --auth-button-text-light-only: #ffffff; - --auth-magic-button-bg-light-only: #8b5cf6; + --auth-magic-button-bg-light-only: #af3434; --auth-magic-button-text-light-only: #ffffff; --auth-bg-color-light-only: #ffffff; --auth-card-bg-light-only: #ffffff; diff --git a/testing/cucumber/features/external.feature b/testing/cucumber/features/external.feature index bf392d720..e8e8d8527 100644 --- a/testing/cucumber/features/external.feature +++ b/testing/cucumber/features/external.feature @@ -233,8 +233,7 @@ Feature: API Validation When I send the API request to the endpoint "/api/v1/convert/pdf/markdown" Then the response status code should be 200 And the response file should have size greater than 100 - And the response file should have extension ".zip" - And the response ZIP should contain 4 files + And the response file should have extension ".md" @positive @pdftocsv diff --git a/testing/test.sh b/testing/test.sh index 0aa30294a..6c7c62c4a 100644 --- a/testing/test.sh +++ b/testing/test.sh @@ -911,7 +911,7 @@ main() { CUCUMBER_JUNIT_DIR="$PROJECT_ROOT/testing/cucumber/junit" mkdir -p "$CUCUMBER_JUNIT_DIR" cd "testing/cucumber" - start_test_timer "Stirling-PDF-Regression" + start_test_timer "Stirling-PDF-Regression $CONTAINER_NAME" # Snapshot docker log line count before behave so we can extract only behave-window logs DOCKER_LOG_BEFORE=$(docker logs "$CONTAINER_NAME" 2>&1 | wc -l) @@ -999,7 +999,7 @@ main() { # Save docker logs from the behave window to a dedicated file local cucumber_log="$REPORT_DIR/cucumber-docker-context.log" docker logs "$CONTAINER_NAME" 2>&1 | tail -n +"$((DOCKER_LOG_BEFORE + 1))" > "$cucumber_log" 2>/dev/null || true - test_failure_logs["Stirling-PDF-Regression"]="$cucumber_log" + test_failure_logs["Stirling-PDF-Regression $CONTAINER_NAME"]="$cucumber_log" gha_group "Docker logs during behave run: $CONTAINER_NAME" tail -100 "$cucumber_log" @@ -1012,7 +1012,7 @@ main() { capture_file_list "$CONTAINER_NAME" "$AFTER_FILE" compare_file_lists "$BEFORE_FILE" "$AFTER_FILE" "$DIFF_FILE" "$CONTAINER_NAME" fi - stop_test_timer "Stirling-PDF-Regression" + stop_test_timer "Stirling-PDF-Regression $CONTAINER_NAME" fi # `down` with the override removes the agent bind-mount cleanly. The # SIGTERM that `down` sends is what triggers dumponexit=true in the