From ece1bb6865544ead506b9e76becf303a7c21c62d Mon Sep 17 00:00:00 2001 From: EthanHealy01 <80844253+EthanHealy01@users.noreply.github.com> Date: Thu, 14 May 2026 17:20:45 +0100 Subject: [PATCH] Feature/pdf to markdown agent (#6271) Co-authored-by: James Brunton --- app/common/build.gradle | 8 + .../SPDF/pdf/parser/CompositeTableParser.java | 73 + .../pdf/parser/LineAlignmentTableParser.java | 528 ++++++ .../software/SPDF/pdf/parser/LineBuilder.java | 139 ++ .../software/SPDF/pdf/parser/PdfIngester.java | 79 + .../software/SPDF/pdf/parser/PdfModels.java | 148 ++ .../SPDF/pdf/parser/TabulaTableParser.java | 256 +++ .../pdf/parser/WordExtractingStripper.java | 113 ++ .../parser/LineAlignmentTableParserTest.java | 153 ++ app/core/build.gradle | 6 - .../api/converters/ExtractCSVController.java | 34 +- .../software/SPDF/pdf/FlexibleCSVWriter.java | 16 - .../converters/ExtractCSVControllerTest.java | 2 + .../SPDF/pdf/FlexibleCSVWriterTest.java | 25 - app/proprietary/build.gradle | 6 - .../model/api/ai/AiWorkflowOutcome.java | 3 +- .../model/api/ai/AiWorkflowResponse.java | 10 + .../proprietary/pdf/FlexibleCSVWriter.java | 17 - .../service/AiWorkflowService.java | 24 + .../service/PdfContentExtractor.java | 115 +- .../service/AiWorkflowServiceTest.java | 25 + .../PageLayoutArtifactContractTest.java | 66 + docs/data-extraction.md | 0 engine/src/stirling/agents/__init__.py | 2 + engine/src/stirling/agents/orchestrator.py | 23 + .../agents/pdf_to_markdown/__init__.py | 3 + .../stirling/agents/pdf_to_markdown/agent.py | 435 +++++ engine/src/stirling/contracts/__init__.py | 22 + engine/src/stirling/contracts/common.py | 16 + engine/src/stirling/contracts/orchestrator.py | 8 +- .../src/stirling/contracts/pdf_to_markdown.py | 105 ++ engine/tests/test_pdf_to_markdown.py | 138 ++ frontend/package-lock.json | 1461 ++++++++++++++++- frontend/package.json | 2 + .../viewer/nonpdf/MarkdownRenderer.tsx | 108 ++ .../components/viewer/nonpdf/TextViewer.tsx | 179 +- 36 files changed, 4081 insertions(+), 267 deletions(-) create mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java create mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java create mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java create mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java create mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfModels.java create mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/TabulaTableParser.java create mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java create mode 100644 app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java delete mode 100644 app/core/src/main/java/stirling/software/SPDF/pdf/FlexibleCSVWriter.java delete mode 100644 app/core/src/test/java/stirling/software/SPDF/pdf/FlexibleCSVWriterTest.java delete mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/pdf/FlexibleCSVWriter.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java create mode 100644 docs/data-extraction.md create mode 100644 engine/src/stirling/agents/pdf_to_markdown/__init__.py create mode 100644 engine/src/stirling/agents/pdf_to_markdown/agent.py create mode 100644 engine/src/stirling/contracts/pdf_to_markdown.py create mode 100644 engine/tests/test_pdf_to_markdown.py create mode 100644 frontend/src/core/components/viewer/nonpdf/MarkdownRenderer.tsx diff --git a/app/common/build.gradle b/app/common/build.gradle index b3cafaf64..0d5586dee 100644 --- a/app/common/build.gradle +++ b/app/common/build.gradle @@ -51,4 +51,12 @@ dependencies { api 'org.simplejavamail:outlook-module:8.12.6' // MSG file support api 'jakarta.mail:jakarta.mail-api:2.1.5' runtimeOnly 'org.eclipse.angus:angus-mail:2.0.5' + + // Tabula table extraction — used by the shared PDF parser and directly by downstream modules. + // api-scoped so downstream modules (core, proprietary) retain it on their compile classpath. + api ('technology.tabula:tabula:1.0.5') { + exclude group: 'org.slf4j', module: 'slf4j-simple' + exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on' + exclude group: 'com.google.code.gson', module: 'gson' + } } diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java new file mode 100644 index 000000000..429f180f3 --- /dev/null +++ b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java @@ -0,0 +1,73 @@ +package stirling.software.SPDF.pdf.parser; + +import static stirling.software.SPDF.pdf.parser.PdfModels.*; + +import java.io.IOException; +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link + * LineAlignmentTableParser}. The first parser returning a result above {@link + * #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page. + */ +@Service +@Primary +@RequiredArgsConstructor +@Slf4j +public class CompositeTableParser implements TableParser { + + /** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */ + static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f; + + private final TabulaTableParser tabulaParser; + private final LineAlignmentTableParser lineAlignmentParser; + + @Override + public List parse(PDDocument document, RawPage rawPage) throws IOException { + // Step 1: Tabula lattice mode (ruled/bordered tables). + List latticeResults = filterConfident(tabulaParser.parse(document, rawPage)); + if (!latticeResults.isEmpty()) { + log.debug( + "Page {}: using Tabula lattice ({} table(s))", + rawPage.pageNumber(), + latticeResults.size()); + return latticeResults; + } + + // Step 2: Tabula stream mode (borderless/whitespace-delimited tables). + // parseStream is not on the TableParser interface — this intentionally couples to the + // concrete TabulaTableParser since stream mode is a Tabula-specific concept. + List streamResults = + filterConfident(tabulaParser.parseStream(document, rawPage)); + if (!streamResults.isEmpty()) { + log.debug( + "Page {}: using Tabula stream ({} table(s))", + rawPage.pageNumber(), + streamResults.size()); + return streamResults; + } + + // Step 3: Geometry-based line-alignment fallback. + List lineResults = lineAlignmentParser.parse(document, rawPage); + if (!lineResults.isEmpty()) { + log.debug( + "Page {}: using LineAlignment ({} table(s))", + rawPage.pageNumber(), + lineResults.size()); + return lineResults; + } + + return List.of(); + } + + private List filterConfident(List tables) { + return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList(); + } +} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java new file mode 100644 index 000000000..b2d8de516 --- /dev/null +++ b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java @@ -0,0 +1,528 @@ +package stirling.software.SPDF.pdf.parser; + +import static stirling.software.SPDF.pdf.parser.PdfModels.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.regex.Pattern; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.springframework.stereotype.Service; + +import lombok.extern.slf4j.Slf4j; + +/** + * Fallback {@link TableParser} for borderless financial tables using text geometry. + * + *

Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge + * positions, groups vertically proximate anchor lines into table candidates, then scores each group + * on column consistency and anchor density (confidence ceiling 0.85). + */ +@Service +@Slf4j +public class LineAlignmentTableParser implements TableParser { + + /** Width in points of each column position bucket. */ + static final float COLUMN_BUCKET_PT = 5f; + + /** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */ + private static final int COLUMN_MATCH_BUCKETS = 2; + + /** Maximum gap (as a multiple of modal line spacing) before splitting a group. */ + private static final float MAX_GAP_FACTOR = 2.5f; + + /** Minimum anchor rows (numeric-heavy) to form a valid table. */ + static final int MIN_TABLE_ROWS = 3; + + /** Minimum confirmed column positions to form a valid table. */ + static final int MIN_COLUMNS = 2; + + /** + * Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A + * rows). + */ + private static final double COLUMN_MIN_FREQUENCY = 0.40; + + /** + * Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency, + * percent, nil dashes. + */ + private static final Pattern NUMERIC = + Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$"); + + /** + * Lines within this y-distance are merged into one row (restores rows split by LineBuilder's + * column-gap logic). + */ + static final float ROW_MERGE_TOLERANCE_PT = 2f; + + // ── public API ─────────────────────────────────────────────────────────────────────────────── + + @Override + public List parse(PDDocument document, RawPage rawPage) throws IOException { + List lines = rawPage.lines(); + if (lines.size() < MIN_TABLE_ROWS) return List.of(); + + float modalSpacing = computeModalSpacing(lines); + List tokenized = + mergeCoincidentLines(lines.stream().map(this::tokenize).toList()); + + List anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList(); + + if (anchors.size() < MIN_TABLE_ROWS) return List.of(); + + List columnGrid = buildColumnGrid(anchors); + if (columnGrid.size() < MIN_COLUMNS) { + log.debug( + "Page {}: LineAlignment — fewer than {} confirmed columns, skipping", + rawPage.pageNumber(), + MIN_COLUMNS); + return List.of(); + } + + List> groups = groupRows(tokenized, columnGrid, modalSpacing); + + List results = new ArrayList<>(); + for (int i = 0; i < groups.size(); i++) { + buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i) + .ifPresent(results::add); + } + + log.debug( + "Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)", + rawPage.pageNumber(), + results.size(), + anchors.size(), + columnGrid.size()); + return results; + } + + // ── coincident-line merging ────────────────────────────────────────────────────────────────── + + /** + * Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves + * split by LineBuilder. + */ + List mergeCoincidentLines(List tokenized) { + if (tokenized.size() < 2) return tokenized; + + List result = new ArrayList<>(); + int i = 0; + + while (i < tokenized.size()) { + float baseY = tokenized.get(i).line().bounds().y(); + int j = i + 1; + while (j < tokenized.size() + && Math.abs(tokenized.get(j).line().bounds().y() - baseY) + <= ROW_MERGE_TOLERANCE_PT) { + j++; + } + + if (j == i + 1) { + result.add(tokenized.get(i)); + } else { + result.add(mergeGroup(tokenized.subList(i, j))); + } + i = j; + } + + return result; + } + + private TokenizedLine mergeGroup(List group) { + List mergedFragments = + group.stream() + .flatMap(tl -> tl.line().fragments().stream()) + .sorted(Comparator.comparingDouble(f -> f.bounds().x())) + .toList(); + + Bounds mergedBounds = + group.stream() + .map(tl -> tl.line().bounds()) + .reduce(Bounds::merge) + .orElse(group.get(0).line().bounds()); + + RawLine mergedLine = + new RawLine( + group.get(0).line().lineId(), + mergedFragments, + mergedBounds, + group.get(0).line().pageNumber()); + + return tokenize(mergedLine); + } + + // ── tokenisation ───────────────────────────────────────────────────────────────────────────── + + /** + * Splits fragments into word-level tokens; x-positions are estimated linearly within each + * fragment. + */ + TokenizedLine tokenize(RawLine line) { + List tokens = new ArrayList<>(); + for (TextFragment frag : line.fragments()) { + tokens.addAll(tokensFromFragment(frag)); + } + List numeric = tokens.stream().filter(LineToken::numeric).toList(); + return new TokenizedLine(line, tokens, numeric); + } + + private List tokensFromFragment(TextFragment frag) { + String raw = frag.text(); + if (raw == null || raw.isBlank()) return List.of(); + + float fragX = frag.bounds().x(); + float fragWidth = frag.bounds().width(); + int rawLen = raw.length(); + + List result = new ArrayList<>(); + int offset = 0; + for (String part : raw.split("\\s+")) { + if (part.isEmpty()) { + offset++; + continue; + } + int idx = raw.indexOf(part, offset); + if (idx < 0) idx = offset; + + float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX; + float tokenRight = + rawLen > 0 + ? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth + : fragX + fragWidth; + + result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches())); + offset = idx + part.length(); + } + return result; + } + + // ── column grid ────────────────────────────────────────────────────────────────────────────── + + /** + * Returns confirmed column right-edge positions — those appearing on ≥ {@value + * #COLUMN_MIN_FREQUENCY} × N anchor lines. + */ + private List buildColumnGrid(List anchors) { + // bucket → set of line indices that contributed a numeric token to that bucket + Map> bucketLines = new HashMap<>(); + for (int i = 0; i < anchors.size(); i++) { + for (LineToken t : anchors.get(i).numeric()) { + int bucket = bucket(t.right()); + bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i); + } + } + + int minHits = + Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY)); + + // Confirmed buckets → average right-edge for that bucket + TreeMap confirmed = new TreeMap<>(); + for (Map.Entry> entry : bucketLines.entrySet()) { + // Count distinct lines + long distinctLines = entry.getValue().stream().distinct().count(); + if (distinctLines >= minHits) { + double avg = + entry.getValue().stream() + .distinct() // weight each line equally regardless of token count + .mapToDouble( + lineIdx -> + avgRightEdgeForBucket( + anchors, lineIdx, entry.getKey())) + .average() + .orElse(entry.getKey() * (double) COLUMN_BUCKET_PT); + confirmed.put(entry.getKey(), (float) avg); + } + } + + return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right) + } + + /** + * Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code + * targetBucket}, falling back to the bucket's nominal centre when no tokens match. + */ + private double avgRightEdgeForBucket( + List anchors, int lineIdx, int targetBucket) { + return anchors.get(lineIdx).numeric().stream() + .filter(t -> bucket(t.right()) == targetBucket) + .mapToDouble(LineToken::right) + .average() + .orElse(targetBucket * (double) COLUMN_BUCKET_PT); + } + + // ── grouping ───────────────────────────────────────────────────────────────────────────────── + + /** + * Groups anchor lines into table candidates, including adjacent label rows; a gap > + * MAX_GAP_FACTOR × modal spacing splits groups. + */ + private List> groupRows( + List all, List columnGrid, float modalSpacing) { + float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f; + + List> groups = new ArrayList<>(); + List current = new ArrayList<>(); + + for (int i = 0; i < all.size(); i++) { + TokenizedLine tl = all.get(i); + boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid); + + if (current.isEmpty()) { + if (fits) current.add(tl); + continue; + } + + float gap = + tl.line().bounds().y() + - current.get(current.size() - 1).line().bounds().bottom(); + + if (gap > maxGap) { + groups.add(current); + current = new ArrayList<>(); + if (fits) current.add(tl); + continue; + } + + if (fits) { + current.add(tl); + } else if (!tl.line().text().isBlank()) { + // Include non-anchor lines (labels) only if they have text and are within + // proximity. + current.add(tl); + } + } + + if (!current.isEmpty()) groups.add(current); + + return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList(); + } + + private boolean hasEnoughAnchorRows(List group, List columnGrid) { + return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count() + >= MIN_TABLE_ROWS; + } + + /** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */ + private boolean matchesGrid(TokenizedLine tl, List columnGrid) { + if (tl.numeric().isEmpty()) return false; + long matches = + tl.numeric().stream() + .filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0) + .count(); + return (double) matches / tl.numeric().size() >= 0.60; + } + + private boolean hasInconsistentColumnMatch(TokenizedLine tl, List columnGrid) { + if (tl.numeric().isEmpty()) return false; + long hits = + tl.numeric().stream() + .filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0) + .count(); + return (double) hits / tl.numeric().size() < 0.60; + } + + // ── fragment assembly ──────────────────────────────────────────────────────────────────────── + + private Optional buildFragment( + List group, List columnGrid, int pageNumber, int tableIndex) { + + long anchorCount = + group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count(); + if (anchorCount < MIN_TABLE_ROWS) return Optional.empty(); + + List warnings = new ArrayList<>(); + List> rawRows = new ArrayList<>(); + List rows = new ArrayList<>(); + + for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) { + TokenizedLine tl = group.get(rowIdx); + List rawRow = buildRawRow(tl, columnGrid); + rawRows.add(Collections.unmodifiableList(rawRow)); + rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid)); + } + + // Column count = 1 label column + confirmed numeric columns + int colCount = columnGrid.size() + 1; + Bounds bounds = computeGroupBounds(group); + float confidence = computeConfidence(group, columnGrid, warnings); + + return Optional.of( + new TableFragment( + "tbl-la-p" + pageNumber + "-" + tableIndex, + pageNumber, + bounds, + List.of(), + Collections.unmodifiableList(rows), + Collections.unmodifiableList(rawRows), + colCount, + confidence, + Collections.unmodifiableList(warnings), + null)); + } + + /** + * Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values. + */ + private List buildRawRow(TokenizedLine tl, List columnGrid) { + String[] cells = new String[columnGrid.size() + 1]; + Arrays.fill(cells, ""); + + // Separate label tokens (those not landing in any confirmed column) from column tokens. + List labelParts = new ArrayList<>(); + for (LineToken token : tl.all()) { + int col = nearestColumnIndex(token.right(), columnGrid); + if (col >= 0 && token.numeric()) { + int cellIdx = col + 1; + cells[cellIdx] = + cells[cellIdx].isEmpty() + ? token.text() + : cells[cellIdx] + " " + token.text(); + } else { + labelParts.add(token.text()); + } + } + cells[0] = String.join(" ", labelParts).trim(); + return Arrays.asList(cells); + } + + private TableRow buildTableRow( + int rowIdx, TokenizedLine tl, List rawRow, List columnGrid) { + List cells = new ArrayList<>(rawRow.size()); + + // Label cell: use the line's full bounds as an approximation. + cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds())); + + for (int col = 0; col < columnGrid.size(); col++) { + String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : ""; + float right = columnGrid.get(col); + float left = col > 0 ? columnGrid.get(col - 1) : right - 50f; + Bounds cellBounds = + new Bounds( + left, + tl.line().bounds().y(), + right - left, + tl.line().bounds().height()); + cells.add(TableCell.of(col + 1, text, cellBounds)); + } + return new TableRow(rowIdx, Collections.unmodifiableList(cells)); + } + + // ── confidence scoring ─────────────────────────────────────────────────────────────────────── + + /** + * Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at + * 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if + * >30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors. + */ + private float computeConfidence( + List group, List columnGrid, List warnings) { + float score = 0.70f; + + long anchorCount = + group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count(); + long totalRows = group.size(); + + // More columns + int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2); + score += extraCols * 0.05f; + + // More anchor rows + if (anchorCount >= 5) score += 0.05f; + if (anchorCount >= 8) score += 0.05f; + + // Inconsistent column matching + long inconsistent = + group.stream() + .filter(TokenizedLine::isAnchor) + .filter(tl -> hasInconsistentColumnMatch(tl, columnGrid)) + .count(); + if (inconsistent > anchorCount * 0.30) { + score -= 0.15f; + warnings.add( + "Column match inconsistent on " + + inconsistent + + "/" + + anchorCount + + " anchor rows"); + } + + // Label-heavy + long nonAnchor = totalRows - anchorCount; + if (nonAnchor > anchorCount) { + score -= 0.10f; + warnings.add( + "Non-anchor rows (" + + nonAnchor + + ") outnumber anchor rows (" + + anchorCount + + ")"); + } + + return Math.max(0f, Math.min(0.85f, score)); + } + + // ── utility ────────────────────────────────────────────────────────────────────────────────── + + /** + * Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value + * #COLUMN_MATCH_BUCKETS} buckets. + */ + private int nearestColumnIndex(float rightEdge, List grid) { + int nearest = -1; + float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f; + for (int i = 0; i < grid.size(); i++) { + float dist = Math.abs(rightEdge - grid.get(i)); + if (dist < minDist) { + minDist = dist; + nearest = i; + } + } + return nearest; + } + + private Bounds computeGroupBounds(List group) { + return group.stream() + .map(tl -> tl.line().bounds()) + .reduce(Bounds::merge) + .orElse(new Bounds(0, 0, 0, 0)); + } + + /** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */ + private float computeModalSpacing(List lines) { + if (lines.size() < 2) return 0f; + Map freq = new HashMap<>(); + for (int i = 1; i < lines.size(); i++) { + float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom(); + if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum); + } + return freq.entrySet().stream() + .max(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .orElse(0f); + } + + private static int bucket(float x) { + return Math.round(x / COLUMN_BUCKET_PT); + } + + // ── private data types ─────────────────────────────────────────────────────────────────────── + + /** A word-level token with an approximate right-edge x-position. */ + record LineToken(String text, float x, float right, boolean numeric) {} + + /** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */ + record TokenizedLine(RawLine line, List all, List numeric) { + boolean isAnchor() { + return numeric.size() >= 2; + } + } +} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java new file mode 100644 index 000000000..6831f6d73 --- /dev/null +++ b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java @@ -0,0 +1,139 @@ +package stirling.software.SPDF.pdf.parser; + +import static stirling.software.SPDF.pdf.parser.PdfModels.*; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.springframework.stereotype.Service; + +import lombok.extern.slf4j.Slf4j; + +/** + * Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity. + * + *

Fragments are on the same line when their baselines are within a font-size-derived tolerance. + * A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code + * max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text. + */ +@Service +@Slf4j +public class LineBuilder { + + /** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */ + private static final float BASELINE_TOLERANCE_FACTOR = 0.5f; + + /** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */ + private static final float MIN_BASELINE_TOLERANCE = 2f; + + /** + * Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below + * two-column gutters. + */ + static final float COLUMN_GAP_RATIO = 0.10f; + + /** Floor for the column-gap threshold so narrow pages don't over-split lines. */ + static final float COLUMN_GAP_MIN_PT = 40f; + + public List build(List fragments, int pageNumber) { + if (fragments.isEmpty()) return List.of(); + + float effectiveWidth = inferEffectiveWidth(fragments); + float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT); + log.debug( + "LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt", + pageNumber, + effectiveWidth, + columnGapThreshold); + + // Sort top-to-bottom first, then left-to-right within the same baseline band. + List sorted = + fragments.stream() + .sorted( + Comparator.comparingDouble(TextFragment::baseline) + .thenComparingDouble(f -> f.bounds().x())) + .toList(); + + List> groups = groupByBaseline(sorted, columnGapThreshold); + + List lines = new ArrayList<>(groups.size()); + for (int i = 0; i < groups.size(); i++) { + List group = + groups.get(i).stream() + .sorted(Comparator.comparingDouble(f -> f.bounds().x())) + .toList(); + + Bounds lineBounds = + group.stream() + .map(TextFragment::bounds) + .reduce(Bounds::merge) + .orElse(new Bounds(0, 0, 0, 0)); + + lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber)); + } + return lines; + } + + private List> groupByBaseline( + List sorted, float columnGapThreshold) { + List> groups = new ArrayList<>(); + List current = new ArrayList<>(); + float currentBaseline = Float.NaN; + + for (TextFragment fragment : sorted) { + if (current.isEmpty()) { + current.add(fragment); + currentBaseline = fragment.baseline(); + continue; + } + + float maxFontSize = + Math.max( + fragment.fontSize(), + (float) + current.stream() + .mapToDouble(TextFragment::fontSize) + .max() + .orElse(0)); + float tolerance = + Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE); + + boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance; + boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold); + + if (sameBaseline && !columnGap) { + current.add(fragment); + // Anchor to the weighted mean baseline so long lines stay stable. + currentBaseline = + (currentBaseline * (current.size() - 1) + fragment.baseline()) + / current.size(); + } else { + groups.add(current); + current = new ArrayList<>(); + current.add(fragment); + currentBaseline = fragment.baseline(); + } + } + + if (!current.isEmpty()) groups.add(current); + return groups; + } + + /** + * True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code + * threshold}. + */ + private static boolean hasColumnGap( + TextFragment next, List group, float threshold) { + float lastRight = group.get(group.size() - 1).bounds().right(); + return next.bounds().x() - lastRight > threshold; + } + + /** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */ + private static float inferEffectiveWidth(List fragments) { + double maxRight = + fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0); + return (float) maxRight * 1.10f; + } +} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java new file mode 100644 index 000000000..a7dc9c282 --- /dev/null +++ b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java @@ -0,0 +1,79 @@ +package stirling.software.SPDF.pdf.parser; + +import static stirling.software.SPDF.pdf.parser.PdfModels.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} → + * {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the + * {@link PDDocument} lifecycle. + */ +@Service +@RequiredArgsConstructor +@Slf4j +public class PdfIngester { + + private final LineBuilder lineBuilder; + private final TableParser tableParser; + + public List parse(PDDocument document) throws IOException { + return parse(document, document.getNumberOfPages()); + } + + public List parse(PDDocument document, int maxPages) throws IOException { + int pageCount = Math.min(document.getNumberOfPages(), maxPages); + List pages = new ArrayList<>(pageCount); + long fragmentsMs = 0; + long tablesMs = 0; + long t0 = System.currentTimeMillis(); + + for (int p = 1; p <= pageCount; p++) { + long ft = System.currentTimeMillis(); + List fragments = extractFragments(document, p); + fragmentsMs += System.currentTimeMillis() - ft; + + PDPage page = document.getPage(p - 1); + PDRectangle mediaBox = page.getMediaBox(); + List lines = lineBuilder.build(fragments, p); + RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines); + + long tt = System.currentTimeMillis(); + List tables = tableParser.parse(document, rawPage); + tablesMs += System.currentTimeMillis() - tt; + + log.debug( + "Page {}: {} fragments → {} lines, {} table(s)", + p, + fragments.size(), + lines.size(), + tables.size()); + pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines)); + } + + log.info( + "[timing] parse pages={} total={}ms fragments={}ms tables={}ms", + pageCount, + System.currentTimeMillis() - t0, + fragmentsMs, + tablesMs); + return pages; + } + + private List extractFragments(PDDocument document, int pageNumber) + throws IOException { + WordExtractingStripper stripper = new WordExtractingStripper(pageNumber); + stripper.getText(document); + return stripper.getFragments(); + } +} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfModels.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfModels.java new file mode 100644 index 000000000..cc26cb01f --- /dev/null +++ b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfModels.java @@ -0,0 +1,148 @@ +package stirling.software.SPDF.pdf.parser; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.pdfbox.pdmodel.PDDocument; + +/** + * All PDF parser model types and the table-parser contract in one place. + * + *

Import as {@code import static stirling.software.SPDF.pdf.parser.PdfModels.*;} to use all + * nested types without qualification. + */ +public final class PdfModels { + + private PdfModels() {} + + // ── Geometry ────────────────────────────────────────────────────────────── + + public record Bounds(float x, float y, float width, float height) { + + public float right() { + return x + width; + } + + public float bottom() { + return y + height; + } + + public static Bounds merge(Bounds a, Bounds b) { + float x = Math.min(a.x, b.x); + float y = Math.min(a.y, b.y); + return new Bounds( + x, y, Math.max(a.right(), b.right()) - x, Math.max(a.bottom(), b.bottom()) - y); + } + } + + // ── Text fragments and lines ────────────────────────────────────────────── + + /** + * A contiguous run of text from a single PDF content-stream string operation. {@code baseline} + * is preserved separately from bounds for line-grouping — characters of different sizes on the + * same visual line share a baseline but differ in top Y. + */ + public record TextFragment( + String fragmentId, + String text, + Bounds bounds, + float baseline, + float fontSize, + String fontName, + boolean bold) {} + + public record RawLine( + String lineId, List fragments, Bounds bounds, int pageNumber) { + + public String text() { + if (fragments.isEmpty()) return ""; + StringBuilder sb = new StringBuilder(); + TextFragment prev = null; + for (TextFragment f : fragments) { + if (prev != null) { + float gap = f.bounds().x() - prev.bounds().right(); + float avgCharWidth = prev.bounds().width() / Math.max(prev.text().length(), 1); + if (gap > avgCharWidth * 0.5f) sb.append(' '); + } + sb.append(f.text()); + prev = f; + } + return sb.toString(); + } + + public float dominantFontSize() { + return fragments.stream() + .collect(Collectors.groupingBy(TextFragment::fontSize, Collectors.counting())) + .entrySet() + .stream() + .max(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .orElse(0f); + } + + public boolean hasBold() { + return fragments.stream().anyMatch(TextFragment::bold); + } + } + + public record RawPage(int pageNumber, float widthPt, float heightPt, List lines) {} + + // ── Table model ─────────────────────────────────────────────────────────── + + /** + * A single cell within a {@link TableRow}. {@code colSpan} and {@code rowSpan} are always 1 in + * v1 — span detection is deferred but the fields keep the schema stable. + */ + public record TableCell(int colIndex, String text, Bounds bounds, int colSpan, int rowSpan) { + + public static TableCell of(int colIndex, String text, Bounds bounds) { + return new TableCell(colIndex, text, bounds, 1, 1); + } + } + + public record TableRow(int rowIndex, List cells) {} + + /** + * A table as it appears on a single page. {@code headers} is empty in v1 — all rows are in + * {@code rows}. {@code rawRows} preserves exact Tabula text output for diagnostics. {@code + * confidence} is a heuristic score in [0.0, 1.0]. {@code continuedFromPage} is null in v1. + */ + public record TableFragment( + String tableId, + int pageNumber, + Bounds bounds, + List headers, + List rows, + List> rawRows, + int columnCount, + float confidence, + List warnings, + Integer continuedFromPage) {} + + // ── Page output ─────────────────────────────────────────────────────────── + + public record ParsedPage( + int pageNumber, + float widthPt, + float heightPt, + List tables, + List layoutLines) {} + + // ── Parser contract ─────────────────────────────────────────────────────── + + /** + * Extracts tables from a single page of a PDF. The caller owns the document lifecycle; + * implementations must not close it. + */ + public interface TableParser { + + /** + * @param document open PDF; must not be closed by the implementation + * @param rawPage page metadata and lines for the page to process + * @return zero or more table fragments found on the page, never null + */ + List parse(PDDocument document, RawPage rawPage) throws IOException; + } +} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/TabulaTableParser.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/TabulaTableParser.java new file mode 100644 index 000000000..b85ddbb08 --- /dev/null +++ b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/TabulaTableParser.java @@ -0,0 +1,256 @@ +package stirling.software.SPDF.pdf.parser; + +import static stirling.software.SPDF.pdf.parser.PdfModels.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.springframework.stereotype.Service; + +import lombok.extern.slf4j.Slf4j; + +import technology.tabula.ObjectExtractor; +import technology.tabula.Page; +import technology.tabula.RectangularTextContainer; +import technology.tabula.Table; +import technology.tabula.extractors.BasicExtractionAlgorithm; +import technology.tabula.extractors.ExtractionAlgorithm; +import technology.tabula.extractors.SpreadsheetExtractionAlgorithm; + +/** + * Primary {@link TableParser} implementation using Tabula's lattice-mode extraction. + * + *

Algorithm

+ * + * Uses {@link SpreadsheetExtractionAlgorithm} (lattice mode), which detects tables from ruled lines + * (horizontal and vertical PDF path operators). This is reliable for tables with visible borders. + * Borderless or whitespace-delimited tables will not be detected — that requires stream mode, which + * is deferred to a later iteration. + * + *

Coordinate system

+ * + * Tabula normalises page coordinates so that (0,0) is the top-left corner and Y increases downward, + * matching the {@link Bounds} convention used throughout this parser. No coordinate transformation + * is needed when mapping Tabula cell bounds to {@link Bounds}. + * + *

Known limitations

+ * + *
    + *
  • Borderless tables are not detected (use stream mode, deferred). + *
  • Colspan and rowspan are not detected; all cells report colSpan=1, rowSpan=1. + *
  • Header rows are not identified; all rows appear in {@code rows}, headers is empty. + *
  • Cross-page table linking is not performed; each page is independent. + *
  • Rotated tables (90°/270° pages) may produce incorrect bounds. + *
+ */ +@Service +@Slf4j +public class TabulaTableParser implements TableParser { + + /** Lattice mode — reliable for tables with visible ruled borders. */ + @Override + public List parse(PDDocument document, RawPage rawPage) throws IOException { + return parseWithAlgorithm( + document, rawPage, new SpreadsheetExtractionAlgorithm(), "lattice"); + } + + /** + * Convenience overload for callers that only have a page number, not a full {@link RawPage}. + */ + public List parse(PDDocument document, int pageNumber) throws IOException { + return parse(document, new RawPage(pageNumber, 0f, 0f, List.of())); + } + + /** Stream mode — whitespace-based column detection for borderless tables. */ + public List parseStream(PDDocument document, RawPage rawPage) + throws IOException { + return parseWithAlgorithm(document, rawPage, new BasicExtractionAlgorithm(), "stream"); + } + + private List parseWithAlgorithm( + PDDocument document, RawPage rawPage, ExtractionAlgorithm algorithm, String modeName) + throws IOException { + int pageNumber = rawPage.pageNumber(); + + List tabulaTables; + try { + // Do NOT use try-with-resources: ObjectExtractor.close() closes the underlying + // PDDocument, which we don't own. The extractor holds no resources of its own. + ObjectExtractor extractor = new ObjectExtractor(document); + Page page = extractor.extract(pageNumber); + tabulaTables = new ArrayList<>(algorithm.extract(page)); + } catch (Exception e) { + log.warn( + "Tabula {} extraction failed on page {}: {}", + modeName, + pageNumber, + e.getMessage()); + return List.of(); + } + + if (tabulaTables.isEmpty()) { + log.debug("Page {}: no tables detected by Tabula ({})", pageNumber, modeName); + return List.of(); + } + + log.debug( + "Page {}: Tabula ({}) detected {} table(s)", + pageNumber, + modeName, + tabulaTables.size()); + + List fragments = new ArrayList<>(tabulaTables.size()); + for (int i = 0; i < tabulaTables.size(); i++) { + fragments.add(toFragment(tabulaTables.get(i), pageNumber, i)); + } + return fragments; + } + + // ── private helpers ────────────────────────────────────────────────────────────────────────── + + private TableFragment toFragment(Table table, int pageNumber, int tableIndex) { + List> tabulaRows = table.getRows(); + List warnings = new ArrayList<>(); + + List> rawRows = buildRawRows(tabulaRows); + int colCount = inferColumnCount(rawRows, warnings); + List rows = buildRows(tabulaRows, colCount, warnings); + float confidence = computeConfidence(rawRows, colCount, warnings); + Bounds bounds = tableBounds(table); + + String tableId = "tbl-p" + pageNumber + "-" + tableIndex; + + if (!warnings.isEmpty()) { + log.warn("Page {}, table {}: {}", pageNumber, tableIndex, warnings); + } + + return new TableFragment( + tableId, + pageNumber, + bounds, + List.of(), // headers: deferred to v2 + rows, + rawRows, + colCount, + confidence, + Collections.unmodifiableList(warnings), + null); // continuedFromPage: deferred to v2 + } + + private List> buildRawRows(List> tabulaRows) { + List> rawRows = new ArrayList<>(tabulaRows.size()); + for (List tabulaRow : tabulaRows) { + List cells = new ArrayList<>(tabulaRow.size()); + for (RectangularTextContainer cell : tabulaRow) { + cells.add(normaliseText(cell.getText())); + } + rawRows.add(Collections.unmodifiableList(cells)); + } + return rawRows; + } + + private List buildRows( + List> tabulaRows, int colCount, List warnings) { + List rows = new ArrayList<>(tabulaRows.size()); + for (int rowIdx = 0; rowIdx < tabulaRows.size(); rowIdx++) { + List tabulaRow = tabulaRows.get(rowIdx); + List cells = new ArrayList<>(tabulaRow.size()); + + for (int colIdx = 0; colIdx < tabulaRow.size(); colIdx++) { + RectangularTextContainer c = tabulaRow.get(colIdx); + Bounds cellBounds = + new Bounds( + (float) c.getX(), + (float) c.getY(), + (float) c.getWidth(), + (float) c.getHeight()); + cells.add(TableCell.of(colIdx, normaliseText(c.getText()), cellBounds)); + } + + if (tabulaRow.size() != colCount) { + warnings.add( + "Row " + + rowIdx + + " has " + + tabulaRow.size() + + " cells; expected " + + colCount); + } + + rows.add(new TableRow(rowIdx, Collections.unmodifiableList(cells))); + } + return rows; + } + + /** + * The canonical column count for a table is the size of the widest row. Tabula can produce + * uneven rows when a cell's ruling lines are partially missing. + */ + private int inferColumnCount(List> rawRows, List warnings) { + if (rawRows.isEmpty()) return 0; + int max = rawRows.stream().mapToInt(List::size).max().orElse(0); + int mode = + rawRows.stream() + .collect( + java.util.stream.Collectors.groupingBy( + List::size, java.util.stream.Collectors.counting())) + .entrySet() + .stream() + .max(java.util.Map.Entry.comparingByValue()) + .map(java.util.Map.Entry::getKey) + .orElse(0); + if (max != mode) { + warnings.add("Inconsistent column count: modal=" + mode + " max=" + max); + } + return mode > 0 ? mode : max; + } + + /** + * Heuristic confidence score in [0.0, 1.0]. + * + *
    + *
  • Starts at 1.0. + *
  • -0.3 if only one column (single-column tables are usually not real tables). + *
  • -0.1 per row with an inconsistent column count, capped at -0.4. + *
  • -0.3 if the empty-cell ratio across all cells exceeds 80%. + *
+ */ + private float computeConfidence( + List> rawRows, int colCount, List warnings) { + if (rawRows.isEmpty() || colCount == 0) return 0f; + + float score = 1.0f; + + if (colCount == 1) score -= 0.3f; + + long inconsistentRows = rawRows.stream().filter(r -> r.size() != colCount).count(); + score -= Math.min(inconsistentRows * 0.1f, 0.4f); + + long totalCells = rawRows.stream().mapToLong(List::size).sum(); + long emptyCells = + rawRows.stream().flatMap(Collection::stream).filter(String::isBlank).count(); + if (totalCells > 0 && (float) emptyCells / totalCells > 0.8f) { + score -= 0.3f; + } + + return Math.max(0f, Math.min(1f, score)); + } + + private Bounds tableBounds(Table table) { + return new Bounds( + (float) table.getX(), + (float) table.getY(), + (float) table.getWidth(), + (float) table.getHeight()); + } + + private String normaliseText(String raw) { + if (raw == null) return ""; + // Tabula wraps multi-line cell content with \r\n — collapse to a single space. + return raw.replace("\r\n", " ").replace("\n", " ").replace("\r", " ").trim(); + } +} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java new file mode 100644 index 000000000..52ab9d9a1 --- /dev/null +++ b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java @@ -0,0 +1,113 @@ +package stirling.software.SPDF.pdf.parser; + +import static stirling.software.SPDF.pdf.parser.PdfModels.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.font.PDFont; +import org.apache.pdfbox.text.PDFTextStripper; +import org.apache.pdfbox.text.TextPosition; + +/** + * Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata. + * + *

Overrides {@link #writeString} to split each content-stream string into word-level {@link + * TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in + * PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline. + */ +class WordExtractingStripper extends PDFTextStripper { + + private final int targetPage; + private final List fragments = new ArrayList<>(); + private int fragmentIndex = 0; + + WordExtractingStripper(int pageNumber) throws IOException { + this.targetPage = pageNumber; + setStartPage(pageNumber); + setEndPage(pageNumber); + setSortByPosition(true); + } + + @Override + protected void startPage(PDPage page) throws IOException { + super.startPage(page); + fragments.clear(); + fragmentIndex = 0; + } + + @Override + protected void writeString(String text, List textPositions) throws IOException { + if (text == null || text.isBlank()) return; + + // Fast path: no whitespace → emit one fragment (most financial PDFs have each + // number as its own string operation, so this is the common case). + if (text.indexOf(' ') < 0) { + emitFragment(text, textPositions); + return; + } + + // Per-word splitting requires 1:1 text-char to TextPosition correspondence. + // Fall back to one fragment when sizes differ (ligatures, encoding edge cases). + if (textPositions.size() != text.length()) { + emitFragment(text, textPositions); + return; + } + + // Emit one TextFragment per whitespace-delimited word with accurate per-word bounds. + int start = 0; + for (int i = 0; i <= text.length(); i++) { + if (i == text.length() || text.charAt(i) == ' ') { + if (start < i) { + emitFragment(text.substring(start, i), textPositions.subList(start, i)); + } + start = i + 1; + } + } + } + + private void emitFragment(String text, List positions) { + if (positions.isEmpty()) return; + + float minX = Float.MAX_VALUE; + float minY = Float.MAX_VALUE; + float maxRight = -Float.MAX_VALUE; + float maxBaseline = -Float.MAX_VALUE; + TextPosition first = null; + + for (TextPosition tp : positions) { + if (tp == null) continue; + if (first == null) first = tp; + + float x = tp.getX(); + // getY() is the baseline; top of character = getY() - getHeight(). + float top = tp.getY() - tp.getHeight(); + float right = x + tp.getWidth(); + float baseline = tp.getY(); + + minX = Math.min(minX, x); + minY = Math.min(minY, top); + maxRight = Math.max(maxRight, right); + maxBaseline = Math.max(maxBaseline, baseline); + } + + if (first == null) return; + + PDFont font = first.getFont(); + String fontName = font != null ? font.getName() : ""; + boolean bold = fontName != null && fontName.toLowerCase().contains("bold"); + // getHeight() gives the rendered glyph height, which is the most reliable visual size. + float fontSize = first.getHeight(); + + Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY); + String id = "tf-p" + targetPage + "-" + fragmentIndex++; + fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold)); + } + + List getFragments() { + return Collections.unmodifiableList(fragments); + } +} diff --git a/app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java b/app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java new file mode 100644 index 000000000..fbbf5af9c --- /dev/null +++ b/app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java @@ -0,0 +1,153 @@ +package stirling.software.SPDF.pdf.parser; + +import static org.assertj.core.api.Assertions.assertThat; +import static stirling.software.SPDF.pdf.parser.PdfModels.*; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and + * column-grid construction. + */ +class LineAlignmentTableParserTest { + + private final LineAlignmentTableParser parser = new LineAlignmentTableParser(); + + // ── mergeCoincidentLines ───────────────────────────────────────────────────────────────────── + + @Test + void mergeCoincidentLines_singleLine_unchanged() { + var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue"))); + assertThat(parser.mergeCoincidentLines(lines)).hasSize(1); + } + + @Test + void mergeCoincidentLines_distinctYLines_unchanged() { + // Two lines at different y positions — must NOT be merged. + var lines = + List.of( + tokenized(rawLine(10f, 100f, "Revenue")), + tokenized(rawLine(10f, 115f, "Cost"))); + assertThat(parser.mergeCoincidentLines(lines)).hasSize(2); + } + + @Test + void mergeCoincidentLines_sameY_merged() { + // Simulates a financial-table row split by LineBuilder at the column gap: + // label fragment at x=72 → "Revenue" + // value fragment at x=350 → "1,234" + // Both have y=100. After merge they should form one TokenizedLine. + var label = rawLine(72f, 100f, "Revenue"); + var value = rawLine(350f, 100f, "1,234"); + + var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value))); + + assertThat(merged).hasSize(1); + // The merged line should contain tokens from both halves. + var tokens = merged.get(0).all(); + assertThat(tokens.stream().map(t -> t.text()).toList()) + .containsExactlyInAnyOrder("Revenue", "1,234"); + } + + @Test + void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() { + var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114 + var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380 + + var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value))); + + var bounds = merged.get(0).line().bounds(); + assertThat(bounds.x()).isEqualTo(72f); + assertThat(bounds.right()).isEqualTo(380f); + } + + @Test + void mergeCoincidentLines_withinTolerance_merged() { + // Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge. + var a = rawLine(10f, 100.0f, "Alpha"); + var b = rawLine(200f, 101.5f, "99"); + + var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b))); + assertThat(merged).hasSize(1); + } + + @Test + void mergeCoincidentLines_beyondTolerance_notMerged() { + // Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge. + var a = rawLine(10f, 100.0f, "Alpha"); + var b = rawLine(200f, 103.0f, "99"); + + var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b))); + assertThat(merged).hasSize(2); + } + + @Test + void mergeCoincidentLines_threeCoincident_allMerged() { + // Three fragments at the same y (e.g. wide financial table with two value columns). + var a = rawLine(72f, 100f, "Revenue"); + var b = rawLine(300f, 100f, "1,234"); + var c = rawLine(400f, 100f, "5,678"); + + var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c))); + assertThat(merged).hasSize(1); + assertThat(merged.get(0).all()).hasSize(3); + } + + @Test + void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() { + var a = rawLine(72f, 100f, "Revenue"); + var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a + var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate + + var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c))); + assertThat(merged).hasSize(2); + } + + @Test + void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() { + // After merging, the combined line should be an anchor (≥2 numeric tokens). + // "Revenue" alone → not an anchor. "1,234 567" alone → anchor. + // Merged → anchor with at least 2 numerics. + var label = rawLine(72f, 100f, "Revenue"); + var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f); + + var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values))); + + assertThat(merged).hasSize(1); + assertThat(merged.get(0).isAnchor()).isTrue(); + } + + // ── helpers ────────────────────────────────────────────────────────────────────────────────── + + /** Creates a RawLine with a single TextFragment of the given text at the given position. */ + private static RawLine rawLine(float x, float y, String text) { + float width = text.length() * 6f; // ~6pt per char — rough but consistent + float height = 12f; + Bounds bounds = new Bounds(x, y, width, height); + TextFragment fragment = + new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false); + return new RawLine("ln-test", List.of(fragment), bounds, 1); + } + + /** + * Creates a RawLine with two TextFragments representing two words separated by a small gap. + * Used to simulate a values-only line with multiple numeric tokens. + */ + private static RawLine rawLineMultiWord( + float x, float y, String word1, float w1, String word2, float w2) { + float height = 12f; + Bounds b1 = new Bounds(x, y, w1, height); + Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height); + TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false); + TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false); + Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height); + return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1); + } + + /** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */ + private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) { + return parser.tokenize(line); + } +} diff --git a/app/core/build.gradle b/app/core/build.gradle index 068320cca..7a30f4766 100644 --- a/app/core/build.gradle +++ b/app/core/build.gradle @@ -87,12 +87,6 @@ dependencies { implementation 'com.sun.xml.bind:jaxb-core:4.0.7' implementation 'org.apache.poi:poi-ooxml:5.5.1' - // https://mvnrepository.com/artifact/technology.tabula/tabula - implementation ('technology.tabula:tabula:1.0.5') { - exclude group: 'org.slf4j', module: 'slf4j-simple' - exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on' - exclude group: 'com.google.code.gson', module: 'gson' - } // CVE-2022-25647: Explicit gson 2.13.2 to prevent unsafe deserialization (tabula would pull 2.8.7) implementation 'com.google.code.gson:gson:2.13.2' implementation 'org.apache.pdfbox:jbig2-imageio:3.0.4' diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractCSVController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractCSVController.java index 8603cec9f..81dc1c358 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractCSVController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractCSVController.java @@ -4,13 +4,13 @@ import java.io.ByteArrayOutputStream; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.QuoteMode; import org.apache.pdfbox.pdmodel.PDDocument; import org.springframework.http.ContentDisposition; @@ -26,24 +26,21 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.SPDF.config.swagger.CsvConversionResponse; import stirling.software.SPDF.model.api.PDFWithPageNums; -import stirling.software.SPDF.pdf.FlexibleCSVWriter; +import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment; +import stirling.software.SPDF.pdf.parser.TabulaTableParser; import stirling.software.common.annotations.AutoJobPostMapping; import stirling.software.common.annotations.api.ConvertApi; import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.util.GeneralUtils; import stirling.software.common.util.WebResponseUtils; -import technology.tabula.ObjectExtractor; -import technology.tabula.Page; -import technology.tabula.Table; -import technology.tabula.extractors.SpreadsheetExtractionAlgorithm; - @ConvertApi @Slf4j @RequiredArgsConstructor public class ExtractCSVController { private final CustomPDFDocumentFactory pdfDocumentFactory; + private final TabulaTableParser tabulaTableParser; @AutoJobPostMapping(value = "/pdf/csv", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) @CsvConversionResponse @@ -58,24 +55,23 @@ public class ExtractCSVController { try (PDDocument document = pdfDocumentFactory.load(request)) { List pages = request.getPageNumbersList(document, true); - SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm(); CSVFormat format = CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build(); for (int pageNum : pages) { - try (ObjectExtractor extractor = new ObjectExtractor(document)) { - log.info("{}", pageNum); - Page page = extractor.extract(pageNum); - List

tables = sea.extract(page); + log.info("{}", pageNum); + List fragments = tabulaTableParser.parse(document, pageNum); - for (int i = 0; i < tables.size(); i++) { - StringWriter sw = new StringWriter(); - FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format); - csvWriter.write(sw, Collections.singletonList(tables.get(i))); - - String entryName = generateEntryName(baseName, pageNum, i + 1); - csvEntries.add(new CsvEntry(entryName, sw.toString())); + for (int i = 0; i < fragments.size(); i++) { + StringWriter sw = new StringWriter(); + try (CSVPrinter printer = format.print(sw)) { + for (List row : fragments.get(i).rawRows()) { + printer.printRecord(row); + } } + csvEntries.add( + new CsvEntry( + generateEntryName(baseName, pageNum, i + 1), sw.toString())); } } diff --git a/app/core/src/main/java/stirling/software/SPDF/pdf/FlexibleCSVWriter.java b/app/core/src/main/java/stirling/software/SPDF/pdf/FlexibleCSVWriter.java deleted file mode 100644 index 94a48d935..000000000 --- a/app/core/src/main/java/stirling/software/SPDF/pdf/FlexibleCSVWriter.java +++ /dev/null @@ -1,16 +0,0 @@ -package stirling.software.SPDF.pdf; - -import org.apache.commons.csv.CSVFormat; - -import technology.tabula.writers.CSVWriter; - -public class FlexibleCSVWriter extends CSVWriter { - - public FlexibleCSVWriter() { - super(); - } - - public FlexibleCSVWriter(CSVFormat csvFormat) { - super(csvFormat); - } -} diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ExtractCSVControllerTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ExtractCSVControllerTest.java index c460fe7ee..a3c83c04f 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ExtractCSVControllerTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/converters/ExtractCSVControllerTest.java @@ -20,6 +20,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.mock.web.MockMultipartFile; import stirling.software.SPDF.model.api.PDFWithPageNums; +import stirling.software.SPDF.pdf.parser.TabulaTableParser; import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.util.GeneralUtils; @@ -27,6 +28,7 @@ import stirling.software.common.util.GeneralUtils; class ExtractCSVControllerTest { @Mock private CustomPDFDocumentFactory pdfDocumentFactory; + @Mock private TabulaTableParser tabulaTableParser; @InjectMocks private ExtractCSVController controller; diff --git a/app/core/src/test/java/stirling/software/SPDF/pdf/FlexibleCSVWriterTest.java b/app/core/src/test/java/stirling/software/SPDF/pdf/FlexibleCSVWriterTest.java deleted file mode 100644 index 77637ece5..000000000 --- a/app/core/src/test/java/stirling/software/SPDF/pdf/FlexibleCSVWriterTest.java +++ /dev/null @@ -1,25 +0,0 @@ -package stirling.software.SPDF.pdf; - -import static org.junit.jupiter.api.Assertions.assertNotNull; - -import org.apache.commons.csv.CSVFormat; -import org.junit.jupiter.api.Test; - -class FlexibleCSVWriterTest { - - @Test - void testDefaultConstructor() { - FlexibleCSVWriter writer = new FlexibleCSVWriter(); - assertNotNull(writer, "The FlexibleCSVWriter instance should not be null"); - } - - @Test - void testConstructorWithCSVFormat() { - CSVFormat csvFormat = CSVFormat.DEFAULT; - FlexibleCSVWriter writer = new FlexibleCSVWriter(csvFormat); - assertNotNull( - writer, - "The FlexibleCSVWriter instance should not be null when initialized with" - + " CSVFormat"); - } -} diff --git a/app/proprietary/build.gradle b/app/proprietary/build.gradle index f32efa75d..e79a44024 100644 --- a/app/proprietary/build.gradle +++ b/app/proprietary/build.gradle @@ -59,12 +59,6 @@ dependencies { // https://mvnrepository.com/artifact/com.bucket4j/bucket4j_jdk17 implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion" - // Tabula table extraction — used by MathAuditorOrchestrator - implementation ('technology.tabula:tabula:1.0.5') { - exclude group: 'org.slf4j', module: 'slf4j-simple' - exclude group: 'org.bouncycastle', module: 'bcprov-jdk15on' - exclude group: 'com.google.code.gson', module: 'gson' - } implementation 'com.google.code.gson:gson:2.13.2' api 'io.micrometer:micrometer-registry-prometheus' diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java index ee2d5a584..a7239e8f9 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java @@ -20,7 +20,8 @@ public enum AiWorkflowOutcome { TOOL_CALL("tool_call"), COMPLETED("completed"), UNSUPPORTED_CAPABILITY("unsupported_capability"), - CANNOT_CONTINUE("cannot_continue"); + CANNOT_CONTINUE("cannot_continue"), + GENERATE_FILE("generate_file"); private final String value; diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResponse.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResponse.java index d1cb186f7..8f0fc631b 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResponse.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowResponse.java @@ -4,6 +4,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonProperty; + import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; @@ -20,6 +22,14 @@ public class AiWorkflowResponse { @Schema(description = "Answer returned by the AI workflow when applicable") private String answer; + @JsonProperty("content") + @Schema(description = "Text content to package as a file (generate_file outcomes)") + private String generatedContent; + + @JsonProperty("filename") + @Schema(description = "Desired output filename for generate_file outcomes") + private String generatedFilename; + @Schema(description = "Summary returned by the AI workflow when applicable") private String summary; diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/FlexibleCSVWriter.java b/app/proprietary/src/main/java/stirling/software/proprietary/pdf/FlexibleCSVWriter.java deleted file mode 100644 index a22357a7e..000000000 --- a/app/proprietary/src/main/java/stirling/software/proprietary/pdf/FlexibleCSVWriter.java +++ /dev/null @@ -1,17 +0,0 @@ -package stirling.software.proprietary.pdf; - -import org.apache.commons.csv.CSVFormat; - -import technology.tabula.writers.CSVWriter; - -/** Exposes Tabula's protected {@link CSVWriter#CSVWriter(CSVFormat)} constructor. */ -public class FlexibleCSVWriter extends CSVWriter { - - public FlexibleCSVWriter() { - super(); - } - - public FlexibleCSVWriter(CSVFormat csvFormat) { - super(csvFormat); - } -} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java index b0352fe20..e219aedd4 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java @@ -160,6 +160,7 @@ public class AiWorkflowService { case TOOL_CALL -> onToolCall(response, filesById, listener); case PLAN -> onPlan(response, filesById, request, listener); case ANSWER -> onAnswer(response, filesById, request, listener); + case GENERATE_FILE -> onGenerateFile(response, listener); case NOT_FOUND, NEED_CLARIFICATION, CANNOT_DO, @@ -364,6 +365,29 @@ public class AiWorkflowService { return new WorkflowState.Terminal(response); } + private WorkflowState onGenerateFile(AiWorkflowResponse response, ProgressListener listener) + throws IOException { + String content = response.getGeneratedContent(); + String filename = response.getGeneratedFilename(); + if (content == null || filename == null || filename.isBlank()) { + return new WorkflowState.Terminal( + cannotContinue( + "AI engine returned generate_file without content or filename.")); + } + listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.PROCESSING)); + String safeFilename = Filenames.toSimpleFileName(filename); + byte[] bytes = content.getBytes(java.nio.charset.StandardCharsets.UTF_8); + org.springframework.core.io.Resource resource = + new org.springframework.core.io.ByteArrayResource(bytes) { + @Override + public String getFilename() { + return safeFilename; + } + }; + return new WorkflowState.Terminal( + buildCompletedResponse(response.getSummary(), List.of(resource), List.of(), null)); + } + @SuppressWarnings("unchecked") private WorkflowState runPlan( List> steps, diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java index 8e89cada3..c7badf99c 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java @@ -3,7 +3,6 @@ package stirling.software.proprietary.service; import java.io.IOException; import java.io.StringWriter; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -12,6 +11,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.QuoteMode; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.text.PDFTextStripper; @@ -21,25 +21,30 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonValue; import lombok.Data; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import stirling.software.SPDF.pdf.parser.PdfIngester; +import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage; +import stirling.software.SPDF.pdf.parser.PdfModels.RawLine; +import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment; +import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment; +import stirling.software.SPDF.pdf.parser.TabulaTableParser; import stirling.software.common.util.ExceptionUtils; import stirling.software.common.util.PdfUtils; import stirling.software.proprietary.model.api.ai.AiPdfContentType; import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest; import stirling.software.proprietary.model.api.ai.AiWorkflowTextSelection; import stirling.software.proprietary.model.api.ai.FolioType; -import stirling.software.proprietary.pdf.FlexibleCSVWriter; - -import technology.tabula.ObjectExtractor; -import technology.tabula.Page; -import technology.tabula.Table; -import technology.tabula.extractors.SpreadsheetExtractionAlgorithm; @Slf4j @Service +@RequiredArgsConstructor public class PdfContentExtractor { + private final TabulaTableParser tabulaTableParser; + private final PdfIngester pdfIngester; + private static final int MAX_CHARACTERS_PER_PAGE = 4_000; private static final int TEXT_PRESENCE_THRESHOLD = 20; @@ -101,21 +106,21 @@ public class PdfContentExtractor { * @return list of CSV strings (one per table), empty if no tables found */ public List extractTablesAsCsv(PDDocument document, int pageNumber) throws IOException { - SpreadsheetExtractionAlgorithm sea = new SpreadsheetExtractionAlgorithm(); + List fragments = tabulaTableParser.parse(document, pageNumber); + if (fragments.isEmpty()) return List.of(); + CSVFormat format = CSVFormat.EXCEL.builder().setEscape('"').setQuoteMode(QuoteMode.ALL).build(); List csvStrings = new ArrayList<>(); - try (ObjectExtractor extractor = new ObjectExtractor(document)) { - Page tabulaPage = extractor.extract(pageNumber); - List
tables = sea.extract(tabulaPage); - - for (Table table : tables) { - StringWriter sw = new StringWriter(); - FlexibleCSVWriter csvWriter = new FlexibleCSVWriter(format); - csvWriter.write(sw, Collections.singletonList(table)); - csvStrings.add(sw.toString()); + for (TableFragment fragment : fragments) { + StringWriter sw = new StringWriter(); + try (CSVPrinter printer = format.print(sw)) { + for (List row : fragment.rawRows()) { + printer.printRecord(row); + } } + csvStrings.add(sw.toString()); } return csvStrings; } @@ -183,6 +188,8 @@ public class PdfContentExtractor { case PAGE_TEXT, FULL_TEXT -> Optional.ofNullable( extractText(lf, fileReq, remainingPages, remainingCharacters)); + case PAGE_LAYOUT -> + Optional.ofNullable(extractPageLayout(lf, remainingPages)); default -> { log.warn( "Content type {} not yet implemented, skipping for {}", @@ -207,6 +214,35 @@ public class PdfContentExtractor { return extracted.isEmpty() ? null : buildExtractedFileText(lf.fileName(), extracted); } + private PageLayoutFileResult extractPageLayout(LoadedFile lf, int maxPages) throws IOException { + List parsedPages = pdfIngester.parse(lf.document(), maxPages); + List pages = new ArrayList<>(); + for (ParsedPage pp : parsedPages) { + if (pp.layoutLines().isEmpty()) continue; + List lines = new ArrayList<>(); + for (RawLine rawLine : pp.layoutLines()) { + List fragments = new ArrayList<>(); + for (TextFragment tf : rawLine.fragments()) { + fragments.add( + new LayoutFragment( + tf.text(), + tf.bounds().x(), + tf.bounds().y(), + tf.bounds().width(), + tf.fontSize(), + tf.bold())); + } + lines.add(new LayoutLine(rawLine.bounds().y(), fragments)); + } + pages.add(new LayoutPage(pp.pageNumber(), lines)); + } + if (pages.isEmpty()) return null; + PageLayoutFileResult result = new PageLayoutFileResult(); + result.setFileName(lf.fileName()); + result.setPages(pages); + return result; + } + private WorkflowArtifact buildArtifact(ArtifactKind kind, List results) { return switch (kind) { case EXTRACTED_TEXT -> { @@ -214,10 +250,12 @@ public class PdfContentExtractor { artifact.setFiles(results.stream().map(ExtractedFileText.class::cast).toList()); yield artifact; } + case PAGE_LAYOUT -> { + PageLayoutArtifact artifact = new PageLayoutArtifact(); + artifact.setFiles(results.stream().map(PageLayoutFileResult.class::cast).toList()); + yield artifact; + } case TOOL_REPORT -> - // TOOL_REPORT artifacts don't come from PDF content extraction — they're - // built by AiWorkflowService from tool-response metadata. Never reached - // from this code path; presence in the enum is to satisfy the switch. throw new IllegalArgumentException( "TOOL_REPORT artifacts are not produced by PdfContentExtractor"); }; @@ -331,6 +369,7 @@ public class PdfContentExtractor { */ enum ArtifactKind { EXTRACTED_TEXT("extracted_text"), + PAGE_LAYOUT("page_layout"), TOOL_REPORT("tool_report"); private final String value; @@ -394,4 +433,40 @@ public class PdfContentExtractor { this.report = report; } } + + // Serialization contract with the Python engine — see PageLayoutArtifactContractTest. + + /** One text fragment with its bounding-box geometry and font properties. */ + record LayoutFragment( + String text, float x, float y, float width, float fontSize, boolean bold) {} + + /** A visual line on the page: y-coordinate and all fragments on that line. */ + record LayoutLine(float y, List fragments) {} + + /** All layout lines for a single page. */ + record LayoutPage(int pageNumber, List lines) {} + + /** Page layout data for one file, as a PdfContentResult. */ + @Data + static final class PageLayoutFileResult implements PdfContentResult { + private String fileName; + private List pages = new ArrayList<>(); + + @Override + public ArtifactKind getArtifactKind() { + return ArtifactKind.PAGE_LAYOUT; + } + + @Override + public int pagesConsumed() { + return pages.size(); + } + } + + /** Artifact carrying full spatial page layout for all input files. */ + @Data + static final class PageLayoutArtifact implements WorkflowArtifact { + private final ArtifactKind kind = ArtifactKind.PAGE_LAYOUT; + private List files = new ArrayList<>(); + } } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java index b3693aa58..5c0939663 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java @@ -408,6 +408,31 @@ class AiWorkflowServiceTest { verify(internalApiClient, times(1)).post(eq(COMPRESS_ENDPOINT), any()); } + @Test + void generateFileStoresContentDirectlyWithoutToolCall() throws IOException { + MockMultipartFile input = pdf("report.pdf", "bytes"); + stubOrchestrator( + """ + { + "outcome":"generate_file", + "content":"# Hello\\n\\nWorld", + "filename":"report-reconstruction.md", + "summary":"Reconstructed the document as a Markdown file." + } + """); + AtomicInteger ids = stubFileStorage(); + + AiWorkflowResponse result = service.orchestrate(requestFor(input, "convert to markdown")); + + assertEquals(AiWorkflowOutcome.COMPLETED, result.getOutcome()); + assertEquals(1, result.getResultFiles().size()); + assertEquals("report-reconstruction.md", result.getResultFiles().get(0).getFileName()); + assertEquals("file-1", result.getResultFiles().get(0).getFileId()); + assertEquals(1, ids.get()); + // No tool endpoint should be called — content goes directly to file storage. + verify(internalApiClient, never()).post(anyString(), any()); + } + @Test void toolCallWithoutEndpointFallsBackToCannotContinue() throws IOException { MockMultipartFile input = pdf("input.pdf", "bytes"); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java new file mode 100644 index 000000000..ae853b2e6 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java @@ -0,0 +1,66 @@ +package stirling.software.proprietary.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import stirling.software.proprietary.service.PdfContentExtractor.LayoutFragment; +import stirling.software.proprietary.service.PdfContentExtractor.LayoutLine; +import stirling.software.proprietary.service.PdfContentExtractor.LayoutPage; +import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutArtifact; +import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutFileResult; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.json.JsonMapper; + +/** + * Contract test: verifies that {@link PageLayoutArtifact} serializes to the JSON field names that + * the Python engine expects in {@code engine/src/stirling/contracts/pdf_to_markdown.py}. + * + *

The companion Python test in {@code tests/test_pdf_to_markdown.py} deserializes the same JSON + * literal and asserts field values. If either side renames a field, one of these tests fails. + */ +class PageLayoutArtifactContractTest { + + static final String CONTRACT_JSON = + """ + {"kind":"page_layout","files":[{"fileName":"test.pdf","pages":[{"pageNumber":1,"lines":[{"y":10.0,"fragments":[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}"""; + + @Test + void pageLayoutArtifact_serialisesToExpectedJson() throws Exception { + LayoutFragment fragment = new LayoutFragment("Hello", 1.0f, 2.0f, 30.0f, 12.0f, true); + LayoutLine line = new LayoutLine(10.0f, List.of(fragment)); + LayoutPage page = new LayoutPage(1, List.of(line)); + + PageLayoutFileResult fileResult = new PageLayoutFileResult(); + fileResult.setFileName("test.pdf"); + fileResult.setPages(List.of(page)); + + PageLayoutArtifact artifact = new PageLayoutArtifact(); + artifact.setFiles(List.of(fileResult)); + + JsonNode json = new JsonMapper().valueToTree(artifact); + + assertEquals("page_layout", json.get("kind").asText()); + + JsonNode file = json.get("files").get(0); + assertEquals("test.pdf", file.get("fileName").asText()); + + JsonNode pg = file.get("pages").get(0); + assertEquals(1, pg.get("pageNumber").asInt()); + + JsonNode ln = pg.get("lines").get(0); + assertEquals(10.0, ln.get("y").asDouble(), 0.001); + + JsonNode frag = ln.get("fragments").get(0); + assertEquals("Hello", frag.get("text").asText()); + assertEquals(1.0, frag.get("x").asDouble(), 0.001); + assertEquals(2.0, frag.get("y").asDouble(), 0.001); + assertEquals(30.0, frag.get("width").asDouble(), 0.001); + assertEquals(12.0, frag.get("fontSize").asDouble(), 0.001); + assertTrue(frag.get("bold").asBoolean()); + } +} diff --git a/docs/data-extraction.md b/docs/data-extraction.md new file mode 100644 index 000000000..e69de29bb diff --git a/engine/src/stirling/agents/__init__.py b/engine/src/stirling/agents/__init__.py index 5410ac098..cddd0275c 100644 --- a/engine/src/stirling/agents/__init__.py +++ b/engine/src/stirling/agents/__init__.py @@ -5,6 +5,7 @@ from .orchestrator import OrchestratorAgent from .pdf_edit import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection from .pdf_questions import PdfQuestionAgent from .pdf_review import PdfReviewAgent +from .pdf_to_markdown import PdfToMarkdownAgent from .user_spec import UserSpecAgent __all__ = [ @@ -15,5 +16,6 @@ __all__ = [ "PdfEditPlanSelection", "PdfQuestionAgent", "PdfReviewAgent", + "PdfToMarkdownAgent", "UserSpecAgent", ] diff --git a/engine/src/stirling/agents/orchestrator.py b/engine/src/stirling/agents/orchestrator.py index 6dedd5079..5de8bc957 100644 --- a/engine/src/stirling/agents/orchestrator.py +++ b/engine/src/stirling/agents/orchestrator.py @@ -11,12 +11,14 @@ from pydantic_ai.tools import RunContext from stirling.agents.pdf_edit import PdfEditAgent from stirling.agents.pdf_questions import PdfQuestionAgent from stirling.agents.pdf_review import PdfReviewAgent +from stirling.agents.pdf_to_markdown import PdfToMarkdownAgent from stirling.agents.user_spec import UserSpecAgent from stirling.contracts import ( AgentDraftWorkflowResponse, ExtractedTextArtifact, OrchestratorRequest, OrchestratorResponse, + PageLayoutArtifact, PdfEditResponse, PdfQuestionOrchestrateResponse, SupportedCapability, @@ -25,6 +27,7 @@ from stirling.contracts import ( format_file_names, ) from stirling.contracts.pdf_edit import EditPlanResponse +from stirling.contracts.pdf_to_markdown import PdfToMarkdownOrchestrateResponse from stirling.services import AppRuntime logger = logging.getLogger(__name__) @@ -68,6 +71,11 @@ class OrchestratorAgent: " feedback')." ), ), + ToolOutput( + self.delegate_pdf_to_markdown, + name="delegate_pdf_to_markdown", + description=("Delegate requests to reconstruct a PDF as a Markdown document."), + ), ToolOutput( self.unsupported_capability, name="unsupported_capability", @@ -84,6 +92,8 @@ class OrchestratorAgent: "Use delegate_pdf_review when the user wants the PDF returned with review" " comments attached — anything like 'review this', 'annotate with comments'," " 'leave feedback on the PDF'. " + "Use delegate_pdf_to_markdown for any request to convert a PDF to Markdown " + "or reconstruct its content as readable text. " "Use unsupported_capability when the user asks about the assistant itself " "or when none of the other outputs fit; supply a helpful message." ), @@ -123,6 +133,8 @@ class OrchestratorAgent: return await self._run_pdf_edit(request) case SupportedCapability.AGENT_DRAFT: return await self._run_agent_draft(request) + case SupportedCapability.PDF_TO_MARKDOWN: + return await self._run_pdf_to_markdown(request) case ( SupportedCapability.ORCHESTRATE | SupportedCapability.AGENT_REVISE @@ -151,6 +163,12 @@ class OrchestratorAgent: async def _run_agent_draft(self, request: OrchestratorRequest) -> AgentDraftWorkflowResponse: return await UserSpecAgent(self.runtime).orchestrate(request) + async def delegate_pdf_to_markdown(self, ctx: RunContext[OrchestratorDeps]) -> PdfToMarkdownOrchestrateResponse: + return await self._run_pdf_to_markdown(ctx.deps.request) + + async def _run_pdf_to_markdown(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse: + return await PdfToMarkdownAgent(self.runtime).orchestrate(request) + async def delegate_pdf_review(self, ctx: RunContext[OrchestratorDeps]) -> EditPlanResponse: return await self._run_pdf_review(ctx.deps.request) @@ -186,5 +204,10 @@ class OrchestratorAgent: file_names = [f.file_name for f in artifact.files] descriptions.append(f"- extracted_text: {total_pages} pages from {file_names}") continue + if isinstance(artifact, PageLayoutArtifact): + total_pages = sum(len(f.pages) for f in artifact.files) + file_names = [f.file_name for f in artifact.files] + descriptions.append(f"- page_layout: {total_pages} pages from {file_names}") + continue descriptions.append("- unknown artifact") return "\n".join(descriptions) diff --git a/engine/src/stirling/agents/pdf_to_markdown/__init__.py b/engine/src/stirling/agents/pdf_to_markdown/__init__.py new file mode 100644 index 000000000..d35ae05c7 --- /dev/null +++ b/engine/src/stirling/agents/pdf_to_markdown/__init__.py @@ -0,0 +1,3 @@ +from .agent import PdfToMarkdownAgent + +__all__ = ["PdfToMarkdownAgent"] diff --git a/engine/src/stirling/agents/pdf_to_markdown/agent.py b/engine/src/stirling/agents/pdf_to_markdown/agent.py new file mode 100644 index 000000000..8c0d7d8ee --- /dev/null +++ b/engine/src/stirling/agents/pdf_to_markdown/agent.py @@ -0,0 +1,435 @@ +"""PDF to Markdown Agent. + +Converts a parsed PDF document into a single clean Markdown document, preserving +headings, paragraphs, and tables in reading order. +""" + +from __future__ import annotations + +import asyncio +import logging +import re +import time + +from pydantic import BaseModel, Field +from pydantic_ai import Agent +from pydantic_ai.output import NativeOutput + +from stirling.contracts import ( + EditCannotDoResponse, + GenerateFileResponse, + NeedContentFileRequest, + NeedContentResponse, + OrchestratorRequest, + PdfContentType, + SupportedCapability, + format_conversation_history, +) +from stirling.contracts.pdf_to_markdown import ( + PageLayout, + PageLayoutArtifact, + PdfToMarkdownCannotDoResponse, + PdfToMarkdownOrchestrateResponse, + PdfToMarkdownRequest, + PdfToMarkdownResponse, + PdfToMarkdownSuccessResponse, +) +from stirling.services import AppRuntime + +logger = logging.getLogger(__name__) + + +# Warn when output tokens are close to the typical model output limit (~8192 for most +# configurations). The actual limit is model-specific; this threshold catches likely truncation. +_OUTPUT_TOKEN_TRUNCATION_THRESHOLD = 7500 + +# Chunking limits — keep each LLM call to a manageable payload size. +# Fragment count is the primary driver of JSON payload size (each fragment carries x/y/width/ +# fontSize/bold metadata beyond its text). Page cap prevents low-text pages accumulating. +_MAX_CHUNK_FRAGMENTS = 1_000 +_MAX_CHUNK_PAGES = 10 + +# Max concurrent LLM calls — limits API rate pressure on large documents. +_MAX_PARALLEL_CHUNKS = 3 + +# ── LLM output model ──────────────────────────────────────────────────────────────────────────── + + +class _ReconstructionOutput(BaseModel): + markdown: str = Field(description="Full document reconstructed as clean Markdown.") + + +# ── Agent ──────────────────────────────────────────────────────────────────────────────────────── + + +class PdfToMarkdownAgent: + def __init__(self, runtime: AppRuntime) -> None: + self.runtime = runtime + self._sem = asyncio.Semaphore(_MAX_PARALLEL_CHUNKS) + self._reconstruct_agent = Agent( + model=runtime.smart_model, + output_type=NativeOutput(_ReconstructionOutput), + system_prompt=( + "You reconstruct PDF pages into clean Markdown from spatial fragment data.\n" + "Input: PAGE LAYOUT — per-fragment x/y/font data for structural analysis.\n\n" + "COLUMN DETECTION (for tables in page_layout):\n" + "- Look at the x-positions of fragments across 3+ consecutive lines.\n" + "- If fragments cluster at the same x-positions across multiple lines, those are table columns.\n" + "- Each distinct x-cluster is one column." + " Name them from the header row (the first line in the cluster).\n" + "- Do NOT merge values from different x-columns into one cell.\n\n" + "ROW DETECTION:\n" + "- Each unique y-coordinate (or group within 3pt) is one table row.\n" + "- Every line of layout data is its own row — do not merge rows.\n" + "- If a column has no fragment on a given y-row, that cell is empty.\n\n" + "TABLE RENDERING:\n" + "- Render as: | col1 | col2 | col3 |\n" + " | --- | --- | --- |\n" + " | val | val | val |\n" + "- One source row = one table row. Never collapse multiple rows into one.\n" + "- Preserve numeric values exactly (no rounding, no formatting changes).\n" + "- Bold cells: wrap with ** in the Markdown cell.\n" + "- CRITICAL: the separator row `| --- | --- |` appears EXACTLY ONCE per table, immediately\n" + " after the header row. NEVER put `| --- |` after a data row or between data rows.\n" + " NEVER put a blank line inside a table. All rows (header + data) must be consecutive.\n" + "- Do NOT produce a header-only table followed by a second table with the data rows.\n" + " One logical table = one markdown table block, with header, one separator, then all data.\n\n" + "GROUP HEADERS (label-only rows inside a table):\n" + "- A row is a group header when: the first column has text AND every numeric column is empty.\n" + "- Do NOT render group headers as table rows with empty cells.\n" + "- Break the table, emit the label as **bold text** on its own line," + " then start a new table for the rows that follow.\n" + "- Example labels: 'Policy functions', 'Non-current assets'.\n\n" + "TOTAL AND SUBTOTAL ROWS:\n" + "- Detect rows whose first cell contains (case-insensitive):" + " total, subtotal, surplus, balance, net, sum.\n" + "- These rows have numeric content — they are NOT group headers.\n" + "- Render the entire row in bold: | **Total income** | **1,234** | **5,678** |\n" + "- Keep total rows attached to the group they summarise.\n\n" + "MULTI-LEVEL TABLES (year or period as a row label):\n" + "- Detect when a row contains only a single label (a year like '2010' or period like 'Q1 2023')" + " with no numeric content, followed by repeated metric rows.\n" + "- Do NOT render the year as a table row.\n" + "- Normalise: add 'Year' as the first column, 'Metric' as the second," + " and repeat the year value on each metric row.\n\n" + "PROSE REGIONS:\n" + "- Lines where x-positions vary across lines (not repeating columns) are prose.\n" + "- Merge lines at the same x-level into paragraphs. Separate indented lines.\n\n" + "HEADINGS:\n" + "- A line is a heading when it is bold OR font_size ≥2pt above body.\n" + " CRITICAL EXCEPTION: a bold fragment is a TABLE HEADER CELL, not a document heading, when\n" + " the same y-row in page_layout contains other fragments at different x-positions.\n" + " Only classify a bold line as a document heading when it is the SOLE fragment on its y-row.\n" + " Example: 'Non-current assets' at y=120 with '2010'@x=350, '2009'@x=420, '2008'@x=490\n" + " → this is a table header row, NOT a heading. Render it as the first cell of the table.\n" + "- Use ## for section headings, ### for sub-headings. Use # only for the document title.\n\n" + "ORDERING:\n" + "- Process content top-to-bottom as it appears on the page.\n" + "- Interleave prose blocks and table blocks in page order.\n" + "- Do not move text that appears before a table to after it, or vice versa.\n\n" + "FIDELITY:\n" + "- Do NOT invent, summarise, or omit any content.\n" + "- Do NOT add commentary, metadata, or JSON — output Markdown only." + ), + model_settings={ + **runtime.smart_model_settings, + "temperature": 0.0, + "max_tokens": _OUTPUT_TOKEN_TRUNCATION_THRESHOLD, + }, + ) + + async def orchestrate(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse: + """Entry point for the orchestrator delegate. + + First turn: requests PAGE_LAYOUT extraction from Java via NeedContentResponse. + Resume turn: runs the LLM reconstruction and returns a write-file plan step. + """ + layout_artifact = next( + (a for a in request.artifacts if isinstance(a, PageLayoutArtifact)), + None, + ) + if layout_artifact is None: + return NeedContentResponse( + resume_with=SupportedCapability.PDF_TO_MARKDOWN, + reason="Page layout data is required to reconstruct the document.", + files=[ + NeedContentFileRequest(file=f, content_types=[PdfContentType.PAGE_LAYOUT]) for f in request.files + ], + max_pages=self.runtime.settings.max_pages, + max_characters=self.runtime.settings.max_characters, + ) + + page_layout = [page for entry in layout_artifact.files for page in entry.pages] + file_names = [f.name for f in request.files] + result = await self.handle( + PdfToMarkdownRequest( + user_message=request.user_message, + file_names=file_names, + conversation_history=request.conversation_history, + page_layout=page_layout, + ) + ) + if isinstance(result, PdfToMarkdownCannotDoResponse): + return EditCannotDoResponse(reason=result.reason) + + base = file_names[0].rsplit(".", 1)[0] if file_names else "document" + return GenerateFileResponse( + content=result.markdown, + filename=f"{base}-reconstruction.md", + summary="Reconstructed the document as a Markdown file.", + ) + + async def handle(self, request: PdfToMarkdownRequest) -> PdfToMarkdownResponse: + total_fragments = sum(len(line.fragments) for page in request.page_layout for line in page.lines) + logger.info( + "[pdf-to-markdown] received layout-pages=%d fragments=%d", + len(request.page_layout), + total_fragments, + ) + + if not request.page_layout: + logger.warning("[pdf-to-markdown] no content extracted from document; returning cannot_do") + return PdfToMarkdownCannotDoResponse( + reason=( + "No content was extracted from the document. " + "The file may be a scanned image PDF with no readable text. " + "Try running OCR on the document first." + ) + ) + + chunks = _build_page_chunks(request.page_layout) + logger.info("[pdf-to-markdown] chunks=%d (max %d in parallel)", len(chunks), _MAX_PARALLEL_CHUNKS) + + if len(chunks) == 1: + return await self._reconstruct_chunk(request, chunks[0], chunk_num=1, total_chunks=1) + + total = len(chunks) + results = await asyncio.gather( + *( + self._reconstruct_chunk(request, chunk, chunk_num=i + 1, total_chunks=total) + for i, chunk in enumerate(chunks) + ) + ) + + markdown_parts: list[str] = [] + for result in results: + if isinstance(result, PdfToMarkdownSuccessResponse) and result.markdown: + markdown_parts.append(result.markdown) + elif isinstance(result, PdfToMarkdownCannotDoResponse): + logger.warning("[pdf-to-markdown] chunk dropped: %s", result.reason) + + if not markdown_parts: + return PdfToMarkdownCannotDoResponse(reason="The document could not be reconstructed. All chunks failed.") + + logger.info("[pdf-to-markdown] assembly: %d/%d chunks produced output", len(markdown_parts), len(chunks)) + return PdfToMarkdownSuccessResponse(markdown="\n\n".join(markdown_parts)) + + async def _reconstruct_chunk( + self, + request: PdfToMarkdownRequest, + pages: list[PageLayout], + chunk_num: int, + total_chunks: int, + ) -> PdfToMarkdownResponse: + chunk_request = PdfToMarkdownRequest( + user_message=request.user_message, + file_names=request.file_names, + conversation_history=request.conversation_history, + page_layout=pages, + ) + try: + async with self._sem: + return await self._reconstruct_document(chunk_request, chunk_num, total_chunks) + except Exception as e: + logger.error("[pdf-to-markdown] chunk %d/%d failed: %s", chunk_num, total_chunks, e, exc_info=True) + return PdfToMarkdownCannotDoResponse( + reason="The document could not be reconstructed. The AI model failed to process it." + ) + + async def _reconstruct_document( + self, request: PdfToMarkdownRequest, chunk_num: int = 1, total_chunks: int = 1 + ) -> PdfToMarkdownSuccessResponse: + content = _build_reconstruction_prompt(request) + logger.info("[timing] chunk %d/%d llm-call prompt-chars=%d", chunk_num, total_chunks, len(content)) + t0 = time.monotonic() + result = await self._reconstruct_agent.run([content]) + llm_ms = int((time.monotonic() - t0) * 1000) + output: _ReconstructionOutput = result.output + usage = result.usage() + logger.info( + "[timing] chunk %d/%d llm-done ms=%d input-tokens=%s output-tokens=%s markdown-chars=%d", + chunk_num, + total_chunks, + llm_ms, + usage.input_tokens, + usage.output_tokens, + len(output.markdown), + ) + if usage.output_tokens and usage.output_tokens >= _OUTPUT_TOKEN_TRUNCATION_THRESHOLD: + logger.warning( + "[timing] chunk %d/%d output likely truncated (output-tokens=%d)", + chunk_num, + total_chunks, + usage.output_tokens, + ) + markdown = _remove_extra_separators(_fix_markdown_tables(_merge_orphaned_table_rows(output.markdown))) + return PdfToMarkdownSuccessResponse(markdown=markdown) + + +# ── Chunking ──────────────────────────────────────────────────────────────────────────────────── + + +def _build_page_chunks(pages: list[PageLayout]) -> list[list[PageLayout]]: + chunks: list[list[PageLayout]] = [] + current: list[PageLayout] = [] + current_fragments = 0 + for page in pages: + page_fragments = sum(len(line.fragments) for line in page.lines) + fragment_full = current and current_fragments + page_fragments > _MAX_CHUNK_FRAGMENTS + page_full = len(current) >= _MAX_CHUNK_PAGES + if fragment_full or page_full: + chunks.append(current) + current = [] + current_fragments = 0 + current.append(page) + current_fragments += page_fragments + if current: + chunks.append(current) + return chunks + + +# ── Prompt builders (module-level, no state) ──────────────────────────────────────────────────── + + +def _build_reconstruction_prompt(request: PdfToMarkdownRequest) -> str: + history = format_conversation_history(request.conversation_history) + file_names = ", ".join(request.file_names) if request.file_names else "Unknown files" + layout_section = _format_layout(request.page_layout) + + return ( + f"Files: {file_names}\n\n" + f"User request: {request.user_message}\n\n" + f"Conversation history:\n{history}\n\n" + "PAGE LAYOUT (structural source — x/y fragment positions):\n" + "Each line is: y=NNN | text@(x,y) fs=N text@(x,y) fs=N ...\n" + "- y=NNN is the vertical position (row). Lines close in y are the same visual row.\n" + "- x=NNN is the horizontal position (column). Consistent x across rows = a column.\n" + "- fs=N is font size. Larger = likely a heading.\n" + "- **bold** markers indicate bold text.\n\n" + f"{layout_section}" + ) + + +# ── LLM output post-processing ────────────────────────────────────────────────────────────────── + + +def _fix_markdown_tables(markdown: str) -> str: + """Remove blank lines between table rows produced by the LLM.""" + lines = markdown.split("\n") + result: list[str] = [] + i = 0 + while i < len(lines): + result.append(lines[i]) + if lines[i].strip().startswith("|"): + j = i + 1 + while j < len(lines) and lines[j].strip() == "": + j += 1 + if j < len(lines) and lines[j].strip().startswith("|"): + i = j + continue + i += 1 + return "\n".join(result) + + +_SEP_CELL = re.compile(r"^:?-+:?$") + + +def _is_sep_row(line: str) -> bool: + """Return True when a pipe row is a Markdown table separator (| --- | --- |).""" + stripped = line.strip() + if not stripped.startswith("|"): + return False + cells = [c.strip() for c in stripped.split("|") if c.strip()] + return bool(cells) and all(_SEP_CELL.match(c) for c in cells) + + +def _merge_orphaned_table_rows(markdown: str) -> str: + """Merge pipe-row blocks that lack a separator into the preceding table. + + When the LLM incorrectly breaks a table (e.g. on a false group-header), it emits + orphaned pipe rows with no header or separator. These are invalid markdown and get + merged back into the preceding table, discarding the intervening non-table content. + """ + lines = markdown.split("\n") + + segments: list[tuple[str, list[str]]] = [] + i = 0 + while i < len(lines): + if lines[i].strip().startswith("|"): + block: list[str] = [] + while i < len(lines) and lines[i].strip().startswith("|"): + block.append(lines[i]) + i += 1 + has_sep = any(_is_sep_row(row) for row in block) + segments.append(("table" if has_sep else "orphan", block)) + else: + block = [] + while i < len(lines) and not lines[i].strip().startswith("|"): + block.append(lines[i]) + i += 1 + segments.append(("prose", block)) + + result: list[tuple[str, list[str]]] = [] + last_table_idx: int | None = None + for seg_type, seg_lines in segments: + if seg_type == "orphan": + if last_table_idx is not None: + result = result[: last_table_idx + 1] + result[-1] = ("table", result[-1][1] + seg_lines) + else: + result.append((seg_type, seg_lines)) + else: + if seg_type == "table": + last_table_idx = len(result) + result.append((seg_type, seg_lines)) + + return "\n".join(line for _, seg_lines in result for line in seg_lines) + + +def _remove_extra_separators(markdown: str) -> str: + """Within each contiguous table block, keep only the first separator row.""" + lines = markdown.split("\n") + result: list[str] = [] + seen_sep = False + + for line in lines: + if not line.strip().startswith("|"): + seen_sep = False + result.append(line) + continue + if _is_sep_row(line): + if seen_sep: + continue + seen_sep = True + result.append(line) + + return "\n".join(result) + + +# ── Formatting helpers (module-level, no state) ────────────────────────────────────────────────── + + +def _format_layout(pages: list[PageLayout]) -> str: + if not pages: + return "None" + parts: list[str] = [] + for page in pages: + line_strs: list[str] = [] + for line in page.lines: + frags = " ".join( + f"{'**' if f.bold else ''}{f.text}{'**' if f.bold else ''}@({f.x:.0f},{f.y:.0f}) fs={f.font_size:.0f}" + for f in line.fragments + ) + line_strs.append(f"y={line.y:.0f} | {frags}") + parts.append(f"--- Page {page.page_number} ---\n" + "\n".join(line_strs)) + return "\n\n".join(parts) diff --git a/engine/src/stirling/contracts/__init__.py b/engine/src/stirling/contracts/__init__.py index 6858ba67e..087dae31a 100644 --- a/engine/src/stirling/contracts/__init__.py +++ b/engine/src/stirling/contracts/__init__.py @@ -14,6 +14,7 @@ from .common import ( ArtifactKind, ConversationMessage, ExtractedFileText, + GenerateFileResponse, MathAuditorToolReportArtifact, NeedContentFileRequest, NeedContentResponse, @@ -87,6 +88,17 @@ from .pdf_questions import ( PdfQuestionResponse, PdfQuestionTerminalResponse, ) +from .pdf_to_markdown import ( + LayoutFragment, + LayoutLine, + PageLayout, + PageLayoutArtifact, + PageLayoutFileEntry, + PdfToMarkdownCannotDoResponse, + PdfToMarkdownOrchestrateResponse, + PdfToMarkdownRequest, + PdfToMarkdownResponse, +) from .progress import ( ProgressEvent, WholeDocCompressionRound, @@ -114,6 +126,10 @@ __all__ = [ "CompletedExecutionAction", "ConversationMessage", "DeleteDocumentResponse", + "PdfToMarkdownCannotDoResponse", + "PdfToMarkdownOrchestrateResponse", + "PdfToMarkdownRequest", + "PdfToMarkdownResponse", "Discrepancy", "DiscrepancyKind", "EditCannotDoResponse", @@ -129,6 +145,7 @@ __all__ = [ "FolioType", "format_conversation_history", "format_file_names", + "GenerateFileResponse", "HealthResponse", "IngestDocumentRequest", "IngestDocumentResponse", @@ -139,7 +156,12 @@ __all__ = [ "NextExecutionAction", "OrchestratorRequest", "OrchestratorResponse", + "LayoutFragment", + "LayoutLine", "Page", + "PageLayout", + "PageLayoutArtifact", + "PageLayoutFileEntry", "PageRange", "PageText", "PdfCommentInstruction", diff --git a/engine/src/stirling/contracts/common.py b/engine/src/stirling/contracts/common.py index 3cd895e14..05103b1a4 100644 --- a/engine/src/stirling/contracts/common.py +++ b/engine/src/stirling/contracts/common.py @@ -61,6 +61,7 @@ class WorkflowOutcome(StrEnum): COMPLETED = "completed" CANNOT_CONTINUE = "cannot_continue" UNSUPPORTED_CAPABILITY = "unsupported_capability" + GENERATE_FILE = "generate_file" class ArtifactKind(StrEnum): @@ -70,6 +71,7 @@ class ArtifactKind(StrEnum): """ EXTRACTED_TEXT = "extracted_text" + PAGE_LAYOUT = "page_layout" TOOL_REPORT = "tool_report" @@ -89,6 +91,7 @@ class SupportedCapability(StrEnum): AGENT_REVISE = "agent_revise" AGENT_NEXT_ACTION = "agent_next_action" MATH_AUDITOR_AGENT = "math_auditor_agent" + PDF_TO_MARKDOWN = "pdf_to_markdown" class ConversationMessage(ApiModel): @@ -200,6 +203,19 @@ class ToolOperationStep(ApiModel): return self +class GenerateFileResponse(ApiModel): + """Return generated text content directly to Java for file packaging. + + Java converts the content string to bytes and stores it as a result file, + avoiding a round-trip through a write-file tool endpoint. + """ + + outcome: Literal[WorkflowOutcome.GENERATE_FILE] = WorkflowOutcome.GENERATE_FILE + content: str + filename: str = Field(pattern=r"^[^/\\]+$", description="Output filename; no path separators.") + summary: str | None = None + + def drop_unknown_tool_endpoints(value: Iterable[str | ToolEndpoint]) -> list[ToolEndpoint]: """Coerce inbound endpoint identifiers into `ToolEndpoint` members, dropping unknowns. diff --git a/engine/src/stirling/contracts/orchestrator.py b/engine/src/stirling/contracts/orchestrator.py index 9972d043b..1bf0f6eb3 100644 --- a/engine/src/stirling/contracts/orchestrator.py +++ b/engine/src/stirling/contracts/orchestrator.py @@ -12,6 +12,7 @@ from .common import ( ArtifactKind, ConversationMessage, ExtractedFileText, + GenerateFileResponse, NeedContentResponse, NeedIngestResponse, SupportedCapability, @@ -22,6 +23,7 @@ from .common import ( from .execution import NextExecutionAction from .pdf_edit import PdfEditTerminalResponse from .pdf_questions import PdfQuestionTerminalResponse +from .pdf_to_markdown import PageLayoutArtifact class ExtractedTextArtifact(ApiModel): @@ -29,7 +31,10 @@ class ExtractedTextArtifact(ApiModel): files: list[ExtractedFileText] = Field(default_factory=list) -WorkflowArtifact = Annotated[ExtractedTextArtifact | ToolReportArtifact, Field(discriminator="kind")] +WorkflowArtifact = Annotated[ + ExtractedTextArtifact | PageLayoutArtifact | ToolReportArtifact, + Field(discriminator="kind"), +] class OrchestratorRequest(ApiModel): @@ -53,6 +58,7 @@ class UnsupportedCapabilityResponse(ApiModel): type OrchestratorResponse = Annotated[ PdfEditTerminalResponse | PdfQuestionTerminalResponse + | GenerateFileResponse | NeedContentResponse | NeedIngestResponse | AgentDraftResponse diff --git a/engine/src/stirling/contracts/pdf_to_markdown.py b/engine/src/stirling/contracts/pdf_to_markdown.py new file mode 100644 index 000000000..4d272e6e2 --- /dev/null +++ b/engine/src/stirling/contracts/pdf_to_markdown.py @@ -0,0 +1,105 @@ +"""Contracts for the PDF to Markdown Agent. + +The agent accepts a parsed document and returns a single Markdown document that +faithfully reconstructs the PDF content — headings, paragraphs, and tables in +reading order, using page_layout as the primary source of truth for structure. + +Java extracts page layout via PdfIngester and returns it as a PageLayoutArtifact +through the orchestrator resume_with pattern. +""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import Field + +from stirling.models import ApiModel + +from .common import ArtifactKind, ConversationMessage, GenerateFileResponse, NeedContentResponse +from .pdf_edit import EditCannotDoResponse + +# ── Input: layout models (mirror Java's RawLine / TextFragment geometry) ──────────────────────── + + +class LayoutFragment(ApiModel): + """One text fragment with its bounding-box geometry and font properties.""" + + text: str + x: float + y: float + width: float + font_size: float + bold: bool + + +class LayoutLine(ApiModel): + """A visual line on the page: one y-coordinate and all fragments on that line.""" + + y: float + fragments: list[LayoutFragment] + + +class PageLayout(ApiModel): + """All layout lines for a single page, in top-to-bottom order.""" + + page_number: int + lines: list[LayoutLine] + + +# ── Artifact: page layout (produced by Java, consumed by orchestrate()) ────────────────────────── + + +class PageLayoutFileEntry(ApiModel): + """Page layout data for one file, as extracted by Java's PdfIngester.""" + + file_name: str + pages: list[PageLayout] = Field(default_factory=list) + + +class PageLayoutArtifact(ApiModel): + """Artifact carrying full spatial page layout for all input files.""" + + kind: Literal[ArtifactKind.PAGE_LAYOUT] = ArtifactKind.PAGE_LAYOUT + files: list[PageLayoutFileEntry] = Field(default_factory=list) + + +# ── Input: full request ────────────────────────────────────────────────────────────────────────── + + +class PdfToMarkdownRequest(ApiModel): + """Request sent by Java after PdfIngester has parsed the document. + + page_layout: per-fragment positional data from the original (y-sorted) line order. + Each fragment carries its x/y position, width, font size, and bold flag. + This is the primary source of truth for column detection and heading hierarchy. + """ + + user_message: str + file_names: list[str] = Field(default_factory=list) + conversation_history: list[ConversationMessage] = Field(default_factory=list) + page_layout: list[PageLayout] = Field(default_factory=list) + + +# ── Output: response variants ──────────────────────────────────────────────────────────────────── + + +class PdfToMarkdownSuccessResponse(ApiModel): + outcome: Literal["document_reconstructed"] = "document_reconstructed" + markdown: str + + +class PdfToMarkdownCannotDoResponse(ApiModel): + outcome: Literal["cannot_do"] = "cannot_do" + reason: str + + +type PdfToMarkdownResponse = Annotated[ + PdfToMarkdownSuccessResponse | PdfToMarkdownCannotDoResponse, + Field(discriminator="outcome"), +] + +type PdfToMarkdownOrchestrateResponse = Annotated[ + GenerateFileResponse | EditCannotDoResponse | NeedContentResponse, + Field(discriminator="outcome"), +] diff --git a/engine/tests/test_pdf_to_markdown.py b/engine/tests/test_pdf_to_markdown.py new file mode 100644 index 000000000..32870a945 --- /dev/null +++ b/engine/tests/test_pdf_to_markdown.py @@ -0,0 +1,138 @@ +"""Tests for PDF to Markdown agent. + +Two cases: +1. Narrative-only page: request validates and routes to reconstruction. +2. Mixed text + table page: layout with table region validates correctly. +""" + +from __future__ import annotations + +from stirling.contracts.pdf_to_markdown import ( + LayoutFragment, + LayoutLine, + PageLayout, + PageLayoutArtifact, + PdfToMarkdownRequest, + PdfToMarkdownSuccessResponse, +) + + +def _frag(text: str, x: float, y: float, font_size: float = 10.0, bold: bool = False) -> LayoutFragment: + return LayoutFragment(text=text, x=x, y=y, width=float(len(text) * 6), font_size=font_size, bold=bold) + + +def _line(y: float, *frags: LayoutFragment) -> LayoutLine: + return LayoutLine(y=y, fragments=list(frags)) + + +# ── Test 1: Narrative-only reconstruction ──────────────────────────────────────────────────────── + + +# ── Contract test: Java serialization ↔ Python deserialization ────────────────────────────────── +# This JSON is also asserted field-by-field in PageLayoutArtifactContractTest.java. +# If either side renames a field, one of these tests fails. +_CONTRACT_JSON = ( + '{"kind":"page_layout","files":[{"fileName":"test.pdf","pages":' + '[{"pageNumber":1,"lines":[{"y":10.0,"fragments":' + '[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}' +) + + +def test_page_layout_artifact_deserialises_java_json() -> None: + artifact = PageLayoutArtifact.model_validate_json(_CONTRACT_JSON) + + assert artifact.kind == "page_layout" + assert artifact.files[0].file_name == "test.pdf" + page = artifact.files[0].pages[0] + assert page.page_number == 1 + line = page.lines[0] + assert line.y == 10.0 + frag = line.fragments[0] + assert frag.text == "Hello" + assert frag.x == 1.0 + assert frag.y == 2.0 + assert frag.width == 30.0 + assert frag.font_size == 12.0 + assert frag.bold is True + + +def test_narrative_reconstruction_request_validates() -> None: + """A prose-only page with no tables produces a valid PdfToMarkdownRequest.""" + layout = PageLayout( + page_number=1, + lines=[ + _line(72.0, _frag("Annual Report 2023", x=72.0, y=72.0, font_size=18.0, bold=True)), + _line(100.0, _frag("Our revenue grew significantly", x=72.0, y=100.0)), + _line(114.0, _frag("during the fiscal year ended", x=72.0, y=114.0)), + _line(128.0, _frag("December 31, 2023.", x=72.0, y=128.0)), + ], + ) + request = PdfToMarkdownRequest( + user_message="reconstruct this document", + page_layout=[layout], + ) + + assert len(request.page_layout) == 1 + assert len(request.page_layout[0].lines) == 4 + assert request.page_layout[0].lines[0].fragments[0].bold is True + assert request.page_layout[0].lines[0].fragments[0].font_size == 18.0 + + +def test_narrative_reconstruction_response_validates() -> None: + """PdfToMarkdownSuccessResponse accepts markdown and returns document_reconstructed outcome.""" + response = PdfToMarkdownSuccessResponse( + markdown="# Annual Report 2023\n\nOur revenue grew significantly during the fiscal year.", + ) + + assert response.outcome == "document_reconstructed" + assert response.markdown.startswith("#") + + +# ── Test 2: Mixed text + table reconstruction ───────────────────────────────────────────────────── + + +def test_mixed_page_layout_validates() -> None: + """A page with both prose lines and a table region produces a valid request.""" + layout = PageLayout( + page_number=1, + lines=[ + # Prose heading + _line(50.0, _frag("Projects in Development", x=72.0, y=50.0, font_size=14.0, bold=True)), + # Table header row + _line( + 80.0, + _frag("Project Name", x=72.0, y=80.0, bold=True), + _frag("Location", x=200.0, y=80.0, bold=True), + _frag("Size (MW)", x=290.0, y=80.0, bold=True), + ), + # Table data rows + _line( + 95.0, + _frag("Chaplin Wind 1", x=72.0, y=95.0), + _frag("Saskatchewan", x=200.0, y=95.0), + _frag("177", x=290.0, y=95.0), + ), + _line( + 110.0, + _frag("Amherst Island 2", x=72.0, y=110.0), + _frag("Ontario", x=200.0, y=110.0), + _frag("75", x=290.0, y=110.0), + ), + # Prose after table + _line(140.0, _frag("Notes:", x=72.0, y=140.0, bold=True)), + _line(154.0, _frag("1 PPA signed", x=85.0, y=154.0)), + ], + ) + request = PdfToMarkdownRequest( + user_message="markdown", + page_layout=[layout], + ) + + assert len(request.page_layout[0].lines) == 6 + # Header line has 3 fragments at distinct x-positions (column detection) + header_line = request.page_layout[0].lines[1] + xs = [f.x for f in header_line.fragments] + assert xs == [72.0, 200.0, 290.0] + # Data rows have matching x-positions + data_row = request.page_layout[0].lines[2] + assert [f.x for f in data_row.fragments] == [72.0, 200.0, 290.0] diff --git a/frontend/package-lock.json b/frontend/package-lock.json index fd45f61e6..5808309d6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -75,9 +75,11 @@ "react-dom": "^19.1.1", "react-easy-crop": "^5.5.6", "react-i18next": "^15.7.3", + "react-markdown": "^9.0.3", "react-rnd": "^10.5.2", "react-router-dom": "^7.9.1", "recharts": "^3.7.0", + "remark-gfm": "^4.0.1", "signature_pad": "^5.0.4", "smol-toml": "^1.4.2", "tailwindcss": "^4.1.13", @@ -4954,6 +4956,15 @@ "@types/d3-selection": "*" } }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/deep-eql": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", @@ -4974,6 +4985,15 @@ "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", "license": "MIT" }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, "node_modules/@types/gapi": { "version": "0.0.47", "resolved": "https://registry.npmjs.org/@types/gapi/-/gapi-0.0.47.tgz", @@ -5029,6 +5049,15 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, "node_modules/@types/http-cache-semantics": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", @@ -5042,6 +5071,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "24.12.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", @@ -5097,6 +5141,12 @@ "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT" }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, "node_modules/@types/use-sync-external-store": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz", @@ -5355,6 +5405,12 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "license": "ISC" + }, "node_modules/@userback/widget": { "version": "0.3.12", "resolved": "https://registry.npmjs.org/@userback/widget/-/widget-0.3.12.tgz", @@ -5962,6 +6018,16 @@ "npm": ">=6" } }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -6384,6 +6450,16 @@ ], "license": "CC-BY-4.0" }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/chai": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", @@ -6424,6 +6500,46 @@ "integrity": "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==", "license": "MIT" }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/check-error": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", @@ -6652,6 +6768,16 @@ "node": ">= 0.8" } }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/commander": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", @@ -7302,6 +7428,19 @@ "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==", "license": "MIT" }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/decompress-response": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-10.0.0.tgz", @@ -7475,7 +7614,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -7644,6 +7782,19 @@ "license": "MIT", "peer": true }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/devtools-protocol": { "version": "0.0.1581282", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1581282.tgz", @@ -8186,6 +8337,16 @@ "node": ">=4.0" } }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/estree-walker": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", @@ -8232,6 +8393,12 @@ "node": ">=12.0.0" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, "node_modules/extract-zip": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", @@ -8902,6 +9069,46 @@ "node": ">= 0.4" } }, + "node_modules/hast-util-to-jsx-runtime": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz", + "integrity": "sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-js": "^1.0.0", + "unist-util-position": "^5.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hoist-non-react-statics": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", @@ -8969,6 +9176,16 @@ "void-elements": "3.1.0" } }, + "node_modules/html-url-attributes": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", + "integrity": "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/http-cache-semantics": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", @@ -9187,6 +9404,12 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/inline-style-parser": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.7.tgz", + "integrity": "sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==", + "license": "MIT" + }, "node_modules/internmap": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", @@ -9206,6 +9429,30 @@ "node": ">= 12" } }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -9240,6 +9487,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-docker": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", @@ -9289,6 +9546,16 @@ "node": ">=0.10.0" } }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-in-ssh": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-in-ssh/-/is-in-ssh-1.0.0.tgz", @@ -9351,6 +9618,18 @@ "node": ">=0.10.0" } }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-potential-custom-element-name": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", @@ -10158,6 +10437,16 @@ "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", "license": "Apache-2.0" }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -10285,6 +10574,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/markdown-table": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -10294,6 +10593,288 @@ "node": ">= 0.4" } }, + "node_modules/mdast-util-find-and-replace": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz", + "integrity": "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-find-and-replace/node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", + "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", + "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", + "integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", + "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", + "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", + "integrity": "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.0.tgz", + "integrity": "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", + "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", + "integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", + "integrity": "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/mdn-data": { "version": "2.27.1", "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", @@ -10301,6 +10882,569 @@ "dev": true, "license": "CC0-1.0" }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.1.tgz", + "integrity": "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -10878,6 +12022,31 @@ "node": ">=6" } }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -11638,6 +12807,16 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "license": "MIT" }, + "node_modules/property-information": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.1.0.tgz", + "integrity": "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/protobufjs": { "version": "7.5.6", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.6.tgz", @@ -11957,6 +13136,33 @@ "integrity": "sha512-W+EWGn2v0ApPKgKKCy/7s7WHXkboGcsrXE+2joLyVxkbyVQfO3MUEaUQDHoSmb8TFFrSKYa9mw64WZHNHSDzYA==", "license": "MIT" }, + "node_modules/react-markdown": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-9.1.0.tgz", + "integrity": "sha512-xaijuJB0kzGiUdG7nc2MOMDUDBWPyGAjZtUrow9XxUeua8IqeP+VlIfAZ3bphpcLTnSZXz6z9jcVC/TCwbfgdw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "hast-util-to-jsx-runtime": "^2.0.0", + "html-url-attributes": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.0.0", + "unified": "^11.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "@types/react": ">=18", + "react": ">=18" + } + }, "node_modules/react-number-format": { "version": "5.4.5", "resolved": "https://registry.npmjs.org/react-number-format/-/react-number-format-5.4.5.tgz", @@ -12364,6 +13570,72 @@ "redux": "^5.0.0" } }, + "node_modules/remark-gfm": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz", + "integrity": "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-gfm": "^3.0.0", + "micromark-extension-gfm": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-rehype": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-11.1.2.tgz", + "integrity": "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-stringify": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", + "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-to-markdown": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -12982,6 +14254,16 @@ "node": ">=0.10.0" } }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/spdx-compare": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/spdx-compare/-/spdx-compare-1.0.0.tgz", @@ -13148,6 +14430,20 @@ "node": ">=8" } }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/stringify-object": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", @@ -13256,6 +14552,24 @@ "dev": true, "license": "MIT" }, + "node_modules/style-to-js": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.21.tgz", + "integrity": "sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ==", + "license": "MIT", + "dependencies": { + "style-to-object": "1.0.14" + } + }, + "node_modules/style-to-object": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.14.tgz", + "integrity": "sha512-LIN7rULI0jBscWQYaSswptyderlarFkjQ+t79nzty8tcIAceVomEVlLzH5VP4Cmsv6MtKhs7qaAiwlcp+Mgaxw==", + "license": "MIT", + "dependencies": { + "inline-style-parser": "0.2.7" + } + }, "node_modules/stylis": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.2.0.tgz", @@ -13748,6 +15062,26 @@ "node": ">=0.6" } }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/ts-api-utils": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", @@ -13947,6 +15281,93 @@ "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "license": "MIT" }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", + "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", + "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", + "integrity": "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", @@ -14118,6 +15539,34 @@ "spdx-expression-parse": "^3.0.0" } }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", + "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/victory-vendor": { "version": "37.3.6", "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-37.3.6.tgz", @@ -14878,6 +16327,16 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } } } } diff --git a/frontend/package.json b/frontend/package.json index 18a89198e..a23ee45d3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -71,9 +71,11 @@ "react-dom": "^19.1.1", "react-easy-crop": "^5.5.6", "react-i18next": "^15.7.3", + "react-markdown": "^9.0.3", "react-rnd": "^10.5.2", "react-router-dom": "^7.9.1", "recharts": "^3.7.0", + "remark-gfm": "^4.0.1", "signature_pad": "^5.0.4", "smol-toml": "^1.4.2", "tailwindcss": "^4.1.13", diff --git a/frontend/src/core/components/viewer/nonpdf/MarkdownRenderer.tsx b/frontend/src/core/components/viewer/nonpdf/MarkdownRenderer.tsx new file mode 100644 index 000000000..585632609 --- /dev/null +++ b/frontend/src/core/components/viewer/nonpdf/MarkdownRenderer.tsx @@ -0,0 +1,108 @@ +import React, { useState } from "react"; +import ReactMarkdown from "react-markdown"; +import type { Components } from "react-markdown"; +import remarkGfm from "remark-gfm"; + +function CopyButton({ text }: { text: string }) { + const [copied, setCopied] = useState(false); + return ( + + ); +} + +const components: Components = { + pre: ({ children }) => { + const codeText = React.isValidElement(children) + ? String((children.props as { children?: unknown }).children ?? "") + : String(children ?? ""); + return ( +

+ 
+          {children}
+
+ +
+ ); + }, + table: ({ children }) => ( +
+
+ {children} +
+ + ), + th: ({ children, style }) => ( + + {children} + + ), + td: ({ children, style }) => ( + + {children} + + ), +}; + +export function renderMarkdown(content: string): React.ReactNode[] { + return [ + + {content} + , + ]; +} diff --git a/frontend/src/core/components/viewer/nonpdf/TextViewer.tsx b/frontend/src/core/components/viewer/nonpdf/TextViewer.tsx index da5c51d21..61c1674f3 100644 --- a/frontend/src/core/components/viewer/nonpdf/TextViewer.tsx +++ b/frontend/src/core/components/viewer/nonpdf/TextViewer.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from "react"; +import React, { useEffect, useMemo, useState } from "react"; import { Box, Center, @@ -12,153 +12,7 @@ import { import { useTranslation } from "react-i18next"; import { formatFileSize } from "@app/utils/fileUtils"; - -// ─── Markdown renderer ──────────────────────────────────────────────────────── - -function renderInline(text: string): React.ReactNode[] { - const parts: React.ReactNode[] = []; - // Patterns: **bold**, *italic*, `code`, [link](url) - const re = /(\*\*(.+?)\*\*|\*(.+?)\*|`([^`]+)`|\[([^\]]+)\]\(([^)]+)\))/g; - let last = 0; - let match; - let key = 0; - while ((match = re.exec(text)) !== null) { - if (match.index > last) parts.push(text.slice(last, match.index)); - if (match[2]) parts.push({match[2]}); - else if (match[3]) parts.push({match[3]}); - else if (match[4]) - parts.push( - - {match[4]} - , - ); - else if (match[5]) - parts.push( - - {match[5]} - , - ); - last = match.index + match[0].length; - } - if (last < text.length) parts.push(text.slice(last)); - return parts; -} - -function renderMarkdown(text: string): React.ReactNode[] { - const lines = text.split("\n"); - const elements: React.ReactNode[] = []; - let inCodeBlock = false; - let codeLines: string[] = []; - let i = 0; - - while (i < lines.length) { - const line = lines[i]; - - if (line.startsWith("```")) { - if (inCodeBlock) { - elements.push( - 
-            {codeLines.join("\n")}
-
, - ); - codeLines = []; - inCodeBlock = false; - } else { - inCodeBlock = true; - } - i++; - continue; - } - - if (inCodeBlock) { - codeLines.push(line); - i++; - continue; - } - - if (line.startsWith("### ")) { - elements.push( - - {renderInline(line.slice(4))} - , - ); - } else if (line.startsWith("## ")) { - elements.push( - - {renderInline(line.slice(3))} - , - ); - } else if (line.startsWith("# ")) { - elements.push( - - {renderInline(line.slice(2))} - , - ); - } else if (line.startsWith("- ") || line.startsWith("* ")) { - elements.push( - - - {"\u2022"} - - - {renderInline(line.slice(2))} - - , - ); - } else if (/^\d+\.\s/.test(line)) { - const num = line.match(/^(\d+)\.\s/)?.[1]; - const rest = line.replace(/^\d+\.\s/, ""); - elements.push( - - - {num}. - - - {renderInline(rest)} - - , - ); - } else if (line.trim() === "" || line === "---" || line === "***") { - elements.push(); - } else { - elements.push( - - {renderInline(line)} - , - ); - } - i++; - } - - return elements; -} - -// ─── Text / Markdown viewer ─────────────────────────────────────────────────── +import { renderMarkdown } from "@app/components/viewer/nonpdf/MarkdownRenderer"; interface TextViewerProps { file: File; @@ -176,6 +30,13 @@ export function TextViewer({ file, isMarkdown }: TextViewerProps) { }, [file]); const lines = content?.split("\n") ?? []; + const renderedMarkdown = useMemo( + () => + content !== null && isMarkdown && renderMd + ? renderMarkdown(content) + : null, + [content, isMarkdown, renderMd], + ); return ( @@ -222,9 +83,17 @@ export function TextViewer({ file, isMarkdown }: TextViewerProps) { {t("viewer.nonPdf.loading")} - ) : isMarkdown && renderMd ? ( - - {renderMarkdown(content)} + ) : renderedMarkdown !== null ? ( + + {renderedMarkdown} ) : ( - {lines.map((line, i) => ( - + {lines.map((line, idx) => ( + {showLineNumbers && ( - {i + 1} + {idx + 1} )} - {line || "\u00A0"} + {line || " "} ))}