From 35a712a2781b1925ff561e20f1b72e5c8ee6eda0 Mon Sep 17 00:00:00 2001 From: EthanHealy01 <80844253+EthanHealy01@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:16:33 +0100 Subject: [PATCH] smart redaction (#6195) Co-authored-by: James Brunton --- .../SPDF/pdf/parser/PageColumnLayout.java | 142 ++ .../SPDF/pdf/parser/PageImageLocator.java | 133 ++ .../JsonListPropertyEditor.java | 48 + .../JsonObjectPropertyEditor.java | 41 + .../StringToArrayListPropertyEditor.java | 53 - .../SPDF/pdf/parser/PageColumnLayoutTest.java | 149 ++ .../StringToArrayListPropertyEditorTest.java | 163 -- .../controller/api/EditTextController.java | 6 +- .../api/security/AllTextLineExtractor.java | 127 ++ .../api/security/ManualRedactionService.java | 403 +++++ .../api/security/MultiPatternTextFinder.java | 174 ++ .../api/security/RedactController.java | 1544 +---------------- .../api/security/RedactExecuteService.java | 850 +++++++++ .../api/security/TextRedactionService.java | 1207 +++++++++++++ .../api/security/RedactExecuteRequest.java | 132 ++ .../api/security/RedactControllerTest.java | 74 +- .../security/RedactExecuteServiceTest.java | 430 +++++ .../service/PdfContentExtractor.java | 202 ++- engine/src/stirling/agents/orchestrator.py | 2 +- engine/src/stirling/agents/pdf_edit.py | 119 +- engine/src/stirling/contracts/__init__.py | 2 +- engine/src/stirling/models/tool_models.py | 81 + engine/tests/test_pdf_edit_agent.py | 27 +- .../core/components/viewer/EmbedPdfViewer.tsx | 9 +- .../formFill/providers/PdfiumFormProvider.ts | 73 +- .../data/usePrototypeToolRegistry.tsx | 8 +- 26 files changed, 4397 insertions(+), 1802 deletions(-) create mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/PageColumnLayout.java create mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/PageImageLocator.java create mode 100644 app/common/src/main/java/stirling/software/common/util/propertyeditor/JsonListPropertyEditor.java create mode 100644 app/common/src/main/java/stirling/software/common/util/propertyeditor/JsonObjectPropertyEditor.java delete mode 100644 app/common/src/main/java/stirling/software/common/util/propertyeditor/StringToArrayListPropertyEditor.java create mode 100644 app/common/src/test/java/stirling/software/SPDF/pdf/parser/PageColumnLayoutTest.java delete mode 100644 app/common/src/test/java/stirling/software/common/util/propertyeditor/StringToArrayListPropertyEditorTest.java create mode 100644 app/core/src/main/java/stirling/software/SPDF/controller/api/security/AllTextLineExtractor.java create mode 100644 app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java create mode 100644 app/core/src/main/java/stirling/software/SPDF/controller/api/security/MultiPatternTextFinder.java create mode 100644 app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java create mode 100644 app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java create mode 100644 app/core/src/main/java/stirling/software/SPDF/model/api/security/RedactExecuteRequest.java create mode 100644 app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactExecuteServiceTest.java diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PageColumnLayout.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PageColumnLayout.java new file mode 100644 index 000000000..e3f000624 --- /dev/null +++ b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PageColumnLayout.java @@ -0,0 +1,142 @@ +package stirling.software.SPDF.pdf.parser; + +import java.util.ArrayList; +import java.util.List; + +/** + * Detects whether a page is one- or two-column from per-line bounding boxes, and classifies an + * X-span into the column it belongs to. Detection is a midpoint vote at {@code pageWidth / 2}. + * + *

Capped at two columns by design — sufficient for the redaction target set (single-column + * documents and IEEE-style two-column papers). 3+ column layouts (newspapers, magazines) and + * off-centre gutters (asymmetric two-column) would need a histogram or clustering approach to + * detect the actual gutter X. (future work) + * + *

Coordinates are PDFTextStripper screen space (top-left origin, Y increases downward). + */ +public final class PageColumnLayout { + + /** + * Slack when checking "crosses a gutter" so single-pixel overshoots don't mark a line as + * spanning. + */ + public static final float SPAN_SLACK_PT = 2f; + + /** + * Slack on each side of the page midpoint inside which a line is considered "spanning" + * (covering both columns) rather than belonging to one side. + */ + private static final float MIDPOINT_SLACK_PT = 30f; + + /** + * Minimum line width (points) for a line to count toward the two-column tally. Avoids false + * positives where right-aligned dates, page numbers, or short "Link" fragments next to a + * heading look like a second column when they're really just inline metadata. + */ + private static final float MIN_COLUMN_LINE_WIDTH_PT = 100f; + + /** + * Minimum number of clearly leftish AND clearly rightish lines (each of width ≥ {@link + * #MIN_COLUMN_LINE_WIDTH_PT}) required to call the page two-column. Anything below this falls + * back to single-column. + */ + private static final int MIN_SIDE_LINES = 3; + + private final List columns; + private final List gutters; + + private PageColumnLayout(List columns, List gutters) { + this.columns = columns; + this.gutters = gutters; + } + + /** + * Determines column layout from per-line bounding boxes ({@code [x1, _, x2, _]}). Counts lines + * whose X-midpoint sits clearly left of, or clearly right of, the page midpoint (with {@link + * #MIDPOINT_SLACK_PT} slack each side). If both sides have at least {@link #MIN_SIDE_LINES} + * lines, the page is treated as two-column with the gutter at the page midpoint. Otherwise it's + * single-column. + * + *

Cross-column lines must already be split: callers should feed boxes from a line extractor + * that splits same-Y glyphs at large X gaps (see {@code AllTextLineExtractor}). Without that + * split, IEEE-style aligned-baseline 2-column PDFs produce one wide merged box per row and the + * side tallies all end up classified as "spanning", falling to single-column. + */ + public static PageColumnLayout fromLineBoxes(List lineBoxes, float pageWidth) { + if (lineBoxes == null || lineBoxes.isEmpty()) { + return new PageColumnLayout(List.of(new float[] {0f, pageWidth}), List.of()); + } + float pageMid = pageWidth / 2f; + int left = 0, right = 0; + for (float[] lb : lineBoxes) { + if (lb == null || lb.length < 3) continue; + float width = lb[2] - lb[0]; + // Skip narrow lines — dates, page numbers, "Link" labels next to a heading should + // not, on their own, make a single-column doc look two-column. + if (width < MIN_COLUMN_LINE_WIDTH_PT) continue; + float mid = (lb[0] + lb[2]) * 0.5f; + if (mid < pageMid - MIDPOINT_SLACK_PT) left++; + else if (mid > pageMid + MIDPOINT_SLACK_PT) right++; + } + if (left < MIN_SIDE_LINES || right < MIN_SIDE_LINES) { + return new PageColumnLayout(List.of(new float[] {0f, pageWidth}), List.of()); + } + float gutterL = pageMid - MIDPOINT_SLACK_PT; + float gutterR = pageMid + MIDPOINT_SLACK_PT; + return new PageColumnLayout( + List.of(new float[] {0f, gutterL}, new float[] {gutterR, pageWidth}), + List.of(new float[] {gutterL, gutterR})); + } + + /** All columns, left-to-right, as {@code [leftX, rightX]} pairs. Never empty. */ + public List columns() { + return columns; + } + + /** Gutters between columns, left-to-right, as {@code [leftX, rightX]} pairs. */ + public List gutters() { + return gutters; + } + + public int columnCount() { + return columns.size(); + } + + /** + * Returns the column index containing the X-midpoint of {@code [x1, x2]}, falling back to the + * closest column if the midpoint sits inside a gutter. + */ + public int columnOf(float x1, float x2) { + float mid = (x1 + x2) * 0.5f; + int best = 0; + float bestDist = Float.MAX_VALUE; + for (int i = 0; i < columns.size(); i++) { + float[] c = columns.get(i); + if (mid >= c[0] && mid <= c[1]) return i; + float dist = mid < c[0] ? c[0] - mid : mid - c[1]; + if (dist < bestDist) { + bestDist = dist; + best = i; + } + } + return best; + } + + /** + * Returns every column index whose X-range overlaps {@code [x1, x2]} with at least {@link + * #SPAN_SLACK_PT} of intrusion. A normal in-column line returns one index; a line crossing a + * gutter returns two or more. + */ + public int[] columnsCrossing(float x1, float x2) { + List hits = new ArrayList<>(); + for (int i = 0; i < columns.size(); i++) { + float[] c = columns.get(i); + float overlap = Math.min(x2, c[1]) - Math.max(x1, c[0]); + if (overlap > SPAN_SLACK_PT) hits.add(i); + } + if (hits.isEmpty()) hits.add(columnOf(x1, x2)); + int[] out = new int[hits.size()]; + for (int i = 0; i < hits.size(); i++) out[i] = hits.get(i); + return out; + } +} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PageImageLocator.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PageImageLocator.java new file mode 100644 index 000000000..dc384b3f5 --- /dev/null +++ b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PageImageLocator.java @@ -0,0 +1,133 @@ +package stirling.software.SPDF.pdf.parser; + +import java.awt.geom.Point2D; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.pdfbox.contentstream.PDFGraphicsStreamEngine; +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.graphics.image.PDImage; +import org.apache.pdfbox.util.Matrix; + +/** + * PDFGraphicsStreamEngine that intercepts {@code drawImage} calls and records each image's bounding + * box in PDF user-space (origin bottom-left, Y up) by transforming the unit square through the + * current transformation matrix (CTM). + * + *

Usage: + * + * 

{@code
+ * PageImageLocator locator = new PageImageLocator(page, pageIndex);
+ * locator.processPage(page);
+ * List boxes = locator.getImageBoxes();
+ * }
+ * + *

Each {@link ImageBox} carries the 0-based page index and the axis-aligned bounding box {@code + * (x1, y1, x2, y2)} in PDF user-space coordinates. + */ +public final class PageImageLocator extends PDFGraphicsStreamEngine { + + /** + * Bounding box of a raster or vector image found on a PDF page. + * + * @param pageIndex 0-based page index + * @param x1 left edge in PDF user-space (origin bottom-left) + * @param y1 bottom edge in PDF user-space + * @param x2 right edge + * @param y2 top edge + */ + public record ImageBox(int pageIndex, float x1, float y1, float x2, float y2) {} + + private final int pageIndex; + private final List imageBoxes = new ArrayList<>(); + private final Point2D.Float currentPoint = new Point2D.Float(); + + /** + * @param page the PDPage to process + * @param pageIndex 0-based index of this page in the document (stored on each returned {@link + * ImageBox}) + */ + public PageImageLocator(PDPage page, int pageIndex) { + super(page); + this.pageIndex = pageIndex; + } + + /** Returns all image bounding boxes collected during {@link #processPage}. */ + public List getImageBoxes() { + return imageBoxes; + } + + @Override + public void drawImage(PDImage pdImage) throws IOException { + Matrix ctm = getGraphicsState().getCurrentTransformationMatrix(); + // An image occupies the unit square (0,0)→(1,1) in image space. + // Transform all four corners through the CTM to get the page-space bounding box. + float a = ctm.getScaleX(); + float b = ctm.getShearY(); + float c = ctm.getShearX(); + float d = ctm.getScaleY(); + float e = ctm.getTranslateX(); + float f = ctm.getTranslateY(); + float[] xs = {e, a + e, c + e, a + c + e}; + float[] ys = {f, b + f, d + f, b + d + f}; + float x1 = Float.MAX_VALUE, y1 = Float.MAX_VALUE; + float x2 = -Float.MAX_VALUE, y2 = -Float.MAX_VALUE; + for (float x : xs) { + x1 = Math.min(x1, x); + x2 = Math.max(x2, x); + } + for (float y : ys) { + y1 = Math.min(y1, y); + y2 = Math.max(y2, y); + } + imageBoxes.add(new ImageBox(pageIndex, x1, y1, x2, y2)); + } + + // ---------- required abstract methods (no-op for path operations) ---------- + + @Override + public void appendRectangle(Point2D p0, Point2D p1, Point2D p2, Point2D p3) {} + + @Override + public void clip(int windingRule) {} + + @Override + public void moveTo(float x, float y) { + currentPoint.setLocation(x, y); + } + + @Override + public void lineTo(float x, float y) { + currentPoint.setLocation(x, y); + } + + @Override + public void curveTo(float x1, float y1, float x2, float y2, float x3, float y3) { + currentPoint.setLocation(x3, y3); + } + + @Override + public Point2D getCurrentPoint() { + return currentPoint; + } + + @Override + public void closePath() {} + + @Override + public void endPath() {} + + @Override + public void strokePath() {} + + @Override + public void fillPath(int windingRule) {} + + @Override + public void fillAndStrokePath(int windingRule) {} + + @Override + public void shadingFill(COSName shadingName) {} +} diff --git a/app/common/src/main/java/stirling/software/common/util/propertyeditor/JsonListPropertyEditor.java b/app/common/src/main/java/stirling/software/common/util/propertyeditor/JsonListPropertyEditor.java new file mode 100644 index 000000000..b0b796c67 --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/util/propertyeditor/JsonListPropertyEditor.java @@ -0,0 +1,48 @@ +package stirling.software.common.util.propertyeditor; + +import java.beans.PropertyEditorSupport; +import java.util.ArrayList; +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.DeserializationFeature; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +/** + * Binds a multipart form value containing a JSON array into a typed {@code List}. Used for + * endpoints that accept structured list parameters via {@code @ModelAttribute} — the form field + * carries the full JSON array as its value and the editor parses it once. + */ +@Slf4j +public class JsonListPropertyEditor extends PropertyEditorSupport { + + private static final ObjectMapper OBJECT_MAPPER = + JsonMapper.builder() + .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .build(); + + private final TypeReference> typeRef; + + public JsonListPropertyEditor(TypeReference> typeRef) { + this.typeRef = typeRef; + } + + @Override + public void setAsText(String text) throws IllegalArgumentException { + if (text == null || text.trim().isEmpty()) { + setValue(new ArrayList()); + return; + } + try { + setValue(OBJECT_MAPPER.readValue(text, typeRef)); + } catch (Exception e) { + log.error("Failed to parse JSON list value", e); + throw new IllegalArgumentException( + "Expected a JSON array but could not parse: " + e.getMessage()); + } + } +} diff --git a/app/common/src/main/java/stirling/software/common/util/propertyeditor/JsonObjectPropertyEditor.java b/app/common/src/main/java/stirling/software/common/util/propertyeditor/JsonObjectPropertyEditor.java new file mode 100644 index 000000000..e714b995b --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/util/propertyeditor/JsonObjectPropertyEditor.java @@ -0,0 +1,41 @@ +package stirling.software.common.util.propertyeditor; + +import java.beans.PropertyEditorSupport; + +import lombok.extern.slf4j.Slf4j; + +import tools.jackson.databind.DeserializationFeature; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; + +/** + * Binds a multipart form value containing a JSON object into a typed {@code T}. Companion to {@link + * JsonListPropertyEditor} for single-object nested fields on {@code @ModelAttribute} endpoints. + */ +@Slf4j +public class JsonObjectPropertyEditor extends PropertyEditorSupport { + + private static final ObjectMapper OBJECT_MAPPER = + JsonMapper.builder().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).build(); + + private final Class type; + + public JsonObjectPropertyEditor(Class type) { + this.type = type; + } + + @Override + public void setAsText(String text) throws IllegalArgumentException { + if (text == null || text.trim().isEmpty()) { + setValue(null); + return; + } + try { + setValue(OBJECT_MAPPER.readValue(text, type)); + } catch (Exception e) { + log.error("Failed to parse JSON object value", e); + throw new IllegalArgumentException( + "Expected a JSON object but could not parse: " + e.getMessage()); + } + } +} diff --git a/app/common/src/main/java/stirling/software/common/util/propertyeditor/StringToArrayListPropertyEditor.java b/app/common/src/main/java/stirling/software/common/util/propertyeditor/StringToArrayListPropertyEditor.java deleted file mode 100644 index 6941e6b6d..000000000 --- a/app/common/src/main/java/stirling/software/common/util/propertyeditor/StringToArrayListPropertyEditor.java +++ /dev/null @@ -1,53 +0,0 @@ -package stirling.software.common.util.propertyeditor; - -import java.beans.PropertyEditorSupport; -import java.util.ArrayList; -import java.util.List; - -import lombok.extern.slf4j.Slf4j; - -import tools.jackson.databind.DeserializationFeature; -import tools.jackson.databind.JavaType; -import tools.jackson.databind.ObjectMapper; -import tools.jackson.databind.json.JsonMapper; - -/** - * Spring property editor that decodes a JSON string into a typed {@link ArrayList}. Used to bind - * complex list parameters (e.g. {@code List}, {@code List}) from - * multipart form fields, where Spring's default binding cannot deserialize a JSON array. - */ -@Slf4j -public class StringToArrayListPropertyEditor extends PropertyEditorSupport { - - private final ObjectMapper objectMapper = - JsonMapper.builder() - .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY) - .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - .build(); - - private final Class elementType; - - public StringToArrayListPropertyEditor(Class elementType) { - this.elementType = elementType; - } - - @Override - public void setAsText(String text) throws IllegalArgumentException { - if (text == null || text.trim().isEmpty()) { - setValue(new ArrayList<>()); - return; - } - try { - JavaType listType = - objectMapper - .getTypeFactory() - .constructCollectionType(ArrayList.class, elementType); - List list = objectMapper.readValue(text, listType); - setValue(list); - } catch (Exception e) { - log.error("Exception while converting {}", e); - throw new IllegalArgumentException( - "Failed to convert java.lang.String to java.util.List"); - } - } -} diff --git a/app/common/src/test/java/stirling/software/SPDF/pdf/parser/PageColumnLayoutTest.java b/app/common/src/test/java/stirling/software/SPDF/pdf/parser/PageColumnLayoutTest.java new file mode 100644 index 000000000..5aa9715b8 --- /dev/null +++ b/app/common/src/test/java/stirling/software/SPDF/pdf/parser/PageColumnLayoutTest.java @@ -0,0 +1,149 @@ +package stirling.software.SPDF.pdf.parser; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link PageColumnLayout} gutter detection and column classification. */ +class PageColumnLayoutTest { + + private static final float PAGE_WIDTH = 612f; // Letter portrait + + // ── single-column ──────────────────────────────────────────────────────────────────────────── + + @Test + void singleColumn_oneColumnNoGutters() { + List lines = List.of(lineBox(72f, 396f)); + + PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH); + + assertThat(layout.columnCount()).isEqualTo(1); + assertThat(layout.gutters()).isEmpty(); + } + + @Test + void singleColumn_classifyAnchor_returnsZero() { + PageColumnLayout layout = + PageColumnLayout.fromLineBoxes(List.of(lineBox(72f, 396f)), PAGE_WIDTH); + + assertThat(layout.columnOf(100f, 200f)).isEqualTo(0); + } + + // ── two-column ─────────────────────────────────────────────────────────────────────────────── + + @Test + void twoColumn_detectsGutter() { + PageColumnLayout layout = + PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH); + + assertThat(layout.columnCount()).isEqualTo(2); + assertThat(layout.gutters()).hasSize(1); + float[] gutter = layout.gutters().get(0); + // Gutter is centered on pageWidth/2 with PageColumnLayout.MIDPOINT_SLACK_PT slack each + // side. + float pageMid = PAGE_WIDTH / 2f; + assertThat(gutter[0]).isBetween(pageMid - 40f, pageMid - 20f); + assertThat(gutter[1]).isBetween(pageMid + 20f, pageMid + 40f); + } + + @Test + void twoColumn_classifyLeftAndRightAnchors() { + PageColumnLayout layout = + PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH); + assertThat(layout.columnOf(100f, 200f)).isEqualTo(0); + assertThat(layout.columnOf(380f, 460f)).isEqualTo(1); + } + + @Test + void twoColumn_columnsCrossing_leftLineOnlyHitsLeft() { + PageColumnLayout layout = + PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH); + assertThat(layout.columnsCrossing(72f, 280f)).containsExactly(0); + assertThat(layout.columnsCrossing(320f, 540f)).containsExactly(1); + } + + @Test + void twoColumn_spanningLine_returnsBothColumns() { + List lines = new ArrayList<>(buildTwoColumnLines(3)); + // Full-width header that crosses pageWidth/2. + lines.add(lineBox(72f, 396f)); + + PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH); + + assertThat(layout.columnsCrossing(72f, 540f)).containsExactly(0, 1); + assertThat(layout.columnsCrossing(72f, 280f)).containsExactly(0); + } + + private static List buildTwoColumnLines(int rowsPerColumn) { + List lines = new ArrayList<>(); + for (int i = 0; i < rowsPerColumn; i++) { + lines.add(lineBox(72f, 136f)); // left column body (72..208) + lines.add(lineBox(320f, 220f)); // right column body (320..540) + } + return lines; + } + + // ── three-column ───────────────────────────────────────────────────────────────────────────── + + @Test + void threeColumn_collapsesToLeftRightSplit() { + // The midpoint-based detector splits the page at pageWidth/2 and treats anything else as + // single-column or spanning. A genuine 3-column layout collapses to 2 columns; the middle + // column's content ends up classified by midpoint as left or right of pageMid. + List lines = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + lines.add(lineBox(72f, 150f)); // 72..222 + lines.add(lineBox(252f, 150f)); // 252..402 + lines.add(lineBox(432f, 150f)); // 432..582 + } + + PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH); + + assertThat(layout.columnCount()).isEqualTo(2); + assertThat(layout.gutters()).hasSize(1); + } + + // ── empty page ─────────────────────────────────────────────────────────────────────────────── + + @Test + void emptyPage_singleColumnFallback() { + PageColumnLayout layout = PageColumnLayout.fromLineBoxes(List.of(), PAGE_WIDTH); + assertThat(layout.columnCount()).isEqualTo(1); + assertThat(layout.gutters()).isEmpty(); + } + + @Test + void onlyShortFragments_singleColumnFallback() { + // Page numbers / decorations — too narrow to vote either side. + List lines = List.of(lineBox(300f, 6f)); + PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH); + assertThat(layout.columnCount()).isEqualTo(1); + } + + // ── narrow gap should not be confused for a gutter ─────────────────────────────────────────── + + @Test + void narrowInternalGap_doesNotProduceGutter() { + // Both halves sit left of the page midpoint, so no line votes for a right column and + // detection falls back to single-column. + List lines = new ArrayList<>(); + lines.add(lineBox(72f, 100f)); + lines.add(lineBox(180f, 100f)); + for (int i = 0; i < 5; i++) { + lines.add(lineBox(72f, 208f)); + } + + PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH); + assertThat(layout.columnCount()).isEqualTo(1); + } + + // ── helpers ────────────────────────────────────────────────────────────────────────────────── + + /** Builds a line bounding box {@code [x1, 0, x1+width, 0]}; Y is unused by detection. */ + private static float[] lineBox(float x1, float width) { + return new float[] {x1, 0f, x1 + width, 0f}; + } +} diff --git a/app/common/src/test/java/stirling/software/common/util/propertyeditor/StringToArrayListPropertyEditorTest.java b/app/common/src/test/java/stirling/software/common/util/propertyeditor/StringToArrayListPropertyEditorTest.java deleted file mode 100644 index a2e03f086..000000000 --- a/app/common/src/test/java/stirling/software/common/util/propertyeditor/StringToArrayListPropertyEditorTest.java +++ /dev/null @@ -1,163 +0,0 @@ -package stirling.software.common.util.propertyeditor; - -import static org.junit.jupiter.api.Assertions.*; - -import java.util.List; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import stirling.software.common.model.api.security.RedactionArea; - -class StringToArrayListPropertyEditorTest { - - private StringToArrayListPropertyEditor editor; - - @BeforeEach - void setUp() { - editor = new StringToArrayListPropertyEditor<>(RedactionArea.class); - } - - @Test - void testSetAsText_ValidJson() { - // Arrange - String json = - "[{\"x\":10.5,\"y\":20.5,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"}]"; - - // Act - editor.setAsText(json); - Object value = editor.getValue(); - - // Assert - assertNotNull(value, "Value should not be null"); - assertInstanceOf(List.class, value, "Value should be a List"); - - @SuppressWarnings("unchecked") - List list = (List) value; - assertEquals(1, list.size(), "List should have 1 entry"); - - RedactionArea area = list.get(0); - assertEquals(10.5, area.getX(), "X should be 10.5"); - assertEquals(20.5, area.getY(), "Y should be 20.5"); - assertEquals(100.0, area.getWidth(), "Width should be 100.0"); - assertEquals(50.0, area.getHeight(), "Height should be 50.0"); - assertEquals(1, area.getPage(), "Page should be 1"); - assertEquals("#FF0000", area.getColor(), "Color should be #FF0000"); - } - - @Test - void testSetAsText_MultipleItems() { - // Arrange - String json = - "[" - + "{\"x\":10.0,\"y\":20.0,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"}," - + "{\"x\":30.0,\"y\":40.0,\"width\":200.0,\"height\":150.0,\"page\":2,\"color\":\"#00FF00\"}" - + "]"; - - // Act - editor.setAsText(json); - Object value = editor.getValue(); - - // Assert - assertNotNull(value, "Value should not be null"); - assertInstanceOf(List.class, value, "Value should be a List"); - - @SuppressWarnings("unchecked") - List list = (List) value; - assertEquals(2, list.size(), "List should have 2 entries"); - - RedactionArea area1 = list.get(0); - assertEquals(10.0, area1.getX(), "X should be 10.0"); - assertEquals(20.0, area1.getY(), "Y should be 20.0"); - assertEquals(1, area1.getPage(), "Page should be 1"); - - RedactionArea area2 = list.get(1); - assertEquals(30.0, area2.getX(), "X should be 30.0"); - assertEquals(40.0, area2.getY(), "Y should be 40.0"); - assertEquals(2, area2.getPage(), "Page should be 2"); - } - - @Test - void testSetAsText_EmptyString() { - // Arrange - String json = ""; - - // Act - editor.setAsText(json); - Object value = editor.getValue(); - - // Assert - assertNotNull(value, "Value should not be null"); - assertInstanceOf(List.class, value, "Value should be a List"); - - @SuppressWarnings("unchecked") - List list = (List) value; - assertTrue(list.isEmpty(), "List should be empty"); - } - - @Test - void testSetAsText_NullString() { - // Act - editor.setAsText(null); - Object value = editor.getValue(); - - // Assert - assertNotNull(value, "Value should not be null"); - assertInstanceOf(List.class, value, "Value should be a List"); - - @SuppressWarnings("unchecked") - List list = (List) value; - assertTrue(list.isEmpty(), "List should be empty"); - } - - @Test - void testSetAsText_SingleItemAsArray() { - // Arrange - note this is a single object, not an array - String json = - "{\"x\":10.0,\"y\":20.0,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"}"; - - // Act - editor.setAsText(json); - Object value = editor.getValue(); - - // Assert - assertNotNull(value, "Value should not be null"); - assertInstanceOf(List.class, value, "Value should be a List"); - - @SuppressWarnings("unchecked") - List list = (List) value; - assertEquals(1, list.size(), "List should have 1 entry"); - - RedactionArea area = list.get(0); - assertEquals(10.0, area.getX(), "X should be 10.0"); - assertEquals(20.0, area.getY(), "Y should be 20.0"); - } - - @Test - void testSetAsText_InvalidJson() { - // Arrange - String json = "invalid json"; - - // Act & Assert - assertThrows(IllegalArgumentException.class, () -> editor.setAsText(json)); - } - - @Test - void testSetAsText_UnknownProperties() { - // Arrange - this JSON contains properties not in RedactionArea - // With FAIL_ON_UNKNOWN_PROPERTIES disabled, this should ignore the unknown properties - String json = "[{\"invalid\":\"structure\"}]"; - - // Act - editor.setAsText(json); - Object value = editor.getValue(); - - // Assert - assertNotNull(value, "Value should not be null"); - assertInstanceOf(List.class, value, "Value should be a List"); - - @SuppressWarnings("unchecked") - List list = (List) value; - assertEquals(1, list.size(), "List should have 1 entry (empty object)"); - } -} diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/EditTextController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/EditTextController.java index ec4420fb1..3a10b1419 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/EditTextController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/EditTextController.java @@ -39,7 +39,9 @@ import stirling.software.common.util.GeneralUtils; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; import stirling.software.common.util.WebResponseUtils; -import stirling.software.common.util.propertyeditor.StringToArrayListPropertyEditor; +import stirling.software.common.util.propertyeditor.JsonListPropertyEditor; + +import tools.jackson.core.type.TypeReference; /** * Find/replace text editing for PDFs. Round-trips through {@link PdfJsonConversionService}: the @@ -73,7 +75,7 @@ public class EditTextController { binder.registerCustomEditor( List.class, "edits", - new StringToArrayListPropertyEditor<>(EditTextOperation.class)); + new JsonListPropertyEditor<>(new TypeReference>() {})); } @AutoJobPostMapping( diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/AllTextLineExtractor.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/AllTextLineExtractor.java new file mode 100644 index 000000000..f744b41af --- /dev/null +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/AllTextLineExtractor.java @@ -0,0 +1,127 @@ +package stirling.software.SPDF.controller.api.security; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.text.PDFTextStripper; +import org.apache.pdfbox.text.TextPosition; + +/** + * PDFTextStripper subclass that collects all text positions and groups them into line-level + * bounding boxes. + * + *

Two outputs are maintained in parallel: + * + *

+ * + *

Lines are flushed not only on Y jumps but also on large X gaps within the same Y row. That way + * left-column glyphs and right-column glyphs that happen to share a baseline (common in IEEE + * conference templates) get emitted as two distinct line boxes instead of one wide merged box. + */ +final class AllTextLineExtractor extends PDFTextStripper { + + /** Min vertical jump (screen Y) before the next glyph is treated as a new line. */ + private static final float LINE_Y_TOLERANCE = 3.0f; + + /** + * Min horizontal gap (screen X) between consecutive glyphs on the same Y that indicates a + * column boundary. Chosen large enough to not split normal inter-word spacing (~6–10pt for 11pt + * text) but small enough to catch standard column gutters (typically ≥15pt). + */ + private static final float COLUMN_GAP_X = 14f; + + private final float pageHeight; + private final List lineBoxes = new ArrayList<>(); + private final List screenLineBoxes = new ArrayList<>(); + + private final List currentLine = new ArrayList<>(); + private float lastScreenY = Float.NaN; + private float lastGlyphRight = Float.NaN; + + AllTextLineExtractor(int pageNumber, float pageHeight) throws IOException { + this.pageHeight = pageHeight; + setStartPage(pageNumber); + setEndPage(pageNumber); + setSortByPosition(true); + } + + List getLineBoxes() { + return lineBoxes; + } + + /** + * Returns line boxes as {@code [x1, screenYtop, x2, screenYbottom]}. {@code screenYtop} is the + * minimum {@code TextPosition.getY() - getHeight()} on the line and {@code screenYbottom} is + * the maximum {@code getY()} (the line's baseline). These values come straight from PDFBox + * without going through {@code pageHeight - …}, so they're stable for ulp-sensitive comparisons + * against anchor screen Ys. + */ + List getScreenLineBoxes() { + return screenLineBoxes; + } + + @Override + protected void writeString(String text, List positions) { + for (TextPosition tp : positions) { + // Skip whitespace-only positions (spaces, newline markers, indent characters). + // These have a TextPosition but no visible glyph; including them causes + // space-only "lines" to produce degenerate segments that appear as thin + // black bars after redaction. + String unicode = tp.getUnicode(); + if (unicode == null || unicode.isBlank()) { + continue; + } + float screenY = tp.getY(); + float screenX = tp.getX(); + boolean yJump = + !Float.isNaN(lastScreenY) && Math.abs(screenY - lastScreenY) > LINE_Y_TOLERANCE; + boolean xJump = + !Float.isNaN(lastGlyphRight) && (screenX - lastGlyphRight) > COLUMN_GAP_X; + if (yJump || xJump) { + flushLine(); + } + lastScreenY = screenY; + lastGlyphRight = screenX + tp.getWidth(); + currentLine.add(tp); + } + } + + @Override + protected void endPage(PDPage page) throws IOException { + flushLine(); + super.endPage(page); + } + + private void flushLine() { + if (currentLine.isEmpty()) { + return; + } + float minX = Float.MAX_VALUE, maxX = -Float.MAX_VALUE; + float minScreenY = Float.MAX_VALUE, maxScreenY = -Float.MAX_VALUE; + for (TextPosition tp : currentLine) { + minX = Math.min(minX, tp.getX()); + maxX = Math.max(maxX, tp.getX() + tp.getWidth()); + minScreenY = Math.min(minScreenY, tp.getY() - tp.getHeight()); + maxScreenY = Math.max(maxScreenY, tp.getY()); + } + emitSegment(minX, maxX, minScreenY, maxScreenY); + currentLine.clear(); + lastScreenY = Float.NaN; + lastGlyphRight = Float.NaN; + } + + private void emitSegment(float minX, float maxX, float minScreenY, float maxScreenY) { + float pdfY1 = pageHeight - maxScreenY; // bottom in PDF coords + float pdfY2 = pageHeight - minScreenY; // top in PDF coords + lineBoxes.add(new float[] {minX, pdfY1, maxX, pdfY2}); + screenLineBoxes.add(new float[] {minX, minScreenY, maxX, maxScreenY}); + } +} diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java new file mode 100644 index 000000000..15a233f1f --- /dev/null +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/ManualRedactionService.java @@ -0,0 +1,403 @@ +package stirling.software.SPDF.controller.api.security; + +import java.awt.Color; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.PDPageTree; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.SPDF.model.PDFText; +import stirling.software.SPDF.model.api.security.ManualRedactPdfRequest; +import stirling.software.SPDF.pdf.parser.PageImageLocator; +import stirling.software.common.model.api.security.RedactionArea; +import stirling.software.common.util.GeneralUtils; +import stirling.software.common.util.PdfUtils; +import stirling.software.common.util.TempFile; +import stirling.software.common.util.TempFileManager; + +@Service +@Slf4j +@RequiredArgsConstructor +class ManualRedactionService { + + private static final float DEFAULT_TEXT_PADDING_MULTIPLIER = 0.6f; + private static final float REDACTION_WIDTH_REDUCTION_FACTOR = 0.9f; + + private final TempFileManager tempFileManager; + + // ----------------------------------------------------------------------- + // Area and page redaction + // ----------------------------------------------------------------------- + + void redactAreas(List redactionAreas, PDDocument document, PDPageTree allPages) + throws IOException { + + if (redactionAreas == null || redactionAreas.isEmpty()) { + return; + } + + Map> redactionsByPage = new HashMap<>(); + + for (RedactionArea redactionArea : redactionAreas) { + if (redactionArea.getPage() == null + || redactionArea.getPage() <= 0 + || redactionArea.getHeight() == null + || redactionArea.getHeight() <= 0.0D + || redactionArea.getWidth() == null + || redactionArea.getWidth() <= 0.0D) { + continue; + } + + redactionsByPage + .computeIfAbsent(redactionArea.getPage(), k -> new ArrayList<>()) + .add(redactionArea); + } + + for (Map.Entry> entry : redactionsByPage.entrySet()) { + Integer pageNumber = entry.getKey(); + List areasForPage = entry.getValue(); + + if (pageNumber > allPages.getCount()) { + continue; + } + + PDPage page = allPages.get(pageNumber - 1); + + try (PDPageContentStream contentStream = + new PDPageContentStream( + document, page, PDPageContentStream.AppendMode.APPEND, true, true)) { + + contentStream.saveGraphicsState(); + for (RedactionArea redactionArea : areasForPage) { + Color redactColor = decodeOrDefault(redactionArea.getColor()); + + contentStream.setNonStrokingColor(redactColor); + + float x = redactionArea.getX().floatValue(); + float y = redactionArea.getY().floatValue(); + float width = redactionArea.getWidth().floatValue(); + float height = redactionArea.getHeight().floatValue(); + + float pdfY = page.getBBox().getHeight() - y - height; + + contentStream.addRect(x, pdfY, width, height); + contentStream.fill(); + } + contentStream.restoreGraphicsState(); + } + } + } + + void redactPages(ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages) + throws IOException { + + Color redactColor = decodeOrDefault(request.getPageRedactionColor()); + List pageNumbers = getPageNumbers(request, allPages.getCount()); + + for (Integer pageNumber : pageNumbers) { + PDPage page = allPages.get(pageNumber); + + try (PDPageContentStream contentStream = + new PDPageContentStream( + document, page, PDPageContentStream.AppendMode.APPEND, true, true)) { + contentStream.setNonStrokingColor(redactColor); + + PDRectangle box = page.getBBox(); + contentStream.addRect(0, 0, box.getWidth(), box.getHeight()); + contentStream.fill(); + } + } + } + + // ----------------------------------------------------------------------- + // Overlay drawing + // ----------------------------------------------------------------------- + + void redactFoundText( + PDDocument document, + List blocks, + float customPadding, + Color redactColor, + boolean isTextRemovalMode) + throws IOException { + + var allPages = document.getDocumentCatalog().getPages(); + + Map> blocksByPage = new HashMap<>(); + for (PDFText block : blocks) { + blocksByPage.computeIfAbsent(block.getPageIndex(), k -> new ArrayList<>()).add(block); + } + + for (Map.Entry> entry : blocksByPage.entrySet()) { + Integer pageIndex = entry.getKey(); + List pageBlocks = entry.getValue(); + + if (pageIndex >= allPages.getCount()) { + continue; + } + + var page = allPages.get(pageIndex); + try (PDPageContentStream contentStream = + new PDPageContentStream( + document, page, PDPageContentStream.AppendMode.APPEND, true, true)) { + + contentStream.saveGraphicsState(); + + try { + contentStream.setNonStrokingColor(redactColor); + PDRectangle pageBox = page.getBBox(); + + for (PDFText block : pageBlocks) { + float padding = + (block.getY2() - block.getY1()) * DEFAULT_TEXT_PADDING_MULTIPLIER + + customPadding; + + float originalWidth = block.getX2() - block.getX1(); + float boxWidth; + float boxX; + + if (isTextRemovalMode) { + boxWidth = originalWidth * REDACTION_WIDTH_REDUCTION_FACTOR; + float widthReduction = originalWidth - boxWidth; + boxX = block.getX1() + (widthReduction / 2); + } else { + boxWidth = originalWidth; + boxX = block.getX1(); + } + + contentStream.addRect( + boxX, + pageBox.getHeight() - block.getY2() - padding, + boxWidth, + block.getY2() - block.getY1() + 2 * padding); + } + + contentStream.fill(); + + } finally { + contentStream.restoreGraphicsState(); + } + } + + // Remove annotations whose bounding rect overlaps a redacted block, to prevent + // users from hovering over redacted URLs and seeing the underlying destination. + try { + float pageH = page.getBBox().getHeight(); + List kept = new ArrayList<>(); + for (PDAnnotation ann : page.getAnnotations()) { + PDRectangle ar = ann.getRectangle(); + boolean overlaps = false; + if (ar != null) { + for (PDFText block : pageBlocks) { + float padding = + (block.getY2() - block.getY1()) + * DEFAULT_TEXT_PADDING_MULTIPLIER + + customPadding; + float bx1 = block.getX1(); + float bx2 = block.getX2(); + float by1 = pageH - block.getY2() - padding; + float by2 = pageH - block.getY1() + padding; + if (ar.getLowerLeftX() < bx2 + && ar.getUpperRightX() > bx1 + && ar.getLowerLeftY() < by2 + && ar.getUpperRightY() > by1) { + overlaps = true; + break; + } + } + } + if (!overlaps) { + kept.add(ann); + } + } + page.setAnnotations(kept); + } catch (Exception e) { + log.debug( + "[redact] could not remove annotations on page {}: {}", + pageIndex, + e.getMessage()); + } + } + } + + void redactImageBoxes(PDDocument document, List imageBoxes, Color color) + throws IOException { + Map> byPage = new HashMap<>(); + for (float[] box : imageBoxes) { + byPage.computeIfAbsent((int) box[0], k -> new ArrayList<>()).add(box); + } + PDPageTree pages = document.getDocumentCatalog().getPages(); + for (Map.Entry> entry : byPage.entrySet()) { + int pageIdx = entry.getKey(); + if (pageIdx < 0 || pageIdx >= pages.getCount()) { + log.warn("[redact/execute] image box references out-of-range page {}", pageIdx); + continue; + } + PDPage page = pages.get(pageIdx); + try (PDPageContentStream cs = + new PDPageContentStream( + document, page, PDPageContentStream.AppendMode.APPEND, true, true)) { + cs.saveGraphicsState(); + cs.setNonStrokingColor(color); + for (float[] box : entry.getValue()) { + float x1 = box[1], y1 = box[2], x2 = box[3], y2 = box[4]; + cs.addRect(x1, y1, x2 - x1, y2 - y1); + } + cs.fill(); + cs.restoreGraphicsState(); + } + } + } + + // ----------------------------------------------------------------------- + // Page element extraction + // ----------------------------------------------------------------------- + + /** + * Returns bounding boxes for every text line and image on {@code page} in PDF user-space + * coordinates: {@code [x1, y1, x2, y2]} (origin bottom-left, Y increases upward). + */ + List extractPageElementBoxes(PDDocument document, PDPage page, int pageIndex) + throws IOException { + List boxes = new ArrayList<>(); + + AllTextLineExtractor textExtractor = + new AllTextLineExtractor(pageIndex + 1, page.getBBox().getHeight()); + textExtractor.getText(document); + boxes.addAll(textExtractor.getLineBoxes()); + + PageImageLocator imgLocator = new PageImageLocator(page, pageIndex); + imgLocator.processPage(page); + for (PageImageLocator.ImageBox imgBox : imgLocator.getImageBoxes()) { + boxes.add(new float[] {imgBox.x1(), imgBox.y1(), imgBox.x2(), imgBox.y2()}); + } + + return boxes; + } + + // ----------------------------------------------------------------------- + // Finalization + // ----------------------------------------------------------------------- + + TempFile finalizeRedaction( + PDDocument document, + Map> allFoundTextsByPage, + String colorString, + float customPadding, + Boolean convertToImage, + boolean isTextRemovalMode) + throws IOException { + + List allFoundTexts = new ArrayList<>(); + for (List pageTexts : allFoundTextsByPage.values()) { + allFoundTexts.addAll(pageTexts); + } + + if (!allFoundTexts.isEmpty()) { + Color redactColor = decodeOrDefault(colorString); + redactFoundText(document, allFoundTexts, customPadding, redactColor, isTextRemovalMode); + cleanDocumentMetadata(document); + } + + if (Boolean.TRUE.equals(convertToImage)) { + try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) { + cleanDocumentMetadata(convertedPdf); + + TempFile tempOut = tempFileManager.createManagedTempFile(".pdf"); + try { + convertedPdf.save(tempOut.getFile()); + } catch (IOException e) { + tempOut.close(); + throw e; + } + + log.info( + "Redaction finalized (image mode): {} pages ➜ {} KB", + convertedPdf.getNumberOfPages(), + tempOut.getFile().length() / 1024); + + return tempOut; + } + } + + TempFile tempOut = tempFileManager.createManagedTempFile(".pdf"); + try { + document.save(tempOut.getFile()); + } catch (IOException e) { + tempOut.close(); + throw e; + } + + log.info( + "Redaction finalized: {} pages ➜ {} KB", + document.getNumberOfPages(), + tempOut.getFile().length() / 1024); + + return tempOut; + } + + private void cleanDocumentMetadata(PDDocument document) { + try { + var documentInfo = document.getDocumentInformation(); + if (documentInfo != null) { + documentInfo.setAuthor(null); + documentInfo.setSubject(null); + documentInfo.setKeywords(null); + documentInfo.setModificationDate(java.util.Calendar.getInstance()); + log.debug("Cleaned document metadata for security"); + } + + if (document.getDocumentCatalog() != null) { + try { + document.getDocumentCatalog().setMetadata(null); + } catch (Exception e) { + log.debug("Could not clear XMP metadata: {}", e.getMessage()); + } + } + + } catch (Exception e) { + log.warn("Failed to clean document metadata: {}", e.getMessage()); + } + } + + // ----------------------------------------------------------------------- + // Utilities + // ----------------------------------------------------------------------- + + static Color decodeOrDefault(String hex) { + if (hex == null) { + return Color.BLACK; + } + + String colorString = hex.startsWith("#") ? hex : "#" + hex; + + try { + return Color.decode(colorString); + } catch (NumberFormatException e) { + return Color.BLACK; + } + } + + private List getPageNumbers(ManualRedactPdfRequest request, int pagesCount) { + String pageNumbersInput = request.getPageNumbers(); + String[] parsedPageNumbers = + pageNumbersInput != null ? pageNumbersInput.split(",") : new String[0]; + List pageNumbers = + GeneralUtils.parsePageList(parsedPageNumbers, pagesCount, false); + Collections.sort(pageNumbers); + return pageNumbers; + } +} diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/MultiPatternTextFinder.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/MultiPatternTextFinder.java new file mode 100644 index 000000000..7771493f2 --- /dev/null +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/MultiPatternTextFinder.java @@ -0,0 +1,174 @@ +package stirling.software.SPDF.controller.api.security; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.text.PDFTextStripper; +import org.apache.pdfbox.text.TextPosition; + +import lombok.extern.slf4j.Slf4j; + +import stirling.software.SPDF.model.PDFText; + +/** + * Scans a PDF document once and matches all provided patterns in a single pass, collecting + * bounding-box positions for every match. Use in place of creating one {@code TextFinder} per + * search term to avoid O(n) full-document scans. + */ +@Slf4j +final class MultiPatternTextFinder extends PDFTextStripper { + + private static final long REGEX_MATCH_TIMEOUT_SECONDS = 30; + private static final ExecutorService REGEX_EXECUTOR = + Executors.newVirtualThreadPerTaskExecutor(); + + private final List patterns; + private final Map> foundTextsByPage = new HashMap<>(); + + private final List pageTextPositions = new ArrayList<>(); + private final StringBuilder pageTextBuilder = new StringBuilder(); + + MultiPatternTextFinder(List patterns) throws IOException { + this.patterns = patterns; + this.setWordSeparator(" "); + this.setLineSeparator("\n"); + } + + Map> getFoundTextsByPage() { + return foundTextsByPage; + } + + @Override + protected void startPage(PDPage page) throws IOException { + super.startPage(page); + pageTextPositions.clear(); + pageTextBuilder.setLength(0); + } + + @Override + protected void writeString(String text, List textPositions) { + pageTextBuilder.append(text); + pageTextPositions.addAll(textPositions); + } + + @Override + protected void writeWordSeparator() { + pageTextBuilder.append(getWordSeparator()); + pageTextPositions.add(null); + } + + @Override + protected void writeLineSeparator() { + pageTextBuilder.append(getLineSeparator()); + pageTextPositions.add(null); + } + + @Override + protected void endPage(PDPage page) throws IOException { + String text = pageTextBuilder.toString(); + if (!text.isEmpty()) { + int pageIndex = getCurrentPageNo() - 1; + for (Pattern pattern : patterns) { + Matcher matcher = pattern.matcher(text); + while (safeFind(matcher)) { + PDFText pdfText = resolveMatchPosition(matcher, pageIndex); + if (pdfText != null) { + foundTextsByPage + .computeIfAbsent(pageIndex, k -> new ArrayList<>()) + .add(pdfText); + } + } + } + } + super.endPage(page); + } + + /** + * Wraps a single {@code matcher.find()} call with a {@value #REGEX_MATCH_TIMEOUT_SECONDS} + * second timeout. Prevents pathological regex backtracking from blocking the request + * indefinitely; per-match timeout so fast legitimate scans are unaffected. + */ + private static boolean safeFind(Matcher matcher) throws IOException { + Future future = + REGEX_EXECUTOR.submit((java.util.concurrent.Callable) matcher::find); + try { + return future.get(REGEX_MATCH_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (TimeoutException e) { + future.cancel(true); + throw new IOException( + "Regex match timed out after " + + REGEX_MATCH_TIMEOUT_SECONDS + + "s — pattern may cause catastrophic backtracking"); + } catch (InterruptedException e) { + future.cancel(true); + Thread.currentThread().interrupt(); + throw new IOException("Regex match interrupted", e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof IOException ioEx) throw ioEx; + throw new IOException("Regex match failed: " + cause.getMessage(), cause); + } + } + + private PDFText resolveMatchPosition(Matcher matcher, int pageIndex) { + int matchStart = matcher.start(); + int matchEnd = matcher.end(); + + float minX = Float.MAX_VALUE; + float minY = Float.MAX_VALUE; + float maxX = Float.MIN_VALUE; + float maxY = Float.MIN_VALUE; + boolean foundPosition = false; + + for (int i = matchStart; i < matchEnd; i++) { + if (i >= pageTextPositions.size()) break; + TextPosition pos = pageTextPositions.get(i); + if (pos != null) { + foundPosition = true; + minX = Math.min(minX, pos.getX()); + maxX = Math.max(maxX, pos.getX() + pos.getWidth()); + minY = Math.min(minY, pos.getY() - pos.getHeight()); + maxY = Math.max(maxY, pos.getY()); + } + } + + if (!foundPosition && matchStart < pageTextPositions.size()) { + for (int i = Math.max(0, matchStart - 5); + i < Math.min(pageTextPositions.size(), matchEnd + 5); + i++) { + TextPosition pos = pageTextPositions.get(i); + if (pos != null) { + foundPosition = true; + minX = Math.min(minX, pos.getX()); + maxX = Math.max(maxX, pos.getX() + pos.getWidth()); + minY = Math.min(minY, pos.getY() - pos.getHeight()); + maxY = Math.max(maxY, pos.getY()); + break; + } + } + } + + if (!foundPosition) { + log.warn( + "Found text match '{}' but no valid position data at {}-{}", + matcher.group(), + matchStart, + matchEnd); + return null; + } + + return new PDFText(pageIndex, minX, minY, maxX, maxY, matcher.group()); + } +} diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java index 33edd89fd..127b43630 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactController.java @@ -1,38 +1,12 @@ package stirling.software.SPDF.controller.api.security; -import java.awt.Color; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import org.apache.pdfbox.contentstream.operator.Operator; -import org.apache.pdfbox.cos.COSArray; -import org.apache.pdfbox.cos.COSBase; -import org.apache.pdfbox.cos.COSFloat; -import org.apache.pdfbox.cos.COSName; -import org.apache.pdfbox.cos.COSNumber; -import org.apache.pdfbox.cos.COSString; -import org.apache.pdfbox.pdfparser.PDFStreamParser; -import org.apache.pdfbox.pdfwriter.ContentStreamWriter; import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.PDPageContentStream; import org.apache.pdfbox.pdmodel.PDPageTree; -import org.apache.pdfbox.pdmodel.PDResources; -import org.apache.pdfbox.pdmodel.common.PDRectangle; -import org.apache.pdfbox.pdmodel.common.PDStream; -import org.apache.pdfbox.pdmodel.font.PDFont; -import org.apache.pdfbox.pdmodel.graphics.PDXObject; -import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject; import org.springframework.core.io.Resource; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -44,54 +18,45 @@ import org.springframework.web.multipart.MultipartFile; import io.github.pixee.security.Filenames; import io.swagger.v3.oas.annotations.Operation; -import lombok.AllArgsConstructor; -import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import stirling.software.SPDF.config.swagger.StandardPdfResponse; import stirling.software.SPDF.model.PDFText; import stirling.software.SPDF.model.api.security.ManualRedactPdfRequest; +import stirling.software.SPDF.model.api.security.RedactExecuteRequest; +import stirling.software.SPDF.model.api.security.RedactExecuteRequest.ImageBox; +import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyle; +import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange; import stirling.software.SPDF.model.api.security.RedactPdfRequest; -import stirling.software.SPDF.pdf.TextFinder; -import stirling.software.SPDF.utils.text.TextEncodingHelper; -import stirling.software.SPDF.utils.text.TextFinderUtils; -import stirling.software.SPDF.utils.text.WidthCalculator; import stirling.software.common.annotations.AutoJobPostMapping; import stirling.software.common.annotations.api.SecurityApi; import stirling.software.common.enumeration.ResourceWeight; import stirling.software.common.model.api.security.RedactionArea; import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.util.ExceptionUtils; -import stirling.software.common.util.GeneralUtils; import stirling.software.common.util.PdfUtils; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; import stirling.software.common.util.WebResponseUtils; -import stirling.software.common.util.propertyeditor.StringToArrayListPropertyEditor; +import stirling.software.common.util.propertyeditor.JsonListPropertyEditor; +import stirling.software.common.util.propertyeditor.JsonObjectPropertyEditor; + +import tools.jackson.core.type.TypeReference; @SecurityApi @Slf4j @RequiredArgsConstructor public class RedactController { - private static final float DEFAULT_TEXT_PADDING_MULTIPLIER = 0.6f; - private static final float PRECISION_THRESHOLD = 1e-3f; - private static final int FONT_SCALE_FACTOR = 1000; - - // Redaction box width reduction factor (10% reduction) - private static final float REDACTION_WIDTH_REDUCTION_FACTOR = 0.9f; - - // Text showing operators - private static final Set TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\""); - - private static final COSString EMPTY_COS_STRING = new COSString(""); - private final CustomPDFDocumentFactory pdfDocumentFactory; private final TempFileManager tempFileManager; + private final ManualRedactionService manualRedactionService; + private final TextRedactionService textRedactionService; + private final RedactExecuteService redactExecuteService; private String removeFileExtension(String filename) { - return GeneralUtils.removeExtension(filename); + return stirling.software.common.util.GeneralUtils.removeExtension(filename); } @InitBinder @@ -99,7 +64,17 @@ public class RedactController { binder.registerCustomEditor( List.class, "redactions", - new StringToArrayListPropertyEditor<>(RedactionArea.class)); + new JsonListPropertyEditor<>(new TypeReference>() {})); + binder.registerCustomEditor( + List.class, + "ranges", + new JsonListPropertyEditor<>(new TypeReference>() {})); + binder.registerCustomEditor( + List.class, + "imageBoxes", + new JsonListPropertyEditor<>(new TypeReference>() {})); + binder.registerCustomEditor( + RedactStyle.class, "style", new JsonObjectPropertyEditor<>(RedactStyle.class)); } @AutoJobPostMapping( @@ -118,14 +93,12 @@ public class RedactController { throws IOException { MultipartFile file = request.getFileInput(); - List redactionAreas = request.getRedactions(); try (PDDocument document = pdfDocumentFactory.load(file)) { PDPageTree allPages = document.getDocumentCatalog().getPages(); - redactPages(request, document, allPages); - - redactAreas(redactionAreas, document, allPages); + manualRedactionService.redactPages(request, document, allPages); + manualRedactionService.redactAreas(request.getRedactions(), document, allPages); if (Boolean.TRUE.equals(request.getConvertPDFToImage())) { try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) { @@ -150,354 +123,6 @@ public class RedactController { } } - private void redactAreas( - List redactionAreas, PDDocument document, PDPageTree allPages) - throws IOException { - - if (redactionAreas == null || redactionAreas.isEmpty()) { - return; - } - - Map> redactionsByPage = new HashMap<>(); - - for (RedactionArea redactionArea : redactionAreas) { - - if (redactionArea.getPage() == null - || redactionArea.getPage() <= 0 - || redactionArea.getHeight() == null - || redactionArea.getHeight() <= 0.0D - || redactionArea.getWidth() == null - || redactionArea.getWidth() <= 0.0D) { - continue; - } - - redactionsByPage - .computeIfAbsent(redactionArea.getPage(), k -> new ArrayList<>()) - .add(redactionArea); - } - - for (Map.Entry> entry : redactionsByPage.entrySet()) { - Integer pageNumber = entry.getKey(); - List areasForPage = entry.getValue(); - - if (pageNumber > allPages.getCount()) { - continue; // Skip if the page number is out of bounds - } - - PDPage page = allPages.get(pageNumber - 1); - - try (PDPageContentStream contentStream = - new PDPageContentStream( - document, page, PDPageContentStream.AppendMode.APPEND, true, true)) { - - contentStream.saveGraphicsState(); - for (RedactionArea redactionArea : areasForPage) { - Color redactColor = decodeOrDefault(redactionArea.getColor()); - - contentStream.setNonStrokingColor(redactColor); - - float x = redactionArea.getX().floatValue(); - float y = redactionArea.getY().floatValue(); - float width = redactionArea.getWidth().floatValue(); - float height = redactionArea.getHeight().floatValue(); - - float pdfY = page.getBBox().getHeight() - y - height; - - contentStream.addRect(x, pdfY, width, height); - contentStream.fill(); - } - contentStream.restoreGraphicsState(); - } - } - } - - private void redactPages( - ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages) - throws IOException { - - Color redactColor = decodeOrDefault(request.getPageRedactionColor()); - List pageNumbers = getPageNumbers(request, allPages.getCount()); - - for (Integer pageNumber : pageNumbers) { - - PDPage page = allPages.get(pageNumber); - - try (PDPageContentStream contentStream = - new PDPageContentStream( - document, page, PDPageContentStream.AppendMode.APPEND, true, true)) { - contentStream.setNonStrokingColor(redactColor); - - PDRectangle box = page.getBBox(); - - contentStream.addRect(0, 0, box.getWidth(), box.getHeight()); - contentStream.fill(); - } - } - } - - private void redactFoundText( - PDDocument document, - List blocks, - float customPadding, - Color redactColor, - boolean isTextRemovalMode) - throws IOException { - - var allPages = document.getDocumentCatalog().getPages(); - - Map> blocksByPage = new HashMap<>(); - for (PDFText block : blocks) { - blocksByPage.computeIfAbsent(block.getPageIndex(), k -> new ArrayList<>()).add(block); - } - - for (Map.Entry> entry : blocksByPage.entrySet()) { - Integer pageIndex = entry.getKey(); - List pageBlocks = entry.getValue(); - - if (pageIndex >= allPages.getCount()) { - continue; // Skip if page index is out of bounds - } - - var page = allPages.get(pageIndex); - try (PDPageContentStream contentStream = - new PDPageContentStream( - document, page, PDPageContentStream.AppendMode.APPEND, true, true)) { - - contentStream.saveGraphicsState(); - - try { - contentStream.setNonStrokingColor(redactColor); - PDRectangle pageBox = page.getBBox(); - - for (PDFText block : pageBlocks) { - float padding = - (block.getY2() - block.getY1()) * DEFAULT_TEXT_PADDING_MULTIPLIER - + customPadding; - - float originalWidth = block.getX2() - block.getX1(); - float boxWidth; - float boxX; - - // Only apply width reduction when text is actually being removed - if (isTextRemovalMode) { - // Calculate reduced width and center the box - boxWidth = - originalWidth - * REDACTION_WIDTH_REDUCTION_FACTOR; // 10% reduction - float widthReduction = originalWidth - boxWidth; - boxX = block.getX1() + (widthReduction / 2); // Center the reduced box - } else { - // Use original width for box-only redaction - boxWidth = originalWidth; - boxX = block.getX1(); - } - - contentStream.addRect( - boxX, - pageBox.getHeight() - block.getY2() - padding, - boxWidth, - block.getY2() - block.getY1() + 2 * padding); - } - - contentStream.fill(); - - } finally { - contentStream.restoreGraphicsState(); - } - } - } - } - - String createPlaceholderWithFont(String originalWord, PDFont font) { - if (originalWord == null || originalWord.isEmpty()) { - return originalWord; - } - - if (font != null && TextEncodingHelper.isFontSubset(font.getName())) { - try { - float originalWidth = safeGetStringWidth(font, originalWord) / FONT_SCALE_FACTOR; - return createAlternativePlaceholder(originalWord, originalWidth, font, 1.0f); - } catch (Exception e) { - log.debug( - "Subset font placeholder creation failed for {}: {}", - font.getName(), - e.getMessage()); - return ""; - } - } - - return " ".repeat(originalWord.length()); - } - - /** - * Enhanced placeholder creation using advanced width calculation. Incorporates font validation - * and sophisticated fallback strategies. - */ - String createPlaceholderWithWidth( - String originalWord, float targetWidth, PDFont font, float fontSize) { - if (originalWord == null || originalWord.isEmpty()) { - return originalWord; - } - - if (font == null || fontSize <= 0) { - return " ".repeat(originalWord.length()); - } - - try { - // Check font reliability before proceeding - if (!WidthCalculator.isWidthCalculationReliable(font)) { - log.debug( - "Font {} unreliable for width calculation, using simple placeholder", - font.getName()); - return " ".repeat(originalWord.length()); - } - - // Use enhanced subset font detection - if (TextEncodingHelper.isFontSubset(font.getName())) { - return createSubsetFontPlaceholder(originalWord, targetWidth, font, fontSize); - } - - // Enhanced space width calculation - float spaceWidth = WidthCalculator.calculateAccurateWidth(font, " ", fontSize); - - if (spaceWidth <= 0) { - return createAlternativePlaceholder(originalWord, targetWidth, font, fontSize); - } - - int spaceCount = Math.max(1, Math.round(targetWidth / spaceWidth)); - - // More conservative space limit based on original word characteristics - int maxSpaces = - Math.max( - originalWord.length() * 2, Math.round(targetWidth / spaceWidth * 1.5f)); - spaceCount = Math.min(spaceCount, maxSpaces); - - return " ".repeat(spaceCount); - - } catch (Exception e) { - log.debug("Enhanced placeholder creation failed: {}", e.getMessage()); - return createAlternativePlaceholder(originalWord, targetWidth, font, fontSize); - } - } - - private String createSubsetFontPlaceholder( - String originalWord, float targetWidth, PDFont font, float fontSize) { - try { - log.debug("Subset font {} - trying to find replacement characters", font.getName()); - String result = createAlternativePlaceholder(originalWord, targetWidth, font, fontSize); - - if (result.isEmpty()) { - log.debug( - "Subset font {} has no suitable replacement characters, using empty string", - font.getName()); - } - - return result; - - } catch (Exception e) { - log.debug("Subset font placeholder creation failed: {}", e.getMessage()); - return ""; - } - } - - private String createAlternativePlaceholder( - String originalWord, float targetWidth, PDFont font, float fontSize) { - try { - String[] alternatives = {" ", ".", "-", "_", "~", "°", "·"}; - - if (TextEncodingHelper.fontSupportsCharacter(font, " ")) { - float spaceWidth = safeGetStringWidth(font, " ") / FONT_SCALE_FACTOR * fontSize; - if (spaceWidth > 0) { - int spaceCount = Math.max(1, Math.round(targetWidth / spaceWidth)); - int maxSpaces = originalWord.length() * 2; - spaceCount = Math.min(spaceCount, maxSpaces); - log.debug("Using spaces for font {}", font.getName()); - return " ".repeat(spaceCount); - } - } - - for (String altChar : alternatives) { - if (" ".equals(altChar)) continue; // Already tried spaces - - try { - if (!TextEncodingHelper.fontSupportsCharacter(font, altChar)) { - continue; - } - - float charWidth = - safeGetStringWidth(font, altChar) / FONT_SCALE_FACTOR * fontSize; - if (charWidth > 0) { - int charCount = Math.max(1, Math.round(targetWidth / charWidth)); - int maxChars = originalWord.length() * 2; - charCount = Math.min(charCount, maxChars); - log.debug( - "Using character '{}' for width calculation but spaces for placeholder in font {}", - altChar, - font.getName()); - - return " ".repeat(charCount); - } - } catch (Exception e) { - } - } - - log.debug( - "All placeholder alternatives failed for font {}, using empty string", - font.getName()); - return ""; - - } catch (Exception e) { - log.debug("Alternative placeholder creation failed: {}", e.getMessage()); - return ""; - } - } - - void writeFilteredContentStream(PDDocument document, PDPage page, List tokens) - throws IOException { - - PDStream newStream = new PDStream(document); - - try { - try (var out = newStream.createOutputStream()) { - ContentStreamWriter writer = new ContentStreamWriter(out); - writer.writeTokens(tokens); - } - - page.setContents(newStream); - - } catch (IOException e) { - throw new IOException("Failed to write filtered content stream to page", e); - } - } - - Color decodeOrDefault(String hex) { - if (hex == null) { - return Color.BLACK; - } - - String colorString = hex.startsWith("#") ? hex : "#" + hex; - - try { - return Color.decode(colorString); - } catch (NumberFormatException e) { - return Color.BLACK; - } - } - - boolean isTextShowingOperator(String opName) { - return TEXT_SHOWING_OPERATORS.contains(opName); - } - - private List getPageNumbers(ManualRedactPdfRequest request, int pagesCount) { - String pageNumbersInput = request.getPageNumbers(); - String[] parsedPageNumbers = - pageNumbersInput != null ? pageNumbersInput.split(",") : new String[0]; - List pageNumbers = - GeneralUtils.parsePageList(parsedPageNumbers, pagesCount, false); - Collections.sort(pageNumbers); - return pageNumbers; - } - @AutoJobPostMapping( value = "/auto-redact", consumes = MediaType.MULTIPART_FORM_DATA_VALUE, @@ -538,7 +163,8 @@ public class RedactController { } Map> allFoundTextsByPage = - findTextToRedact(document, listOfText, useRegex, wholeWordSearchBool); + textRedactionService.findTextToRedact( + document, listOfText, useRegex, wholeWordSearchBool); int totalMatches = allFoundTextsByPage.values().stream().mapToInt(List::size).sum(); log.info( @@ -549,23 +175,22 @@ public class RedactController { useRegex, wholeWordSearchBool); + String filename = + removeFileExtension( + Objects.requireNonNull( + Filenames.toSimpleFileName( + request.getFileInput().getOriginalFilename()))) + + "_redacted.pdf"; + if (allFoundTextsByPage.isEmpty()) { log.info("No text found matching redaction patterns"); - return WebResponseUtils.pdfDocToWebResponse( - document, - removeFileExtension( - Objects.requireNonNull( - Filenames.toSimpleFileName( - request.getFileInput() - .getOriginalFilename()))) - + "_redacted.pdf", - tempFileManager); + return WebResponseUtils.pdfDocToWebResponse(document, filename, tempFileManager); } boolean fallbackToBoxOnlyMode; try { fallbackToBoxOnlyMode = - performTextReplacement( + textRedactionService.performTextReplacement( document, allFoundTextsByPage, listOfText, @@ -585,44 +210,31 @@ public class RedactController { fallbackDocument = pdfDocumentFactory.load(request.getFileInput()); allFoundTextsByPage = - findTextToRedact( + textRedactionService.findTextToRedact( fallbackDocument, listOfText, useRegex, wholeWordSearchBool); TempFile finalized = - finalizeRedaction( + manualRedactionService.finalizeRedaction( fallbackDocument, allFoundTextsByPage, request.getRedactColor(), request.getCustomPadding(), request.getConvertPDFToImage(), - false); // Box-only mode, use original box sizes + false); - return WebResponseUtils.pdfFileToWebResponse( - finalized, - removeFileExtension( - Objects.requireNonNull( - Filenames.toSimpleFileName( - request.getFileInput() - .getOriginalFilename()))) - + "_redacted.pdf"); + return WebResponseUtils.pdfFileToWebResponse(finalized, filename); } TempFile finalized = - finalizeRedaction( + manualRedactionService.finalizeRedaction( document, allFoundTextsByPage, request.getRedactColor(), request.getCustomPadding(), request.getConvertPDFToImage(), - true); // Text removal mode, use reduced box sizes + true); - return WebResponseUtils.pdfFileToWebResponse( - finalized, - removeFileExtension( - Objects.requireNonNull( - Filenames.toSimpleFileName( - request.getFileInput().getOriginalFilename()))) - + "_redacted.pdf"); + return WebResponseUtils.pdfFileToWebResponse(finalized, filename); } catch (Exception e) { log.error("Redaction operation failed: {}", e.getMessage(), e); @@ -649,1055 +261,33 @@ public class RedactController { } } - private Map> findTextToRedact( - PDDocument document, String[] listOfText, boolean useRegex, boolean wholeWordSearch) { - Map> allFoundTextsByPage = new HashMap<>(); - - for (String text : listOfText) { - text = text.trim(); - if (text.isEmpty()) continue; - - log.debug( - "Searching for text: '{}' (regex: {}, wholeWord: {})", - text, - useRegex, - wholeWordSearch); - - try { - TextFinder textFinder = new TextFinder(text, useRegex, wholeWordSearch); - textFinder.getText(document); - - List foundTexts = textFinder.getFoundTexts(); - log.debug("TextFinder found {} instances of '{}'", foundTexts.size(), text); - - for (PDFText found : foundTexts) { - allFoundTextsByPage - .computeIfAbsent(found.getPageIndex(), k -> new ArrayList<>()) - .add(found); - log.debug( - "Added match on page {} at ({},{},{},{}): '{}'", - found.getPageIndex(), - found.getX1(), - found.getY1(), - found.getX2(), - found.getY2(), - found.getText()); - } - } catch (Exception e) { - log.error("Error processing search term '{}': {}", text, e.getMessage()); - } - } - - return allFoundTextsByPage; - } - - private boolean performTextReplacement( - PDDocument document, - Map> allFoundTextsByPage, - String[] listOfText, - boolean useRegex, - boolean wholeWordSearchBool) { - if (allFoundTextsByPage.isEmpty()) { - return false; - } - - if (detectCustomEncodingFonts(document)) { - log.warn( - "Custom encoded fonts detected (non-standard encodings / DictionaryEncoding / damaged fonts). " - + "Text replacement is unreliable for these fonts. Falling back to box-only redaction mode."); - return true; // signal caller to fall back - } - - try { - Set allSearchTerms = - Arrays.stream(listOfText) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toSet()); - - int pageCount = 0; - for (PDPage page : document.getPages()) { - pageCount++; - List filteredTokens = - createTokensWithoutTargetText( - document, page, allSearchTerms, useRegex, wholeWordSearchBool); - writeFilteredContentStream(document, page, filteredTokens); - } - log.info("Successfully performed text replacement redaction on {} pages.", pageCount); - return false; - } catch (Exception e) { - log.error( - "Text replacement redaction failed due to font or encoding issues. " - + "Will fall back to box-only redaction mode. Error: {}", - e.getMessage()); - return true; - } - } - - private TempFile finalizeRedaction( - PDDocument document, - Map> allFoundTextsByPage, - String colorString, - float customPadding, - Boolean convertToImage, - boolean isTextRemovalMode) + @AutoJobPostMapping( + value = "/redact-execute", + consumes = MediaType.MULTIPART_FORM_DATA_VALUE, + resourceWeight = ResourceWeight.LARGE_WEIGHT) + @StandardPdfResponse + @Operation( + operationId = "redactExecute", + summary = "Execute a unified redaction plan on a PDF", + description = + "Unified redaction endpoint that accepts exact strings, regex patterns, and " + + "page numbers in a single request. Supports execution strategy hints. " + + "Input:PDF Output:PDF Type:SISO") + public ResponseEntity executeRedaction(@ModelAttribute RedactExecuteRequest request) throws IOException { - List allFoundTexts = new ArrayList<>(); - for (List pageTexts : allFoundTextsByPage.values()) { - allFoundTexts.addAll(pageTexts); + if (request.getFileInput() == null) { + throw ExceptionUtils.createFileNullOrEmptyException(); } - if (!allFoundTexts.isEmpty()) { - Color redactColor = decodeOrDefault(colorString); - - redactFoundText(document, allFoundTexts, customPadding, redactColor, isTextRemovalMode); - - cleanDocumentMetadata(document); - } - - if (Boolean.TRUE.equals(convertToImage)) { - try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) { - cleanDocumentMetadata(convertedPdf); - - TempFile tempOut = tempFileManager.createManagedTempFile(".pdf"); - try { - convertedPdf.save(tempOut.getFile()); - } catch (IOException e) { - tempOut.close(); - throw e; - } - - log.info( - "Redaction finalized (image mode): {} pages ➜ {} KB", - convertedPdf.getNumberOfPages(), - tempOut.getFile().length() / 1024); - - return tempOut; - } - } - - TempFile tempOut = tempFileManager.createManagedTempFile(".pdf"); - try { - document.save(tempOut.getFile()); - } catch (IOException e) { - tempOut.close(); - throw e; - } - - log.info( - "Redaction finalized: {} pages ➜ {} KB", - document.getNumberOfPages(), - tempOut.getFile().length() / 1024); - - return tempOut; - } - - private void cleanDocumentMetadata(PDDocument document) { - try { - var documentInfo = document.getDocumentInformation(); - if (documentInfo != null) { - documentInfo.setAuthor(null); - documentInfo.setSubject(null); - documentInfo.setKeywords(null); - - documentInfo.setModificationDate(java.util.Calendar.getInstance()); - - log.debug("Cleaned document metadata for security"); - } - - if (document.getDocumentCatalog() != null) { - try { - document.getDocumentCatalog().setMetadata(null); - } catch (Exception e) { - log.debug("Could not clear XMP metadata: {}", e.getMessage()); - } - } - - } catch (Exception e) { - log.warn("Failed to clean document metadata: {}", e.getMessage()); - } - } - - List createTokensWithoutTargetText( - PDDocument document, - PDPage page, - Set targetWords, - boolean useRegex, - boolean wholeWordSearch) - throws IOException { - - PDFStreamParser parser = new PDFStreamParser(page); - List tokens = new ArrayList<>(); - Object token; - while ((token = parser.parseNextToken()) != null) { - tokens.add(token); - } - - PDResources resources = page.getResources(); - if (resources != null) { - processPageXObjects(document, resources, targetWords, useRegex, wholeWordSearch); - } - - List textSegments = extractTextSegments(page, tokens); - - String completeText = buildCompleteText(textSegments); - - List matches = - findAllMatches(completeText, targetWords, useRegex, wholeWordSearch); - - return applyRedactionsToTokens(tokens, textSegments, matches); - } - - private void processPageXObjects( - PDDocument document, - PDResources resources, - Set targetWords, - boolean useRegex, - boolean wholeWordSearch) { - - for (COSName xobjName : resources.getXObjectNames()) { - try { - PDXObject xobj = resources.getXObject(xobjName); - if (xobj instanceof PDFormXObject formXObj) { - processFormXObject(document, formXObj, targetWords, useRegex, wholeWordSearch); - log.debug("Processed Form XObject: {}", xobjName.getName()); - } - } catch (Exception e) { - log.warn("Failed to process XObject {}: {}", xobjName.getName(), e.getMessage()); - } - } - } - - @Data - private static class GraphicsState { - private PDFont font = null; - private float fontSize = 0; - } - - @Data - @AllArgsConstructor - private static class TextSegment { - private int tokenIndex; - private String operatorName; - private String text; - private int startPos; - private int endPos; - private PDFont font; - private float fontSize; - } - - @Data - @AllArgsConstructor - private static class MatchRange { - private int startPos; - private int endPos; - } - - private List extractTextSegments(PDPage page, List tokens) { - - List segments = new ArrayList<>(); - int currentTextPos = 0; - GraphicsState graphicsState = new GraphicsState(); - PDResources resources = page.getResources(); - - for (int i = 0; i < tokens.size(); i++) { - Object currentToken = tokens.get(i); - - if (currentToken instanceof Operator op) { - String opName = op.getName(); - - if ("Tf".equals(opName) && i >= 2) { - try { - COSName fontName = (COSName) tokens.get(i - 2); - COSBase fontSizeBase = (COSBase) tokens.get(i - 1); - if (fontSizeBase instanceof COSNumber cosNumber) { - graphicsState.setFont(resources.getFont(fontName)); - graphicsState.setFontSize(cosNumber.floatValue()); - } - } catch (ClassCastException | IOException e) { - log.debug( - "Failed to extract font and font size from Tf operator: {}", - e.getMessage()); - } - } - - currentTextPos = - getCurrentTextPos( - tokens, segments, currentTextPos, graphicsState, i, opName); - } - } - - return segments; - } - - private String buildCompleteText(List segments) { - StringBuilder sb = new StringBuilder(); - for (TextSegment segment : segments) { - sb.append(segment.text); - } - return sb.toString(); - } - - private List findAllMatches( - String completeText, - Set targetWords, - boolean useRegex, - boolean wholeWordSearch) { - - // Use the new utility for creating optimized patterns - List patterns = - TextFinderUtils.createOptimizedSearchPatterns( - targetWords, useRegex, wholeWordSearch); - - return patterns.stream() - .flatMap( - pattern -> { - try { - return pattern.matcher(completeText).results(); - } catch (Exception e) { - log.debug( - "Pattern matching failed for pattern {}: {}", - pattern.pattern(), - e.getMessage()); - return java.util.stream.Stream.empty(); - } - }) - .map(matchResult -> new MatchRange(matchResult.start(), matchResult.end())) - .sorted(Comparator.comparingInt(MatchRange::getStartPos)) - .collect(Collectors.toList()); - } - - private List applyRedactionsToTokens( - List tokens, List textSegments, List matches) { - - long startTime = System.currentTimeMillis(); - - try { - List newTokens = new ArrayList<>(tokens); - - Map> matchesBySegment = new HashMap<>(); - for (MatchRange match : matches) { - for (int i = 0; i < textSegments.size(); i++) { - TextSegment segment = textSegments.get(i); - int overlapStart = Math.max(match.startPos, segment.startPos); - int overlapEnd = Math.min(match.endPos, segment.endPos); - if (overlapStart < overlapEnd) { - matchesBySegment.computeIfAbsent(i, k -> new ArrayList<>()).add(match); - } - } - } - - List tasks = new ArrayList<>(); - for (Map.Entry> entry : matchesBySegment.entrySet()) { - int segmentIndex = entry.getKey(); - List segmentMatches = entry.getValue(); - TextSegment segment = textSegments.get(segmentIndex); - - if ("Tj".equals(segment.operatorName) || "'".equals(segment.operatorName)) { - String newText = applyRedactionsToSegmentText(segment, segmentMatches); - try { - float adjustment = calculateWidthAdjustment(segment, segmentMatches); - tasks.add(new ModificationTask(segment, newText, adjustment)); - } catch (Exception e) { - log.debug( - "Width adjustment calculation failed for segment: {}", - e.getMessage()); - } - } else if ("TJ".equals(segment.operatorName)) { - tasks.add(new ModificationTask(segment, null, 0)); - } - } - - tasks.sort((a, b) -> Integer.compare(b.segment.tokenIndex, a.segment.tokenIndex)); - - for (ModificationTask task : tasks) { - List segmentMatches = - matchesBySegment.getOrDefault( - textSegments.indexOf(task.segment), Collections.emptyList()); - modifyTokenForRedaction( - newTokens, task.segment, task.newText, task.adjustment, segmentMatches); - } - - return newTokens; - - } finally { - long processingTime = System.currentTimeMillis() - startTime; - log.debug( - "Token redaction processing completed in {} ms for {} matches", - processingTime, - matches.size()); - } - } - - @Data - @AllArgsConstructor - private static class ModificationTask { - private TextSegment segment; - private String newText; // Only for Tj - private float adjustment; // Only for Tj - } - - private String applyRedactionsToSegmentText(TextSegment segment, List matches) { - String text = segment.getText(); - - if (segment.getFont() != null - && !TextEncodingHelper.isTextSegmentRemovable(segment.getFont(), text)) { - log.debug( - "Skipping text segment '{}' - font {} cannot process this text reliably", - text, - segment.getFont().getName()); - return text; // Return original text unchanged - } - - StringBuilder result = new StringBuilder(text); - - for (MatchRange match : matches) { - int segmentStart = Math.max(0, match.getStartPos() - segment.getStartPos()); - int segmentEnd = Math.min(text.length(), match.getEndPos() - segment.getStartPos()); - - if (segmentStart < text.length() && segmentEnd > segmentStart) { - String originalPart = text.substring(segmentStart, segmentEnd); - - if (segment.getFont() != null - && !TextEncodingHelper.isTextSegmentRemovable( - segment.getFont(), originalPart)) { - log.debug( - "Skipping text part '{}' within segment - cannot be processed reliably", - originalPart); - continue; // Skip this match, process others - } - - float originalWidth = 0; - if (segment.getFont() != null && segment.getFontSize() > 0) { - try { - originalWidth = - safeGetStringWidth(segment.getFont(), originalPart) - / FONT_SCALE_FACTOR - * segment.getFontSize(); - } catch (Exception e) { - log.debug( - "Failed to calculate original width for placeholder: {}", - e.getMessage()); - } - } - - String placeholder = - (originalWidth > 0) - ? createPlaceholderWithWidth( - originalPart, - originalWidth, - segment.getFont(), - segment.getFontSize()) - : createPlaceholderWithFont(originalPart, segment.getFont()); - - result.replace(segmentStart, segmentEnd, placeholder); - } - } - - return result.toString(); - } - - private float safeGetStringWidth(PDFont font, String text) { - if (font == null || text == null || text.isEmpty()) { - return 0; - } - - if (!WidthCalculator.isWidthCalculationReliable(font)) { - log.debug( - "Font {} flagged as unreliable for width calculation, using fallback", - font.getName()); - return calculateConservativeWidth(font, text); - } - - if (!TextEncodingHelper.canEncodeCharacters(font, text)) { - log.debug( - "Text cannot be encoded by font {}, using character-based fallback", - font.getName()); - return calculateCharacterBasedWidth(font, text); - } - - try { - float width = font.getStringWidth(text); - log.debug("Direct width calculation successful for '{}': {}", text, width); - return width; - - } catch (Exception e) { - log.debug( - "Direct width calculation failed for font {}: {}", - font.getName(), - e.getMessage()); - return calculateFallbackWidth(font, text); - } - } - - private float calculateCharacterBasedWidth(PDFont font, String text) { - try { - float totalWidth = 0; - for (int i = 0; i < text.length(); i++) { - String character = text.substring(i, i + 1); - try { - // Validate character encoding first - if (!TextEncodingHelper.fontSupportsCharacter(font, character)) { - totalWidth += font.getAverageFontWidth(); - continue; - } - - byte[] encoded = font.encode(character); - if (encoded.length > 0) { - int glyphCode = encoded[0] & 0xFF; - float glyphWidth = font.getWidth(glyphCode); - - // Try alternative width methods if primary fails - if (glyphWidth == 0) { - try { - glyphWidth = font.getWidthFromFont(glyphCode); - } catch (Exception e2) { - glyphWidth = font.getAverageFontWidth(); - } - } - - totalWidth += glyphWidth; - } else { - totalWidth += font.getAverageFontWidth(); - } - } catch (Exception e2) { - // Character processing failed, use average width - totalWidth += font.getAverageFontWidth(); - } - } - - log.debug("Character-based width calculation: {}", totalWidth); - return totalWidth; - - } catch (Exception e) { - log.debug("Character-based width calculation failed: {}", e.getMessage()); - return calculateConservativeWidth(font, text); - } - } - - private float calculateFallbackWidth(PDFont font, String text) { - try { - // Method 1: Font bounding box approach - if (font.getFontDescriptor() != null - && font.getFontDescriptor().getFontBoundingBox() != null) { - - PDRectangle bbox = font.getFontDescriptor().getFontBoundingBox(); - float avgCharWidth = bbox.getWidth() * 0.6f; // Conservative estimate - float fallbackWidth = text.length() * avgCharWidth; - - log.debug("Bounding box fallback width: {}", fallbackWidth); - return fallbackWidth; - } - - // Method 2: Average font width - try { - float avgWidth = font.getAverageFontWidth(); - if (avgWidth > 0) { - float fallbackWidth = text.length() * avgWidth; - log.debug("Average width fallback: {}", fallbackWidth); - return fallbackWidth; - } - } catch (Exception e2) { - log.debug("Average font width calculation failed: {}", e2.getMessage()); - } - - // Method 3: Conservative estimate based on font metrics - return calculateConservativeWidth(font, text); - - } catch (Exception e) { - log.debug("Fallback width calculation failed: {}", e.getMessage()); - return calculateConservativeWidth(font, text); - } - } - - private float calculateConservativeWidth(PDFont font, String text) { - float conservativeWidth = text.length() * 500f; - - log.debug( - "Conservative width estimate for font {} text '{}': {}", - font.getName(), - text, - conservativeWidth); - return conservativeWidth; - } - - private float calculateWidthAdjustment(TextSegment segment, List matches) { - try { - if (segment.getFont() == null || segment.getFontSize() <= 0) { - return 0; - } - - String fontName = segment.getFont().getName(); - if (fontName != null - && (fontName.contains("HOEPAP") || TextEncodingHelper.isFontSubset(fontName))) { - log.debug("Skipping width adjustment for problematic/subset font: {}", fontName); - return 0; - } - - float totalOriginal = 0; - float totalPlaceholder = 0; - - String text = segment.getText(); - - for (MatchRange match : matches) { - int segStart = Math.max(0, match.getStartPos() - segment.getStartPos()); - int segEnd = Math.min(text.length(), match.getEndPos() - segment.getStartPos()); - - if (segStart < text.length() && segEnd > segStart) { - String originalPart = text.substring(segStart, segEnd); - - float originalWidth = - safeGetStringWidth(segment.getFont(), originalPart) - / FONT_SCALE_FACTOR - * segment.getFontSize(); - - String placeholderPart = - createPlaceholderWithWidth( - originalPart, - originalWidth, - segment.getFont(), - segment.getFontSize()); - - float origUnits = safeGetStringWidth(segment.getFont(), originalPart); - float placeUnits = safeGetStringWidth(segment.getFont(), placeholderPart); - - float orig = (origUnits / FONT_SCALE_FACTOR) * segment.getFontSize(); - float place = (placeUnits / FONT_SCALE_FACTOR) * segment.getFontSize(); - - totalOriginal += orig; - totalPlaceholder += place; - } - } - - float adjustment = totalOriginal - totalPlaceholder; - - float maxReasonableAdjustment = - Math.max( - segment.getText().length() * segment.getFontSize() * 2, - totalOriginal * 1.5f // Allow up to 50% more than original width - ); - - if (Math.abs(adjustment) > maxReasonableAdjustment) { - log.debug( - "Width adjustment {} seems unreasonable for text length {}, capping to 0", - adjustment, - segment.getText().length()); - return 0; - } - - return adjustment; - } catch (Exception ex) { - log.debug("Width adjustment failed: {}", ex.getMessage()); - return 0; - } - } - - private void modifyTokenForRedaction( - List tokens, - TextSegment segment, - String newText, - float adjustment, - List matches) { - - if (segment.getTokenIndex() < 0 || segment.getTokenIndex() >= tokens.size()) { - return; - } - - Object token = tokens.get(segment.getTokenIndex()); - String operatorName = segment.getOperatorName(); - - try { - if (("Tj".equals(operatorName) || "'".equals(operatorName)) - && token instanceof COSString) { - - if (Math.abs(adjustment) < PRECISION_THRESHOLD) { - if (newText.isEmpty()) { - tokens.set(segment.getTokenIndex(), EMPTY_COS_STRING); - } else { - tokens.set(segment.getTokenIndex(), new COSString(newText)); - } - } else { - COSArray newArray = new COSArray(); - newArray.add(new COSString(newText)); - if (segment.getFontSize() > 0) { - - float kerning = (-adjustment / segment.getFontSize()) * FONT_SCALE_FACTOR; - - newArray.add(new COSFloat(kerning)); - } - tokens.set(segment.getTokenIndex(), newArray); - - int operatorIndex = segment.getTokenIndex() + 1; - if (operatorIndex < tokens.size() - && tokens.get(operatorIndex) instanceof Operator op - && op.getName().equals(operatorName)) { - tokens.set(operatorIndex, Operator.getOperator("TJ")); - } - } - } else if ("TJ".equals(operatorName) && token instanceof COSArray) { - COSArray newArray = createRedactedTJArray((COSArray) token, segment, matches); - tokens.set(segment.getTokenIndex(), newArray); - } - } catch (Exception e) { - log.debug( - "Token modification failed for segment at index {}: {}", - segment.getTokenIndex(), - e.getMessage()); - } - } - - private COSArray createRedactedTJArray( - COSArray originalArray, TextSegment segment, List matches) { - try { - COSArray newArray = new COSArray(); - int textOffsetInSegment = 0; - - for (COSBase element : originalArray) { - if (element instanceof COSString cosString) { - String originalText = cosString.getString(); - - if (segment.getFont() != null - && !TextEncodingHelper.isTextSegmentRemovable( - segment.getFont(), originalText)) { - log.debug( - "Skipping TJ text part '{}' - cannot be processed reliably with font {}", - originalText, - segment.getFont().getName()); - newArray.add(element); // Keep original unchanged - textOffsetInSegment += originalText.length(); - continue; - } - - StringBuilder newText = new StringBuilder(originalText); - boolean modified = false; - - for (MatchRange match : matches) { - int stringStartInPage = segment.getStartPos() + textOffsetInSegment; - int stringEndInPage = stringStartInPage + originalText.length(); - - int overlapStart = Math.max(match.getStartPos(), stringStartInPage); - int overlapEnd = Math.min(match.getEndPos(), stringEndInPage); - - if (overlapStart < overlapEnd) { - int redactionStartInString = overlapStart - stringStartInPage; - int redactionEndInString = overlapEnd - stringStartInPage; - if (redactionStartInString >= 0 - && redactionEndInString <= originalText.length()) { - String originalPart = - originalText.substring( - redactionStartInString, redactionEndInString); - - if (segment.getFont() != null - && !TextEncodingHelper.isTextSegmentRemovable( - segment.getFont(), originalPart)) { - log.debug( - "Skipping TJ text part '{}' - cannot be redacted reliably", - originalPart); - continue; // Skip this redaction, keep original text - } - - modified = true; - float originalWidth = 0; - if (segment.getFont() != null && segment.getFontSize() > 0) { - try { - originalWidth = - safeGetStringWidth(segment.getFont(), originalPart) - / FONT_SCALE_FACTOR - * segment.getFontSize(); - } catch (Exception e) { - log.debug( - "Failed to calculate original width for TJ placeholder: {}", - e.getMessage()); - } - } - - String placeholder = - (originalWidth > 0) - ? createPlaceholderWithWidth( - originalPart, - originalWidth, - segment.getFont(), - segment.getFontSize()) - : createPlaceholderWithFont( - originalPart, segment.getFont()); - - newText.replace( - redactionStartInString, redactionEndInString, placeholder); - } - } - } - - String modifiedString = newText.toString(); - newArray.add(new COSString(modifiedString)); - - if (modified && segment.getFont() != null && segment.getFontSize() > 0) { - try { - float originalWidth = - safeGetStringWidth(segment.getFont(), originalText) - / FONT_SCALE_FACTOR - * segment.getFontSize(); - float modifiedWidth = - safeGetStringWidth(segment.getFont(), modifiedString) - / FONT_SCALE_FACTOR - * segment.getFontSize(); - float adjustment = originalWidth - modifiedWidth; - if (Math.abs(adjustment) > PRECISION_THRESHOLD) { - float kerning = - (-adjustment / segment.getFontSize()) - * FONT_SCALE_FACTOR - * 1.10f; - - newArray.add(new COSFloat(kerning)); - } - } catch (Exception e) { - log.debug( - "Width adjustment calculation failed for segment: {}", - e.getMessage()); - } - } - - textOffsetInSegment += originalText.length(); - } else { - newArray.add(element); - } - } - return newArray; - } catch (Exception e) { - return originalArray; - } - } - - private String extractTextFromToken(Object token, String operatorName) { - return switch (operatorName) { - case "Tj", "'" -> { - if (token instanceof COSString cosString) { - yield cosString.getString(); - } - yield ""; - } - case "TJ" -> { - if (token instanceof COSArray cosArray) { - StringBuilder sb = new StringBuilder(); - for (COSBase element : cosArray) { - if (element instanceof COSString cosString) { - sb.append(cosString.getString()); - } - } - yield sb.toString(); - } - yield ""; - } - default -> ""; - }; - } - - private boolean detectCustomEncodingFonts(PDDocument document) { - try { - var documentCatalog = document.getDocumentCatalog(); - if (documentCatalog == null) { - return false; - } - - int totalFonts = 0; - int customEncodedFonts = 0; - int subsetFonts = 0; - int unreliableFonts = 0; - - for (PDPage page : document.getPages()) { - if (TextFinderUtils.hasProblematicFonts(page)) { - log.debug("Page contains fonts flagged as problematic by TextFinderUtils"); - } - - PDResources resources = page.getResources(); - if (resources == null) { - continue; - } - - for (COSName fontName : resources.getFontNames()) { - try { - PDFont font = resources.getFont(fontName); - if (font != null) { - totalFonts++; - - // Enhanced analysis using helper classes - boolean isSubset = TextEncodingHelper.isFontSubset(font.getName()); - boolean hasCustomEncoding = TextEncodingHelper.hasCustomEncoding(font); - boolean isReliable = WidthCalculator.isWidthCalculationReliable(font); - boolean canCalculateWidths = - TextEncodingHelper.canCalculateBasicWidths(font); - - if (isSubset) { - subsetFonts++; - } - - if (hasCustomEncoding) { - customEncodedFonts++; - log.debug("Font {} has custom encoding", font.getName()); - } - - if (!isReliable || !canCalculateWidths) { - unreliableFonts++; - log.debug( - "Font {} flagged as unreliable: reliable={}, canCalculateWidths={}", - font.getName(), - isReliable, - canCalculateWidths); - } - - if (!TextFinderUtils.validateFontReliability(font)) { - log.debug( - "Font {} failed comprehensive reliability check", - font.getName()); - } - } - } catch (Exception e) { - log.debug( - "Font loading/analysis failed for {}: {}", - fontName.getName(), - e.getMessage()); - customEncodedFonts++; - unreliableFonts++; - totalFonts++; - } - } - } - - log.info( - "Enhanced font analysis: {}/{} custom encoding, {}/{} subset, {}/{} unreliable fonts", - customEncodedFonts, - totalFonts, - subsetFonts, - totalFonts, - unreliableFonts, - totalFonts); - - // Consider document problematic if we have custom encodings or unreliable fonts - return customEncodedFonts > 0 || unreliableFonts > 0; - - } catch (Exception e) { - log.warn("Enhanced font detection analysis failed: {}", e.getMessage()); - return true; // Assume problematic if analysis fails - } - } - - private void processFormXObject( - PDDocument document, - PDFormXObject formXObject, - Set targetWords, - boolean useRegex, - boolean wholeWordSearch) { - - try { - PDResources xobjResources = formXObject.getResources(); - if (xobjResources == null) { - return; - } - - for (COSName xobjName : xobjResources.getXObjectNames()) { - PDXObject nestedXObj = xobjResources.getXObject(xobjName); - if (nestedXObj instanceof PDFormXObject nestedFormXObj) { - processFormXObject( - document, nestedFormXObj, targetWords, useRegex, wholeWordSearch); - } - } - - PDFStreamParser parser = new PDFStreamParser(formXObject); - List tokens = new ArrayList<>(); - Object token; - while ((token = parser.parseNextToken()) != null) { - tokens.add(token); - } - - List textSegments = extractTextSegmentsFromXObject(xobjResources, tokens); - String completeText = buildCompleteText(textSegments); - - List matches = - findAllMatches(completeText, targetWords, useRegex, wholeWordSearch); - - if (!matches.isEmpty()) { - List redactedTokens = - applyRedactionsToTokens(tokens, textSegments, matches); - writeRedactedContentToXObject(document, formXObject, redactedTokens); - log.debug("Processed {} redactions in Form XObject", matches.size()); - } - - } catch (Exception e) { - log.warn("Failed to process Form XObject: {}", e.getMessage()); - } - } - - private List extractTextSegmentsFromXObject( - PDResources resources, List tokens) { - List segments = new ArrayList<>(); - int currentTextPos = 0; - GraphicsState graphicsState = new GraphicsState(); - - for (int i = 0; i < tokens.size(); i++) { - Object currentToken = tokens.get(i); - - if (currentToken instanceof Operator op) { - String opName = op.getName(); - - if ("Tf".equals(opName) && i >= 2) { - try { - COSName fontName = (COSName) tokens.get(i - 2); - COSBase fontSizeBase = (COSBase) tokens.get(i - 1); - if (fontSizeBase instanceof COSNumber cosNumber) { - graphicsState.setFont(resources.getFont(fontName)); - graphicsState.setFontSize(cosNumber.floatValue()); - } - } catch (ClassCastException | IOException e) { - log.debug("Font extraction failed in XObject: {}", e.getMessage()); - } - } - - currentTextPos = - getCurrentTextPos( - tokens, segments, currentTextPos, graphicsState, i, opName); - } - } - - return segments; - } - - private int getCurrentTextPos( - List tokens, - List segments, - int currentTextPos, - GraphicsState graphicsState, - int i, - String opName) { - if (isTextShowingOperator(opName) && i > 0) { - String textContent = extractTextFromToken(tokens.get(i - 1), opName); - if (!textContent.isEmpty()) { - segments.add( - new TextSegment( - i - 1, - opName, - textContent, - currentTextPos, - currentTextPos + textContent.length(), - graphicsState.font, - graphicsState.fontSize)); - currentTextPos += textContent.length(); - } - } - return currentTextPos; - } - - private void writeRedactedContentToXObject( - PDDocument document, PDFormXObject formXObject, List redactedTokens) - throws IOException { - - PDStream newStream = new PDStream(document); - - try (var out = newStream.createOutputStream()) { - ContentStreamWriter writer = new ContentStreamWriter(out); - writer.writeTokens(redactedTokens); - } - - formXObject.getCOSObject().removeItem(COSName.CONTENTS); - formXObject.getCOSObject().setItem(COSName.CONTENTS, newStream.getCOSObject()); + String filename = + removeFileExtension( + Objects.requireNonNull( + Filenames.toSimpleFileName( + request.getFileInput().getOriginalFilename()))) + + "_redacted.pdf"; + + TempFile out = redactExecuteService.execute(request); + return WebResponseUtils.pdfFileToWebResponse(out, filename); } } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java new file mode 100644 index 000000000..4a53be97b --- /dev/null +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/RedactExecuteService.java @@ -0,0 +1,850 @@ +package stirling.software.SPDF.controller.api.security; + +import java.awt.Color; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.PDPageTree; +import org.apache.pdfbox.pdmodel.PDResources; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.SPDF.model.PDFText; +import stirling.software.SPDF.model.api.security.RedactExecuteRequest; +import stirling.software.SPDF.model.api.security.RedactExecuteRequest.ImageBox; +import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyle; +import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange; +import stirling.software.SPDF.pdf.parser.PageColumnLayout; +import stirling.software.SPDF.pdf.parser.PageImageLocator; +import stirling.software.common.service.CustomPDFDocumentFactory; +import stirling.software.common.util.ExceptionUtils; +import stirling.software.common.util.TempFile; + +@Service +@Slf4j +@RequiredArgsConstructor +class RedactExecuteService { + + private final CustomPDFDocumentFactory pdfDocumentFactory; + private final ManualRedactionService manualRedactionService; + private final TextRedactionService textRedactionService; + + TempFile execute(RedactExecuteRequest request) throws IOException { + RedactStyle style = request.getStyle() != null ? request.getStyle() : new RedactStyle(); + List textValues = orEmpty(request.getTextValues()); + List regexPatterns = orEmpty(request.getRegexPatterns()); + List wipePages = orEmpty(request.getWipePages()); + List ranges = orEmpty(request.getRanges()); + List imageBoxes = orEmpty(request.getImageBoxes()); + + boolean hasTargets = + !textValues.isEmpty() + || !regexPatterns.isEmpty() + || !wipePages.isEmpty() + || !ranges.isEmpty() + || !imageBoxes.isEmpty() + || request.getRedactImagePages() != null; + + if (!hasTargets) { + throw ExceptionUtils.createIllegalArgumentException( + "error.redaction.no.targets", "No redaction targets provided"); + } + + boolean overlayOnly = + RedactExecuteRequest.RedactionStrategy.OVERLAY_ONLY.equals(style.getStrategy()); + boolean imageFinalize = + RedactExecuteRequest.RedactionStrategy.IMAGE_FINALIZE.equals(style.getStrategy()); + boolean convertToImage = imageFinalize || style.isConvertToImage(); + + boolean hasTextOps = !textValues.isEmpty() || !regexPatterns.isEmpty(); + + log.info( + "[redact/execute] strategy={} textValues={} regexPatterns={} wipePages={} ranges={} imageBoxes={} imagePages={}", + style.getStrategy(), + textValues.size(), + regexPatterns.size(), + wipePages.size(), + ranges.size(), + imageBoxes.size(), + request.getRedactImagePages()); + + if (request.getFileInput() == null) { + throw ExceptionUtils.createFileNullOrEmptyException(); + } + + PDDocument document = null; + try { + document = pdfDocumentFactory.load(request.getFileInput()); + + // Single-pass text scan: collect all text-based targets so we run the PDF + // stripper only once across the entire execute() call rather than once per target. + Map> foundTexts = + hasTextOps ? collectTextMatches(document, request) : new HashMap<>(); + + int totalMatches = foundTexts.values().stream().mapToInt(List::size).sum(); + log.info( + "[redact/execute] scan complete: {} text matches across {} pages", + totalMatches, + foundTexts.size()); + + // Text removal (content-stream rewriting) — skipped in overlay-only mode. + boolean needsOverlayOnly = overlayOnly; + if (hasTextOps && !foundTexts.isEmpty() && !overlayOnly) { + needsOverlayOnly = applyTextRemoval(document, request); + } else if (overlayOnly) { + log.info( + "[redact/execute] overlay-only mode requested — skipping content-stream rewriting"); + } + + // Reload fresh document on fallback so we overlay onto clean content. + if (needsOverlayOnly && !foundTexts.isEmpty()) { + log.info("[redact/execute] reloading document for clean overlay pass"); + document.close(); + document = pdfDocumentFactory.load(request.getFileInput()); + foundTexts.clear(); + if (hasTextOps) { + foundTexts.putAll(collectTextMatches(document, request)); + } + } + + // Non-text operations. + Map layoutCache = new HashMap<>(); + + if (!wipePages.isEmpty()) { + applyPageWipe(document, wipePages, style); + } + + for (TextRange range : ranges) { + applyRangeRedaction(document, range, style, layoutCache); + } + + for (ImageBox box : imageBoxes) { + applyImageBoxRedaction(document, box, style); + } + + if (request.getRedactImagePages() != null) { + applyAllImagesRedaction(document, request.getRedactImagePages(), style); + } + + return manualRedactionService.finalizeRedaction( + document, + foundTexts, + style.getColor(), + style.getPadding(), + convertToImage, + !needsOverlayOnly); + + } catch (Exception e) { + log.error("Execute redaction failed: {}", e.getMessage(), e); + throw new RuntimeException("Failed to perform PDF redaction: " + e.getMessage(), e); + } finally { + if (document != null) { + try { + document.close(); + } catch (IOException e) { + log.warn("Failed to close document: {}", e.getMessage()); + } + } + } + } + + // ----------------------------------------------------------------------- + // Single-pass text scan (one stripper pass per execute() call) + // ----------------------------------------------------------------------- + + /** + * Runs a single PDF text-stripper pass over all text-based targets and returns the merged hit + * map. + */ + private Map> collectTextMatches( + PDDocument document, RedactExecuteRequest request) { + Map> found = new HashMap<>(); + + String[] terms = cleanStrings(request.getTextValues()); + if (terms.length > 0) { + textRedactionService + .findTextToRedact(document, terms, false, false) + .forEach( + (page, hits) -> + found.computeIfAbsent(page, k -> new ArrayList<>()) + .addAll(hits)); + } + + String[] patterns = cleanStrings(request.getRegexPatterns()); + if (patterns.length > 0) { + textRedactionService + .findTextToRedact(document, patterns, true, false) + .forEach( + (page, hits) -> + found.computeIfAbsent(page, k -> new ArrayList<>()) + .addAll(hits)); + } + + return found; + } + + // ----------------------------------------------------------------------- + // Text removal (content-stream rewriting) + // ----------------------------------------------------------------------- + + /** + * Attempts content-stream text removal for all text/regex targets. Returns {@code true} if the + * document fell back to overlay-only mode. + */ + private boolean applyTextRemoval(PDDocument document, RedactExecuteRequest request) { + try { + boolean fallback = false; + + String[] terms = cleanStrings(request.getTextValues()); + if (terms.length > 0) { + Map> exactFound = + textRedactionService.findTextToRedact(document, terms, false, false); + if (!exactFound.isEmpty()) { + fallback |= + textRedactionService.performTextReplacement( + document, exactFound, terms, false, false); + } + } + + String[] patterns = cleanStrings(request.getRegexPatterns()); + if (patterns.length > 0) { + Map> regexFound = + textRedactionService.findTextToRedact(document, patterns, true, false); + if (!regexFound.isEmpty()) { + fallback |= + textRedactionService.performTextReplacement( + document, regexFound, patterns, true, false); + } + } + + if (fallback) { + log.warn( + "[redact/execute] font compatibility issue — falling back to overlay-only"); + } else { + log.info("[redact/execute] content-stream text removal applied successfully"); + } + return fallback; + } catch (Exception e) { + log.warn( + "[redact/execute] text removal failed, falling back to overlay: {}", + e.getMessage()); + return true; + } + } + + // ----------------------------------------------------------------------- + // Per-operation dispatch methods + // ----------------------------------------------------------------------- + + private void applyPageWipe(PDDocument document, List pageNumbers, RedactStyle style) + throws IOException { + List pageIndices = toZeroBasedIndices(pageNumbers); + if (pageIndices.isEmpty()) return; + + PDPageTree allPages = document.getDocumentCatalog().getPages(); + Color pageColor = ManualRedactionService.decodeOrDefault(style.getColor()); + Collections.sort(pageIndices); + log.info("[redact/execute] full-page wipe: {} pages ({})", pageIndices.size(), pageIndices); + + Map> pageElementBoxes = new HashMap<>(); + for (Integer idx : pageIndices) { + if (idx >= 0 && idx < allPages.getCount()) { + try { + pageElementBoxes.put( + idx, + manualRedactionService.extractPageElementBoxes( + document, allPages.get(idx), idx)); + } catch (Exception e) { + log.warn( + "[redact/execute] element extraction failed for page {}: {}", + idx, + e.getMessage()); + } + } + } + + for (Integer idx : pageIndices) { + if (idx >= 0 && idx < allPages.getCount()) { + PDPage page = allPages.get(idx); + List elementBoxes = + pageElementBoxes.getOrDefault(idx, Collections.emptyList()); + page.getCOSObject().removeItem(COSName.CONTENTS); + page.setResources(new PDResources()); + try (PDPageContentStream cs = new PDPageContentStream(document, page)) { + cs.setNonStrokingColor(pageColor); + if (elementBoxes.isEmpty()) { + PDRectangle box = page.getBBox(); + cs.addRect(0, 0, box.getWidth(), box.getHeight()); + } else { + log.info( + "[redact/execute] page {}: drawing {} element boxes", + idx + 1, + elementBoxes.size()); + for (float[] r : elementBoxes) { + cs.addRect(r[0], r[1], r[2] - r[0], r[3] - r[1]); + } + } + cs.fill(); + } + } + } + } + + private void applyRangeRedaction( + PDDocument document, + TextRange range, + RedactStyle style, + Map layoutCache) + throws IOException { + String rangeStart = trimOrEmpty(range.startString()); + String rangeEnd = trimOrEmpty(range.endString()); + log.info("[redact/execute] range redaction: start='{}' end='{}'", rangeStart, rangeEnd); + try { + List blocks = collectRangeBlocks(document, rangeStart, rangeEnd, layoutCache); + if (!blocks.isEmpty()) { + manualRedactionService.redactFoundText( + document, + blocks, + style.getPadding(), + ManualRedactionService.decodeOrDefault(style.getColor()), + false); + } else { + log.warn( + "[redact/execute] range not found: start='{}' end='{}'", + rangeStart, + rangeEnd); + } + } catch (Exception e) { + log.warn("[redact/execute] range redaction failed: {}", e.getMessage()); + } + } + + private void applyImageBoxRedaction(PDDocument document, ImageBox box, RedactStyle style) + throws IOException { + List boxes = + List.of( + new float[] { + (float) box.pageIndex(), box.x1(), box.y1(), box.x2(), box.y2() + }); + log.info("[redact/execute] image box overlay on page {}", box.pageIndex()); + Color boxColor = ManualRedactionService.decodeOrDefault(style.getColor()); + manualRedactionService.redactImageBoxes(document, boxes, boxColor); + } + + private void applyAllImagesRedaction( + PDDocument document, List pageNumbers, RedactStyle style) throws IOException { + PDPageTree allPages = document.getDocumentCatalog().getPages(); + Color imgColor = ManualRedactionService.decodeOrDefault(style.getColor()); + + List imagePageIndices = toZeroBasedIndices(pageNumbers); + if (imagePageIndices.isEmpty()) { + imagePageIndices = new ArrayList<>(); + for (int i = 0; i < allPages.getCount(); i++) { + imagePageIndices.add(i); + } + } + + List detectedBoxes = new ArrayList<>(); + for (int pageIdx : imagePageIndices) { + if (pageIdx < 0 || pageIdx >= allPages.getCount()) continue; + try { + PDPage page = allPages.get(pageIdx); + PageImageLocator locator = new PageImageLocator(page, pageIdx); + locator.processPage(page); + for (PageImageLocator.ImageBox ib : locator.getImageBoxes()) { + detectedBoxes.add(new float[] {pageIdx, ib.x1(), ib.y1(), ib.x2(), ib.y2()}); + } + } catch (Exception e) { + log.warn( + "[redact/execute] image detection failed for page {}: {}", + pageIdx + 1, + e.getMessage()); + } + } + + log.info( + "[redact/execute] auto image detection: {} images across {} pages", + detectedBoxes.size(), + imagePageIndices.size()); + + if (!detectedBoxes.isEmpty()) { + manualRedactionService.redactImageBoxes(document, detectedBoxes, imgColor); + } + } + + // ----------------------------------------------------------------------- + // Range collection helpers + // ----------------------------------------------------------------------- + + /** + * Locates {@code startStr} in the document and returns {@link PDFText} blocks for every text + * line and image from that point up to (but NOT including) the line where {@code endStr} + * begins. If {@code endStr} is blank, redacts from {@code startStr} to the end of the document. + * + *

Multi-column pages follow reading order: down the start column, jump to the top of the + * next column, continue to the end anchor. Single-column pages reduce to a plain Y-band check. + */ + List collectRangeBlocks( + PDDocument document, + String startStr, + String endStr, + Map layoutCache) + throws IOException { + + PDPageTree allPages = document.getDocumentCatalog().getPages(); + int totalPages = allPages.getCount(); + + Map> startMatchesByPage = findWithFallbacks(document, startStr); + if (startMatchesByPage.isEmpty()) { + log.warn("[redact/execute] range start not found: '{}'", startStr); + return Collections.emptyList(); + } + + List starts = toAnchors(document, startMatchesByPage, layoutCache); + starts.sort(READING_ORDER); + log.info( + "[redact/execute] start='{}' matched {} anchor(s): {}", + startStr, + starts.size(), + anchorSummary(starts)); + + boolean openEnded = (endStr == null || endStr.isBlank()); + List ends = new ArrayList<>(); + if (!openEnded) { + Map> endMatchesByPage = findWithFallbacks(document, endStr); + if (endMatchesByPage.isEmpty()) { + log.warn( + "[redact/execute] range end '{}' not found in document - skipping range" + + " (start='{}')", + endStr, + startStr); + return Collections.emptyList(); + } + ends = toAnchors(document, endMatchesByPage, layoutCache); + ends.sort(READING_ORDER); + log.info( + "[redact/execute] end='{}' matched {} anchor(s): {}", + endStr, + ends.size(), + anchorSummary(ends)); + } + + List blocks = new ArrayList<>(); + for (Anchor start : starts) { + Anchor end = null; + int endPage; + if (openEnded) { + endPage = totalPages - 1; + } else { + for (Anchor candidate : ends) { + if (READING_ORDER.compare(candidate, start) > 0) { + end = candidate; + break; + } + } + if (end == null) { + log.warn( + "[redact/execute] no end anchor after start at (page={}, col={}, y={}) — skipping", + start.page + 1, + start.col, + start.y); + continue; + } + endPage = end.page; + } + + log.info( + "[redact/execute] range pages {}-{}: start='{}' (col {}) end='{}'", + start.page + 1, + endPage + 1, + startStr, + start.col, + openEnded ? "" : endStr); + + collectBlocksForRange(document, allPages, start, end, openEnded, blocks, layoutCache); + } + + log.info( + "[redact/execute] range '{}'→'{}': {} total blocks", + startStr, + openEnded ? "" : endStr, + blocks.size()); + return blocks; + } + + /** + * Collects all redactable content (text line segments and images) between two anchor positions. + * + *

Line boxes are cached per page number in {@code lineBoxCache} and reused across range + * iterations within one execute() call, avoiding redundant {@link AllTextLineExtractor} passes. + */ + private void collectBlocksForRange( + PDDocument document, + PDPageTree allPages, + Anchor start, + Anchor end, + boolean openEnded, + List blocks, + Map layoutCache) + throws IOException { + + int startPage = start.page; + int endPage = openEnded ? allPages.getCount() - 1 : end.page; + int endCol = + openEnded ? layoutFor(document, endPage, layoutCache).columnCount() - 1 : end.col; + float startY = start.y; + // Use bottom of end anchor so the end anchor line itself is included (inclusive range). + float endY = openEnded ? Float.POSITIVE_INFINITY : end.text.getY2(); + + // Line-box cache: populated lazily per page, reused across range iterations. + // Cannot use computeIfAbsent because AllTextLineExtractor's constructor throws IOException. + Map> lineBoxCache = new HashMap<>(); + + for (int pageIdx = startPage; pageIdx <= endPage; pageIdx++) { + PDPage page = allPages.get(pageIdx); + float pageHeight = page.getBBox().getHeight(); + PageColumnLayout layout = layoutFor(document, pageIdx, layoutCache); + + List screenLineBoxes = lineBoxCache.get(pageIdx); + if (screenLineBoxes == null) { + AllTextLineExtractor textExtractor = + new AllTextLineExtractor(pageIdx + 1, pageHeight); + textExtractor.getText(document); + screenLineBoxes = textExtractor.getScreenLineBoxes(); + lineBoxCache.put(pageIdx, screenLineBoxes); + } + + for (float[] sb : screenLineBoxes) { + emitColumnSlices( + pageIdx, layout, sb[0], sb[2], sb[1], sb[3], start.col, startPage, startY, + endCol, endPage, endY, blocks); + } + + PageImageLocator imgLocator = new PageImageLocator(page, pageIdx); + imgLocator.processPage(page); + for (PageImageLocator.ImageBox ib : imgLocator.getImageBoxes()) { + // ImageBox coordinates are in PDF user-space (Y up); convert to screen-Y (Y down). + float screenY1 = pageHeight - ib.y2(); + float screenY2 = pageHeight - ib.y1(); + emitColumnSlices( + pageIdx, layout, ib.x1(), ib.x2(), screenY1, screenY2, start.col, startPage, + startY, endCol, endPage, endY, blocks); + } + } + } + + /** Emits each per-column sub-box accepted by the reading-order predicate. */ + private static void emitColumnSlices( + int pageIdx, + PageColumnLayout layout, + float x1, + float x2, + float yTop, + float yBottom, + int startCol, + int startPage, + float startY, + int endCol, + int endPage, + float endY, + List blocks) { + int[] cols = layout.columnsCrossing(x1, x2); + if (cols.length == 1) { + if (inColumnZone( + pageIdx, cols[0], yTop, yBottom, startPage, startCol, startY, endPage, endCol, + endY)) { + blocks.add(new PDFText(pageIdx, x1, yTop, x2, yBottom, "")); + } + return; + } + for (int col : cols) { + if (!inColumnZone( + pageIdx, col, yTop, yBottom, startPage, startCol, startY, endPage, endCol, + endY)) { + return; + } + } + blocks.add(new PDFText(pageIdx, x1, yTop, x2, yBottom, "")); + } + + /** + * Reading-order predicate: true when (col, yBottom) on page {@code pageIdx} sits between the + * start anchor (inclusive) and end anchor (inclusive). + */ + static boolean inColumnZone( + int pageIdx, + int col, + float yTop, + float yBottom, + int startPage, + int startCol, + float startY, + int endPage, + int endCol, + float endY) { + if (pageIdx > startPage && pageIdx < endPage) return true; + if (pageIdx == startPage && pageIdx == endPage) { + if (startCol == endCol) { + return col == startCol && yBottom >= startY && yBottom <= endY; + } + if (startCol < endCol) { + if (col < startCol || col > endCol) return false; + if (col == startCol) return yBottom >= startY; + if (col == endCol) return yBottom <= endY; + return true; + } + return col == startCol && yBottom >= startY; + } + if (pageIdx == startPage) { + if (col == startCol) return yBottom >= startY; + return col > startCol; + } + if (pageIdx == endPage) { + if (col == endCol) return yBottom <= endY; + return col < endCol; + } + return false; + } + + /** Lazily builds and caches the column layout for a single page. */ + private PageColumnLayout layoutFor( + PDDocument document, int pageIdx, Map cache) + throws IOException { + PageColumnLayout cached = cache.get(pageIdx); + if (cached != null) return cached; + PDPage page = document.getDocumentCatalog().getPages().get(pageIdx); + float pageWidth = page.getBBox().getWidth(); + float pageHeight = page.getBBox().getHeight(); + AllTextLineExtractor extractor = new AllTextLineExtractor(pageIdx + 1, pageHeight); + extractor.getText(document); + PageColumnLayout layout = + PageColumnLayout.fromLineBoxes(extractor.getLineBoxes(), pageWidth); + if (layout.columnCount() > 1) { + float[] g = layout.gutters().get(0); + log.info( + "[redact/execute] page {} layout: 2 cols, gutter x=[{}, {}]", + pageIdx + 1, + g[0], + g[1]); + } else { + log.info("[redact/execute] page {} layout: 1 col (single-column mode)", pageIdx + 1); + } + cache.put(pageIdx, layout); + return layout; + } + + private List toAnchors( + PDDocument document, + Map> matchesByPage, + Map layoutCache) + throws IOException { + List out = new ArrayList<>(); + for (int page : matchesByPage.keySet().stream().sorted().toList()) { + PageColumnLayout layout = layoutFor(document, page, layoutCache); + for (PDFText hit : matchesByPage.get(page)) { + int col = layout.columnOf(hit.getX1(), hit.getX2()); + out.add(new Anchor(page, col, hit.getY1(), hit)); + } + } + return out; + } + + /** Lexicographic ordering by (page, column, screenY). */ + private static final Comparator READING_ORDER = + Comparator.comparingInt((Anchor a) -> a.page) + .thenComparingInt(a -> a.col) + .thenComparingDouble(a -> a.y); + + private static String anchorSummary(List anchors) { + StringBuilder sb = new StringBuilder(); + int max = Math.min(anchors.size(), 5); + for (int i = 0; i < max; i++) { + Anchor a = anchors.get(i); + if (i > 0) sb.append(", "); + sb.append(String.format("(p=%d,c=%d,y=%.1f)", a.page + 1, a.col, a.y)); + } + if (anchors.size() > max) sb.append(", …"); + return sb.toString(); + } + + private record Anchor(int page, int col, float y, PDFText text) {} + + /** + * Tries progressively more permissive variants: raw (regex then literal), letter-spacing + * collapsed, then a punctuation-tolerant regex over alphanumeric runs. + */ + private Map> findWithFallbacks(PDDocument document, String raw) { + String trimmed = raw.trim(); + String collapsed = collapseLetterSpacing(trimmed); + String tolerant = punctuationTolerantRegex(trimmed); + + List candidates = new ArrayList<>(); + candidates.add(new Candidate(trimmed, true)); + candidates.add(new Candidate(trimmed, false)); + if (!collapsed.equals(trimmed)) { + candidates.add(new Candidate(collapsed, true)); + candidates.add(new Candidate(collapsed, false)); + } + if (tolerant != null && !tolerant.equals(trimmed)) { + candidates.add(new Candidate(tolerant, true)); + } + + // If the anchor spans multiple lines (model provided entire paragraph instead of a short + // phrase), try just the first non-empty line — it's usually sufficient to locate the + // position and avoids mismatches from mid-paragraph text extraction artifacts. + if (trimmed.contains("\n")) { + String firstLine = + Arrays.stream(trimmed.split("\n")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .findFirst() + .orElse(null); + if (firstLine != null && firstLine.length() >= 4) { + String firstLineCollapsed = collapseLetterSpacing(firstLine); + String firstLineTolerant = punctuationTolerantRegex(firstLine); + candidates.add(new Candidate(firstLine, false)); + if (!firstLineCollapsed.equals(firstLine)) { + candidates.add(new Candidate(firstLineCollapsed, false)); + } + if (firstLineTolerant != null && !firstLineTolerant.equals(firstLine)) { + candidates.add(new Candidate(firstLineTolerant, true)); + } + } + } + + for (Candidate c : candidates) { + Map> m = + textRedactionService.findTextToRedact( + document, new String[] {c.pattern}, c.useRegex, false); + if (!m.isEmpty()) { + if (!c.pattern.equals(trimmed)) { + log.info( + "[redact/execute] range boundary matched via fallback: '{}' → '{}'", + trimmed, + c.pattern); + } + return m; + } + } + return Collections.emptyMap(); + } + + private record Candidate(String pattern, boolean useRegex) {} + + // ----------------------------------------------------------------------- + // Static helpers + // ----------------------------------------------------------------------- + + /** + * Joins {@code raw}'s alphanumeric runs with {@code \W*} so anchors match across punctuation + * drift. Returns {@code null} when fewer than two tokens exist. + */ + private static String punctuationTolerantRegex(String raw) { + List tokens = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + for (int i = 0; i < raw.length(); i++) { + char ch = raw.charAt(i); + if (Character.isLetterOrDigit(ch)) { + current.append(ch); + } else if (current.length() > 0) { + tokens.add(current.toString()); + current.setLength(0); + } + } + if (current.length() > 0) tokens.add(current.toString()); + if (tokens.size() < 2) return null; + StringBuilder out = new StringBuilder(); + for (int i = 0; i < tokens.size(); i++) { + if (i > 0) out.append("\\W*"); + out.append(Pattern.quote(tokens.get(i))); + } + return out.toString(); + } + + /** + * Collapses letter-spaced text produced by position-sorted text extraction. + * + *

When a PDF text stripper runs with {@code setSortByPosition(true)}, letter-spaced headings + * come out as {@code "T a b l e o f c o n t e n t s"}. This method converts the spaced form + * back to words. + */ + private static String collapseLetterSpacing(String text) { + String[] tokens = text.split(" ", -1); + StringBuilder result = new StringBuilder(); + StringBuilder current = new StringBuilder(); + for (String token : tokens) { + if (token.isEmpty()) { + if (current.length() > 0) { + if (result.length() > 0) result.append(' '); + result.append(current); + current.setLength(0); + } + } else if (token.length() == 1) { + current.append(token); + } else { + if (current.length() > 0) { + if (result.length() > 0) result.append(' '); + result.append(current); + current.setLength(0); + } + if (result.length() > 0) result.append(' '); + result.append(token); + } + } + if (current.length() > 0) { + if (result.length() > 0) result.append(' '); + result.append(current); + } + return result.toString().trim(); + } + + private static List orEmpty(List list) { + return list != null ? list : List.of(); + } + + private static String[] cleanStrings(List input) { + if (input == null || input.isEmpty()) { + return new String[0]; + } + return input.stream() + .filter(s -> s != null) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toArray(String[]::new); + } + + /** + * Converts 1-based page numbers from the request to the 0-based indices used internally. + * Out-of-range and non-positive values are silently dropped. + */ + private static List toZeroBasedIndices(List oneBasedPageNumbers) { + if (oneBasedPageNumbers == null || oneBasedPageNumbers.isEmpty()) { + return new ArrayList<>(); + } + List result = new ArrayList<>(); + for (Integer page : oneBasedPageNumbers) { + if (page != null && page > 0) { + result.add(page - 1); + } + } + return result; + } + + private static String trimOrEmpty(String s) { + return s == null ? "" : s.trim(); + } +} diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java new file mode 100644 index 000000000..a9633ffa3 --- /dev/null +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/security/TextRedactionService.java @@ -0,0 +1,1207 @@ +package stirling.software.SPDF.controller.api.security; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.pdfbox.contentstream.operator.Operator; +import org.apache.pdfbox.cos.COSArray; +import org.apache.pdfbox.cos.COSBase; +import org.apache.pdfbox.cos.COSFloat; +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.cos.COSNumber; +import org.apache.pdfbox.cos.COSString; +import org.apache.pdfbox.pdfparser.PDFStreamParser; +import org.apache.pdfbox.pdfwriter.ContentStreamWriter; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDResources; +import org.apache.pdfbox.pdmodel.common.PDStream; +import org.apache.pdfbox.pdmodel.font.PDFont; +import org.apache.pdfbox.pdmodel.graphics.PDXObject; +import org.apache.pdfbox.pdmodel.graphics.form.PDFormXObject; +import org.springframework.stereotype.Service; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.SPDF.model.PDFText; +import stirling.software.SPDF.utils.text.TextEncodingHelper; +import stirling.software.SPDF.utils.text.TextFinderUtils; +import stirling.software.SPDF.utils.text.WidthCalculator; + +@Service +@Slf4j +class TextRedactionService { + + private static final int MAX_XOBJECT_DEPTH = 10; + private static final float PRECISION_THRESHOLD = 1e-3f; + private static final int FONT_SCALE_FACTOR = 1000; + private static final Set TEXT_SHOWING_OPERATORS = Set.of("Tj", "TJ", "'", "\""); + private static final COSString EMPTY_COS_STRING = new COSString(""); + + // ----------------------------------------------------------------------- + // Public API + // ----------------------------------------------------------------------- + + Map> findTextToRedact( + PDDocument document, String[] listOfText, boolean useRegex, boolean wholeWordSearch) { + + Set terms = + Arrays.stream(listOfText) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toSet()); + + if (terms.isEmpty()) { + return new HashMap<>(); + } + + List patterns = + TextFinderUtils.createOptimizedSearchPatterns(terms, useRegex, wholeWordSearch); + + if (patterns.isEmpty()) { + return new HashMap<>(); + } + + log.debug( + "Scanning document once for {} pattern(s) (useRegex={}, wholeWord={})", + patterns.size(), + useRegex, + wholeWordSearch); + + try { + MultiPatternTextFinder finder = new MultiPatternTextFinder(patterns); + finder.getText(document); + Map> result = finder.getFoundTextsByPage(); + int total = result.values().stream().mapToInt(List::size).sum(); + log.debug("Multi-pattern scan: {} match(es) across {} page(s)", total, result.size()); + return result; + } catch (Exception e) { + log.error("Multi-pattern text search failed: {}", e.getMessage()); + return new HashMap<>(); + } + } + + boolean performTextReplacement( + PDDocument document, + Map> allFoundTextsByPage, + String[] listOfText, + boolean useRegex, + boolean wholeWordSearchBool) { + if (allFoundTextsByPage.isEmpty()) { + return false; + } + + if (detectCustomEncodingFonts(document)) { + log.warn( + "Custom encoded fonts detected (non-standard encodings / DictionaryEncoding / damaged fonts). " + + "Text replacement is unreliable for these fonts. Falling back to box-only redaction mode."); + return true; + } + + try { + Set allSearchTerms = + Arrays.stream(listOfText) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toSet()); + + int pageCount = 0; + for (PDPage page : document.getPages()) { + pageCount++; + List filteredTokens = + createTokensWithoutTargetText( + document, page, allSearchTerms, useRegex, wholeWordSearchBool); + writeFilteredContentStream(document, page, filteredTokens); + } + log.info("Successfully performed text replacement redaction on {} pages.", pageCount); + return false; + } catch (Exception e) { + log.error( + "Text replacement redaction failed due to font or encoding issues. " + + "Will fall back to box-only redaction mode. Error: {}", + e.getMessage()); + return true; + } + } + + // ----------------------------------------------------------------------- + // Content stream manipulation + // ----------------------------------------------------------------------- + + List createTokensWithoutTargetText( + PDDocument document, + PDPage page, + Set targetWords, + boolean useRegex, + boolean wholeWordSearch) + throws IOException { + + PDFStreamParser parser = new PDFStreamParser(page); + List tokens = new ArrayList<>(); + Object token; + while ((token = parser.parseNextToken()) != null) { + tokens.add(token); + } + + PDResources resources = page.getResources(); + if (resources != null) { + processPageXObjects(document, resources, targetWords, useRegex, wholeWordSearch); + } + + List textSegments = extractTextSegments(page, tokens); + String completeText = buildCompleteText(textSegments); + List matches = + findAllMatches(completeText, targetWords, useRegex, wholeWordSearch); + + return applyRedactionsToTokens(tokens, textSegments, matches); + } + + void writeFilteredContentStream(PDDocument document, PDPage page, List tokens) + throws IOException { + + PDStream newStream = new PDStream(document); + + try { + try (var out = newStream.createOutputStream()) { + ContentStreamWriter writer = new ContentStreamWriter(out); + writer.writeTokens(tokens); + } + page.setContents(newStream); + } catch (IOException e) { + throw new IOException("Failed to write filtered content stream to page", e); + } + } + + boolean isTextShowingOperator(String opName) { + return TEXT_SHOWING_OPERATORS.contains(opName); + } + + boolean detectCustomEncodingFonts(PDDocument document) { + try { + var documentCatalog = document.getDocumentCatalog(); + if (documentCatalog == null) { + return false; + } + + int totalFonts = 0; + int customEncodedFonts = 0; + int subsetFonts = 0; + int unreliableFonts = 0; + + for (PDPage page : document.getPages()) { + if (TextFinderUtils.hasProblematicFonts(page)) { + log.debug("Page contains fonts flagged as problematic by TextFinderUtils"); + } + + PDResources resources = page.getResources(); + if (resources == null) { + continue; + } + + for (COSName fontName : resources.getFontNames()) { + try { + PDFont font = resources.getFont(fontName); + if (font != null) { + totalFonts++; + + boolean isSubset = TextEncodingHelper.isFontSubset(font.getName()); + boolean hasCustomEncoding = TextEncodingHelper.hasCustomEncoding(font); + boolean isReliable = WidthCalculator.isWidthCalculationReliable(font); + boolean canCalculateWidths = + TextEncodingHelper.canCalculateBasicWidths(font); + + if (isSubset) { + subsetFonts++; + } + if (hasCustomEncoding) { + customEncodedFonts++; + log.debug("Font {} has custom encoding", font.getName()); + } + if (!isReliable || !canCalculateWidths) { + unreliableFonts++; + log.debug( + "Font {} flagged as unreliable: reliable={}, canCalculateWidths={}", + font.getName(), + isReliable, + canCalculateWidths); + } + if (!TextFinderUtils.validateFontReliability(font)) { + log.debug( + "Font {} failed comprehensive reliability check", + font.getName()); + } + } + } catch (Exception e) { + log.debug( + "Font loading/analysis failed for {}: {}", + fontName.getName(), + e.getMessage()); + customEncodedFonts++; + unreliableFonts++; + totalFonts++; + } + } + } + + log.info( + "Enhanced font analysis: {}/{} custom encoding, {}/{} subset, {}/{} unreliable fonts", + customEncodedFonts, + totalFonts, + subsetFonts, + totalFonts, + unreliableFonts, + totalFonts); + + return customEncodedFonts > 0 || unreliableFonts > 0; + + } catch (Exception e) { + log.warn("Enhanced font detection analysis failed: {}", e.getMessage()); + return true; + } + } + + // ----------------------------------------------------------------------- + // Placeholder creation + // ----------------------------------------------------------------------- + + String createPlaceholderWithFont(String originalWord, PDFont font) { + if (originalWord == null || originalWord.isEmpty()) { + return originalWord; + } + + if (font != null && TextEncodingHelper.isFontSubset(font.getName())) { + try { + float originalWidth = safeGetStringWidth(font, originalWord) / FONT_SCALE_FACTOR; + return createAlternativePlaceholder(originalWord, originalWidth, font, 1.0f); + } catch (Exception e) { + log.debug( + "Subset font placeholder creation failed for {}: {}", + font.getName(), + e.getMessage()); + return ""; + } + } + + return " ".repeat(originalWord.length()); + } + + String createPlaceholderWithWidth( + String originalWord, float targetWidth, PDFont font, float fontSize) { + if (originalWord == null || originalWord.isEmpty()) { + return originalWord; + } + + if (font == null || fontSize <= 0) { + return " ".repeat(originalWord.length()); + } + + try { + if (!WidthCalculator.isWidthCalculationReliable(font)) { + log.debug( + "Font {} unreliable for width calculation, using simple placeholder", + font.getName()); + return " ".repeat(originalWord.length()); + } + + if (TextEncodingHelper.isFontSubset(font.getName())) { + return createSubsetFontPlaceholder(originalWord, targetWidth, font, fontSize); + } + + float spaceWidth = WidthCalculator.calculateAccurateWidth(font, " ", fontSize); + + if (spaceWidth <= 0) { + return createAlternativePlaceholder(originalWord, targetWidth, font, fontSize); + } + + int spaceCount = Math.max(1, Math.round(targetWidth / spaceWidth)); + int maxSpaces = + Math.max( + originalWord.length() * 2, Math.round(targetWidth / spaceWidth * 1.5f)); + spaceCount = Math.min(spaceCount, maxSpaces); + + return " ".repeat(spaceCount); + + } catch (Exception e) { + log.debug("Enhanced placeholder creation failed: {}", e.getMessage()); + return createAlternativePlaceholder(originalWord, targetWidth, font, fontSize); + } + } + + private String createSubsetFontPlaceholder( + String originalWord, float targetWidth, PDFont font, float fontSize) { + try { + log.debug("Subset font {} - trying to find replacement characters", font.getName()); + String result = createAlternativePlaceholder(originalWord, targetWidth, font, fontSize); + + if (result.isEmpty()) { + log.debug( + "Subset font {} has no suitable replacement characters, using empty string", + font.getName()); + } + + return result; + + } catch (Exception e) { + log.debug("Subset font placeholder creation failed: {}", e.getMessage()); + return ""; + } + } + + private String createAlternativePlaceholder( + String originalWord, float targetWidth, PDFont font, float fontSize) { + try { + String[] alternatives = {" ", ".", "-", "_", "~", "°", "·"}; + + if (TextEncodingHelper.fontSupportsCharacter(font, " ")) { + float spaceWidth = safeGetStringWidth(font, " ") / FONT_SCALE_FACTOR * fontSize; + if (spaceWidth > 0) { + int spaceCount = Math.max(1, Math.round(targetWidth / spaceWidth)); + int maxSpaces = originalWord.length() * 2; + spaceCount = Math.min(spaceCount, maxSpaces); + log.debug("Using spaces for font {}", font.getName()); + return " ".repeat(spaceCount); + } + } + + for (String altChar : alternatives) { + if (" ".equals(altChar)) continue; + + try { + if (!TextEncodingHelper.fontSupportsCharacter(font, altChar)) { + continue; + } + + float charWidth = + safeGetStringWidth(font, altChar) / FONT_SCALE_FACTOR * fontSize; + if (charWidth > 0) { + int charCount = Math.max(1, Math.round(targetWidth / charWidth)); + int maxChars = originalWord.length() * 2; + charCount = Math.min(charCount, maxChars); + log.debug( + "Using character '{}' for width calculation but spaces for placeholder in font {}", + altChar, + font.getName()); + return " ".repeat(charCount); + } + } catch (Exception e) { + // try next alternative + } + } + + log.debug( + "All placeholder alternatives failed for font {}, using empty string", + font.getName()); + return ""; + + } catch (Exception e) { + log.debug("Alternative placeholder creation failed: {}", e.getMessage()); + return ""; + } + } + + // ----------------------------------------------------------------------- + // Width calculation + // ----------------------------------------------------------------------- + + private float safeGetStringWidth(PDFont font, String text) { + if (font == null || text == null || text.isEmpty()) { + return 0; + } + + if (!WidthCalculator.isWidthCalculationReliable(font)) { + log.debug( + "Font {} flagged as unreliable for width calculation, using fallback", + font.getName()); + return calculateConservativeWidth(font, text); + } + + if (!TextEncodingHelper.canEncodeCharacters(font, text)) { + log.debug( + "Text cannot be encoded by font {}, using character-based fallback", + font.getName()); + return calculateCharacterBasedWidth(font, text); + } + + try { + float width = font.getStringWidth(text); + log.debug("Direct width calculation successful for '{}': {}", text, width); + return width; + + } catch (Exception e) { + log.debug( + "Direct width calculation failed for font {}: {}", + font.getName(), + e.getMessage()); + return calculateFallbackWidth(font, text); + } + } + + private float calculateCharacterBasedWidth(PDFont font, String text) { + try { + float totalWidth = 0; + for (int i = 0; i < text.length(); i++) { + String character = text.substring(i, i + 1); + try { + if (!TextEncodingHelper.fontSupportsCharacter(font, character)) { + totalWidth += font.getAverageFontWidth(); + continue; + } + + byte[] encoded = font.encode(character); + if (encoded.length > 0) { + int glyphCode = encoded[0] & 0xFF; + float glyphWidth = font.getWidth(glyphCode); + + if (glyphWidth == 0) { + try { + glyphWidth = font.getWidthFromFont(glyphCode); + } catch (Exception e2) { + glyphWidth = font.getAverageFontWidth(); + } + } + + totalWidth += glyphWidth; + } else { + totalWidth += font.getAverageFontWidth(); + } + } catch (Exception e2) { + totalWidth += font.getAverageFontWidth(); + } + } + + log.debug("Character-based width calculation: {}", totalWidth); + return totalWidth; + + } catch (Exception e) { + log.debug("Character-based width calculation failed: {}", e.getMessage()); + return calculateConservativeWidth(font, text); + } + } + + private float calculateFallbackWidth(PDFont font, String text) { + try { + if (font.getFontDescriptor() != null + && font.getFontDescriptor().getFontBoundingBox() != null) { + + org.apache.pdfbox.pdmodel.common.PDRectangle bbox = + font.getFontDescriptor().getFontBoundingBox(); + float avgCharWidth = bbox.getWidth() * 0.6f; + float fallbackWidth = text.length() * avgCharWidth; + + log.debug("Bounding box fallback width: {}", fallbackWidth); + return fallbackWidth; + } + + try { + float avgWidth = font.getAverageFontWidth(); + if (avgWidth > 0) { + float fallbackWidth = text.length() * avgWidth; + log.debug("Average width fallback: {}", fallbackWidth); + return fallbackWidth; + } + } catch (Exception e2) { + log.debug("Average font width calculation failed: {}", e2.getMessage()); + } + + return calculateConservativeWidth(font, text); + + } catch (Exception e) { + log.debug("Fallback width calculation failed: {}", e.getMessage()); + return calculateConservativeWidth(font, text); + } + } + + private float calculateConservativeWidth(PDFont font, String text) { + float conservativeWidth = text.length() * 500f; + log.debug( + "Conservative width estimate for font {} text '{}': {}", + font.getName(), + text, + conservativeWidth); + return conservativeWidth; + } + + private float calculateWidthAdjustment(TextSegment segment, List matches) { + try { + if (segment.getFont() == null || segment.getFontSize() <= 0) { + return 0; + } + + String fontName = segment.getFont().getName(); + if (fontName != null + && (fontName.contains("HOEPAP") || TextEncodingHelper.isFontSubset(fontName))) { + log.debug("Skipping width adjustment for problematic/subset font: {}", fontName); + return 0; + } + + float totalOriginal = 0; + float totalPlaceholder = 0; + String text = segment.getText(); + + for (MatchRange match : matches) { + int segStart = Math.max(0, match.getStartPos() - segment.getStartPos()); + int segEnd = Math.min(text.length(), match.getEndPos() - segment.getStartPos()); + + if (segStart < text.length() && segEnd > segStart) { + String originalPart = text.substring(segStart, segEnd); + + float originalWidth = + safeGetStringWidth(segment.getFont(), originalPart) + / FONT_SCALE_FACTOR + * segment.getFontSize(); + + String placeholderPart = + createPlaceholderWithWidth( + originalPart, + originalWidth, + segment.getFont(), + segment.getFontSize()); + + float origUnits = safeGetStringWidth(segment.getFont(), originalPart); + float placeUnits = safeGetStringWidth(segment.getFont(), placeholderPart); + + float orig = (origUnits / FONT_SCALE_FACTOR) * segment.getFontSize(); + float place = (placeUnits / FONT_SCALE_FACTOR) * segment.getFontSize(); + + totalOriginal += orig; + totalPlaceholder += place; + } + } + + float adjustment = totalOriginal - totalPlaceholder; + + float maxReasonableAdjustment = + Math.max( + segment.getText().length() * segment.getFontSize() * 2, + totalOriginal * 1.5f); + + if (Math.abs(adjustment) > maxReasonableAdjustment) { + log.debug( + "Width adjustment {} seems unreasonable for text length {}, capping to 0", + adjustment, + segment.getText().length()); + return 0; + } + + return adjustment; + } catch (Exception ex) { + log.debug("Width adjustment failed: {}", ex.getMessage()); + return 0; + } + } + + // ----------------------------------------------------------------------- + // Token and segment operations + // ----------------------------------------------------------------------- + + private void processPageXObjects( + PDDocument document, + PDResources resources, + Set targetWords, + boolean useRegex, + boolean wholeWordSearch) { + processPageXObjects( + document, resources, targetWords, useRegex, wholeWordSearch, 0, new HashSet<>()); + } + + private void processPageXObjects( + PDDocument document, + PDResources resources, + Set targetWords, + boolean useRegex, + boolean wholeWordSearch, + int depth, + Set visited) { + + if (depth > MAX_XOBJECT_DEPTH) { + log.warn("[redact] XObject nesting depth {} exceeded limit, stopping traversal", depth); + return; + } + + for (COSName xobjName : resources.getXObjectNames()) { + try { + PDXObject xobj = resources.getXObject(xobjName); + if (xobj instanceof PDFormXObject formXObj) { + if (!visited.add(formXObj.getCOSObject())) { + log.debug( + "[redact] Cycle detected in XObject graph, skipping {}", + xobjName.getName()); + continue; + } + processFormXObject( + document, + formXObj, + targetWords, + useRegex, + wholeWordSearch, + depth + 1, + visited); + log.debug("Processed Form XObject: {}", xobjName.getName()); + } + } catch (Exception e) { + log.warn("Failed to process XObject {}: {}", xobjName.getName(), e.getMessage()); + } + } + } + + private void processFormXObject( + PDDocument document, + PDFormXObject formXObject, + Set targetWords, + boolean useRegex, + boolean wholeWordSearch, + int depth, + Set visited) { + + try { + PDResources xobjResources = formXObject.getResources(); + if (xobjResources == null) { + return; + } + + processPageXObjects( + document, + xobjResources, + targetWords, + useRegex, + wholeWordSearch, + depth, + visited); + + PDFStreamParser parser = new PDFStreamParser(formXObject); + List tokens = new ArrayList<>(); + Object token; + while ((token = parser.parseNextToken()) != null) { + tokens.add(token); + } + + List textSegments = extractTextSegmentsFromXObject(xobjResources, tokens); + String completeText = buildCompleteText(textSegments); + List matches = + findAllMatches(completeText, targetWords, useRegex, wholeWordSearch); + + if (!matches.isEmpty()) { + List redactedTokens = + applyRedactionsToTokens(tokens, textSegments, matches); + writeRedactedContentToXObject(document, formXObject, redactedTokens); + log.debug("Processed {} redactions in Form XObject", matches.size()); + } + + } catch (Exception e) { + log.warn("Failed to process Form XObject: {}", e.getMessage()); + } + } + + private void writeRedactedContentToXObject( + PDDocument document, PDFormXObject formXObject, List redactedTokens) + throws IOException { + + PDStream newStream = new PDStream(document); + + try (var out = newStream.createOutputStream()) { + ContentStreamWriter writer = new ContentStreamWriter(out); + writer.writeTokens(redactedTokens); + } + + formXObject.getCOSObject().removeItem(COSName.CONTENTS); + formXObject.getCOSObject().setItem(COSName.CONTENTS, newStream.getCOSObject()); + } + + private List extractTextSegments(PDPage page, List tokens) { + List segments = new ArrayList<>(); + int currentTextPos = 0; + GraphicsState graphicsState = new GraphicsState(); + PDResources resources = page.getResources(); + + for (int i = 0; i < tokens.size(); i++) { + Object currentToken = tokens.get(i); + + if (currentToken instanceof Operator op) { + String opName = op.getName(); + + if ("Tf".equals(opName) && i >= 2) { + try { + COSName fontName = (COSName) tokens.get(i - 2); + COSBase fontSizeBase = (COSBase) tokens.get(i - 1); + if (fontSizeBase instanceof COSNumber cosNumber) { + graphicsState.setFont(resources.getFont(fontName)); + graphicsState.setFontSize(cosNumber.floatValue()); + } + } catch (ClassCastException | IOException e) { + log.debug( + "Failed to extract font and font size from Tf operator: {}", + e.getMessage()); + } + } + + currentTextPos = + getCurrentTextPos( + tokens, segments, currentTextPos, graphicsState, i, opName); + } + } + + return segments; + } + + private List extractTextSegmentsFromXObject( + PDResources resources, List tokens) { + List segments = new ArrayList<>(); + int currentTextPos = 0; + GraphicsState graphicsState = new GraphicsState(); + + for (int i = 0; i < tokens.size(); i++) { + Object currentToken = tokens.get(i); + + if (currentToken instanceof Operator op) { + String opName = op.getName(); + + if ("Tf".equals(opName) && i >= 2) { + try { + COSName fontName = (COSName) tokens.get(i - 2); + COSBase fontSizeBase = (COSBase) tokens.get(i - 1); + if (fontSizeBase instanceof COSNumber cosNumber) { + graphicsState.setFont(resources.getFont(fontName)); + graphicsState.setFontSize(cosNumber.floatValue()); + } + } catch (ClassCastException | IOException e) { + log.debug("Font extraction failed in XObject: {}", e.getMessage()); + } + } + + currentTextPos = + getCurrentTextPos( + tokens, segments, currentTextPos, graphicsState, i, opName); + } + } + + return segments; + } + + private int getCurrentTextPos( + List tokens, + List segments, + int currentTextPos, + GraphicsState graphicsState, + int i, + String opName) { + if (isTextShowingOperator(opName) && i > 0) { + String textContent = extractTextFromToken(tokens.get(i - 1), opName); + if (!textContent.isEmpty()) { + segments.add( + new TextSegment( + i - 1, + opName, + textContent, + currentTextPos, + currentTextPos + textContent.length(), + graphicsState.font, + graphicsState.fontSize)); + currentTextPos += textContent.length(); + } + } + return currentTextPos; + } + + private String buildCompleteText(List segments) { + StringBuilder sb = new StringBuilder(); + for (TextSegment segment : segments) { + sb.append(segment.text); + } + return sb.toString(); + } + + private List findAllMatches( + String completeText, + Set targetWords, + boolean useRegex, + boolean wholeWordSearch) { + + List patterns = + TextFinderUtils.createOptimizedSearchPatterns( + targetWords, useRegex, wholeWordSearch); + + return patterns.stream() + .flatMap( + pattern -> { + try { + return pattern.matcher(completeText).results(); + } catch (Exception e) { + log.debug( + "Pattern matching failed for pattern {}: {}", + pattern.pattern(), + e.getMessage()); + return java.util.stream.Stream.empty(); + } + }) + .map(matchResult -> new MatchRange(matchResult.start(), matchResult.end())) + .sorted(Comparator.comparingInt(MatchRange::getStartPos)) + .collect(Collectors.toList()); + } + + private List applyRedactionsToTokens( + List tokens, List textSegments, List matches) { + + long startTime = System.currentTimeMillis(); + + try { + List newTokens = new ArrayList<>(tokens); + + Map> matchesBySegment = new HashMap<>(); + for (MatchRange match : matches) { + for (int i = 0; i < textSegments.size(); i++) { + TextSegment segment = textSegments.get(i); + int overlapStart = Math.max(match.startPos, segment.startPos); + int overlapEnd = Math.min(match.endPos, segment.endPos); + if (overlapStart < overlapEnd) { + matchesBySegment.computeIfAbsent(i, k -> new ArrayList<>()).add(match); + } + } + } + + List tasks = new ArrayList<>(); + for (Map.Entry> entry : matchesBySegment.entrySet()) { + int segmentIndex = entry.getKey(); + List segmentMatches = entry.getValue(); + TextSegment segment = textSegments.get(segmentIndex); + + if ("Tj".equals(segment.operatorName) || "'".equals(segment.operatorName)) { + String newText = applyRedactionsToSegmentText(segment, segmentMatches); + try { + float adjustment = calculateWidthAdjustment(segment, segmentMatches); + tasks.add(new ModificationTask(segment, newText, adjustment)); + } catch (Exception e) { + log.debug( + "Width adjustment calculation failed for segment: {}", + e.getMessage()); + } + } else if ("TJ".equals(segment.operatorName)) { + tasks.add(new ModificationTask(segment, null, 0)); + } + } + + tasks.sort((a, b) -> Integer.compare(b.segment.tokenIndex, a.segment.tokenIndex)); + + for (ModificationTask task : tasks) { + List segmentMatches = + matchesBySegment.getOrDefault( + textSegments.indexOf(task.segment), + java.util.Collections.emptyList()); + modifyTokenForRedaction( + newTokens, task.segment, task.newText, task.adjustment, segmentMatches); + } + + return newTokens; + + } finally { + long processingTime = System.currentTimeMillis() - startTime; + log.debug( + "Token redaction processing completed in {} ms for {} matches", + processingTime, + matches.size()); + } + } + + private String applyRedactionsToSegmentText(TextSegment segment, List matches) { + String text = segment.getText(); + + if (segment.getFont() != null + && !TextEncodingHelper.isTextSegmentRemovable(segment.getFont(), text)) { + log.debug( + "Skipping text segment '{}' - font {} cannot process this text reliably", + text, + segment.getFont().getName()); + return text; + } + + StringBuilder result = new StringBuilder(text); + + for (MatchRange match : matches) { + int segmentStart = Math.max(0, match.getStartPos() - segment.getStartPos()); + int segmentEnd = Math.min(text.length(), match.getEndPos() - segment.getStartPos()); + + if (segmentStart < text.length() && segmentEnd > segmentStart) { + String originalPart = text.substring(segmentStart, segmentEnd); + + if (segment.getFont() != null + && !TextEncodingHelper.isTextSegmentRemovable( + segment.getFont(), originalPart)) { + log.debug( + "Skipping text part '{}' within segment - cannot be processed reliably", + originalPart); + continue; + } + + float originalWidth = 0; + if (segment.getFont() != null && segment.getFontSize() > 0) { + try { + originalWidth = + safeGetStringWidth(segment.getFont(), originalPart) + / FONT_SCALE_FACTOR + * segment.getFontSize(); + } catch (Exception e) { + log.debug( + "Failed to calculate original width for placeholder: {}", + e.getMessage()); + } + } + + String placeholder = + (originalWidth > 0) + ? createPlaceholderWithWidth( + originalPart, + originalWidth, + segment.getFont(), + segment.getFontSize()) + : createPlaceholderWithFont(originalPart, segment.getFont()); + + result.replace(segmentStart, segmentEnd, placeholder); + } + } + + return result.toString(); + } + + private void modifyTokenForRedaction( + List tokens, + TextSegment segment, + String newText, + float adjustment, + List matches) { + + if (segment.getTokenIndex() < 0 || segment.getTokenIndex() >= tokens.size()) { + return; + } + + Object token = tokens.get(segment.getTokenIndex()); + String operatorName = segment.getOperatorName(); + + try { + if (("Tj".equals(operatorName) || "'".equals(operatorName)) + && token instanceof COSString) { + + if (Math.abs(adjustment) < PRECISION_THRESHOLD) { + if (newText.isEmpty()) { + tokens.set(segment.getTokenIndex(), EMPTY_COS_STRING); + } else { + tokens.set(segment.getTokenIndex(), new COSString(newText)); + } + } else { + COSArray newArray = new COSArray(); + newArray.add(new COSString(newText)); + if (segment.getFontSize() > 0) { + float kerning = (-adjustment / segment.getFontSize()) * FONT_SCALE_FACTOR; + newArray.add(new COSFloat(kerning)); + } + tokens.set(segment.getTokenIndex(), newArray); + + int operatorIndex = segment.getTokenIndex() + 1; + if (operatorIndex < tokens.size() + && tokens.get(operatorIndex) instanceof Operator op + && op.getName().equals(operatorName)) { + tokens.set(operatorIndex, Operator.getOperator("TJ")); + } + } + } else if ("TJ".equals(operatorName) && token instanceof COSArray) { + COSArray newArray = createRedactedTJArray((COSArray) token, segment, matches); + tokens.set(segment.getTokenIndex(), newArray); + } + } catch (Exception e) { + log.debug( + "Token modification failed for segment at index {}: {}", + segment.getTokenIndex(), + e.getMessage()); + } + } + + private COSArray createRedactedTJArray( + COSArray originalArray, TextSegment segment, List matches) { + try { + COSArray newArray = new COSArray(); + int textOffsetInSegment = 0; + + for (COSBase element : originalArray) { + if (element instanceof COSString cosString) { + String originalText = cosString.getString(); + + if (segment.getFont() != null + && !TextEncodingHelper.isTextSegmentRemovable( + segment.getFont(), originalText)) { + log.debug( + "Skipping TJ text part '{}' - cannot be processed reliably with font {}", + originalText, + segment.getFont().getName()); + newArray.add(element); + textOffsetInSegment += originalText.length(); + continue; + } + + StringBuilder newText = new StringBuilder(originalText); + boolean modified = false; + + for (MatchRange match : matches) { + int stringStartInPage = segment.getStartPos() + textOffsetInSegment; + int stringEndInPage = stringStartInPage + originalText.length(); + + int overlapStart = Math.max(match.getStartPos(), stringStartInPage); + int overlapEnd = Math.min(match.getEndPos(), stringEndInPage); + + if (overlapStart < overlapEnd) { + int redactionStartInString = overlapStart - stringStartInPage; + int redactionEndInString = overlapEnd - stringStartInPage; + if (redactionStartInString >= 0 + && redactionEndInString <= originalText.length()) { + String originalPart = + originalText.substring( + redactionStartInString, redactionEndInString); + + if (segment.getFont() != null + && !TextEncodingHelper.isTextSegmentRemovable( + segment.getFont(), originalPart)) { + log.debug( + "Skipping TJ text part '{}' - cannot be redacted reliably", + originalPart); + continue; + } + + modified = true; + float originalWidth = 0; + if (segment.getFont() != null && segment.getFontSize() > 0) { + try { + originalWidth = + safeGetStringWidth(segment.getFont(), originalPart) + / FONT_SCALE_FACTOR + * segment.getFontSize(); + } catch (Exception e) { + log.debug( + "Failed to calculate original width for TJ placeholder: {}", + e.getMessage()); + } + } + + String placeholder = + (originalWidth > 0) + ? createPlaceholderWithWidth( + originalPart, + originalWidth, + segment.getFont(), + segment.getFontSize()) + : createPlaceholderWithFont( + originalPart, segment.getFont()); + + newText.replace( + redactionStartInString, redactionEndInString, placeholder); + } + } + } + + String modifiedString = newText.toString(); + newArray.add(new COSString(modifiedString)); + + if (modified && segment.getFont() != null && segment.getFontSize() > 0) { + try { + float originalWidth = + safeGetStringWidth(segment.getFont(), originalText) + / FONT_SCALE_FACTOR + * segment.getFontSize(); + float modifiedWidth = + safeGetStringWidth(segment.getFont(), modifiedString) + / FONT_SCALE_FACTOR + * segment.getFontSize(); + float adjustment = originalWidth - modifiedWidth; + if (Math.abs(adjustment) > PRECISION_THRESHOLD) { + float kerning = + (-adjustment / segment.getFontSize()) + * FONT_SCALE_FACTOR + * 1.10f; + newArray.add(new COSFloat(kerning)); + } + } catch (Exception e) { + log.debug( + "Width adjustment calculation failed for segment: {}", + e.getMessage()); + } + } + + textOffsetInSegment += originalText.length(); + } else { + newArray.add(element); + } + } + return newArray; + } catch (Exception e) { + return originalArray; + } + } + + private String extractTextFromToken(Object token, String operatorName) { + return switch (operatorName) { + case "Tj", "'" -> { + if (token instanceof COSString cosString) { + yield cosString.getString(); + } + yield ""; + } + case "TJ" -> { + if (token instanceof COSArray cosArray) { + StringBuilder sb = new StringBuilder(); + for (COSBase element : cosArray) { + if (element instanceof COSString cosString) { + sb.append(cosString.getString()); + } + } + yield sb.toString(); + } + yield ""; + } + default -> ""; + }; + } + + // ----------------------------------------------------------------------- + // Inner data classes + // ----------------------------------------------------------------------- + + @Data + private static class GraphicsState { + private PDFont font = null; + private float fontSize = 0; + } + + @Data + @AllArgsConstructor + static class TextSegment { + private int tokenIndex; + private String operatorName; + private String text; + private int startPos; + private int endPos; + private PDFont font; + private float fontSize; + } + + @Data + @AllArgsConstructor + static class MatchRange { + private int startPos; + private int endPos; + } + + @Data + @AllArgsConstructor + private static class ModificationTask { + private TextSegment segment; + private String newText; + private float adjustment; + } +} diff --git a/app/core/src/main/java/stirling/software/SPDF/model/api/security/RedactExecuteRequest.java b/app/core/src/main/java/stirling/software/SPDF/model/api/security/RedactExecuteRequest.java new file mode 100644 index 000000000..ffa2af9df --- /dev/null +++ b/app/core/src/main/java/stirling/software/SPDF/model/api/security/RedactExecuteRequest.java @@ -0,0 +1,132 @@ +package stirling.software.SPDF.model.api.security; + +import java.util.ArrayList; +import java.util.List; + +import io.swagger.v3.oas.annotations.media.Schema; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import stirling.software.common.model.api.PDFFile; + +@Data +@EqualsAndHashCode(callSuper = true) +public class RedactExecuteRequest extends PDFFile { + + @Schema( + description = + "Exact strings to find and black out. One entry per phrase to redact." + + " Best for known names, identifiers, and specific text found in the document.") + private List textValues = new ArrayList<>(); + + @Schema( + description = + "Regex patterns to match and redact. Each match anywhere in the document is blacked out." + + " Uses Java/PCRE regex syntax. Well-suited for strings that follow known patterns, like" + + " phone numbers, email addresses, national ID numbers, or" + + " dates (which can appear with different separators, optional country codes," + + " etc.). For fixed known strings such as names, use textValues instead.") + private List regexPatterns = new ArrayList<>(); + + @Schema( + description = + "1-indexed page numbers to wipe entirely (all content removed from those pages).") + private List wipePages = new ArrayList<>(); + + @Schema( + description = + "Text ranges to redact by specifying a start and end anchor phrase. All" + + " content between the two phrases (inclusive) is redacted. Anchors" + + " work best when short and unique. They must appear" + + " verbatim in the document.") + private List ranges = new ArrayList<>(); + + @Schema( + description = + "Rectangular areas to black out, each defined by a page number and bounding box coordinates.") + private List imageBoxes = new ArrayList<>(); + + @Schema( + description = + "1-indexed page numbers to redact all detected images from. Pass an empty list to redact images from every page. Omit or pass null to skip image redaction entirely.") + private List redactImagePages; + + @Schema(description = "Redaction style options") + private RedactStyle style = new RedactStyle(); + + public record TextRange( + @Schema( + description = + "A short, distinctive phrase (5–15 words) that marks where" + + " redaction begins (inclusive). Must appear verbatim in" + + " the document — e.g. a section heading or a unique" + + " sentence fragment.", + requiredMode = Schema.RequiredMode.REQUIRED, + minLength = 1) + String startString, + @Schema( + description = + "A short, distinctive phrase (5–15 words) that marks where" + + " redaction ends (inclusive). Must appear verbatim in the" + + " document. Shorter phrases match more reliably.", + requiredMode = Schema.RequiredMode.REQUIRED, + minLength = 1) + String endString) { + public TextRange { + if (endString == null) endString = ""; + } + } + + public record ImageBox( + @Schema( + description = "0-indexed page number (first page = 0).", + requiredMode = Schema.RequiredMode.REQUIRED) + int pageIndex, + @Schema( + description = + "Left x coordinate of the redaction rectangle in PDF user-space points.", + requiredMode = Schema.RequiredMode.REQUIRED) + float x1, + @Schema( + description = + "Top y coordinate of the redaction rectangle in PDF user-space points.", + requiredMode = Schema.RequiredMode.REQUIRED) + float y1, + @Schema( + description = + "Right x coordinate of the redaction rectangle in PDF user-space points.", + requiredMode = Schema.RequiredMode.REQUIRED) + float x2, + @Schema( + description = + "Bottom y coordinate of the redaction rectangle in PDF user-space points.", + requiredMode = Schema.RequiredMode.REQUIRED) + float y2) {} + + public enum RedactionStrategy { + AUTO, + OVERLAY_ONLY, + IMAGE_FINALIZE + } + + @Data + public static class RedactStyle { + @Schema(description = "Hex redaction box color", defaultValue = "#000000") + private String color = "#000000"; + + @Schema( + description = "Extra padding around each box in points", + type = "number", + defaultValue = "0") + private float padding = 0f; + + @Schema(description = "Rasterize output to prevent text extraction", defaultValue = "false") + private boolean convertToImage = false; + + @Schema( + description = "Execution strategy hint for the redaction pipeline", + defaultValue = "AUTO") + private RedactionStrategy strategy = RedactionStrategy.AUTO; + } +} diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerTest.java index 584ad3744..15774415a 100644 --- a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactControllerTest.java @@ -37,7 +37,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; @@ -77,8 +76,11 @@ class RedactControllerTest { @Mock private CustomPDFDocumentFactory pdfDocumentFactory; @Mock private TempFileManager tempFileManager; + @Mock private RedactExecuteService redactExecuteService; - @InjectMocks private RedactController redactController; + private TextRedactionService textRedactionService; + private ManualRedactionService manualRedactionService; + private RedactController redactController; private MockMultipartFile mockPdfFile; private PDDocument mockDocument; @@ -201,7 +203,17 @@ class RedactControllerTest { .save(any(File.class)); doNothing().when(mockDocument).close(); - // Initialize a real document for unit tests + // Build real service instances so tests exercise actual logic + textRedactionService = new TextRedactionService(); + manualRedactionService = new ManualRedactionService(tempFileManager); + redactController = + new RedactController( + pdfDocumentFactory, + tempFileManager, + manualRedactionService, + textRedactionService, + redactExecuteService); + setupRealDocument(); } @@ -819,9 +831,9 @@ class RedactControllerTest { contentStream.newLineAtOffset(50, 750); contentStream.showText("This is "); - contentStream.newLineAtOffset(-10, 0); // Simulate positioning + contentStream.newLineAtOffset(-10, 0); contentStream.showText("secret"); - contentStream.newLineAtOffset(10, 0); // Reset positioning + contentStream.newLineAtOffset(10, 0); contentStream.showText(" information"); contentStream.endText(); } @@ -1005,7 +1017,7 @@ class RedactControllerTest { contentStream.showText("Original content"); contentStream.endText(); } - return redactController.createTokensWithoutTargetText( + return textRedactionService.createTokensWithoutTargetText( realDocument, pageForTokenExtraction, Collections.emptySet(), false, false); } @@ -1016,28 +1028,28 @@ class RedactControllerTest { @Test @DisplayName("Should decode valid hex color with hash") void decodeValidHexColorWithHash() { - Color result = redactController.decodeOrDefault("#FF0000"); + Color result = ManualRedactionService.decodeOrDefault("#FF0000"); assertEquals(Color.RED, result); } @Test @DisplayName("Should decode valid hex color without hash") void decodeValidHexColorWithoutHash() { - Color result = redactController.decodeOrDefault("FF0000"); + Color result = ManualRedactionService.decodeOrDefault("FF0000"); assertEquals(Color.RED, result); } @Test @DisplayName("Should default to black for null color") void defaultToBlackForNullColor() { - Color result = redactController.decodeOrDefault(null); + Color result = ManualRedactionService.decodeOrDefault(null); assertEquals(Color.BLACK, result); } @Test @DisplayName("Should default to black for invalid color") void defaultToBlackForInvalidColor() { - Color result = redactController.decodeOrDefault("invalid-color"); + Color result = ManualRedactionService.decodeOrDefault("invalid-color"); assertEquals(Color.BLACK, result); } @@ -1049,7 +1061,7 @@ class RedactControllerTest { }) @DisplayName("Should handle various valid color formats") void handleVariousValidColorFormats(String colorInput) { - Color result = redactController.decodeOrDefault(colorInput); + Color result = ManualRedactionService.decodeOrDefault(colorInput); assertNotNull(result); assertTrue( result.getRed() >= 0 && result.getRed() <= 255, @@ -1065,8 +1077,8 @@ class RedactControllerTest { @Test @DisplayName("Should handle short hex codes appropriately") void handleShortHexCodes() { - Color result1 = redactController.decodeOrDefault("123"); - Color result2 = redactController.decodeOrDefault("#12"); + Color result1 = ManualRedactionService.decodeOrDefault("123"); + Color result2 = ManualRedactionService.decodeOrDefault("#12"); assertNotNull(result1); assertNotNull(result2); @@ -1094,7 +1106,7 @@ class RedactControllerTest { Set targetWords = Set.of("confidential"); List tokens = - redactController.createTokensWithoutTargetText( + textRedactionService.createTokensWithoutTargetText( realDocument, realPage, targetWords, false, false); assertNotNull(tokens); @@ -1115,7 +1127,7 @@ class RedactControllerTest { Set targetWords = Set.of("secret"); List tokens = - redactController.createTokensWithoutTargetText( + textRedactionService.createTokensWithoutTargetText( realDocument, realPage, targetWords, false, false); assertNotNull(tokens); @@ -1148,7 +1160,7 @@ class RedactControllerTest { List originalTokens = getOriginalTokens(); List filteredTokens = - redactController.createTokensWithoutTargetText( + textRedactionService.createTokensWithoutTargetText( realDocument, realPage, targetWords, false, false); long originalNonTextCount = @@ -1156,7 +1168,7 @@ class RedactControllerTest { .filter( token -> token instanceof Operator op - && !redactController.isTextShowingOperator( + && !textRedactionService.isTextShowingOperator( op.getName())) .count(); @@ -1165,7 +1177,7 @@ class RedactControllerTest { .filter( token -> token instanceof Operator op - && !redactController.isTextShowingOperator( + && !textRedactionService.isTextShowingOperator( op.getName())) .count(); @@ -1184,7 +1196,7 @@ class RedactControllerTest { Set targetWords = Set.of("\\d{3}-\\d{2}-\\d{4}"); // SSN pattern List tokens = - redactController.createTokensWithoutTargetText( + textRedactionService.createTokensWithoutTargetText( realDocument, realPage, targetWords, true, false); String reconstructedText = extractTextFromTokens(tokens); @@ -1200,7 +1212,7 @@ class RedactControllerTest { Set targetWords = Set.of("test"); List tokens = - redactController.createTokensWithoutTargetText( + textRedactionService.createTokensWithoutTargetText( realDocument, realPage, targetWords, false, true); String reconstructedText = extractTextFromTokens(tokens); @@ -1217,7 +1229,7 @@ class RedactControllerTest { Set targetWords = Set.of("sensitive"); List tokens = - redactController.createTokensWithoutTargetText( + textRedactionService.createTokensWithoutTargetText( realDocument, realPage, targetWords, false, false); String reconstructedText = extractTextFromTokens(tokens); @@ -1231,7 +1243,7 @@ class RedactControllerTest { void shouldWriteTokensToNewContentStream() throws Exception { List tokens = createSampleTokenList(); - redactController.writeFilteredContentStream(realDocument, realPage, tokens); + textRedactionService.writeFilteredContentStream(realDocument, realPage, tokens); assertNotNull(realPage.getContents(), "Page should have content stream"); @@ -1249,7 +1261,7 @@ class RedactControllerTest { assertDoesNotThrow( () -> - redactController.writeFilteredContentStream( + textRedactionService.writeFilteredContentStream( realDocument, realPage, emptyTokens)); assertNotNull(realPage.getContents(), "Page should still have content stream"); @@ -1262,7 +1274,7 @@ class RedactControllerTest { String originalContent = extractTextFromModifiedPage(realPage); List newTokens = createSampleTokenList(); - redactController.writeFilteredContentStream(realDocument, realPage, newTokens); + textRedactionService.writeFilteredContentStream(realDocument, realPage, newTokens); String newContent = extractTextFromModifiedPage(realPage); assertNotEquals(originalContent, newContent, "Content stream should be replaced"); @@ -1273,7 +1285,7 @@ class RedactControllerTest { void shouldCreateWidthMatchingPlaceholder() { String originalText = "confidential"; String placeholder = - redactController.createPlaceholderWithFont( + textRedactionService.createPlaceholderWithFont( originalText, new PDType1Font(Standard14Fonts.FontName.HELVETICA)); assertEquals( @@ -1287,7 +1299,7 @@ class RedactControllerTest { void shouldHandleSpecialCharactersInPlaceholder() { String originalText = "café naïve"; String placeholder = - redactController.createPlaceholderWithFont( + textRedactionService.createPlaceholderWithFont( originalText, new PDType1Font(Standard14Fonts.FontName.HELVETICA)); assertEquals(originalText.length(), placeholder.length()); @@ -1303,10 +1315,10 @@ class RedactControllerTest { Set targetWords = Set.of("secret"); List filteredTokens = - redactController.createTokensWithoutTargetText( + textRedactionService.createTokensWithoutTargetText( realDocument, realPage, targetWords, false, false); - redactController.writeFilteredContentStream(realDocument, realPage, filteredTokens); + textRedactionService.writeFilteredContentStream(realDocument, realPage, filteredTokens); assertNotNull(realPage.getContents()); String finalText = extractTextFromModifiedPage(realPage); @@ -1322,7 +1334,7 @@ class RedactControllerTest { Set targetWords = Set.of("confidential"); List filteredTokens = - redactController.createTokensWithoutTargetText( + textRedactionService.createTokensWithoutTargetText( realDocument, realPage, targetWords, false, false); long filteredPositioning = @@ -1377,7 +1389,7 @@ class RedactControllerTest { Set targetWords = Set.of("confidential"); List tokens = - redactController.createTokensWithoutTargetText( + textRedactionService.createTokensWithoutTargetText( realDocument, realPage, targetWords, false, false); assertNotNull(tokens); @@ -1404,14 +1416,12 @@ class RedactControllerTest { @Test @DisplayName("Should handle documents with multiple text blocks") void shouldHandleDocumentsWithMultipleTextBlocks() throws Exception { - // Create a document with multiple text blocks realPage = new PDPage(PDRectangle.A4); while (realDocument.getNumberOfPages() > 0) { realDocument.removePage(0); } realDocument.addPage(realPage); - // Create resources PDResources resources = new PDResources(); resources.put( COSName.getPDFName("F1"), new PDType1Font(Standard14Fonts.FontName.HELVETICA)); diff --git a/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactExecuteServiceTest.java b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactExecuteServiceTest.java new file mode 100644 index 000000000..8a8b33564 --- /dev/null +++ b/app/core/src/test/java/stirling/software/SPDF/controller/api/security/RedactExecuteServiceTest.java @@ -0,0 +1,430 @@ +package stirling.software.SPDF.controller.api.security; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import stirling.software.SPDF.model.PDFText; +import stirling.software.SPDF.pdf.parser.PageColumnLayout; + +/** + * Integration tests for {@link RedactExecuteService#collectRangeBlocks(PDDocument, String, String, + * Map)}. Each test builds a synthetic PDF (single-column or two-column) with text-positioning that + * matches what a real document would produce, then asserts that the redaction range produces blocks + * confined to the expected X/Y region. + */ +class RedactExecuteServiceTest { + + private static final float PAGE_WIDTH = PDRectangle.LETTER.getWidth(); // 612 + private static final float PAGE_HEIGHT = PDRectangle.LETTER.getHeight(); // 792 + + private static final float LEFT_X = 72f; + private static final float RIGHT_X = 330f; + private static final float COL_WIDTH = 220f; + private static final float LINE_HEIGHT = 14f; + private static final float TOP_Y = PAGE_HEIGHT - 80f; + private static final float FONT_SIZE = 11f; + + private final RedactExecuteService service = + new RedactExecuteService(null, null, new TextRedactionService()); + + @Nested + @DisplayName("Single-column documents") + class SingleColumn { + + @Test + void redactBetweenMarkers_inclusive() throws IOException { + try (PDDocument doc = buildSingleColumnDoc()) { + Map cache = new HashMap<>(); + List blocks = + service.collectRangeBlocks(doc, "START-HERE", "STOP-HERE", cache); + + assertThat(blocks) + .as("blocks should be produced for single-column range") + .isNotEmpty(); + + // Blocks are in screen coords (top-left, Y down). START-HERE is drawn at the top + // of the page; STOP-HERE four lines below. Screen Y grows downward, so the + // anchors' screen-Y tops sit roughly around screenTop(0) and screenTop(4). + // The end anchor is inclusive, so blocks may extend to the bottom of line 4. + float screenTopOfStart = screenTopOfLine(0); + float screenBottomOfEnd = screenTopOfLine(4) + LINE_HEIGHT; + for (PDFText block : blocks) { + assertThat(block.getY1()) + .as("block top must be at or below the start anchor's top") + .isGreaterThanOrEqualTo(screenTopOfStart - 1f); + assertThat(block.getY2()) + .as( + "block bottom must not extend past the end anchor's bottom (end is inclusive)") + .isLessThanOrEqualTo(screenBottomOfEnd + 1f); + assertThat(block.getX2()) + .as("block should not extend into a hypothetical right column") + .isLessThan(PAGE_WIDTH / 2f + 50f); + } + } + } + + @Test + void missingStartString_noBlocks() throws IOException { + try (PDDocument doc = buildSingleColumnDoc()) { + Map cache = new HashMap<>(); + List blocks = + service.collectRangeBlocks(doc, "MISSING-START", "STOP-HERE", cache); + + assertThat(blocks).isEmpty(); + } + } + + @Test + void cvStyleHeadingPlusRightAlignedDate_stillTreatedAsSingleColumn() throws IOException { + // CV-style page: single-column body, but each section heading shares its row with a + // right-aligned date. The X-gap splitter emits the heading and the date as separate + // line boxes; this must NOT trip 2-column detection (the date is too narrow to be a + // real column), otherwise the cross-page redaction predicate over-includes wrong + // regions. + try (PDDocument doc = buildCvStyleDoc()) { + Map cache = new HashMap<>(); + List blocks = + service.collectRangeBlocks(doc, "SECTION-A", "SECTION-C", cache); + + assertThat(blocks) + .as("CV-style redaction between section headings must produce blocks") + .isNotEmpty(); + + PageColumnLayout layout = cache.get(0); + assertThat(layout.columnCount()) + .as("CV-style page with heading+date rows must remain single-column") + .isEqualTo(1); + } + } + + @Test + void punctuationDriftInAnchors_stillMatchesViaTolerantFallback() throws IOException { + // Simulates the LLM paraphrasing the heading by inserting a colon that isn't in the + // source ("#3 Character substitution" → "#3: Character substitution"). The + // punctuation-tolerant regex fallback should still find the line. + try (PDDocument doc = buildHeadingPdf()) { + Map cache = new HashMap<>(); + List blocks = + service.collectRangeBlocks( + doc, "#3: Character substitution", "#6: Image resolution", cache); + + assertThat(blocks) + .as("anchor with extra punctuation should still resolve via fallback") + .isNotEmpty(); + } + } + } + + @Nested + @DisplayName("Two-column documents") + class TwoColumn { + + @Test + void rangeInLeftColumn_redactsOnlyLeftColumn() throws IOException { + try (PDDocument doc = buildTwoColumnDoc()) { + Map cache = new HashMap<>(); + List blocks = service.collectRangeBlocks(doc, "L-START", "L-END", cache); + + assertThat(blocks).as("left-only range must produce blocks").isNotEmpty(); + + float gutterMid = (LEFT_X + COL_WIDTH + RIGHT_X) / 2f; + for (PDFText block : blocks) { + float midX = (block.getX1() + block.getX2()) / 2f; + assertThat(midX) + .as("every block must sit in the left column, never the right") + .isLessThan(gutterMid); + } + } + } + + @Test + void rangeInRightColumn_redactsOnlyRightColumn() throws IOException { + try (PDDocument doc = buildTwoColumnDoc()) { + Map cache = new HashMap<>(); + List blocks = service.collectRangeBlocks(doc, "R-START", "R-END", cache); + + assertThat(blocks).as("right-only range must produce blocks").isNotEmpty(); + + float gutterMid = (LEFT_X + COL_WIDTH + RIGHT_X) / 2f; + for (PDFText block : blocks) { + float midX = (block.getX1() + block.getX2()) / 2f; + assertThat(midX) + .as("every block must sit in the right column, never the left") + .isGreaterThan(gutterMid); + } + } + } + + @Test + void twoColumnWithTocAbove_pairsAcrossColumns() throws IOException { + // Reproduces magic.pdf-style stacked layout: a multi-line TOC near the top, then a + // 2-column body where the start anchor is in left col (lower screen Y) and the end + // anchor is in right col (higher screen Y). Original pairing failed here because + // end.y < start.y in screen coords. + try (PDDocument doc = buildTwoColumnWithTocDoc()) { + Map cache = new HashMap<>(); + List blocks = + service.collectRangeBlocks(doc, "BODY-L-3", "BODY-R-1", cache); + + assertThat(blocks) + .as("cross-column body redaction must produce blocks despite stacked TOC") + .isNotEmpty(); + } + } + + @Test + void crossColumnReadingOrder_leftBottomToRightTop_producesBothSides() throws IOException { + // This is the case the original code couldn't handle at all: end Y < start Y. + try (PDDocument doc = buildTwoColumnDoc()) { + Map cache = new HashMap<>(); + List blocks = + service.collectRangeBlocks(doc, "L-MIDDLE", "R-MIDDLE", cache); + + assertThat(blocks) + .as("cross-column range must produce blocks, not be silently dropped") + .isNotEmpty(); + + float gutterMid = (LEFT_X + COL_WIDTH + RIGHT_X) / 2f; + boolean sawLeft = false; + boolean sawRight = false; + for (PDFText block : blocks) { + float midX = (block.getX1() + block.getX2()) / 2f; + if (midX < gutterMid) sawLeft = true; + else sawRight = true; + } + assertThat(sawLeft).as("left column should contain at least one block").isTrue(); + assertThat(sawRight).as("right column should contain at least one block").isTrue(); + } + } + } + + // ── document fixtures ──────────────────────────────────────────────────────────────────────── + + /** + * Single-column page laid out as one column starting at LEFT_X. Lines: 0: START-HERE (start + * anchor) 1: line one 2: line two 3: line three 4: STOP-HERE (end anchor) 5: line five (must + * NOT be redacted) + */ + private PDDocument buildSingleColumnDoc() throws IOException { + PDDocument doc = new PDDocument(); + PDPage page = new PDPage(PDRectangle.LETTER); + doc.addPage(page); + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE); + String[] lines = { + "START-HERE", "line one", "line two", "line three", "STOP-HERE", "line five" + }; + for (int i = 0; i < lines.length; i++) { + cs.beginText(); + cs.newLineAtOffset(LEFT_X, yForLine(i)); + cs.showText(lines[i]); + cs.endText(); + } + } + return doc; + } + + /** + * Two-column page. Lines per column, top to bottom: Left: L-TOP, L-START, L-MIDDLE, L-END, + * L-BOTTOM Right: R-TOP, R-MIDDLE, R-START, R-END, R-BOTTOM + */ + private PDDocument buildTwoColumnDoc() throws IOException { + PDDocument doc = new PDDocument(); + PDPage page = new PDPage(PDRectangle.LETTER); + doc.addPage(page); + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE); + // Body lines are padded to make each column genuinely wide enough that column + // detection (which ignores narrow lines) treats both sides as real columns. + String fill = " " + "x".repeat(26); + String[] left = { + "L-TOP" + fill, + "L-START" + fill, + "L-MIDDLE" + fill, + "L-END" + fill, + "L-BOTTOM" + fill + }; + String[] right = { + "R-TOP" + fill, + "R-MIDDLE" + fill, + "R-START" + fill, + "R-END" + fill, + "R-BOTTOM" + fill + }; + for (int i = 0; i < left.length; i++) { + cs.beginText(); + cs.newLineAtOffset(LEFT_X, yForLine(i)); + cs.showText(left[i]); + cs.endText(); + } + // Aligned baselines per row (IEEE template style) — AllTextLineExtractor must split + // these at the column gap rather than merge same-row left+right glyphs into a wide + // box. + for (int i = 0; i < right.length; i++) { + cs.beginText(); + cs.newLineAtOffset(RIGHT_X, yForLine(i)); + cs.showText(right[i]); + cs.endText(); + } + } + return doc; + } + + /** + * Single-column page with feature headings: #1..#7 each followed by body text. The PDF text is + * exactly "#3 Character substitution" (no colon) — the test then queries with a colon to + * exercise the punctuation-tolerant fallback. + */ + private PDDocument buildHeadingPdf() throws IOException { + PDDocument doc = new PDDocument(); + PDPage page = new PDPage(PDRectangle.LETTER); + doc.addPage(page); + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE); + String[] lines = { + "#1 Auto layout", + "Body about auto layout.", + "#2 Smart selection", + "Body about smart selection.", + "#3 Character substitution", + "Body about character substitution.", + "#4 Rounded borders", + "Body about rounded borders.", + "#5 Auto contrast", + "Body about auto contrast.", + "#6 Image resolution", + "Body about image resolution.", + "#7 Columns", + "Body about columns." + }; + for (int i = 0; i < lines.length; i++) { + cs.beginText(); + cs.newLineAtOffset(LEFT_X, yForLine(i)); + cs.showText(lines[i]); + cs.endText(); + } + } + return doc; + } + + /** + * Two-column page like {@code magic.pdf}: a few full-width header lines, a 2-column TOC stacked + * on top of the 2-column body, where TOC's right half lives inside what would otherwise be the + * body's gutter. Body left column has BODY-L-1..3, right column has BODY-R-1..3. + */ + private PDDocument buildTwoColumnWithTocDoc() throws IOException { + PDDocument doc = new PDDocument(); + PDPage page = new PDPage(PDRectangle.LETTER); + doc.addPage(page); + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE); + // Header — full width, lines 0..1. + for (int i = 0; i < 2; i++) { + cs.beginText(); + cs.newLineAtOffset(LEFT_X, yForLine(i)); + cs.showText("FULL WIDTH HEADER LINE " + i + " ACROSS BOTH COLUMNS OF THE PAGE"); + cs.endText(); + } + // TOC, 2 columns of entries. TOC right half sits where the body gutter would be — + // exactly the layout that broke the histogram-based detector on magic.pdf. + float tocLeftX = 101f; + float tocRightX = 230f; + for (int i = 0; i < 5; i++) { + float y = yForLine(3 + i); + cs.beginText(); + cs.newLineAtOffset(tocLeftX, y); + cs.showText("TOC entry left " + i); + cs.endText(); + cs.beginText(); + cs.newLineAtOffset(tocRightX, y); + cs.showText("TOC entry right " + i); + cs.endText(); + } + // Body — 2-column with aligned baselines per row (IEEE-style). + String fill = " " + "x".repeat(26); + String[] bodyLeft = {"BODY-L-1" + fill, "BODY-L-2" + fill, "BODY-L-3" + fill}; + String[] bodyRight = {"BODY-R-1" + fill, "BODY-R-2" + fill, "BODY-R-3" + fill}; + for (int i = 0; i < bodyLeft.length; i++) { + cs.beginText(); + cs.newLineAtOffset(LEFT_X, yForLine(10 + i)); + cs.showText(bodyLeft[i]); + cs.endText(); + cs.beginText(); + cs.newLineAtOffset(RIGHT_X, yForLine(10 + i)); + cs.showText(bodyRight[i]); + cs.endText(); + } + } + return doc; + } + + /** + * CV-style page: single-column body with a few section headings, each followed on the same + * baseline by a right-aligned date string. {@link AllTextLineExtractor} will split each + * heading+date row into two line boxes; column detection must reject this as a fake two-column + * layout because the dates are too narrow to be a real column body. + */ + private PDDocument buildCvStyleDoc() throws IOException { + PDDocument doc = new PDDocument(); + PDPage page = new PDPage(PDRectangle.LETTER); + doc.addPage(page); + try (PDPageContentStream cs = new PDPageContentStream(doc, page)) { + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE); + + float dateX = PAGE_WIDTH - 144f; // right-aligned dates near the right margin + + // Section A: heading + date, then 3 body lines. + writeAt(cs, LEFT_X, yForLine(0), "SECTION-A"); + writeAt(cs, dateX, yForLine(0), "Jan 2020"); + writeAt(cs, LEFT_X, yForLine(1), "Body line A1 with enough width to look like body"); + writeAt(cs, LEFT_X, yForLine(2), "Body line A2 with enough width to look like body"); + writeAt(cs, LEFT_X, yForLine(3), "Body line A3 with enough width to look like body"); + + // Section B (in the redact range): heading + date + 3 body lines. + writeAt(cs, LEFT_X, yForLine(5), "SECTION-B"); + writeAt(cs, dateX, yForLine(5), "Feb 2021"); + writeAt(cs, LEFT_X, yForLine(6), "Body line B1 with enough width to look like body"); + writeAt(cs, LEFT_X, yForLine(7), "Body line B2 with enough width to look like body"); + writeAt(cs, LEFT_X, yForLine(8), "Body line B3 with enough width to look like body"); + + // Section C (end anchor): heading + date. + writeAt(cs, LEFT_X, yForLine(10), "SECTION-C"); + writeAt(cs, dateX, yForLine(10), "Mar 2022"); + } + return doc; + } + + private static void writeAt(PDPageContentStream cs, float x, float y, String text) + throws IOException { + cs.beginText(); + cs.newLineAtOffset(x, y); + cs.showText(text); + cs.endText(); + } + + /** PDF user-space Y baseline for line index {@code i} (0-based, top to bottom). */ + private static float yForLine(int lineIndex) { + return TOP_Y - lineIndex * LINE_HEIGHT; + } + + /** Approximate screen-Y of the top of line {@code i} (top-left origin). */ + private static float screenTopOfLine(int lineIndex) { + // baseline_pdf → baseline_screen flips against page height; glyph top ≈ baseline - font + // size. + return PAGE_HEIGHT - yForLine(lineIndex) - FONT_SIZE; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java index c7badf99c..c06007f31 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java @@ -8,13 +8,18 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.QuoteMode; import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.common.PDRectangle; import org.apache.pdfbox.text.PDFTextStripper; +import org.apache.pdfbox.text.TextPosition; import org.springframework.stereotype.Service; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -24,6 +29,7 @@ import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import stirling.software.SPDF.pdf.parser.PageImageLocator; import stirling.software.SPDF.pdf.parser.PdfIngester; import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage; import stirling.software.SPDF.pdf.parser.PdfModels.RawLine; @@ -32,6 +38,7 @@ import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment; import stirling.software.SPDF.pdf.parser.TabulaTableParser; import stirling.software.common.util.ExceptionUtils; import stirling.software.common.util.PdfUtils; +import stirling.software.common.util.RegexPatternUtils; import stirling.software.proprietary.model.api.ai.AiPdfContentType; import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest; import stirling.software.proprietary.model.api.ai.AiWorkflowTextSelection; @@ -122,6 +129,7 @@ public class PdfContentExtractor { } csvStrings.add(sw.toString()); } + return csvStrings; } @@ -295,7 +303,6 @@ public class PdfContentExtractor { private List extractPageText( PDDocument document, List selectedPages, int maxCharacters) throws IOException { - PDFTextStripper textStripper = new PDFTextStripper(); List pages = new ArrayList<>(); int remainingCharacters = maxCharacters; @@ -304,10 +311,29 @@ public class PdfContentExtractor { break; } + PDFTextStripper textStripper = new PDFTextStripper(); + textStripper.setSortByPosition(true); textStripper.setStartPage(pageNumber); textStripper.setEndPage(pageNumber); String pageText = textStripper.getText(document).trim(); + + // Prepend page dimensions so the AI agent can reason about absolute coordinates. + PDPage page = document.getPage(pageNumber - 1); + PDRectangle bbox = page.getBBox(); + String dimensionHeader = + String.format( + "--- Page dimensions: %.0fx%.0f pts" + + " (PDF user-space: origin bottom-left, Y up) ---\n", + bbox.getWidth(), bbox.getHeight()); + pageText = dimensionHeader + pageText; + + // Append image metadata so the AI agent can reason about images spatially. + String imageAnnotation = buildImageAnnotation(document, pageNumber - 1); + if (!imageAnnotation.isEmpty()) { + pageText = pageText + imageAnnotation; + } + if (pageText.isBlank()) { continue; } @@ -327,6 +353,56 @@ public class PdfContentExtractor { return pages; } + /** + * Builds a human-readable description of all images on a page to append to page text. Uses PDF + * user-space coordinates (origin bottom-left, Y up) so the AI can reference exact bounding + * boxes when requesting image redaction. + */ + private String buildImageAnnotation(PDDocument document, int pageIndex) { + try { + List images = extractImagePositions(document, pageIndex); + if (images.isEmpty()) { + return ""; + } + PDPage page = document.getPage(pageIndex); + PDRectangle bbox = page.getBBox(); + float pageWidth = bbox.getWidth(); + float pageHeight = bbox.getHeight(); + + StringBuilder sb = new StringBuilder("\n\n--- Images on this page ---"); + for (int i = 0; i < images.size(); i++) { + ImageBlock img = images.get(i); + String position = spatialLabel(img, pageWidth, pageHeight); + float w = img.x2() - img.x1(); + float h = img.y2() - img.y1(); + sb.append( + String.format( + "\nImage %d: position=%s, size=%.0fx%.0f pts," + + " bounds=(x1=%.0f, y1=%.0f, x2=%.0f, y2=%.0f)", + i + 1, position, w, h, img.x1(), img.y1(), img.x2(), img.y2())); + } + return sb.toString(); + } catch (Exception e) { + log.debug( + "Failed to extract image positions for page {}: {}", pageIndex, e.getMessage()); + return ""; + } + } + + /** + * Returns a human-readable spatial label (e.g. "top-left", "center") for an image based on its + * centre relative to the page dimensions. Coordinates are in PDF user-space (Y up). + */ + private static String spatialLabel(ImageBlock img, float pageWidth, float pageHeight) { + float cx = (img.x1() + img.x2()) / 2f; + float cy = (img.y1() + img.y2()) / 2f; + + String horiz = cx < pageWidth / 3f ? "left" : cx < 2 * pageWidth / 3f ? "center" : "right"; + // PDF Y increases upward, so higher Y = higher on the page = "top" + String vert = cy > 2 * pageHeight / 3f ? "top" : cy > pageHeight / 3f ? "middle" : "bottom"; + return vert + "-" + horiz; + } + private ExtractedFileText buildExtractedFileText( String fileName, List pages) { ExtractedFileText fileText = new ExtractedFileText(); @@ -347,6 +423,130 @@ public class PdfContentExtractor { return text.substring(0, end); } + // ----------------------------------------------------------------------- + // Text position finding + // ----------------------------------------------------------------------- + + /** + * A located text match inside a PDF: 0-based page index and bounding box in PDFBox coordinates + * (origin bottom-left). + */ + public record TextBlock(int pageIndex, float x1, float y1, float x2, float y2) {} + + /** + * An image found on a PDF page: 0-based page index and bounding box in PDF user-space + * coordinates (origin bottom-left, Y increases upward). + */ + public record ImageBlock(int pageIndex, float x1, float y1, float x2, float y2) {} + + /** + * Extract the bounding boxes of all raster/vector images on the given (0-based) page. + * + * @param document the open PDF + * @param pageIndex 0-based page index + * @return list of located images in document order + */ + public List extractImagePositions(PDDocument document, int pageIndex) + throws IOException { + PDPage page = document.getPage(pageIndex); + PageImageLocator locator = new PageImageLocator(page, pageIndex); + locator.processPage(page); + return locator.getImageBoxes().stream() + .map(b -> new ImageBlock(b.pageIndex(), b.x1(), b.y1(), b.x2(), b.y2())) + .toList(); + } + + /** + * Find all occurrences of {@code pattern} in {@code document} and return their bounding boxes. + * + * @param document the open PDF + * @param pattern the search string or regex + * @param useRegex {@code true} to treat {@code pattern} as a regular expression + * @return list of located matches, in page order + */ + public List findTextPositions(PDDocument document, String pattern, boolean useRegex) + throws IOException { + LocalTextFinder finder = new LocalTextFinder(pattern, useRegex); + finder.getText(document); + return finder.found; + } + + private static final class LocalTextFinder extends PDFTextStripper { + + private final String searchTerm; + private final boolean useRegex; + final List found = new ArrayList<>(); + + private final List pagePositions = new ArrayList<>(); + private final StringBuilder pageText = new StringBuilder(); + + LocalTextFinder(String searchTerm, boolean useRegex) throws IOException { + this.searchTerm = searchTerm; + this.useRegex = useRegex; + setWordSeparator(" "); + setLineSeparator("\n"); + } + + @Override + protected void startPage(PDPage page) throws IOException { + super.startPage(page); + pagePositions.clear(); + pageText.setLength(0); + } + + @Override + protected void writeString(String text, List positions) { + pageText.append(text); + pagePositions.addAll(positions); + } + + @Override + protected void writeWordSeparator() { + pageText.append(getWordSeparator()); + pagePositions.add(null); + } + + @Override + protected void writeLineSeparator() { + pageText.append(getLineSeparator()); + pagePositions.add(null); + } + + @Override + protected void endPage(PDPage page) throws IOException { + String text = pageText.toString(); + if (!text.isEmpty() && searchTerm != null && !searchTerm.isBlank()) { + String term = searchTerm.trim(); + String regex = useRegex ? term : "\\Q" + term + "\\E"; + Pattern pat = RegexPatternUtils.getInstance().createSearchPattern(regex, true); + Matcher matcher = pat.matcher(text); + while (matcher.find()) { + float minX = Float.MAX_VALUE; + float minY = Float.MAX_VALUE; + float maxX = -Float.MAX_VALUE; + float maxY = -Float.MAX_VALUE; + boolean hit = false; + for (int i = matcher.start(); i < matcher.end(); i++) { + if (i < pagePositions.size()) { + TextPosition tp = pagePositions.get(i); + if (tp != null) { + hit = true; + minX = Math.min(minX, tp.getX()); + maxX = Math.max(maxX, tp.getX() + tp.getWidth()); + minY = Math.min(minY, tp.getY() - tp.getHeight()); + maxY = Math.max(maxY, tp.getY()); + } + } + } + if (hit) { + found.add(new TextBlock(getCurrentPageNo() - 1, minX, minY, maxX, maxY)); + } + } + } + super.endPage(page); + } + } + // --- Types shared with AiWorkflowService (package-private) --- interface PdfContentResult { diff --git a/engine/src/stirling/agents/orchestrator.py b/engine/src/stirling/agents/orchestrator.py index 04e2d4517..4dbf0b65a 100644 --- a/engine/src/stirling/agents/orchestrator.py +++ b/engine/src/stirling/agents/orchestrator.py @@ -86,7 +86,7 @@ class OrchestratorAgent: system_prompt=( "You are the top-level orchestrator. " "Choose exactly one output function that best handles the request. " - "Use delegate_pdf_edit for requested modifications of single or multiple PDFs. " + "Use delegate_pdf_edit for any requested modification of one or more PDFs. " "Use delegate_pdf_question for questions about the contents of the attached PDFs. " "Use delegate_user_spec for requests to create or define an agent spec. " "Use delegate_pdf_review when the user wants the PDF returned with review" diff --git a/engine/src/stirling/agents/pdf_edit.py b/engine/src/stirling/agents/pdf_edit.py index 99901a4b6..cdc4fea4f 100644 --- a/engine/src/stirling/agents/pdf_edit.py +++ b/engine/src/stirling/agents/pdf_edit.py @@ -39,11 +39,39 @@ class PdfEditPlanSelection(ApiModel): summary: str -type PdfEditPlanOutput = PdfEditPlanSelection | EditClarificationRequest | EditCannotDoResponse | NeedContentResponse +class PdfEditNeedContentSelection(ApiModel): + """LLM-facing variant of need_content: the model signals it needs document content and gives + a reason. File objects are resolved by Python from request.files by matching names, so the + LLM can't fabricate file ids — it only selects by the display names it sees in the prompt. + """ + + outcome: Literal["need_content"] = "need_content" + reason: str + file_names: list[str] | None = Field( + default=None, + description=( + "Names of files whose content is needed. " + "Use the exact names shown in the prompt. " + "Omit or leave empty to request content from all files." + ), + ) + max_pages: int | None = None + max_characters: int | None = None + + +type PdfEditPlanOutput = ( + PdfEditPlanSelection | EditClarificationRequest | EditCannotDoResponse | PdfEditNeedContentSelection +) class PdfEditSelectionAgent: - def __init__(self, runtime: AppRuntime, base_system_prompt: str, *, allow_need_content: bool) -> None: + def __init__( + self, + runtime: AppRuntime, + base_system_prompt: str, + *, + allow_need_content: bool, + ) -> None: self.runtime = runtime output_types: list[type[PdfEditPlanOutput]] = [ PdfEditPlanSelection, @@ -52,11 +80,12 @@ class PdfEditSelectionAgent: ] system_prompt = base_system_prompt if allow_need_content: - output_types.append(NeedContentResponse) + output_types.append(PdfEditNeedContentSelection) system_prompt += ( " Return need_content when planning a correct answer requires inspecting the actual PDF " "page text (e.g. 'split after every page that says NEW PAGE', " - "'rotate pages that mention draft')." + "'rotate pages that mention draft'). " + "Set file_names to only the files that need to be read; omit it to read all files." ) self.agent = Agent( model=runtime.smart_model, @@ -101,14 +130,18 @@ class PdfEditParameterSelector: parameter_result = await self.agent.run( prompt, output_type=NativeOutput(parameter_model), - instructions=( - f"Generate only the parameters for the PDF operation `{operation_id.name}`. " - "Do not include fields from any other operation." - ), + instructions=self._get_operation_instructions(operation_id), ) logger.debug("[pdf-edit params %s] output: %s", operation_id.name, Pretty(parameter_result.output)) return parameter_result.output + @staticmethod + def _get_operation_instructions(operation_id: ToolEndpoint) -> str: + return ( + f"Generate only the parameters for the PDF operation `{operation_id.name}`. " + "Do not include fields from any other operation." + ) + def _build_parameter_prompt( self, request: PdfEditRequest, @@ -173,8 +206,7 @@ class PdfEditAgent: request.enabled_endpoints, request.user_message, ) - supported_operations = self._get_supported_operations(request) - unavailable_operations = self._get_unavailable_operations(supported_operations) + supported_operations, unavailable_operations = self._classify_operations(request) if not supported_operations: return EditCannotDoResponse(reason="No PDF edit operations are available on this server.") selection = await self._select_plan( @@ -183,9 +215,9 @@ class PdfEditAgent: if isinstance(selection, EditClarificationRequest | EditCannotDoResponse): logger.info("[pdf-edit] selection -> %s: %s", selection.outcome, Pretty(selection)) return selection - if isinstance(selection, NeedContentResponse): + if isinstance(selection, PdfEditNeedContentSelection): logger.info("[pdf-edit] selection -> need_content: %s", selection.reason) - return self._fill_need_content_defaults(selection, request) + return self._build_need_content_response(selection, request) enabled = set(supported_operations) unsupported = [op for op in selection.operations if op not in enabled] if unsupported: @@ -228,7 +260,9 @@ class PdfEditAgent: ) -> PdfEditPlanOutput: can_request_content = allow_need_content and not has_page_text(request.page_text) agent = self._build_selection_agent( - supported_operations, unavailable_operations, allow_need_content=can_request_content + supported_operations, + unavailable_operations, + allow_need_content=can_request_content, ) return await agent.select(self._build_selection_prompt(request, supported_operations, unavailable_operations)) @@ -263,9 +297,11 @@ class PdfEditAgent: "merging, or extracting pages then re-inserting them). " "Only return cannot_do when no sequence of the supported operations could achieve the request. " "Do not produce operation parameters in this stage. " - "Return need_clarification when the request is genuinely ambiguous. " "Return plan when a reasonable multi-step plan can be created. " - "Never return partial plans." + "Never return partial plans. " + "Return need_clarification only when the request is genuinely ambiguous in a way " + "that no reasonable interpretation could produce a correct plan — do not ask to " + "confirm details that are already clear from the user's message." ), allow_need_content=allow_need_content, ) @@ -291,13 +327,31 @@ class PdfEditAgent: f"Extracted page text:\n{format_page_text(request.page_text)}" ) - def _get_supported_operations(self, request: PdfEditRequest) -> Iterable[ToolEndpoint]: - return request.enabled_endpoints + # Endpoints that exist on the server and are callable via the direct API or the manual UI, + # but are never offered to the AI agent as a routing option. + # + # Why: REDACT_EXECUTE is the preferred AI-driven redaction route. AUTO_REDACT and REDACT are + # legacy endpoints that remain fully functional for human callers (the manual redact UI, direct + # API consumers, pipelines) but would produce a worse experience if the AI routed to them — + # they accept a simpler, less expressive schema and pre-date the unified operation model. + # Hiding them here channels all AI redaction traffic through REDACT_EXECUTE without disabling + # the legacy endpoints for anyone else. + # + # How to reuse: add an endpoint here whenever a legacy endpoint has a preferred replacement + # that the AI should use exclusively. The endpoint remains live on the server; only the AI + # planner is prevented from selecting it. + _AGENT_HIDDEN_ENDPOINTS: frozenset[ToolEndpoint] = frozenset({ToolEndpoint.AUTO_REDACT, ToolEndpoint.REDACT}) - @staticmethod - def _get_unavailable_operations(supported_operations: Iterable[ToolEndpoint]) -> Iterable[ToolEndpoint]: - supported_set = set(supported_operations) - return [op for op in OPERATIONS if op not in supported_set] + def _classify_operations(self, request: PdfEditRequest) -> tuple[list[ToolEndpoint], list[ToolEndpoint]]: + """Split the universe of operations into (supported, unavailable) from the agent's + point of view. Endpoints in `_AGENT_HIDDEN_ENDPOINTS` are filtered out regardless + of enabled state — they exist on the server but only callers outside the AI + pipeline (the manual redact UI, direct API consumers) can invoke them. + """ + enabled_set = set(request.enabled_endpoints) + supported = [op for op in request.enabled_endpoints if op not in self._AGENT_HIDDEN_ENDPOINTS] + unavailable = [op for op in OPERATIONS if op not in enabled_set and op not in self._AGENT_HIDDEN_ENDPOINTS] + return supported, unavailable @staticmethod def _get_operations_prompt(operations: Iterable[ToolEndpoint]) -> str: @@ -326,18 +380,29 @@ class PdfEditAgent: lines.append(f" {name}") return "\n".join(lines) - def _fill_need_content_defaults( + def _build_need_content_response( self, - selection: NeedContentResponse, + selection: PdfEditNeedContentSelection, request: PdfEditRequest, ) -> NeedContentResponse: - files = selection.files or [ - NeedContentFileRequest(file=file, content_types=[PdfContentType.PAGE_TEXT]) for file in request.files - ] + # File objects are resolved here by matching names against request.files so the LLM + # can't fabricate file ids — it selects by display name, Python provides the AiFile. + if selection.file_names: + requested = set(selection.file_names) + files = [f for f in request.files if f.name in requested] + if not files: + # Names didn't match anything; fall back to all files rather than sending nothing. + logger.warning( + "[pdf-edit] need_content file_names %s matched no request files — using all", + selection.file_names, + ) + files = request.files + else: + files = request.files return NeedContentResponse( resume_with=SupportedCapability.PDF_EDIT, reason=selection.reason, - files=files, + files=[NeedContentFileRequest(file=file, content_types=[PdfContentType.PAGE_TEXT]) for file in files], max_pages=selection.max_pages or self.runtime.settings.max_pages, max_characters=selection.max_characters or self.runtime.settings.max_characters, ) diff --git a/engine/src/stirling/contracts/__init__.py b/engine/src/stirling/contracts/__init__.py index 407f2ca51..696749d7d 100644 --- a/engine/src/stirling/contracts/__init__.py +++ b/engine/src/stirling/contracts/__init__.py @@ -116,7 +116,6 @@ from .progress import ( ) __all__ = [ - "AiFile", "AgentDraft", "AgentDraftRequest", "AgentDraftResponse", @@ -127,6 +126,7 @@ __all__ = [ "AgentRevisionWorkflowResponse", "AgentSpec", "AgentSpecStep", + "AiFile", "AiToolAgentStep", "ArtifactKind", "CannotContinueExecutionAction", diff --git a/engine/src/stirling/models/tool_models.py b/engine/src/stirling/models/tool_models.py index 91ff27b5b..e470eb190 100644 --- a/engine/src/stirling/models/tool_models.py +++ b/engine/src/stirling/models/tool_models.py @@ -542,6 +542,18 @@ class HtmlToPdfParams(ApiModel): zoom: float = Field(1, description="Zoom level for displaying the website. Default is '1'.") +class ImageBox(ApiModel): + """ + Rectangular areas to black out, each defined by a page number and bounding box coordinates. + """ + + page_index: int = Field(..., description="0-indexed page number (first page = 0).") + x1: float = Field(..., description="Left x coordinate of the redaction rectangle in PDF user-space points.") + x2: float = Field(..., description="Right x coordinate of the redaction rectangle in PDF user-space points.") + y1: float = Field(..., description="Top y coordinate of the redaction rectangle in PDF user-space points.") + y2: float = Field(..., description="Bottom y coordinate of the redaction rectangle in PDF user-space points.") + + class ColorType(StrEnum): """ The color type of the output image(s) @@ -984,6 +996,27 @@ class RearrangePagesParams(ApiModel): ) +class Strategy(StrEnum): + """ + Execution strategy hint for the redaction pipeline + """ + + auto = "AUTO" + overlay_only = "OVERLAY_ONLY" + image_finalize = "IMAGE_FINALIZE" + + +class Style(ApiModel): + """ + Redaction style options + """ + + color: str = Field("#000000", description="Hex redaction box color") + convert_to_image: bool = Field(False, description="Rasterize output to prevent text extraction") + padding: float = Field(0, description="Extra padding around each box in points") + strategy: Strategy = Field(Strategy.auto, description="Execution strategy hint for the redaction pipeline") + + class RedactionArea(ApiModel): """ A list of areas that should be redacted @@ -1279,6 +1312,23 @@ class SvgToPdfParams(ApiModel): ) +class TextRange(ApiModel): + """ + Text ranges to redact by specifying a start and end anchor phrase. All content between the two phrases (inclusive) is redacted. Anchors work best when short and unique. They must appear verbatim in the document. + """ + + end_string: str = Field( + ..., + description="A short, distinctive phrase (5–15 words) that marks where redaction ends (inclusive). Must appear verbatim in the document. Shorter phrases match more reliably.", + min_length=1, + ) + start_string: str = Field( + ..., + description="A short, distinctive phrase (5–15 words) that marks where redaction begins (inclusive). Must appear verbatim in the document — e.g. a section heading or a unique sentence fragment.", + min_length=1, + ) + + class TimestampPdfParams(ApiModel): tsa_url: str = Field( "http://timestamp.digicert.com", @@ -1346,6 +1396,32 @@ class VectorToPdfParams(ApiModel): prepress: Prepress = Field(Prepress.boolean_false, description="Apply Ghostscript prepress settings") +class RedactExecuteParams(ApiModel): + image_boxes: list[ImageBox] | None = Field( + None, description="Rectangular areas to black out, each defined by a page number and bounding box coordinates." + ) + ranges: list[TextRange] | None = Field( + None, + description="Text ranges to redact by specifying a start and end anchor phrase. All content between the two phrases (inclusive) is redacted. Anchors work best when short and unique. They must appear verbatim in the document.", + ) + redact_image_pages: list[int] | None = Field( + None, + description="1-indexed page numbers to redact all detected images from. Pass an empty list to redact images from every page. Omit or pass null to skip image redaction entirely.", + ) + regex_patterns: list[str] | None = Field( + None, + description="Regex patterns to match and redact. Each match anywhere in the document is blacked out. Uses Java/PCRE regex syntax. Well-suited for strings that follow known patterns, like phone numbers, email addresses, national ID numbers, or dates (which can appear with different separators, optional country codes, etc.). For fixed known strings such as names, use textValues instead.", + ) + style: Style | None = Field(None, description="Redaction style options") + text_values: list[str] | None = Field( + None, + description="Exact strings to find and black out. One entry per phrase to redact. Best for known names, identifiers, and specific text found in the document.", + ) + wipe_pages: list[int] | None = Field( + None, description="1-indexed page numbers to wipe entirely (all content removed from those pages)." + ) + + class RedactParams(ApiModel): convert_pdf_to_image: bool = Field(False, description="Convert the redacted PDF to an image") page_numbers: str = Field( @@ -1420,6 +1496,7 @@ class Model( | SessionsParams | ValidateCertificateParams | RedactParams + | RedactExecuteParams | RemovePasswordParams | SanitizePdfParams | TimestampPdfParams @@ -1488,6 +1565,7 @@ class Model( | SessionsParams | ValidateCertificateParams | RedactParams + | RedactExecuteParams | RemovePasswordParams | SanitizePdfParams | TimestampPdfParams @@ -1557,6 +1635,7 @@ type ParamToolModel = ( | SessionsParams | ValidateCertificateParams | RedactParams + | RedactExecuteParams | RemovePasswordParams | SanitizePdfParams | TimestampPdfParams @@ -1627,6 +1706,7 @@ class ToolEndpoint(StrEnum): SESSIONS = "/api/v1/security/cert-sign/sessions" VALIDATE_CERTIFICATE = "/api/v1/security/cert-sign/validate-certificate" REDACT = "/api/v1/security/redact" + REDACT_EXECUTE = "/api/v1/security/redact-execute" REMOVE_PASSWORD = "/api/v1/security/remove-password" SANITIZE_PDF = "/api/v1/security/sanitize-pdf" TIMESTAMP_PDF = "/api/v1/security/timestamp-pdf" @@ -1695,6 +1775,7 @@ OPERATIONS: dict[ToolEndpoint, ParamToolModelType] = { ToolEndpoint.SESSIONS: SessionsParams, ToolEndpoint.VALIDATE_CERTIFICATE: ValidateCertificateParams, ToolEndpoint.REDACT: RedactParams, + ToolEndpoint.REDACT_EXECUTE: RedactExecuteParams, ToolEndpoint.REMOVE_PASSWORD: RemovePasswordParams, ToolEndpoint.SANITIZE_PDF: SanitizePdfParams, ToolEndpoint.TIMESTAMP_PDF: TimestampPdfParams, diff --git a/engine/tests/test_pdf_edit_agent.py b/engine/tests/test_pdf_edit_agent.py index 52c40ce9b..0ba5eca6c 100644 --- a/engine/tests/test_pdf_edit_agent.py +++ b/engine/tests/test_pdf_edit_agent.py @@ -6,7 +6,7 @@ from dataclasses import dataclass import pytest from stirling.agents import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection -from stirling.agents.pdf_edit import PdfEditPlanOutput +from stirling.agents.pdf_edit import PdfEditNeedContentSelection, PdfEditPlanOutput from stirling.contracts import ( AiFile, EditCannotDoResponse, @@ -82,12 +82,12 @@ class StubPdfEditAgent(PdfEditAgent): if parameter_selector is not None: self.parameter_selector = parameter_selector - def _get_supported_operations(self, request: PdfEditRequest) -> Iterable[ToolEndpoint]: + def _classify_operations(self, request: PdfEditRequest) -> tuple[list[ToolEndpoint], list[ToolEndpoint]]: # Tests construct requests without `enabled_endpoints`; pretend everything is enabled # unless the test explicitly supplies an enabled set. - if request.enabled_endpoints: - return request.enabled_endpoints - return OPERATIONS + if not request.enabled_endpoints: + return list(OPERATIONS), [] + return super()._classify_operations(request) async def _select_plan( self, @@ -194,12 +194,8 @@ async def test_pdf_edit_agent_returns_need_content_without_building_plan(runtime parameter_selector = RecordingParameterSelector() agent = StubPdfEditAgent( runtime, - NeedContentResponse( - resume_with=SupportedCapability.PDF_EDIT, + PdfEditNeedContentSelection( reason="Need page text to locate the NEW PAGE markers.", - files=[], - max_pages=0, - max_characters=0, ), parameter_selector=parameter_selector, ) @@ -274,8 +270,8 @@ async def test_pdf_edit_selection_agent_excludes_need_content_from_schema_when_n can_request = PdfEditSelectionAgent(runtime, "base", allow_need_content=True) cannot_request = PdfEditSelectionAgent(runtime, "base", allow_need_content=False) - assert NeedContentResponse in _agent_output_types(can_request) - assert NeedContentResponse not in _agent_output_types(cannot_request) + assert PdfEditNeedContentSelection in _agent_output_types(can_request) + assert PdfEditNeedContentSelection not in _agent_output_types(cannot_request) def _agent_output_types(agent: object) -> list[type]: @@ -336,7 +332,7 @@ async def test_pdf_edit_agent_supported_operations_defaults_to_empty( runtime: AppRuntime, ) -> None: agent = PdfEditAgent(runtime) - supported = agent._get_supported_operations(PdfEditRequest(user_message="hi")) + supported, _ = agent._classify_operations(PdfEditRequest(user_message="hi")) assert list(supported) == [] @@ -351,7 +347,7 @@ async def test_pdf_edit_agent_supported_operations_uses_provided_list( enabled_endpoints=[ToolEndpoint.FLATTEN, ToolEndpoint.ROTATE_PDF], ) - supported = agent._get_supported_operations(request) + supported, _ = agent._classify_operations(request) assert list(supported) == [ToolEndpoint.FLATTEN, ToolEndpoint.ROTATE_PDF] @@ -372,8 +368,7 @@ def test_pdf_edit_selection_prompt_includes_unavailable_operations(runtime: AppR user_message="Run OCR.", enabled_endpoints=[ToolEndpoint.FLATTEN], ) - supported = agent._get_supported_operations(request) - unavailable = agent._get_unavailable_operations(supported) + supported, unavailable = agent._classify_operations(request) prompt = agent._build_selection_prompt(request, supported, unavailable) diff --git a/frontend/editor/src/core/components/viewer/EmbedPdfViewer.tsx b/frontend/editor/src/core/components/viewer/EmbedPdfViewer.tsx index f3e9b8a93..1af42d712 100644 --- a/frontend/editor/src/core/components/viewer/EmbedPdfViewer.tsx +++ b/frontend/editor/src/core/components/viewer/EmbedPdfViewer.tsx @@ -64,15 +64,16 @@ async function extractPageMeasureScales( // Parse a Measure dict into a MeasureScale, or return null if malformed. const parseScale = (measureObj: unknown) => { if (!(measureObj instanceof PDFDict)) return null; - const rObj = measureObj.lookup(PDFName.of("R")); + // @cantoo/pdf-lib ships without individual .d.ts files so instanceof can't narrow `unknown` + const m = measureObj as PDFDict; + const rObj = m.lookup(PDFName.of("R")); const ratioLabel = rObj instanceof PDFString || rObj instanceof PDFHexString ? rObj.decodeText() : ""; // D = distance array, X = x-axis fallback - let fmtArray = measureObj.lookup(PDFName.of("D")); - if (!(fmtArray instanceof PDFArray)) - fmtArray = measureObj.lookup(PDFName.of("X")); + let fmtArray = m.lookup(PDFName.of("D")); + if (!(fmtArray instanceof PDFArray)) fmtArray = m.lookup(PDFName.of("X")); if (!(fmtArray instanceof PDFArray)) return null; const firstFmt = fmtArray.lookup(0); if (!(firstFmt instanceof PDFDict)) return null; diff --git a/frontend/editor/src/core/tools/formFill/providers/PdfiumFormProvider.ts b/frontend/editor/src/core/tools/formFill/providers/PdfiumFormProvider.ts index b560c2867..27dc19595 100644 --- a/frontend/editor/src/core/tools/formFill/providers/PdfiumFormProvider.ts +++ b/frontend/editor/src/core/tools/formFill/providers/PdfiumFormProvider.ts @@ -22,6 +22,20 @@ import type { ButtonAction, } from "@app/tools/formFill/types"; import type { IFormDataProvider } from "@app/tools/formFill/providers/types"; +import type { + PDFDict, + PDFString, + PDFHexString, + PDFName, +} from "@cantoo/pdf-lib"; + +interface PDFAcroField { + dict: PDFDict; + getWidgets?: () => Array<{ dict: PDFDict }>; +} +interface PDFFieldInternal { + acroField?: PDFAcroField; +} import { closeDocAndFreeBuffer, extractFormFields, @@ -238,7 +252,11 @@ export class PdfiumFormProvider implements IFormDataProvider { ); const altName = readUtf16(m, altBuf, altLen); m.pdfium.wasmExports.free(altBuf); - (nameToField.get(name) as any)._tooltip = altName || null; + ( + nameToField.get(name) as PdfiumFormField & { + _tooltip?: string | null; + } + )._tooltip = altName || null; } enriched.add(name); } @@ -299,7 +317,7 @@ export class PdfiumFormProvider implements IFormDataProvider { const decodeText = (obj: unknown): string => { if (obj instanceof PDFString || obj instanceof PDFHexString) - return obj.decodeText(); + return (obj as PDFString | PDFHexString).decodeText(); return String(obj ?? ""); }; @@ -312,7 +330,8 @@ export class PdfiumFormProvider implements IFormDataProvider { ) continue; - const acroDict = (field.acroField as any).dict; + const acroDict = (field as unknown as PDFFieldInternal).acroField! + .dict; const optRaw = acroDict.lookup(PDFName.of("Opt")); if (!(optRaw instanceof PDFArray)) continue; @@ -374,8 +393,14 @@ export class PdfiumFormProvider implements IFormDataProvider { if (buttons.length === 0) return result; try { - const { PDFDocument, PDFName, PDFString, PDFHexString, PDFDict } = - await import("@cantoo/pdf-lib"); + const { + PDFDocument, + PDFName, + PDFString, + PDFHexString, + PDFDict, + PDFNumber, + } = await import("@cantoo/pdf-lib"); const doc = await PDFDocument.load(data, { ignoreEncryption: true, @@ -385,53 +410,53 @@ export class PdfiumFormProvider implements IFormDataProvider { const decodeText = (obj: unknown): string | null => { if (obj instanceof PDFString || obj instanceof PDFHexString) - return obj.decodeText(); + return (obj as PDFString | PDFHexString).decodeText(); if (obj instanceof PDFName) - return (obj as any).asString?.() ?? obj.toString().replace(/^\//, ""); + return (obj as PDFName).asString() ?? String(obj).replace(/^\//, ""); return null; }; const parseActionDict = (aObj: unknown): ButtonAction | null => { if (!(aObj instanceof PDFDict)) return null; - const sObj = aObj.lookup(PDFName.of("S")); + // @cantoo/pdf-lib ships without individual .d.ts files so instanceof can't narrow `unknown` + const a = aObj as PDFDict; + const sObj = a.lookup(PDFName.of("S")); if (!(sObj instanceof PDFName)) return null; const actionType: string = - (sObj as any).asString?.() ?? sObj.toString().replace(/^\//, ""); + (sObj as PDFName).asString() ?? String(sObj).replace(/^\//, ""); switch (actionType) { case "Named": { - const nObj = aObj.lookup(PDFName.of("N")); + const nObj = a.lookup(PDFName.of("N")); const name = nObj instanceof PDFName - ? ((nObj as any).asString?.() ?? - nObj.toString().replace(/^\//, "")) + ? ((nObj as PDFName).asString() ?? + String(nObj).replace(/^\//, "")) : ""; return { type: "named", namedAction: name }; } case "JavaScript": { - const jsObj = aObj.lookup(PDFName.of("JS")); + const jsObj = a.lookup(PDFName.of("JS")); const js = decodeText(jsObj) ?? jsObj?.toString() ?? ""; return { type: "javascript", javascript: js }; } case "SubmitForm": { - const fObj = aObj.lookup(PDFName.of("F")); + const fObj = a.lookup(PDFName.of("F")); let url = ""; if (fObj instanceof PDFDict) { url = decodeText(fObj.lookup(PDFName.of("F"))) ?? ""; } else if (fObj) { - url = decodeText(fObj) ?? fObj.toString(); + url = decodeText(fObj) ?? String(fObj); } - const flagsObj = aObj.lookup(PDFName.of("Flags")); + const flagsObj = a.lookup(PDFName.of("Flags")); const flags = - typeof (flagsObj as any)?.asNumber === "function" - ? (flagsObj as any).asNumber() - : 0; + flagsObj instanceof PDFNumber ? flagsObj.asNumber() : 0; return { type: "submitForm", url, submitFlags: flags }; } case "ResetForm": return { type: "resetForm" }; case "URI": { - const uriObj = aObj.lookup(PDFName.of("URI")); + const uriObj = a.lookup(PDFName.of("URI")); return { type: "uri", url: decodeText(uriObj) ?? "" }; } default: @@ -439,7 +464,7 @@ export class PdfiumFormProvider implements IFormDataProvider { } }; - const getMkCaption = (dict: any): string | null => { + const getMkCaption = (dict: PDFDict): string | null => { try { const mkObj = dict.lookup(PDFName.of("MK")); if (!(mkObj instanceof PDFDict)) return null; @@ -450,7 +475,7 @@ export class PdfiumFormProvider implements IFormDataProvider { } }; - const getActionFromDict = (dict: any): ButtonAction | null => { + const getActionFromDict = (dict: PDFDict): ButtonAction | null => { try { return parseActionDict(dict.lookup(PDFName.of("A"))); } catch { @@ -465,13 +490,13 @@ export class PdfiumFormProvider implements IFormDataProvider { if (!buttonNames.has(name)) continue; try { - const acroField = (field as any).acroField; + const acroField = (field as unknown as PDFFieldInternal).acroField; if (!acroField?.dict) continue; const info: { label?: string; action?: ButtonAction } = {}; // Try widget dicts first (each widget can have its own /MK and /A) - const widgets: any[] = (acroField as any).getWidgets?.() ?? []; + const widgets = acroField.getWidgets?.() ?? []; for (const widget of widgets) { if (!info.label) { const label = getMkCaption(widget.dict); diff --git a/frontend/editor/src/prototypes/data/usePrototypeToolRegistry.tsx b/frontend/editor/src/prototypes/data/usePrototypeToolRegistry.tsx index 2ac730dac..3ebcfeade 100644 --- a/frontend/editor/src/prototypes/data/usePrototypeToolRegistry.tsx +++ b/frontend/editor/src/prototypes/data/usePrototypeToolRegistry.tsx @@ -25,7 +25,13 @@ export function usePrototypeToolRegistry(): PrototypeToolRegistry { () => ({ pdfCommentAgent: { - icon: , + icon: ( + + ), name: t("home.pdfCommentAgent.title", "Add AI comments"), component: PdfCommentAgent, description: t(