mirror of
https://github.com/arsvendg/Stirling-PDF.git
synced 2026-07-14 10:34:06 +02:00
smart redaction (#6195)
Co-authored-by: James Brunton <[email protected]>
This commit is contained in:
EthanHealy01
co-authored by
James Brunton
James Brunton
parent
7f3ca7ea70
commit
35a712a278
@@ -0,0 +1,142 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Detects whether a page is one- or two-column from per-line bounding boxes, and classifies an
|
||||
* X-span into the column it belongs to. Detection is a midpoint vote at {@code pageWidth / 2}.
|
||||
*
|
||||
* <p>Capped at two columns by design — sufficient for the redaction target set (single-column
|
||||
* documents and IEEE-style two-column papers). 3+ column layouts (newspapers, magazines) and
|
||||
* off-centre gutters (asymmetric two-column) would need a histogram or clustering approach to
|
||||
* detect the actual gutter X. (future work)
|
||||
*
|
||||
* <p>Coordinates are PDFTextStripper screen space (top-left origin, Y increases downward).
|
||||
*/
|
||||
public final class PageColumnLayout {
|
||||
|
||||
/**
|
||||
* Slack when checking "crosses a gutter" so single-pixel overshoots don't mark a line as
|
||||
* spanning.
|
||||
*/
|
||||
public static final float SPAN_SLACK_PT = 2f;
|
||||
|
||||
/**
|
||||
* Slack on each side of the page midpoint inside which a line is considered "spanning"
|
||||
* (covering both columns) rather than belonging to one side.
|
||||
*/
|
||||
private static final float MIDPOINT_SLACK_PT = 30f;
|
||||
|
||||
/**
|
||||
* Minimum line width (points) for a line to count toward the two-column tally. Avoids false
|
||||
* positives where right-aligned dates, page numbers, or short "Link" fragments next to a
|
||||
* heading look like a second column when they're really just inline metadata.
|
||||
*/
|
||||
private static final float MIN_COLUMN_LINE_WIDTH_PT = 100f;
|
||||
|
||||
/**
|
||||
* Minimum number of clearly leftish AND clearly rightish lines (each of width ≥ {@link
|
||||
* #MIN_COLUMN_LINE_WIDTH_PT}) required to call the page two-column. Anything below this falls
|
||||
* back to single-column.
|
||||
*/
|
||||
private static final int MIN_SIDE_LINES = 3;
|
||||
|
||||
private final List<float[]> columns;
|
||||
private final List<float[]> gutters;
|
||||
|
||||
private PageColumnLayout(List<float[]> columns, List<float[]> gutters) {
|
||||
this.columns = columns;
|
||||
this.gutters = gutters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines column layout from per-line bounding boxes ({@code [x1, _, x2, _]}). Counts lines
|
||||
* whose X-midpoint sits clearly left of, or clearly right of, the page midpoint (with {@link
|
||||
* #MIDPOINT_SLACK_PT} slack each side). If both sides have at least {@link #MIN_SIDE_LINES}
|
||||
* lines, the page is treated as two-column with the gutter at the page midpoint. Otherwise it's
|
||||
* single-column.
|
||||
*
|
||||
* <p>Cross-column lines must already be split: callers should feed boxes from a line extractor
|
||||
* that splits same-Y glyphs at large X gaps (see {@code AllTextLineExtractor}). Without that
|
||||
* split, IEEE-style aligned-baseline 2-column PDFs produce one wide merged box per row and the
|
||||
* side tallies all end up classified as "spanning", falling to single-column.
|
||||
*/
|
||||
public static PageColumnLayout fromLineBoxes(List<float[]> lineBoxes, float pageWidth) {
|
||||
if (lineBoxes == null || lineBoxes.isEmpty()) {
|
||||
return new PageColumnLayout(List.of(new float[] {0f, pageWidth}), List.of());
|
||||
}
|
||||
float pageMid = pageWidth / 2f;
|
||||
int left = 0, right = 0;
|
||||
for (float[] lb : lineBoxes) {
|
||||
if (lb == null || lb.length < 3) continue;
|
||||
float width = lb[2] - lb[0];
|
||||
// Skip narrow lines — dates, page numbers, "Link" labels next to a heading should
|
||||
// not, on their own, make a single-column doc look two-column.
|
||||
if (width < MIN_COLUMN_LINE_WIDTH_PT) continue;
|
||||
float mid = (lb[0] + lb[2]) * 0.5f;
|
||||
if (mid < pageMid - MIDPOINT_SLACK_PT) left++;
|
||||
else if (mid > pageMid + MIDPOINT_SLACK_PT) right++;
|
||||
}
|
||||
if (left < MIN_SIDE_LINES || right < MIN_SIDE_LINES) {
|
||||
return new PageColumnLayout(List.of(new float[] {0f, pageWidth}), List.of());
|
||||
}
|
||||
float gutterL = pageMid - MIDPOINT_SLACK_PT;
|
||||
float gutterR = pageMid + MIDPOINT_SLACK_PT;
|
||||
return new PageColumnLayout(
|
||||
List.of(new float[] {0f, gutterL}, new float[] {gutterR, pageWidth}),
|
||||
List.of(new float[] {gutterL, gutterR}));
|
||||
}
|
||||
|
||||
/** All columns, left-to-right, as {@code [leftX, rightX]} pairs. Never empty. */
|
||||
public List<float[]> columns() {
|
||||
return columns;
|
||||
}
|
||||
|
||||
/** Gutters between columns, left-to-right, as {@code [leftX, rightX]} pairs. */
|
||||
public List<float[]> gutters() {
|
||||
return gutters;
|
||||
}
|
||||
|
||||
public int columnCount() {
|
||||
return columns.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the column index containing the X-midpoint of {@code [x1, x2]}, falling back to the
|
||||
* closest column if the midpoint sits inside a gutter.
|
||||
*/
|
||||
public int columnOf(float x1, float x2) {
|
||||
float mid = (x1 + x2) * 0.5f;
|
||||
int best = 0;
|
||||
float bestDist = Float.MAX_VALUE;
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
float[] c = columns.get(i);
|
||||
if (mid >= c[0] && mid <= c[1]) return i;
|
||||
float dist = mid < c[0] ? c[0] - mid : mid - c[1];
|
||||
if (dist < bestDist) {
|
||||
bestDist = dist;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns every column index whose X-range overlaps {@code [x1, x2]} with at least {@link
|
||||
* #SPAN_SLACK_PT} of intrusion. A normal in-column line returns one index; a line crossing a
|
||||
* gutter returns two or more.
|
||||
*/
|
||||
public int[] columnsCrossing(float x1, float x2) {
|
||||
List<Integer> hits = new ArrayList<>();
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
float[] c = columns.get(i);
|
||||
float overlap = Math.min(x2, c[1]) - Math.max(x1, c[0]);
|
||||
if (overlap > SPAN_SLACK_PT) hits.add(i);
|
||||
}
|
||||
if (hits.isEmpty()) hits.add(columnOf(x1, x2));
|
||||
int[] out = new int[hits.size()];
|
||||
for (int i = 0; i < hits.size(); i++) out[i] = hits.get(i);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.contentstream.PDFGraphicsStreamEngine;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImage;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
|
||||
/**
|
||||
* PDFGraphicsStreamEngine that intercepts {@code drawImage} calls and records each image's bounding
|
||||
* box in PDF user-space (origin bottom-left, Y up) by transforming the unit square through the
|
||||
* current transformation matrix (CTM).
|
||||
*
|
||||
* <p>Usage:
|
||||
*
|
||||
* <pre>{@code
|
||||
* PageImageLocator locator = new PageImageLocator(page, pageIndex);
|
||||
* locator.processPage(page);
|
||||
* List<ImageBox> boxes = locator.getImageBoxes();
|
||||
* }</pre>
|
||||
*
|
||||
* <p>Each {@link ImageBox} carries the 0-based page index and the axis-aligned bounding box {@code
|
||||
* (x1, y1, x2, y2)} in PDF user-space coordinates.
|
||||
*/
|
||||
public final class PageImageLocator extends PDFGraphicsStreamEngine {
|
||||
|
||||
/**
|
||||
* Bounding box of a raster or vector image found on a PDF page.
|
||||
*
|
||||
* @param pageIndex 0-based page index
|
||||
* @param x1 left edge in PDF user-space (origin bottom-left)
|
||||
* @param y1 bottom edge in PDF user-space
|
||||
* @param x2 right edge
|
||||
* @param y2 top edge
|
||||
*/
|
||||
public record ImageBox(int pageIndex, float x1, float y1, float x2, float y2) {}
|
||||
|
||||
private final int pageIndex;
|
||||
private final List<ImageBox> imageBoxes = new ArrayList<>();
|
||||
private final Point2D.Float currentPoint = new Point2D.Float();
|
||||
|
||||
/**
|
||||
* @param page the PDPage to process
|
||||
* @param pageIndex 0-based index of this page in the document (stored on each returned {@link
|
||||
* ImageBox})
|
||||
*/
|
||||
public PageImageLocator(PDPage page, int pageIndex) {
|
||||
super(page);
|
||||
this.pageIndex = pageIndex;
|
||||
}
|
||||
|
||||
/** Returns all image bounding boxes collected during {@link #processPage}. */
|
||||
public List<ImageBox> getImageBoxes() {
|
||||
return imageBoxes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void drawImage(PDImage pdImage) throws IOException {
|
||||
Matrix ctm = getGraphicsState().getCurrentTransformationMatrix();
|
||||
// An image occupies the unit square (0,0)→(1,1) in image space.
|
||||
// Transform all four corners through the CTM to get the page-space bounding box.
|
||||
float a = ctm.getScaleX();
|
||||
float b = ctm.getShearY();
|
||||
float c = ctm.getShearX();
|
||||
float d = ctm.getScaleY();
|
||||
float e = ctm.getTranslateX();
|
||||
float f = ctm.getTranslateY();
|
||||
float[] xs = {e, a + e, c + e, a + c + e};
|
||||
float[] ys = {f, b + f, d + f, b + d + f};
|
||||
float x1 = Float.MAX_VALUE, y1 = Float.MAX_VALUE;
|
||||
float x2 = -Float.MAX_VALUE, y2 = -Float.MAX_VALUE;
|
||||
for (float x : xs) {
|
||||
x1 = Math.min(x1, x);
|
||||
x2 = Math.max(x2, x);
|
||||
}
|
||||
for (float y : ys) {
|
||||
y1 = Math.min(y1, y);
|
||||
y2 = Math.max(y2, y);
|
||||
}
|
||||
imageBoxes.add(new ImageBox(pageIndex, x1, y1, x2, y2));
|
||||
}
|
||||
|
||||
// ---------- required abstract methods (no-op for path operations) ----------
|
||||
|
||||
@Override
|
||||
public void appendRectangle(Point2D p0, Point2D p1, Point2D p2, Point2D p3) {}
|
||||
|
||||
@Override
|
||||
public void clip(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void moveTo(float x, float y) {
|
||||
currentPoint.setLocation(x, y);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void lineTo(float x, float y) {
|
||||
currentPoint.setLocation(x, y);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void curveTo(float x1, float y1, float x2, float y2, float x3, float y3) {
|
||||
currentPoint.setLocation(x3, y3);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Point2D getCurrentPoint() {
|
||||
return currentPoint;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closePath() {}
|
||||
|
||||
@Override
|
||||
public void endPath() {}
|
||||
|
||||
@Override
|
||||
public void strokePath() {}
|
||||
|
||||
@Override
|
||||
public void fillPath(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void fillAndStrokePath(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void shadingFill(COSName shadingName) {}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Binds a multipart form value containing a JSON array into a typed {@code List<T>}. Used for
|
||||
* endpoints that accept structured list parameters via {@code @ModelAttribute} — the form field
|
||||
* carries the full JSON array as its value and the editor parses it once.
|
||||
*/
|
||||
@Slf4j
|
||||
public class JsonListPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER =
|
||||
JsonMapper.builder()
|
||||
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.build();
|
||||
|
||||
private final TypeReference<? extends List<T>> typeRef;
|
||||
|
||||
public JsonListPropertyEditor(TypeReference<? extends List<T>> typeRef) {
|
||||
this.typeRef = typeRef;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(new ArrayList<T>());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
setValue(OBJECT_MAPPER.readValue(text, typeRef));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to parse JSON list value", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Expected a JSON array but could not parse: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Binds a multipart form value containing a JSON object into a typed {@code T}. Companion to {@link
|
||||
* JsonListPropertyEditor} for single-object nested fields on {@code @ModelAttribute} endpoints.
|
||||
*/
|
||||
@Slf4j
|
||||
public class JsonObjectPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER =
|
||||
JsonMapper.builder().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).build();
|
||||
|
||||
private final Class<T> type;
|
||||
|
||||
public JsonObjectPropertyEditor(Class<T> type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(null);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
setValue(OBJECT_MAPPER.readValue(text, type));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to parse JSON object value", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Expected a JSON object but could not parse: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
-53
@@ -1,53 +0,0 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.JavaType;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Spring property editor that decodes a JSON string into a typed {@link ArrayList}. Used to bind
|
||||
* complex list parameters (e.g. {@code List<RedactionArea>}, {@code List<EditTextOperation>}) from
|
||||
* multipart form fields, where Spring's default binding cannot deserialize a JSON array.
|
||||
*/
|
||||
@Slf4j
|
||||
public class StringToArrayListPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private final ObjectMapper objectMapper =
|
||||
JsonMapper.builder()
|
||||
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.build();
|
||||
|
||||
private final Class<T> elementType;
|
||||
|
||||
public StringToArrayListPropertyEditor(Class<T> elementType) {
|
||||
this.elementType = elementType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(new ArrayList<>());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
JavaType listType =
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructCollectionType(ArrayList.class, elementType);
|
||||
List<T> list = objectMapper.readValue(text, listType);
|
||||
setValue(list);
|
||||
} catch (Exception e) {
|
||||
log.error("Exception while converting {}", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Failed to convert java.lang.String to java.util.List");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Unit tests for {@link PageColumnLayout} gutter detection and column classification. */
|
||||
class PageColumnLayoutTest {
|
||||
|
||||
private static final float PAGE_WIDTH = 612f; // Letter portrait
|
||||
|
||||
// ── single-column ────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void singleColumn_oneColumnNoGutters() {
|
||||
List<float[]> lines = List.of(lineBox(72f, 396f));
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
assertThat(layout.gutters()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void singleColumn_classifyAnchor_returnsZero() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(List.of(lineBox(72f, 396f)), PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnOf(100f, 200f)).isEqualTo(0);
|
||||
}
|
||||
|
||||
// ── two-column ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void twoColumn_detectsGutter() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(2);
|
||||
assertThat(layout.gutters()).hasSize(1);
|
||||
float[] gutter = layout.gutters().get(0);
|
||||
// Gutter is centered on pageWidth/2 with PageColumnLayout.MIDPOINT_SLACK_PT slack each
|
||||
// side.
|
||||
float pageMid = PAGE_WIDTH / 2f;
|
||||
assertThat(gutter[0]).isBetween(pageMid - 40f, pageMid - 20f);
|
||||
assertThat(gutter[1]).isBetween(pageMid + 20f, pageMid + 40f);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_classifyLeftAndRightAnchors() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
assertThat(layout.columnOf(100f, 200f)).isEqualTo(0);
|
||||
assertThat(layout.columnOf(380f, 460f)).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_columnsCrossing_leftLineOnlyHitsLeft() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
assertThat(layout.columnsCrossing(72f, 280f)).containsExactly(0);
|
||||
assertThat(layout.columnsCrossing(320f, 540f)).containsExactly(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_spanningLine_returnsBothColumns() {
|
||||
List<float[]> lines = new ArrayList<>(buildTwoColumnLines(3));
|
||||
// Full-width header that crosses pageWidth/2.
|
||||
lines.add(lineBox(72f, 396f));
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnsCrossing(72f, 540f)).containsExactly(0, 1);
|
||||
assertThat(layout.columnsCrossing(72f, 280f)).containsExactly(0);
|
||||
}
|
||||
|
||||
private static List<float[]> buildTwoColumnLines(int rowsPerColumn) {
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
for (int i = 0; i < rowsPerColumn; i++) {
|
||||
lines.add(lineBox(72f, 136f)); // left column body (72..208)
|
||||
lines.add(lineBox(320f, 220f)); // right column body (320..540)
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// ── three-column ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void threeColumn_collapsesToLeftRightSplit() {
|
||||
// The midpoint-based detector splits the page at pageWidth/2 and treats anything else as
|
||||
// single-column or spanning. A genuine 3-column layout collapses to 2 columns; the middle
|
||||
// column's content ends up classified by midpoint as left or right of pageMid.
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
for (int i = 0; i < 6; i++) {
|
||||
lines.add(lineBox(72f, 150f)); // 72..222
|
||||
lines.add(lineBox(252f, 150f)); // 252..402
|
||||
lines.add(lineBox(432f, 150f)); // 432..582
|
||||
}
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(2);
|
||||
assertThat(layout.gutters()).hasSize(1);
|
||||
}
|
||||
|
||||
// ── empty page ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void emptyPage_singleColumnFallback() {
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(List.of(), PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
assertThat(layout.gutters()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void onlyShortFragments_singleColumnFallback() {
|
||||
// Page numbers / decorations — too narrow to vote either side.
|
||||
List<float[]> lines = List.of(lineBox(300f, 6f));
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
}
|
||||
|
||||
// ── narrow gap should not be confused for a gutter ───────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void narrowInternalGap_doesNotProduceGutter() {
|
||||
// Both halves sit left of the page midpoint, so no line votes for a right column and
|
||||
// detection falls back to single-column.
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
lines.add(lineBox(72f, 100f));
|
||||
lines.add(lineBox(180f, 100f));
|
||||
for (int i = 0; i < 5; i++) {
|
||||
lines.add(lineBox(72f, 208f));
|
||||
}
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Builds a line bounding box {@code [x1, 0, x1+width, 0]}; Y is unused by detection. */
|
||||
private static float[] lineBox(float x1, float width) {
|
||||
return new float[] {x1, 0f, x1 + width, 0f};
|
||||
}
|
||||
}
|
||||
-163
@@ -1,163 +0,0 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
|
||||
class StringToArrayListPropertyEditorTest {
|
||||
|
||||
private StringToArrayListPropertyEditor<RedactionArea> editor;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
editor = new StringToArrayListPropertyEditor<>(RedactionArea.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_ValidJson() {
|
||||
// Arrange
|
||||
String json =
|
||||
"[{\"x\":10.5,\"y\":20.5,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"}]";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(1, list.size(), "List should have 1 entry");
|
||||
|
||||
RedactionArea area = list.get(0);
|
||||
assertEquals(10.5, area.getX(), "X should be 10.5");
|
||||
assertEquals(20.5, area.getY(), "Y should be 20.5");
|
||||
assertEquals(100.0, area.getWidth(), "Width should be 100.0");
|
||||
assertEquals(50.0, area.getHeight(), "Height should be 50.0");
|
||||
assertEquals(1, area.getPage(), "Page should be 1");
|
||||
assertEquals("#FF0000", area.getColor(), "Color should be #FF0000");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_MultipleItems() {
|
||||
// Arrange
|
||||
String json =
|
||||
"["
|
||||
+ "{\"x\":10.0,\"y\":20.0,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"},"
|
||||
+ "{\"x\":30.0,\"y\":40.0,\"width\":200.0,\"height\":150.0,\"page\":2,\"color\":\"#00FF00\"}"
|
||||
+ "]";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(2, list.size(), "List should have 2 entries");
|
||||
|
||||
RedactionArea area1 = list.get(0);
|
||||
assertEquals(10.0, area1.getX(), "X should be 10.0");
|
||||
assertEquals(20.0, area1.getY(), "Y should be 20.0");
|
||||
assertEquals(1, area1.getPage(), "Page should be 1");
|
||||
|
||||
RedactionArea area2 = list.get(1);
|
||||
assertEquals(30.0, area2.getX(), "X should be 30.0");
|
||||
assertEquals(40.0, area2.getY(), "Y should be 40.0");
|
||||
assertEquals(2, area2.getPage(), "Page should be 2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_EmptyString() {
|
||||
// Arrange
|
||||
String json = "";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertTrue(list.isEmpty(), "List should be empty");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_NullString() {
|
||||
// Act
|
||||
editor.setAsText(null);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertTrue(list.isEmpty(), "List should be empty");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_SingleItemAsArray() {
|
||||
// Arrange - note this is a single object, not an array
|
||||
String json =
|
||||
"{\"x\":10.0,\"y\":20.0,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"}";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(1, list.size(), "List should have 1 entry");
|
||||
|
||||
RedactionArea area = list.get(0);
|
||||
assertEquals(10.0, area.getX(), "X should be 10.0");
|
||||
assertEquals(20.0, area.getY(), "Y should be 20.0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_InvalidJson() {
|
||||
// Arrange
|
||||
String json = "invalid json";
|
||||
|
||||
// Act & Assert
|
||||
assertThrows(IllegalArgumentException.class, () -> editor.setAsText(json));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_UnknownProperties() {
|
||||
// Arrange - this JSON contains properties not in RedactionArea
|
||||
// With FAIL_ON_UNKNOWN_PROPERTIES disabled, this should ignore the unknown properties
|
||||
String json = "[{\"invalid\":\"structure\"}]";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(1, list.size(), "List should have 1 entry (empty object)");
|
||||
}
|
||||
}
|
||||
+4
-2
@@ -39,7 +39,9 @@ import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.common.util.propertyeditor.StringToArrayListPropertyEditor;
|
||||
import stirling.software.common.util.propertyeditor.JsonListPropertyEditor;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
|
||||
/**
|
||||
* Find/replace text editing for PDFs. Round-trips through {@link PdfJsonConversionService}: the
|
||||
@@ -73,7 +75,7 @@ public class EditTextController {
|
||||
binder.registerCustomEditor(
|
||||
List.class,
|
||||
"edits",
|
||||
new StringToArrayListPropertyEditor<>(EditTextOperation.class));
|
||||
new JsonListPropertyEditor<>(new TypeReference<List<EditTextOperation>>() {}));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* PDFTextStripper subclass that collects all text positions and groups them into line-level
|
||||
* bounding boxes.
|
||||
*
|
||||
* <p>Two outputs are maintained in parallel:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@link #getLineBoxes()} returns {@code [x1, pdfYbottom, x2, pdfYtop]} in PDF user-space
|
||||
* (origin bottom-left, Y up). This is what existing callers expect.
|
||||
* <li>{@link #getScreenLineBoxes()} returns {@code [x1, screenYtop, x2, screenYbottom]} computed
|
||||
* directly from glyph positions without a PDF↔screen round-trip — used by column-aware
|
||||
* redaction where ulp-level drift in the round-trip caused false rejects against anchors.
|
||||
* </ul>
|
||||
*
|
||||
* <p>Lines are flushed not only on Y jumps but also on large X gaps within the same Y row. That way
|
||||
* left-column glyphs and right-column glyphs that happen to share a baseline (common in IEEE
|
||||
* conference templates) get emitted as two distinct line boxes instead of one wide merged box.
|
||||
*/
|
||||
final class AllTextLineExtractor extends PDFTextStripper {
|
||||
|
||||
/** Min vertical jump (screen Y) before the next glyph is treated as a new line. */
|
||||
private static final float LINE_Y_TOLERANCE = 3.0f;
|
||||
|
||||
/**
|
||||
* Min horizontal gap (screen X) between consecutive glyphs on the same Y that indicates a
|
||||
* column boundary. Chosen large enough to not split normal inter-word spacing (~6–10pt for 11pt
|
||||
* text) but small enough to catch standard column gutters (typically ≥15pt).
|
||||
*/
|
||||
private static final float COLUMN_GAP_X = 14f;
|
||||
|
||||
private final float pageHeight;
|
||||
private final List<float[]> lineBoxes = new ArrayList<>();
|
||||
private final List<float[]> screenLineBoxes = new ArrayList<>();
|
||||
|
||||
private final List<TextPosition> currentLine = new ArrayList<>();
|
||||
private float lastScreenY = Float.NaN;
|
||||
private float lastGlyphRight = Float.NaN;
|
||||
|
||||
AllTextLineExtractor(int pageNumber, float pageHeight) throws IOException {
|
||||
this.pageHeight = pageHeight;
|
||||
setStartPage(pageNumber);
|
||||
setEndPage(pageNumber);
|
||||
setSortByPosition(true);
|
||||
}
|
||||
|
||||
List<float[]> getLineBoxes() {
|
||||
return lineBoxes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns line boxes as {@code [x1, screenYtop, x2, screenYbottom]}. {@code screenYtop} is the
|
||||
* minimum {@code TextPosition.getY() - getHeight()} on the line and {@code screenYbottom} is
|
||||
* the maximum {@code getY()} (the line's baseline). These values come straight from PDFBox
|
||||
* without going through {@code pageHeight - …}, so they're stable for ulp-sensitive comparisons
|
||||
* against anchor screen Ys.
|
||||
*/
|
||||
List<float[]> getScreenLineBoxes() {
|
||||
return screenLineBoxes;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> positions) {
|
||||
for (TextPosition tp : positions) {
|
||||
// Skip whitespace-only positions (spaces, newline markers, indent characters).
|
||||
// These have a TextPosition but no visible glyph; including them causes
|
||||
// space-only "lines" to produce degenerate segments that appear as thin
|
||||
// black bars after redaction.
|
||||
String unicode = tp.getUnicode();
|
||||
if (unicode == null || unicode.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
float screenY = tp.getY();
|
||||
float screenX = tp.getX();
|
||||
boolean yJump =
|
||||
!Float.isNaN(lastScreenY) && Math.abs(screenY - lastScreenY) > LINE_Y_TOLERANCE;
|
||||
boolean xJump =
|
||||
!Float.isNaN(lastGlyphRight) && (screenX - lastGlyphRight) > COLUMN_GAP_X;
|
||||
if (yJump || xJump) {
|
||||
flushLine();
|
||||
}
|
||||
lastScreenY = screenY;
|
||||
lastGlyphRight = screenX + tp.getWidth();
|
||||
currentLine.add(tp);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void endPage(PDPage page) throws IOException {
|
||||
flushLine();
|
||||
super.endPage(page);
|
||||
}
|
||||
|
||||
private void flushLine() {
|
||||
if (currentLine.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
float minX = Float.MAX_VALUE, maxX = -Float.MAX_VALUE;
|
||||
float minScreenY = Float.MAX_VALUE, maxScreenY = -Float.MAX_VALUE;
|
||||
for (TextPosition tp : currentLine) {
|
||||
minX = Math.min(minX, tp.getX());
|
||||
maxX = Math.max(maxX, tp.getX() + tp.getWidth());
|
||||
minScreenY = Math.min(minScreenY, tp.getY() - tp.getHeight());
|
||||
maxScreenY = Math.max(maxScreenY, tp.getY());
|
||||
}
|
||||
emitSegment(minX, maxX, minScreenY, maxScreenY);
|
||||
currentLine.clear();
|
||||
lastScreenY = Float.NaN;
|
||||
lastGlyphRight = Float.NaN;
|
||||
}
|
||||
|
||||
private void emitSegment(float minX, float maxX, float minScreenY, float maxScreenY) {
|
||||
float pdfY1 = pageHeight - maxScreenY; // bottom in PDF coords
|
||||
float pdfY2 = pageHeight - minScreenY; // top in PDF coords
|
||||
lineBoxes.add(new float[] {minX, pdfY1, maxX, pdfY2});
|
||||
screenLineBoxes.add(new float[] {minX, minScreenY, maxX, maxScreenY});
|
||||
}
|
||||
}
|
||||
+403
@@ -0,0 +1,403 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDPageTree;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.model.api.security.ManualRedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
class ManualRedactionService {
|
||||
|
||||
private static final float DEFAULT_TEXT_PADDING_MULTIPLIER = 0.6f;
|
||||
private static final float REDACTION_WIDTH_REDUCTION_FACTOR = 0.9f;
|
||||
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Area and page redaction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactAreas(List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
if (redactionAreas == null || redactionAreas.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<Integer, List<RedactionArea>> redactionsByPage = new HashMap<>();
|
||||
|
||||
for (RedactionArea redactionArea : redactionAreas) {
|
||||
if (redactionArea.getPage() == null
|
||||
|| redactionArea.getPage() <= 0
|
||||
|| redactionArea.getHeight() == null
|
||||
|| redactionArea.getHeight() <= 0.0D
|
||||
|| redactionArea.getWidth() == null
|
||||
|| redactionArea.getWidth() <= 0.0D) {
|
||||
continue;
|
||||
}
|
||||
|
||||
redactionsByPage
|
||||
.computeIfAbsent(redactionArea.getPage(), k -> new ArrayList<>())
|
||||
.add(redactionArea);
|
||||
}
|
||||
|
||||
for (Map.Entry<Integer, List<RedactionArea>> entry : redactionsByPage.entrySet()) {
|
||||
Integer pageNumber = entry.getKey();
|
||||
List<RedactionArea> areasForPage = entry.getValue();
|
||||
|
||||
if (pageNumber > allPages.getCount()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
PDPage page = allPages.get(pageNumber - 1);
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
|
||||
contentStream.saveGraphicsState();
|
||||
for (RedactionArea redactionArea : areasForPage) {
|
||||
Color redactColor = decodeOrDefault(redactionArea.getColor());
|
||||
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
float x = redactionArea.getX().floatValue();
|
||||
float y = redactionArea.getY().floatValue();
|
||||
float width = redactionArea.getWidth().floatValue();
|
||||
float height = redactionArea.getHeight().floatValue();
|
||||
|
||||
float pdfY = page.getBBox().getHeight() - y - height;
|
||||
|
||||
contentStream.addRect(x, pdfY, width, height);
|
||||
contentStream.fill();
|
||||
}
|
||||
contentStream.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void redactPages(ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
Color redactColor = decodeOrDefault(request.getPageRedactionColor());
|
||||
List<Integer> pageNumbers = getPageNumbers(request, allPages.getCount());
|
||||
|
||||
for (Integer pageNumber : pageNumbers) {
|
||||
PDPage page = allPages.get(pageNumber);
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
PDRectangle box = page.getBBox();
|
||||
contentStream.addRect(0, 0, box.getWidth(), box.getHeight());
|
||||
contentStream.fill();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Overlay drawing
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactFoundText(
|
||||
PDDocument document,
|
||||
List<PDFText> blocks,
|
||||
float customPadding,
|
||||
Color redactColor,
|
||||
boolean isTextRemovalMode)
|
||||
throws IOException {
|
||||
|
||||
var allPages = document.getDocumentCatalog().getPages();
|
||||
|
||||
Map<Integer, List<PDFText>> blocksByPage = new HashMap<>();
|
||||
for (PDFText block : blocks) {
|
||||
blocksByPage.computeIfAbsent(block.getPageIndex(), k -> new ArrayList<>()).add(block);
|
||||
}
|
||||
|
||||
for (Map.Entry<Integer, List<PDFText>> entry : blocksByPage.entrySet()) {
|
||||
Integer pageIndex = entry.getKey();
|
||||
List<PDFText> pageBlocks = entry.getValue();
|
||||
|
||||
if (pageIndex >= allPages.getCount()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
var page = allPages.get(pageIndex);
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
|
||||
contentStream.saveGraphicsState();
|
||||
|
||||
try {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
PDRectangle pageBox = page.getBBox();
|
||||
|
||||
for (PDFText block : pageBlocks) {
|
||||
float padding =
|
||||
(block.getY2() - block.getY1()) * DEFAULT_TEXT_PADDING_MULTIPLIER
|
||||
+ customPadding;
|
||||
|
||||
float originalWidth = block.getX2() - block.getX1();
|
||||
float boxWidth;
|
||||
float boxX;
|
||||
|
||||
if (isTextRemovalMode) {
|
||||
boxWidth = originalWidth * REDACTION_WIDTH_REDUCTION_FACTOR;
|
||||
float widthReduction = originalWidth - boxWidth;
|
||||
boxX = block.getX1() + (widthReduction / 2);
|
||||
} else {
|
||||
boxWidth = originalWidth;
|
||||
boxX = block.getX1();
|
||||
}
|
||||
|
||||
contentStream.addRect(
|
||||
boxX,
|
||||
pageBox.getHeight() - block.getY2() - padding,
|
||||
boxWidth,
|
||||
block.getY2() - block.getY1() + 2 * padding);
|
||||
}
|
||||
|
||||
contentStream.fill();
|
||||
|
||||
} finally {
|
||||
contentStream.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
|
||||
// Remove annotations whose bounding rect overlaps a redacted block, to prevent
|
||||
// users from hovering over redacted URLs and seeing the underlying destination.
|
||||
try {
|
||||
float pageH = page.getBBox().getHeight();
|
||||
List<PDAnnotation> kept = new ArrayList<>();
|
||||
for (PDAnnotation ann : page.getAnnotations()) {
|
||||
PDRectangle ar = ann.getRectangle();
|
||||
boolean overlaps = false;
|
||||
if (ar != null) {
|
||||
for (PDFText block : pageBlocks) {
|
||||
float padding =
|
||||
(block.getY2() - block.getY1())
|
||||
* DEFAULT_TEXT_PADDING_MULTIPLIER
|
||||
+ customPadding;
|
||||
float bx1 = block.getX1();
|
||||
float bx2 = block.getX2();
|
||||
float by1 = pageH - block.getY2() - padding;
|
||||
float by2 = pageH - block.getY1() + padding;
|
||||
if (ar.getLowerLeftX() < bx2
|
||||
&& ar.getUpperRightX() > bx1
|
||||
&& ar.getLowerLeftY() < by2
|
||||
&& ar.getUpperRightY() > by1) {
|
||||
overlaps = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!overlaps) {
|
||||
kept.add(ann);
|
||||
}
|
||||
}
|
||||
page.setAnnotations(kept);
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"[redact] could not remove annotations on page {}: {}",
|
||||
pageIndex,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void redactImageBoxes(PDDocument document, List<float[]> imageBoxes, Color color)
|
||||
throws IOException {
|
||||
Map<Integer, List<float[]>> byPage = new HashMap<>();
|
||||
for (float[] box : imageBoxes) {
|
||||
byPage.computeIfAbsent((int) box[0], k -> new ArrayList<>()).add(box);
|
||||
}
|
||||
PDPageTree pages = document.getDocumentCatalog().getPages();
|
||||
for (Map.Entry<Integer, List<float[]>> entry : byPage.entrySet()) {
|
||||
int pageIdx = entry.getKey();
|
||||
if (pageIdx < 0 || pageIdx >= pages.getCount()) {
|
||||
log.warn("[redact/execute] image box references out-of-range page {}", pageIdx);
|
||||
continue;
|
||||
}
|
||||
PDPage page = pages.get(pageIdx);
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.setNonStrokingColor(color);
|
||||
for (float[] box : entry.getValue()) {
|
||||
float x1 = box[1], y1 = box[2], x2 = box[3], y2 = box[4];
|
||||
cs.addRect(x1, y1, x2 - x1, y2 - y1);
|
||||
}
|
||||
cs.fill();
|
||||
cs.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Page element extraction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Returns bounding boxes for every text line and image on {@code page} in PDF user-space
|
||||
* coordinates: {@code [x1, y1, x2, y2]} (origin bottom-left, Y increases upward).
|
||||
*/
|
||||
List<float[]> extractPageElementBoxes(PDDocument document, PDPage page, int pageIndex)
|
||||
throws IOException {
|
||||
List<float[]> boxes = new ArrayList<>();
|
||||
|
||||
AllTextLineExtractor textExtractor =
|
||||
new AllTextLineExtractor(pageIndex + 1, page.getBBox().getHeight());
|
||||
textExtractor.getText(document);
|
||||
boxes.addAll(textExtractor.getLineBoxes());
|
||||
|
||||
PageImageLocator imgLocator = new PageImageLocator(page, pageIndex);
|
||||
imgLocator.processPage(page);
|
||||
for (PageImageLocator.ImageBox imgBox : imgLocator.getImageBoxes()) {
|
||||
boxes.add(new float[] {imgBox.x1(), imgBox.y1(), imgBox.x2(), imgBox.y2()});
|
||||
}
|
||||
|
||||
return boxes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Finalization
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
TempFile finalizeRedaction(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDFText>> allFoundTextsByPage,
|
||||
String colorString,
|
||||
float customPadding,
|
||||
Boolean convertToImage,
|
||||
boolean isTextRemovalMode)
|
||||
throws IOException {
|
||||
|
||||
List<PDFText> allFoundTexts = new ArrayList<>();
|
||||
for (List<PDFText> pageTexts : allFoundTextsByPage.values()) {
|
||||
allFoundTexts.addAll(pageTexts);
|
||||
}
|
||||
|
||||
if (!allFoundTexts.isEmpty()) {
|
||||
Color redactColor = decodeOrDefault(colorString);
|
||||
redactFoundText(document, allFoundTexts, customPadding, redactColor, isTextRemovalMode);
|
||||
cleanDocumentMetadata(document);
|
||||
}
|
||||
|
||||
if (Boolean.TRUE.equals(convertToImage)) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
cleanDocumentMetadata(convertedPdf);
|
||||
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
convertedPdf.save(tempOut.getFile());
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized (image mode): {} pages ➜ {} KB",
|
||||
convertedPdf.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
return tempOut;
|
||||
}
|
||||
}
|
||||
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
document.save(tempOut.getFile());
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized: {} pages ➜ {} KB",
|
||||
document.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
return tempOut;
|
||||
}
|
||||
|
||||
private void cleanDocumentMetadata(PDDocument document) {
|
||||
try {
|
||||
var documentInfo = document.getDocumentInformation();
|
||||
if (documentInfo != null) {
|
||||
documentInfo.setAuthor(null);
|
||||
documentInfo.setSubject(null);
|
||||
documentInfo.setKeywords(null);
|
||||
documentInfo.setModificationDate(java.util.Calendar.getInstance());
|
||||
log.debug("Cleaned document metadata for security");
|
||||
}
|
||||
|
||||
if (document.getDocumentCatalog() != null) {
|
||||
try {
|
||||
document.getDocumentCatalog().setMetadata(null);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not clear XMP metadata: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to clean document metadata: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Utilities
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
static Color decodeOrDefault(String hex) {
|
||||
if (hex == null) {
|
||||
return Color.BLACK;
|
||||
}
|
||||
|
||||
String colorString = hex.startsWith("#") ? hex : "#" + hex;
|
||||
|
||||
try {
|
||||
return Color.decode(colorString);
|
||||
} catch (NumberFormatException e) {
|
||||
return Color.BLACK;
|
||||
}
|
||||
}
|
||||
|
||||
private List<Integer> getPageNumbers(ManualRedactPdfRequest request, int pagesCount) {
|
||||
String pageNumbersInput = request.getPageNumbers();
|
||||
String[] parsedPageNumbers =
|
||||
pageNumbersInput != null ? pageNumbersInput.split(",") : new String[0];
|
||||
List<Integer> pageNumbers =
|
||||
GeneralUtils.parsePageList(parsedPageNumbers, pagesCount, false);
|
||||
Collections.sort(pageNumbers);
|
||||
return pageNumbers;
|
||||
}
|
||||
}
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
|
||||
/**
|
||||
* Scans a PDF document once and matches all provided patterns in a single pass, collecting
|
||||
* bounding-box positions for every match. Use in place of creating one {@code TextFinder} per
|
||||
* search term to avoid O(n) full-document scans.
|
||||
*/
|
||||
@Slf4j
|
||||
final class MultiPatternTextFinder extends PDFTextStripper {
|
||||
|
||||
private static final long REGEX_MATCH_TIMEOUT_SECONDS = 30;
|
||||
private static final ExecutorService REGEX_EXECUTOR =
|
||||
Executors.newVirtualThreadPerTaskExecutor();
|
||||
|
||||
private final List<Pattern> patterns;
|
||||
private final Map<Integer, List<PDFText>> foundTextsByPage = new HashMap<>();
|
||||
|
||||
private final List<TextPosition> pageTextPositions = new ArrayList<>();
|
||||
private final StringBuilder pageTextBuilder = new StringBuilder();
|
||||
|
||||
MultiPatternTextFinder(List<Pattern> patterns) throws IOException {
|
||||
this.patterns = patterns;
|
||||
this.setWordSeparator(" ");
|
||||
this.setLineSeparator("\n");
|
||||
}
|
||||
|
||||
Map<Integer, List<PDFText>> getFoundTextsByPage() {
|
||||
return foundTextsByPage;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startPage(PDPage page) throws IOException {
|
||||
super.startPage(page);
|
||||
pageTextPositions.clear();
|
||||
pageTextBuilder.setLength(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> textPositions) {
|
||||
pageTextBuilder.append(text);
|
||||
pageTextPositions.addAll(textPositions);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeWordSeparator() {
|
||||
pageTextBuilder.append(getWordSeparator());
|
||||
pageTextPositions.add(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeLineSeparator() {
|
||||
pageTextBuilder.append(getLineSeparator());
|
||||
pageTextPositions.add(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void endPage(PDPage page) throws IOException {
|
||||
String text = pageTextBuilder.toString();
|
||||
if (!text.isEmpty()) {
|
||||
int pageIndex = getCurrentPageNo() - 1;
|
||||
for (Pattern pattern : patterns) {
|
||||
Matcher matcher = pattern.matcher(text);
|
||||
while (safeFind(matcher)) {
|
||||
PDFText pdfText = resolveMatchPosition(matcher, pageIndex);
|
||||
if (pdfText != null) {
|
||||
foundTextsByPage
|
||||
.computeIfAbsent(pageIndex, k -> new ArrayList<>())
|
||||
.add(pdfText);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
super.endPage(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a single {@code matcher.find()} call with a {@value #REGEX_MATCH_TIMEOUT_SECONDS}
|
||||
* second timeout. Prevents pathological regex backtracking from blocking the request
|
||||
* indefinitely; per-match timeout so fast legitimate scans are unaffected.
|
||||
*/
|
||||
private static boolean safeFind(Matcher matcher) throws IOException {
|
||||
Future<Boolean> future =
|
||||
REGEX_EXECUTOR.submit((java.util.concurrent.Callable<Boolean>) matcher::find);
|
||||
try {
|
||||
return future.get(REGEX_MATCH_TIMEOUT_SECONDS, TimeUnit.SECONDS);
|
||||
} catch (TimeoutException e) {
|
||||
future.cancel(true);
|
||||
throw new IOException(
|
||||
"Regex match timed out after "
|
||||
+ REGEX_MATCH_TIMEOUT_SECONDS
|
||||
+ "s — pattern may cause catastrophic backtracking");
|
||||
} catch (InterruptedException e) {
|
||||
future.cancel(true);
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("Regex match interrupted", e);
|
||||
} catch (ExecutionException e) {
|
||||
Throwable cause = e.getCause();
|
||||
if (cause instanceof IOException ioEx) throw ioEx;
|
||||
throw new IOException("Regex match failed: " + cause.getMessage(), cause);
|
||||
}
|
||||
}
|
||||
|
||||
private PDFText resolveMatchPosition(Matcher matcher, int pageIndex) {
|
||||
int matchStart = matcher.start();
|
||||
int matchEnd = matcher.end();
|
||||
|
||||
float minX = Float.MAX_VALUE;
|
||||
float minY = Float.MAX_VALUE;
|
||||
float maxX = Float.MIN_VALUE;
|
||||
float maxY = Float.MIN_VALUE;
|
||||
boolean foundPosition = false;
|
||||
|
||||
for (int i = matchStart; i < matchEnd; i++) {
|
||||
if (i >= pageTextPositions.size()) break;
|
||||
TextPosition pos = pageTextPositions.get(i);
|
||||
if (pos != null) {
|
||||
foundPosition = true;
|
||||
minX = Math.min(minX, pos.getX());
|
||||
maxX = Math.max(maxX, pos.getX() + pos.getWidth());
|
||||
minY = Math.min(minY, pos.getY() - pos.getHeight());
|
||||
maxY = Math.max(maxY, pos.getY());
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundPosition && matchStart < pageTextPositions.size()) {
|
||||
for (int i = Math.max(0, matchStart - 5);
|
||||
i < Math.min(pageTextPositions.size(), matchEnd + 5);
|
||||
i++) {
|
||||
TextPosition pos = pageTextPositions.get(i);
|
||||
if (pos != null) {
|
||||
foundPosition = true;
|
||||
minX = Math.min(minX, pos.getX());
|
||||
maxX = Math.max(maxX, pos.getX() + pos.getWidth());
|
||||
minY = Math.min(minY, pos.getY() - pos.getHeight());
|
||||
maxY = Math.max(maxY, pos.getY());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundPosition) {
|
||||
log.warn(
|
||||
"Found text match '{}' but no valid position data at {}-{}",
|
||||
matcher.group(),
|
||||
matchStart,
|
||||
matchEnd);
|
||||
return null;
|
||||
}
|
||||
|
||||
return new PDFText(pageIndex, minX, minY, maxX, maxY, matcher.group());
|
||||
}
|
||||
}
|
||||
+67
-1477
File diff suppressed because it is too large
Load Diff
+850
@@ -0,0 +1,850 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDPageTree;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.ImageBox;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyle;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
|
||||
import stirling.software.SPDF.pdf.parser.PageColumnLayout;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
class RedactExecuteService {
|
||||
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final ManualRedactionService manualRedactionService;
|
||||
private final TextRedactionService textRedactionService;
|
||||
|
||||
TempFile execute(RedactExecuteRequest request) throws IOException {
|
||||
RedactStyle style = request.getStyle() != null ? request.getStyle() : new RedactStyle();
|
||||
List<String> textValues = orEmpty(request.getTextValues());
|
||||
List<String> regexPatterns = orEmpty(request.getRegexPatterns());
|
||||
List<Integer> wipePages = orEmpty(request.getWipePages());
|
||||
List<TextRange> ranges = orEmpty(request.getRanges());
|
||||
List<ImageBox> imageBoxes = orEmpty(request.getImageBoxes());
|
||||
|
||||
boolean hasTargets =
|
||||
!textValues.isEmpty()
|
||||
|| !regexPatterns.isEmpty()
|
||||
|| !wipePages.isEmpty()
|
||||
|| !ranges.isEmpty()
|
||||
|| !imageBoxes.isEmpty()
|
||||
|| request.getRedactImagePages() != null;
|
||||
|
||||
if (!hasTargets) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.redaction.no.targets", "No redaction targets provided");
|
||||
}
|
||||
|
||||
boolean overlayOnly =
|
||||
RedactExecuteRequest.RedactionStrategy.OVERLAY_ONLY.equals(style.getStrategy());
|
||||
boolean imageFinalize =
|
||||
RedactExecuteRequest.RedactionStrategy.IMAGE_FINALIZE.equals(style.getStrategy());
|
||||
boolean convertToImage = imageFinalize || style.isConvertToImage();
|
||||
|
||||
boolean hasTextOps = !textValues.isEmpty() || !regexPatterns.isEmpty();
|
||||
|
||||
log.info(
|
||||
"[redact/execute] strategy={} textValues={} regexPatterns={} wipePages={} ranges={} imageBoxes={} imagePages={}",
|
||||
style.getStrategy(),
|
||||
textValues.size(),
|
||||
regexPatterns.size(),
|
||||
wipePages.size(),
|
||||
ranges.size(),
|
||||
imageBoxes.size(),
|
||||
request.getRedactImagePages());
|
||||
|
||||
if (request.getFileInput() == null) {
|
||||
throw ExceptionUtils.createFileNullOrEmptyException();
|
||||
}
|
||||
|
||||
PDDocument document = null;
|
||||
try {
|
||||
document = pdfDocumentFactory.load(request.getFileInput());
|
||||
|
||||
// Single-pass text scan: collect all text-based targets so we run the PDF
|
||||
// stripper only once across the entire execute() call rather than once per target.
|
||||
Map<Integer, List<PDFText>> foundTexts =
|
||||
hasTextOps ? collectTextMatches(document, request) : new HashMap<>();
|
||||
|
||||
int totalMatches = foundTexts.values().stream().mapToInt(List::size).sum();
|
||||
log.info(
|
||||
"[redact/execute] scan complete: {} text matches across {} pages",
|
||||
totalMatches,
|
||||
foundTexts.size());
|
||||
|
||||
// Text removal (content-stream rewriting) — skipped in overlay-only mode.
|
||||
boolean needsOverlayOnly = overlayOnly;
|
||||
if (hasTextOps && !foundTexts.isEmpty() && !overlayOnly) {
|
||||
needsOverlayOnly = applyTextRemoval(document, request);
|
||||
} else if (overlayOnly) {
|
||||
log.info(
|
||||
"[redact/execute] overlay-only mode requested — skipping content-stream rewriting");
|
||||
}
|
||||
|
||||
// Reload fresh document on fallback so we overlay onto clean content.
|
||||
if (needsOverlayOnly && !foundTexts.isEmpty()) {
|
||||
log.info("[redact/execute] reloading document for clean overlay pass");
|
||||
document.close();
|
||||
document = pdfDocumentFactory.load(request.getFileInput());
|
||||
foundTexts.clear();
|
||||
if (hasTextOps) {
|
||||
foundTexts.putAll(collectTextMatches(document, request));
|
||||
}
|
||||
}
|
||||
|
||||
// Non-text operations.
|
||||
Map<Integer, PageColumnLayout> layoutCache = new HashMap<>();
|
||||
|
||||
if (!wipePages.isEmpty()) {
|
||||
applyPageWipe(document, wipePages, style);
|
||||
}
|
||||
|
||||
for (TextRange range : ranges) {
|
||||
applyRangeRedaction(document, range, style, layoutCache);
|
||||
}
|
||||
|
||||
for (ImageBox box : imageBoxes) {
|
||||
applyImageBoxRedaction(document, box, style);
|
||||
}
|
||||
|
||||
if (request.getRedactImagePages() != null) {
|
||||
applyAllImagesRedaction(document, request.getRedactImagePages(), style);
|
||||
}
|
||||
|
||||
return manualRedactionService.finalizeRedaction(
|
||||
document,
|
||||
foundTexts,
|
||||
style.getColor(),
|
||||
style.getPadding(),
|
||||
convertToImage,
|
||||
!needsOverlayOnly);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Execute redaction failed: {}", e.getMessage(), e);
|
||||
throw new RuntimeException("Failed to perform PDF redaction: " + e.getMessage(), e);
|
||||
} finally {
|
||||
if (document != null) {
|
||||
try {
|
||||
document.close();
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to close document: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Single-pass text scan (one stripper pass per execute() call)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Runs a single PDF text-stripper pass over all text-based targets and returns the merged hit
|
||||
* map.
|
||||
*/
|
||||
private Map<Integer, List<PDFText>> collectTextMatches(
|
||||
PDDocument document, RedactExecuteRequest request) {
|
||||
Map<Integer, List<PDFText>> found = new HashMap<>();
|
||||
|
||||
String[] terms = cleanStrings(request.getTextValues());
|
||||
if (terms.length > 0) {
|
||||
textRedactionService
|
||||
.findTextToRedact(document, terms, false, false)
|
||||
.forEach(
|
||||
(page, hits) ->
|
||||
found.computeIfAbsent(page, k -> new ArrayList<>())
|
||||
.addAll(hits));
|
||||
}
|
||||
|
||||
String[] patterns = cleanStrings(request.getRegexPatterns());
|
||||
if (patterns.length > 0) {
|
||||
textRedactionService
|
||||
.findTextToRedact(document, patterns, true, false)
|
||||
.forEach(
|
||||
(page, hits) ->
|
||||
found.computeIfAbsent(page, k -> new ArrayList<>())
|
||||
.addAll(hits));
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Text removal (content-stream rewriting)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Attempts content-stream text removal for all text/regex targets. Returns {@code true} if the
|
||||
* document fell back to overlay-only mode.
|
||||
*/
|
||||
private boolean applyTextRemoval(PDDocument document, RedactExecuteRequest request) {
|
||||
try {
|
||||
boolean fallback = false;
|
||||
|
||||
String[] terms = cleanStrings(request.getTextValues());
|
||||
if (terms.length > 0) {
|
||||
Map<Integer, List<PDFText>> exactFound =
|
||||
textRedactionService.findTextToRedact(document, terms, false, false);
|
||||
if (!exactFound.isEmpty()) {
|
||||
fallback |=
|
||||
textRedactionService.performTextReplacement(
|
||||
document, exactFound, terms, false, false);
|
||||
}
|
||||
}
|
||||
|
||||
String[] patterns = cleanStrings(request.getRegexPatterns());
|
||||
if (patterns.length > 0) {
|
||||
Map<Integer, List<PDFText>> regexFound =
|
||||
textRedactionService.findTextToRedact(document, patterns, true, false);
|
||||
if (!regexFound.isEmpty()) {
|
||||
fallback |=
|
||||
textRedactionService.performTextReplacement(
|
||||
document, regexFound, patterns, true, false);
|
||||
}
|
||||
}
|
||||
|
||||
if (fallback) {
|
||||
log.warn(
|
||||
"[redact/execute] font compatibility issue — falling back to overlay-only");
|
||||
} else {
|
||||
log.info("[redact/execute] content-stream text removal applied successfully");
|
||||
}
|
||||
return fallback;
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"[redact/execute] text removal failed, falling back to overlay: {}",
|
||||
e.getMessage());
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Per-operation dispatch methods
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void applyPageWipe(PDDocument document, List<Integer> pageNumbers, RedactStyle style)
|
||||
throws IOException {
|
||||
List<Integer> pageIndices = toZeroBasedIndices(pageNumbers);
|
||||
if (pageIndices.isEmpty()) return;
|
||||
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
Color pageColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
Collections.sort(pageIndices);
|
||||
log.info("[redact/execute] full-page wipe: {} pages ({})", pageIndices.size(), pageIndices);
|
||||
|
||||
Map<Integer, List<float[]>> pageElementBoxes = new HashMap<>();
|
||||
for (Integer idx : pageIndices) {
|
||||
if (idx >= 0 && idx < allPages.getCount()) {
|
||||
try {
|
||||
pageElementBoxes.put(
|
||||
idx,
|
||||
manualRedactionService.extractPageElementBoxes(
|
||||
document, allPages.get(idx), idx));
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"[redact/execute] element extraction failed for page {}: {}",
|
||||
idx,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (Integer idx : pageIndices) {
|
||||
if (idx >= 0 && idx < allPages.getCount()) {
|
||||
PDPage page = allPages.get(idx);
|
||||
List<float[]> elementBoxes =
|
||||
pageElementBoxes.getOrDefault(idx, Collections.emptyList());
|
||||
page.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
page.setResources(new PDResources());
|
||||
try (PDPageContentStream cs = new PDPageContentStream(document, page)) {
|
||||
cs.setNonStrokingColor(pageColor);
|
||||
if (elementBoxes.isEmpty()) {
|
||||
PDRectangle box = page.getBBox();
|
||||
cs.addRect(0, 0, box.getWidth(), box.getHeight());
|
||||
} else {
|
||||
log.info(
|
||||
"[redact/execute] page {}: drawing {} element boxes",
|
||||
idx + 1,
|
||||
elementBoxes.size());
|
||||
for (float[] r : elementBoxes) {
|
||||
cs.addRect(r[0], r[1], r[2] - r[0], r[3] - r[1]);
|
||||
}
|
||||
}
|
||||
cs.fill();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void applyRangeRedaction(
|
||||
PDDocument document,
|
||||
TextRange range,
|
||||
RedactStyle style,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
String rangeStart = trimOrEmpty(range.startString());
|
||||
String rangeEnd = trimOrEmpty(range.endString());
|
||||
log.info("[redact/execute] range redaction: start='{}' end='{}'", rangeStart, rangeEnd);
|
||||
try {
|
||||
List<PDFText> blocks = collectRangeBlocks(document, rangeStart, rangeEnd, layoutCache);
|
||||
if (!blocks.isEmpty()) {
|
||||
manualRedactionService.redactFoundText(
|
||||
document,
|
||||
blocks,
|
||||
style.getPadding(),
|
||||
ManualRedactionService.decodeOrDefault(style.getColor()),
|
||||
false);
|
||||
} else {
|
||||
log.warn(
|
||||
"[redact/execute] range not found: start='{}' end='{}'",
|
||||
rangeStart,
|
||||
rangeEnd);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("[redact/execute] range redaction failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void applyImageBoxRedaction(PDDocument document, ImageBox box, RedactStyle style)
|
||||
throws IOException {
|
||||
List<float[]> boxes =
|
||||
List.of(
|
||||
new float[] {
|
||||
(float) box.pageIndex(), box.x1(), box.y1(), box.x2(), box.y2()
|
||||
});
|
||||
log.info("[redact/execute] image box overlay on page {}", box.pageIndex());
|
||||
Color boxColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
manualRedactionService.redactImageBoxes(document, boxes, boxColor);
|
||||
}
|
||||
|
||||
private void applyAllImagesRedaction(
|
||||
PDDocument document, List<Integer> pageNumbers, RedactStyle style) throws IOException {
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
Color imgColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
|
||||
List<Integer> imagePageIndices = toZeroBasedIndices(pageNumbers);
|
||||
if (imagePageIndices.isEmpty()) {
|
||||
imagePageIndices = new ArrayList<>();
|
||||
for (int i = 0; i < allPages.getCount(); i++) {
|
||||
imagePageIndices.add(i);
|
||||
}
|
||||
}
|
||||
|
||||
List<float[]> detectedBoxes = new ArrayList<>();
|
||||
for (int pageIdx : imagePageIndices) {
|
||||
if (pageIdx < 0 || pageIdx >= allPages.getCount()) continue;
|
||||
try {
|
||||
PDPage page = allPages.get(pageIdx);
|
||||
PageImageLocator locator = new PageImageLocator(page, pageIdx);
|
||||
locator.processPage(page);
|
||||
for (PageImageLocator.ImageBox ib : locator.getImageBoxes()) {
|
||||
detectedBoxes.add(new float[] {pageIdx, ib.x1(), ib.y1(), ib.x2(), ib.y2()});
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"[redact/execute] image detection failed for page {}: {}",
|
||||
pageIdx + 1,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] auto image detection: {} images across {} pages",
|
||||
detectedBoxes.size(),
|
||||
imagePageIndices.size());
|
||||
|
||||
if (!detectedBoxes.isEmpty()) {
|
||||
manualRedactionService.redactImageBoxes(document, detectedBoxes, imgColor);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Range collection helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Locates {@code startStr} in the document and returns {@link PDFText} blocks for every text
|
||||
* line and image from that point up to (but NOT including) the line where {@code endStr}
|
||||
* begins. If {@code endStr} is blank, redacts from {@code startStr} to the end of the document.
|
||||
*
|
||||
* <p>Multi-column pages follow reading order: down the start column, jump to the top of the
|
||||
* next column, continue to the end anchor. Single-column pages reduce to a plain Y-band check.
|
||||
*/
|
||||
List<PDFText> collectRangeBlocks(
|
||||
PDDocument document,
|
||||
String startStr,
|
||||
String endStr,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
int totalPages = allPages.getCount();
|
||||
|
||||
Map<Integer, List<PDFText>> startMatchesByPage = findWithFallbacks(document, startStr);
|
||||
if (startMatchesByPage.isEmpty()) {
|
||||
log.warn("[redact/execute] range start not found: '{}'", startStr);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
List<Anchor> starts = toAnchors(document, startMatchesByPage, layoutCache);
|
||||
starts.sort(READING_ORDER);
|
||||
log.info(
|
||||
"[redact/execute] start='{}' matched {} anchor(s): {}",
|
||||
startStr,
|
||||
starts.size(),
|
||||
anchorSummary(starts));
|
||||
|
||||
boolean openEnded = (endStr == null || endStr.isBlank());
|
||||
List<Anchor> ends = new ArrayList<>();
|
||||
if (!openEnded) {
|
||||
Map<Integer, List<PDFText>> endMatchesByPage = findWithFallbacks(document, endStr);
|
||||
if (endMatchesByPage.isEmpty()) {
|
||||
log.warn(
|
||||
"[redact/execute] range end '{}' not found in document - skipping range"
|
||||
+ " (start='{}')",
|
||||
endStr,
|
||||
startStr);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
ends = toAnchors(document, endMatchesByPage, layoutCache);
|
||||
ends.sort(READING_ORDER);
|
||||
log.info(
|
||||
"[redact/execute] end='{}' matched {} anchor(s): {}",
|
||||
endStr,
|
||||
ends.size(),
|
||||
anchorSummary(ends));
|
||||
}
|
||||
|
||||
List<PDFText> blocks = new ArrayList<>();
|
||||
for (Anchor start : starts) {
|
||||
Anchor end = null;
|
||||
int endPage;
|
||||
if (openEnded) {
|
||||
endPage = totalPages - 1;
|
||||
} else {
|
||||
for (Anchor candidate : ends) {
|
||||
if (READING_ORDER.compare(candidate, start) > 0) {
|
||||
end = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (end == null) {
|
||||
log.warn(
|
||||
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) — skipping",
|
||||
start.page + 1,
|
||||
start.col,
|
||||
start.y);
|
||||
continue;
|
||||
}
|
||||
endPage = end.page;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] range pages {}-{}: start='{}' (col {}) end='{}'",
|
||||
start.page + 1,
|
||||
endPage + 1,
|
||||
startStr,
|
||||
start.col,
|
||||
openEnded ? "<end of document>" : endStr);
|
||||
|
||||
collectBlocksForRange(document, allPages, start, end, openEnded, blocks, layoutCache);
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] range '{}'→'{}': {} total blocks",
|
||||
startStr,
|
||||
openEnded ? "<end of document>" : endStr,
|
||||
blocks.size());
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects all redactable content (text line segments and images) between two anchor positions.
|
||||
*
|
||||
* <p>Line boxes are cached per page number in {@code lineBoxCache} and reused across range
|
||||
* iterations within one execute() call, avoiding redundant {@link AllTextLineExtractor} passes.
|
||||
*/
|
||||
private void collectBlocksForRange(
|
||||
PDDocument document,
|
||||
PDPageTree allPages,
|
||||
Anchor start,
|
||||
Anchor end,
|
||||
boolean openEnded,
|
||||
List<PDFText> blocks,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
|
||||
int startPage = start.page;
|
||||
int endPage = openEnded ? allPages.getCount() - 1 : end.page;
|
||||
int endCol =
|
||||
openEnded ? layoutFor(document, endPage, layoutCache).columnCount() - 1 : end.col;
|
||||
float startY = start.y;
|
||||
// Use bottom of end anchor so the end anchor line itself is included (inclusive range).
|
||||
float endY = openEnded ? Float.POSITIVE_INFINITY : end.text.getY2();
|
||||
|
||||
// Line-box cache: populated lazily per page, reused across range iterations.
|
||||
// Cannot use computeIfAbsent because AllTextLineExtractor's constructor throws IOException.
|
||||
Map<Integer, List<float[]>> lineBoxCache = new HashMap<>();
|
||||
|
||||
for (int pageIdx = startPage; pageIdx <= endPage; pageIdx++) {
|
||||
PDPage page = allPages.get(pageIdx);
|
||||
float pageHeight = page.getBBox().getHeight();
|
||||
PageColumnLayout layout = layoutFor(document, pageIdx, layoutCache);
|
||||
|
||||
List<float[]> screenLineBoxes = lineBoxCache.get(pageIdx);
|
||||
if (screenLineBoxes == null) {
|
||||
AllTextLineExtractor textExtractor =
|
||||
new AllTextLineExtractor(pageIdx + 1, pageHeight);
|
||||
textExtractor.getText(document);
|
||||
screenLineBoxes = textExtractor.getScreenLineBoxes();
|
||||
lineBoxCache.put(pageIdx, screenLineBoxes);
|
||||
}
|
||||
|
||||
for (float[] sb : screenLineBoxes) {
|
||||
emitColumnSlices(
|
||||
pageIdx, layout, sb[0], sb[2], sb[1], sb[3], start.col, startPage, startY,
|
||||
endCol, endPage, endY, blocks);
|
||||
}
|
||||
|
||||
PageImageLocator imgLocator = new PageImageLocator(page, pageIdx);
|
||||
imgLocator.processPage(page);
|
||||
for (PageImageLocator.ImageBox ib : imgLocator.getImageBoxes()) {
|
||||
// ImageBox coordinates are in PDF user-space (Y up); convert to screen-Y (Y down).
|
||||
float screenY1 = pageHeight - ib.y2();
|
||||
float screenY2 = pageHeight - ib.y1();
|
||||
emitColumnSlices(
|
||||
pageIdx, layout, ib.x1(), ib.x2(), screenY1, screenY2, start.col, startPage,
|
||||
startY, endCol, endPage, endY, blocks);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Emits each per-column sub-box accepted by the reading-order predicate. */
|
||||
private static void emitColumnSlices(
|
||||
int pageIdx,
|
||||
PageColumnLayout layout,
|
||||
float x1,
|
||||
float x2,
|
||||
float yTop,
|
||||
float yBottom,
|
||||
int startCol,
|
||||
int startPage,
|
||||
float startY,
|
||||
int endCol,
|
||||
int endPage,
|
||||
float endY,
|
||||
List<PDFText> blocks) {
|
||||
int[] cols = layout.columnsCrossing(x1, x2);
|
||||
if (cols.length == 1) {
|
||||
if (inColumnZone(
|
||||
pageIdx, cols[0], yTop, yBottom, startPage, startCol, startY, endPage, endCol,
|
||||
endY)) {
|
||||
blocks.add(new PDFText(pageIdx, x1, yTop, x2, yBottom, ""));
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (int col : cols) {
|
||||
if (!inColumnZone(
|
||||
pageIdx, col, yTop, yBottom, startPage, startCol, startY, endPage, endCol,
|
||||
endY)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
blocks.add(new PDFText(pageIdx, x1, yTop, x2, yBottom, ""));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reading-order predicate: true when (col, yBottom) on page {@code pageIdx} sits between the
|
||||
* start anchor (inclusive) and end anchor (inclusive).
|
||||
*/
|
||||
static boolean inColumnZone(
|
||||
int pageIdx,
|
||||
int col,
|
||||
float yTop,
|
||||
float yBottom,
|
||||
int startPage,
|
||||
int startCol,
|
||||
float startY,
|
||||
int endPage,
|
||||
int endCol,
|
||||
float endY) {
|
||||
if (pageIdx > startPage && pageIdx < endPage) return true;
|
||||
if (pageIdx == startPage && pageIdx == endPage) {
|
||||
if (startCol == endCol) {
|
||||
return col == startCol && yBottom >= startY && yBottom <= endY;
|
||||
}
|
||||
if (startCol < endCol) {
|
||||
if (col < startCol || col > endCol) return false;
|
||||
if (col == startCol) return yBottom >= startY;
|
||||
if (col == endCol) return yBottom <= endY;
|
||||
return true;
|
||||
}
|
||||
return col == startCol && yBottom >= startY;
|
||||
}
|
||||
if (pageIdx == startPage) {
|
||||
if (col == startCol) return yBottom >= startY;
|
||||
return col > startCol;
|
||||
}
|
||||
if (pageIdx == endPage) {
|
||||
if (col == endCol) return yBottom <= endY;
|
||||
return col < endCol;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Lazily builds and caches the column layout for a single page. */
|
||||
private PageColumnLayout layoutFor(
|
||||
PDDocument document, int pageIdx, Map<Integer, PageColumnLayout> cache)
|
||||
throws IOException {
|
||||
PageColumnLayout cached = cache.get(pageIdx);
|
||||
if (cached != null) return cached;
|
||||
PDPage page = document.getDocumentCatalog().getPages().get(pageIdx);
|
||||
float pageWidth = page.getBBox().getWidth();
|
||||
float pageHeight = page.getBBox().getHeight();
|
||||
AllTextLineExtractor extractor = new AllTextLineExtractor(pageIdx + 1, pageHeight);
|
||||
extractor.getText(document);
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(extractor.getLineBoxes(), pageWidth);
|
||||
if (layout.columnCount() > 1) {
|
||||
float[] g = layout.gutters().get(0);
|
||||
log.info(
|
||||
"[redact/execute] page {} layout: 2 cols, gutter x=[{}, {}]",
|
||||
pageIdx + 1,
|
||||
g[0],
|
||||
g[1]);
|
||||
} else {
|
||||
log.info("[redact/execute] page {} layout: 1 col (single-column mode)", pageIdx + 1);
|
||||
}
|
||||
cache.put(pageIdx, layout);
|
||||
return layout;
|
||||
}
|
||||
|
||||
private List<Anchor> toAnchors(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDFText>> matchesByPage,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
List<Anchor> out = new ArrayList<>();
|
||||
for (int page : matchesByPage.keySet().stream().sorted().toList()) {
|
||||
PageColumnLayout layout = layoutFor(document, page, layoutCache);
|
||||
for (PDFText hit : matchesByPage.get(page)) {
|
||||
int col = layout.columnOf(hit.getX1(), hit.getX2());
|
||||
out.add(new Anchor(page, col, hit.getY1(), hit));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Lexicographic ordering by (page, column, screenY). */
|
||||
private static final Comparator<Anchor> READING_ORDER =
|
||||
Comparator.comparingInt((Anchor a) -> a.page)
|
||||
.thenComparingInt(a -> a.col)
|
||||
.thenComparingDouble(a -> a.y);
|
||||
|
||||
private static String anchorSummary(List<Anchor> anchors) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int max = Math.min(anchors.size(), 5);
|
||||
for (int i = 0; i < max; i++) {
|
||||
Anchor a = anchors.get(i);
|
||||
if (i > 0) sb.append(", ");
|
||||
sb.append(String.format("(p=%d,c=%d,y=%.1f)", a.page + 1, a.col, a.y));
|
||||
}
|
||||
if (anchors.size() > max) sb.append(", …");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private record Anchor(int page, int col, float y, PDFText text) {}
|
||||
|
||||
/**
|
||||
* Tries progressively more permissive variants: raw (regex then literal), letter-spacing
|
||||
* collapsed, then a punctuation-tolerant regex over alphanumeric runs.
|
||||
*/
|
||||
private Map<Integer, List<PDFText>> findWithFallbacks(PDDocument document, String raw) {
|
||||
String trimmed = raw.trim();
|
||||
String collapsed = collapseLetterSpacing(trimmed);
|
||||
String tolerant = punctuationTolerantRegex(trimmed);
|
||||
|
||||
List<Candidate> candidates = new ArrayList<>();
|
||||
candidates.add(new Candidate(trimmed, true));
|
||||
candidates.add(new Candidate(trimmed, false));
|
||||
if (!collapsed.equals(trimmed)) {
|
||||
candidates.add(new Candidate(collapsed, true));
|
||||
candidates.add(new Candidate(collapsed, false));
|
||||
}
|
||||
if (tolerant != null && !tolerant.equals(trimmed)) {
|
||||
candidates.add(new Candidate(tolerant, true));
|
||||
}
|
||||
|
||||
// If the anchor spans multiple lines (model provided entire paragraph instead of a short
|
||||
// phrase), try just the first non-empty line — it's usually sufficient to locate the
|
||||
// position and avoids mismatches from mid-paragraph text extraction artifacts.
|
||||
if (trimmed.contains("\n")) {
|
||||
String firstLine =
|
||||
Arrays.stream(trimmed.split("\n"))
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
if (firstLine != null && firstLine.length() >= 4) {
|
||||
String firstLineCollapsed = collapseLetterSpacing(firstLine);
|
||||
String firstLineTolerant = punctuationTolerantRegex(firstLine);
|
||||
candidates.add(new Candidate(firstLine, false));
|
||||
if (!firstLineCollapsed.equals(firstLine)) {
|
||||
candidates.add(new Candidate(firstLineCollapsed, false));
|
||||
}
|
||||
if (firstLineTolerant != null && !firstLineTolerant.equals(firstLine)) {
|
||||
candidates.add(new Candidate(firstLineTolerant, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (Candidate c : candidates) {
|
||||
Map<Integer, List<PDFText>> m =
|
||||
textRedactionService.findTextToRedact(
|
||||
document, new String[] {c.pattern}, c.useRegex, false);
|
||||
if (!m.isEmpty()) {
|
||||
if (!c.pattern.equals(trimmed)) {
|
||||
log.info(
|
||||
"[redact/execute] range boundary matched via fallback: '{}' → '{}'",
|
||||
trimmed,
|
||||
c.pattern);
|
||||
}
|
||||
return m;
|
||||
}
|
||||
}
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
private record Candidate(String pattern, boolean useRegex) {}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Static helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Joins {@code raw}'s alphanumeric runs with {@code \W*} so anchors match across punctuation
|
||||
* drift. Returns {@code null} when fewer than two tokens exist.
|
||||
*/
|
||||
private static String punctuationTolerantRegex(String raw) {
|
||||
List<String> tokens = new ArrayList<>();
|
||||
StringBuilder current = new StringBuilder();
|
||||
for (int i = 0; i < raw.length(); i++) {
|
||||
char ch = raw.charAt(i);
|
||||
if (Character.isLetterOrDigit(ch)) {
|
||||
current.append(ch);
|
||||
} else if (current.length() > 0) {
|
||||
tokens.add(current.toString());
|
||||
current.setLength(0);
|
||||
}
|
||||
}
|
||||
if (current.length() > 0) tokens.add(current.toString());
|
||||
if (tokens.size() < 2) return null;
|
||||
StringBuilder out = new StringBuilder();
|
||||
for (int i = 0; i < tokens.size(); i++) {
|
||||
if (i > 0) out.append("\\W*");
|
||||
out.append(Pattern.quote(tokens.get(i)));
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapses letter-spaced text produced by position-sorted text extraction.
|
||||
*
|
||||
* <p>When a PDF text stripper runs with {@code setSortByPosition(true)}, letter-spaced headings
|
||||
* come out as {@code "T a b l e o f c o n t e n t s"}. This method converts the spaced form
|
||||
* back to words.
|
||||
*/
|
||||
private static String collapseLetterSpacing(String text) {
|
||||
String[] tokens = text.split(" ", -1);
|
||||
StringBuilder result = new StringBuilder();
|
||||
StringBuilder current = new StringBuilder();
|
||||
for (String token : tokens) {
|
||||
if (token.isEmpty()) {
|
||||
if (current.length() > 0) {
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(current);
|
||||
current.setLength(0);
|
||||
}
|
||||
} else if (token.length() == 1) {
|
||||
current.append(token);
|
||||
} else {
|
||||
if (current.length() > 0) {
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(current);
|
||||
current.setLength(0);
|
||||
}
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(token);
|
||||
}
|
||||
}
|
||||
if (current.length() > 0) {
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(current);
|
||||
}
|
||||
return result.toString().trim();
|
||||
}
|
||||
|
||||
private static <T> List<T> orEmpty(List<T> list) {
|
||||
return list != null ? list : List.of();
|
||||
}
|
||||
|
||||
private static String[] cleanStrings(List<String> input) {
|
||||
if (input == null || input.isEmpty()) {
|
||||
return new String[0];
|
||||
}
|
||||
return input.stream()
|
||||
.filter(s -> s != null)
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.toArray(String[]::new);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts 1-based page numbers from the request to the 0-based indices used internally.
|
||||
* Out-of-range and non-positive values are silently dropped.
|
||||
*/
|
||||
private static List<Integer> toZeroBasedIndices(List<Integer> oneBasedPageNumbers) {
|
||||
if (oneBasedPageNumbers == null || oneBasedPageNumbers.isEmpty()) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
List<Integer> result = new ArrayList<>();
|
||||
for (Integer page : oneBasedPageNumbers) {
|
||||
if (page != null && page > 0) {
|
||||
result.add(page - 1);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String trimOrEmpty(String s) {
|
||||
return s == null ? "" : s.trim();
|
||||
}
|
||||
}
|
||||
+1207
File diff suppressed because it is too large
Load Diff
+132
@@ -0,0 +1,132 @@
|
||||
package stirling.software.SPDF.model.api.security;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class RedactExecuteRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Exact strings to find and black out. One entry per phrase to redact."
|
||||
+ " Best for known names, identifiers, and specific text found in the document.")
|
||||
private List<String> textValues = new ArrayList<>();
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Regex patterns to match and redact. Each match anywhere in the document is blacked out."
|
||||
+ " Uses Java/PCRE regex syntax. Well-suited for strings that follow known patterns, like"
|
||||
+ " phone numbers, email addresses, national ID numbers, or"
|
||||
+ " dates (which can appear with different separators, optional country codes,"
|
||||
+ " etc.). For fixed known strings such as names, use textValues instead.")
|
||||
private List<String> regexPatterns = new ArrayList<>();
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"1-indexed page numbers to wipe entirely (all content removed from those pages).")
|
||||
private List<Integer> wipePages = new ArrayList<>();
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Text ranges to redact by specifying a start and end anchor phrase. All"
|
||||
+ " content between the two phrases (inclusive) is redacted. Anchors"
|
||||
+ " work best when short and unique. They must appear"
|
||||
+ " verbatim in the document.")
|
||||
private List<TextRange> ranges = new ArrayList<>();
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Rectangular areas to black out, each defined by a page number and bounding box coordinates.")
|
||||
private List<ImageBox> imageBoxes = new ArrayList<>();
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"1-indexed page numbers to redact all detected images from. Pass an empty list to redact images from every page. Omit or pass null to skip image redaction entirely.")
|
||||
private List<Integer> redactImagePages;
|
||||
|
||||
@Schema(description = "Redaction style options")
|
||||
private RedactStyle style = new RedactStyle();
|
||||
|
||||
public record TextRange(
|
||||
@Schema(
|
||||
description =
|
||||
"A short, distinctive phrase (5–15 words) that marks where"
|
||||
+ " redaction begins (inclusive). Must appear verbatim in"
|
||||
+ " the document — e.g. a section heading or a unique"
|
||||
+ " sentence fragment.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
minLength = 1)
|
||||
String startString,
|
||||
@Schema(
|
||||
description =
|
||||
"A short, distinctive phrase (5–15 words) that marks where"
|
||||
+ " redaction ends (inclusive). Must appear verbatim in the"
|
||||
+ " document. Shorter phrases match more reliably.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
minLength = 1)
|
||||
String endString) {
|
||||
public TextRange {
|
||||
if (endString == null) endString = "";
|
||||
}
|
||||
}
|
||||
|
||||
public record ImageBox(
|
||||
@Schema(
|
||||
description = "0-indexed page number (first page = 0).",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
int pageIndex,
|
||||
@Schema(
|
||||
description =
|
||||
"Left x coordinate of the redaction rectangle in PDF user-space points.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
float x1,
|
||||
@Schema(
|
||||
description =
|
||||
"Top y coordinate of the redaction rectangle in PDF user-space points.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
float y1,
|
||||
@Schema(
|
||||
description =
|
||||
"Right x coordinate of the redaction rectangle in PDF user-space points.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
float x2,
|
||||
@Schema(
|
||||
description =
|
||||
"Bottom y coordinate of the redaction rectangle in PDF user-space points.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
float y2) {}
|
||||
|
||||
public enum RedactionStrategy {
|
||||
AUTO,
|
||||
OVERLAY_ONLY,
|
||||
IMAGE_FINALIZE
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class RedactStyle {
|
||||
@Schema(description = "Hex redaction box color", defaultValue = "#000000")
|
||||
private String color = "#000000";
|
||||
|
||||
@Schema(
|
||||
description = "Extra padding around each box in points",
|
||||
type = "number",
|
||||
defaultValue = "0")
|
||||
private float padding = 0f;
|
||||
|
||||
@Schema(description = "Rasterize output to prevent text extraction", defaultValue = "false")
|
||||
private boolean convertToImage = false;
|
||||
|
||||
@Schema(
|
||||
description = "Execution strategy hint for the redaction pipeline",
|
||||
defaultValue = "AUTO")
|
||||
private RedactionStrategy strategy = RedactionStrategy.AUTO;
|
||||
}
|
||||
}
|
||||
+42
-32
@@ -37,7 +37,6 @@ import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.junit.jupiter.MockitoSettings;
|
||||
@@ -77,8 +76,11 @@ class RedactControllerTest {
|
||||
|
||||
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
@Mock private TempFileManager tempFileManager;
|
||||
@Mock private RedactExecuteService redactExecuteService;
|
||||
|
||||
@InjectMocks private RedactController redactController;
|
||||
private TextRedactionService textRedactionService;
|
||||
private ManualRedactionService manualRedactionService;
|
||||
private RedactController redactController;
|
||||
|
||||
private MockMultipartFile mockPdfFile;
|
||||
private PDDocument mockDocument;
|
||||
@@ -201,7 +203,17 @@ class RedactControllerTest {
|
||||
.save(any(File.class));
|
||||
doNothing().when(mockDocument).close();
|
||||
|
||||
// Initialize a real document for unit tests
|
||||
// Build real service instances so tests exercise actual logic
|
||||
textRedactionService = new TextRedactionService();
|
||||
manualRedactionService = new ManualRedactionService(tempFileManager);
|
||||
redactController =
|
||||
new RedactController(
|
||||
pdfDocumentFactory,
|
||||
tempFileManager,
|
||||
manualRedactionService,
|
||||
textRedactionService,
|
||||
redactExecuteService);
|
||||
|
||||
setupRealDocument();
|
||||
}
|
||||
|
||||
@@ -819,9 +831,9 @@ class RedactControllerTest {
|
||||
contentStream.newLineAtOffset(50, 750);
|
||||
|
||||
contentStream.showText("This is ");
|
||||
contentStream.newLineAtOffset(-10, 0); // Simulate positioning
|
||||
contentStream.newLineAtOffset(-10, 0);
|
||||
contentStream.showText("secret");
|
||||
contentStream.newLineAtOffset(10, 0); // Reset positioning
|
||||
contentStream.newLineAtOffset(10, 0);
|
||||
contentStream.showText(" information");
|
||||
contentStream.endText();
|
||||
}
|
||||
@@ -1005,7 +1017,7 @@ class RedactControllerTest {
|
||||
contentStream.showText("Original content");
|
||||
contentStream.endText();
|
||||
}
|
||||
return redactController.createTokensWithoutTargetText(
|
||||
return textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, pageForTokenExtraction, Collections.emptySet(), false, false);
|
||||
}
|
||||
|
||||
@@ -1016,28 +1028,28 @@ class RedactControllerTest {
|
||||
@Test
|
||||
@DisplayName("Should decode valid hex color with hash")
|
||||
void decodeValidHexColorWithHash() {
|
||||
Color result = redactController.decodeOrDefault("#FF0000");
|
||||
Color result = ManualRedactionService.decodeOrDefault("#FF0000");
|
||||
assertEquals(Color.RED, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Should decode valid hex color without hash")
|
||||
void decodeValidHexColorWithoutHash() {
|
||||
Color result = redactController.decodeOrDefault("FF0000");
|
||||
Color result = ManualRedactionService.decodeOrDefault("FF0000");
|
||||
assertEquals(Color.RED, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Should default to black for null color")
|
||||
void defaultToBlackForNullColor() {
|
||||
Color result = redactController.decodeOrDefault(null);
|
||||
Color result = ManualRedactionService.decodeOrDefault(null);
|
||||
assertEquals(Color.BLACK, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Should default to black for invalid color")
|
||||
void defaultToBlackForInvalidColor() {
|
||||
Color result = redactController.decodeOrDefault("invalid-color");
|
||||
Color result = ManualRedactionService.decodeOrDefault("invalid-color");
|
||||
assertEquals(Color.BLACK, result);
|
||||
}
|
||||
|
||||
@@ -1049,7 +1061,7 @@ class RedactControllerTest {
|
||||
})
|
||||
@DisplayName("Should handle various valid color formats")
|
||||
void handleVariousValidColorFormats(String colorInput) {
|
||||
Color result = redactController.decodeOrDefault(colorInput);
|
||||
Color result = ManualRedactionService.decodeOrDefault(colorInput);
|
||||
assertNotNull(result);
|
||||
assertTrue(
|
||||
result.getRed() >= 0 && result.getRed() <= 255,
|
||||
@@ -1065,8 +1077,8 @@ class RedactControllerTest {
|
||||
@Test
|
||||
@DisplayName("Should handle short hex codes appropriately")
|
||||
void handleShortHexCodes() {
|
||||
Color result1 = redactController.decodeOrDefault("123");
|
||||
Color result2 = redactController.decodeOrDefault("#12");
|
||||
Color result1 = ManualRedactionService.decodeOrDefault("123");
|
||||
Color result2 = ManualRedactionService.decodeOrDefault("#12");
|
||||
|
||||
assertNotNull(result1);
|
||||
assertNotNull(result2);
|
||||
@@ -1094,7 +1106,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("confidential");
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
assertNotNull(tokens);
|
||||
@@ -1115,7 +1127,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("secret");
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
assertNotNull(tokens);
|
||||
@@ -1148,7 +1160,7 @@ class RedactControllerTest {
|
||||
|
||||
List<Object> originalTokens = getOriginalTokens();
|
||||
List<Object> filteredTokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
long originalNonTextCount =
|
||||
@@ -1156,7 +1168,7 @@ class RedactControllerTest {
|
||||
.filter(
|
||||
token ->
|
||||
token instanceof Operator op
|
||||
&& !redactController.isTextShowingOperator(
|
||||
&& !textRedactionService.isTextShowingOperator(
|
||||
op.getName()))
|
||||
.count();
|
||||
|
||||
@@ -1165,7 +1177,7 @@ class RedactControllerTest {
|
||||
.filter(
|
||||
token ->
|
||||
token instanceof Operator op
|
||||
&& !redactController.isTextShowingOperator(
|
||||
&& !textRedactionService.isTextShowingOperator(
|
||||
op.getName()))
|
||||
.count();
|
||||
|
||||
@@ -1184,7 +1196,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("\\d{3}-\\d{2}-\\d{4}"); // SSN pattern
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, true, false);
|
||||
|
||||
String reconstructedText = extractTextFromTokens(tokens);
|
||||
@@ -1200,7 +1212,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("test");
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, true);
|
||||
|
||||
String reconstructedText = extractTextFromTokens(tokens);
|
||||
@@ -1217,7 +1229,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("sensitive");
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
String reconstructedText = extractTextFromTokens(tokens);
|
||||
@@ -1231,7 +1243,7 @@ class RedactControllerTest {
|
||||
void shouldWriteTokensToNewContentStream() throws Exception {
|
||||
List<Object> tokens = createSampleTokenList();
|
||||
|
||||
redactController.writeFilteredContentStream(realDocument, realPage, tokens);
|
||||
textRedactionService.writeFilteredContentStream(realDocument, realPage, tokens);
|
||||
|
||||
assertNotNull(realPage.getContents(), "Page should have content stream");
|
||||
|
||||
@@ -1249,7 +1261,7 @@ class RedactControllerTest {
|
||||
|
||||
assertDoesNotThrow(
|
||||
() ->
|
||||
redactController.writeFilteredContentStream(
|
||||
textRedactionService.writeFilteredContentStream(
|
||||
realDocument, realPage, emptyTokens));
|
||||
|
||||
assertNotNull(realPage.getContents(), "Page should still have content stream");
|
||||
@@ -1262,7 +1274,7 @@ class RedactControllerTest {
|
||||
String originalContent = extractTextFromModifiedPage(realPage);
|
||||
|
||||
List<Object> newTokens = createSampleTokenList();
|
||||
redactController.writeFilteredContentStream(realDocument, realPage, newTokens);
|
||||
textRedactionService.writeFilteredContentStream(realDocument, realPage, newTokens);
|
||||
|
||||
String newContent = extractTextFromModifiedPage(realPage);
|
||||
assertNotEquals(originalContent, newContent, "Content stream should be replaced");
|
||||
@@ -1273,7 +1285,7 @@ class RedactControllerTest {
|
||||
void shouldCreateWidthMatchingPlaceholder() {
|
||||
String originalText = "confidential";
|
||||
String placeholder =
|
||||
redactController.createPlaceholderWithFont(
|
||||
textRedactionService.createPlaceholderWithFont(
|
||||
originalText, new PDType1Font(Standard14Fonts.FontName.HELVETICA));
|
||||
|
||||
assertEquals(
|
||||
@@ -1287,7 +1299,7 @@ class RedactControllerTest {
|
||||
void shouldHandleSpecialCharactersInPlaceholder() {
|
||||
String originalText = "café naïve";
|
||||
String placeholder =
|
||||
redactController.createPlaceholderWithFont(
|
||||
textRedactionService.createPlaceholderWithFont(
|
||||
originalText, new PDType1Font(Standard14Fonts.FontName.HELVETICA));
|
||||
|
||||
assertEquals(originalText.length(), placeholder.length());
|
||||
@@ -1303,10 +1315,10 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("secret");
|
||||
|
||||
List<Object> filteredTokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
redactController.writeFilteredContentStream(realDocument, realPage, filteredTokens);
|
||||
textRedactionService.writeFilteredContentStream(realDocument, realPage, filteredTokens);
|
||||
assertNotNull(realPage.getContents());
|
||||
|
||||
String finalText = extractTextFromModifiedPage(realPage);
|
||||
@@ -1322,7 +1334,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("confidential");
|
||||
|
||||
List<Object> filteredTokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
long filteredPositioning =
|
||||
@@ -1377,7 +1389,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("confidential");
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
assertNotNull(tokens);
|
||||
@@ -1404,14 +1416,12 @@ class RedactControllerTest {
|
||||
@Test
|
||||
@DisplayName("Should handle documents with multiple text blocks")
|
||||
void shouldHandleDocumentsWithMultipleTextBlocks() throws Exception {
|
||||
// Create a document with multiple text blocks
|
||||
realPage = new PDPage(PDRectangle.A4);
|
||||
while (realDocument.getNumberOfPages() > 0) {
|
||||
realDocument.removePage(0);
|
||||
}
|
||||
realDocument.addPage(realPage);
|
||||
|
||||
// Create resources
|
||||
PDResources resources = new PDResources();
|
||||
resources.put(
|
||||
COSName.getPDFName("F1"), new PDType1Font(Standard14Fonts.FontName.HELVETICA));
|
||||
|
||||
+430
@@ -0,0 +1,430 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.pdf.parser.PageColumnLayout;
|
||||
|
||||
/**
|
||||
* Integration tests for {@link RedactExecuteService#collectRangeBlocks(PDDocument, String, String,
|
||||
* Map)}. Each test builds a synthetic PDF (single-column or two-column) with text-positioning that
|
||||
* matches what a real document would produce, then asserts that the redaction range produces blocks
|
||||
* confined to the expected X/Y region.
|
||||
*/
|
||||
class RedactExecuteServiceTest {
|
||||
|
||||
private static final float PAGE_WIDTH = PDRectangle.LETTER.getWidth(); // 612
|
||||
private static final float PAGE_HEIGHT = PDRectangle.LETTER.getHeight(); // 792
|
||||
|
||||
private static final float LEFT_X = 72f;
|
||||
private static final float RIGHT_X = 330f;
|
||||
private static final float COL_WIDTH = 220f;
|
||||
private static final float LINE_HEIGHT = 14f;
|
||||
private static final float TOP_Y = PAGE_HEIGHT - 80f;
|
||||
private static final float FONT_SIZE = 11f;
|
||||
|
||||
private final RedactExecuteService service =
|
||||
new RedactExecuteService(null, null, new TextRedactionService());
|
||||
|
||||
@Nested
|
||||
@DisplayName("Single-column documents")
|
||||
class SingleColumn {
|
||||
|
||||
@Test
|
||||
void redactBetweenMarkers_inclusive() throws IOException {
|
||||
try (PDDocument doc = buildSingleColumnDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(doc, "START-HERE", "STOP-HERE", cache);
|
||||
|
||||
assertThat(blocks)
|
||||
.as("blocks should be produced for single-column range")
|
||||
.isNotEmpty();
|
||||
|
||||
// Blocks are in screen coords (top-left, Y down). START-HERE is drawn at the top
|
||||
// of the page; STOP-HERE four lines below. Screen Y grows downward, so the
|
||||
// anchors' screen-Y tops sit roughly around screenTop(0) and screenTop(4).
|
||||
// The end anchor is inclusive, so blocks may extend to the bottom of line 4.
|
||||
float screenTopOfStart = screenTopOfLine(0);
|
||||
float screenBottomOfEnd = screenTopOfLine(4) + LINE_HEIGHT;
|
||||
for (PDFText block : blocks) {
|
||||
assertThat(block.getY1())
|
||||
.as("block top must be at or below the start anchor's top")
|
||||
.isGreaterThanOrEqualTo(screenTopOfStart - 1f);
|
||||
assertThat(block.getY2())
|
||||
.as(
|
||||
"block bottom must not extend past the end anchor's bottom (end is inclusive)")
|
||||
.isLessThanOrEqualTo(screenBottomOfEnd + 1f);
|
||||
assertThat(block.getX2())
|
||||
.as("block should not extend into a hypothetical right column")
|
||||
.isLessThan(PAGE_WIDTH / 2f + 50f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingStartString_noBlocks() throws IOException {
|
||||
try (PDDocument doc = buildSingleColumnDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(doc, "MISSING-START", "STOP-HERE", cache);
|
||||
|
||||
assertThat(blocks).isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void cvStyleHeadingPlusRightAlignedDate_stillTreatedAsSingleColumn() throws IOException {
|
||||
// CV-style page: single-column body, but each section heading shares its row with a
|
||||
// right-aligned date. The X-gap splitter emits the heading and the date as separate
|
||||
// line boxes; this must NOT trip 2-column detection (the date is too narrow to be a
|
||||
// real column), otherwise the cross-page redaction predicate over-includes wrong
|
||||
// regions.
|
||||
try (PDDocument doc = buildCvStyleDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(doc, "SECTION-A", "SECTION-C", cache);
|
||||
|
||||
assertThat(blocks)
|
||||
.as("CV-style redaction between section headings must produce blocks")
|
||||
.isNotEmpty();
|
||||
|
||||
PageColumnLayout layout = cache.get(0);
|
||||
assertThat(layout.columnCount())
|
||||
.as("CV-style page with heading+date rows must remain single-column")
|
||||
.isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void punctuationDriftInAnchors_stillMatchesViaTolerantFallback() throws IOException {
|
||||
// Simulates the LLM paraphrasing the heading by inserting a colon that isn't in the
|
||||
// source ("#3 Character substitution" → "#3: Character substitution"). The
|
||||
// punctuation-tolerant regex fallback should still find the line.
|
||||
try (PDDocument doc = buildHeadingPdf()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(
|
||||
doc, "#3: Character substitution", "#6: Image resolution", cache);
|
||||
|
||||
assertThat(blocks)
|
||||
.as("anchor with extra punctuation should still resolve via fallback")
|
||||
.isNotEmpty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("Two-column documents")
|
||||
class TwoColumn {
|
||||
|
||||
@Test
|
||||
void rangeInLeftColumn_redactsOnlyLeftColumn() throws IOException {
|
||||
try (PDDocument doc = buildTwoColumnDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks = service.collectRangeBlocks(doc, "L-START", "L-END", cache);
|
||||
|
||||
assertThat(blocks).as("left-only range must produce blocks").isNotEmpty();
|
||||
|
||||
float gutterMid = (LEFT_X + COL_WIDTH + RIGHT_X) / 2f;
|
||||
for (PDFText block : blocks) {
|
||||
float midX = (block.getX1() + block.getX2()) / 2f;
|
||||
assertThat(midX)
|
||||
.as("every block must sit in the left column, never the right")
|
||||
.isLessThan(gutterMid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void rangeInRightColumn_redactsOnlyRightColumn() throws IOException {
|
||||
try (PDDocument doc = buildTwoColumnDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks = service.collectRangeBlocks(doc, "R-START", "R-END", cache);
|
||||
|
||||
assertThat(blocks).as("right-only range must produce blocks").isNotEmpty();
|
||||
|
||||
float gutterMid = (LEFT_X + COL_WIDTH + RIGHT_X) / 2f;
|
||||
for (PDFText block : blocks) {
|
||||
float midX = (block.getX1() + block.getX2()) / 2f;
|
||||
assertThat(midX)
|
||||
.as("every block must sit in the right column, never the left")
|
||||
.isGreaterThan(gutterMid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumnWithTocAbove_pairsAcrossColumns() throws IOException {
|
||||
// Reproduces magic.pdf-style stacked layout: a multi-line TOC near the top, then a
|
||||
// 2-column body where the start anchor is in left col (lower screen Y) and the end
|
||||
// anchor is in right col (higher screen Y). Original pairing failed here because
|
||||
// end.y < start.y in screen coords.
|
||||
try (PDDocument doc = buildTwoColumnWithTocDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(doc, "BODY-L-3", "BODY-R-1", cache);
|
||||
|
||||
assertThat(blocks)
|
||||
.as("cross-column body redaction must produce blocks despite stacked TOC")
|
||||
.isNotEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void crossColumnReadingOrder_leftBottomToRightTop_producesBothSides() throws IOException {
|
||||
// This is the case the original code couldn't handle at all: end Y < start Y.
|
||||
try (PDDocument doc = buildTwoColumnDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(doc, "L-MIDDLE", "R-MIDDLE", cache);
|
||||
|
||||
assertThat(blocks)
|
||||
.as("cross-column range must produce blocks, not be silently dropped")
|
||||
.isNotEmpty();
|
||||
|
||||
float gutterMid = (LEFT_X + COL_WIDTH + RIGHT_X) / 2f;
|
||||
boolean sawLeft = false;
|
||||
boolean sawRight = false;
|
||||
for (PDFText block : blocks) {
|
||||
float midX = (block.getX1() + block.getX2()) / 2f;
|
||||
if (midX < gutterMid) sawLeft = true;
|
||||
else sawRight = true;
|
||||
}
|
||||
assertThat(sawLeft).as("left column should contain at least one block").isTrue();
|
||||
assertThat(sawRight).as("right column should contain at least one block").isTrue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── document fixtures ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Single-column page laid out as one column starting at LEFT_X. Lines: 0: START-HERE (start
|
||||
* anchor) 1: line one 2: line two 3: line three 4: STOP-HERE (end anchor) 5: line five (must
|
||||
* NOT be redacted)
|
||||
*/
|
||||
private PDDocument buildSingleColumnDoc() throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
|
||||
String[] lines = {
|
||||
"START-HERE", "line one", "line two", "line three", "STOP-HERE", "line five"
|
||||
};
|
||||
for (int i = 0; i < lines.length; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(LEFT_X, yForLine(i));
|
||||
cs.showText(lines[i]);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-column page. Lines per column, top to bottom: Left: L-TOP, L-START, L-MIDDLE, L-END,
|
||||
* L-BOTTOM Right: R-TOP, R-MIDDLE, R-START, R-END, R-BOTTOM
|
||||
*/
|
||||
private PDDocument buildTwoColumnDoc() throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
|
||||
// Body lines are padded to make each column genuinely wide enough that column
|
||||
// detection (which ignores narrow lines) treats both sides as real columns.
|
||||
String fill = " " + "x".repeat(26);
|
||||
String[] left = {
|
||||
"L-TOP" + fill,
|
||||
"L-START" + fill,
|
||||
"L-MIDDLE" + fill,
|
||||
"L-END" + fill,
|
||||
"L-BOTTOM" + fill
|
||||
};
|
||||
String[] right = {
|
||||
"R-TOP" + fill,
|
||||
"R-MIDDLE" + fill,
|
||||
"R-START" + fill,
|
||||
"R-END" + fill,
|
||||
"R-BOTTOM" + fill
|
||||
};
|
||||
for (int i = 0; i < left.length; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(LEFT_X, yForLine(i));
|
||||
cs.showText(left[i]);
|
||||
cs.endText();
|
||||
}
|
||||
// Aligned baselines per row (IEEE template style) — AllTextLineExtractor must split
|
||||
// these at the column gap rather than merge same-row left+right glyphs into a wide
|
||||
// box.
|
||||
for (int i = 0; i < right.length; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(RIGHT_X, yForLine(i));
|
||||
cs.showText(right[i]);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single-column page with feature headings: #1..#7 each followed by body text. The PDF text is
|
||||
* exactly "#3 Character substitution" (no colon) — the test then queries with a colon to
|
||||
* exercise the punctuation-tolerant fallback.
|
||||
*/
|
||||
private PDDocument buildHeadingPdf() throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
|
||||
String[] lines = {
|
||||
"#1 Auto layout",
|
||||
"Body about auto layout.",
|
||||
"#2 Smart selection",
|
||||
"Body about smart selection.",
|
||||
"#3 Character substitution",
|
||||
"Body about character substitution.",
|
||||
"#4 Rounded borders",
|
||||
"Body about rounded borders.",
|
||||
"#5 Auto contrast",
|
||||
"Body about auto contrast.",
|
||||
"#6 Image resolution",
|
||||
"Body about image resolution.",
|
||||
"#7 Columns",
|
||||
"Body about columns."
|
||||
};
|
||||
for (int i = 0; i < lines.length; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(LEFT_X, yForLine(i));
|
||||
cs.showText(lines[i]);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-column page like {@code magic.pdf}: a few full-width header lines, a 2-column TOC stacked
|
||||
* on top of the 2-column body, where TOC's right half lives inside what would otherwise be the
|
||||
* body's gutter. Body left column has BODY-L-1..3, right column has BODY-R-1..3.
|
||||
*/
|
||||
private PDDocument buildTwoColumnWithTocDoc() throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
|
||||
// Header — full width, lines 0..1.
|
||||
for (int i = 0; i < 2; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(LEFT_X, yForLine(i));
|
||||
cs.showText("FULL WIDTH HEADER LINE " + i + " ACROSS BOTH COLUMNS OF THE PAGE");
|
||||
cs.endText();
|
||||
}
|
||||
// TOC, 2 columns of entries. TOC right half sits where the body gutter would be —
|
||||
// exactly the layout that broke the histogram-based detector on magic.pdf.
|
||||
float tocLeftX = 101f;
|
||||
float tocRightX = 230f;
|
||||
for (int i = 0; i < 5; i++) {
|
||||
float y = yForLine(3 + i);
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(tocLeftX, y);
|
||||
cs.showText("TOC entry left " + i);
|
||||
cs.endText();
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(tocRightX, y);
|
||||
cs.showText("TOC entry right " + i);
|
||||
cs.endText();
|
||||
}
|
||||
// Body — 2-column with aligned baselines per row (IEEE-style).
|
||||
String fill = " " + "x".repeat(26);
|
||||
String[] bodyLeft = {"BODY-L-1" + fill, "BODY-L-2" + fill, "BODY-L-3" + fill};
|
||||
String[] bodyRight = {"BODY-R-1" + fill, "BODY-R-2" + fill, "BODY-R-3" + fill};
|
||||
for (int i = 0; i < bodyLeft.length; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(LEFT_X, yForLine(10 + i));
|
||||
cs.showText(bodyLeft[i]);
|
||||
cs.endText();
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(RIGHT_X, yForLine(10 + i));
|
||||
cs.showText(bodyRight[i]);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* CV-style page: single-column body with a few section headings, each followed on the same
|
||||
* baseline by a right-aligned date string. {@link AllTextLineExtractor} will split each
|
||||
* heading+date row into two line boxes; column detection must reject this as a fake two-column
|
||||
* layout because the dates are too narrow to be a real column body.
|
||||
*/
|
||||
private PDDocument buildCvStyleDoc() throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
|
||||
|
||||
float dateX = PAGE_WIDTH - 144f; // right-aligned dates near the right margin
|
||||
|
||||
// Section A: heading + date, then 3 body lines.
|
||||
writeAt(cs, LEFT_X, yForLine(0), "SECTION-A");
|
||||
writeAt(cs, dateX, yForLine(0), "Jan 2020");
|
||||
writeAt(cs, LEFT_X, yForLine(1), "Body line A1 with enough width to look like body");
|
||||
writeAt(cs, LEFT_X, yForLine(2), "Body line A2 with enough width to look like body");
|
||||
writeAt(cs, LEFT_X, yForLine(3), "Body line A3 with enough width to look like body");
|
||||
|
||||
// Section B (in the redact range): heading + date + 3 body lines.
|
||||
writeAt(cs, LEFT_X, yForLine(5), "SECTION-B");
|
||||
writeAt(cs, dateX, yForLine(5), "Feb 2021");
|
||||
writeAt(cs, LEFT_X, yForLine(6), "Body line B1 with enough width to look like body");
|
||||
writeAt(cs, LEFT_X, yForLine(7), "Body line B2 with enough width to look like body");
|
||||
writeAt(cs, LEFT_X, yForLine(8), "Body line B3 with enough width to look like body");
|
||||
|
||||
// Section C (end anchor): heading + date.
|
||||
writeAt(cs, LEFT_X, yForLine(10), "SECTION-C");
|
||||
writeAt(cs, dateX, yForLine(10), "Mar 2022");
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
private static void writeAt(PDPageContentStream cs, float x, float y, String text)
|
||||
throws IOException {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(x, y);
|
||||
cs.showText(text);
|
||||
cs.endText();
|
||||
}
|
||||
|
||||
/** PDF user-space Y baseline for line index {@code i} (0-based, top to bottom). */
|
||||
private static float yForLine(int lineIndex) {
|
||||
return TOP_Y - lineIndex * LINE_HEIGHT;
|
||||
}
|
||||
|
||||
/** Approximate screen-Y of the top of line {@code i} (top-left origin). */
|
||||
private static float screenTopOfLine(int lineIndex) {
|
||||
// baseline_pdf → baseline_screen flips against page height; glyph top ≈ baseline - font
|
||||
// size.
|
||||
return PAGE_HEIGHT - yForLine(lineIndex) - FONT_SIZE;
|
||||
}
|
||||
}
|
||||
+201
-1
@@ -8,13 +8,18 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
import org.apache.commons.csv.CSVPrinter;
|
||||
import org.apache.commons.csv.QuoteMode;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
@@ -24,6 +29,7 @@ import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.SPDF.pdf.parser.PdfIngester;
|
||||
import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage;
|
||||
import stirling.software.SPDF.pdf.parser.PdfModels.RawLine;
|
||||
@@ -32,6 +38,7 @@ import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment;
|
||||
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
import stirling.software.common.util.RegexPatternUtils;
|
||||
import stirling.software.proprietary.model.api.ai.AiPdfContentType;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowTextSelection;
|
||||
@@ -122,6 +129,7 @@ public class PdfContentExtractor {
|
||||
}
|
||||
csvStrings.add(sw.toString());
|
||||
}
|
||||
|
||||
return csvStrings;
|
||||
}
|
||||
|
||||
@@ -295,7 +303,6 @@ public class PdfContentExtractor {
|
||||
private List<AiWorkflowTextSelection> extractPageText(
|
||||
PDDocument document, List<Integer> selectedPages, int maxCharacters)
|
||||
throws IOException {
|
||||
PDFTextStripper textStripper = new PDFTextStripper();
|
||||
List<AiWorkflowTextSelection> pages = new ArrayList<>();
|
||||
int remainingCharacters = maxCharacters;
|
||||
|
||||
@@ -304,10 +311,29 @@ public class PdfContentExtractor {
|
||||
break;
|
||||
}
|
||||
|
||||
PDFTextStripper textStripper = new PDFTextStripper();
|
||||
textStripper.setSortByPosition(true);
|
||||
textStripper.setStartPage(pageNumber);
|
||||
textStripper.setEndPage(pageNumber);
|
||||
|
||||
String pageText = textStripper.getText(document).trim();
|
||||
|
||||
// Prepend page dimensions so the AI agent can reason about absolute coordinates.
|
||||
PDPage page = document.getPage(pageNumber - 1);
|
||||
PDRectangle bbox = page.getBBox();
|
||||
String dimensionHeader =
|
||||
String.format(
|
||||
"--- Page dimensions: %.0fx%.0f pts"
|
||||
+ " (PDF user-space: origin bottom-left, Y up) ---\n",
|
||||
bbox.getWidth(), bbox.getHeight());
|
||||
pageText = dimensionHeader + pageText;
|
||||
|
||||
// Append image metadata so the AI agent can reason about images spatially.
|
||||
String imageAnnotation = buildImageAnnotation(document, pageNumber - 1);
|
||||
if (!imageAnnotation.isEmpty()) {
|
||||
pageText = pageText + imageAnnotation;
|
||||
}
|
||||
|
||||
if (pageText.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
@@ -327,6 +353,56 @@ public class PdfContentExtractor {
|
||||
return pages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a human-readable description of all images on a page to append to page text. Uses PDF
|
||||
* user-space coordinates (origin bottom-left, Y up) so the AI can reference exact bounding
|
||||
* boxes when requesting image redaction.
|
||||
*/
|
||||
private String buildImageAnnotation(PDDocument document, int pageIndex) {
|
||||
try {
|
||||
List<ImageBlock> images = extractImagePositions(document, pageIndex);
|
||||
if (images.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
PDRectangle bbox = page.getBBox();
|
||||
float pageWidth = bbox.getWidth();
|
||||
float pageHeight = bbox.getHeight();
|
||||
|
||||
StringBuilder sb = new StringBuilder("\n\n--- Images on this page ---");
|
||||
for (int i = 0; i < images.size(); i++) {
|
||||
ImageBlock img = images.get(i);
|
||||
String position = spatialLabel(img, pageWidth, pageHeight);
|
||||
float w = img.x2() - img.x1();
|
||||
float h = img.y2() - img.y1();
|
||||
sb.append(
|
||||
String.format(
|
||||
"\nImage %d: position=%s, size=%.0fx%.0f pts,"
|
||||
+ " bounds=(x1=%.0f, y1=%.0f, x2=%.0f, y2=%.0f)",
|
||||
i + 1, position, w, h, img.x1(), img.y1(), img.x2(), img.y2()));
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"Failed to extract image positions for page {}: {}", pageIndex, e.getMessage());
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a human-readable spatial label (e.g. "top-left", "center") for an image based on its
|
||||
* centre relative to the page dimensions. Coordinates are in PDF user-space (Y up).
|
||||
*/
|
||||
private static String spatialLabel(ImageBlock img, float pageWidth, float pageHeight) {
|
||||
float cx = (img.x1() + img.x2()) / 2f;
|
||||
float cy = (img.y1() + img.y2()) / 2f;
|
||||
|
||||
String horiz = cx < pageWidth / 3f ? "left" : cx < 2 * pageWidth / 3f ? "center" : "right";
|
||||
// PDF Y increases upward, so higher Y = higher on the page = "top"
|
||||
String vert = cy > 2 * pageHeight / 3f ? "top" : cy > pageHeight / 3f ? "middle" : "bottom";
|
||||
return vert + "-" + horiz;
|
||||
}
|
||||
|
||||
private ExtractedFileText buildExtractedFileText(
|
||||
String fileName, List<AiWorkflowTextSelection> pages) {
|
||||
ExtractedFileText fileText = new ExtractedFileText();
|
||||
@@ -347,6 +423,130 @@ public class PdfContentExtractor {
|
||||
return text.substring(0, end);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Text position finding
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* A located text match inside a PDF: 0-based page index and bounding box in PDFBox coordinates
|
||||
* (origin bottom-left).
|
||||
*/
|
||||
public record TextBlock(int pageIndex, float x1, float y1, float x2, float y2) {}
|
||||
|
||||
/**
|
||||
* An image found on a PDF page: 0-based page index and bounding box in PDF user-space
|
||||
* coordinates (origin bottom-left, Y increases upward).
|
||||
*/
|
||||
public record ImageBlock(int pageIndex, float x1, float y1, float x2, float y2) {}
|
||||
|
||||
/**
|
||||
* Extract the bounding boxes of all raster/vector images on the given (0-based) page.
|
||||
*
|
||||
* @param document the open PDF
|
||||
* @param pageIndex 0-based page index
|
||||
* @return list of located images in document order
|
||||
*/
|
||||
public List<ImageBlock> extractImagePositions(PDDocument document, int pageIndex)
|
||||
throws IOException {
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
PageImageLocator locator = new PageImageLocator(page, pageIndex);
|
||||
locator.processPage(page);
|
||||
return locator.getImageBoxes().stream()
|
||||
.map(b -> new ImageBlock(b.pageIndex(), b.x1(), b.y1(), b.x2(), b.y2()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all occurrences of {@code pattern} in {@code document} and return their bounding boxes.
|
||||
*
|
||||
* @param document the open PDF
|
||||
* @param pattern the search string or regex
|
||||
* @param useRegex {@code true} to treat {@code pattern} as a regular expression
|
||||
* @return list of located matches, in page order
|
||||
*/
|
||||
public List<TextBlock> findTextPositions(PDDocument document, String pattern, boolean useRegex)
|
||||
throws IOException {
|
||||
LocalTextFinder finder = new LocalTextFinder(pattern, useRegex);
|
||||
finder.getText(document);
|
||||
return finder.found;
|
||||
}
|
||||
|
||||
private static final class LocalTextFinder extends PDFTextStripper {
|
||||
|
||||
private final String searchTerm;
|
||||
private final boolean useRegex;
|
||||
final List<TextBlock> found = new ArrayList<>();
|
||||
|
||||
private final List<TextPosition> pagePositions = new ArrayList<>();
|
||||
private final StringBuilder pageText = new StringBuilder();
|
||||
|
||||
LocalTextFinder(String searchTerm, boolean useRegex) throws IOException {
|
||||
this.searchTerm = searchTerm;
|
||||
this.useRegex = useRegex;
|
||||
setWordSeparator(" ");
|
||||
setLineSeparator("\n");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startPage(PDPage page) throws IOException {
|
||||
super.startPage(page);
|
||||
pagePositions.clear();
|
||||
pageText.setLength(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> positions) {
|
||||
pageText.append(text);
|
||||
pagePositions.addAll(positions);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeWordSeparator() {
|
||||
pageText.append(getWordSeparator());
|
||||
pagePositions.add(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeLineSeparator() {
|
||||
pageText.append(getLineSeparator());
|
||||
pagePositions.add(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void endPage(PDPage page) throws IOException {
|
||||
String text = pageText.toString();
|
||||
if (!text.isEmpty() && searchTerm != null && !searchTerm.isBlank()) {
|
||||
String term = searchTerm.trim();
|
||||
String regex = useRegex ? term : "\\Q" + term + "\\E";
|
||||
Pattern pat = RegexPatternUtils.getInstance().createSearchPattern(regex, true);
|
||||
Matcher matcher = pat.matcher(text);
|
||||
while (matcher.find()) {
|
||||
float minX = Float.MAX_VALUE;
|
||||
float minY = Float.MAX_VALUE;
|
||||
float maxX = -Float.MAX_VALUE;
|
||||
float maxY = -Float.MAX_VALUE;
|
||||
boolean hit = false;
|
||||
for (int i = matcher.start(); i < matcher.end(); i++) {
|
||||
if (i < pagePositions.size()) {
|
||||
TextPosition tp = pagePositions.get(i);
|
||||
if (tp != null) {
|
||||
hit = true;
|
||||
minX = Math.min(minX, tp.getX());
|
||||
maxX = Math.max(maxX, tp.getX() + tp.getWidth());
|
||||
minY = Math.min(minY, tp.getY() - tp.getHeight());
|
||||
maxY = Math.max(maxY, tp.getY());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (hit) {
|
||||
found.add(new TextBlock(getCurrentPageNo() - 1, minX, minY, maxX, maxY));
|
||||
}
|
||||
}
|
||||
}
|
||||
super.endPage(page);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Types shared with AiWorkflowService (package-private) ---
|
||||
|
||||
interface PdfContentResult {
|
||||
|
||||
@@ -86,7 +86,7 @@ class OrchestratorAgent:
|
||||
system_prompt=(
|
||||
"You are the top-level orchestrator. "
|
||||
"Choose exactly one output function that best handles the request. "
|
||||
"Use delegate_pdf_edit for requested modifications of single or multiple PDFs. "
|
||||
"Use delegate_pdf_edit for any requested modification of one or more PDFs. "
|
||||
"Use delegate_pdf_question for questions about the contents of the attached PDFs. "
|
||||
"Use delegate_user_spec for requests to create or define an agent spec. "
|
||||
"Use delegate_pdf_review when the user wants the PDF returned with review"
|
||||
|
||||
@@ -39,11 +39,39 @@ class PdfEditPlanSelection(ApiModel):
|
||||
summary: str
|
||||
|
||||
|
||||
type PdfEditPlanOutput = PdfEditPlanSelection | EditClarificationRequest | EditCannotDoResponse | NeedContentResponse
|
||||
class PdfEditNeedContentSelection(ApiModel):
|
||||
"""LLM-facing variant of need_content: the model signals it needs document content and gives
|
||||
a reason. File objects are resolved by Python from request.files by matching names, so the
|
||||
LLM can't fabricate file ids — it only selects by the display names it sees in the prompt.
|
||||
"""
|
||||
|
||||
outcome: Literal["need_content"] = "need_content"
|
||||
reason: str
|
||||
file_names: list[str] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Names of files whose content is needed. "
|
||||
"Use the exact names shown in the prompt. "
|
||||
"Omit or leave empty to request content from all files."
|
||||
),
|
||||
)
|
||||
max_pages: int | None = None
|
||||
max_characters: int | None = None
|
||||
|
||||
|
||||
type PdfEditPlanOutput = (
|
||||
PdfEditPlanSelection | EditClarificationRequest | EditCannotDoResponse | PdfEditNeedContentSelection
|
||||
)
|
||||
|
||||
|
||||
class PdfEditSelectionAgent:
|
||||
def __init__(self, runtime: AppRuntime, base_system_prompt: str, *, allow_need_content: bool) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
runtime: AppRuntime,
|
||||
base_system_prompt: str,
|
||||
*,
|
||||
allow_need_content: bool,
|
||||
) -> None:
|
||||
self.runtime = runtime
|
||||
output_types: list[type[PdfEditPlanOutput]] = [
|
||||
PdfEditPlanSelection,
|
||||
@@ -52,11 +80,12 @@ class PdfEditSelectionAgent:
|
||||
]
|
||||
system_prompt = base_system_prompt
|
||||
if allow_need_content:
|
||||
output_types.append(NeedContentResponse)
|
||||
output_types.append(PdfEditNeedContentSelection)
|
||||
system_prompt += (
|
||||
" Return need_content when planning a correct answer requires inspecting the actual PDF "
|
||||
"page text (e.g. 'split after every page that says NEW PAGE', "
|
||||
"'rotate pages that mention draft')."
|
||||
"'rotate pages that mention draft'). "
|
||||
"Set file_names to only the files that need to be read; omit it to read all files."
|
||||
)
|
||||
self.agent = Agent(
|
||||
model=runtime.smart_model,
|
||||
@@ -101,14 +130,18 @@ class PdfEditParameterSelector:
|
||||
parameter_result = await self.agent.run(
|
||||
prompt,
|
||||
output_type=NativeOutput(parameter_model),
|
||||
instructions=(
|
||||
f"Generate only the parameters for the PDF operation `{operation_id.name}`. "
|
||||
"Do not include fields from any other operation."
|
||||
),
|
||||
instructions=self._get_operation_instructions(operation_id),
|
||||
)
|
||||
logger.debug("[pdf-edit params %s] output: %s", operation_id.name, Pretty(parameter_result.output))
|
||||
return parameter_result.output
|
||||
|
||||
@staticmethod
|
||||
def _get_operation_instructions(operation_id: ToolEndpoint) -> str:
|
||||
return (
|
||||
f"Generate only the parameters for the PDF operation `{operation_id.name}`. "
|
||||
"Do not include fields from any other operation."
|
||||
)
|
||||
|
||||
def _build_parameter_prompt(
|
||||
self,
|
||||
request: PdfEditRequest,
|
||||
@@ -173,8 +206,7 @@ class PdfEditAgent:
|
||||
request.enabled_endpoints,
|
||||
request.user_message,
|
||||
)
|
||||
supported_operations = self._get_supported_operations(request)
|
||||
unavailable_operations = self._get_unavailable_operations(supported_operations)
|
||||
supported_operations, unavailable_operations = self._classify_operations(request)
|
||||
if not supported_operations:
|
||||
return EditCannotDoResponse(reason="No PDF edit operations are available on this server.")
|
||||
selection = await self._select_plan(
|
||||
@@ -183,9 +215,9 @@ class PdfEditAgent:
|
||||
if isinstance(selection, EditClarificationRequest | EditCannotDoResponse):
|
||||
logger.info("[pdf-edit] selection -> %s: %s", selection.outcome, Pretty(selection))
|
||||
return selection
|
||||
if isinstance(selection, NeedContentResponse):
|
||||
if isinstance(selection, PdfEditNeedContentSelection):
|
||||
logger.info("[pdf-edit] selection -> need_content: %s", selection.reason)
|
||||
return self._fill_need_content_defaults(selection, request)
|
||||
return self._build_need_content_response(selection, request)
|
||||
enabled = set(supported_operations)
|
||||
unsupported = [op for op in selection.operations if op not in enabled]
|
||||
if unsupported:
|
||||
@@ -228,7 +260,9 @@ class PdfEditAgent:
|
||||
) -> PdfEditPlanOutput:
|
||||
can_request_content = allow_need_content and not has_page_text(request.page_text)
|
||||
agent = self._build_selection_agent(
|
||||
supported_operations, unavailable_operations, allow_need_content=can_request_content
|
||||
supported_operations,
|
||||
unavailable_operations,
|
||||
allow_need_content=can_request_content,
|
||||
)
|
||||
return await agent.select(self._build_selection_prompt(request, supported_operations, unavailable_operations))
|
||||
|
||||
@@ -263,9 +297,11 @@ class PdfEditAgent:
|
||||
"merging, or extracting pages then re-inserting them). "
|
||||
"Only return cannot_do when no sequence of the supported operations could achieve the request. "
|
||||
"Do not produce operation parameters in this stage. "
|
||||
"Return need_clarification when the request is genuinely ambiguous. "
|
||||
"Return plan when a reasonable multi-step plan can be created. "
|
||||
"Never return partial plans."
|
||||
"Never return partial plans. "
|
||||
"Return need_clarification only when the request is genuinely ambiguous in a way "
|
||||
"that no reasonable interpretation could produce a correct plan — do not ask to "
|
||||
"confirm details that are already clear from the user's message."
|
||||
),
|
||||
allow_need_content=allow_need_content,
|
||||
)
|
||||
@@ -291,13 +327,31 @@ class PdfEditAgent:
|
||||
f"Extracted page text:\n{format_page_text(request.page_text)}"
|
||||
)
|
||||
|
||||
def _get_supported_operations(self, request: PdfEditRequest) -> Iterable[ToolEndpoint]:
|
||||
return request.enabled_endpoints
|
||||
# Endpoints that exist on the server and are callable via the direct API or the manual UI,
|
||||
# but are never offered to the AI agent as a routing option.
|
||||
#
|
||||
# Why: REDACT_EXECUTE is the preferred AI-driven redaction route. AUTO_REDACT and REDACT are
|
||||
# legacy endpoints that remain fully functional for human callers (the manual redact UI, direct
|
||||
# API consumers, pipelines) but would produce a worse experience if the AI routed to them —
|
||||
# they accept a simpler, less expressive schema and pre-date the unified operation model.
|
||||
# Hiding them here channels all AI redaction traffic through REDACT_EXECUTE without disabling
|
||||
# the legacy endpoints for anyone else.
|
||||
#
|
||||
# How to reuse: add an endpoint here whenever a legacy endpoint has a preferred replacement
|
||||
# that the AI should use exclusively. The endpoint remains live on the server; only the AI
|
||||
# planner is prevented from selecting it.
|
||||
_AGENT_HIDDEN_ENDPOINTS: frozenset[ToolEndpoint] = frozenset({ToolEndpoint.AUTO_REDACT, ToolEndpoint.REDACT})
|
||||
|
||||
@staticmethod
|
||||
def _get_unavailable_operations(supported_operations: Iterable[ToolEndpoint]) -> Iterable[ToolEndpoint]:
|
||||
supported_set = set(supported_operations)
|
||||
return [op for op in OPERATIONS if op not in supported_set]
|
||||
def _classify_operations(self, request: PdfEditRequest) -> tuple[list[ToolEndpoint], list[ToolEndpoint]]:
|
||||
"""Split the universe of operations into (supported, unavailable) from the agent's
|
||||
point of view. Endpoints in `_AGENT_HIDDEN_ENDPOINTS` are filtered out regardless
|
||||
of enabled state — they exist on the server but only callers outside the AI
|
||||
pipeline (the manual redact UI, direct API consumers) can invoke them.
|
||||
"""
|
||||
enabled_set = set(request.enabled_endpoints)
|
||||
supported = [op for op in request.enabled_endpoints if op not in self._AGENT_HIDDEN_ENDPOINTS]
|
||||
unavailable = [op for op in OPERATIONS if op not in enabled_set and op not in self._AGENT_HIDDEN_ENDPOINTS]
|
||||
return supported, unavailable
|
||||
|
||||
@staticmethod
|
||||
def _get_operations_prompt(operations: Iterable[ToolEndpoint]) -> str:
|
||||
@@ -326,18 +380,29 @@ class PdfEditAgent:
|
||||
lines.append(f" {name}")
|
||||
return "\n".join(lines)
|
||||
|
||||
def _fill_need_content_defaults(
|
||||
def _build_need_content_response(
|
||||
self,
|
||||
selection: NeedContentResponse,
|
||||
selection: PdfEditNeedContentSelection,
|
||||
request: PdfEditRequest,
|
||||
) -> NeedContentResponse:
|
||||
files = selection.files or [
|
||||
NeedContentFileRequest(file=file, content_types=[PdfContentType.PAGE_TEXT]) for file in request.files
|
||||
]
|
||||
# File objects are resolved here by matching names against request.files so the LLM
|
||||
# can't fabricate file ids — it selects by display name, Python provides the AiFile.
|
||||
if selection.file_names:
|
||||
requested = set(selection.file_names)
|
||||
files = [f for f in request.files if f.name in requested]
|
||||
if not files:
|
||||
# Names didn't match anything; fall back to all files rather than sending nothing.
|
||||
logger.warning(
|
||||
"[pdf-edit] need_content file_names %s matched no request files — using all",
|
||||
selection.file_names,
|
||||
)
|
||||
files = request.files
|
||||
else:
|
||||
files = request.files
|
||||
return NeedContentResponse(
|
||||
resume_with=SupportedCapability.PDF_EDIT,
|
||||
reason=selection.reason,
|
||||
files=files,
|
||||
files=[NeedContentFileRequest(file=file, content_types=[PdfContentType.PAGE_TEXT]) for file in files],
|
||||
max_pages=selection.max_pages or self.runtime.settings.max_pages,
|
||||
max_characters=selection.max_characters or self.runtime.settings.max_characters,
|
||||
)
|
||||
|
||||
@@ -116,7 +116,6 @@ from .progress import (
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AiFile",
|
||||
"AgentDraft",
|
||||
"AgentDraftRequest",
|
||||
"AgentDraftResponse",
|
||||
@@ -127,6 +126,7 @@ __all__ = [
|
||||
"AgentRevisionWorkflowResponse",
|
||||
"AgentSpec",
|
||||
"AgentSpecStep",
|
||||
"AiFile",
|
||||
"AiToolAgentStep",
|
||||
"ArtifactKind",
|
||||
"CannotContinueExecutionAction",
|
||||
|
||||
@@ -542,6 +542,18 @@ class HtmlToPdfParams(ApiModel):
|
||||
zoom: float = Field(1, description="Zoom level for displaying the website. Default is '1'.")
|
||||
|
||||
|
||||
class ImageBox(ApiModel):
|
||||
"""
|
||||
Rectangular areas to black out, each defined by a page number and bounding box coordinates.
|
||||
"""
|
||||
|
||||
page_index: int = Field(..., description="0-indexed page number (first page = 0).")
|
||||
x1: float = Field(..., description="Left x coordinate of the redaction rectangle in PDF user-space points.")
|
||||
x2: float = Field(..., description="Right x coordinate of the redaction rectangle in PDF user-space points.")
|
||||
y1: float = Field(..., description="Top y coordinate of the redaction rectangle in PDF user-space points.")
|
||||
y2: float = Field(..., description="Bottom y coordinate of the redaction rectangle in PDF user-space points.")
|
||||
|
||||
|
||||
class ColorType(StrEnum):
|
||||
"""
|
||||
The color type of the output image(s)
|
||||
@@ -984,6 +996,27 @@ class RearrangePagesParams(ApiModel):
|
||||
)
|
||||
|
||||
|
||||
class Strategy(StrEnum):
|
||||
"""
|
||||
Execution strategy hint for the redaction pipeline
|
||||
"""
|
||||
|
||||
auto = "AUTO"
|
||||
overlay_only = "OVERLAY_ONLY"
|
||||
image_finalize = "IMAGE_FINALIZE"
|
||||
|
||||
|
||||
class Style(ApiModel):
|
||||
"""
|
||||
Redaction style options
|
||||
"""
|
||||
|
||||
color: str = Field("#000000", description="Hex redaction box color")
|
||||
convert_to_image: bool = Field(False, description="Rasterize output to prevent text extraction")
|
||||
padding: float = Field(0, description="Extra padding around each box in points")
|
||||
strategy: Strategy = Field(Strategy.auto, description="Execution strategy hint for the redaction pipeline")
|
||||
|
||||
|
||||
class RedactionArea(ApiModel):
|
||||
"""
|
||||
A list of areas that should be redacted
|
||||
@@ -1279,6 +1312,23 @@ class SvgToPdfParams(ApiModel):
|
||||
)
|
||||
|
||||
|
||||
class TextRange(ApiModel):
|
||||
"""
|
||||
Text ranges to redact by specifying a start and end anchor phrase. All content between the two phrases (inclusive) is redacted. Anchors work best when short and unique. They must appear verbatim in the document.
|
||||
"""
|
||||
|
||||
end_string: str = Field(
|
||||
...,
|
||||
description="A short, distinctive phrase (5–15 words) that marks where redaction ends (inclusive). Must appear verbatim in the document. Shorter phrases match more reliably.",
|
||||
min_length=1,
|
||||
)
|
||||
start_string: str = Field(
|
||||
...,
|
||||
description="A short, distinctive phrase (5–15 words) that marks where redaction begins (inclusive). Must appear verbatim in the document — e.g. a section heading or a unique sentence fragment.",
|
||||
min_length=1,
|
||||
)
|
||||
|
||||
|
||||
class TimestampPdfParams(ApiModel):
|
||||
tsa_url: str = Field(
|
||||
"http://timestamp.digicert.com",
|
||||
@@ -1346,6 +1396,32 @@ class VectorToPdfParams(ApiModel):
|
||||
prepress: Prepress = Field(Prepress.boolean_false, description="Apply Ghostscript prepress settings")
|
||||
|
||||
|
||||
class RedactExecuteParams(ApiModel):
|
||||
image_boxes: list[ImageBox] | None = Field(
|
||||
None, description="Rectangular areas to black out, each defined by a page number and bounding box coordinates."
|
||||
)
|
||||
ranges: list[TextRange] | None = Field(
|
||||
None,
|
||||
description="Text ranges to redact by specifying a start and end anchor phrase. All content between the two phrases (inclusive) is redacted. Anchors work best when short and unique. They must appear verbatim in the document.",
|
||||
)
|
||||
redact_image_pages: list[int] | None = Field(
|
||||
None,
|
||||
description="1-indexed page numbers to redact all detected images from. Pass an empty list to redact images from every page. Omit or pass null to skip image redaction entirely.",
|
||||
)
|
||||
regex_patterns: list[str] | None = Field(
|
||||
None,
|
||||
description="Regex patterns to match and redact. Each match anywhere in the document is blacked out. Uses Java/PCRE regex syntax. Well-suited for strings that follow known patterns, like phone numbers, email addresses, national ID numbers, or dates (which can appear with different separators, optional country codes, etc.). For fixed known strings such as names, use textValues instead.",
|
||||
)
|
||||
style: Style | None = Field(None, description="Redaction style options")
|
||||
text_values: list[str] | None = Field(
|
||||
None,
|
||||
description="Exact strings to find and black out. One entry per phrase to redact. Best for known names, identifiers, and specific text found in the document.",
|
||||
)
|
||||
wipe_pages: list[int] | None = Field(
|
||||
None, description="1-indexed page numbers to wipe entirely (all content removed from those pages)."
|
||||
)
|
||||
|
||||
|
||||
class RedactParams(ApiModel):
|
||||
convert_pdf_to_image: bool = Field(False, description="Convert the redacted PDF to an image")
|
||||
page_numbers: str = Field(
|
||||
@@ -1420,6 +1496,7 @@ class Model(
|
||||
| SessionsParams
|
||||
| ValidateCertificateParams
|
||||
| RedactParams
|
||||
| RedactExecuteParams
|
||||
| RemovePasswordParams
|
||||
| SanitizePdfParams
|
||||
| TimestampPdfParams
|
||||
@@ -1488,6 +1565,7 @@ class Model(
|
||||
| SessionsParams
|
||||
| ValidateCertificateParams
|
||||
| RedactParams
|
||||
| RedactExecuteParams
|
||||
| RemovePasswordParams
|
||||
| SanitizePdfParams
|
||||
| TimestampPdfParams
|
||||
@@ -1557,6 +1635,7 @@ type ParamToolModel = (
|
||||
| SessionsParams
|
||||
| ValidateCertificateParams
|
||||
| RedactParams
|
||||
| RedactExecuteParams
|
||||
| RemovePasswordParams
|
||||
| SanitizePdfParams
|
||||
| TimestampPdfParams
|
||||
@@ -1627,6 +1706,7 @@ class ToolEndpoint(StrEnum):
|
||||
SESSIONS = "/api/v1/security/cert-sign/sessions"
|
||||
VALIDATE_CERTIFICATE = "/api/v1/security/cert-sign/validate-certificate"
|
||||
REDACT = "/api/v1/security/redact"
|
||||
REDACT_EXECUTE = "/api/v1/security/redact-execute"
|
||||
REMOVE_PASSWORD = "/api/v1/security/remove-password"
|
||||
SANITIZE_PDF = "/api/v1/security/sanitize-pdf"
|
||||
TIMESTAMP_PDF = "/api/v1/security/timestamp-pdf"
|
||||
@@ -1695,6 +1775,7 @@ OPERATIONS: dict[ToolEndpoint, ParamToolModelType] = {
|
||||
ToolEndpoint.SESSIONS: SessionsParams,
|
||||
ToolEndpoint.VALIDATE_CERTIFICATE: ValidateCertificateParams,
|
||||
ToolEndpoint.REDACT: RedactParams,
|
||||
ToolEndpoint.REDACT_EXECUTE: RedactExecuteParams,
|
||||
ToolEndpoint.REMOVE_PASSWORD: RemovePasswordParams,
|
||||
ToolEndpoint.SANITIZE_PDF: SanitizePdfParams,
|
||||
ToolEndpoint.TIMESTAMP_PDF: TimestampPdfParams,
|
||||
|
||||
@@ -6,7 +6,7 @@ from dataclasses import dataclass
|
||||
import pytest
|
||||
|
||||
from stirling.agents import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection
|
||||
from stirling.agents.pdf_edit import PdfEditPlanOutput
|
||||
from stirling.agents.pdf_edit import PdfEditNeedContentSelection, PdfEditPlanOutput
|
||||
from stirling.contracts import (
|
||||
AiFile,
|
||||
EditCannotDoResponse,
|
||||
@@ -82,12 +82,12 @@ class StubPdfEditAgent(PdfEditAgent):
|
||||
if parameter_selector is not None:
|
||||
self.parameter_selector = parameter_selector
|
||||
|
||||
def _get_supported_operations(self, request: PdfEditRequest) -> Iterable[ToolEndpoint]:
|
||||
def _classify_operations(self, request: PdfEditRequest) -> tuple[list[ToolEndpoint], list[ToolEndpoint]]:
|
||||
# Tests construct requests without `enabled_endpoints`; pretend everything is enabled
|
||||
# unless the test explicitly supplies an enabled set.
|
||||
if request.enabled_endpoints:
|
||||
return request.enabled_endpoints
|
||||
return OPERATIONS
|
||||
if not request.enabled_endpoints:
|
||||
return list(OPERATIONS), []
|
||||
return super()._classify_operations(request)
|
||||
|
||||
async def _select_plan(
|
||||
self,
|
||||
@@ -194,12 +194,8 @@ async def test_pdf_edit_agent_returns_need_content_without_building_plan(runtime
|
||||
parameter_selector = RecordingParameterSelector()
|
||||
agent = StubPdfEditAgent(
|
||||
runtime,
|
||||
NeedContentResponse(
|
||||
resume_with=SupportedCapability.PDF_EDIT,
|
||||
PdfEditNeedContentSelection(
|
||||
reason="Need page text to locate the NEW PAGE markers.",
|
||||
files=[],
|
||||
max_pages=0,
|
||||
max_characters=0,
|
||||
),
|
||||
parameter_selector=parameter_selector,
|
||||
)
|
||||
@@ -274,8 +270,8 @@ async def test_pdf_edit_selection_agent_excludes_need_content_from_schema_when_n
|
||||
can_request = PdfEditSelectionAgent(runtime, "base", allow_need_content=True)
|
||||
cannot_request = PdfEditSelectionAgent(runtime, "base", allow_need_content=False)
|
||||
|
||||
assert NeedContentResponse in _agent_output_types(can_request)
|
||||
assert NeedContentResponse not in _agent_output_types(cannot_request)
|
||||
assert PdfEditNeedContentSelection in _agent_output_types(can_request)
|
||||
assert PdfEditNeedContentSelection not in _agent_output_types(cannot_request)
|
||||
|
||||
|
||||
def _agent_output_types(agent: object) -> list[type]:
|
||||
@@ -336,7 +332,7 @@ async def test_pdf_edit_agent_supported_operations_defaults_to_empty(
|
||||
runtime: AppRuntime,
|
||||
) -> None:
|
||||
agent = PdfEditAgent(runtime)
|
||||
supported = agent._get_supported_operations(PdfEditRequest(user_message="hi"))
|
||||
supported, _ = agent._classify_operations(PdfEditRequest(user_message="hi"))
|
||||
|
||||
assert list(supported) == []
|
||||
|
||||
@@ -351,7 +347,7 @@ async def test_pdf_edit_agent_supported_operations_uses_provided_list(
|
||||
enabled_endpoints=[ToolEndpoint.FLATTEN, ToolEndpoint.ROTATE_PDF],
|
||||
)
|
||||
|
||||
supported = agent._get_supported_operations(request)
|
||||
supported, _ = agent._classify_operations(request)
|
||||
|
||||
assert list(supported) == [ToolEndpoint.FLATTEN, ToolEndpoint.ROTATE_PDF]
|
||||
|
||||
@@ -372,8 +368,7 @@ def test_pdf_edit_selection_prompt_includes_unavailable_operations(runtime: AppR
|
||||
user_message="Run OCR.",
|
||||
enabled_endpoints=[ToolEndpoint.FLATTEN],
|
||||
)
|
||||
supported = agent._get_supported_operations(request)
|
||||
unavailable = agent._get_unavailable_operations(supported)
|
||||
supported, unavailable = agent._classify_operations(request)
|
||||
|
||||
prompt = agent._build_selection_prompt(request, supported, unavailable)
|
||||
|
||||
|
||||
@@ -64,15 +64,16 @@ async function extractPageMeasureScales(
|
||||
// Parse a Measure dict into a MeasureScale, or return null if malformed.
|
||||
const parseScale = (measureObj: unknown) => {
|
||||
if (!(measureObj instanceof PDFDict)) return null;
|
||||
const rObj = measureObj.lookup(PDFName.of("R"));
|
||||
// @cantoo/pdf-lib ships without individual .d.ts files so instanceof can't narrow `unknown`
|
||||
const m = measureObj as PDFDict;
|
||||
const rObj = m.lookup(PDFName.of("R"));
|
||||
const ratioLabel =
|
||||
rObj instanceof PDFString || rObj instanceof PDFHexString
|
||||
? rObj.decodeText()
|
||||
: "";
|
||||
// D = distance array, X = x-axis fallback
|
||||
let fmtArray = measureObj.lookup(PDFName.of("D"));
|
||||
if (!(fmtArray instanceof PDFArray))
|
||||
fmtArray = measureObj.lookup(PDFName.of("X"));
|
||||
let fmtArray = m.lookup(PDFName.of("D"));
|
||||
if (!(fmtArray instanceof PDFArray)) fmtArray = m.lookup(PDFName.of("X"));
|
||||
if (!(fmtArray instanceof PDFArray)) return null;
|
||||
const firstFmt = fmtArray.lookup(0);
|
||||
if (!(firstFmt instanceof PDFDict)) return null;
|
||||
|
||||
@@ -22,6 +22,20 @@ import type {
|
||||
ButtonAction,
|
||||
} from "@app/tools/formFill/types";
|
||||
import type { IFormDataProvider } from "@app/tools/formFill/providers/types";
|
||||
import type {
|
||||
PDFDict,
|
||||
PDFString,
|
||||
PDFHexString,
|
||||
PDFName,
|
||||
} from "@cantoo/pdf-lib";
|
||||
|
||||
interface PDFAcroField {
|
||||
dict: PDFDict;
|
||||
getWidgets?: () => Array<{ dict: PDFDict }>;
|
||||
}
|
||||
interface PDFFieldInternal {
|
||||
acroField?: PDFAcroField;
|
||||
}
|
||||
import {
|
||||
closeDocAndFreeBuffer,
|
||||
extractFormFields,
|
||||
@@ -238,7 +252,11 @@ export class PdfiumFormProvider implements IFormDataProvider {
|
||||
);
|
||||
const altName = readUtf16(m, altBuf, altLen);
|
||||
m.pdfium.wasmExports.free(altBuf);
|
||||
(nameToField.get(name) as any)._tooltip = altName || null;
|
||||
(
|
||||
nameToField.get(name) as PdfiumFormField & {
|
||||
_tooltip?: string | null;
|
||||
}
|
||||
)._tooltip = altName || null;
|
||||
}
|
||||
enriched.add(name);
|
||||
}
|
||||
@@ -299,7 +317,7 @@ export class PdfiumFormProvider implements IFormDataProvider {
|
||||
|
||||
const decodeText = (obj: unknown): string => {
|
||||
if (obj instanceof PDFString || obj instanceof PDFHexString)
|
||||
return obj.decodeText();
|
||||
return (obj as PDFString | PDFHexString).decodeText();
|
||||
return String(obj ?? "");
|
||||
};
|
||||
|
||||
@@ -312,7 +330,8 @@ export class PdfiumFormProvider implements IFormDataProvider {
|
||||
)
|
||||
continue;
|
||||
|
||||
const acroDict = (field.acroField as any).dict;
|
||||
const acroDict = (field as unknown as PDFFieldInternal).acroField!
|
||||
.dict;
|
||||
const optRaw = acroDict.lookup(PDFName.of("Opt"));
|
||||
if (!(optRaw instanceof PDFArray)) continue;
|
||||
|
||||
@@ -374,8 +393,14 @@ export class PdfiumFormProvider implements IFormDataProvider {
|
||||
if (buttons.length === 0) return result;
|
||||
|
||||
try {
|
||||
const { PDFDocument, PDFName, PDFString, PDFHexString, PDFDict } =
|
||||
await import("@cantoo/pdf-lib");
|
||||
const {
|
||||
PDFDocument,
|
||||
PDFName,
|
||||
PDFString,
|
||||
PDFHexString,
|
||||
PDFDict,
|
||||
PDFNumber,
|
||||
} = await import("@cantoo/pdf-lib");
|
||||
|
||||
const doc = await PDFDocument.load(data, {
|
||||
ignoreEncryption: true,
|
||||
@@ -385,53 +410,53 @@ export class PdfiumFormProvider implements IFormDataProvider {
|
||||
|
||||
const decodeText = (obj: unknown): string | null => {
|
||||
if (obj instanceof PDFString || obj instanceof PDFHexString)
|
||||
return obj.decodeText();
|
||||
return (obj as PDFString | PDFHexString).decodeText();
|
||||
if (obj instanceof PDFName)
|
||||
return (obj as any).asString?.() ?? obj.toString().replace(/^\//, "");
|
||||
return (obj as PDFName).asString() ?? String(obj).replace(/^\//, "");
|
||||
return null;
|
||||
};
|
||||
|
||||
const parseActionDict = (aObj: unknown): ButtonAction | null => {
|
||||
if (!(aObj instanceof PDFDict)) return null;
|
||||
const sObj = aObj.lookup(PDFName.of("S"));
|
||||
// @cantoo/pdf-lib ships without individual .d.ts files so instanceof can't narrow `unknown`
|
||||
const a = aObj as PDFDict;
|
||||
const sObj = a.lookup(PDFName.of("S"));
|
||||
if (!(sObj instanceof PDFName)) return null;
|
||||
const actionType: string =
|
||||
(sObj as any).asString?.() ?? sObj.toString().replace(/^\//, "");
|
||||
(sObj as PDFName).asString() ?? String(sObj).replace(/^\//, "");
|
||||
|
||||
switch (actionType) {
|
||||
case "Named": {
|
||||
const nObj = aObj.lookup(PDFName.of("N"));
|
||||
const nObj = a.lookup(PDFName.of("N"));
|
||||
const name =
|
||||
nObj instanceof PDFName
|
||||
? ((nObj as any).asString?.() ??
|
||||
nObj.toString().replace(/^\//, ""))
|
||||
? ((nObj as PDFName).asString() ??
|
||||
String(nObj).replace(/^\//, ""))
|
||||
: "";
|
||||
return { type: "named", namedAction: name };
|
||||
}
|
||||
case "JavaScript": {
|
||||
const jsObj = aObj.lookup(PDFName.of("JS"));
|
||||
const jsObj = a.lookup(PDFName.of("JS"));
|
||||
const js = decodeText(jsObj) ?? jsObj?.toString() ?? "";
|
||||
return { type: "javascript", javascript: js };
|
||||
}
|
||||
case "SubmitForm": {
|
||||
const fObj = aObj.lookup(PDFName.of("F"));
|
||||
const fObj = a.lookup(PDFName.of("F"));
|
||||
let url = "";
|
||||
if (fObj instanceof PDFDict) {
|
||||
url = decodeText(fObj.lookup(PDFName.of("F"))) ?? "";
|
||||
} else if (fObj) {
|
||||
url = decodeText(fObj) ?? fObj.toString();
|
||||
url = decodeText(fObj) ?? String(fObj);
|
||||
}
|
||||
const flagsObj = aObj.lookup(PDFName.of("Flags"));
|
||||
const flagsObj = a.lookup(PDFName.of("Flags"));
|
||||
const flags =
|
||||
typeof (flagsObj as any)?.asNumber === "function"
|
||||
? (flagsObj as any).asNumber()
|
||||
: 0;
|
||||
flagsObj instanceof PDFNumber ? flagsObj.asNumber() : 0;
|
||||
return { type: "submitForm", url, submitFlags: flags };
|
||||
}
|
||||
case "ResetForm":
|
||||
return { type: "resetForm" };
|
||||
case "URI": {
|
||||
const uriObj = aObj.lookup(PDFName.of("URI"));
|
||||
const uriObj = a.lookup(PDFName.of("URI"));
|
||||
return { type: "uri", url: decodeText(uriObj) ?? "" };
|
||||
}
|
||||
default:
|
||||
@@ -439,7 +464,7 @@ export class PdfiumFormProvider implements IFormDataProvider {
|
||||
}
|
||||
};
|
||||
|
||||
const getMkCaption = (dict: any): string | null => {
|
||||
const getMkCaption = (dict: PDFDict): string | null => {
|
||||
try {
|
||||
const mkObj = dict.lookup(PDFName.of("MK"));
|
||||
if (!(mkObj instanceof PDFDict)) return null;
|
||||
@@ -450,7 +475,7 @@ export class PdfiumFormProvider implements IFormDataProvider {
|
||||
}
|
||||
};
|
||||
|
||||
const getActionFromDict = (dict: any): ButtonAction | null => {
|
||||
const getActionFromDict = (dict: PDFDict): ButtonAction | null => {
|
||||
try {
|
||||
return parseActionDict(dict.lookup(PDFName.of("A")));
|
||||
} catch {
|
||||
@@ -465,13 +490,13 @@ export class PdfiumFormProvider implements IFormDataProvider {
|
||||
if (!buttonNames.has(name)) continue;
|
||||
|
||||
try {
|
||||
const acroField = (field as any).acroField;
|
||||
const acroField = (field as unknown as PDFFieldInternal).acroField;
|
||||
if (!acroField?.dict) continue;
|
||||
|
||||
const info: { label?: string; action?: ButtonAction } = {};
|
||||
|
||||
// Try widget dicts first (each widget can have its own /MK and /A)
|
||||
const widgets: any[] = (acroField as any).getWidgets?.() ?? [];
|
||||
const widgets = acroField.getWidgets?.() ?? [];
|
||||
for (const widget of widgets) {
|
||||
if (!info.label) {
|
||||
const label = getMkCaption(widget.dict);
|
||||
|
||||
@@ -25,7 +25,13 @@ export function usePrototypeToolRegistry(): PrototypeToolRegistry {
|
||||
() =>
|
||||
({
|
||||
pdfCommentAgent: {
|
||||
icon: <LocalIcon icon="add-comment" width="1.5rem" height="1.5rem" />,
|
||||
icon: (
|
||||
<LocalIcon
|
||||
icon="add-comment-outline-rounded"
|
||||
width="1.5rem"
|
||||
height="1.5rem"
|
||||
/>
|
||||
),
|
||||
name: t("home.pdfCommentAgent.title", "Add AI comments"),
|
||||
component: PdfCommentAgent,
|
||||
description: t(
|
||||
|
||||
Reference in New Issue
Block a user