mirror of
https://github.com/arsvendg/Stirling-PDF.git
synced 2026-07-14 10:34:06 +02:00
smart redaction (#6195)
Co-authored-by: James Brunton <[email protected]>
This commit is contained in:
EthanHealy01
co-authored by
James Brunton
James Brunton
parent
7f3ca7ea70
commit
35a712a278
@@ -0,0 +1,142 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Detects whether a page is one- or two-column from per-line bounding boxes, and classifies an
|
||||
* X-span into the column it belongs to. Detection is a midpoint vote at {@code pageWidth / 2}.
|
||||
*
|
||||
* <p>Capped at two columns by design — sufficient for the redaction target set (single-column
|
||||
* documents and IEEE-style two-column papers). 3+ column layouts (newspapers, magazines) and
|
||||
* off-centre gutters (asymmetric two-column) would need a histogram or clustering approach to
|
||||
* detect the actual gutter X. (future work)
|
||||
*
|
||||
* <p>Coordinates are PDFTextStripper screen space (top-left origin, Y increases downward).
|
||||
*/
|
||||
public final class PageColumnLayout {
|
||||
|
||||
/**
|
||||
* Slack when checking "crosses a gutter" so single-pixel overshoots don't mark a line as
|
||||
* spanning.
|
||||
*/
|
||||
public static final float SPAN_SLACK_PT = 2f;
|
||||
|
||||
/**
|
||||
* Slack on each side of the page midpoint inside which a line is considered "spanning"
|
||||
* (covering both columns) rather than belonging to one side.
|
||||
*/
|
||||
private static final float MIDPOINT_SLACK_PT = 30f;
|
||||
|
||||
/**
|
||||
* Minimum line width (points) for a line to count toward the two-column tally. Avoids false
|
||||
* positives where right-aligned dates, page numbers, or short "Link" fragments next to a
|
||||
* heading look like a second column when they're really just inline metadata.
|
||||
*/
|
||||
private static final float MIN_COLUMN_LINE_WIDTH_PT = 100f;
|
||||
|
||||
/**
|
||||
* Minimum number of clearly leftish AND clearly rightish lines (each of width ≥ {@link
|
||||
* #MIN_COLUMN_LINE_WIDTH_PT}) required to call the page two-column. Anything below this falls
|
||||
* back to single-column.
|
||||
*/
|
||||
private static final int MIN_SIDE_LINES = 3;
|
||||
|
||||
private final List<float[]> columns;
|
||||
private final List<float[]> gutters;
|
||||
|
||||
private PageColumnLayout(List<float[]> columns, List<float[]> gutters) {
|
||||
this.columns = columns;
|
||||
this.gutters = gutters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines column layout from per-line bounding boxes ({@code [x1, _, x2, _]}). Counts lines
|
||||
* whose X-midpoint sits clearly left of, or clearly right of, the page midpoint (with {@link
|
||||
* #MIDPOINT_SLACK_PT} slack each side). If both sides have at least {@link #MIN_SIDE_LINES}
|
||||
* lines, the page is treated as two-column with the gutter at the page midpoint. Otherwise it's
|
||||
* single-column.
|
||||
*
|
||||
* <p>Cross-column lines must already be split: callers should feed boxes from a line extractor
|
||||
* that splits same-Y glyphs at large X gaps (see {@code AllTextLineExtractor}). Without that
|
||||
* split, IEEE-style aligned-baseline 2-column PDFs produce one wide merged box per row and the
|
||||
* side tallies all end up classified as "spanning", falling to single-column.
|
||||
*/
|
||||
public static PageColumnLayout fromLineBoxes(List<float[]> lineBoxes, float pageWidth) {
|
||||
if (lineBoxes == null || lineBoxes.isEmpty()) {
|
||||
return new PageColumnLayout(List.of(new float[] {0f, pageWidth}), List.of());
|
||||
}
|
||||
float pageMid = pageWidth / 2f;
|
||||
int left = 0, right = 0;
|
||||
for (float[] lb : lineBoxes) {
|
||||
if (lb == null || lb.length < 3) continue;
|
||||
float width = lb[2] - lb[0];
|
||||
// Skip narrow lines — dates, page numbers, "Link" labels next to a heading should
|
||||
// not, on their own, make a single-column doc look two-column.
|
||||
if (width < MIN_COLUMN_LINE_WIDTH_PT) continue;
|
||||
float mid = (lb[0] + lb[2]) * 0.5f;
|
||||
if (mid < pageMid - MIDPOINT_SLACK_PT) left++;
|
||||
else if (mid > pageMid + MIDPOINT_SLACK_PT) right++;
|
||||
}
|
||||
if (left < MIN_SIDE_LINES || right < MIN_SIDE_LINES) {
|
||||
return new PageColumnLayout(List.of(new float[] {0f, pageWidth}), List.of());
|
||||
}
|
||||
float gutterL = pageMid - MIDPOINT_SLACK_PT;
|
||||
float gutterR = pageMid + MIDPOINT_SLACK_PT;
|
||||
return new PageColumnLayout(
|
||||
List.of(new float[] {0f, gutterL}, new float[] {gutterR, pageWidth}),
|
||||
List.of(new float[] {gutterL, gutterR}));
|
||||
}
|
||||
|
||||
/** All columns, left-to-right, as {@code [leftX, rightX]} pairs. Never empty. */
|
||||
public List<float[]> columns() {
|
||||
return columns;
|
||||
}
|
||||
|
||||
/** Gutters between columns, left-to-right, as {@code [leftX, rightX]} pairs. */
|
||||
public List<float[]> gutters() {
|
||||
return gutters;
|
||||
}
|
||||
|
||||
public int columnCount() {
|
||||
return columns.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the column index containing the X-midpoint of {@code [x1, x2]}, falling back to the
|
||||
* closest column if the midpoint sits inside a gutter.
|
||||
*/
|
||||
public int columnOf(float x1, float x2) {
|
||||
float mid = (x1 + x2) * 0.5f;
|
||||
int best = 0;
|
||||
float bestDist = Float.MAX_VALUE;
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
float[] c = columns.get(i);
|
||||
if (mid >= c[0] && mid <= c[1]) return i;
|
||||
float dist = mid < c[0] ? c[0] - mid : mid - c[1];
|
||||
if (dist < bestDist) {
|
||||
bestDist = dist;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns every column index whose X-range overlaps {@code [x1, x2]} with at least {@link
|
||||
* #SPAN_SLACK_PT} of intrusion. A normal in-column line returns one index; a line crossing a
|
||||
* gutter returns two or more.
|
||||
*/
|
||||
public int[] columnsCrossing(float x1, float x2) {
|
||||
List<Integer> hits = new ArrayList<>();
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
float[] c = columns.get(i);
|
||||
float overlap = Math.min(x2, c[1]) - Math.max(x1, c[0]);
|
||||
if (overlap > SPAN_SLACK_PT) hits.add(i);
|
||||
}
|
||||
if (hits.isEmpty()) hits.add(columnOf(x1, x2));
|
||||
int[] out = new int[hits.size()];
|
||||
for (int i = 0; i < hits.size(); i++) out[i] = hits.get(i);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.contentstream.PDFGraphicsStreamEngine;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImage;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
|
||||
/**
|
||||
* PDFGraphicsStreamEngine that intercepts {@code drawImage} calls and records each image's bounding
|
||||
* box in PDF user-space (origin bottom-left, Y up) by transforming the unit square through the
|
||||
* current transformation matrix (CTM).
|
||||
*
|
||||
* <p>Usage:
|
||||
*
|
||||
* <pre>{@code
|
||||
* PageImageLocator locator = new PageImageLocator(page, pageIndex);
|
||||
* locator.processPage(page);
|
||||
* List<ImageBox> boxes = locator.getImageBoxes();
|
||||
* }</pre>
|
||||
*
|
||||
* <p>Each {@link ImageBox} carries the 0-based page index and the axis-aligned bounding box {@code
|
||||
* (x1, y1, x2, y2)} in PDF user-space coordinates.
|
||||
*/
|
||||
public final class PageImageLocator extends PDFGraphicsStreamEngine {
|
||||
|
||||
/**
|
||||
* Bounding box of a raster or vector image found on a PDF page.
|
||||
*
|
||||
* @param pageIndex 0-based page index
|
||||
* @param x1 left edge in PDF user-space (origin bottom-left)
|
||||
* @param y1 bottom edge in PDF user-space
|
||||
* @param x2 right edge
|
||||
* @param y2 top edge
|
||||
*/
|
||||
public record ImageBox(int pageIndex, float x1, float y1, float x2, float y2) {}
|
||||
|
||||
private final int pageIndex;
|
||||
private final List<ImageBox> imageBoxes = new ArrayList<>();
|
||||
private final Point2D.Float currentPoint = new Point2D.Float();
|
||||
|
||||
/**
|
||||
* @param page the PDPage to process
|
||||
* @param pageIndex 0-based index of this page in the document (stored on each returned {@link
|
||||
* ImageBox})
|
||||
*/
|
||||
public PageImageLocator(PDPage page, int pageIndex) {
|
||||
super(page);
|
||||
this.pageIndex = pageIndex;
|
||||
}
|
||||
|
||||
/** Returns all image bounding boxes collected during {@link #processPage}. */
|
||||
public List<ImageBox> getImageBoxes() {
|
||||
return imageBoxes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void drawImage(PDImage pdImage) throws IOException {
|
||||
Matrix ctm = getGraphicsState().getCurrentTransformationMatrix();
|
||||
// An image occupies the unit square (0,0)→(1,1) in image space.
|
||||
// Transform all four corners through the CTM to get the page-space bounding box.
|
||||
float a = ctm.getScaleX();
|
||||
float b = ctm.getShearY();
|
||||
float c = ctm.getShearX();
|
||||
float d = ctm.getScaleY();
|
||||
float e = ctm.getTranslateX();
|
||||
float f = ctm.getTranslateY();
|
||||
float[] xs = {e, a + e, c + e, a + c + e};
|
||||
float[] ys = {f, b + f, d + f, b + d + f};
|
||||
float x1 = Float.MAX_VALUE, y1 = Float.MAX_VALUE;
|
||||
float x2 = -Float.MAX_VALUE, y2 = -Float.MAX_VALUE;
|
||||
for (float x : xs) {
|
||||
x1 = Math.min(x1, x);
|
||||
x2 = Math.max(x2, x);
|
||||
}
|
||||
for (float y : ys) {
|
||||
y1 = Math.min(y1, y);
|
||||
y2 = Math.max(y2, y);
|
||||
}
|
||||
imageBoxes.add(new ImageBox(pageIndex, x1, y1, x2, y2));
|
||||
}
|
||||
|
||||
// ---------- required abstract methods (no-op for path operations) ----------
|
||||
|
||||
@Override
|
||||
public void appendRectangle(Point2D p0, Point2D p1, Point2D p2, Point2D p3) {}
|
||||
|
||||
@Override
|
||||
public void clip(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void moveTo(float x, float y) {
|
||||
currentPoint.setLocation(x, y);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void lineTo(float x, float y) {
|
||||
currentPoint.setLocation(x, y);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void curveTo(float x1, float y1, float x2, float y2, float x3, float y3) {
|
||||
currentPoint.setLocation(x3, y3);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Point2D getCurrentPoint() {
|
||||
return currentPoint;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void closePath() {}
|
||||
|
||||
@Override
|
||||
public void endPath() {}
|
||||
|
||||
@Override
|
||||
public void strokePath() {}
|
||||
|
||||
@Override
|
||||
public void fillPath(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void fillAndStrokePath(int windingRule) {}
|
||||
|
||||
@Override
|
||||
public void shadingFill(COSName shadingName) {}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Binds a multipart form value containing a JSON array into a typed {@code List<T>}. Used for
|
||||
* endpoints that accept structured list parameters via {@code @ModelAttribute} — the form field
|
||||
* carries the full JSON array as its value and the editor parses it once.
|
||||
*/
|
||||
@Slf4j
|
||||
public class JsonListPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER =
|
||||
JsonMapper.builder()
|
||||
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.build();
|
||||
|
||||
private final TypeReference<? extends List<T>> typeRef;
|
||||
|
||||
public JsonListPropertyEditor(TypeReference<? extends List<T>> typeRef) {
|
||||
this.typeRef = typeRef;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(new ArrayList<T>());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
setValue(OBJECT_MAPPER.readValue(text, typeRef));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to parse JSON list value", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Expected a JSON array but could not parse: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Binds a multipart form value containing a JSON object into a typed {@code T}. Companion to {@link
|
||||
* JsonListPropertyEditor} for single-object nested fields on {@code @ModelAttribute} endpoints.
|
||||
*/
|
||||
@Slf4j
|
||||
public class JsonObjectPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER =
|
||||
JsonMapper.builder().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).build();
|
||||
|
||||
private final Class<T> type;
|
||||
|
||||
public JsonObjectPropertyEditor(Class<T> type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(null);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
setValue(OBJECT_MAPPER.readValue(text, type));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to parse JSON object value", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Expected a JSON object but could not parse: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
-53
@@ -1,53 +0,0 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import java.beans.PropertyEditorSupport;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import tools.jackson.databind.DeserializationFeature;
|
||||
import tools.jackson.databind.JavaType;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
import tools.jackson.databind.json.JsonMapper;
|
||||
|
||||
/**
|
||||
* Spring property editor that decodes a JSON string into a typed {@link ArrayList}. Used to bind
|
||||
* complex list parameters (e.g. {@code List<RedactionArea>}, {@code List<EditTextOperation>}) from
|
||||
* multipart form fields, where Spring's default binding cannot deserialize a JSON array.
|
||||
*/
|
||||
@Slf4j
|
||||
public class StringToArrayListPropertyEditor<T> extends PropertyEditorSupport {
|
||||
|
||||
private final ObjectMapper objectMapper =
|
||||
JsonMapper.builder()
|
||||
.enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY)
|
||||
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.build();
|
||||
|
||||
private final Class<T> elementType;
|
||||
|
||||
public StringToArrayListPropertyEditor(Class<T> elementType) {
|
||||
this.elementType = elementType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setAsText(String text) throws IllegalArgumentException {
|
||||
if (text == null || text.trim().isEmpty()) {
|
||||
setValue(new ArrayList<>());
|
||||
return;
|
||||
}
|
||||
try {
|
||||
JavaType listType =
|
||||
objectMapper
|
||||
.getTypeFactory()
|
||||
.constructCollectionType(ArrayList.class, elementType);
|
||||
List<T> list = objectMapper.readValue(text, listType);
|
||||
setValue(list);
|
||||
} catch (Exception e) {
|
||||
log.error("Exception while converting {}", e);
|
||||
throw new IllegalArgumentException(
|
||||
"Failed to convert java.lang.String to java.util.List");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package stirling.software.SPDF.pdf.parser;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Unit tests for {@link PageColumnLayout} gutter detection and column classification. */
|
||||
class PageColumnLayoutTest {
|
||||
|
||||
private static final float PAGE_WIDTH = 612f; // Letter portrait
|
||||
|
||||
// ── single-column ────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void singleColumn_oneColumnNoGutters() {
|
||||
List<float[]> lines = List.of(lineBox(72f, 396f));
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
assertThat(layout.gutters()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void singleColumn_classifyAnchor_returnsZero() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(List.of(lineBox(72f, 396f)), PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnOf(100f, 200f)).isEqualTo(0);
|
||||
}
|
||||
|
||||
// ── two-column ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void twoColumn_detectsGutter() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(2);
|
||||
assertThat(layout.gutters()).hasSize(1);
|
||||
float[] gutter = layout.gutters().get(0);
|
||||
// Gutter is centered on pageWidth/2 with PageColumnLayout.MIDPOINT_SLACK_PT slack each
|
||||
// side.
|
||||
float pageMid = PAGE_WIDTH / 2f;
|
||||
assertThat(gutter[0]).isBetween(pageMid - 40f, pageMid - 20f);
|
||||
assertThat(gutter[1]).isBetween(pageMid + 20f, pageMid + 40f);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_classifyLeftAndRightAnchors() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
assertThat(layout.columnOf(100f, 200f)).isEqualTo(0);
|
||||
assertThat(layout.columnOf(380f, 460f)).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_columnsCrossing_leftLineOnlyHitsLeft() {
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(buildTwoColumnLines(3), PAGE_WIDTH);
|
||||
assertThat(layout.columnsCrossing(72f, 280f)).containsExactly(0);
|
||||
assertThat(layout.columnsCrossing(320f, 540f)).containsExactly(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumn_spanningLine_returnsBothColumns() {
|
||||
List<float[]> lines = new ArrayList<>(buildTwoColumnLines(3));
|
||||
// Full-width header that crosses pageWidth/2.
|
||||
lines.add(lineBox(72f, 396f));
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnsCrossing(72f, 540f)).containsExactly(0, 1);
|
||||
assertThat(layout.columnsCrossing(72f, 280f)).containsExactly(0);
|
||||
}
|
||||
|
||||
private static List<float[]> buildTwoColumnLines(int rowsPerColumn) {
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
for (int i = 0; i < rowsPerColumn; i++) {
|
||||
lines.add(lineBox(72f, 136f)); // left column body (72..208)
|
||||
lines.add(lineBox(320f, 220f)); // right column body (320..540)
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// ── three-column ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void threeColumn_collapsesToLeftRightSplit() {
|
||||
// The midpoint-based detector splits the page at pageWidth/2 and treats anything else as
|
||||
// single-column or spanning. A genuine 3-column layout collapses to 2 columns; the middle
|
||||
// column's content ends up classified by midpoint as left or right of pageMid.
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
for (int i = 0; i < 6; i++) {
|
||||
lines.add(lineBox(72f, 150f)); // 72..222
|
||||
lines.add(lineBox(252f, 150f)); // 252..402
|
||||
lines.add(lineBox(432f, 150f)); // 432..582
|
||||
}
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
|
||||
assertThat(layout.columnCount()).isEqualTo(2);
|
||||
assertThat(layout.gutters()).hasSize(1);
|
||||
}
|
||||
|
||||
// ── empty page ───────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void emptyPage_singleColumnFallback() {
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(List.of(), PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
assertThat(layout.gutters()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void onlyShortFragments_singleColumnFallback() {
|
||||
// Page numbers / decorations — too narrow to vote either side.
|
||||
List<float[]> lines = List.of(lineBox(300f, 6f));
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
}
|
||||
|
||||
// ── narrow gap should not be confused for a gutter ───────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void narrowInternalGap_doesNotProduceGutter() {
|
||||
// Both halves sit left of the page midpoint, so no line votes for a right column and
|
||||
// detection falls back to single-column.
|
||||
List<float[]> lines = new ArrayList<>();
|
||||
lines.add(lineBox(72f, 100f));
|
||||
lines.add(lineBox(180f, 100f));
|
||||
for (int i = 0; i < 5; i++) {
|
||||
lines.add(lineBox(72f, 208f));
|
||||
}
|
||||
|
||||
PageColumnLayout layout = PageColumnLayout.fromLineBoxes(lines, PAGE_WIDTH);
|
||||
assertThat(layout.columnCount()).isEqualTo(1);
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Builds a line bounding box {@code [x1, 0, x1+width, 0]}; Y is unused by detection. */
|
||||
private static float[] lineBox(float x1, float width) {
|
||||
return new float[] {x1, 0f, x1 + width, 0f};
|
||||
}
|
||||
}
|
||||
-163
@@ -1,163 +0,0 @@
|
||||
package stirling.software.common.util.propertyeditor;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
|
||||
class StringToArrayListPropertyEditorTest {
|
||||
|
||||
private StringToArrayListPropertyEditor<RedactionArea> editor;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
editor = new StringToArrayListPropertyEditor<>(RedactionArea.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_ValidJson() {
|
||||
// Arrange
|
||||
String json =
|
||||
"[{\"x\":10.5,\"y\":20.5,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"}]";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(1, list.size(), "List should have 1 entry");
|
||||
|
||||
RedactionArea area = list.get(0);
|
||||
assertEquals(10.5, area.getX(), "X should be 10.5");
|
||||
assertEquals(20.5, area.getY(), "Y should be 20.5");
|
||||
assertEquals(100.0, area.getWidth(), "Width should be 100.0");
|
||||
assertEquals(50.0, area.getHeight(), "Height should be 50.0");
|
||||
assertEquals(1, area.getPage(), "Page should be 1");
|
||||
assertEquals("#FF0000", area.getColor(), "Color should be #FF0000");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_MultipleItems() {
|
||||
// Arrange
|
||||
String json =
|
||||
"["
|
||||
+ "{\"x\":10.0,\"y\":20.0,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"},"
|
||||
+ "{\"x\":30.0,\"y\":40.0,\"width\":200.0,\"height\":150.0,\"page\":2,\"color\":\"#00FF00\"}"
|
||||
+ "]";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(2, list.size(), "List should have 2 entries");
|
||||
|
||||
RedactionArea area1 = list.get(0);
|
||||
assertEquals(10.0, area1.getX(), "X should be 10.0");
|
||||
assertEquals(20.0, area1.getY(), "Y should be 20.0");
|
||||
assertEquals(1, area1.getPage(), "Page should be 1");
|
||||
|
||||
RedactionArea area2 = list.get(1);
|
||||
assertEquals(30.0, area2.getX(), "X should be 30.0");
|
||||
assertEquals(40.0, area2.getY(), "Y should be 40.0");
|
||||
assertEquals(2, area2.getPage(), "Page should be 2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_EmptyString() {
|
||||
// Arrange
|
||||
String json = "";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertTrue(list.isEmpty(), "List should be empty");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_NullString() {
|
||||
// Act
|
||||
editor.setAsText(null);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertTrue(list.isEmpty(), "List should be empty");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_SingleItemAsArray() {
|
||||
// Arrange - note this is a single object, not an array
|
||||
String json =
|
||||
"{\"x\":10.0,\"y\":20.0,\"width\":100.0,\"height\":50.0,\"page\":1,\"color\":\"#FF0000\"}";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(1, list.size(), "List should have 1 entry");
|
||||
|
||||
RedactionArea area = list.get(0);
|
||||
assertEquals(10.0, area.getX(), "X should be 10.0");
|
||||
assertEquals(20.0, area.getY(), "Y should be 20.0");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_InvalidJson() {
|
||||
// Arrange
|
||||
String json = "invalid json";
|
||||
|
||||
// Act & Assert
|
||||
assertThrows(IllegalArgumentException.class, () -> editor.setAsText(json));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetAsText_UnknownProperties() {
|
||||
// Arrange - this JSON contains properties not in RedactionArea
|
||||
// With FAIL_ON_UNKNOWN_PROPERTIES disabled, this should ignore the unknown properties
|
||||
String json = "[{\"invalid\":\"structure\"}]";
|
||||
|
||||
// Act
|
||||
editor.setAsText(json);
|
||||
Object value = editor.getValue();
|
||||
|
||||
// Assert
|
||||
assertNotNull(value, "Value should not be null");
|
||||
assertInstanceOf(List.class, value, "Value should be a List");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RedactionArea> list = (List<RedactionArea>) value;
|
||||
assertEquals(1, list.size(), "List should have 1 entry (empty object)");
|
||||
}
|
||||
}
|
||||
+4
-2
@@ -39,7 +39,9 @@ import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.WebResponseUtils;
|
||||
import stirling.software.common.util.propertyeditor.StringToArrayListPropertyEditor;
|
||||
import stirling.software.common.util.propertyeditor.JsonListPropertyEditor;
|
||||
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
|
||||
/**
|
||||
* Find/replace text editing for PDFs. Round-trips through {@link PdfJsonConversionService}: the
|
||||
@@ -73,7 +75,7 @@ public class EditTextController {
|
||||
binder.registerCustomEditor(
|
||||
List.class,
|
||||
"edits",
|
||||
new StringToArrayListPropertyEditor<>(EditTextOperation.class));
|
||||
new JsonListPropertyEditor<>(new TypeReference<List<EditTextOperation>>() {}));
|
||||
}
|
||||
|
||||
@AutoJobPostMapping(
|
||||
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
/**
|
||||
* PDFTextStripper subclass that collects all text positions and groups them into line-level
|
||||
* bounding boxes.
|
||||
*
|
||||
* <p>Two outputs are maintained in parallel:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@link #getLineBoxes()} returns {@code [x1, pdfYbottom, x2, pdfYtop]} in PDF user-space
|
||||
* (origin bottom-left, Y up). This is what existing callers expect.
|
||||
* <li>{@link #getScreenLineBoxes()} returns {@code [x1, screenYtop, x2, screenYbottom]} computed
|
||||
* directly from glyph positions without a PDF↔screen round-trip — used by column-aware
|
||||
* redaction where ulp-level drift in the round-trip caused false rejects against anchors.
|
||||
* </ul>
|
||||
*
|
||||
* <p>Lines are flushed not only on Y jumps but also on large X gaps within the same Y row. That way
|
||||
* left-column glyphs and right-column glyphs that happen to share a baseline (common in IEEE
|
||||
* conference templates) get emitted as two distinct line boxes instead of one wide merged box.
|
||||
*/
|
||||
final class AllTextLineExtractor extends PDFTextStripper {
|
||||
|
||||
/** Min vertical jump (screen Y) before the next glyph is treated as a new line. */
|
||||
private static final float LINE_Y_TOLERANCE = 3.0f;
|
||||
|
||||
/**
|
||||
* Min horizontal gap (screen X) between consecutive glyphs on the same Y that indicates a
|
||||
* column boundary. Chosen large enough to not split normal inter-word spacing (~6–10pt for 11pt
|
||||
* text) but small enough to catch standard column gutters (typically ≥15pt).
|
||||
*/
|
||||
private static final float COLUMN_GAP_X = 14f;
|
||||
|
||||
private final float pageHeight;
|
||||
private final List<float[]> lineBoxes = new ArrayList<>();
|
||||
private final List<float[]> screenLineBoxes = new ArrayList<>();
|
||||
|
||||
private final List<TextPosition> currentLine = new ArrayList<>();
|
||||
private float lastScreenY = Float.NaN;
|
||||
private float lastGlyphRight = Float.NaN;
|
||||
|
||||
AllTextLineExtractor(int pageNumber, float pageHeight) throws IOException {
|
||||
this.pageHeight = pageHeight;
|
||||
setStartPage(pageNumber);
|
||||
setEndPage(pageNumber);
|
||||
setSortByPosition(true);
|
||||
}
|
||||
|
||||
List<float[]> getLineBoxes() {
|
||||
return lineBoxes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns line boxes as {@code [x1, screenYtop, x2, screenYbottom]}. {@code screenYtop} is the
|
||||
* minimum {@code TextPosition.getY() - getHeight()} on the line and {@code screenYbottom} is
|
||||
* the maximum {@code getY()} (the line's baseline). These values come straight from PDFBox
|
||||
* without going through {@code pageHeight - …}, so they're stable for ulp-sensitive comparisons
|
||||
* against anchor screen Ys.
|
||||
*/
|
||||
List<float[]> getScreenLineBoxes() {
|
||||
return screenLineBoxes;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> positions) {
|
||||
for (TextPosition tp : positions) {
|
||||
// Skip whitespace-only positions (spaces, newline markers, indent characters).
|
||||
// These have a TextPosition but no visible glyph; including them causes
|
||||
// space-only "lines" to produce degenerate segments that appear as thin
|
||||
// black bars after redaction.
|
||||
String unicode = tp.getUnicode();
|
||||
if (unicode == null || unicode.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
float screenY = tp.getY();
|
||||
float screenX = tp.getX();
|
||||
boolean yJump =
|
||||
!Float.isNaN(lastScreenY) && Math.abs(screenY - lastScreenY) > LINE_Y_TOLERANCE;
|
||||
boolean xJump =
|
||||
!Float.isNaN(lastGlyphRight) && (screenX - lastGlyphRight) > COLUMN_GAP_X;
|
||||
if (yJump || xJump) {
|
||||
flushLine();
|
||||
}
|
||||
lastScreenY = screenY;
|
||||
lastGlyphRight = screenX + tp.getWidth();
|
||||
currentLine.add(tp);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void endPage(PDPage page) throws IOException {
|
||||
flushLine();
|
||||
super.endPage(page);
|
||||
}
|
||||
|
||||
private void flushLine() {
|
||||
if (currentLine.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
float minX = Float.MAX_VALUE, maxX = -Float.MAX_VALUE;
|
||||
float minScreenY = Float.MAX_VALUE, maxScreenY = -Float.MAX_VALUE;
|
||||
for (TextPosition tp : currentLine) {
|
||||
minX = Math.min(minX, tp.getX());
|
||||
maxX = Math.max(maxX, tp.getX() + tp.getWidth());
|
||||
minScreenY = Math.min(minScreenY, tp.getY() - tp.getHeight());
|
||||
maxScreenY = Math.max(maxScreenY, tp.getY());
|
||||
}
|
||||
emitSegment(minX, maxX, minScreenY, maxScreenY);
|
||||
currentLine.clear();
|
||||
lastScreenY = Float.NaN;
|
||||
lastGlyphRight = Float.NaN;
|
||||
}
|
||||
|
||||
private void emitSegment(float minX, float maxX, float minScreenY, float maxScreenY) {
|
||||
float pdfY1 = pageHeight - maxScreenY; // bottom in PDF coords
|
||||
float pdfY2 = pageHeight - minScreenY; // top in PDF coords
|
||||
lineBoxes.add(new float[] {minX, pdfY1, maxX, pdfY2});
|
||||
screenLineBoxes.add(new float[] {minX, minScreenY, maxX, maxScreenY});
|
||||
}
|
||||
}
|
||||
+403
@@ -0,0 +1,403 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDPageTree;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.model.api.security.ManualRedactPdfRequest;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.common.model.api.security.RedactionArea;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
class ManualRedactionService {
|
||||
|
||||
private static final float DEFAULT_TEXT_PADDING_MULTIPLIER = 0.6f;
|
||||
private static final float REDACTION_WIDTH_REDUCTION_FACTOR = 0.9f;
|
||||
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Area and page redaction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactAreas(List<RedactionArea> redactionAreas, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
if (redactionAreas == null || redactionAreas.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<Integer, List<RedactionArea>> redactionsByPage = new HashMap<>();
|
||||
|
||||
for (RedactionArea redactionArea : redactionAreas) {
|
||||
if (redactionArea.getPage() == null
|
||||
|| redactionArea.getPage() <= 0
|
||||
|| redactionArea.getHeight() == null
|
||||
|| redactionArea.getHeight() <= 0.0D
|
||||
|| redactionArea.getWidth() == null
|
||||
|| redactionArea.getWidth() <= 0.0D) {
|
||||
continue;
|
||||
}
|
||||
|
||||
redactionsByPage
|
||||
.computeIfAbsent(redactionArea.getPage(), k -> new ArrayList<>())
|
||||
.add(redactionArea);
|
||||
}
|
||||
|
||||
for (Map.Entry<Integer, List<RedactionArea>> entry : redactionsByPage.entrySet()) {
|
||||
Integer pageNumber = entry.getKey();
|
||||
List<RedactionArea> areasForPage = entry.getValue();
|
||||
|
||||
if (pageNumber > allPages.getCount()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
PDPage page = allPages.get(pageNumber - 1);
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
|
||||
contentStream.saveGraphicsState();
|
||||
for (RedactionArea redactionArea : areasForPage) {
|
||||
Color redactColor = decodeOrDefault(redactionArea.getColor());
|
||||
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
float x = redactionArea.getX().floatValue();
|
||||
float y = redactionArea.getY().floatValue();
|
||||
float width = redactionArea.getWidth().floatValue();
|
||||
float height = redactionArea.getHeight().floatValue();
|
||||
|
||||
float pdfY = page.getBBox().getHeight() - y - height;
|
||||
|
||||
contentStream.addRect(x, pdfY, width, height);
|
||||
contentStream.fill();
|
||||
}
|
||||
contentStream.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void redactPages(ManualRedactPdfRequest request, PDDocument document, PDPageTree allPages)
|
||||
throws IOException {
|
||||
|
||||
Color redactColor = decodeOrDefault(request.getPageRedactionColor());
|
||||
List<Integer> pageNumbers = getPageNumbers(request, allPages.getCount());
|
||||
|
||||
for (Integer pageNumber : pageNumbers) {
|
||||
PDPage page = allPages.get(pageNumber);
|
||||
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
|
||||
PDRectangle box = page.getBBox();
|
||||
contentStream.addRect(0, 0, box.getWidth(), box.getHeight());
|
||||
contentStream.fill();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Overlay drawing
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
void redactFoundText(
|
||||
PDDocument document,
|
||||
List<PDFText> blocks,
|
||||
float customPadding,
|
||||
Color redactColor,
|
||||
boolean isTextRemovalMode)
|
||||
throws IOException {
|
||||
|
||||
var allPages = document.getDocumentCatalog().getPages();
|
||||
|
||||
Map<Integer, List<PDFText>> blocksByPage = new HashMap<>();
|
||||
for (PDFText block : blocks) {
|
||||
blocksByPage.computeIfAbsent(block.getPageIndex(), k -> new ArrayList<>()).add(block);
|
||||
}
|
||||
|
||||
for (Map.Entry<Integer, List<PDFText>> entry : blocksByPage.entrySet()) {
|
||||
Integer pageIndex = entry.getKey();
|
||||
List<PDFText> pageBlocks = entry.getValue();
|
||||
|
||||
if (pageIndex >= allPages.getCount()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
var page = allPages.get(pageIndex);
|
||||
try (PDPageContentStream contentStream =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
|
||||
contentStream.saveGraphicsState();
|
||||
|
||||
try {
|
||||
contentStream.setNonStrokingColor(redactColor);
|
||||
PDRectangle pageBox = page.getBBox();
|
||||
|
||||
for (PDFText block : pageBlocks) {
|
||||
float padding =
|
||||
(block.getY2() - block.getY1()) * DEFAULT_TEXT_PADDING_MULTIPLIER
|
||||
+ customPadding;
|
||||
|
||||
float originalWidth = block.getX2() - block.getX1();
|
||||
float boxWidth;
|
||||
float boxX;
|
||||
|
||||
if (isTextRemovalMode) {
|
||||
boxWidth = originalWidth * REDACTION_WIDTH_REDUCTION_FACTOR;
|
||||
float widthReduction = originalWidth - boxWidth;
|
||||
boxX = block.getX1() + (widthReduction / 2);
|
||||
} else {
|
||||
boxWidth = originalWidth;
|
||||
boxX = block.getX1();
|
||||
}
|
||||
|
||||
contentStream.addRect(
|
||||
boxX,
|
||||
pageBox.getHeight() - block.getY2() - padding,
|
||||
boxWidth,
|
||||
block.getY2() - block.getY1() + 2 * padding);
|
||||
}
|
||||
|
||||
contentStream.fill();
|
||||
|
||||
} finally {
|
||||
contentStream.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
|
||||
// Remove annotations whose bounding rect overlaps a redacted block, to prevent
|
||||
// users from hovering over redacted URLs and seeing the underlying destination.
|
||||
try {
|
||||
float pageH = page.getBBox().getHeight();
|
||||
List<PDAnnotation> kept = new ArrayList<>();
|
||||
for (PDAnnotation ann : page.getAnnotations()) {
|
||||
PDRectangle ar = ann.getRectangle();
|
||||
boolean overlaps = false;
|
||||
if (ar != null) {
|
||||
for (PDFText block : pageBlocks) {
|
||||
float padding =
|
||||
(block.getY2() - block.getY1())
|
||||
* DEFAULT_TEXT_PADDING_MULTIPLIER
|
||||
+ customPadding;
|
||||
float bx1 = block.getX1();
|
||||
float bx2 = block.getX2();
|
||||
float by1 = pageH - block.getY2() - padding;
|
||||
float by2 = pageH - block.getY1() + padding;
|
||||
if (ar.getLowerLeftX() < bx2
|
||||
&& ar.getUpperRightX() > bx1
|
||||
&& ar.getLowerLeftY() < by2
|
||||
&& ar.getUpperRightY() > by1) {
|
||||
overlaps = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!overlaps) {
|
||||
kept.add(ann);
|
||||
}
|
||||
}
|
||||
page.setAnnotations(kept);
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"[redact] could not remove annotations on page {}: {}",
|
||||
pageIndex,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void redactImageBoxes(PDDocument document, List<float[]> imageBoxes, Color color)
|
||||
throws IOException {
|
||||
Map<Integer, List<float[]>> byPage = new HashMap<>();
|
||||
for (float[] box : imageBoxes) {
|
||||
byPage.computeIfAbsent((int) box[0], k -> new ArrayList<>()).add(box);
|
||||
}
|
||||
PDPageTree pages = document.getDocumentCatalog().getPages();
|
||||
for (Map.Entry<Integer, List<float[]>> entry : byPage.entrySet()) {
|
||||
int pageIdx = entry.getKey();
|
||||
if (pageIdx < 0 || pageIdx >= pages.getCount()) {
|
||||
log.warn("[redact/execute] image box references out-of-range page {}", pageIdx);
|
||||
continue;
|
||||
}
|
||||
PDPage page = pages.get(pageIdx);
|
||||
try (PDPageContentStream cs =
|
||||
new PDPageContentStream(
|
||||
document, page, PDPageContentStream.AppendMode.APPEND, true, true)) {
|
||||
cs.saveGraphicsState();
|
||||
cs.setNonStrokingColor(color);
|
||||
for (float[] box : entry.getValue()) {
|
||||
float x1 = box[1], y1 = box[2], x2 = box[3], y2 = box[4];
|
||||
cs.addRect(x1, y1, x2 - x1, y2 - y1);
|
||||
}
|
||||
cs.fill();
|
||||
cs.restoreGraphicsState();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Page element extraction
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Returns bounding boxes for every text line and image on {@code page} in PDF user-space
|
||||
* coordinates: {@code [x1, y1, x2, y2]} (origin bottom-left, Y increases upward).
|
||||
*/
|
||||
List<float[]> extractPageElementBoxes(PDDocument document, PDPage page, int pageIndex)
|
||||
throws IOException {
|
||||
List<float[]> boxes = new ArrayList<>();
|
||||
|
||||
AllTextLineExtractor textExtractor =
|
||||
new AllTextLineExtractor(pageIndex + 1, page.getBBox().getHeight());
|
||||
textExtractor.getText(document);
|
||||
boxes.addAll(textExtractor.getLineBoxes());
|
||||
|
||||
PageImageLocator imgLocator = new PageImageLocator(page, pageIndex);
|
||||
imgLocator.processPage(page);
|
||||
for (PageImageLocator.ImageBox imgBox : imgLocator.getImageBoxes()) {
|
||||
boxes.add(new float[] {imgBox.x1(), imgBox.y1(), imgBox.x2(), imgBox.y2()});
|
||||
}
|
||||
|
||||
return boxes;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Finalization
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
TempFile finalizeRedaction(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDFText>> allFoundTextsByPage,
|
||||
String colorString,
|
||||
float customPadding,
|
||||
Boolean convertToImage,
|
||||
boolean isTextRemovalMode)
|
||||
throws IOException {
|
||||
|
||||
List<PDFText> allFoundTexts = new ArrayList<>();
|
||||
for (List<PDFText> pageTexts : allFoundTextsByPage.values()) {
|
||||
allFoundTexts.addAll(pageTexts);
|
||||
}
|
||||
|
||||
if (!allFoundTexts.isEmpty()) {
|
||||
Color redactColor = decodeOrDefault(colorString);
|
||||
redactFoundText(document, allFoundTexts, customPadding, redactColor, isTextRemovalMode);
|
||||
cleanDocumentMetadata(document);
|
||||
}
|
||||
|
||||
if (Boolean.TRUE.equals(convertToImage)) {
|
||||
try (PDDocument convertedPdf = PdfUtils.convertPdfToPdfImage(document)) {
|
||||
cleanDocumentMetadata(convertedPdf);
|
||||
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
convertedPdf.save(tempOut.getFile());
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized (image mode): {} pages ➜ {} KB",
|
||||
convertedPdf.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
return tempOut;
|
||||
}
|
||||
}
|
||||
|
||||
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
|
||||
try {
|
||||
document.save(tempOut.getFile());
|
||||
} catch (IOException e) {
|
||||
tempOut.close();
|
||||
throw e;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Redaction finalized: {} pages ➜ {} KB",
|
||||
document.getNumberOfPages(),
|
||||
tempOut.getFile().length() / 1024);
|
||||
|
||||
return tempOut;
|
||||
}
|
||||
|
||||
private void cleanDocumentMetadata(PDDocument document) {
|
||||
try {
|
||||
var documentInfo = document.getDocumentInformation();
|
||||
if (documentInfo != null) {
|
||||
documentInfo.setAuthor(null);
|
||||
documentInfo.setSubject(null);
|
||||
documentInfo.setKeywords(null);
|
||||
documentInfo.setModificationDate(java.util.Calendar.getInstance());
|
||||
log.debug("Cleaned document metadata for security");
|
||||
}
|
||||
|
||||
if (document.getDocumentCatalog() != null) {
|
||||
try {
|
||||
document.getDocumentCatalog().setMetadata(null);
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not clear XMP metadata: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to clean document metadata: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Utilities
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
static Color decodeOrDefault(String hex) {
|
||||
if (hex == null) {
|
||||
return Color.BLACK;
|
||||
}
|
||||
|
||||
String colorString = hex.startsWith("#") ? hex : "#" + hex;
|
||||
|
||||
try {
|
||||
return Color.decode(colorString);
|
||||
} catch (NumberFormatException e) {
|
||||
return Color.BLACK;
|
||||
}
|
||||
}
|
||||
|
||||
private List<Integer> getPageNumbers(ManualRedactPdfRequest request, int pagesCount) {
|
||||
String pageNumbersInput = request.getPageNumbers();
|
||||
String[] parsedPageNumbers =
|
||||
pageNumbersInput != null ? pageNumbersInput.split(",") : new String[0];
|
||||
List<Integer> pageNumbers =
|
||||
GeneralUtils.parsePageList(parsedPageNumbers, pagesCount, false);
|
||||
Collections.sort(pageNumbers);
|
||||
return pageNumbers;
|
||||
}
|
||||
}
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
|
||||
/**
|
||||
* Scans a PDF document once and matches all provided patterns in a single pass, collecting
|
||||
* bounding-box positions for every match. Use in place of creating one {@code TextFinder} per
|
||||
* search term to avoid O(n) full-document scans.
|
||||
*/
|
||||
@Slf4j
|
||||
final class MultiPatternTextFinder extends PDFTextStripper {
|
||||
|
||||
private static final long REGEX_MATCH_TIMEOUT_SECONDS = 30;
|
||||
private static final ExecutorService REGEX_EXECUTOR =
|
||||
Executors.newVirtualThreadPerTaskExecutor();
|
||||
|
||||
private final List<Pattern> patterns;
|
||||
private final Map<Integer, List<PDFText>> foundTextsByPage = new HashMap<>();
|
||||
|
||||
private final List<TextPosition> pageTextPositions = new ArrayList<>();
|
||||
private final StringBuilder pageTextBuilder = new StringBuilder();
|
||||
|
||||
MultiPatternTextFinder(List<Pattern> patterns) throws IOException {
|
||||
this.patterns = patterns;
|
||||
this.setWordSeparator(" ");
|
||||
this.setLineSeparator("\n");
|
||||
}
|
||||
|
||||
Map<Integer, List<PDFText>> getFoundTextsByPage() {
|
||||
return foundTextsByPage;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startPage(PDPage page) throws IOException {
|
||||
super.startPage(page);
|
||||
pageTextPositions.clear();
|
||||
pageTextBuilder.setLength(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> textPositions) {
|
||||
pageTextBuilder.append(text);
|
||||
pageTextPositions.addAll(textPositions);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeWordSeparator() {
|
||||
pageTextBuilder.append(getWordSeparator());
|
||||
pageTextPositions.add(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeLineSeparator() {
|
||||
pageTextBuilder.append(getLineSeparator());
|
||||
pageTextPositions.add(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void endPage(PDPage page) throws IOException {
|
||||
String text = pageTextBuilder.toString();
|
||||
if (!text.isEmpty()) {
|
||||
int pageIndex = getCurrentPageNo() - 1;
|
||||
for (Pattern pattern : patterns) {
|
||||
Matcher matcher = pattern.matcher(text);
|
||||
while (safeFind(matcher)) {
|
||||
PDFText pdfText = resolveMatchPosition(matcher, pageIndex);
|
||||
if (pdfText != null) {
|
||||
foundTextsByPage
|
||||
.computeIfAbsent(pageIndex, k -> new ArrayList<>())
|
||||
.add(pdfText);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
super.endPage(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a single {@code matcher.find()} call with a {@value #REGEX_MATCH_TIMEOUT_SECONDS}
|
||||
* second timeout. Prevents pathological regex backtracking from blocking the request
|
||||
* indefinitely; per-match timeout so fast legitimate scans are unaffected.
|
||||
*/
|
||||
private static boolean safeFind(Matcher matcher) throws IOException {
|
||||
Future<Boolean> future =
|
||||
REGEX_EXECUTOR.submit((java.util.concurrent.Callable<Boolean>) matcher::find);
|
||||
try {
|
||||
return future.get(REGEX_MATCH_TIMEOUT_SECONDS, TimeUnit.SECONDS);
|
||||
} catch (TimeoutException e) {
|
||||
future.cancel(true);
|
||||
throw new IOException(
|
||||
"Regex match timed out after "
|
||||
+ REGEX_MATCH_TIMEOUT_SECONDS
|
||||
+ "s — pattern may cause catastrophic backtracking");
|
||||
} catch (InterruptedException e) {
|
||||
future.cancel(true);
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IOException("Regex match interrupted", e);
|
||||
} catch (ExecutionException e) {
|
||||
Throwable cause = e.getCause();
|
||||
if (cause instanceof IOException ioEx) throw ioEx;
|
||||
throw new IOException("Regex match failed: " + cause.getMessage(), cause);
|
||||
}
|
||||
}
|
||||
|
||||
private PDFText resolveMatchPosition(Matcher matcher, int pageIndex) {
|
||||
int matchStart = matcher.start();
|
||||
int matchEnd = matcher.end();
|
||||
|
||||
float minX = Float.MAX_VALUE;
|
||||
float minY = Float.MAX_VALUE;
|
||||
float maxX = Float.MIN_VALUE;
|
||||
float maxY = Float.MIN_VALUE;
|
||||
boolean foundPosition = false;
|
||||
|
||||
for (int i = matchStart; i < matchEnd; i++) {
|
||||
if (i >= pageTextPositions.size()) break;
|
||||
TextPosition pos = pageTextPositions.get(i);
|
||||
if (pos != null) {
|
||||
foundPosition = true;
|
||||
minX = Math.min(minX, pos.getX());
|
||||
maxX = Math.max(maxX, pos.getX() + pos.getWidth());
|
||||
minY = Math.min(minY, pos.getY() - pos.getHeight());
|
||||
maxY = Math.max(maxY, pos.getY());
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundPosition && matchStart < pageTextPositions.size()) {
|
||||
for (int i = Math.max(0, matchStart - 5);
|
||||
i < Math.min(pageTextPositions.size(), matchEnd + 5);
|
||||
i++) {
|
||||
TextPosition pos = pageTextPositions.get(i);
|
||||
if (pos != null) {
|
||||
foundPosition = true;
|
||||
minX = Math.min(minX, pos.getX());
|
||||
maxX = Math.max(maxX, pos.getX() + pos.getWidth());
|
||||
minY = Math.min(minY, pos.getY() - pos.getHeight());
|
||||
maxY = Math.max(maxY, pos.getY());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundPosition) {
|
||||
log.warn(
|
||||
"Found text match '{}' but no valid position data at {}-{}",
|
||||
matcher.group(),
|
||||
matchStart,
|
||||
matchEnd);
|
||||
return null;
|
||||
}
|
||||
|
||||
return new PDFText(pageIndex, minX, minY, maxX, maxY, matcher.group());
|
||||
}
|
||||
}
|
||||
+67
-1477
File diff suppressed because it is too large
Load Diff
+850
@@ -0,0 +1,850 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.PDPageTree;
|
||||
import org.apache.pdfbox.pdmodel.PDResources;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.ImageBox;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.RedactStyle;
|
||||
import stirling.software.SPDF.model.api.security.RedactExecuteRequest.TextRange;
|
||||
import stirling.software.SPDF.pdf.parser.PageColumnLayout;
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.common.service.CustomPDFDocumentFactory;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.TempFile;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
class RedactExecuteService {
|
||||
|
||||
private final CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
private final ManualRedactionService manualRedactionService;
|
||||
private final TextRedactionService textRedactionService;
|
||||
|
||||
TempFile execute(RedactExecuteRequest request) throws IOException {
|
||||
RedactStyle style = request.getStyle() != null ? request.getStyle() : new RedactStyle();
|
||||
List<String> textValues = orEmpty(request.getTextValues());
|
||||
List<String> regexPatterns = orEmpty(request.getRegexPatterns());
|
||||
List<Integer> wipePages = orEmpty(request.getWipePages());
|
||||
List<TextRange> ranges = orEmpty(request.getRanges());
|
||||
List<ImageBox> imageBoxes = orEmpty(request.getImageBoxes());
|
||||
|
||||
boolean hasTargets =
|
||||
!textValues.isEmpty()
|
||||
|| !regexPatterns.isEmpty()
|
||||
|| !wipePages.isEmpty()
|
||||
|| !ranges.isEmpty()
|
||||
|| !imageBoxes.isEmpty()
|
||||
|| request.getRedactImagePages() != null;
|
||||
|
||||
if (!hasTargets) {
|
||||
throw ExceptionUtils.createIllegalArgumentException(
|
||||
"error.redaction.no.targets", "No redaction targets provided");
|
||||
}
|
||||
|
||||
boolean overlayOnly =
|
||||
RedactExecuteRequest.RedactionStrategy.OVERLAY_ONLY.equals(style.getStrategy());
|
||||
boolean imageFinalize =
|
||||
RedactExecuteRequest.RedactionStrategy.IMAGE_FINALIZE.equals(style.getStrategy());
|
||||
boolean convertToImage = imageFinalize || style.isConvertToImage();
|
||||
|
||||
boolean hasTextOps = !textValues.isEmpty() || !regexPatterns.isEmpty();
|
||||
|
||||
log.info(
|
||||
"[redact/execute] strategy={} textValues={} regexPatterns={} wipePages={} ranges={} imageBoxes={} imagePages={}",
|
||||
style.getStrategy(),
|
||||
textValues.size(),
|
||||
regexPatterns.size(),
|
||||
wipePages.size(),
|
||||
ranges.size(),
|
||||
imageBoxes.size(),
|
||||
request.getRedactImagePages());
|
||||
|
||||
if (request.getFileInput() == null) {
|
||||
throw ExceptionUtils.createFileNullOrEmptyException();
|
||||
}
|
||||
|
||||
PDDocument document = null;
|
||||
try {
|
||||
document = pdfDocumentFactory.load(request.getFileInput());
|
||||
|
||||
// Single-pass text scan: collect all text-based targets so we run the PDF
|
||||
// stripper only once across the entire execute() call rather than once per target.
|
||||
Map<Integer, List<PDFText>> foundTexts =
|
||||
hasTextOps ? collectTextMatches(document, request) : new HashMap<>();
|
||||
|
||||
int totalMatches = foundTexts.values().stream().mapToInt(List::size).sum();
|
||||
log.info(
|
||||
"[redact/execute] scan complete: {} text matches across {} pages",
|
||||
totalMatches,
|
||||
foundTexts.size());
|
||||
|
||||
// Text removal (content-stream rewriting) — skipped in overlay-only mode.
|
||||
boolean needsOverlayOnly = overlayOnly;
|
||||
if (hasTextOps && !foundTexts.isEmpty() && !overlayOnly) {
|
||||
needsOverlayOnly = applyTextRemoval(document, request);
|
||||
} else if (overlayOnly) {
|
||||
log.info(
|
||||
"[redact/execute] overlay-only mode requested — skipping content-stream rewriting");
|
||||
}
|
||||
|
||||
// Reload fresh document on fallback so we overlay onto clean content.
|
||||
if (needsOverlayOnly && !foundTexts.isEmpty()) {
|
||||
log.info("[redact/execute] reloading document for clean overlay pass");
|
||||
document.close();
|
||||
document = pdfDocumentFactory.load(request.getFileInput());
|
||||
foundTexts.clear();
|
||||
if (hasTextOps) {
|
||||
foundTexts.putAll(collectTextMatches(document, request));
|
||||
}
|
||||
}
|
||||
|
||||
// Non-text operations.
|
||||
Map<Integer, PageColumnLayout> layoutCache = new HashMap<>();
|
||||
|
||||
if (!wipePages.isEmpty()) {
|
||||
applyPageWipe(document, wipePages, style);
|
||||
}
|
||||
|
||||
for (TextRange range : ranges) {
|
||||
applyRangeRedaction(document, range, style, layoutCache);
|
||||
}
|
||||
|
||||
for (ImageBox box : imageBoxes) {
|
||||
applyImageBoxRedaction(document, box, style);
|
||||
}
|
||||
|
||||
if (request.getRedactImagePages() != null) {
|
||||
applyAllImagesRedaction(document, request.getRedactImagePages(), style);
|
||||
}
|
||||
|
||||
return manualRedactionService.finalizeRedaction(
|
||||
document,
|
||||
foundTexts,
|
||||
style.getColor(),
|
||||
style.getPadding(),
|
||||
convertToImage,
|
||||
!needsOverlayOnly);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("Execute redaction failed: {}", e.getMessage(), e);
|
||||
throw new RuntimeException("Failed to perform PDF redaction: " + e.getMessage(), e);
|
||||
} finally {
|
||||
if (document != null) {
|
||||
try {
|
||||
document.close();
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to close document: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Single-pass text scan (one stripper pass per execute() call)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Runs a single PDF text-stripper pass over all text-based targets and returns the merged hit
|
||||
* map.
|
||||
*/
|
||||
private Map<Integer, List<PDFText>> collectTextMatches(
|
||||
PDDocument document, RedactExecuteRequest request) {
|
||||
Map<Integer, List<PDFText>> found = new HashMap<>();
|
||||
|
||||
String[] terms = cleanStrings(request.getTextValues());
|
||||
if (terms.length > 0) {
|
||||
textRedactionService
|
||||
.findTextToRedact(document, terms, false, false)
|
||||
.forEach(
|
||||
(page, hits) ->
|
||||
found.computeIfAbsent(page, k -> new ArrayList<>())
|
||||
.addAll(hits));
|
||||
}
|
||||
|
||||
String[] patterns = cleanStrings(request.getRegexPatterns());
|
||||
if (patterns.length > 0) {
|
||||
textRedactionService
|
||||
.findTextToRedact(document, patterns, true, false)
|
||||
.forEach(
|
||||
(page, hits) ->
|
||||
found.computeIfAbsent(page, k -> new ArrayList<>())
|
||||
.addAll(hits));
|
||||
}
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Text removal (content-stream rewriting)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Attempts content-stream text removal for all text/regex targets. Returns {@code true} if the
|
||||
* document fell back to overlay-only mode.
|
||||
*/
|
||||
private boolean applyTextRemoval(PDDocument document, RedactExecuteRequest request) {
|
||||
try {
|
||||
boolean fallback = false;
|
||||
|
||||
String[] terms = cleanStrings(request.getTextValues());
|
||||
if (terms.length > 0) {
|
||||
Map<Integer, List<PDFText>> exactFound =
|
||||
textRedactionService.findTextToRedact(document, terms, false, false);
|
||||
if (!exactFound.isEmpty()) {
|
||||
fallback |=
|
||||
textRedactionService.performTextReplacement(
|
||||
document, exactFound, terms, false, false);
|
||||
}
|
||||
}
|
||||
|
||||
String[] patterns = cleanStrings(request.getRegexPatterns());
|
||||
if (patterns.length > 0) {
|
||||
Map<Integer, List<PDFText>> regexFound =
|
||||
textRedactionService.findTextToRedact(document, patterns, true, false);
|
||||
if (!regexFound.isEmpty()) {
|
||||
fallback |=
|
||||
textRedactionService.performTextReplacement(
|
||||
document, regexFound, patterns, true, false);
|
||||
}
|
||||
}
|
||||
|
||||
if (fallback) {
|
||||
log.warn(
|
||||
"[redact/execute] font compatibility issue — falling back to overlay-only");
|
||||
} else {
|
||||
log.info("[redact/execute] content-stream text removal applied successfully");
|
||||
}
|
||||
return fallback;
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"[redact/execute] text removal failed, falling back to overlay: {}",
|
||||
e.getMessage());
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Per-operation dispatch methods
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private void applyPageWipe(PDDocument document, List<Integer> pageNumbers, RedactStyle style)
|
||||
throws IOException {
|
||||
List<Integer> pageIndices = toZeroBasedIndices(pageNumbers);
|
||||
if (pageIndices.isEmpty()) return;
|
||||
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
Color pageColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
Collections.sort(pageIndices);
|
||||
log.info("[redact/execute] full-page wipe: {} pages ({})", pageIndices.size(), pageIndices);
|
||||
|
||||
Map<Integer, List<float[]>> pageElementBoxes = new HashMap<>();
|
||||
for (Integer idx : pageIndices) {
|
||||
if (idx >= 0 && idx < allPages.getCount()) {
|
||||
try {
|
||||
pageElementBoxes.put(
|
||||
idx,
|
||||
manualRedactionService.extractPageElementBoxes(
|
||||
document, allPages.get(idx), idx));
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"[redact/execute] element extraction failed for page {}: {}",
|
||||
idx,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (Integer idx : pageIndices) {
|
||||
if (idx >= 0 && idx < allPages.getCount()) {
|
||||
PDPage page = allPages.get(idx);
|
||||
List<float[]> elementBoxes =
|
||||
pageElementBoxes.getOrDefault(idx, Collections.emptyList());
|
||||
page.getCOSObject().removeItem(COSName.CONTENTS);
|
||||
page.setResources(new PDResources());
|
||||
try (PDPageContentStream cs = new PDPageContentStream(document, page)) {
|
||||
cs.setNonStrokingColor(pageColor);
|
||||
if (elementBoxes.isEmpty()) {
|
||||
PDRectangle box = page.getBBox();
|
||||
cs.addRect(0, 0, box.getWidth(), box.getHeight());
|
||||
} else {
|
||||
log.info(
|
||||
"[redact/execute] page {}: drawing {} element boxes",
|
||||
idx + 1,
|
||||
elementBoxes.size());
|
||||
for (float[] r : elementBoxes) {
|
||||
cs.addRect(r[0], r[1], r[2] - r[0], r[3] - r[1]);
|
||||
}
|
||||
}
|
||||
cs.fill();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void applyRangeRedaction(
|
||||
PDDocument document,
|
||||
TextRange range,
|
||||
RedactStyle style,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
String rangeStart = trimOrEmpty(range.startString());
|
||||
String rangeEnd = trimOrEmpty(range.endString());
|
||||
log.info("[redact/execute] range redaction: start='{}' end='{}'", rangeStart, rangeEnd);
|
||||
try {
|
||||
List<PDFText> blocks = collectRangeBlocks(document, rangeStart, rangeEnd, layoutCache);
|
||||
if (!blocks.isEmpty()) {
|
||||
manualRedactionService.redactFoundText(
|
||||
document,
|
||||
blocks,
|
||||
style.getPadding(),
|
||||
ManualRedactionService.decodeOrDefault(style.getColor()),
|
||||
false);
|
||||
} else {
|
||||
log.warn(
|
||||
"[redact/execute] range not found: start='{}' end='{}'",
|
||||
rangeStart,
|
||||
rangeEnd);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("[redact/execute] range redaction failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void applyImageBoxRedaction(PDDocument document, ImageBox box, RedactStyle style)
|
||||
throws IOException {
|
||||
List<float[]> boxes =
|
||||
List.of(
|
||||
new float[] {
|
||||
(float) box.pageIndex(), box.x1(), box.y1(), box.x2(), box.y2()
|
||||
});
|
||||
log.info("[redact/execute] image box overlay on page {}", box.pageIndex());
|
||||
Color boxColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
manualRedactionService.redactImageBoxes(document, boxes, boxColor);
|
||||
}
|
||||
|
||||
private void applyAllImagesRedaction(
|
||||
PDDocument document, List<Integer> pageNumbers, RedactStyle style) throws IOException {
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
Color imgColor = ManualRedactionService.decodeOrDefault(style.getColor());
|
||||
|
||||
List<Integer> imagePageIndices = toZeroBasedIndices(pageNumbers);
|
||||
if (imagePageIndices.isEmpty()) {
|
||||
imagePageIndices = new ArrayList<>();
|
||||
for (int i = 0; i < allPages.getCount(); i++) {
|
||||
imagePageIndices.add(i);
|
||||
}
|
||||
}
|
||||
|
||||
List<float[]> detectedBoxes = new ArrayList<>();
|
||||
for (int pageIdx : imagePageIndices) {
|
||||
if (pageIdx < 0 || pageIdx >= allPages.getCount()) continue;
|
||||
try {
|
||||
PDPage page = allPages.get(pageIdx);
|
||||
PageImageLocator locator = new PageImageLocator(page, pageIdx);
|
||||
locator.processPage(page);
|
||||
for (PageImageLocator.ImageBox ib : locator.getImageBoxes()) {
|
||||
detectedBoxes.add(new float[] {pageIdx, ib.x1(), ib.y1(), ib.x2(), ib.y2()});
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"[redact/execute] image detection failed for page {}: {}",
|
||||
pageIdx + 1,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] auto image detection: {} images across {} pages",
|
||||
detectedBoxes.size(),
|
||||
imagePageIndices.size());
|
||||
|
||||
if (!detectedBoxes.isEmpty()) {
|
||||
manualRedactionService.redactImageBoxes(document, detectedBoxes, imgColor);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Range collection helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Locates {@code startStr} in the document and returns {@link PDFText} blocks for every text
|
||||
* line and image from that point up to (but NOT including) the line where {@code endStr}
|
||||
* begins. If {@code endStr} is blank, redacts from {@code startStr} to the end of the document.
|
||||
*
|
||||
* <p>Multi-column pages follow reading order: down the start column, jump to the top of the
|
||||
* next column, continue to the end anchor. Single-column pages reduce to a plain Y-band check.
|
||||
*/
|
||||
List<PDFText> collectRangeBlocks(
|
||||
PDDocument document,
|
||||
String startStr,
|
||||
String endStr,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
|
||||
PDPageTree allPages = document.getDocumentCatalog().getPages();
|
||||
int totalPages = allPages.getCount();
|
||||
|
||||
Map<Integer, List<PDFText>> startMatchesByPage = findWithFallbacks(document, startStr);
|
||||
if (startMatchesByPage.isEmpty()) {
|
||||
log.warn("[redact/execute] range start not found: '{}'", startStr);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
List<Anchor> starts = toAnchors(document, startMatchesByPage, layoutCache);
|
||||
starts.sort(READING_ORDER);
|
||||
log.info(
|
||||
"[redact/execute] start='{}' matched {} anchor(s): {}",
|
||||
startStr,
|
||||
starts.size(),
|
||||
anchorSummary(starts));
|
||||
|
||||
boolean openEnded = (endStr == null || endStr.isBlank());
|
||||
List<Anchor> ends = new ArrayList<>();
|
||||
if (!openEnded) {
|
||||
Map<Integer, List<PDFText>> endMatchesByPage = findWithFallbacks(document, endStr);
|
||||
if (endMatchesByPage.isEmpty()) {
|
||||
log.warn(
|
||||
"[redact/execute] range end '{}' not found in document - skipping range"
|
||||
+ " (start='{}')",
|
||||
endStr,
|
||||
startStr);
|
||||
return Collections.emptyList();
|
||||
}
|
||||
ends = toAnchors(document, endMatchesByPage, layoutCache);
|
||||
ends.sort(READING_ORDER);
|
||||
log.info(
|
||||
"[redact/execute] end='{}' matched {} anchor(s): {}",
|
||||
endStr,
|
||||
ends.size(),
|
||||
anchorSummary(ends));
|
||||
}
|
||||
|
||||
List<PDFText> blocks = new ArrayList<>();
|
||||
for (Anchor start : starts) {
|
||||
Anchor end = null;
|
||||
int endPage;
|
||||
if (openEnded) {
|
||||
endPage = totalPages - 1;
|
||||
} else {
|
||||
for (Anchor candidate : ends) {
|
||||
if (READING_ORDER.compare(candidate, start) > 0) {
|
||||
end = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (end == null) {
|
||||
log.warn(
|
||||
"[redact/execute] no end anchor after start at (page={}, col={}, y={}) — skipping",
|
||||
start.page + 1,
|
||||
start.col,
|
||||
start.y);
|
||||
continue;
|
||||
}
|
||||
endPage = end.page;
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] range pages {}-{}: start='{}' (col {}) end='{}'",
|
||||
start.page + 1,
|
||||
endPage + 1,
|
||||
startStr,
|
||||
start.col,
|
||||
openEnded ? "<end of document>" : endStr);
|
||||
|
||||
collectBlocksForRange(document, allPages, start, end, openEnded, blocks, layoutCache);
|
||||
}
|
||||
|
||||
log.info(
|
||||
"[redact/execute] range '{}'→'{}': {} total blocks",
|
||||
startStr,
|
||||
openEnded ? "<end of document>" : endStr,
|
||||
blocks.size());
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects all redactable content (text line segments and images) between two anchor positions.
|
||||
*
|
||||
* <p>Line boxes are cached per page number in {@code lineBoxCache} and reused across range
|
||||
* iterations within one execute() call, avoiding redundant {@link AllTextLineExtractor} passes.
|
||||
*/
|
||||
private void collectBlocksForRange(
|
||||
PDDocument document,
|
||||
PDPageTree allPages,
|
||||
Anchor start,
|
||||
Anchor end,
|
||||
boolean openEnded,
|
||||
List<PDFText> blocks,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
|
||||
int startPage = start.page;
|
||||
int endPage = openEnded ? allPages.getCount() - 1 : end.page;
|
||||
int endCol =
|
||||
openEnded ? layoutFor(document, endPage, layoutCache).columnCount() - 1 : end.col;
|
||||
float startY = start.y;
|
||||
// Use bottom of end anchor so the end anchor line itself is included (inclusive range).
|
||||
float endY = openEnded ? Float.POSITIVE_INFINITY : end.text.getY2();
|
||||
|
||||
// Line-box cache: populated lazily per page, reused across range iterations.
|
||||
// Cannot use computeIfAbsent because AllTextLineExtractor's constructor throws IOException.
|
||||
Map<Integer, List<float[]>> lineBoxCache = new HashMap<>();
|
||||
|
||||
for (int pageIdx = startPage; pageIdx <= endPage; pageIdx++) {
|
||||
PDPage page = allPages.get(pageIdx);
|
||||
float pageHeight = page.getBBox().getHeight();
|
||||
PageColumnLayout layout = layoutFor(document, pageIdx, layoutCache);
|
||||
|
||||
List<float[]> screenLineBoxes = lineBoxCache.get(pageIdx);
|
||||
if (screenLineBoxes == null) {
|
||||
AllTextLineExtractor textExtractor =
|
||||
new AllTextLineExtractor(pageIdx + 1, pageHeight);
|
||||
textExtractor.getText(document);
|
||||
screenLineBoxes = textExtractor.getScreenLineBoxes();
|
||||
lineBoxCache.put(pageIdx, screenLineBoxes);
|
||||
}
|
||||
|
||||
for (float[] sb : screenLineBoxes) {
|
||||
emitColumnSlices(
|
||||
pageIdx, layout, sb[0], sb[2], sb[1], sb[3], start.col, startPage, startY,
|
||||
endCol, endPage, endY, blocks);
|
||||
}
|
||||
|
||||
PageImageLocator imgLocator = new PageImageLocator(page, pageIdx);
|
||||
imgLocator.processPage(page);
|
||||
for (PageImageLocator.ImageBox ib : imgLocator.getImageBoxes()) {
|
||||
// ImageBox coordinates are in PDF user-space (Y up); convert to screen-Y (Y down).
|
||||
float screenY1 = pageHeight - ib.y2();
|
||||
float screenY2 = pageHeight - ib.y1();
|
||||
emitColumnSlices(
|
||||
pageIdx, layout, ib.x1(), ib.x2(), screenY1, screenY2, start.col, startPage,
|
||||
startY, endCol, endPage, endY, blocks);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Emits each per-column sub-box accepted by the reading-order predicate. */
|
||||
private static void emitColumnSlices(
|
||||
int pageIdx,
|
||||
PageColumnLayout layout,
|
||||
float x1,
|
||||
float x2,
|
||||
float yTop,
|
||||
float yBottom,
|
||||
int startCol,
|
||||
int startPage,
|
||||
float startY,
|
||||
int endCol,
|
||||
int endPage,
|
||||
float endY,
|
||||
List<PDFText> blocks) {
|
||||
int[] cols = layout.columnsCrossing(x1, x2);
|
||||
if (cols.length == 1) {
|
||||
if (inColumnZone(
|
||||
pageIdx, cols[0], yTop, yBottom, startPage, startCol, startY, endPage, endCol,
|
||||
endY)) {
|
||||
blocks.add(new PDFText(pageIdx, x1, yTop, x2, yBottom, ""));
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (int col : cols) {
|
||||
if (!inColumnZone(
|
||||
pageIdx, col, yTop, yBottom, startPage, startCol, startY, endPage, endCol,
|
||||
endY)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
blocks.add(new PDFText(pageIdx, x1, yTop, x2, yBottom, ""));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reading-order predicate: true when (col, yBottom) on page {@code pageIdx} sits between the
|
||||
* start anchor (inclusive) and end anchor (inclusive).
|
||||
*/
|
||||
static boolean inColumnZone(
|
||||
int pageIdx,
|
||||
int col,
|
||||
float yTop,
|
||||
float yBottom,
|
||||
int startPage,
|
||||
int startCol,
|
||||
float startY,
|
||||
int endPage,
|
||||
int endCol,
|
||||
float endY) {
|
||||
if (pageIdx > startPage && pageIdx < endPage) return true;
|
||||
if (pageIdx == startPage && pageIdx == endPage) {
|
||||
if (startCol == endCol) {
|
||||
return col == startCol && yBottom >= startY && yBottom <= endY;
|
||||
}
|
||||
if (startCol < endCol) {
|
||||
if (col < startCol || col > endCol) return false;
|
||||
if (col == startCol) return yBottom >= startY;
|
||||
if (col == endCol) return yBottom <= endY;
|
||||
return true;
|
||||
}
|
||||
return col == startCol && yBottom >= startY;
|
||||
}
|
||||
if (pageIdx == startPage) {
|
||||
if (col == startCol) return yBottom >= startY;
|
||||
return col > startCol;
|
||||
}
|
||||
if (pageIdx == endPage) {
|
||||
if (col == endCol) return yBottom <= endY;
|
||||
return col < endCol;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Lazily builds and caches the column layout for a single page. */
|
||||
private PageColumnLayout layoutFor(
|
||||
PDDocument document, int pageIdx, Map<Integer, PageColumnLayout> cache)
|
||||
throws IOException {
|
||||
PageColumnLayout cached = cache.get(pageIdx);
|
||||
if (cached != null) return cached;
|
||||
PDPage page = document.getDocumentCatalog().getPages().get(pageIdx);
|
||||
float pageWidth = page.getBBox().getWidth();
|
||||
float pageHeight = page.getBBox().getHeight();
|
||||
AllTextLineExtractor extractor = new AllTextLineExtractor(pageIdx + 1, pageHeight);
|
||||
extractor.getText(document);
|
||||
PageColumnLayout layout =
|
||||
PageColumnLayout.fromLineBoxes(extractor.getLineBoxes(), pageWidth);
|
||||
if (layout.columnCount() > 1) {
|
||||
float[] g = layout.gutters().get(0);
|
||||
log.info(
|
||||
"[redact/execute] page {} layout: 2 cols, gutter x=[{}, {}]",
|
||||
pageIdx + 1,
|
||||
g[0],
|
||||
g[1]);
|
||||
} else {
|
||||
log.info("[redact/execute] page {} layout: 1 col (single-column mode)", pageIdx + 1);
|
||||
}
|
||||
cache.put(pageIdx, layout);
|
||||
return layout;
|
||||
}
|
||||
|
||||
private List<Anchor> toAnchors(
|
||||
PDDocument document,
|
||||
Map<Integer, List<PDFText>> matchesByPage,
|
||||
Map<Integer, PageColumnLayout> layoutCache)
|
||||
throws IOException {
|
||||
List<Anchor> out = new ArrayList<>();
|
||||
for (int page : matchesByPage.keySet().stream().sorted().toList()) {
|
||||
PageColumnLayout layout = layoutFor(document, page, layoutCache);
|
||||
for (PDFText hit : matchesByPage.get(page)) {
|
||||
int col = layout.columnOf(hit.getX1(), hit.getX2());
|
||||
out.add(new Anchor(page, col, hit.getY1(), hit));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Lexicographic ordering by (page, column, screenY). */
|
||||
private static final Comparator<Anchor> READING_ORDER =
|
||||
Comparator.comparingInt((Anchor a) -> a.page)
|
||||
.thenComparingInt(a -> a.col)
|
||||
.thenComparingDouble(a -> a.y);
|
||||
|
||||
private static String anchorSummary(List<Anchor> anchors) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int max = Math.min(anchors.size(), 5);
|
||||
for (int i = 0; i < max; i++) {
|
||||
Anchor a = anchors.get(i);
|
||||
if (i > 0) sb.append(", ");
|
||||
sb.append(String.format("(p=%d,c=%d,y=%.1f)", a.page + 1, a.col, a.y));
|
||||
}
|
||||
if (anchors.size() > max) sb.append(", …");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private record Anchor(int page, int col, float y, PDFText text) {}
|
||||
|
||||
/**
|
||||
* Tries progressively more permissive variants: raw (regex then literal), letter-spacing
|
||||
* collapsed, then a punctuation-tolerant regex over alphanumeric runs.
|
||||
*/
|
||||
private Map<Integer, List<PDFText>> findWithFallbacks(PDDocument document, String raw) {
|
||||
String trimmed = raw.trim();
|
||||
String collapsed = collapseLetterSpacing(trimmed);
|
||||
String tolerant = punctuationTolerantRegex(trimmed);
|
||||
|
||||
List<Candidate> candidates = new ArrayList<>();
|
||||
candidates.add(new Candidate(trimmed, true));
|
||||
candidates.add(new Candidate(trimmed, false));
|
||||
if (!collapsed.equals(trimmed)) {
|
||||
candidates.add(new Candidate(collapsed, true));
|
||||
candidates.add(new Candidate(collapsed, false));
|
||||
}
|
||||
if (tolerant != null && !tolerant.equals(trimmed)) {
|
||||
candidates.add(new Candidate(tolerant, true));
|
||||
}
|
||||
|
||||
// If the anchor spans multiple lines (model provided entire paragraph instead of a short
|
||||
// phrase), try just the first non-empty line — it's usually sufficient to locate the
|
||||
// position and avoids mismatches from mid-paragraph text extraction artifacts.
|
||||
if (trimmed.contains("\n")) {
|
||||
String firstLine =
|
||||
Arrays.stream(trimmed.split("\n"))
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
if (firstLine != null && firstLine.length() >= 4) {
|
||||
String firstLineCollapsed = collapseLetterSpacing(firstLine);
|
||||
String firstLineTolerant = punctuationTolerantRegex(firstLine);
|
||||
candidates.add(new Candidate(firstLine, false));
|
||||
if (!firstLineCollapsed.equals(firstLine)) {
|
||||
candidates.add(new Candidate(firstLineCollapsed, false));
|
||||
}
|
||||
if (firstLineTolerant != null && !firstLineTolerant.equals(firstLine)) {
|
||||
candidates.add(new Candidate(firstLineTolerant, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (Candidate c : candidates) {
|
||||
Map<Integer, List<PDFText>> m =
|
||||
textRedactionService.findTextToRedact(
|
||||
document, new String[] {c.pattern}, c.useRegex, false);
|
||||
if (!m.isEmpty()) {
|
||||
if (!c.pattern.equals(trimmed)) {
|
||||
log.info(
|
||||
"[redact/execute] range boundary matched via fallback: '{}' → '{}'",
|
||||
trimmed,
|
||||
c.pattern);
|
||||
}
|
||||
return m;
|
||||
}
|
||||
}
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
private record Candidate(String pattern, boolean useRegex) {}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Static helpers
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Joins {@code raw}'s alphanumeric runs with {@code \W*} so anchors match across punctuation
|
||||
* drift. Returns {@code null} when fewer than two tokens exist.
|
||||
*/
|
||||
private static String punctuationTolerantRegex(String raw) {
|
||||
List<String> tokens = new ArrayList<>();
|
||||
StringBuilder current = new StringBuilder();
|
||||
for (int i = 0; i < raw.length(); i++) {
|
||||
char ch = raw.charAt(i);
|
||||
if (Character.isLetterOrDigit(ch)) {
|
||||
current.append(ch);
|
||||
} else if (current.length() > 0) {
|
||||
tokens.add(current.toString());
|
||||
current.setLength(0);
|
||||
}
|
||||
}
|
||||
if (current.length() > 0) tokens.add(current.toString());
|
||||
if (tokens.size() < 2) return null;
|
||||
StringBuilder out = new StringBuilder();
|
||||
for (int i = 0; i < tokens.size(); i++) {
|
||||
if (i > 0) out.append("\\W*");
|
||||
out.append(Pattern.quote(tokens.get(i)));
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapses letter-spaced text produced by position-sorted text extraction.
|
||||
*
|
||||
* <p>When a PDF text stripper runs with {@code setSortByPosition(true)}, letter-spaced headings
|
||||
* come out as {@code "T a b l e o f c o n t e n t s"}. This method converts the spaced form
|
||||
* back to words.
|
||||
*/
|
||||
private static String collapseLetterSpacing(String text) {
|
||||
String[] tokens = text.split(" ", -1);
|
||||
StringBuilder result = new StringBuilder();
|
||||
StringBuilder current = new StringBuilder();
|
||||
for (String token : tokens) {
|
||||
if (token.isEmpty()) {
|
||||
if (current.length() > 0) {
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(current);
|
||||
current.setLength(0);
|
||||
}
|
||||
} else if (token.length() == 1) {
|
||||
current.append(token);
|
||||
} else {
|
||||
if (current.length() > 0) {
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(current);
|
||||
current.setLength(0);
|
||||
}
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(token);
|
||||
}
|
||||
}
|
||||
if (current.length() > 0) {
|
||||
if (result.length() > 0) result.append(' ');
|
||||
result.append(current);
|
||||
}
|
||||
return result.toString().trim();
|
||||
}
|
||||
|
||||
private static <T> List<T> orEmpty(List<T> list) {
|
||||
return list != null ? list : List.of();
|
||||
}
|
||||
|
||||
private static String[] cleanStrings(List<String> input) {
|
||||
if (input == null || input.isEmpty()) {
|
||||
return new String[0];
|
||||
}
|
||||
return input.stream()
|
||||
.filter(s -> s != null)
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.toArray(String[]::new);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts 1-based page numbers from the request to the 0-based indices used internally.
|
||||
* Out-of-range and non-positive values are silently dropped.
|
||||
*/
|
||||
private static List<Integer> toZeroBasedIndices(List<Integer> oneBasedPageNumbers) {
|
||||
if (oneBasedPageNumbers == null || oneBasedPageNumbers.isEmpty()) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
List<Integer> result = new ArrayList<>();
|
||||
for (Integer page : oneBasedPageNumbers) {
|
||||
if (page != null && page > 0) {
|
||||
result.add(page - 1);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String trimOrEmpty(String s) {
|
||||
return s == null ? "" : s.trim();
|
||||
}
|
||||
}
|
||||
+1207
File diff suppressed because it is too large
Load Diff
+132
@@ -0,0 +1,132 @@
|
||||
package stirling.software.SPDF.model.api.security;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class RedactExecuteRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Exact strings to find and black out. One entry per phrase to redact."
|
||||
+ " Best for known names, identifiers, and specific text found in the document.")
|
||||
private List<String> textValues = new ArrayList<>();
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Regex patterns to match and redact. Each match anywhere in the document is blacked out."
|
||||
+ " Uses Java/PCRE regex syntax. Well-suited for strings that follow known patterns, like"
|
||||
+ " phone numbers, email addresses, national ID numbers, or"
|
||||
+ " dates (which can appear with different separators, optional country codes,"
|
||||
+ " etc.). For fixed known strings such as names, use textValues instead.")
|
||||
private List<String> regexPatterns = new ArrayList<>();
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"1-indexed page numbers to wipe entirely (all content removed from those pages).")
|
||||
private List<Integer> wipePages = new ArrayList<>();
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Text ranges to redact by specifying a start and end anchor phrase. All"
|
||||
+ " content between the two phrases (inclusive) is redacted. Anchors"
|
||||
+ " work best when short and unique. They must appear"
|
||||
+ " verbatim in the document.")
|
||||
private List<TextRange> ranges = new ArrayList<>();
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"Rectangular areas to black out, each defined by a page number and bounding box coordinates.")
|
||||
private List<ImageBox> imageBoxes = new ArrayList<>();
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"1-indexed page numbers to redact all detected images from. Pass an empty list to redact images from every page. Omit or pass null to skip image redaction entirely.")
|
||||
private List<Integer> redactImagePages;
|
||||
|
||||
@Schema(description = "Redaction style options")
|
||||
private RedactStyle style = new RedactStyle();
|
||||
|
||||
public record TextRange(
|
||||
@Schema(
|
||||
description =
|
||||
"A short, distinctive phrase (5–15 words) that marks where"
|
||||
+ " redaction begins (inclusive). Must appear verbatim in"
|
||||
+ " the document — e.g. a section heading or a unique"
|
||||
+ " sentence fragment.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
minLength = 1)
|
||||
String startString,
|
||||
@Schema(
|
||||
description =
|
||||
"A short, distinctive phrase (5–15 words) that marks where"
|
||||
+ " redaction ends (inclusive). Must appear verbatim in the"
|
||||
+ " document. Shorter phrases match more reliably.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
minLength = 1)
|
||||
String endString) {
|
||||
public TextRange {
|
||||
if (endString == null) endString = "";
|
||||
}
|
||||
}
|
||||
|
||||
public record ImageBox(
|
||||
@Schema(
|
||||
description = "0-indexed page number (first page = 0).",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
int pageIndex,
|
||||
@Schema(
|
||||
description =
|
||||
"Left x coordinate of the redaction rectangle in PDF user-space points.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
float x1,
|
||||
@Schema(
|
||||
description =
|
||||
"Top y coordinate of the redaction rectangle in PDF user-space points.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
float y1,
|
||||
@Schema(
|
||||
description =
|
||||
"Right x coordinate of the redaction rectangle in PDF user-space points.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
float x2,
|
||||
@Schema(
|
||||
description =
|
||||
"Bottom y coordinate of the redaction rectangle in PDF user-space points.",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
float y2) {}
|
||||
|
||||
public enum RedactionStrategy {
|
||||
AUTO,
|
||||
OVERLAY_ONLY,
|
||||
IMAGE_FINALIZE
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class RedactStyle {
|
||||
@Schema(description = "Hex redaction box color", defaultValue = "#000000")
|
||||
private String color = "#000000";
|
||||
|
||||
@Schema(
|
||||
description = "Extra padding around each box in points",
|
||||
type = "number",
|
||||
defaultValue = "0")
|
||||
private float padding = 0f;
|
||||
|
||||
@Schema(description = "Rasterize output to prevent text extraction", defaultValue = "false")
|
||||
private boolean convertToImage = false;
|
||||
|
||||
@Schema(
|
||||
description = "Execution strategy hint for the redaction pipeline",
|
||||
defaultValue = "AUTO")
|
||||
private RedactionStrategy strategy = RedactionStrategy.AUTO;
|
||||
}
|
||||
}
|
||||
+42
-32
@@ -37,7 +37,6 @@ import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.junit.jupiter.MockitoSettings;
|
||||
@@ -77,8 +76,11 @@ class RedactControllerTest {
|
||||
|
||||
@Mock private CustomPDFDocumentFactory pdfDocumentFactory;
|
||||
@Mock private TempFileManager tempFileManager;
|
||||
@Mock private RedactExecuteService redactExecuteService;
|
||||
|
||||
@InjectMocks private RedactController redactController;
|
||||
private TextRedactionService textRedactionService;
|
||||
private ManualRedactionService manualRedactionService;
|
||||
private RedactController redactController;
|
||||
|
||||
private MockMultipartFile mockPdfFile;
|
||||
private PDDocument mockDocument;
|
||||
@@ -201,7 +203,17 @@ class RedactControllerTest {
|
||||
.save(any(File.class));
|
||||
doNothing().when(mockDocument).close();
|
||||
|
||||
// Initialize a real document for unit tests
|
||||
// Build real service instances so tests exercise actual logic
|
||||
textRedactionService = new TextRedactionService();
|
||||
manualRedactionService = new ManualRedactionService(tempFileManager);
|
||||
redactController =
|
||||
new RedactController(
|
||||
pdfDocumentFactory,
|
||||
tempFileManager,
|
||||
manualRedactionService,
|
||||
textRedactionService,
|
||||
redactExecuteService);
|
||||
|
||||
setupRealDocument();
|
||||
}
|
||||
|
||||
@@ -819,9 +831,9 @@ class RedactControllerTest {
|
||||
contentStream.newLineAtOffset(50, 750);
|
||||
|
||||
contentStream.showText("This is ");
|
||||
contentStream.newLineAtOffset(-10, 0); // Simulate positioning
|
||||
contentStream.newLineAtOffset(-10, 0);
|
||||
contentStream.showText("secret");
|
||||
contentStream.newLineAtOffset(10, 0); // Reset positioning
|
||||
contentStream.newLineAtOffset(10, 0);
|
||||
contentStream.showText(" information");
|
||||
contentStream.endText();
|
||||
}
|
||||
@@ -1005,7 +1017,7 @@ class RedactControllerTest {
|
||||
contentStream.showText("Original content");
|
||||
contentStream.endText();
|
||||
}
|
||||
return redactController.createTokensWithoutTargetText(
|
||||
return textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, pageForTokenExtraction, Collections.emptySet(), false, false);
|
||||
}
|
||||
|
||||
@@ -1016,28 +1028,28 @@ class RedactControllerTest {
|
||||
@Test
|
||||
@DisplayName("Should decode valid hex color with hash")
|
||||
void decodeValidHexColorWithHash() {
|
||||
Color result = redactController.decodeOrDefault("#FF0000");
|
||||
Color result = ManualRedactionService.decodeOrDefault("#FF0000");
|
||||
assertEquals(Color.RED, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Should decode valid hex color without hash")
|
||||
void decodeValidHexColorWithoutHash() {
|
||||
Color result = redactController.decodeOrDefault("FF0000");
|
||||
Color result = ManualRedactionService.decodeOrDefault("FF0000");
|
||||
assertEquals(Color.RED, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Should default to black for null color")
|
||||
void defaultToBlackForNullColor() {
|
||||
Color result = redactController.decodeOrDefault(null);
|
||||
Color result = ManualRedactionService.decodeOrDefault(null);
|
||||
assertEquals(Color.BLACK, result);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Should default to black for invalid color")
|
||||
void defaultToBlackForInvalidColor() {
|
||||
Color result = redactController.decodeOrDefault("invalid-color");
|
||||
Color result = ManualRedactionService.decodeOrDefault("invalid-color");
|
||||
assertEquals(Color.BLACK, result);
|
||||
}
|
||||
|
||||
@@ -1049,7 +1061,7 @@ class RedactControllerTest {
|
||||
})
|
||||
@DisplayName("Should handle various valid color formats")
|
||||
void handleVariousValidColorFormats(String colorInput) {
|
||||
Color result = redactController.decodeOrDefault(colorInput);
|
||||
Color result = ManualRedactionService.decodeOrDefault(colorInput);
|
||||
assertNotNull(result);
|
||||
assertTrue(
|
||||
result.getRed() >= 0 && result.getRed() <= 255,
|
||||
@@ -1065,8 +1077,8 @@ class RedactControllerTest {
|
||||
@Test
|
||||
@DisplayName("Should handle short hex codes appropriately")
|
||||
void handleShortHexCodes() {
|
||||
Color result1 = redactController.decodeOrDefault("123");
|
||||
Color result2 = redactController.decodeOrDefault("#12");
|
||||
Color result1 = ManualRedactionService.decodeOrDefault("123");
|
||||
Color result2 = ManualRedactionService.decodeOrDefault("#12");
|
||||
|
||||
assertNotNull(result1);
|
||||
assertNotNull(result2);
|
||||
@@ -1094,7 +1106,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("confidential");
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
assertNotNull(tokens);
|
||||
@@ -1115,7 +1127,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("secret");
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
assertNotNull(tokens);
|
||||
@@ -1148,7 +1160,7 @@ class RedactControllerTest {
|
||||
|
||||
List<Object> originalTokens = getOriginalTokens();
|
||||
List<Object> filteredTokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
long originalNonTextCount =
|
||||
@@ -1156,7 +1168,7 @@ class RedactControllerTest {
|
||||
.filter(
|
||||
token ->
|
||||
token instanceof Operator op
|
||||
&& !redactController.isTextShowingOperator(
|
||||
&& !textRedactionService.isTextShowingOperator(
|
||||
op.getName()))
|
||||
.count();
|
||||
|
||||
@@ -1165,7 +1177,7 @@ class RedactControllerTest {
|
||||
.filter(
|
||||
token ->
|
||||
token instanceof Operator op
|
||||
&& !redactController.isTextShowingOperator(
|
||||
&& !textRedactionService.isTextShowingOperator(
|
||||
op.getName()))
|
||||
.count();
|
||||
|
||||
@@ -1184,7 +1196,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("\\d{3}-\\d{2}-\\d{4}"); // SSN pattern
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, true, false);
|
||||
|
||||
String reconstructedText = extractTextFromTokens(tokens);
|
||||
@@ -1200,7 +1212,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("test");
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, true);
|
||||
|
||||
String reconstructedText = extractTextFromTokens(tokens);
|
||||
@@ -1217,7 +1229,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("sensitive");
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
String reconstructedText = extractTextFromTokens(tokens);
|
||||
@@ -1231,7 +1243,7 @@ class RedactControllerTest {
|
||||
void shouldWriteTokensToNewContentStream() throws Exception {
|
||||
List<Object> tokens = createSampleTokenList();
|
||||
|
||||
redactController.writeFilteredContentStream(realDocument, realPage, tokens);
|
||||
textRedactionService.writeFilteredContentStream(realDocument, realPage, tokens);
|
||||
|
||||
assertNotNull(realPage.getContents(), "Page should have content stream");
|
||||
|
||||
@@ -1249,7 +1261,7 @@ class RedactControllerTest {
|
||||
|
||||
assertDoesNotThrow(
|
||||
() ->
|
||||
redactController.writeFilteredContentStream(
|
||||
textRedactionService.writeFilteredContentStream(
|
||||
realDocument, realPage, emptyTokens));
|
||||
|
||||
assertNotNull(realPage.getContents(), "Page should still have content stream");
|
||||
@@ -1262,7 +1274,7 @@ class RedactControllerTest {
|
||||
String originalContent = extractTextFromModifiedPage(realPage);
|
||||
|
||||
List<Object> newTokens = createSampleTokenList();
|
||||
redactController.writeFilteredContentStream(realDocument, realPage, newTokens);
|
||||
textRedactionService.writeFilteredContentStream(realDocument, realPage, newTokens);
|
||||
|
||||
String newContent = extractTextFromModifiedPage(realPage);
|
||||
assertNotEquals(originalContent, newContent, "Content stream should be replaced");
|
||||
@@ -1273,7 +1285,7 @@ class RedactControllerTest {
|
||||
void shouldCreateWidthMatchingPlaceholder() {
|
||||
String originalText = "confidential";
|
||||
String placeholder =
|
||||
redactController.createPlaceholderWithFont(
|
||||
textRedactionService.createPlaceholderWithFont(
|
||||
originalText, new PDType1Font(Standard14Fonts.FontName.HELVETICA));
|
||||
|
||||
assertEquals(
|
||||
@@ -1287,7 +1299,7 @@ class RedactControllerTest {
|
||||
void shouldHandleSpecialCharactersInPlaceholder() {
|
||||
String originalText = "café naïve";
|
||||
String placeholder =
|
||||
redactController.createPlaceholderWithFont(
|
||||
textRedactionService.createPlaceholderWithFont(
|
||||
originalText, new PDType1Font(Standard14Fonts.FontName.HELVETICA));
|
||||
|
||||
assertEquals(originalText.length(), placeholder.length());
|
||||
@@ -1303,10 +1315,10 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("secret");
|
||||
|
||||
List<Object> filteredTokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
redactController.writeFilteredContentStream(realDocument, realPage, filteredTokens);
|
||||
textRedactionService.writeFilteredContentStream(realDocument, realPage, filteredTokens);
|
||||
assertNotNull(realPage.getContents());
|
||||
|
||||
String finalText = extractTextFromModifiedPage(realPage);
|
||||
@@ -1322,7 +1334,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("confidential");
|
||||
|
||||
List<Object> filteredTokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
long filteredPositioning =
|
||||
@@ -1377,7 +1389,7 @@ class RedactControllerTest {
|
||||
Set<String> targetWords = Set.of("confidential");
|
||||
|
||||
List<Object> tokens =
|
||||
redactController.createTokensWithoutTargetText(
|
||||
textRedactionService.createTokensWithoutTargetText(
|
||||
realDocument, realPage, targetWords, false, false);
|
||||
|
||||
assertNotNull(tokens);
|
||||
@@ -1404,14 +1416,12 @@ class RedactControllerTest {
|
||||
@Test
|
||||
@DisplayName("Should handle documents with multiple text blocks")
|
||||
void shouldHandleDocumentsWithMultipleTextBlocks() throws Exception {
|
||||
// Create a document with multiple text blocks
|
||||
realPage = new PDPage(PDRectangle.A4);
|
||||
while (realDocument.getNumberOfPages() > 0) {
|
||||
realDocument.removePage(0);
|
||||
}
|
||||
realDocument.addPage(realPage);
|
||||
|
||||
// Create resources
|
||||
PDResources resources = new PDResources();
|
||||
resources.put(
|
||||
COSName.getPDFName("F1"), new PDType1Font(Standard14Fonts.FontName.HELVETICA));
|
||||
|
||||
+430
@@ -0,0 +1,430 @@
|
||||
package stirling.software.SPDF.controller.api.security;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import stirling.software.SPDF.model.PDFText;
|
||||
import stirling.software.SPDF.pdf.parser.PageColumnLayout;
|
||||
|
||||
/**
|
||||
* Integration tests for {@link RedactExecuteService#collectRangeBlocks(PDDocument, String, String,
|
||||
* Map)}. Each test builds a synthetic PDF (single-column or two-column) with text-positioning that
|
||||
* matches what a real document would produce, then asserts that the redaction range produces blocks
|
||||
* confined to the expected X/Y region.
|
||||
*/
|
||||
class RedactExecuteServiceTest {
|
||||
|
||||
private static final float PAGE_WIDTH = PDRectangle.LETTER.getWidth(); // 612
|
||||
private static final float PAGE_HEIGHT = PDRectangle.LETTER.getHeight(); // 792
|
||||
|
||||
private static final float LEFT_X = 72f;
|
||||
private static final float RIGHT_X = 330f;
|
||||
private static final float COL_WIDTH = 220f;
|
||||
private static final float LINE_HEIGHT = 14f;
|
||||
private static final float TOP_Y = PAGE_HEIGHT - 80f;
|
||||
private static final float FONT_SIZE = 11f;
|
||||
|
||||
private final RedactExecuteService service =
|
||||
new RedactExecuteService(null, null, new TextRedactionService());
|
||||
|
||||
@Nested
|
||||
@DisplayName("Single-column documents")
|
||||
class SingleColumn {
|
||||
|
||||
@Test
|
||||
void redactBetweenMarkers_inclusive() throws IOException {
|
||||
try (PDDocument doc = buildSingleColumnDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(doc, "START-HERE", "STOP-HERE", cache);
|
||||
|
||||
assertThat(blocks)
|
||||
.as("blocks should be produced for single-column range")
|
||||
.isNotEmpty();
|
||||
|
||||
// Blocks are in screen coords (top-left, Y down). START-HERE is drawn at the top
|
||||
// of the page; STOP-HERE four lines below. Screen Y grows downward, so the
|
||||
// anchors' screen-Y tops sit roughly around screenTop(0) and screenTop(4).
|
||||
// The end anchor is inclusive, so blocks may extend to the bottom of line 4.
|
||||
float screenTopOfStart = screenTopOfLine(0);
|
||||
float screenBottomOfEnd = screenTopOfLine(4) + LINE_HEIGHT;
|
||||
for (PDFText block : blocks) {
|
||||
assertThat(block.getY1())
|
||||
.as("block top must be at or below the start anchor's top")
|
||||
.isGreaterThanOrEqualTo(screenTopOfStart - 1f);
|
||||
assertThat(block.getY2())
|
||||
.as(
|
||||
"block bottom must not extend past the end anchor's bottom (end is inclusive)")
|
||||
.isLessThanOrEqualTo(screenBottomOfEnd + 1f);
|
||||
assertThat(block.getX2())
|
||||
.as("block should not extend into a hypothetical right column")
|
||||
.isLessThan(PAGE_WIDTH / 2f + 50f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingStartString_noBlocks() throws IOException {
|
||||
try (PDDocument doc = buildSingleColumnDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(doc, "MISSING-START", "STOP-HERE", cache);
|
||||
|
||||
assertThat(blocks).isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void cvStyleHeadingPlusRightAlignedDate_stillTreatedAsSingleColumn() throws IOException {
|
||||
// CV-style page: single-column body, but each section heading shares its row with a
|
||||
// right-aligned date. The X-gap splitter emits the heading and the date as separate
|
||||
// line boxes; this must NOT trip 2-column detection (the date is too narrow to be a
|
||||
// real column), otherwise the cross-page redaction predicate over-includes wrong
|
||||
// regions.
|
||||
try (PDDocument doc = buildCvStyleDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(doc, "SECTION-A", "SECTION-C", cache);
|
||||
|
||||
assertThat(blocks)
|
||||
.as("CV-style redaction between section headings must produce blocks")
|
||||
.isNotEmpty();
|
||||
|
||||
PageColumnLayout layout = cache.get(0);
|
||||
assertThat(layout.columnCount())
|
||||
.as("CV-style page with heading+date rows must remain single-column")
|
||||
.isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void punctuationDriftInAnchors_stillMatchesViaTolerantFallback() throws IOException {
|
||||
// Simulates the LLM paraphrasing the heading by inserting a colon that isn't in the
|
||||
// source ("#3 Character substitution" → "#3: Character substitution"). The
|
||||
// punctuation-tolerant regex fallback should still find the line.
|
||||
try (PDDocument doc = buildHeadingPdf()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(
|
||||
doc, "#3: Character substitution", "#6: Image resolution", cache);
|
||||
|
||||
assertThat(blocks)
|
||||
.as("anchor with extra punctuation should still resolve via fallback")
|
||||
.isNotEmpty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("Two-column documents")
|
||||
class TwoColumn {
|
||||
|
||||
@Test
|
||||
void rangeInLeftColumn_redactsOnlyLeftColumn() throws IOException {
|
||||
try (PDDocument doc = buildTwoColumnDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks = service.collectRangeBlocks(doc, "L-START", "L-END", cache);
|
||||
|
||||
assertThat(blocks).as("left-only range must produce blocks").isNotEmpty();
|
||||
|
||||
float gutterMid = (LEFT_X + COL_WIDTH + RIGHT_X) / 2f;
|
||||
for (PDFText block : blocks) {
|
||||
float midX = (block.getX1() + block.getX2()) / 2f;
|
||||
assertThat(midX)
|
||||
.as("every block must sit in the left column, never the right")
|
||||
.isLessThan(gutterMid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void rangeInRightColumn_redactsOnlyRightColumn() throws IOException {
|
||||
try (PDDocument doc = buildTwoColumnDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks = service.collectRangeBlocks(doc, "R-START", "R-END", cache);
|
||||
|
||||
assertThat(blocks).as("right-only range must produce blocks").isNotEmpty();
|
||||
|
||||
float gutterMid = (LEFT_X + COL_WIDTH + RIGHT_X) / 2f;
|
||||
for (PDFText block : blocks) {
|
||||
float midX = (block.getX1() + block.getX2()) / 2f;
|
||||
assertThat(midX)
|
||||
.as("every block must sit in the right column, never the left")
|
||||
.isGreaterThan(gutterMid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoColumnWithTocAbove_pairsAcrossColumns() throws IOException {
|
||||
// Reproduces magic.pdf-style stacked layout: a multi-line TOC near the top, then a
|
||||
// 2-column body where the start anchor is in left col (lower screen Y) and the end
|
||||
// anchor is in right col (higher screen Y). Original pairing failed here because
|
||||
// end.y < start.y in screen coords.
|
||||
try (PDDocument doc = buildTwoColumnWithTocDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(doc, "BODY-L-3", "BODY-R-1", cache);
|
||||
|
||||
assertThat(blocks)
|
||||
.as("cross-column body redaction must produce blocks despite stacked TOC")
|
||||
.isNotEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void crossColumnReadingOrder_leftBottomToRightTop_producesBothSides() throws IOException {
|
||||
// This is the case the original code couldn't handle at all: end Y < start Y.
|
||||
try (PDDocument doc = buildTwoColumnDoc()) {
|
||||
Map<Integer, PageColumnLayout> cache = new HashMap<>();
|
||||
List<PDFText> blocks =
|
||||
service.collectRangeBlocks(doc, "L-MIDDLE", "R-MIDDLE", cache);
|
||||
|
||||
assertThat(blocks)
|
||||
.as("cross-column range must produce blocks, not be silently dropped")
|
||||
.isNotEmpty();
|
||||
|
||||
float gutterMid = (LEFT_X + COL_WIDTH + RIGHT_X) / 2f;
|
||||
boolean sawLeft = false;
|
||||
boolean sawRight = false;
|
||||
for (PDFText block : blocks) {
|
||||
float midX = (block.getX1() + block.getX2()) / 2f;
|
||||
if (midX < gutterMid) sawLeft = true;
|
||||
else sawRight = true;
|
||||
}
|
||||
assertThat(sawLeft).as("left column should contain at least one block").isTrue();
|
||||
assertThat(sawRight).as("right column should contain at least one block").isTrue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── document fixtures ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Single-column page laid out as one column starting at LEFT_X. Lines: 0: START-HERE (start
|
||||
* anchor) 1: line one 2: line two 3: line three 4: STOP-HERE (end anchor) 5: line five (must
|
||||
* NOT be redacted)
|
||||
*/
|
||||
private PDDocument buildSingleColumnDoc() throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
|
||||
String[] lines = {
|
||||
"START-HERE", "line one", "line two", "line three", "STOP-HERE", "line five"
|
||||
};
|
||||
for (int i = 0; i < lines.length; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(LEFT_X, yForLine(i));
|
||||
cs.showText(lines[i]);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-column page. Lines per column, top to bottom: Left: L-TOP, L-START, L-MIDDLE, L-END,
|
||||
* L-BOTTOM Right: R-TOP, R-MIDDLE, R-START, R-END, R-BOTTOM
|
||||
*/
|
||||
private PDDocument buildTwoColumnDoc() throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
|
||||
// Body lines are padded to make each column genuinely wide enough that column
|
||||
// detection (which ignores narrow lines) treats both sides as real columns.
|
||||
String fill = " " + "x".repeat(26);
|
||||
String[] left = {
|
||||
"L-TOP" + fill,
|
||||
"L-START" + fill,
|
||||
"L-MIDDLE" + fill,
|
||||
"L-END" + fill,
|
||||
"L-BOTTOM" + fill
|
||||
};
|
||||
String[] right = {
|
||||
"R-TOP" + fill,
|
||||
"R-MIDDLE" + fill,
|
||||
"R-START" + fill,
|
||||
"R-END" + fill,
|
||||
"R-BOTTOM" + fill
|
||||
};
|
||||
for (int i = 0; i < left.length; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(LEFT_X, yForLine(i));
|
||||
cs.showText(left[i]);
|
||||
cs.endText();
|
||||
}
|
||||
// Aligned baselines per row (IEEE template style) — AllTextLineExtractor must split
|
||||
// these at the column gap rather than merge same-row left+right glyphs into a wide
|
||||
// box.
|
||||
for (int i = 0; i < right.length; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(RIGHT_X, yForLine(i));
|
||||
cs.showText(right[i]);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single-column page with feature headings: #1..#7 each followed by body text. The PDF text is
|
||||
* exactly "#3 Character substitution" (no colon) — the test then queries with a colon to
|
||||
* exercise the punctuation-tolerant fallback.
|
||||
*/
|
||||
private PDDocument buildHeadingPdf() throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
|
||||
String[] lines = {
|
||||
"#1 Auto layout",
|
||||
"Body about auto layout.",
|
||||
"#2 Smart selection",
|
||||
"Body about smart selection.",
|
||||
"#3 Character substitution",
|
||||
"Body about character substitution.",
|
||||
"#4 Rounded borders",
|
||||
"Body about rounded borders.",
|
||||
"#5 Auto contrast",
|
||||
"Body about auto contrast.",
|
||||
"#6 Image resolution",
|
||||
"Body about image resolution.",
|
||||
"#7 Columns",
|
||||
"Body about columns."
|
||||
};
|
||||
for (int i = 0; i < lines.length; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(LEFT_X, yForLine(i));
|
||||
cs.showText(lines[i]);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-column page like {@code magic.pdf}: a few full-width header lines, a 2-column TOC stacked
|
||||
* on top of the 2-column body, where TOC's right half lives inside what would otherwise be the
|
||||
* body's gutter. Body left column has BODY-L-1..3, right column has BODY-R-1..3.
|
||||
*/
|
||||
private PDDocument buildTwoColumnWithTocDoc() throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
|
||||
// Header — full width, lines 0..1.
|
||||
for (int i = 0; i < 2; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(LEFT_X, yForLine(i));
|
||||
cs.showText("FULL WIDTH HEADER LINE " + i + " ACROSS BOTH COLUMNS OF THE PAGE");
|
||||
cs.endText();
|
||||
}
|
||||
// TOC, 2 columns of entries. TOC right half sits where the body gutter would be —
|
||||
// exactly the layout that broke the histogram-based detector on magic.pdf.
|
||||
float tocLeftX = 101f;
|
||||
float tocRightX = 230f;
|
||||
for (int i = 0; i < 5; i++) {
|
||||
float y = yForLine(3 + i);
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(tocLeftX, y);
|
||||
cs.showText("TOC entry left " + i);
|
||||
cs.endText();
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(tocRightX, y);
|
||||
cs.showText("TOC entry right " + i);
|
||||
cs.endText();
|
||||
}
|
||||
// Body — 2-column with aligned baselines per row (IEEE-style).
|
||||
String fill = " " + "x".repeat(26);
|
||||
String[] bodyLeft = {"BODY-L-1" + fill, "BODY-L-2" + fill, "BODY-L-3" + fill};
|
||||
String[] bodyRight = {"BODY-R-1" + fill, "BODY-R-2" + fill, "BODY-R-3" + fill};
|
||||
for (int i = 0; i < bodyLeft.length; i++) {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(LEFT_X, yForLine(10 + i));
|
||||
cs.showText(bodyLeft[i]);
|
||||
cs.endText();
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(RIGHT_X, yForLine(10 + i));
|
||||
cs.showText(bodyRight[i]);
|
||||
cs.endText();
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* CV-style page: single-column body with a few section headings, each followed on the same
|
||||
* baseline by a right-aligned date string. {@link AllTextLineExtractor} will split each
|
||||
* heading+date row into two line boxes; column detection must reject this as a fake two-column
|
||||
* layout because the dates are too narrow to be a real column body.
|
||||
*/
|
||||
private PDDocument buildCvStyleDoc() throws IOException {
|
||||
PDDocument doc = new PDDocument();
|
||||
PDPage page = new PDPage(PDRectangle.LETTER);
|
||||
doc.addPage(page);
|
||||
try (PDPageContentStream cs = new PDPageContentStream(doc, page)) {
|
||||
cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), FONT_SIZE);
|
||||
|
||||
float dateX = PAGE_WIDTH - 144f; // right-aligned dates near the right margin
|
||||
|
||||
// Section A: heading + date, then 3 body lines.
|
||||
writeAt(cs, LEFT_X, yForLine(0), "SECTION-A");
|
||||
writeAt(cs, dateX, yForLine(0), "Jan 2020");
|
||||
writeAt(cs, LEFT_X, yForLine(1), "Body line A1 with enough width to look like body");
|
||||
writeAt(cs, LEFT_X, yForLine(2), "Body line A2 with enough width to look like body");
|
||||
writeAt(cs, LEFT_X, yForLine(3), "Body line A3 with enough width to look like body");
|
||||
|
||||
// Section B (in the redact range): heading + date + 3 body lines.
|
||||
writeAt(cs, LEFT_X, yForLine(5), "SECTION-B");
|
||||
writeAt(cs, dateX, yForLine(5), "Feb 2021");
|
||||
writeAt(cs, LEFT_X, yForLine(6), "Body line B1 with enough width to look like body");
|
||||
writeAt(cs, LEFT_X, yForLine(7), "Body line B2 with enough width to look like body");
|
||||
writeAt(cs, LEFT_X, yForLine(8), "Body line B3 with enough width to look like body");
|
||||
|
||||
// Section C (end anchor): heading + date.
|
||||
writeAt(cs, LEFT_X, yForLine(10), "SECTION-C");
|
||||
writeAt(cs, dateX, yForLine(10), "Mar 2022");
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
private static void writeAt(PDPageContentStream cs, float x, float y, String text)
|
||||
throws IOException {
|
||||
cs.beginText();
|
||||
cs.newLineAtOffset(x, y);
|
||||
cs.showText(text);
|
||||
cs.endText();
|
||||
}
|
||||
|
||||
/** PDF user-space Y baseline for line index {@code i} (0-based, top to bottom). */
|
||||
private static float yForLine(int lineIndex) {
|
||||
return TOP_Y - lineIndex * LINE_HEIGHT;
|
||||
}
|
||||
|
||||
/** Approximate screen-Y of the top of line {@code i} (top-left origin). */
|
||||
private static float screenTopOfLine(int lineIndex) {
|
||||
// baseline_pdf → baseline_screen flips against page height; glyph top ≈ baseline - font
|
||||
// size.
|
||||
return PAGE_HEIGHT - yForLine(lineIndex) - FONT_SIZE;
|
||||
}
|
||||
}
|
||||
+201
-1
@@ -8,13 +8,18 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
import org.apache.commons.csv.CSVPrinter;
|
||||
import org.apache.commons.csv.QuoteMode;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
@@ -24,6 +29,7 @@ import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.pdf.parser.PageImageLocator;
|
||||
import stirling.software.SPDF.pdf.parser.PdfIngester;
|
||||
import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage;
|
||||
import stirling.software.SPDF.pdf.parser.PdfModels.RawLine;
|
||||
@@ -32,6 +38,7 @@ import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment;
|
||||
import stirling.software.SPDF.pdf.parser.TabulaTableParser;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.PdfUtils;
|
||||
import stirling.software.common.util.RegexPatternUtils;
|
||||
import stirling.software.proprietary.model.api.ai.AiPdfContentType;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowFileRequest;
|
||||
import stirling.software.proprietary.model.api.ai.AiWorkflowTextSelection;
|
||||
@@ -122,6 +129,7 @@ public class PdfContentExtractor {
|
||||
}
|
||||
csvStrings.add(sw.toString());
|
||||
}
|
||||
|
||||
return csvStrings;
|
||||
}
|
||||
|
||||
@@ -295,7 +303,6 @@ public class PdfContentExtractor {
|
||||
private List<AiWorkflowTextSelection> extractPageText(
|
||||
PDDocument document, List<Integer> selectedPages, int maxCharacters)
|
||||
throws IOException {
|
||||
PDFTextStripper textStripper = new PDFTextStripper();
|
||||
List<AiWorkflowTextSelection> pages = new ArrayList<>();
|
||||
int remainingCharacters = maxCharacters;
|
||||
|
||||
@@ -304,10 +311,29 @@ public class PdfContentExtractor {
|
||||
break;
|
||||
}
|
||||
|
||||
PDFTextStripper textStripper = new PDFTextStripper();
|
||||
textStripper.setSortByPosition(true);
|
||||
textStripper.setStartPage(pageNumber);
|
||||
textStripper.setEndPage(pageNumber);
|
||||
|
||||
String pageText = textStripper.getText(document).trim();
|
||||
|
||||
// Prepend page dimensions so the AI agent can reason about absolute coordinates.
|
||||
PDPage page = document.getPage(pageNumber - 1);
|
||||
PDRectangle bbox = page.getBBox();
|
||||
String dimensionHeader =
|
||||
String.format(
|
||||
"--- Page dimensions: %.0fx%.0f pts"
|
||||
+ " (PDF user-space: origin bottom-left, Y up) ---\n",
|
||||
bbox.getWidth(), bbox.getHeight());
|
||||
pageText = dimensionHeader + pageText;
|
||||
|
||||
// Append image metadata so the AI agent can reason about images spatially.
|
||||
String imageAnnotation = buildImageAnnotation(document, pageNumber - 1);
|
||||
if (!imageAnnotation.isEmpty()) {
|
||||
pageText = pageText + imageAnnotation;
|
||||
}
|
||||
|
||||
if (pageText.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
@@ -327,6 +353,56 @@ public class PdfContentExtractor {
|
||||
return pages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a human-readable description of all images on a page to append to page text. Uses PDF
|
||||
* user-space coordinates (origin bottom-left, Y up) so the AI can reference exact bounding
|
||||
* boxes when requesting image redaction.
|
||||
*/
|
||||
private String buildImageAnnotation(PDDocument document, int pageIndex) {
|
||||
try {
|
||||
List<ImageBlock> images = extractImagePositions(document, pageIndex);
|
||||
if (images.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
PDRectangle bbox = page.getBBox();
|
||||
float pageWidth = bbox.getWidth();
|
||||
float pageHeight = bbox.getHeight();
|
||||
|
||||
StringBuilder sb = new StringBuilder("\n\n--- Images on this page ---");
|
||||
for (int i = 0; i < images.size(); i++) {
|
||||
ImageBlock img = images.get(i);
|
||||
String position = spatialLabel(img, pageWidth, pageHeight);
|
||||
float w = img.x2() - img.x1();
|
||||
float h = img.y2() - img.y1();
|
||||
sb.append(
|
||||
String.format(
|
||||
"\nImage %d: position=%s, size=%.0fx%.0f pts,"
|
||||
+ " bounds=(x1=%.0f, y1=%.0f, x2=%.0f, y2=%.0f)",
|
||||
i + 1, position, w, h, img.x1(), img.y1(), img.x2(), img.y2()));
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"Failed to extract image positions for page {}: {}", pageIndex, e.getMessage());
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a human-readable spatial label (e.g. "top-left", "center") for an image based on its
|
||||
* centre relative to the page dimensions. Coordinates are in PDF user-space (Y up).
|
||||
*/
|
||||
private static String spatialLabel(ImageBlock img, float pageWidth, float pageHeight) {
|
||||
float cx = (img.x1() + img.x2()) / 2f;
|
||||
float cy = (img.y1() + img.y2()) / 2f;
|
||||
|
||||
String horiz = cx < pageWidth / 3f ? "left" : cx < 2 * pageWidth / 3f ? "center" : "right";
|
||||
// PDF Y increases upward, so higher Y = higher on the page = "top"
|
||||
String vert = cy > 2 * pageHeight / 3f ? "top" : cy > pageHeight / 3f ? "middle" : "bottom";
|
||||
return vert + "-" + horiz;
|
||||
}
|
||||
|
||||
private ExtractedFileText buildExtractedFileText(
|
||||
String fileName, List<AiWorkflowTextSelection> pages) {
|
||||
ExtractedFileText fileText = new ExtractedFileText();
|
||||
@@ -347,6 +423,130 @@ public class PdfContentExtractor {
|
||||
return text.substring(0, end);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Text position finding
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* A located text match inside a PDF: 0-based page index and bounding box in PDFBox coordinates
|
||||
* (origin bottom-left).
|
||||
*/
|
||||
public record TextBlock(int pageIndex, float x1, float y1, float x2, float y2) {}
|
||||
|
||||
/**
|
||||
* An image found on a PDF page: 0-based page index and bounding box in PDF user-space
|
||||
* coordinates (origin bottom-left, Y increases upward).
|
||||
*/
|
||||
public record ImageBlock(int pageIndex, float x1, float y1, float x2, float y2) {}
|
||||
|
||||
/**
|
||||
* Extract the bounding boxes of all raster/vector images on the given (0-based) page.
|
||||
*
|
||||
* @param document the open PDF
|
||||
* @param pageIndex 0-based page index
|
||||
* @return list of located images in document order
|
||||
*/
|
||||
public List<ImageBlock> extractImagePositions(PDDocument document, int pageIndex)
|
||||
throws IOException {
|
||||
PDPage page = document.getPage(pageIndex);
|
||||
PageImageLocator locator = new PageImageLocator(page, pageIndex);
|
||||
locator.processPage(page);
|
||||
return locator.getImageBoxes().stream()
|
||||
.map(b -> new ImageBlock(b.pageIndex(), b.x1(), b.y1(), b.x2(), b.y2()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all occurrences of {@code pattern} in {@code document} and return their bounding boxes.
|
||||
*
|
||||
* @param document the open PDF
|
||||
* @param pattern the search string or regex
|
||||
* @param useRegex {@code true} to treat {@code pattern} as a regular expression
|
||||
* @return list of located matches, in page order
|
||||
*/
|
||||
public List<TextBlock> findTextPositions(PDDocument document, String pattern, boolean useRegex)
|
||||
throws IOException {
|
||||
LocalTextFinder finder = new LocalTextFinder(pattern, useRegex);
|
||||
finder.getText(document);
|
||||
return finder.found;
|
||||
}
|
||||
|
||||
private static final class LocalTextFinder extends PDFTextStripper {
|
||||
|
||||
private final String searchTerm;
|
||||
private final boolean useRegex;
|
||||
final List<TextBlock> found = new ArrayList<>();
|
||||
|
||||
private final List<TextPosition> pagePositions = new ArrayList<>();
|
||||
private final StringBuilder pageText = new StringBuilder();
|
||||
|
||||
LocalTextFinder(String searchTerm, boolean useRegex) throws IOException {
|
||||
this.searchTerm = searchTerm;
|
||||
this.useRegex = useRegex;
|
||||
setWordSeparator(" ");
|
||||
setLineSeparator("\n");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startPage(PDPage page) throws IOException {
|
||||
super.startPage(page);
|
||||
pagePositions.clear();
|
||||
pageText.setLength(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeString(String text, List<TextPosition> positions) {
|
||||
pageText.append(text);
|
||||
pagePositions.addAll(positions);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeWordSeparator() {
|
||||
pageText.append(getWordSeparator());
|
||||
pagePositions.add(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void writeLineSeparator() {
|
||||
pageText.append(getLineSeparator());
|
||||
pagePositions.add(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void endPage(PDPage page) throws IOException {
|
||||
String text = pageText.toString();
|
||||
if (!text.isEmpty() && searchTerm != null && !searchTerm.isBlank()) {
|
||||
String term = searchTerm.trim();
|
||||
String regex = useRegex ? term : "\\Q" + term + "\\E";
|
||||
Pattern pat = RegexPatternUtils.getInstance().createSearchPattern(regex, true);
|
||||
Matcher matcher = pat.matcher(text);
|
||||
while (matcher.find()) {
|
||||
float minX = Float.MAX_VALUE;
|
||||
float minY = Float.MAX_VALUE;
|
||||
float maxX = -Float.MAX_VALUE;
|
||||
float maxY = -Float.MAX_VALUE;
|
||||
boolean hit = false;
|
||||
for (int i = matcher.start(); i < matcher.end(); i++) {
|
||||
if (i < pagePositions.size()) {
|
||||
TextPosition tp = pagePositions.get(i);
|
||||
if (tp != null) {
|
||||
hit = true;
|
||||
minX = Math.min(minX, tp.getX());
|
||||
maxX = Math.max(maxX, tp.getX() + tp.getWidth());
|
||||
minY = Math.min(minY, tp.getY() - tp.getHeight());
|
||||
maxY = Math.max(maxY, tp.getY());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (hit) {
|
||||
found.add(new TextBlock(getCurrentPageNo() - 1, minX, minY, maxX, maxY));
|
||||
}
|
||||
}
|
||||
}
|
||||
super.endPage(page);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Types shared with AiWorkflowService (package-private) ---
|
||||
|
||||
interface PdfContentResult {
|
||||
|
||||
Reference in New Issue
Block a user