mirror of
https://github.com/arsvendg/Stirling-PDF.git
synced 2026-07-16 11:23:10 +02:00
Change AI engine to execute tools in Java instead of on frontend (#6116)
# Description of Changes Redesign AI engine so that it autogenerates the `tool_models.py` file from the OpenAPI spec so the Python has access to the Java API parameters and the full list of Java tools that it can run. CI ensures that whenever someone modifies a tool endpoint that the AI enigne tool models get updated as well (the dev gets told to run `task engine:tool-models`). There's loads of advantages to having the Java be the one that actually executes the tools, rather than the frontend as it was previously set up to theoretically use: - The AI gets much better descriptions of the params from the API docs - It'll be usable headless in the future so a Java daemon could run to execute ops on files in a folder without the need for the UI to run - The Java already has all the logic it needs to execute the tools - We don't need to parse the TypeScript to find the API (which is hard because the TS wasn't designed to be computer-read to extract the API) I've also hooked up the prototype frontend to ensure it's working properly, and have built it in a way that all the tool names can be translated properly, which was always an issue with previous prototypes of this. --------- Co-authored-by: Anthony Stirling <[email protected]> Co-authored-by: EthanHealy01 <[email protected]>
This commit is contained in:
James Brunton
co-authored by
Anthony Stirling
EthanHealy01
Anthony Stirling
EthanHealy01
parent
cc9650e7a3
commit
e5767ed58b
+3
-1
@@ -124,7 +124,9 @@ public class SplitPdfByChaptersController {
|
||||
@MultiFileResponse
|
||||
@Operation(
|
||||
summary = "Split PDFs by Chapters",
|
||||
description = "Splits a PDF into chapters and returns a ZIP file.")
|
||||
description =
|
||||
"Splits a PDF into chapters and returns a ZIP file. Input:PDF Output:ZIP-PDF"
|
||||
+ " Type:SISO")
|
||||
public ResponseEntity<StreamingResponseBody> splitPdf(
|
||||
@ModelAttribute SplitPdfByChaptersRequest request) throws Exception {
|
||||
MultipartFile file = request.getFileInput();
|
||||
|
||||
+19
-200
@@ -11,36 +11,26 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.*;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RequestCallback;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.github.pixee.security.Filenames;
|
||||
import io.github.pixee.security.ZipSecurity;
|
||||
|
||||
import jakarta.servlet.ServletContext;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.SPDF.SPDFApplication;
|
||||
import stirling.software.SPDF.model.PipelineConfig;
|
||||
import stirling.software.SPDF.model.PipelineOperation;
|
||||
import stirling.software.SPDF.model.PipelineResult;
|
||||
import stirling.software.SPDF.service.ApiDocService;
|
||||
import stirling.software.common.model.enumeration.Role;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.common.util.TempFile;
|
||||
import stirling.software.common.service.InternalApiClient;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.ZipExtractionUtils;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@@ -48,20 +38,16 @@ public class PipelineProcessor {
|
||||
|
||||
private final ApiDocService apiDocService;
|
||||
|
||||
private final UserServiceInterface userService;
|
||||
|
||||
private final ServletContext servletContext;
|
||||
private final InternalApiClient internalApiClient;
|
||||
|
||||
private final TempFileManager tempFileManager;
|
||||
|
||||
public PipelineProcessor(
|
||||
ApiDocService apiDocService,
|
||||
@Autowired(required = false) UserServiceInterface userService,
|
||||
ServletContext servletContext,
|
||||
InternalApiClient internalApiClient,
|
||||
TempFileManager tempFileManager) {
|
||||
this.apiDocService = apiDocService;
|
||||
this.userService = userService;
|
||||
this.servletContext = servletContext;
|
||||
this.internalApiClient = internalApiClient;
|
||||
this.tempFileManager = tempFileManager;
|
||||
}
|
||||
|
||||
@@ -84,48 +70,6 @@ public class PipelineProcessor {
|
||||
return name.substring(0, underscoreIndex) + extension;
|
||||
}
|
||||
|
||||
// Allowlist of URL path prefixes permitted through the pipeline.
|
||||
private static final List<String> ALLOWED_PIPELINE_PATH_PREFIXES =
|
||||
List.of(
|
||||
"/api/v1/general/",
|
||||
"/api/v1/misc/",
|
||||
"/api/v1/security/",
|
||||
"/api/v1/convert/",
|
||||
"/api/v1/filter/");
|
||||
|
||||
private void validatePipelineUrl(String url) {
|
||||
// Strip scheme+host to get the path portion for comparison
|
||||
String path = url;
|
||||
int schemeEnd = url.indexOf("://");
|
||||
if (schemeEnd != -1) {
|
||||
int pathStart = url.indexOf('/', schemeEnd + 3);
|
||||
path = pathStart != -1 ? url.substring(pathStart) : "/";
|
||||
}
|
||||
final String pathToCheck = path;
|
||||
boolean allowed = ALLOWED_PIPELINE_PATH_PREFIXES.stream().anyMatch(pathToCheck::contains);
|
||||
if (!allowed) {
|
||||
log.warn("Blocked pipeline request to disallowed URL: {}", url);
|
||||
throw new SecurityException(
|
||||
"Pipeline operation not permitted for endpoint: " + pathToCheck);
|
||||
}
|
||||
}
|
||||
|
||||
private String getApiKeyForUser() {
|
||||
if (userService == null) return "";
|
||||
String username = userService.getCurrentUsername();
|
||||
if (username != null && !username.equals("anonymousUser")) {
|
||||
return userService.getApiKeyForUser(username);
|
||||
}
|
||||
// Scheduled/internal context — no user in security context
|
||||
return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId());
|
||||
}
|
||||
|
||||
private String getBaseUrl() {
|
||||
String contextPath = servletContext.getContextPath();
|
||||
String port = SPDFApplication.getStaticPort();
|
||||
return "http://localhost:" + port + contextPath + "/";
|
||||
}
|
||||
|
||||
PipelineResult runPipelineAgainstFiles(List<Resource> outputFiles, PipelineConfig config)
|
||||
throws Exception {
|
||||
PipelineResult result = new PipelineResult();
|
||||
@@ -153,7 +97,6 @@ public class PipelineProcessor {
|
||||
"Invalid operation: " + operation + " with parameters: " + parameters);
|
||||
}
|
||||
|
||||
String url = getBaseUrl() + operation;
|
||||
List<Resource> newOutputFiles = new ArrayList<>();
|
||||
if (!isMultiInputOperation) {
|
||||
for (Resource file : outputFiles) {
|
||||
@@ -175,12 +118,15 @@ public class PipelineProcessor {
|
||||
body.add(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
ResponseEntity<Resource> response = sendWebRequest(url, body);
|
||||
ResponseEntity<Resource> response =
|
||||
internalApiClient.post(operation, body);
|
||||
// If the operation is filter and the response body is null or empty,
|
||||
// skip
|
||||
// this
|
||||
// file
|
||||
if (response.getBody() instanceof TempFileResource tempFileResource) {
|
||||
if (response.getBody()
|
||||
instanceof
|
||||
InternalApiClient.TempFileResource tempFileResource) {
|
||||
result.addTempFile(tempFileResource.getTempFile());
|
||||
}
|
||||
|
||||
@@ -257,8 +203,9 @@ public class PipelineProcessor {
|
||||
body.add(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
ResponseEntity<Resource> response = sendWebRequest(url, body);
|
||||
if (response.getBody() instanceof TempFileResource tempFileResource) {
|
||||
ResponseEntity<Resource> response = internalApiClient.post(operation, body);
|
||||
if (response.getBody()
|
||||
instanceof InternalApiClient.TempFileResource tempFileResource) {
|
||||
result.addTempFile(tempFileResource.getTempFile());
|
||||
}
|
||||
// Handle the response
|
||||
@@ -312,43 +259,6 @@ public class PipelineProcessor {
|
||||
return result;
|
||||
}
|
||||
|
||||
/* package */ ResponseEntity<Resource> sendWebRequest(
|
||||
String url, MultiValueMap<String, Object> body) {
|
||||
validatePipelineUrl(url);
|
||||
RestTemplate restTemplate = new RestTemplate();
|
||||
// Set up headers, including API key
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
String apiKey = getApiKeyForUser();
|
||||
if (apiKey != null && !apiKey.isEmpty()) {
|
||||
headers.add("X-API-KEY", apiKey);
|
||||
}
|
||||
|
||||
// Let the message converter set the multipart boundary/content type
|
||||
HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
|
||||
|
||||
RequestCallback requestCallback =
|
||||
restTemplate.httpEntityCallback(entity, Resource.class /* response type hint */);
|
||||
return restTemplate.execute(
|
||||
url,
|
||||
HttpMethod.POST,
|
||||
requestCallback,
|
||||
response -> {
|
||||
try {
|
||||
TempFile tempFile = tempFileManager.createManagedTempFile("pipeline");
|
||||
Files.copy(
|
||||
response.getBody(),
|
||||
tempFile.getPath(),
|
||||
java.nio.file.StandardCopyOption.REPLACE_EXISTING);
|
||||
TempFileResource resource = new TempFileResource(tempFile);
|
||||
return ResponseEntity.status(response.getStatusCode())
|
||||
.headers(response.getHeaders())
|
||||
.body(resource);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private List<Resource> processOutputFiles(
|
||||
String operation,
|
||||
ResponseEntity<Resource> response,
|
||||
@@ -367,13 +277,15 @@ public class PipelineProcessor {
|
||||
newFilename = removeTrailingNaming(extractFilename(response));
|
||||
}
|
||||
// Check if the response body is a zip file
|
||||
if (isZip(response.getBody(), newFilename)) {
|
||||
if (ZipExtractionUtils.isZip(response.getBody(), newFilename)) {
|
||||
// Unzip the file and add all the files to the new output files
|
||||
newOutputFiles.addAll(unzip(response.getBody(), result));
|
||||
newOutputFiles.addAll(
|
||||
ZipExtractionUtils.extractZip(
|
||||
response.getBody(), tempFileManager, result::addTempFile));
|
||||
} else {
|
||||
final Resource tempResource = response.getBody();
|
||||
if (tempResource instanceof TempFileResource) {
|
||||
result.addTempFile(((TempFileResource) tempResource).getTempFile());
|
||||
if (tempResource instanceof InternalApiClient.TempFileResource tfr) {
|
||||
result.addTempFile(tfr.getTempFile());
|
||||
}
|
||||
Resource outputResource =
|
||||
new FileSystemResource(tempResource.getFile()) {
|
||||
@@ -456,97 +368,4 @@ public class PipelineProcessor {
|
||||
log.info("Files successfully loaded. Starting processing...");
|
||||
return outputFiles;
|
||||
}
|
||||
|
||||
private boolean isZip(Resource data, String filename) throws IOException {
|
||||
if (data == null || data.contentLength() < 4) {
|
||||
return false;
|
||||
}
|
||||
if (filename != null) {
|
||||
String lower = filename.toLowerCase();
|
||||
if (lower.endsWith(".cbz")) {
|
||||
// Treat CBZ as non-zip for our unzipping purposes
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Check the first four bytes of the data against the standard zip magic number
|
||||
try (InputStream is = data.getInputStream()) {
|
||||
byte[] header = new byte[4];
|
||||
if (is.read(header) < 4) {
|
||||
return false;
|
||||
}
|
||||
return header[0] == 0x50 && header[1] == 0x4B && header[2] == 0x03 && header[3] == 0x04;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isZip(Resource data) throws IOException {
|
||||
return isZip(data, null);
|
||||
}
|
||||
|
||||
private static final int MAX_UNZIP_DEPTH = 10;
|
||||
|
||||
private List<Resource> unzip(Resource data, PipelineResult result) throws IOException {
|
||||
return unzip(data, result, 0);
|
||||
}
|
||||
|
||||
private List<Resource> unzip(Resource data, PipelineResult result, int depth)
|
||||
throws IOException {
|
||||
if (depth > MAX_UNZIP_DEPTH) {
|
||||
log.warn(
|
||||
"ZIP nesting depth {} exceeds limit {}, treating as file",
|
||||
depth,
|
||||
MAX_UNZIP_DEPTH);
|
||||
return List.of(data);
|
||||
}
|
||||
log.info("Unzipping data of length: {}", data.contentLength());
|
||||
List<Resource> unzippedFiles = new ArrayList<>();
|
||||
try (InputStream bais = data.getInputStream();
|
||||
ZipInputStream zis = ZipSecurity.createHardenedInputStream(bais)) {
|
||||
ZipEntry entry;
|
||||
while ((entry = zis.getNextEntry()) != null) {
|
||||
if (entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
TempFile tempFile = tempFileManager.createManagedTempFile("unzip");
|
||||
result.addTempFile(tempFile);
|
||||
try (OutputStream os = Files.newOutputStream(tempFile.getPath())) {
|
||||
byte[] buffer = new byte[4096];
|
||||
int count;
|
||||
while ((count = zis.read(buffer)) != -1) {
|
||||
os.write(buffer, 0, count);
|
||||
}
|
||||
}
|
||||
final String filename = entry.getName();
|
||||
Resource fileResource =
|
||||
new FileSystemResource(tempFile.getFile()) {
|
||||
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
};
|
||||
// If the unzipped file is a zip file, unzip it
|
||||
if (isZip(fileResource, filename)) {
|
||||
log.info("File {} is a zip file. Unzipping...", filename);
|
||||
unzippedFiles.addAll(unzip(fileResource, result, depth + 1));
|
||||
} else {
|
||||
unzippedFiles.add(fileResource);
|
||||
}
|
||||
}
|
||||
}
|
||||
log.info("Unzipping completed. {} files were unzipped.", unzippedFiles.size());
|
||||
return unzippedFiles;
|
||||
}
|
||||
|
||||
private static class TempFileResource extends FileSystemResource {
|
||||
private final TempFile tempFile;
|
||||
|
||||
public TempFileResource(TempFile tempFile) {
|
||||
super(tempFile.getFile());
|
||||
this.tempFile = tempFile;
|
||||
}
|
||||
|
||||
public TempFile getTempFile() {
|
||||
return tempFile;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -13,7 +13,8 @@ public class RotatePDFRequest extends PDFFile {
|
||||
|
||||
@Schema(
|
||||
description =
|
||||
"The angle by which to rotate the PDF file. This should be a multiple of 90.",
|
||||
"The clockwise angle by which to rotate the PDF file. Must be a multiple of"
|
||||
+ " 90.",
|
||||
type = "integer",
|
||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||
allowableValues = {"0", "90", "180", "270"})
|
||||
|
||||
@@ -6,6 +6,7 @@ import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpEntity;
|
||||
@@ -30,7 +31,14 @@ import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ApiDocService {
|
||||
public class ApiDocService implements stirling.software.common.service.ToolMetadataService {
|
||||
|
||||
// Matches a bare "Output:ZIP" declaration (i.e. ZIP is not followed by "-" or "/").
|
||||
// Bare ZIP means the archive itself is the deliverable (e.g. get-attachments), so it
|
||||
// should not be auto-unpacked. Wrapper forms like Output:ZIP-PDF or Output:IMAGE/ZIP
|
||||
// use ZIP as transport for multiple typed results and are safe to unpack.
|
||||
private static final Pattern BARE_ZIP_OUTPUT =
|
||||
Pattern.compile("Output\\s*:\\s*ZIP(?![-/])", Pattern.CASE_INSENSITIVE);
|
||||
|
||||
private final Map<String, ApiEndpoint> apiDocumentation = new HashMap<>();
|
||||
|
||||
@@ -149,6 +157,7 @@ public class ApiDocService {
|
||||
return endpoint.areParametersValid(parameters);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiInput(String operationName) {
|
||||
if (apiDocsJsonRootNode == null || apiDocumentation.isEmpty()) {
|
||||
loadApiDocumentation();
|
||||
@@ -166,5 +175,36 @@ public class ApiDocService {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean shouldUnpackZipResponse(String operationName) {
|
||||
if (apiDocsJsonRootNode == null || apiDocumentation.isEmpty()) {
|
||||
loadApiDocumentation();
|
||||
}
|
||||
if (!apiDocumentation.containsKey(operationName)) {
|
||||
return false;
|
||||
}
|
||||
ApiEndpoint endpoint = apiDocumentation.get(operationName);
|
||||
String description = endpoint.getDescription();
|
||||
Matcher typeMatcher =
|
||||
RegexPatternUtils.getInstance().getApiDocTypePattern().matcher(description);
|
||||
if (typeMatcher.find()) {
|
||||
String type = typeMatcher.group(1);
|
||||
// Multi-output endpoints (SIMO/MIMO) return a ZIP of their outputs.
|
||||
if (type.endsWith("MO")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Matcher outputMatcher =
|
||||
RegexPatternUtils.getInstance().getApiDocOutputTypePattern().matcher(description);
|
||||
if (outputMatcher.find()) {
|
||||
String output = outputMatcher.group(1).toUpperCase(Locale.ROOT);
|
||||
if (output.startsWith("ZIP")) {
|
||||
// Bare "Output:ZIP" is a single-archive deliverable, not a transport.
|
||||
return !BARE_ZIP_OUTPUT.matcher(description).find();
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Model class for API Endpoint
|
||||
|
||||
+8
-110
@@ -4,8 +4,6 @@ import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
@@ -15,24 +13,18 @@ import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockedConstruction;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.http.client.ClientHttpResponse;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
|
||||
import jakarta.servlet.ServletContext;
|
||||
|
||||
import stirling.software.SPDF.model.PipelineConfig;
|
||||
import stirling.software.SPDF.model.PipelineOperation;
|
||||
import stirling.software.SPDF.model.PipelineResult;
|
||||
import stirling.software.SPDF.service.ApiDocService;
|
||||
import stirling.software.common.service.UserServiceInterface;
|
||||
import stirling.software.common.service.InternalApiClient;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@@ -40,9 +32,7 @@ class PipelineProcessorTest {
|
||||
|
||||
@Mock ApiDocService apiDocService;
|
||||
|
||||
@Mock UserServiceInterface userService;
|
||||
|
||||
@Mock ServletContext servletContext;
|
||||
@Mock InternalApiClient internalApiClient;
|
||||
|
||||
@Mock TempFileManager tempFileManager;
|
||||
|
||||
@@ -51,9 +41,7 @@ class PipelineProcessorTest {
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
pipelineProcessor =
|
||||
spy(
|
||||
new PipelineProcessor(
|
||||
apiDocService, userService, servletContext, tempFileManager));
|
||||
new PipelineProcessor(apiDocService, internalApiClient, tempFileManager);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -65,7 +53,6 @@ class PipelineProcessorTest {
|
||||
config.setOperations(List.of(op));
|
||||
|
||||
Resource file = new MyFileByteArrayResource();
|
||||
|
||||
List<Resource> files = List.of(file);
|
||||
|
||||
when(apiDocService.isMultiInput("/api/v1/filter/filter-page-count")).thenReturn(false);
|
||||
@@ -74,13 +61,11 @@ class PipelineProcessorTest {
|
||||
when(apiDocService.isValidOperation(eq("/api/v1/filter/filter-page-count"), anyMap()))
|
||||
.thenReturn(true);
|
||||
|
||||
// Use a FileSystemResource backed by a temp file to avoid FileNotFoundException
|
||||
Path emptyTemp = Files.createTempFile("empty", ".tmp");
|
||||
Resource emptyResource = new FileSystemResource(emptyTemp.toFile());
|
||||
|
||||
doReturn(new ResponseEntity<>(emptyResource, HttpStatus.OK))
|
||||
.when(pipelineProcessor)
|
||||
.sendWebRequest(anyString(), any());
|
||||
when(internalApiClient.post(anyString(), any()))
|
||||
.thenReturn(new ResponseEntity<>(emptyResource, HttpStatus.OK));
|
||||
|
||||
PipelineResult result = pipelineProcessor.runPipelineAgainstFiles(files, config);
|
||||
|
||||
@@ -118,105 +103,18 @@ class PipelineProcessorTest {
|
||||
when(apiDocService.getExtensionTypes(anyBoolean(), anyString())).thenReturn(List.of("pdf"));
|
||||
when(apiDocService.isValidOperation(anyString(), anyMap())).thenReturn(true);
|
||||
|
||||
doReturn(new ResponseEntity<>(outputResource, HttpStatus.OK))
|
||||
.when(pipelineProcessor)
|
||||
.sendWebRequest(anyString(), any());
|
||||
when(internalApiClient.post(anyString(), any()))
|
||||
.thenReturn(new ResponseEntity<>(outputResource, HttpStatus.OK));
|
||||
|
||||
PipelineResult result = pipelineProcessor.runPipelineAgainstFiles(files, config);
|
||||
|
||||
verify(pipelineProcessor).sendWebRequest(anyString(), any());
|
||||
verify(internalApiClient).post(anyString(), any());
|
||||
|
||||
assertFalse(result.isHasErrors());
|
||||
|
||||
// Clean up
|
||||
Files.deleteIfExists(tempPath);
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendWebRequestDoesNotForceContentType() throws Exception {
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add(
|
||||
"fileInput",
|
||||
new ByteArrayResource("data".getBytes(StandardCharsets.UTF_8)) {
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return "input.pdf";
|
||||
}
|
||||
});
|
||||
|
||||
Path tempPath = Files.createTempFile("pipeline-test", ".tmp");
|
||||
var tempFile = mock(stirling.software.common.util.TempFile.class);
|
||||
when(tempFile.getPath()).thenReturn(tempPath);
|
||||
when(tempFile.getFile()).thenReturn(tempPath.toFile());
|
||||
when(tempFileManager.createManagedTempFile("pipeline")).thenReturn(tempFile);
|
||||
|
||||
var capturedHeaders = new org.springframework.http.HttpHeaders[1];
|
||||
|
||||
try (MockedConstruction<org.springframework.web.client.RestTemplate> ignored =
|
||||
mockConstruction(
|
||||
org.springframework.web.client.RestTemplate.class,
|
||||
(mock, context) -> {
|
||||
when(mock.httpEntityCallback(any(), eq(Resource.class)))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
var entity = invocation.getArgument(0);
|
||||
capturedHeaders[0] =
|
||||
((org.springframework.http.HttpEntity<?>)
|
||||
entity)
|
||||
.getHeaders();
|
||||
return (org.springframework.web.client
|
||||
.RequestCallback)
|
||||
request -> {};
|
||||
});
|
||||
|
||||
when(mock.execute(
|
||||
anyString(),
|
||||
eq(org.springframework.http.HttpMethod.POST),
|
||||
any(),
|
||||
any()))
|
||||
.thenAnswer(
|
||||
invocation -> {
|
||||
@SuppressWarnings("unchecked")
|
||||
var extractor =
|
||||
(org.springframework.web.client
|
||||
.ResponseExtractor<
|
||||
ResponseEntity<Resource>>)
|
||||
invocation.getArgument(3);
|
||||
ClientHttpResponse response =
|
||||
mock(ClientHttpResponse.class);
|
||||
when(response.getBody())
|
||||
.thenReturn(
|
||||
new ByteArrayInputStream(
|
||||
"ok"
|
||||
.getBytes(
|
||||
StandardCharsets
|
||||
.UTF_8)));
|
||||
var headers =
|
||||
new org.springframework.http.HttpHeaders();
|
||||
headers.add(
|
||||
org.springframework.http.HttpHeaders
|
||||
.CONTENT_DISPOSITION,
|
||||
"attachment; filename=\"out.pdf\"");
|
||||
when(response.getHeaders()).thenReturn(headers);
|
||||
lenient()
|
||||
.when(response.getStatusCode())
|
||||
.thenReturn(HttpStatus.OK);
|
||||
return extractor.extractData(response);
|
||||
});
|
||||
})) {
|
||||
ResponseEntity<Resource> response =
|
||||
pipelineProcessor.sendWebRequest(
|
||||
"http://localhost/api/v1/general/merge-pdfs", body);
|
||||
|
||||
assertNotNull(response);
|
||||
assertEquals(HttpStatus.OK, response.getStatusCode());
|
||||
assertNotNull(response.getBody());
|
||||
assertNull(capturedHeaders[0].getContentType());
|
||||
} finally {
|
||||
Files.deleteIfExists(tempPath);
|
||||
}
|
||||
}
|
||||
|
||||
private static class MyFileByteArrayResource extends ByteArrayResource {
|
||||
public MyFileByteArrayResource() {
|
||||
super("data".getBytes());
|
||||
|
||||
@@ -9,6 +9,8 @@ import java.util.Map;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
@@ -348,6 +350,123 @@ class ApiDocServiceTest {
|
||||
assertTrue(apiDocService.isMultiInput("/miso"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUnpackZipResponseDetectsMultiOutputType() throws Exception {
|
||||
String json = "{\"description\": \"Output:PDF Type:SIMO\"}";
|
||||
JsonNode postNode = mapper.readTree(json);
|
||||
ApiEndpoint endpoint = new ApiEndpoint("/split", postNode);
|
||||
setApiDocumentation(Map.of("/split", endpoint));
|
||||
setApiDocsJsonRootNode();
|
||||
assertTrue(apiDocService.shouldUnpackZipResponse("/split"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUnpackZipResponseDetectsMimoType() throws Exception {
|
||||
String json = "{\"description\": \"Output:PDF Type:MIMO\"}";
|
||||
JsonNode postNode = mapper.readTree(json);
|
||||
ApiEndpoint endpoint = new ApiEndpoint("/overlay", postNode);
|
||||
setApiDocumentation(Map.of("/overlay", endpoint));
|
||||
setApiDocsJsonRootNode();
|
||||
assertTrue(apiDocService.shouldUnpackZipResponse("/overlay"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUnpackZipResponseDetectsZipOutputDeclaration() throws Exception {
|
||||
String json = "{\"description\": \"Output:ZIP-PDF Type:SISO\"}";
|
||||
JsonNode postNode = mapper.readTree(json);
|
||||
ApiEndpoint endpoint = new ApiEndpoint("/split-by-sections", postNode);
|
||||
setApiDocumentation(Map.of("/split-by-sections", endpoint));
|
||||
setApiDocsJsonRootNode();
|
||||
assertTrue(apiDocService.shouldUnpackZipResponse("/split-by-sections"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUnpackZipResponseReturnsFalseForSisoPdf() throws Exception {
|
||||
String json = "{\"description\": \"Input:PDF Output:PDF Type:SISO\"}";
|
||||
JsonNode postNode = mapper.readTree(json);
|
||||
ApiEndpoint endpoint = new ApiEndpoint("/rotate", postNode);
|
||||
setApiDocumentation(Map.of("/rotate", endpoint));
|
||||
setApiDocsJsonRootNode();
|
||||
assertFalse(apiDocService.shouldUnpackZipResponse("/rotate"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUnpackZipResponseReturnsFalseForUnknownOperation() throws Exception {
|
||||
setApiDocumentation(Map.of());
|
||||
assertFalse(apiDocService.shouldUnpackZipResponse("/unknown"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage test: every Stirling endpoint whose ZIP response is a transport for multiple typed
|
||||
* results (SIMO/MIMO or Output:ZIP-PDF / Output:IMAGE/ZIP etc.) must be classified as {@code
|
||||
* shouldUnpackZipResponse = true}. Descriptions below are the real
|
||||
* {@code @Operation(description=...)} strings from each controller, so if a controller is
|
||||
* renamed, tweaked or introduced without a {@code Type:} / {@code Output:ZIP-*} tag, this test
|
||||
* breaks, surfacing the bug before {@code AiWorkflowService} silently registers a multi-result
|
||||
* ZIP as a single file.
|
||||
*
|
||||
* <p>Add a new row here whenever a new unpack-eligible endpoint is introduced. Descriptions can
|
||||
* be trimmed to the part containing the relevant tags.
|
||||
*/
|
||||
@ParameterizedTest(name = "{0} → shouldUnpackZipResponse")
|
||||
@CsvSource(
|
||||
textBlock =
|
||||
"""
|
||||
/api/v1/general/split-pages, 'Split pages. Input:PDF Output:PDF Type:SIMO'
|
||||
/api/v1/general/split-pdf-by-sections, 'Split. Input:PDF Output:ZIP-PDF Type:SISO'
|
||||
/api/v1/general/split-by-size-or-count, 'Split by size. Input:PDF Output:ZIP-PDF Type:SISO'
|
||||
/api/v1/general/split-pdf-by-chapters, 'Split by chapters. Input:PDF Output:ZIP-PDF Type:SISO'
|
||||
/api/v1/general/split-for-poster-print, 'Poster split. Input: PDF Output: ZIP-PDF Type: SISO'
|
||||
/api/v1/general/overlay-pdfs, 'Overlay PDFs. Input:PDF Output:PDF Type:MIMO'
|
||||
/api/v1/misc/auto-split-pdf, 'Auto split. Input:PDF Output:ZIP-PDF Type:SISO'
|
||||
/api/v1/misc/extract-images, 'Extract images. Output:IMAGE/ZIP Type:SIMO'
|
||||
/api/v1/misc/extract-image-scans, 'Extract image scans. Input:PDF Output:IMAGE/ZIP Type:SIMO'
|
||||
""")
|
||||
void shouldUnpackZipResponseClassifiesKnownUnpackableEndpoints(
|
||||
String endpoint, String description) throws Exception {
|
||||
String json = mapper.writeValueAsString(Map.of("description", description));
|
||||
JsonNode postNode = mapper.readTree(json);
|
||||
setApiDocumentation(Map.of(endpoint, new ApiEndpoint(endpoint, postNode)));
|
||||
setApiDocsJsonRootNode();
|
||||
assertTrue(
|
||||
apiDocService.shouldUnpackZipResponse(endpoint),
|
||||
() ->
|
||||
"Expected shouldUnpackZipResponse=true for "
|
||||
+ endpoint
|
||||
+ " with description: "
|
||||
+ description);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inverse coverage: endpoints whose ZIP response is the deliverable itself (or that return
|
||||
* single non-ZIP files) must not be flagged for unpacking. Catches regressions where a change
|
||||
* to the classifier accidentally widens the positive match.
|
||||
*/
|
||||
@ParameterizedTest(name = "{0} → !shouldUnpackZipResponse")
|
||||
@CsvSource(
|
||||
textBlock =
|
||||
"""
|
||||
/api/v1/general/rotate-pdf, 'Rotate. Input:PDF Output:PDF Type:SISO'
|
||||
/api/v1/general/merge-pdfs, 'Merge. Input:PDF Output:PDF Type:MISO'
|
||||
/api/v1/misc/compress-pdf, 'Compress. Input:PDF Output:PDF Type:SISO'
|
||||
/api/v1/misc/flatten, 'Flatten forms. Input:PDF Output:PDF Type:SISO'
|
||||
/api/v1/security/get-attachments, 'Extract attachments. Input:PDF Output:ZIP Type:SISO'
|
||||
""")
|
||||
void shouldUnpackZipResponseRejectsNonUnpackableEndpoints(String endpoint, String description)
|
||||
throws Exception {
|
||||
String json = mapper.writeValueAsString(Map.of("description", description));
|
||||
JsonNode postNode = mapper.readTree(json);
|
||||
setApiDocumentation(Map.of(endpoint, new ApiEndpoint(endpoint, postNode)));
|
||||
setApiDocsJsonRootNode();
|
||||
assertFalse(
|
||||
apiDocService.shouldUnpackZipResponse(endpoint),
|
||||
() ->
|
||||
"Expected shouldUnpackZipResponse=false for "
|
||||
+ endpoint
|
||||
+ " with description: "
|
||||
+ description);
|
||||
}
|
||||
|
||||
@Test
|
||||
void constructorAcceptsNullUserService() {
|
||||
ApiDocService service = new ApiDocService(mapper, servletContext, null);
|
||||
|
||||
Reference in New Issue
Block a user