arsvendg/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/arsvendg/Stirling-PDF.git synced 2026-07-16 19:33:11 +02:00
Code Issues Actions 3 Packages Projects Releases 1 Wiki Activity

Support multiple pipeline watch directories and configurable pipeline base path (#5545)

Browse Source 
### Motivation
- Allow operators to configure a pipeline base directory and multiple
watched folders so the pipeline can monitor several directories and
subdirectories concurrently.
- Ensure scanning traverses subdirectories while skipping internal
processing folders (e.g. `processing`) and preserve existing behavior
for finished/output paths.
- Expose the new options in the server `settings.yml.template` and the
admin UI so paths can be edited from the web console.

### Description
- Added new `pipelineDir` and `watchedFoldersDirs` fields to
`ApplicationProperties.CustomPaths.Pipeline` and kept backward
compatibility with `watchedFoldersDir`
(app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java).
- Resolved pipeline base and multiple watched folder paths in
`RuntimePathConfig` and exposed `getPipelineWatchedFoldersPaths()`
(app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java).
- Updated `FileMonitor` to accept and register multiple root paths
instead of a single root
(app/common/src/main/java/stirling/software/common/util/FileMonitor.java).
- Updated `PipelineDirectoryProcessor` to iterate all configured watched
roots and to walk subdirectories while ignoring `processing` dirs
(app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java).
- Exposed the new settings in `settings.yml.template` and the admin UI,
including a multi-line `Textarea` to edit `watchedFoldersDirs`
(app/core/src/main/resources/settings.yml.template,
frontend/src/proprietary/components/shared/config/configSections/AdminGeneralSection.tsx).
- Adjusted unit test setup to account for list-based watched folders
(app/common/src/test/java/stirling/software/common/util/FileMonitorTest.java).

### Testing
- Ran formatting and build checks with `./gradlew spotlessApply` and
`./gradlew build` using Java 21 via
`JAVA_HOME=/root/.local/share/mise/installs/java/21.0.2
PATH=/root/.local/share/mise/installs/java/21.0.2/bin:$PATH ./gradlew
...`, but both runs failed due to Gradle plugin resolution being blocked
in this environment (plugin portal/network 403), so full
compilation/formatting could not complete.
- Confirmed the code compiles locally was not possible here; unit test
`FileMonitorTest` was updated to use the new API but was not executed
due to the blocked build.
- Changes were committed (`Support multiple pipeline watch directories`)
and the repository diff contains the listed file modifications.

------
[Codex
Task](https://chatgpt.com/codex/tasks/task_b_69741ecd17c883288d8085a63ccd66f4)
This commit is contained in:
Anthony Stirling
2026-01-31 20:59:25 +00:00
committed by GitHub
parent 2ae413c5ea
commit 4f404a1ccf
9 changed files with 452 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java
+155 -6
View File
@@ -1,10 +1,14 @@
package stirling.software.common.configuration;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Configuration;
@@ -41,6 +45,7 @@ public class RuntimePathConfig {
// Pipeline paths
private final String pipelineWatchedFoldersPath;
private final List<String> pipelineWatchedFoldersPaths;
private final String pipelineFinishedFoldersPath;
private final String pipelineDefaultWebUiConfigs;
private final String pipelinePath;
@@ -49,20 +54,27 @@ public class RuntimePathConfig {
this.properties = properties;
this.basePath = InstallationPathConfig.getPath();
this.pipelinePath = Path.of(basePath, "pipeline").toString();
String defaultWatchedFolders = Path.of(this.pipelinePath, "watchedFolders").toString();
String defaultFinishedFolders = Path.of(this.pipelinePath, "finishedFolders").toString();
String defaultWebUIConfigs = Path.of(this.pipelinePath, "defaultWebUIConfigs").toString();
System system = properties.getSystem();
CustomPaths customPaths = system.getCustomPaths();
Pipeline pipeline = customPaths.getPipeline();
this.pipelineWatchedFoldersPath =
this.pipelinePath =
resolvePath(
Path.of(basePath, "pipeline").toString(),
pipeline != null ? pipeline.getPipelineDir() : null);
String defaultWatchedFolders = Path.of(this.pipelinePath, "watchedFolders").toString();
String defaultFinishedFolders = Path.of(this.pipelinePath, "finishedFolders").toString();
String defaultWebUIConfigs = Path.of(this.pipelinePath, "defaultWebUIConfigs").toString();
List<String> watchedFoldersDirs =
sanitizePathList(pipeline != null ? pipeline.getWatchedFoldersDirs() : null);
this.pipelineWatchedFoldersPaths =
resolveWatchedFolderPaths(
defaultWatchedFolders,
watchedFoldersDirs,
pipeline != null ? pipeline.getWatchedFoldersDir() : null);
this.pipelineWatchedFoldersPath = this.pipelineWatchedFoldersPaths.get(0);
this.pipelineFinishedFoldersPath =
resolvePath(
defaultFinishedFolders,
@@ -72,6 +84,9 @@ public class RuntimePathConfig {
defaultWebUIConfigs,
pipeline != null ? pipeline.getWebUIConfigsDir() : null);
// Validate path conflicts after all paths are resolved
validatePipelinePaths();
boolean isDocker = isRunningInDocker();
// Initialize Operation paths
@@ -129,6 +144,140 @@ public class RuntimePathConfig {
return StringUtils.isNotBlank(customPath) ? customPath : defaultPath;
}
private List<String> resolveWatchedFolderPaths(
String defaultPath, List<String> watchedFoldersDirs, String legacyWatchedFolder) {
List<String> rawPaths = new ArrayList<>();
// Collect paths from new config
if (watchedFoldersDirs != null && !watchedFoldersDirs.isEmpty()) {
rawPaths.addAll(watchedFoldersDirs);
}
// Fall back to legacy config
else if (StringUtils.isNotBlank(legacyWatchedFolder)) {
rawPaths.add(legacyWatchedFolder);
}
// Fall back to default
else {
rawPaths.add(defaultPath);
}
// Validate, normalize, and deduplicate paths
List<String> validatedPaths = validateAndNormalizePaths(rawPaths);
// Ensure we have at least one valid path (critical for system to function)
if (validatedPaths.isEmpty()) {
log.warn(
"No valid watched folder paths configured, falling back to default: {}",
defaultPath);
validatedPaths.add(defaultPath);
}
// Detect overlapping paths (warning only, not blocking)
detectOverlappingPaths(validatedPaths);
return validatedPaths;
}
private List<String> sanitizePathList(List<String> paths) {
if (paths == null || paths.isEmpty()) {
return Collections.emptyList();
}
List<String> sanitized = new ArrayList<>();
for (String path : paths) {
if (StringUtils.isNotBlank(path)) {
sanitized.add(path.trim());
}
}
return sanitized;
}
private List<String> validateAndNormalizePaths(List<String> paths) {
Set<String> normalizedPaths = new LinkedHashSet<>(); // Preserves order, prevents duplicates
for (String pathStr : paths) {
if (StringUtils.isBlank(pathStr)) {
continue;
}
try {
// Normalize to absolute path
Path path = Paths.get(pathStr.trim()).toAbsolutePath().normalize();
String normalizedPath = path.toString();
// Check for duplicates
if (normalizedPaths.contains(normalizedPath)) {
log.debug("Skipping duplicate watched folder path: {}", pathStr);
continue;
}
normalizedPaths.add(normalizedPath);
log.info("Registered watched folder path: {}", normalizedPath);
} catch (InvalidPathException e) {
log.error(
"Invalid watched folder path '{}' - skipping: {}", pathStr, e.getMessage());
}
}
return new ArrayList<>(normalizedPaths);
}
private void detectOverlappingPaths(List<String> paths) {
for (int i = 0; i < paths.size(); i++) {
Path path1 = Paths.get(paths.get(i));
for (int j = i + 1; j < paths.size(); j++) {
Path path2 = Paths.get(paths.get(j));
// Check if one path is a parent of the other
if (path1.startsWith(path2)) {
log.warn(
"Watched folder path '{}' is nested inside '{}' - this may cause duplicate processing",
path1,
path2);
} else if (path2.startsWith(path1)) {
log.warn(
"Watched folder path '{}' is nested inside '{}' - this may cause duplicate processing",
path2,
path1);
}
}
}
}
private void validatePipelinePaths() {
try {
Path finishedPath = Paths.get(pipelineFinishedFoldersPath).toAbsolutePath().normalize();
for (String watchedPathStr : pipelineWatchedFoldersPaths) {
Path watchedPath = Paths.get(watchedPathStr).toAbsolutePath().normalize();
// Check if watched folder is same as finished folder
if (watchedPath.equals(finishedPath)) {
log.error(
"CRITICAL: Watched folder '{}' is the same as finished folder '{}' - this will cause processing loops!",
watchedPath,
finishedPath);
}
// Check if watched folder contains finished folder
else if (finishedPath.startsWith(watchedPath)) {
log.warn(
"Finished folder '{}' is nested inside watched folder '{}' - this may cause issues",
finishedPath,
watchedPath);
}
// Check if finished folder contains watched folder
else if (watchedPath.startsWith(finishedPath)) {
log.error(
"CRITICAL: Watched folder '{}' is nested inside finished folder '{}' - this will cause processing loops!",
watchedPath,
finishedPath);
}
}
} catch (Exception e) {
log.error("Error validating pipeline paths: {}", e.getMessage());
}
}
private boolean isRunningInDocker() {
return Files.exists(Path.of("/.dockerenv"));
}
app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java
+2
View File
@@ -459,7 +459,9 @@ public class ApplicationProperties {
@Data
public static class Pipeline {
private String pipelineDir;
private String watchedFoldersDir;
private List<String> watchedFoldersDirs = new ArrayList<>();
private String finishedFoldersDir;
private String webUIConfigsDir;
}
app/common/src/main/java/stirling/software/common/util/FileMonitor.java
+32 -10
View File
@@ -29,7 +29,7 @@ public class FileMonitor {
private final ConcurrentHashMap.KeySetView<Path, Boolean> readyForProcessingFiles;
private final WatchService watchService;
private final Predicate<Path> pathFilter;
private final Path rootDir;
private final List<Path> rootDirs;
private Set<Path> stagingFiles;
/**
@@ -47,8 +47,28 @@ public class FileMonitor {
this.pathFilter = pathFilter;
this.readyForProcessingFiles = ConcurrentHashMap.newKeySet();
this.watchService = FileSystems.getDefault().newWatchService();
log.info("Monitoring directory: {}", runtimePathConfig.getPipelineWatchedFoldersPath());
this.rootDir = Path.of(runtimePathConfig.getPipelineWatchedFoldersPath());
List<String> watchedFoldersDirs = runtimePathConfig.getPipelineWatchedFoldersPaths();
List<Path> validRootDirs = new ArrayList<>();
for (String pathStr : watchedFoldersDirs) {
try {
Path path = Path.of(pathStr);
validRootDirs.add(path);
log.info("Monitoring directory: {}", path);
} catch (Exception e) {
log.error(
"Failed to initialize monitoring for path '{}': {}",
pathStr,
e.getMessage());
}
}
this.rootDirs = Collections.unmodifiableList(validRootDirs);
if (this.rootDirs.isEmpty()) {
log.error("No valid directories to monitor - FileMonitor will not function");
}
}
private boolean shouldNotProcess(Path path) {
@@ -85,13 +105,15 @@ public class FileMonitor {
readyForProcessingFiles.clear();
if (path2KeyMapping.isEmpty()) {
log.warn("not monitoring any directory, even the root directory itself: {}", rootDir);
if (Files.exists(
rootDir)) { // if the root directory exists, re-register the root directory
try {
recursivelyRegisterEntry(rootDir);
} catch (IOException e) {
log.error("unable to register monitoring", e);
log.warn("Not monitoring any directories; attempting to re-register root paths.");
for (Path rootDir : rootDirs) {
if (Files.exists(
rootDir)) { // if the root directory exists, re-register the root directory
try {
recursivelyRegisterEntry(rootDir);
} catch (IOException e) {
log.error("unable to register monitoring for {}", rootDir, e);
}
}
}
}
app/common/src/test/java/stirling/software/common/util/FileMonitorTest.java
+3 -1
View File
@@ -9,6 +9,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.time.Instant;
import java.util.List;
import java.util.function.Predicate;
import org.junit.jupiter.api.BeforeEach;
@@ -34,7 +35,8 @@ class FileMonitorTest {
@BeforeEach
void setUp() throws IOException {
when(runtimePathConfig.getPipelineWatchedFoldersPath()).thenReturn(tempDir.toString());
when(runtimePathConfig.getPipelineWatchedFoldersPaths())
.thenReturn(List.of(tempDir.toString()));
// This mock is used in all tests except testPathFilter
// We use lenient to avoid UnnecessaryStubbingException in that test
app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java
+59 -6
View File
@@ -3,6 +3,7 @@ package stirling.software.SPDF.controller.api.pipeline;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystemException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -14,6 +15,7 @@ import java.time.LocalDate;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -41,14 +43,20 @@ import stirling.software.common.util.FileMonitor;
@Slf4j
public class PipelineDirectoryProcessor {
private static final int MAX_DIRECTORY_DEPTH = 50; // Prevent excessive recursion
private final ObjectMapper objectMapper;
private final ApiDocService apiDocService;
private final PipelineProcessor processor;
private final FileMonitor fileMonitor;
private final PostHogService postHogService;
private final String watchedFoldersDir;
private final List<String> watchedFoldersDirs;
private final String finishedFoldersDir;
// Track processed directories in current scan to prevent duplicates
private final ThreadLocal<java.util.Set<Path>> processedDirsInScan =
ThreadLocal.withInitial(java.util.HashSet::new);
public PipelineDirectoryProcessor(
ObjectMapper objectMapper,
ApiDocService apiDocService,
@@ -61,13 +69,26 @@ public class PipelineDirectoryProcessor {
this.processor = processor;
this.fileMonitor = fileMonitor;
this.postHogService = postHogService;
this.watchedFoldersDir = runtimePathConfig.getPipelineWatchedFoldersPath();
this.watchedFoldersDirs = runtimePathConfig.getPipelineWatchedFoldersPaths();
this.finishedFoldersDir = runtimePathConfig.getPipelineFinishedFoldersPath();
}
@Scheduled(fixedRate = 60000)
public void scanFolders() {
Path watchedFolderPath = Paths.get(watchedFoldersDir).toAbsolutePath();
// Clear the processed directories set for this scan cycle
processedDirsInScan.get().clear();
try {
for (String watchedFoldersDir : watchedFoldersDirs) {
scanWatchedFolder(Paths.get(watchedFoldersDir).toAbsolutePath());
}
} finally {
// Clean up ThreadLocal to prevent memory leaks
processedDirsInScan.remove();
}
}
private void scanWatchedFolder(Path watchedFolderPath) {
if (!Files.exists(watchedFolderPath)) {
try {
Files.createDirectories(watchedFolderPath);
@@ -78,16 +99,34 @@ public class PipelineDirectoryProcessor {
}
}
// Validate the path is a directory and readable
if (!Files.isDirectory(watchedFolderPath)) {
log.error("Path is not a directory: {}", watchedFolderPath);
return;
}
if (!Files.isReadable(watchedFolderPath)) {
log.error("Directory is not readable: {}", watchedFolderPath);
return;
}
try {
// Use FOLLOW_LINKS to follow symlinks, with max depth to prevent infinite loops
Files.walkFileTree(
watchedFolderPath,
EnumSet.of(FileVisitOption.FOLLOW_LINKS),
MAX_DIRECTORY_DEPTH,
new SimpleFileVisitor<>() {
@Override
public FileVisitResult preVisitDirectory(
Path dir, BasicFileAttributes attrs) {
try {
String dirName =
dir.getFileName() != null
? dir.getFileName().toString()
: "";
// Skip root directory and "processing" subdirectories
if (!dir.equals(watchedFolderPath) && !dir.endsWith("processing")) {
if (!dir.equals(watchedFolderPath)
&& !"processing".equals(dirName)) {
handleDirectory(dir);
}
} catch (Exception e) {
@@ -98,8 +137,11 @@ public class PipelineDirectoryProcessor {
@Override
public FileVisitResult visitFileFailed(Path path, IOException exc) {
// Handle broken symlinks or inaccessible directories
log.error("Error accessing path: {}", path, exc);
// Handle broken symlinks, permission issues, or inaccessible
// directories
if (exc != null) {
log.debug("Cannot access path '{}': {}", path, exc.getMessage());
}
return FileVisitResult.CONTINUE;
}
});
@@ -109,6 +151,17 @@ public class PipelineDirectoryProcessor {
}
public void handleDirectory(Path dir) throws IOException {
// Normalize path to absolute to prevent duplicate processing from different path
// representations
Path normalizedDir = dir.toAbsolutePath().normalize();
// Check if we've already processed this directory in this scan cycle
java.util.Set<Path> processedDirs = processedDirsInScan.get();
if (!processedDirs.add(normalizedDir)) {
log.debug("Directory already processed in this scan cycle: {}", normalizedDir);
return;
}
log.info("Handling directory: {}", dir);
Path processingDir = createProcessingDirectory(dir);
Optional<Path> jsonFileOptional = findJsonFile(dir);
app/core/src/main/resources/settings.yml.template
+2
View File
@@ -203,7 +203,9 @@ system:
name: postgres # set the name of your database. Should match the name of the database you create
customPaths:
pipeline:
pipelineDir: "" # Defaults to /pipeline
watchedFoldersDir: "" # Defaults to /pipeline/watchedFolders
watchedFoldersDirs: [] # List of watched folder directories. Defaults to watchedFoldersDir or /pipeline/watchedFolders.
finishedFoldersDir: "" # Defaults to /pipeline/finishedFolders
operations:
weasyprint: "" # Defaults to /opt/venv/bin/weasyprint
app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/AdminSettingsController.java
+55
View File
@@ -186,6 +186,13 @@ public class AdminSettingsController {
+ HtmlUtils.htmlEscape(key)));
}
// Validate pipeline path settings
String validationError = validatePipelinePathSetting(key, value);
if (validationError != null) {
return ResponseEntity.badRequest()
.body(Map.of("error", HtmlUtils.htmlEscape(validationError)));
}
log.info("Admin updating setting: {} = {}", key, value);
GeneralUtils.saveKeyToSettings(key, value);
@@ -642,6 +649,54 @@ public class AdminSettingsController {
return true;
}
private String validatePipelinePathSetting(String key, Object value) {
// Validate pipeline path settings
if (key.startsWith("system.customPaths.pipeline.watchedFoldersDirs")
&& value instanceof java.util.List) {
@SuppressWarnings("unchecked")
java.util.List<String> paths = (java.util.List<String>) value;
// Check for empty or all-blank paths
if (paths.isEmpty()) {
return null; // Empty is OK, will use default
}
// Validate each path
java.util.Set<String> normalizedPaths = new java.util.HashSet<>();
for (String path : paths) {
if (path != null && !path.trim().isEmpty()) {
try {
java.nio.file.Path normalized =
java.nio.file.Paths.get(path.trim()).toAbsolutePath().normalize();
String normalizedStr = normalized.toString();
// Check for duplicates
if (normalizedPaths.contains(normalizedStr)) {
return "Duplicate path detected: " + path;
}
normalizedPaths.add(normalizedStr);
} catch (java.nio.file.InvalidPathException e) {
return "Invalid path: " + path + " - " + e.getMessage();
}
}
}
// Check for overlapping paths
java.util.List<String> pathList = new java.util.ArrayList<>(normalizedPaths);
for (int i = 0; i < pathList.size(); i++) {
java.nio.file.Path path1 = java.nio.file.Paths.get(pathList.get(i));
for (int j = i + 1; j < pathList.size(); j++) {
java.nio.file.Path path2 = java.nio.file.Paths.get(pathList.get(j));
if (path1.startsWith(path2) || path2.startsWith(path1)) {
return "Overlapping paths detected: " + path1 + " and " + path2;
}
}
}
}
return null; // Valid
}
private Object getSettingByKey(String key) {
if (key == null || key.trim().isEmpty()) {
return null;