mirror of
https://github.com/arsvendg/Stirling-PDF.git
synced 2026-07-14 10:34:06 +02:00
Support multiple pipeline watch directories and configurable pipeline base path (#5545)
### Motivation - Allow operators to configure a pipeline base directory and multiple watched folders so the pipeline can monitor several directories and subdirectories concurrently. - Ensure scanning traverses subdirectories while skipping internal processing folders (e.g. `processing`) and preserve existing behavior for finished/output paths. - Expose the new options in the server `settings.yml.template` and the admin UI so paths can be edited from the web console. ### Description - Added new `pipelineDir` and `watchedFoldersDirs` fields to `ApplicationProperties.CustomPaths.Pipeline` and kept backward compatibility with `watchedFoldersDir` (app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java). - Resolved pipeline base and multiple watched folder paths in `RuntimePathConfig` and exposed `getPipelineWatchedFoldersPaths()` (app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java). - Updated `FileMonitor` to accept and register multiple root paths instead of a single root (app/common/src/main/java/stirling/software/common/util/FileMonitor.java). - Updated `PipelineDirectoryProcessor` to iterate all configured watched roots and to walk subdirectories while ignoring `processing` dirs (app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java). - Exposed the new settings in `settings.yml.template` and the admin UI, including a multi-line `Textarea` to edit `watchedFoldersDirs` (app/core/src/main/resources/settings.yml.template, frontend/src/proprietary/components/shared/config/configSections/AdminGeneralSection.tsx). - Adjusted unit test setup to account for list-based watched folders (app/common/src/test/java/stirling/software/common/util/FileMonitorTest.java). ### Testing - Ran formatting and build checks with `./gradlew spotlessApply` and `./gradlew build` using Java 21 via `JAVA_HOME=/root/.local/share/mise/installs/java/21.0.2 PATH=/root/.local/share/mise/installs/java/21.0.2/bin:$PATH ./gradlew ...`, but both runs failed due to Gradle plugin resolution being blocked in this environment (plugin portal/network 403), so full compilation/formatting could not complete. - Confirmed the code compiles locally was not possible here; unit test `FileMonitorTest` was updated to use the new API but was not executed due to the blocked build. - Changes were committed (`Support multiple pipeline watch directories`) and the repository diff contains the listed file modifications. ------ [Codex Task](https://chatgpt.com/codex/tasks/task_b_69741ecd17c883288d8085a63ccd66f4)
This commit is contained in:
Anthony Stirling
+155
-6
@@ -1,10 +1,14 @@
|
||||
package stirling.software.common.configuration;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.InvalidPathException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
@@ -41,6 +45,7 @@ public class RuntimePathConfig {
|
||||
|
||||
// Pipeline paths
|
||||
private final String pipelineWatchedFoldersPath;
|
||||
private final List<String> pipelineWatchedFoldersPaths;
|
||||
private final String pipelineFinishedFoldersPath;
|
||||
private final String pipelineDefaultWebUiConfigs;
|
||||
private final String pipelinePath;
|
||||
@@ -49,20 +54,27 @@ public class RuntimePathConfig {
|
||||
this.properties = properties;
|
||||
this.basePath = InstallationPathConfig.getPath();
|
||||
|
||||
this.pipelinePath = Path.of(basePath, "pipeline").toString();
|
||||
String defaultWatchedFolders = Path.of(this.pipelinePath, "watchedFolders").toString();
|
||||
String defaultFinishedFolders = Path.of(this.pipelinePath, "finishedFolders").toString();
|
||||
String defaultWebUIConfigs = Path.of(this.pipelinePath, "defaultWebUIConfigs").toString();
|
||||
|
||||
System system = properties.getSystem();
|
||||
CustomPaths customPaths = system.getCustomPaths();
|
||||
|
||||
Pipeline pipeline = customPaths.getPipeline();
|
||||
|
||||
this.pipelineWatchedFoldersPath =
|
||||
this.pipelinePath =
|
||||
resolvePath(
|
||||
Path.of(basePath, "pipeline").toString(),
|
||||
pipeline != null ? pipeline.getPipelineDir() : null);
|
||||
String defaultWatchedFolders = Path.of(this.pipelinePath, "watchedFolders").toString();
|
||||
String defaultFinishedFolders = Path.of(this.pipelinePath, "finishedFolders").toString();
|
||||
String defaultWebUIConfigs = Path.of(this.pipelinePath, "defaultWebUIConfigs").toString();
|
||||
|
||||
List<String> watchedFoldersDirs =
|
||||
sanitizePathList(pipeline != null ? pipeline.getWatchedFoldersDirs() : null);
|
||||
this.pipelineWatchedFoldersPaths =
|
||||
resolveWatchedFolderPaths(
|
||||
defaultWatchedFolders,
|
||||
watchedFoldersDirs,
|
||||
pipeline != null ? pipeline.getWatchedFoldersDir() : null);
|
||||
this.pipelineWatchedFoldersPath = this.pipelineWatchedFoldersPaths.get(0);
|
||||
this.pipelineFinishedFoldersPath =
|
||||
resolvePath(
|
||||
defaultFinishedFolders,
|
||||
@@ -72,6 +84,9 @@ public class RuntimePathConfig {
|
||||
defaultWebUIConfigs,
|
||||
pipeline != null ? pipeline.getWebUIConfigsDir() : null);
|
||||
|
||||
// Validate path conflicts after all paths are resolved
|
||||
validatePipelinePaths();
|
||||
|
||||
boolean isDocker = isRunningInDocker();
|
||||
|
||||
// Initialize Operation paths
|
||||
@@ -129,6 +144,140 @@ public class RuntimePathConfig {
|
||||
return StringUtils.isNotBlank(customPath) ? customPath : defaultPath;
|
||||
}
|
||||
|
||||
private List<String> resolveWatchedFolderPaths(
|
||||
String defaultPath, List<String> watchedFoldersDirs, String legacyWatchedFolder) {
|
||||
List<String> rawPaths = new ArrayList<>();
|
||||
|
||||
// Collect paths from new config
|
||||
if (watchedFoldersDirs != null && !watchedFoldersDirs.isEmpty()) {
|
||||
rawPaths.addAll(watchedFoldersDirs);
|
||||
}
|
||||
// Fall back to legacy config
|
||||
else if (StringUtils.isNotBlank(legacyWatchedFolder)) {
|
||||
rawPaths.add(legacyWatchedFolder);
|
||||
}
|
||||
// Fall back to default
|
||||
else {
|
||||
rawPaths.add(defaultPath);
|
||||
}
|
||||
|
||||
// Validate, normalize, and deduplicate paths
|
||||
List<String> validatedPaths = validateAndNormalizePaths(rawPaths);
|
||||
|
||||
// Ensure we have at least one valid path (critical for system to function)
|
||||
if (validatedPaths.isEmpty()) {
|
||||
log.warn(
|
||||
"No valid watched folder paths configured, falling back to default: {}",
|
||||
defaultPath);
|
||||
validatedPaths.add(defaultPath);
|
||||
}
|
||||
|
||||
// Detect overlapping paths (warning only, not blocking)
|
||||
detectOverlappingPaths(validatedPaths);
|
||||
|
||||
return validatedPaths;
|
||||
}
|
||||
|
||||
private List<String> sanitizePathList(List<String> paths) {
|
||||
if (paths == null || paths.isEmpty()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
List<String> sanitized = new ArrayList<>();
|
||||
for (String path : paths) {
|
||||
if (StringUtils.isNotBlank(path)) {
|
||||
sanitized.add(path.trim());
|
||||
}
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
private List<String> validateAndNormalizePaths(List<String> paths) {
|
||||
Set<String> normalizedPaths = new LinkedHashSet<>(); // Preserves order, prevents duplicates
|
||||
|
||||
for (String pathStr : paths) {
|
||||
if (StringUtils.isBlank(pathStr)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Normalize to absolute path
|
||||
Path path = Paths.get(pathStr.trim()).toAbsolutePath().normalize();
|
||||
String normalizedPath = path.toString();
|
||||
|
||||
// Check for duplicates
|
||||
if (normalizedPaths.contains(normalizedPath)) {
|
||||
log.debug("Skipping duplicate watched folder path: {}", pathStr);
|
||||
continue;
|
||||
}
|
||||
|
||||
normalizedPaths.add(normalizedPath);
|
||||
log.info("Registered watched folder path: {}", normalizedPath);
|
||||
|
||||
} catch (InvalidPathException e) {
|
||||
log.error(
|
||||
"Invalid watched folder path '{}' - skipping: {}", pathStr, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
return new ArrayList<>(normalizedPaths);
|
||||
}
|
||||
|
||||
private void detectOverlappingPaths(List<String> paths) {
|
||||
for (int i = 0; i < paths.size(); i++) {
|
||||
Path path1 = Paths.get(paths.get(i));
|
||||
for (int j = i + 1; j < paths.size(); j++) {
|
||||
Path path2 = Paths.get(paths.get(j));
|
||||
|
||||
// Check if one path is a parent of the other
|
||||
if (path1.startsWith(path2)) {
|
||||
log.warn(
|
||||
"Watched folder path '{}' is nested inside '{}' - this may cause duplicate processing",
|
||||
path1,
|
||||
path2);
|
||||
} else if (path2.startsWith(path1)) {
|
||||
log.warn(
|
||||
"Watched folder path '{}' is nested inside '{}' - this may cause duplicate processing",
|
||||
path2,
|
||||
path1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void validatePipelinePaths() {
|
||||
try {
|
||||
Path finishedPath = Paths.get(pipelineFinishedFoldersPath).toAbsolutePath().normalize();
|
||||
|
||||
for (String watchedPathStr : pipelineWatchedFoldersPaths) {
|
||||
Path watchedPath = Paths.get(watchedPathStr).toAbsolutePath().normalize();
|
||||
|
||||
// Check if watched folder is same as finished folder
|
||||
if (watchedPath.equals(finishedPath)) {
|
||||
log.error(
|
||||
"CRITICAL: Watched folder '{}' is the same as finished folder '{}' - this will cause processing loops!",
|
||||
watchedPath,
|
||||
finishedPath);
|
||||
}
|
||||
// Check if watched folder contains finished folder
|
||||
else if (finishedPath.startsWith(watchedPath)) {
|
||||
log.warn(
|
||||
"Finished folder '{}' is nested inside watched folder '{}' - this may cause issues",
|
||||
finishedPath,
|
||||
watchedPath);
|
||||
}
|
||||
// Check if finished folder contains watched folder
|
||||
else if (watchedPath.startsWith(finishedPath)) {
|
||||
log.error(
|
||||
"CRITICAL: Watched folder '{}' is nested inside finished folder '{}' - this will cause processing loops!",
|
||||
watchedPath,
|
||||
finishedPath);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error validating pipeline paths: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isRunningInDocker() {
|
||||
return Files.exists(Path.of("/.dockerenv"));
|
||||
}
|
||||
|
||||
@@ -459,7 +459,9 @@ public class ApplicationProperties {
|
||||
|
||||
@Data
|
||||
public static class Pipeline {
|
||||
private String pipelineDir;
|
||||
private String watchedFoldersDir;
|
||||
private List<String> watchedFoldersDirs = new ArrayList<>();
|
||||
private String finishedFoldersDir;
|
||||
private String webUIConfigsDir;
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ public class FileMonitor {
|
||||
private final ConcurrentHashMap.KeySetView<Path, Boolean> readyForProcessingFiles;
|
||||
private final WatchService watchService;
|
||||
private final Predicate<Path> pathFilter;
|
||||
private final Path rootDir;
|
||||
private final List<Path> rootDirs;
|
||||
private Set<Path> stagingFiles;
|
||||
|
||||
/**
|
||||
@@ -47,8 +47,28 @@ public class FileMonitor {
|
||||
this.pathFilter = pathFilter;
|
||||
this.readyForProcessingFiles = ConcurrentHashMap.newKeySet();
|
||||
this.watchService = FileSystems.getDefault().newWatchService();
|
||||
log.info("Monitoring directory: {}", runtimePathConfig.getPipelineWatchedFoldersPath());
|
||||
this.rootDir = Path.of(runtimePathConfig.getPipelineWatchedFoldersPath());
|
||||
|
||||
List<String> watchedFoldersDirs = runtimePathConfig.getPipelineWatchedFoldersPaths();
|
||||
List<Path> validRootDirs = new ArrayList<>();
|
||||
|
||||
for (String pathStr : watchedFoldersDirs) {
|
||||
try {
|
||||
Path path = Path.of(pathStr);
|
||||
validRootDirs.add(path);
|
||||
log.info("Monitoring directory: {}", path);
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Failed to initialize monitoring for path '{}': {}",
|
||||
pathStr,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
this.rootDirs = Collections.unmodifiableList(validRootDirs);
|
||||
|
||||
if (this.rootDirs.isEmpty()) {
|
||||
log.error("No valid directories to monitor - FileMonitor will not function");
|
||||
}
|
||||
}
|
||||
|
||||
private boolean shouldNotProcess(Path path) {
|
||||
@@ -85,13 +105,15 @@ public class FileMonitor {
|
||||
readyForProcessingFiles.clear();
|
||||
|
||||
if (path2KeyMapping.isEmpty()) {
|
||||
log.warn("not monitoring any directory, even the root directory itself: {}", rootDir);
|
||||
if (Files.exists(
|
||||
rootDir)) { // if the root directory exists, re-register the root directory
|
||||
try {
|
||||
recursivelyRegisterEntry(rootDir);
|
||||
} catch (IOException e) {
|
||||
log.error("unable to register monitoring", e);
|
||||
log.warn("Not monitoring any directories; attempting to re-register root paths.");
|
||||
for (Path rootDir : rootDirs) {
|
||||
if (Files.exists(
|
||||
rootDir)) { // if the root directory exists, re-register the root directory
|
||||
try {
|
||||
recursivelyRegisterEntry(rootDir);
|
||||
} catch (IOException e) {
|
||||
log.error("unable to register monitoring for {}", rootDir, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.FileTime;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -34,7 +35,8 @@ class FileMonitorTest {
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws IOException {
|
||||
when(runtimePathConfig.getPipelineWatchedFoldersPath()).thenReturn(tempDir.toString());
|
||||
when(runtimePathConfig.getPipelineWatchedFoldersPaths())
|
||||
.thenReturn(List.of(tempDir.toString()));
|
||||
|
||||
// This mock is used in all tests except testPathFilter
|
||||
// We use lenient to avoid UnnecessaryStubbingException in that test
|
||||
|
||||
+59
-6
@@ -3,6 +3,7 @@ package stirling.software.SPDF.controller.api.pipeline;
|
||||
import java.io.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.FileSystemException;
|
||||
import java.nio.file.FileVisitOption;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
@@ -14,6 +15,7 @@ import java.time.LocalDate;
|
||||
import java.time.LocalTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
@@ -41,14 +43,20 @@ import stirling.software.common.util.FileMonitor;
|
||||
@Slf4j
|
||||
public class PipelineDirectoryProcessor {
|
||||
|
||||
private static final int MAX_DIRECTORY_DEPTH = 50; // Prevent excessive recursion
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ApiDocService apiDocService;
|
||||
private final PipelineProcessor processor;
|
||||
private final FileMonitor fileMonitor;
|
||||
private final PostHogService postHogService;
|
||||
private final String watchedFoldersDir;
|
||||
private final List<String> watchedFoldersDirs;
|
||||
private final String finishedFoldersDir;
|
||||
|
||||
// Track processed directories in current scan to prevent duplicates
|
||||
private final ThreadLocal<java.util.Set<Path>> processedDirsInScan =
|
||||
ThreadLocal.withInitial(java.util.HashSet::new);
|
||||
|
||||
public PipelineDirectoryProcessor(
|
||||
ObjectMapper objectMapper,
|
||||
ApiDocService apiDocService,
|
||||
@@ -61,13 +69,26 @@ public class PipelineDirectoryProcessor {
|
||||
this.processor = processor;
|
||||
this.fileMonitor = fileMonitor;
|
||||
this.postHogService = postHogService;
|
||||
this.watchedFoldersDir = runtimePathConfig.getPipelineWatchedFoldersPath();
|
||||
this.watchedFoldersDirs = runtimePathConfig.getPipelineWatchedFoldersPaths();
|
||||
this.finishedFoldersDir = runtimePathConfig.getPipelineFinishedFoldersPath();
|
||||
}
|
||||
|
||||
@Scheduled(fixedRate = 60000)
|
||||
public void scanFolders() {
|
||||
Path watchedFolderPath = Paths.get(watchedFoldersDir).toAbsolutePath();
|
||||
// Clear the processed directories set for this scan cycle
|
||||
processedDirsInScan.get().clear();
|
||||
|
||||
try {
|
||||
for (String watchedFoldersDir : watchedFoldersDirs) {
|
||||
scanWatchedFolder(Paths.get(watchedFoldersDir).toAbsolutePath());
|
||||
}
|
||||
} finally {
|
||||
// Clean up ThreadLocal to prevent memory leaks
|
||||
processedDirsInScan.remove();
|
||||
}
|
||||
}
|
||||
|
||||
private void scanWatchedFolder(Path watchedFolderPath) {
|
||||
if (!Files.exists(watchedFolderPath)) {
|
||||
try {
|
||||
Files.createDirectories(watchedFolderPath);
|
||||
@@ -78,16 +99,34 @@ public class PipelineDirectoryProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the path is a directory and readable
|
||||
if (!Files.isDirectory(watchedFolderPath)) {
|
||||
log.error("Path is not a directory: {}", watchedFolderPath);
|
||||
return;
|
||||
}
|
||||
if (!Files.isReadable(watchedFolderPath)) {
|
||||
log.error("Directory is not readable: {}", watchedFolderPath);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Use FOLLOW_LINKS to follow symlinks, with max depth to prevent infinite loops
|
||||
Files.walkFileTree(
|
||||
watchedFolderPath,
|
||||
EnumSet.of(FileVisitOption.FOLLOW_LINKS),
|
||||
MAX_DIRECTORY_DEPTH,
|
||||
new SimpleFileVisitor<>() {
|
||||
@Override
|
||||
public FileVisitResult preVisitDirectory(
|
||||
Path dir, BasicFileAttributes attrs) {
|
||||
try {
|
||||
String dirName =
|
||||
dir.getFileName() != null
|
||||
? dir.getFileName().toString()
|
||||
: "";
|
||||
// Skip root directory and "processing" subdirectories
|
||||
if (!dir.equals(watchedFolderPath) && !dir.endsWith("processing")) {
|
||||
if (!dir.equals(watchedFolderPath)
|
||||
&& !"processing".equals(dirName)) {
|
||||
handleDirectory(dir);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
@@ -98,8 +137,11 @@ public class PipelineDirectoryProcessor {
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFileFailed(Path path, IOException exc) {
|
||||
// Handle broken symlinks or inaccessible directories
|
||||
log.error("Error accessing path: {}", path, exc);
|
||||
// Handle broken symlinks, permission issues, or inaccessible
|
||||
// directories
|
||||
if (exc != null) {
|
||||
log.debug("Cannot access path '{}': {}", path, exc.getMessage());
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
});
|
||||
@@ -109,6 +151,17 @@ public class PipelineDirectoryProcessor {
|
||||
}
|
||||
|
||||
public void handleDirectory(Path dir) throws IOException {
|
||||
// Normalize path to absolute to prevent duplicate processing from different path
|
||||
// representations
|
||||
Path normalizedDir = dir.toAbsolutePath().normalize();
|
||||
|
||||
// Check if we've already processed this directory in this scan cycle
|
||||
java.util.Set<Path> processedDirs = processedDirsInScan.get();
|
||||
if (!processedDirs.add(normalizedDir)) {
|
||||
log.debug("Directory already processed in this scan cycle: {}", normalizedDir);
|
||||
return;
|
||||
}
|
||||
|
||||
log.info("Handling directory: {}", dir);
|
||||
Path processingDir = createProcessingDirectory(dir);
|
||||
Optional<Path> jsonFileOptional = findJsonFile(dir);
|
||||
|
||||
@@ -203,7 +203,9 @@ system:
|
||||
name: postgres # set the name of your database. Should match the name of the database you create
|
||||
customPaths:
|
||||
pipeline:
|
||||
pipelineDir: "" # Defaults to /pipeline
|
||||
watchedFoldersDir: "" # Defaults to /pipeline/watchedFolders
|
||||
watchedFoldersDirs: [] # List of watched folder directories. Defaults to watchedFoldersDir or /pipeline/watchedFolders.
|
||||
finishedFoldersDir: "" # Defaults to /pipeline/finishedFolders
|
||||
operations:
|
||||
weasyprint: "" # Defaults to /opt/venv/bin/weasyprint
|
||||
|
||||
+55
@@ -186,6 +186,13 @@ public class AdminSettingsController {
|
||||
+ HtmlUtils.htmlEscape(key)));
|
||||
}
|
||||
|
||||
// Validate pipeline path settings
|
||||
String validationError = validatePipelinePathSetting(key, value);
|
||||
if (validationError != null) {
|
||||
return ResponseEntity.badRequest()
|
||||
.body(Map.of("error", HtmlUtils.htmlEscape(validationError)));
|
||||
}
|
||||
|
||||
log.info("Admin updating setting: {} = {}", key, value);
|
||||
GeneralUtils.saveKeyToSettings(key, value);
|
||||
|
||||
@@ -642,6 +649,54 @@ public class AdminSettingsController {
|
||||
return true;
|
||||
}
|
||||
|
||||
private String validatePipelinePathSetting(String key, Object value) {
|
||||
// Validate pipeline path settings
|
||||
if (key.startsWith("system.customPaths.pipeline.watchedFoldersDirs")
|
||||
&& value instanceof java.util.List) {
|
||||
@SuppressWarnings("unchecked")
|
||||
java.util.List<String> paths = (java.util.List<String>) value;
|
||||
|
||||
// Check for empty or all-blank paths
|
||||
if (paths.isEmpty()) {
|
||||
return null; // Empty is OK, will use default
|
||||
}
|
||||
|
||||
// Validate each path
|
||||
java.util.Set<String> normalizedPaths = new java.util.HashSet<>();
|
||||
for (String path : paths) {
|
||||
if (path != null && !path.trim().isEmpty()) {
|
||||
try {
|
||||
java.nio.file.Path normalized =
|
||||
java.nio.file.Paths.get(path.trim()).toAbsolutePath().normalize();
|
||||
String normalizedStr = normalized.toString();
|
||||
|
||||
// Check for duplicates
|
||||
if (normalizedPaths.contains(normalizedStr)) {
|
||||
return "Duplicate path detected: " + path;
|
||||
}
|
||||
normalizedPaths.add(normalizedStr);
|
||||
} catch (java.nio.file.InvalidPathException e) {
|
||||
return "Invalid path: " + path + " - " + e.getMessage();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for overlapping paths
|
||||
java.util.List<String> pathList = new java.util.ArrayList<>(normalizedPaths);
|
||||
for (int i = 0; i < pathList.size(); i++) {
|
||||
java.nio.file.Path path1 = java.nio.file.Paths.get(pathList.get(i));
|
||||
for (int j = i + 1; j < pathList.size(); j++) {
|
||||
java.nio.file.Path path2 = java.nio.file.Paths.get(pathList.get(j));
|
||||
if (path1.startsWith(path2) || path2.startsWith(path1)) {
|
||||
return "Overlapping paths detected: " + path1 + " and " + path2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null; // Valid
|
||||
}
|
||||
|
||||
private Object getSettingByKey(String key) {
|
||||
if (key == null || key.trim().isEmpty()) {
|
||||
return null;
|
||||
|
||||
@@ -4514,10 +4514,18 @@ description = "Configure custom file system paths for pipeline processing and ex
|
||||
[admin.settings.general.customPaths.pipeline]
|
||||
label = "Pipeline Directories"
|
||||
|
||||
[admin.settings.general.customPaths.pipeline.pipelineDir]
|
||||
label = "Pipeline Directory"
|
||||
description = "Base directory for pipeline resources (leave empty for default: /pipeline)"
|
||||
|
||||
[admin.settings.general.customPaths.pipeline.watchedFoldersDir]
|
||||
label = "Watched Folders Directory"
|
||||
description = "Directory where pipeline monitors for incoming PDFs (leave empty for default: /pipeline/watchedFolders)"
|
||||
|
||||
[admin.settings.general.customPaths.pipeline.watchedFoldersDirs]
|
||||
label = "Watched Folders Directories"
|
||||
description = "Directories where pipeline monitors for incoming PDFs (one per line or comma-separated; leave empty for default: /pipeline/watchedFolders)"
|
||||
|
||||
[admin.settings.general.customPaths.pipeline.finishedFoldersDir]
|
||||
label = "Finished Folders Directory"
|
||||
description = "Directory where processed PDFs are outputted (leave empty for default: /pipeline/finishedFolders)"
|
||||
|
||||
+136
-12
@@ -1,7 +1,7 @@
|
||||
import { useEffect, useState, useRef, useCallback, useMemo } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useLocation, useNavigate } from 'react-router-dom';
|
||||
import { TextInput, Switch, Button, Stack, Paper, Text, Loader, Group, MultiSelect, Badge, SegmentedControl, Select } from '@mantine/core';
|
||||
import { TextInput, Textarea, Switch, Button, Stack, Paper, Text, Loader, Group, MultiSelect, Badge, SegmentedControl, Select } from '@mantine/core';
|
||||
import { alert } from '@app/components/toast';
|
||||
import RestartConfirmationModal from '@app/components/shared/config/RestartConfirmationModal';
|
||||
import { useRestartServer } from '@app/components/shared/config/useRestartServer';
|
||||
@@ -31,7 +31,9 @@ interface GeneralSettingsData {
|
||||
};
|
||||
customPaths?: {
|
||||
pipeline?: {
|
||||
pipelineDir?: string;
|
||||
watchedFoldersDir?: string;
|
||||
watchedFoldersDirs?: string[];
|
||||
finishedFoldersDir?: string;
|
||||
};
|
||||
operations?: {
|
||||
@@ -61,6 +63,17 @@ export default function AdminGeneralSection() {
|
||||
.sort((a, b) => a.label.localeCompare(b.label)),
|
||||
[]
|
||||
);
|
||||
const parseWatchedFoldersInput = useCallback((value: string) => {
|
||||
const paths = value
|
||||
.split(/[\n,;]+/)
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
// Deduplicate paths (case-sensitive, exact match)
|
||||
const uniquePaths = Array.from(new Set(paths));
|
||||
|
||||
return uniquePaths;
|
||||
}, []);
|
||||
|
||||
// Track original settings for dirty detection
|
||||
const [originalSettingsSnapshot, setOriginalSettingsSnapshot] = useState<string>('');
|
||||
@@ -91,17 +104,31 @@ export default function AdminGeneralSection() {
|
||||
|
||||
ui.languages = Array.isArray(ui.languages) ? toUnderscoreLanguages(ui.languages) : [];
|
||||
|
||||
const pipelinePaths = system.customPaths?.pipeline || {};
|
||||
const watchedFoldersDirs = Array.isArray(pipelinePaths.watchedFoldersDirs)
|
||||
? pipelinePaths.watchedFoldersDirs
|
||||
: [];
|
||||
const normalizedWatchedFoldersDirs =
|
||||
watchedFoldersDirs.length > 0
|
||||
? watchedFoldersDirs
|
||||
: (pipelinePaths.watchedFoldersDir ? [pipelinePaths.watchedFoldersDir] : []);
|
||||
|
||||
const result: any = {
|
||||
ui,
|
||||
system,
|
||||
customPaths: system.customPaths || {
|
||||
customPaths: {
|
||||
...(system.customPaths || {}),
|
||||
pipeline: {
|
||||
watchedFoldersDir: '',
|
||||
finishedFoldersDir: ''
|
||||
...pipelinePaths,
|
||||
pipelineDir: pipelinePaths.pipelineDir || '',
|
||||
watchedFoldersDir: pipelinePaths.watchedFoldersDir || '',
|
||||
watchedFoldersDirs: normalizedWatchedFoldersDirs,
|
||||
finishedFoldersDir: pipelinePaths.finishedFoldersDir || ''
|
||||
},
|
||||
operations: {
|
||||
weasyprint: '',
|
||||
unoconvert: ''
|
||||
...(system.customPaths?.operations || {}),
|
||||
weasyprint: system.customPaths?.operations?.weasyprint || '',
|
||||
unoconvert: system.customPaths?.operations?.unoconvert || ''
|
||||
}
|
||||
},
|
||||
customMetadata: premium.proFeatures?.customMetadata || {
|
||||
@@ -154,7 +181,9 @@ export default function AdminGeneralSection() {
|
||||
};
|
||||
|
||||
if (settings.customPaths) {
|
||||
deltaSettings['system.customPaths.pipeline.pipelineDir'] = settings.customPaths?.pipeline?.pipelineDir;
|
||||
deltaSettings['system.customPaths.pipeline.watchedFoldersDir'] = settings.customPaths?.pipeline?.watchedFoldersDir;
|
||||
deltaSettings['system.customPaths.pipeline.watchedFoldersDirs'] = settings.customPaths?.pipeline?.watchedFoldersDirs;
|
||||
deltaSettings['system.customPaths.pipeline.finishedFoldersDir'] = settings.customPaths?.pipeline?.finishedFoldersDir;
|
||||
deltaSettings['system.customPaths.operations.weasyprint'] = settings.customPaths?.operations?.weasyprint;
|
||||
deltaSettings['system.customPaths.operations.unoconvert'] = settings.customPaths?.operations?.unoconvert;
|
||||
@@ -171,6 +200,54 @@ export default function AdminGeneralSection() {
|
||||
() => toUnderscoreLanguages(settings.ui?.languages || []),
|
||||
[settings.ui?.languages]
|
||||
);
|
||||
const watchedFoldersInput = useMemo(() => (
|
||||
(settings.customPaths?.pipeline?.watchedFoldersDirs || []).join('\n')
|
||||
), [settings.customPaths?.pipeline?.watchedFoldersDirs]);
|
||||
|
||||
const watchedFoldersValidation = useMemo(() => {
|
||||
const paths = settings.customPaths?.pipeline?.watchedFoldersDirs || [];
|
||||
const finishedPath = settings.customPaths?.pipeline?.finishedFoldersDir || '';
|
||||
const warnings: string[] = [];
|
||||
|
||||
// Normalize paths for comparison (handle both Windows and Unix paths)
|
||||
const normalizePath = (p: string) => p.replace(/\\/g, '/').replace(/\/+$/, '');
|
||||
|
||||
// Check for overlapping watched folders
|
||||
if (paths.length >= 2) {
|
||||
for (let i = 0; i < paths.length; i++) {
|
||||
for (let j = i + 1; j < paths.length; j++) {
|
||||
const path1 = normalizePath(paths[i]);
|
||||
const path2 = normalizePath(paths[j]);
|
||||
|
||||
if (path1 === path2) {
|
||||
warnings.push(`Duplicate path detected: '${paths[i]}'`);
|
||||
} else if (path1.startsWith(path2 + '/')) {
|
||||
warnings.push(`'${paths[i]}' is nested inside '${paths[j]}' - may cause duplicate processing`);
|
||||
} else if (path2.startsWith(path1 + '/')) {
|
||||
warnings.push(`'${paths[j]}' is nested inside '${paths[i]}' - may cause duplicate processing`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for conflicts with finished folder
|
||||
if (finishedPath && paths.length > 0) {
|
||||
const normalizedFinished = normalizePath(finishedPath);
|
||||
for (const watchedPath of paths) {
|
||||
const normalizedWatched = normalizePath(watchedPath);
|
||||
|
||||
if (normalizedWatched === normalizedFinished) {
|
||||
warnings.push(`CRITICAL: Watched folder '${watchedPath}' is the same as finished folder - will cause processing loops!`);
|
||||
} else if (normalizedFinished.startsWith(normalizedWatched + '/')) {
|
||||
warnings.push(`Finished folder is nested inside watched folder '${watchedPath}' - may cause issues`);
|
||||
} else if (normalizedWatched.startsWith(normalizedFinished + '/')) {
|
||||
warnings.push(`CRITICAL: Watched folder '${watchedPath}' is nested inside finished folder - will cause processing loops!`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return warnings.length > 0 ? warnings : null;
|
||||
}, [settings.customPaths?.pipeline?.watchedFoldersDirs, settings.customPaths?.pipeline?.finishedFoldersDir]);
|
||||
|
||||
// Filter default locale options based on available languages setting
|
||||
const defaultLocaleOptions = useMemo(() => {
|
||||
@@ -651,27 +728,74 @@ export default function AdminGeneralSection() {
|
||||
<TextInput
|
||||
label={
|
||||
<Group gap="xs">
|
||||
<span>{t('admin.settings.general.customPaths.pipeline.watchedFoldersDir.label', 'Watched Folders Directory')}</span>
|
||||
<PendingBadge show={isFieldPending('customPaths.pipeline.watchedFoldersDir')} />
|
||||
<span>{t('admin.settings.general.customPaths.pipeline.pipelineDir.label', 'Pipeline Directory')}</span>
|
||||
<PendingBadge show={isFieldPending('customPaths.pipeline.pipelineDir')} />
|
||||
</Group>
|
||||
}
|
||||
description={t('admin.settings.general.customPaths.pipeline.watchedFoldersDir.description', 'Directory where pipeline monitors for incoming PDFs (leave empty for default: /pipeline/watchedFolders)')}
|
||||
value={settings.customPaths?.pipeline?.watchedFoldersDir || ''}
|
||||
description={t('admin.settings.general.customPaths.pipeline.pipelineDir.description', 'Base directory for pipeline resources (leave empty for default: /pipeline)')}
|
||||
value={settings.customPaths?.pipeline?.pipelineDir || ''}
|
||||
onChange={(e) => setSettings({
|
||||
...settings,
|
||||
customPaths: {
|
||||
...settings.customPaths,
|
||||
pipeline: {
|
||||
...settings.customPaths?.pipeline,
|
||||
watchedFoldersDir: e.target.value
|
||||
pipelineDir: e.target.value
|
||||
}
|
||||
}
|
||||
})}
|
||||
placeholder="/pipeline/watchedFolders"
|
||||
placeholder="/pipeline"
|
||||
disabled={!loginEnabled}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Textarea
|
||||
label={
|
||||
<Group gap="xs">
|
||||
<span>{t('admin.settings.general.customPaths.pipeline.watchedFoldersDirs.label', 'Watched Folders Directories')}</span>
|
||||
<PendingBadge show={
|
||||
isFieldPending('customPaths.pipeline.watchedFoldersDirs')
|
||||
|| isFieldPending('customPaths.pipeline.watchedFoldersDir')
|
||||
} />
|
||||
</Group>
|
||||
}
|
||||
description={t('admin.settings.general.customPaths.pipeline.watchedFoldersDirs.description', 'Directories where pipeline monitors for incoming PDFs (one per line or comma-separated; leave empty for default: /pipeline/watchedFolders)')}
|
||||
value={watchedFoldersInput}
|
||||
onChange={(e) => {
|
||||
const parsedDirs = parseWatchedFoldersInput(e.target.value);
|
||||
setSettings({
|
||||
...settings,
|
||||
customPaths: {
|
||||
...settings.customPaths,
|
||||
pipeline: {
|
||||
...settings.customPaths?.pipeline,
|
||||
watchedFoldersDir: parsedDirs[0] || '',
|
||||
watchedFoldersDirs: parsedDirs
|
||||
}
|
||||
}
|
||||
});
|
||||
}}
|
||||
placeholder="/pipeline/watchedFolders"
|
||||
minRows={3}
|
||||
autosize
|
||||
disabled={!loginEnabled}
|
||||
/>
|
||||
{watchedFoldersValidation && (
|
||||
<Stack gap="xs" mt="xs">
|
||||
{watchedFoldersValidation.map((warning, idx) => (
|
||||
<Text
|
||||
key={idx}
|
||||
size="sm"
|
||||
c={warning.includes('CRITICAL') ? 'red' : 'yellow'}
|
||||
>
|
||||
{warning}
|
||||
</Text>
|
||||
))}
|
||||
</Stack>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<TextInput
|
||||
label={
|
||||
|
||||
Reference in New Issue
Block a user