Support multiple pipeline watch directories and configurable pipeline base path (#5545)

### Motivation
- Allow operators to configure a pipeline base directory and multiple
watched folders so the pipeline can monitor several directories and
subdirectories concurrently.
- Ensure scanning traverses subdirectories while skipping internal
processing folders (e.g. `processing`) and preserve existing behavior
for finished/output paths.
- Expose the new options in the server `settings.yml.template` and the
admin UI so paths can be edited from the web console.

### Description
- Added new `pipelineDir` and `watchedFoldersDirs` fields to
`ApplicationProperties.CustomPaths.Pipeline` and kept backward
compatibility with `watchedFoldersDir`
(app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java).
- Resolved pipeline base and multiple watched folder paths in
`RuntimePathConfig` and exposed `getPipelineWatchedFoldersPaths()`
(app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java).
- Updated `FileMonitor` to accept and register multiple root paths
instead of a single root
(app/common/src/main/java/stirling/software/common/util/FileMonitor.java).
- Updated `PipelineDirectoryProcessor` to iterate all configured watched
roots and to walk subdirectories while ignoring `processing` dirs
(app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java).
- Exposed the new settings in `settings.yml.template` and the admin UI,
including a multi-line `Textarea` to edit `watchedFoldersDirs`
(app/core/src/main/resources/settings.yml.template,
frontend/src/proprietary/components/shared/config/configSections/AdminGeneralSection.tsx).
- Adjusted unit test setup to account for list-based watched folders
(app/common/src/test/java/stirling/software/common/util/FileMonitorTest.java).

### Testing
- Ran formatting and build checks with `./gradlew spotlessApply` and
`./gradlew build` using Java 21 via
`JAVA_HOME=/root/.local/share/mise/installs/java/21.0.2
PATH=/root/.local/share/mise/installs/java/21.0.2/bin:$PATH ./gradlew
...`, but both runs failed due to Gradle plugin resolution being blocked
in this environment (plugin portal/network 403), so full
compilation/formatting could not complete.
- Confirmed the code compiles locally was not possible here; unit test
`FileMonitorTest` was updated to use the new API but was not executed
due to the blocked build.
- Changes were committed (`Support multiple pipeline watch directories`)
and the repository diff contains the listed file modifications.

------
[Codex
Task](https://chatgpt.com/codex/tasks/task_b_69741ecd17c883288d8085a63ccd66f4)

app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java

@@ -3,6 +3,7 @@ package stirling.software.SPDF.controller.api.pipeline;
 import java.io.*;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystemException;
+import java.nio.file.FileVisitOption;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -14,6 +15,7 @@ import java.time.LocalDate;
 import java.time.LocalTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -41,14 +43,20 @@ import stirling.software.common.util.FileMonitor;
 @Slf4j
 public class PipelineDirectoryProcessor {
 
+    private static final int MAX_DIRECTORY_DEPTH = 50; // Prevent excessive recursion
+
     private final ObjectMapper objectMapper;
     private final ApiDocService apiDocService;
     private final PipelineProcessor processor;
     private final FileMonitor fileMonitor;
     private final PostHogService postHogService;
-    private final String watchedFoldersDir;
+    private final List<String> watchedFoldersDirs;
     private final String finishedFoldersDir;
 
+    // Track processed directories in current scan to prevent duplicates
+    private final ThreadLocal<java.util.Set<Path>> processedDirsInScan =
+            ThreadLocal.withInitial(java.util.HashSet::new);
+
     public PipelineDirectoryProcessor(
             ObjectMapper objectMapper,
             ApiDocService apiDocService,
@@ -61,13 +69,26 @@ public class PipelineDirectoryProcessor {
         this.processor = processor;
         this.fileMonitor = fileMonitor;
         this.postHogService = postHogService;
-        this.watchedFoldersDir = runtimePathConfig.getPipelineWatchedFoldersPath();
+        this.watchedFoldersDirs = runtimePathConfig.getPipelineWatchedFoldersPaths();
         this.finishedFoldersDir = runtimePathConfig.getPipelineFinishedFoldersPath();
     }
 
     @Scheduled(fixedRate = 60000)
     public void scanFolders() {
-        Path watchedFolderPath = Paths.get(watchedFoldersDir).toAbsolutePath();
+        // Clear the processed directories set for this scan cycle
+        processedDirsInScan.get().clear();
+
+        try {
+            for (String watchedFoldersDir : watchedFoldersDirs) {
+                scanWatchedFolder(Paths.get(watchedFoldersDir).toAbsolutePath());
+            }
+        } finally {
+            // Clean up ThreadLocal to prevent memory leaks
+            processedDirsInScan.remove();
+        }
+    }
+
+    private void scanWatchedFolder(Path watchedFolderPath) {
         if (!Files.exists(watchedFolderPath)) {
             try {
                 Files.createDirectories(watchedFolderPath);
@@ -78,16 +99,34 @@ public class PipelineDirectoryProcessor {
             }
         }
 
+        // Validate the path is a directory and readable
+        if (!Files.isDirectory(watchedFolderPath)) {
+            log.error("Path is not a directory: {}", watchedFolderPath);
+            return;
+        }
+        if (!Files.isReadable(watchedFolderPath)) {
+            log.error("Directory is not readable: {}", watchedFolderPath);
+            return;
+        }
+
         try {
+            // Use FOLLOW_LINKS to follow symlinks, with max depth to prevent infinite loops
             Files.walkFileTree(
                     watchedFolderPath,
+                    EnumSet.of(FileVisitOption.FOLLOW_LINKS),
+                    MAX_DIRECTORY_DEPTH,
                     new SimpleFileVisitor<>() {
                         @Override
                         public FileVisitResult preVisitDirectory(
                                 Path dir, BasicFileAttributes attrs) {
                             try {
+                                String dirName =
+                                        dir.getFileName() != null
+                                                ? dir.getFileName().toString()
+                                                : "";
                                 // Skip root directory and "processing" subdirectories
-                                if (!dir.equals(watchedFolderPath) && !dir.endsWith("processing")) {
+                                if (!dir.equals(watchedFolderPath)
+                                        && !"processing".equals(dirName)) {
                                     handleDirectory(dir);
                                 }
                             } catch (Exception e) {
@@ -98,8 +137,11 @@ public class PipelineDirectoryProcessor {
 
                         @Override
                         public FileVisitResult visitFileFailed(Path path, IOException exc) {
-                            // Handle broken symlinks or inaccessible directories
-                            log.error("Error accessing path: {}", path, exc);
+                            // Handle broken symlinks, permission issues, or inaccessible
+                            // directories
+                            if (exc != null) {
+                                log.debug("Cannot access path '{}': {}", path, exc.getMessage());
+                            }
                             return FileVisitResult.CONTINUE;
                         }
                     });
@@ -109,6 +151,17 @@ public class PipelineDirectoryProcessor {
     }
 
     public void handleDirectory(Path dir) throws IOException {
+        // Normalize path to absolute to prevent duplicate processing from different path
+        // representations
+        Path normalizedDir = dir.toAbsolutePath().normalize();
+
+        // Check if we've already processed this directory in this scan cycle
+        java.util.Set<Path> processedDirs = processedDirsInScan.get();
+        if (!processedDirs.add(normalizedDir)) {
+            log.debug("Directory already processed in this scan cycle: {}", normalizedDir);
+            return;
+        }
+
         log.info("Handling directory: {}", dir);
         Path processingDir = createProcessingDirectory(dir);
         Optional<Path> jsonFileOptional = findJsonFile(dir);