mirror of
https://github.com/arsvendg/Stirling-PDF.git
synced 2026-07-16 11:23:10 +02:00
Support multiple pipeline watch directories and configurable pipeline base path (#5545)
### Motivation - Allow operators to configure a pipeline base directory and multiple watched folders so the pipeline can monitor several directories and subdirectories concurrently. - Ensure scanning traverses subdirectories while skipping internal processing folders (e.g. `processing`) and preserve existing behavior for finished/output paths. - Expose the new options in the server `settings.yml.template` and the admin UI so paths can be edited from the web console. ### Description - Added new `pipelineDir` and `watchedFoldersDirs` fields to `ApplicationProperties.CustomPaths.Pipeline` and kept backward compatibility with `watchedFoldersDir` (app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java). - Resolved pipeline base and multiple watched folder paths in `RuntimePathConfig` and exposed `getPipelineWatchedFoldersPaths()` (app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java). - Updated `FileMonitor` to accept and register multiple root paths instead of a single root (app/common/src/main/java/stirling/software/common/util/FileMonitor.java). - Updated `PipelineDirectoryProcessor` to iterate all configured watched roots and to walk subdirectories while ignoring `processing` dirs (app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java). - Exposed the new settings in `settings.yml.template` and the admin UI, including a multi-line `Textarea` to edit `watchedFoldersDirs` (app/core/src/main/resources/settings.yml.template, frontend/src/proprietary/components/shared/config/configSections/AdminGeneralSection.tsx). - Adjusted unit test setup to account for list-based watched folders (app/common/src/test/java/stirling/software/common/util/FileMonitorTest.java). ### Testing - Ran formatting and build checks with `./gradlew spotlessApply` and `./gradlew build` using Java 21 via `JAVA_HOME=/root/.local/share/mise/installs/java/21.0.2 PATH=/root/.local/share/mise/installs/java/21.0.2/bin:$PATH ./gradlew ...`, but both runs failed due to Gradle plugin resolution being blocked in this environment (plugin portal/network 403), so full compilation/formatting could not complete. - Confirmed the code compiles locally was not possible here; unit test `FileMonitorTest` was updated to use the new API but was not executed due to the blocked build. - Changes were committed (`Support multiple pipeline watch directories`) and the repository diff contains the listed file modifications. ------ [Codex Task](https://chatgpt.com/codex/tasks/task_b_69741ecd17c883288d8085a63ccd66f4)
This commit is contained in:
Anthony Stirling
@@ -4514,10 +4514,18 @@ description = "Configure custom file system paths for pipeline processing and ex
|
||||
[admin.settings.general.customPaths.pipeline]
|
||||
label = "Pipeline Directories"
|
||||
|
||||
[admin.settings.general.customPaths.pipeline.pipelineDir]
|
||||
label = "Pipeline Directory"
|
||||
description = "Base directory for pipeline resources (leave empty for default: /pipeline)"
|
||||
|
||||
[admin.settings.general.customPaths.pipeline.watchedFoldersDir]
|
||||
label = "Watched Folders Directory"
|
||||
description = "Directory where pipeline monitors for incoming PDFs (leave empty for default: /pipeline/watchedFolders)"
|
||||
|
||||
[admin.settings.general.customPaths.pipeline.watchedFoldersDirs]
|
||||
label = "Watched Folders Directories"
|
||||
description = "Directories where pipeline monitors for incoming PDFs (one per line or comma-separated; leave empty for default: /pipeline/watchedFolders)"
|
||||
|
||||
[admin.settings.general.customPaths.pipeline.finishedFoldersDir]
|
||||
label = "Finished Folders Directory"
|
||||
description = "Directory where processed PDFs are outputted (leave empty for default: /pipeline/finishedFolders)"
|
||||
|
||||
+136
-12
@@ -1,7 +1,7 @@
|
||||
import { useEffect, useState, useRef, useCallback, useMemo } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useLocation, useNavigate } from 'react-router-dom';
|
||||
import { TextInput, Switch, Button, Stack, Paper, Text, Loader, Group, MultiSelect, Badge, SegmentedControl, Select } from '@mantine/core';
|
||||
import { TextInput, Textarea, Switch, Button, Stack, Paper, Text, Loader, Group, MultiSelect, Badge, SegmentedControl, Select } from '@mantine/core';
|
||||
import { alert } from '@app/components/toast';
|
||||
import RestartConfirmationModal from '@app/components/shared/config/RestartConfirmationModal';
|
||||
import { useRestartServer } from '@app/components/shared/config/useRestartServer';
|
||||
@@ -31,7 +31,9 @@ interface GeneralSettingsData {
|
||||
};
|
||||
customPaths?: {
|
||||
pipeline?: {
|
||||
pipelineDir?: string;
|
||||
watchedFoldersDir?: string;
|
||||
watchedFoldersDirs?: string[];
|
||||
finishedFoldersDir?: string;
|
||||
};
|
||||
operations?: {
|
||||
@@ -61,6 +63,17 @@ export default function AdminGeneralSection() {
|
||||
.sort((a, b) => a.label.localeCompare(b.label)),
|
||||
[]
|
||||
);
|
||||
const parseWatchedFoldersInput = useCallback((value: string) => {
|
||||
const paths = value
|
||||
.split(/[\n,;]+/)
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
// Deduplicate paths (case-sensitive, exact match)
|
||||
const uniquePaths = Array.from(new Set(paths));
|
||||
|
||||
return uniquePaths;
|
||||
}, []);
|
||||
|
||||
// Track original settings for dirty detection
|
||||
const [originalSettingsSnapshot, setOriginalSettingsSnapshot] = useState<string>('');
|
||||
@@ -91,17 +104,31 @@ export default function AdminGeneralSection() {
|
||||
|
||||
ui.languages = Array.isArray(ui.languages) ? toUnderscoreLanguages(ui.languages) : [];
|
||||
|
||||
const pipelinePaths = system.customPaths?.pipeline || {};
|
||||
const watchedFoldersDirs = Array.isArray(pipelinePaths.watchedFoldersDirs)
|
||||
? pipelinePaths.watchedFoldersDirs
|
||||
: [];
|
||||
const normalizedWatchedFoldersDirs =
|
||||
watchedFoldersDirs.length > 0
|
||||
? watchedFoldersDirs
|
||||
: (pipelinePaths.watchedFoldersDir ? [pipelinePaths.watchedFoldersDir] : []);
|
||||
|
||||
const result: any = {
|
||||
ui,
|
||||
system,
|
||||
customPaths: system.customPaths || {
|
||||
customPaths: {
|
||||
...(system.customPaths || {}),
|
||||
pipeline: {
|
||||
watchedFoldersDir: '',
|
||||
finishedFoldersDir: ''
|
||||
...pipelinePaths,
|
||||
pipelineDir: pipelinePaths.pipelineDir || '',
|
||||
watchedFoldersDir: pipelinePaths.watchedFoldersDir || '',
|
||||
watchedFoldersDirs: normalizedWatchedFoldersDirs,
|
||||
finishedFoldersDir: pipelinePaths.finishedFoldersDir || ''
|
||||
},
|
||||
operations: {
|
||||
weasyprint: '',
|
||||
unoconvert: ''
|
||||
...(system.customPaths?.operations || {}),
|
||||
weasyprint: system.customPaths?.operations?.weasyprint || '',
|
||||
unoconvert: system.customPaths?.operations?.unoconvert || ''
|
||||
}
|
||||
},
|
||||
customMetadata: premium.proFeatures?.customMetadata || {
|
||||
@@ -154,7 +181,9 @@ export default function AdminGeneralSection() {
|
||||
};
|
||||
|
||||
if (settings.customPaths) {
|
||||
deltaSettings['system.customPaths.pipeline.pipelineDir'] = settings.customPaths?.pipeline?.pipelineDir;
|
||||
deltaSettings['system.customPaths.pipeline.watchedFoldersDir'] = settings.customPaths?.pipeline?.watchedFoldersDir;
|
||||
deltaSettings['system.customPaths.pipeline.watchedFoldersDirs'] = settings.customPaths?.pipeline?.watchedFoldersDirs;
|
||||
deltaSettings['system.customPaths.pipeline.finishedFoldersDir'] = settings.customPaths?.pipeline?.finishedFoldersDir;
|
||||
deltaSettings['system.customPaths.operations.weasyprint'] = settings.customPaths?.operations?.weasyprint;
|
||||
deltaSettings['system.customPaths.operations.unoconvert'] = settings.customPaths?.operations?.unoconvert;
|
||||
@@ -171,6 +200,54 @@ export default function AdminGeneralSection() {
|
||||
() => toUnderscoreLanguages(settings.ui?.languages || []),
|
||||
[settings.ui?.languages]
|
||||
);
|
||||
const watchedFoldersInput = useMemo(() => (
|
||||
(settings.customPaths?.pipeline?.watchedFoldersDirs || []).join('\n')
|
||||
), [settings.customPaths?.pipeline?.watchedFoldersDirs]);
|
||||
|
||||
const watchedFoldersValidation = useMemo(() => {
|
||||
const paths = settings.customPaths?.pipeline?.watchedFoldersDirs || [];
|
||||
const finishedPath = settings.customPaths?.pipeline?.finishedFoldersDir || '';
|
||||
const warnings: string[] = [];
|
||||
|
||||
// Normalize paths for comparison (handle both Windows and Unix paths)
|
||||
const normalizePath = (p: string) => p.replace(/\\/g, '/').replace(/\/+$/, '');
|
||||
|
||||
// Check for overlapping watched folders
|
||||
if (paths.length >= 2) {
|
||||
for (let i = 0; i < paths.length; i++) {
|
||||
for (let j = i + 1; j < paths.length; j++) {
|
||||
const path1 = normalizePath(paths[i]);
|
||||
const path2 = normalizePath(paths[j]);
|
||||
|
||||
if (path1 === path2) {
|
||||
warnings.push(`Duplicate path detected: '${paths[i]}'`);
|
||||
} else if (path1.startsWith(path2 + '/')) {
|
||||
warnings.push(`'${paths[i]}' is nested inside '${paths[j]}' - may cause duplicate processing`);
|
||||
} else if (path2.startsWith(path1 + '/')) {
|
||||
warnings.push(`'${paths[j]}' is nested inside '${paths[i]}' - may cause duplicate processing`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for conflicts with finished folder
|
||||
if (finishedPath && paths.length > 0) {
|
||||
const normalizedFinished = normalizePath(finishedPath);
|
||||
for (const watchedPath of paths) {
|
||||
const normalizedWatched = normalizePath(watchedPath);
|
||||
|
||||
if (normalizedWatched === normalizedFinished) {
|
||||
warnings.push(`CRITICAL: Watched folder '${watchedPath}' is the same as finished folder - will cause processing loops!`);
|
||||
} else if (normalizedFinished.startsWith(normalizedWatched + '/')) {
|
||||
warnings.push(`Finished folder is nested inside watched folder '${watchedPath}' - may cause issues`);
|
||||
} else if (normalizedWatched.startsWith(normalizedFinished + '/')) {
|
||||
warnings.push(`CRITICAL: Watched folder '${watchedPath}' is nested inside finished folder - will cause processing loops!`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return warnings.length > 0 ? warnings : null;
|
||||
}, [settings.customPaths?.pipeline?.watchedFoldersDirs, settings.customPaths?.pipeline?.finishedFoldersDir]);
|
||||
|
||||
// Filter default locale options based on available languages setting
|
||||
const defaultLocaleOptions = useMemo(() => {
|
||||
@@ -651,27 +728,74 @@ export default function AdminGeneralSection() {
|
||||
<TextInput
|
||||
label={
|
||||
<Group gap="xs">
|
||||
<span>{t('admin.settings.general.customPaths.pipeline.watchedFoldersDir.label', 'Watched Folders Directory')}</span>
|
||||
<PendingBadge show={isFieldPending('customPaths.pipeline.watchedFoldersDir')} />
|
||||
<span>{t('admin.settings.general.customPaths.pipeline.pipelineDir.label', 'Pipeline Directory')}</span>
|
||||
<PendingBadge show={isFieldPending('customPaths.pipeline.pipelineDir')} />
|
||||
</Group>
|
||||
}
|
||||
description={t('admin.settings.general.customPaths.pipeline.watchedFoldersDir.description', 'Directory where pipeline monitors for incoming PDFs (leave empty for default: /pipeline/watchedFolders)')}
|
||||
value={settings.customPaths?.pipeline?.watchedFoldersDir || ''}
|
||||
description={t('admin.settings.general.customPaths.pipeline.pipelineDir.description', 'Base directory for pipeline resources (leave empty for default: /pipeline)')}
|
||||
value={settings.customPaths?.pipeline?.pipelineDir || ''}
|
||||
onChange={(e) => setSettings({
|
||||
...settings,
|
||||
customPaths: {
|
||||
...settings.customPaths,
|
||||
pipeline: {
|
||||
...settings.customPaths?.pipeline,
|
||||
watchedFoldersDir: e.target.value
|
||||
pipelineDir: e.target.value
|
||||
}
|
||||
}
|
||||
})}
|
||||
placeholder="/pipeline/watchedFolders"
|
||||
placeholder="/pipeline"
|
||||
disabled={!loginEnabled}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Textarea
|
||||
label={
|
||||
<Group gap="xs">
|
||||
<span>{t('admin.settings.general.customPaths.pipeline.watchedFoldersDirs.label', 'Watched Folders Directories')}</span>
|
||||
<PendingBadge show={
|
||||
isFieldPending('customPaths.pipeline.watchedFoldersDirs')
|
||||
|| isFieldPending('customPaths.pipeline.watchedFoldersDir')
|
||||
} />
|
||||
</Group>
|
||||
}
|
||||
description={t('admin.settings.general.customPaths.pipeline.watchedFoldersDirs.description', 'Directories where pipeline monitors for incoming PDFs (one per line or comma-separated; leave empty for default: /pipeline/watchedFolders)')}
|
||||
value={watchedFoldersInput}
|
||||
onChange={(e) => {
|
||||
const parsedDirs = parseWatchedFoldersInput(e.target.value);
|
||||
setSettings({
|
||||
...settings,
|
||||
customPaths: {
|
||||
...settings.customPaths,
|
||||
pipeline: {
|
||||
...settings.customPaths?.pipeline,
|
||||
watchedFoldersDir: parsedDirs[0] || '',
|
||||
watchedFoldersDirs: parsedDirs
|
||||
}
|
||||
}
|
||||
});
|
||||
}}
|
||||
placeholder="/pipeline/watchedFolders"
|
||||
minRows={3}
|
||||
autosize
|
||||
disabled={!loginEnabled}
|
||||
/>
|
||||
{watchedFoldersValidation && (
|
||||
<Stack gap="xs" mt="xs">
|
||||
{watchedFoldersValidation.map((warning, idx) => (
|
||||
<Text
|
||||
key={idx}
|
||||
size="sm"
|
||||
c={warning.includes('CRITICAL') ? 'red' : 'yellow'}
|
||||
>
|
||||
{warning}
|
||||
</Text>
|
||||
))}
|
||||
</Stack>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<TextInput
|
||||
label={
|
||||
|
||||
Reference in New Issue
Block a user