arsvendg/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/arsvendg/Stirling-PDF.git synced 2026-07-16 19:33:11 +02:00
Code Issues Actions 4 Packages Projects Releases 1 Wiki Activity

Support multiple pipeline watch directories and configurable pipeline base path (#5545)

Browse Source 
### Motivation
- Allow operators to configure a pipeline base directory and multiple
watched folders so the pipeline can monitor several directories and
subdirectories concurrently.
- Ensure scanning traverses subdirectories while skipping internal
processing folders (e.g. `processing`) and preserve existing behavior
for finished/output paths.
- Expose the new options in the server `settings.yml.template` and the
admin UI so paths can be edited from the web console.

### Description
- Added new `pipelineDir` and `watchedFoldersDirs` fields to
`ApplicationProperties.CustomPaths.Pipeline` and kept backward
compatibility with `watchedFoldersDir`
(app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java).
- Resolved pipeline base and multiple watched folder paths in
`RuntimePathConfig` and exposed `getPipelineWatchedFoldersPaths()`
(app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java).
- Updated `FileMonitor` to accept and register multiple root paths
instead of a single root
(app/common/src/main/java/stirling/software/common/util/FileMonitor.java).
- Updated `PipelineDirectoryProcessor` to iterate all configured watched
roots and to walk subdirectories while ignoring `processing` dirs
(app/core/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java).
- Exposed the new settings in `settings.yml.template` and the admin UI,
including a multi-line `Textarea` to edit `watchedFoldersDirs`
(app/core/src/main/resources/settings.yml.template,
frontend/src/proprietary/components/shared/config/configSections/AdminGeneralSection.tsx).
- Adjusted unit test setup to account for list-based watched folders
(app/common/src/test/java/stirling/software/common/util/FileMonitorTest.java).

### Testing
- Ran formatting and build checks with `./gradlew spotlessApply` and
`./gradlew build` using Java 21 via
`JAVA_HOME=/root/.local/share/mise/installs/java/21.0.2
PATH=/root/.local/share/mise/installs/java/21.0.2/bin:$PATH ./gradlew
...`, but both runs failed due to Gradle plugin resolution being blocked
in this environment (plugin portal/network 403), so full
compilation/formatting could not complete.
- Confirmed the code compiles locally was not possible here; unit test
`FileMonitorTest` was updated to use the new API but was not executed
due to the blocked build.
- Changes were committed (`Support multiple pipeline watch directories`)
and the repository diff contains the listed file modifications.

------
[Codex
Task](https://chatgpt.com/codex/tasks/task_b_69741ecd17c883288d8085a63ccd66f4)
This commit is contained in:
Anthony Stirling
2026-01-31 20:59:25 +00:00
committed by GitHub
parent 2ae413c5ea
commit 4f404a1ccf
9 changed files with 452 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/proprietary/components/shared/config/configSections/AdminGeneralSection.tsx
+136 -12
View File
@@ -1,7 +1,7 @@
import { useEffect, useState, useRef, useCallback, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { useLocation, useNavigate } from 'react-router-dom';
import { TextInput, Switch, Button, Stack, Paper, Text, Loader, Group, MultiSelect, Badge, SegmentedControl, Select } from '@mantine/core';
import { TextInput, Textarea, Switch, Button, Stack, Paper, Text, Loader, Group, MultiSelect, Badge, SegmentedControl, Select } from '@mantine/core';
import { alert } from '@app/components/toast';
import RestartConfirmationModal from '@app/components/shared/config/RestartConfirmationModal';
import { useRestartServer } from '@app/components/shared/config/useRestartServer';
@@ -31,7 +31,9 @@ interface GeneralSettingsData {
};
customPaths?: {
pipeline?: {
pipelineDir?: string;
watchedFoldersDir?: string;
watchedFoldersDirs?: string[];
finishedFoldersDir?: string;
};
operations?: {
@@ -61,6 +63,17 @@ export default function AdminGeneralSection() {
.sort((a, b) => a.label.localeCompare(b.label)),
[]
);
const parseWatchedFoldersInput = useCallback((value: string) => {
const paths = value
.split(/[\n,;]+/)
.map((entry) => entry.trim())
.filter(Boolean);
// Deduplicate paths (case-sensitive, exact match)
const uniquePaths = Array.from(new Set(paths));
return uniquePaths;
}, []);
// Track original settings for dirty detection
const [originalSettingsSnapshot, setOriginalSettingsSnapshot] = useState<string>('');
@@ -91,17 +104,31 @@ export default function AdminGeneralSection() {
ui.languages = Array.isArray(ui.languages) ? toUnderscoreLanguages(ui.languages) : [];
const pipelinePaths = system.customPaths?.pipeline || {};
const watchedFoldersDirs = Array.isArray(pipelinePaths.watchedFoldersDirs)
? pipelinePaths.watchedFoldersDirs
: [];
const normalizedWatchedFoldersDirs =
watchedFoldersDirs.length > 0
? watchedFoldersDirs
: (pipelinePaths.watchedFoldersDir ? [pipelinePaths.watchedFoldersDir] : []);
const result: any = {
ui,
system,
customPaths: system.customPaths || {
customPaths: {
...(system.customPaths || {}),
pipeline: {
watchedFoldersDir: '',
finishedFoldersDir: ''
...pipelinePaths,
pipelineDir: pipelinePaths.pipelineDir || '',
watchedFoldersDir: pipelinePaths.watchedFoldersDir || '',
watchedFoldersDirs: normalizedWatchedFoldersDirs,
finishedFoldersDir: pipelinePaths.finishedFoldersDir || ''
},
operations: {
weasyprint: '',
unoconvert: ''
...(system.customPaths?.operations || {}),
weasyprint: system.customPaths?.operations?.weasyprint || '',
unoconvert: system.customPaths?.operations?.unoconvert || ''
}
},
customMetadata: premium.proFeatures?.customMetadata || {
@@ -154,7 +181,9 @@ export default function AdminGeneralSection() {
};
if (settings.customPaths) {
deltaSettings['system.customPaths.pipeline.pipelineDir'] = settings.customPaths?.pipeline?.pipelineDir;
deltaSettings['system.customPaths.pipeline.watchedFoldersDir'] = settings.customPaths?.pipeline?.watchedFoldersDir;
deltaSettings['system.customPaths.pipeline.watchedFoldersDirs'] = settings.customPaths?.pipeline?.watchedFoldersDirs;
deltaSettings['system.customPaths.pipeline.finishedFoldersDir'] = settings.customPaths?.pipeline?.finishedFoldersDir;
deltaSettings['system.customPaths.operations.weasyprint'] = settings.customPaths?.operations?.weasyprint;
deltaSettings['system.customPaths.operations.unoconvert'] = settings.customPaths?.operations?.unoconvert;
@@ -171,6 +200,54 @@ export default function AdminGeneralSection() {
() => toUnderscoreLanguages(settings.ui?.languages || []),
[settings.ui?.languages]
);
const watchedFoldersInput = useMemo(() => (
(settings.customPaths?.pipeline?.watchedFoldersDirs || []).join('\n')
), [settings.customPaths?.pipeline?.watchedFoldersDirs]);
const watchedFoldersValidation = useMemo(() => {
const paths = settings.customPaths?.pipeline?.watchedFoldersDirs || [];
const finishedPath = settings.customPaths?.pipeline?.finishedFoldersDir || '';
const warnings: string[] = [];
// Normalize paths for comparison (handle both Windows and Unix paths)
const normalizePath = (p: string) => p.replace(/\\/g, '/').replace(/\/+$/, '');
// Check for overlapping watched folders
if (paths.length >= 2) {
for (let i = 0; i < paths.length; i++) {
for (let j = i + 1; j < paths.length; j++) {
const path1 = normalizePath(paths[i]);
const path2 = normalizePath(paths[j]);
if (path1 === path2) {
warnings.push(`Duplicate path detected: '${paths[i]}'`);
} else if (path1.startsWith(path2 + '/')) {
warnings.push(`'${paths[i]}' is nested inside '${paths[j]}' - may cause duplicate processing`);
} else if (path2.startsWith(path1 + '/')) {
warnings.push(`'${paths[j]}' is nested inside '${paths[i]}' - may cause duplicate processing`);
}
}
}
}
// Check for conflicts with finished folder
if (finishedPath && paths.length > 0) {
const normalizedFinished = normalizePath(finishedPath);
for (const watchedPath of paths) {
const normalizedWatched = normalizePath(watchedPath);
if (normalizedWatched === normalizedFinished) {
warnings.push(`CRITICAL: Watched folder '${watchedPath}' is the same as finished folder - will cause processing loops!`);
} else if (normalizedFinished.startsWith(normalizedWatched + '/')) {
warnings.push(`Finished folder is nested inside watched folder '${watchedPath}' - may cause issues`);
} else if (normalizedWatched.startsWith(normalizedFinished + '/')) {
warnings.push(`CRITICAL: Watched folder '${watchedPath}' is nested inside finished folder - will cause processing loops!`);
}
}
}
return warnings.length > 0 ? warnings : null;
}, [settings.customPaths?.pipeline?.watchedFoldersDirs, settings.customPaths?.pipeline?.finishedFoldersDir]);
// Filter default locale options based on available languages setting
const defaultLocaleOptions = useMemo(() => {
@@ -651,27 +728,74 @@ export default function AdminGeneralSection() {
<TextInput
label={
<Group gap="xs">
<span>{t('admin.settings.general.customPaths.pipeline.watchedFoldersDir.label', 'Watched Folders Directory')}</span>
<PendingBadge show={isFieldPending('customPaths.pipeline.watchedFoldersDir')} />
<span>{t('admin.settings.general.customPaths.pipeline.pipelineDir.label', 'Pipeline Directory')}</span>
<PendingBadge show={isFieldPending('customPaths.pipeline.pipelineDir')} />
</Group>
}
description={t('admin.settings.general.customPaths.pipeline.watchedFoldersDir.description', 'Directory where pipeline monitors for incoming PDFs (leave empty for default: /pipeline/watchedFolders)')}
value={settings.customPaths?.pipeline?.watchedFoldersDir || ''}
description={t('admin.settings.general.customPaths.pipeline.pipelineDir.description', 'Base directory for pipeline resources (leave empty for default: /pipeline)')}
value={settings.customPaths?.pipeline?.pipelineDir || ''}
onChange={(e) => setSettings({
...settings,
customPaths: {
...settings.customPaths,
pipeline: {
...settings.customPaths?.pipeline,
watchedFoldersDir: e.target.value
pipelineDir: e.target.value
}
}
})}
placeholder="/pipeline/watchedFolders"
placeholder="/pipeline"
disabled={!loginEnabled}
/>
</div>
<div>
<Textarea
label={
<Group gap="xs">
<span>{t('admin.settings.general.customPaths.pipeline.watchedFoldersDirs.label', 'Watched Folders Directories')}</span>
<PendingBadge show={
isFieldPending('customPaths.pipeline.watchedFoldersDirs')
|| isFieldPending('customPaths.pipeline.watchedFoldersDir')
} />
</Group>
}
description={t('admin.settings.general.customPaths.pipeline.watchedFoldersDirs.description', 'Directories where pipeline monitors for incoming PDFs (one per line or comma-separated; leave empty for default: /pipeline/watchedFolders)')}
value={watchedFoldersInput}
onChange={(e) => {
const parsedDirs = parseWatchedFoldersInput(e.target.value);
setSettings({
...settings,
customPaths: {
...settings.customPaths,
pipeline: {
...settings.customPaths?.pipeline,
watchedFoldersDir: parsedDirs[0] || '',
watchedFoldersDirs: parsedDirs
}
}
});
}}
placeholder="/pipeline/watchedFolders"
minRows={3}
autosize
disabled={!loginEnabled}
/>
{watchedFoldersValidation && (
<Stack gap="xs" mt="xs">
{watchedFoldersValidation.map((warning, idx) => (
<Text
key={idx}
size="sm"
c={warning.includes('CRITICAL') ? 'red' : 'yellow'}
>
{warning}
</Text>
))}
</Stack>
)}
</div>
<div>
<TextInput
label={