From 3675db5907b24c918abf821b79604d71a8865331 Mon Sep 17 00:00:00 2001 From: James Brunton Date: Wed, 10 Jun 2026 11:55:29 +0100 Subject: [PATCH 1/5] Add new triggers, sources and sinks to policies (#6543) # Description of Changes Add new triggers: - Schedule (fires every X amount of time) - Folder watch (fires whenever the OS tells us a folder has a new file in it; on Mac this is technically a 2s schedule but that's just how Java implements it) Add new sources: - Folder (reads from this directory) Add new sinks: - Inline (stores in FileStorage) - Folder (stores in specified directory) Still want to do S3 buckets and web hooks and stuff, but they can come in a future PR. I'm hoping this should make it sufficient to be able to integrate with processing folders frontend etc. I've also changed it so that policies can have multiple sources and triggers at once, which seems like it might be useful. --- .../common/model/ApplicationProperties.java | 40 +++ .../src/main/resources/settings.yml.template | 12 + .../policy/config/FolderAccessGuard.java | 106 ++++++ .../policy/controller/PolicyController.java | 56 +++- .../policy/engine/PolicyRunRegistry.java | 12 +- .../policy/engine/PolicyRunner.java | 116 +++++++ .../policy/engine/PolicyValidator.java | 74 +++++ .../policy/input/FolderInputSource.java | 186 +++++++++++ .../proprietary/policy/input/InputSource.java | 49 +++ .../policy/input/ResolvedInput.java | 23 ++ .../proprietary/policy/model/InputSpec.java | 26 ++ .../proprietary/policy/model/OutputSpec.java | 5 + .../proprietary/policy/model/Policy.java | 33 +- .../proprietary/policy/model/Schedule.java | 141 ++++++++ .../policy/model/TriggerConfig.java | 20 +- .../policy/output/FolderOutputSink.java | 130 ++++++++ .../policy/output/PolicyOutputSink.java | 6 + .../policy/store/InProcessPolicyStore.java | 2 + .../policy/store/JpaPolicyStore.java | 3 +- .../policy/trigger/FolderWatchTrigger.java | 302 ++++++++++++++++++ .../policy/trigger/ManualTrigger.java | 41 --- .../policy/trigger/PolicyTrigger.java | 32 +- .../policy/trigger/PolicyTriggerManager.java | 56 ++++ .../policy/trigger/ScheduleTrigger.java | 158 +++++++++ .../policy/config/FolderAccessGuardTest.java | 99 ++++++ .../policy/engine/PolicyEngineTest.java | 5 +- .../policy/engine/PolicyRunRegistryTest.java | 3 +- .../policy/engine/PolicyRunnerTest.java | 159 +++++++++ .../policy/engine/PolicyValidatorTest.java | 114 +++++++ .../policy/input/FolderInputSourceTest.java | 137 ++++++++ .../policy/output/FolderOutputSinkTest.java | 105 ++++++ .../store/InProcessPolicyStoreTest.java | 23 +- .../policy/store/JpaPolicyStoreTest.java | 20 +- .../trigger/FolderWatchTriggerTest.java | 178 +++++++++++ .../trigger/PolicyTriggerManagerTest.java | 51 +++ .../policy/trigger/ScheduleTriggerTest.java | 147 +++++++++ 36 files changed, 2556 insertions(+), 114 deletions(-) create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/config/FolderAccessGuard.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyValidator.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolvedInput.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/InputSpec.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Schedule.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java delete mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManager.java create mode 100644 app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/config/FolderAccessGuardTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyValidatorTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManagerTest.java create mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java diff --git a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java index 00e26ec64..067be54eb 100644 --- a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java +++ b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java @@ -80,6 +80,7 @@ public class ApplicationProperties { private Mcp mcp = new Mcp(); private InternalApi internalApi = new InternalApi(); private Cluster cluster = new Cluster(); + private Policies policies = new Policies(); @Bean public PropertySource dynamicYamlPropertySource(ConfigurableEnvironment environment) @@ -203,6 +204,45 @@ public class ApplicationProperties { } } + @Data + public static class Policies { + /** + * Absolute directories that policy folder input sources and output sinks may read from or + * write to. Empty (the default) disables folder access entirely, so a policy can never be + * pointed at an arbitrary server path. Stirling's own config directory is always + * off-limits, and folder access is always disabled in SaaS mode regardless of this list. + */ + private List allowedFolderRoots = new java.util.ArrayList<>(); + + /** How often (seconds) the schedule trigger checks for policies whose schedule is due. */ + private long scheduleSweepSeconds = 60; + + /** + * How often (seconds) the folder-watch trigger reconciles its watch registrations and + * re-runs every folder-watch policy as a safety net for filesystem events that were missed + * (NFS, bind mounts, inotify-queue overflow). + */ + private long watchReconcileSeconds = 300; + + /** + * How long (milliseconds) the folder-watch trigger keeps draining filesystem events after + * the first, so a burst from a single file copy coalesces into one run instead of many. + */ + private long watchQuietPeriodMs = 500; + + /** + * SSE emitter timeout (milliseconds) for streamed runs; generous for long multi-step runs. + */ + private long streamTimeoutMs = 1800000; + + /** + * How long (minutes) a finished run's in-memory state is retained before eviction, + * mirroring the job-result expiry so rich run state does not outlive the process. Active + * and paused runs are kept regardless of age. + */ + private int runExpiryMinutes = 30; + } + @Data public static class PdfEditor { private Cache cache = new Cache(); diff --git a/app/core/src/main/resources/settings.yml.template b/app/core/src/main/resources/settings.yml.template index d4ecb35f7..413f39706 100644 --- a/app/core/src/main/resources/settings.yml.template +++ b/app/core/src/main/resources/settings.yml.template @@ -364,6 +364,18 @@ aiEngine: url: http://localhost:5001 # URL of the Python AI engine timeoutSeconds: 120 # Timeout in seconds for AI engine requests +policies: + # Folder automations can read from and write to the directories you allow here, so treat this as a + # security boundary. Leave allowedFolderRoots empty (default) to disable folder sources/outputs + # entirely; list absolute directories to permit folder access only within them. Stirling's own + # config directory is always off-limits, and folder access is always disabled in SaaS mode. + allowedFolderRoots: [] # e.g. ["/data/inbox", "/data/outbox"] + scheduleSweepSeconds: 60 # How often (seconds) scheduled policies are checked for being due + watchReconcileSeconds: 300 # How often (seconds) folder-watch re-syncs watches and re-runs as a safety net for missed events + watchQuietPeriodMs: 500 # How long (ms) folder-watch coalesces a burst of file events into a single run + streamTimeoutMs: 1800000 # SSE timeout (ms) for live run-progress streams + runExpiryMinutes: 30 # How long (minutes) a finished run's in-memory state is kept before eviction + # Model Context Protocol (MCP) server. Exposes Stirling's PDF tools (grouped by namespace) # plus the AI agents to MCP clients (Inspector, Claude Desktop, custom). OAuth-protected. # Disabled by default - enable explicitly per deployment after configuring mcp.auth. diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/FolderAccessGuard.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/FolderAccessGuard.java new file mode 100644 index 000000000..3abf919f1 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/config/FolderAccessGuard.java @@ -0,0 +1,106 @@ +package stirling.software.proprietary.policy.config; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +import stirling.software.common.configuration.InstallationPathConfig; +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.model.Policy; + +/** + * The single authority on which filesystem locations a policy may read from or write to. Folder + * sources and sinks take a configured directory, so without this a user who can save a policy could + * point one at Stirling's own config/secrets directory and exfiltrate (or overwrite) it. Every + * folder source and sink runs its directory through {@link #requirePermitted(Path)} at save time + * and again at run time. + * + *

Enforced fail-closed, in order: + * + *

+ * + *

Paths are compared after normalisation, so {@code ..} segments cannot walk out of an allowed + * root. (Symlink escape is not defended here; an operator who configures an allowed root containing + * a symlink to a sensitive location is trusted.) + */ +@Component +public class FolderAccessGuard { + + public static final String FOLDER_TYPE = "folder"; + + private final boolean saasActive; + private final List allowedRoots; + private final List protectedRoots; + + public FolderAccessGuard(ApplicationProperties applicationProperties, Environment environment) { + this.saasActive = Arrays.asList(environment.getActiveProfiles()).contains("saas"); + this.allowedRoots = + normalizeAll(applicationProperties.getPolicies().getAllowedFolderRoots()); + this.protectedRoots = List.of(normalize(Path.of(InstallationPathConfig.getConfigPath()))); + } + + /** + * Check that {@code dir} is a permitted folder location, returning its normalised absolute + * form. + * + * @throws IllegalArgumentException if folder access is disabled (SaaS or no roots configured), + * the path is inside a protected directory, or it falls outside every allowed root + */ + public Path requirePermitted(Path dir) { + if (saasActive) { + throw new IllegalArgumentException( + "folder sources and outputs are not available in SaaS mode"); + } + Path normalized = normalize(dir); + for (Path protectedRoot : protectedRoots) { + if (normalized.startsWith(protectedRoot)) { + throw new IllegalArgumentException( + "folder may not point inside a protected Stirling directory"); + } + } + if (allowedRoots.isEmpty()) { + throw new IllegalArgumentException( + "folder access is disabled; set policies.allowedFolderRoots to permit it"); + } + boolean within = allowedRoots.stream().anyMatch(normalized::startsWith); + if (!within) { + throw new IllegalArgumentException( + "folder '" + normalized + "' is outside the allowed folder roots"); + } + return normalized; + } + + /** Whether this policy reads from or writes to a folder, and so is subject to these rules. */ + public boolean usesFolderAccess(Policy policy) { + boolean readsFolder = + policy.sources().stream().anyMatch(spec -> FOLDER_TYPE.equals(spec.type())); + boolean writesFolder = + policy.output() != null && FOLDER_TYPE.equals(policy.output().type()); + return readsFolder || writesFolder; + } + + private static List normalizeAll(List roots) { + List result = new ArrayList<>(); + for (String root : roots) { + if (root != null && !root.isBlank()) { + result.add(normalize(Path.of(root))); + } + } + return result; + } + + private static Path normalize(Path path) { + return path.toAbsolutePath().normalize(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java index 0a94eef1f..df65fbd99 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java @@ -6,7 +6,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; import org.springframework.http.HttpStatus; @@ -34,11 +33,16 @@ import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import stirling.software.common.model.ApplicationProperties; import stirling.software.common.model.job.JobResponse; +import stirling.software.common.service.UserServiceInterface; import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; +import stirling.software.proprietary.policy.config.FolderAccessGuard; import stirling.software.proprietary.policy.engine.PolicyRunHandle; import stirling.software.proprietary.policy.engine.PolicyRunRegistry; +import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.engine.PolicyValidator; import stirling.software.proprietary.policy.model.PipelineDefinition; import stirling.software.proprietary.policy.model.Policy; import stirling.software.proprietary.policy.model.PolicyInputs; @@ -47,7 +51,6 @@ import stirling.software.proprietary.policy.model.PolicyRunStatus; import stirling.software.proprietary.policy.model.PolicyRunView; import stirling.software.proprietary.policy.progress.PolicyProgressListener; import stirling.software.proprietary.policy.store.PolicyStore; -import stirling.software.proprietary.policy.trigger.ManualTrigger; import stirling.software.proprietary.security.config.PremiumEndpoint; import tools.jackson.core.JacksonException; @@ -71,16 +74,16 @@ import tools.jackson.databind.ObjectMapper; @Tag(name = "Policies", description = "Run tool pipelines on the backend") public class PolicyController { - private final ManualTrigger manualTrigger; + private final PolicyRunner policyRunner; private final PolicyRunRegistry runRegistry; private final PolicyStore policyStore; + private final PolicyValidator policyValidator; + private final FolderAccessGuard folderAccessGuard; + private final UserServiceInterface userService; + private final ApplicationProperties applicationProperties; private final ObjectMapper objectMapper; private final TempFileManager tempFileManager; - /** SSE emitter timeout, generous enough for long multi-step runs on large files. */ - @Value("${stirling.policies.streamTimeoutMs:1800000}") - private long streamTimeoutMs; - @PostMapping(value = "/run", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) @Operation( summary = "Run a tool pipeline", @@ -95,7 +98,8 @@ public class PolicyController { throws IOException { PipelineDefinition definition = parseDefinition(json); PolicyInputs inputs = collectInputs(request); - String runId = manualTrigger.fire(definition, inputs, PolicyProgressListener.NOOP).runId(); + String runId = + policyRunner.runAdHoc(definition, inputs, PolicyProgressListener.NOOP).runId(); return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null)); } @@ -112,10 +116,11 @@ public class PolicyController { PipelineDefinition definition = parseDefinition(json); PolicyInputs inputs = collectInputs(request); - SseEmitter emitter = new SseEmitter(streamTimeoutMs); + SseEmitter emitter = + new SseEmitter(applicationProperties.getPolicies().getStreamTimeoutMs()); emitter.onError(e -> log.warn("Policy run SSE emitter error", e)); - PolicyRunHandle handle = manualTrigger.fire(definition, inputs, streamListener(emitter)); + PolicyRunHandle handle = policyRunner.runAdHoc(definition, inputs, streamListener(emitter)); // Close the stream with a terminal event once the run finishes. whenComplete runs on the // engine's worker thread after the run is done, so this never races the step events. handle.completion() @@ -155,7 +160,34 @@ public class PolicyController { "Stores a policy (trigger config + steps + output + metadata). A blank id is" + " assigned; returns the stored policy with its id.") public ResponseEntity savePolicy(@RequestBody String json) { - return ResponseEntity.ok(policyStore.save(parsePolicy(json))); + Policy policy = parsePolicy(json); + requireAuthorizedForFolderAccess(policy); + try { + policyValidator.validate(policy); + } catch (IllegalArgumentException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); + } + return ResponseEntity.ok(policyStore.save(policy)); + } + + /** + * A policy that reads from or writes to a server folder grants whoever saves it access to that + * path, so restrict it to administrators on multi-user deployments. Single-user deployments + * (login disabled, e.g. desktop) trust the local operator. The {@link FolderAccessGuard} still + * enforces SaaS-off and the path allowlist during validation regardless of who saves. + */ + private void requireAuthorizedForFolderAccess(Policy policy) { + if (!folderAccessGuard.usesFolderAccess(policy)) { + return; + } + if (!applicationProperties.getSecurity().isEnableLogin()) { + return; + } + if (!userService.isCurrentUserAdmin()) { + throw new ResponseStatusException( + HttpStatus.FORBIDDEN, + "Folder sources and outputs may only be configured by an administrator"); + } } @GetMapping @@ -199,7 +231,7 @@ public class PolicyController { new ResponseStatusException( HttpStatus.NOT_FOUND, "No policy: " + policyId)); PolicyInputs inputs = collectInputs(request); - String runId = manualTrigger.run(policy, inputs, PolicyProgressListener.NOOP).runId(); + String runId = policyRunner.runWith(policy, inputs, PolicyProgressListener.NOOP).runId(); return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null)); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java index 0b613726d..acf5a3168 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java @@ -9,13 +9,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; +import stirling.software.common.model.ApplicationProperties; import stirling.software.proprietary.policy.model.PolicyRun; /** @@ -24,9 +24,9 @@ import stirling.software.proprietary.policy.model.PolicyRun; * TaskManager}. * *

Finished runs are evicted on a fixed interval once they age past {@code - * stirling.policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a - * run's rich in-memory state does not outlive the process. Only terminal runs are evicted; active - * and paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not + * policies.runExpiryMinutes}, mirroring the job-result expiry in {@code TaskManager} so a run's + * rich in-memory state does not outlive the process. Only terminal runs are evicted; active and + * paused ({@code WAITING_FOR_INPUT}) runs are retained regardless of age. Result files are not * touched here: a run shares its runId with a {@code TaskManager} job, which owns file-lifecycle * cleanup, so eviction only frees this map's entry. */ @@ -41,8 +41,8 @@ public class PolicyRunRegistry { Executors.newSingleThreadScheduledExecutor( Thread.ofVirtual().name("policy-run-cleanup-", 0).factory()); - public PolicyRunRegistry( - @Value("${stirling.policies.runExpiryMinutes:30}") int runExpiryMinutes) { + public PolicyRunRegistry(ApplicationProperties applicationProperties) { + int runExpiryMinutes = applicationProperties.getPolicies().getRunExpiryMinutes(); this.runExpiry = Duration.ofMinutes(runExpiryMinutes); cleanupExecutor.scheduleAtFixedRate(this::evictExpiredRuns, 10, 10, TimeUnit.MINUTES); log.debug( diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java new file mode 100644 index 000000000..8685bf020 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java @@ -0,0 +1,116 @@ +package stirling.software.proprietary.policy.engine; + +import java.io.IOException; +import java.util.List; +import java.util.function.Consumer; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.PipelineDefinition; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.PolicyRunStatus; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +/** + * Runs policies, and is the one place that knows how to turn a policy's configured {@link InputSpec + * sources} into actual runs. Triggers (schedule, and future webhook/folder-watch) decide + * when to run and call {@link #run(Policy)}; they never touch sources themselves. The + * controller uses the supplied-input and ad-hoc entry points for on-demand work. + * + *

This is the seam that keeps triggers and sources independent: a trigger depends on the runner, + * the runner depends on the {@link InputSource} beans, and a source depends on neither - it just + * yields {@link ResolvedInput units of work}, each carrying its own completion hook. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class PolicyRunner { + + private final PolicyEngine policyEngine; + private final List inputSources; + + /** + * Run a policy by pulling from every source it configures: each source yields zero or more + * units of work, and each unit becomes its own run so one failure does not affect the others. A + * policy with no sources runs once with no input files (a generator pipeline). Used by + * automatic triggers. + */ + public void run(Policy policy) { + List sources = policy.sources(); + if (sources.isEmpty()) { + startRun(policy, PolicyInputs.of(List.of()), unused -> {}); + return; + } + for (InputSpec spec : sources) { + pullAndRun(policy, spec); + } + } + + /** + * Run a stored policy on files supplied directly by the caller (e.g. a manual run with + * uploads), bypassing its configured sources. Returns the run handle so callers can stream + * progress. + */ + public PolicyRunHandle runWith( + Policy policy, PolicyInputs inputs, PolicyProgressListener listener) { + return policyEngine.runPolicy(policy, inputs, listener); + } + + /** Run an ad-hoc pipeline with no stored policy (AI/Automate one-offs). */ + public PolicyRunHandle runAdHoc( + PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) { + return policyEngine.submit(definition, inputs, listener); + } + + private void pullAndRun(Policy policy, InputSpec spec) { + InputSource source = sourceFor(spec); + if (source == null) { + log.warn( + "No input source for type '{}' (policy {}); skipping", + spec.type(), + policy.id()); + return; + } + List work; + try { + work = source.resolve(spec); + } catch (IOException | RuntimeException e) { + log.warn( + "Failed to resolve source '{}' for policy {}: {}", + spec.type(), + policy.id(), + e.getMessage()); + return; + } + for (ResolvedInput unit : work) { + startRun(policy, unit.inputs(), unit.onComplete()); + } + } + + private void startRun(Policy policy, PolicyInputs inputs, Consumer onComplete) { + log.info("Running policy {} ({})", policy.id(), policy.name()); + PolicyRunHandle handle = + policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP); + handle.completion() + .whenComplete((run, throwable) -> onComplete.accept(succeeded(run, throwable))); + } + + private static boolean succeeded(PolicyRun run, Throwable throwable) { + return throwable == null && run != null && run.getStatus() == PolicyRunStatus.COMPLETED; + } + + private InputSource sourceFor(InputSpec spec) { + return inputSources.stream() + .filter(source -> source.supports(spec)) + .findFirst() + .orElse(null); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyValidator.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyValidator.java new file mode 100644 index 000000000..2aa4d98be --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyValidator.java @@ -0,0 +1,74 @@ +package stirling.software.proprietary.policy.engine; + +import java.util.List; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; + +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.TriggerConfig; +import stirling.software.proprietary.policy.output.PolicyOutputSink; +import stirling.software.proprietary.policy.trigger.PolicyTrigger; + +/** + * Validates a policy's trigger, sources, and output configuration by delegating each facet to the + * bean that handles its type. Called when a policy is saved so a misconfigured schedule, missing + * folder directory, or unknown type fails fast instead of silently misbehaving at run time. + * + *

The trigger is optional (a {@code null} trigger is a manual-only policy and needs no + * validation); every configured source is validated. + */ +@Service +@RequiredArgsConstructor +public class PolicyValidator { + + private final List triggers; + private final List inputSources; + private final List outputSinks; + + /** + * @throws IllegalArgumentException if any facet's type is unknown or its configuration is + * invalid + */ + public void validate(Policy policy) { + if (policy.trigger() != null) { + triggerFor(policy.trigger()).validate(policy); + } + for (InputSpec source : policy.sources()) { + inputSourceFor(source).validate(source); + } + outputSinkFor(policy.output()).validate(policy.output()); + } + + private PolicyTrigger triggerFor(TriggerConfig config) { + return triggers.stream() + .filter(trigger -> trigger.type().equals(config.type())) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "unknown trigger type: " + config.type())); + } + + private InputSource inputSourceFor(InputSpec spec) { + return inputSources.stream() + .filter(source -> source.supports(spec)) + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "unknown input source type: " + spec.type())); + } + + private PolicyOutputSink outputSinkFor(OutputSpec spec) { + return outputSinks.stream() + .filter(sink -> sink.supports(spec)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException("unknown output type: " + spec.type())); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java new file mode 100644 index 000000000..d67820275 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/FolderInputSource.java @@ -0,0 +1,186 @@ +package stirling.software.proprietary.policy.input; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.util.FileReadinessChecker; +import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.PolicyInputs; + +/** + * Reads input files from a directory. Each ready file becomes its own unit of work (one run per + * file) so a failure on one file does not affect the others. + * + *

Two modes via the {@code mode} option: + * + *

+ * + * Readiness is checked first (via {@link FileReadinessChecker}) so files mid-write are skipped. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class FolderInputSource implements InputSource { + + private static final String TYPE = FolderAccessGuard.FOLDER_TYPE; + // Bookkeeping lives under one hidden namespace dir so the watched folder stays tidy. + private static final String WORK_SUBDIR = ".stirling"; + private static final String PROCESSING_SUBDIR = "processing"; + private static final String DONE_SUBDIR = "done"; + private static final String ERROR_SUBDIR = "error"; + + private final FileReadinessChecker readinessChecker; + private final FolderAccessGuard accessGuard; + + @Override + public String type() { + return TYPE; + } + + @Override + public boolean supports(InputSpec spec) { + return spec != null && TYPE.equals(spec.type()); + } + + @Override + public void validate(InputSpec spec) { + accessGuard.requirePermitted(FolderConfig.from(spec.options()).directory()); + } + + @Override + public List watchTargets(InputSpec spec) { + return List.of(FolderConfig.from(spec.options()).directory()); + } + + @Override + public List resolve(InputSpec spec) throws IOException { + FolderConfig config = FolderConfig.from(spec.options()); + Path inputDir = accessGuard.requirePermitted(config.directory()); + if (!Files.isDirectory(inputDir)) { + log.debug("Folder input dir does not exist: {}", inputDir); + return List.of(); + } + + List ready = new ArrayList<>(); + try (Stream entries = Files.list(inputDir)) { + entries.filter(Files::isRegularFile) + .filter(readinessChecker::isReady) + .forEach(ready::add); + } + + List work = new ArrayList<>(); + for (Path file : ready) { + if (config.snapshot()) { + work.add(ResolvedInput.of(PolicyInputs.of(List.of(fileResource(file))))); + } else { + Path claimed = claim(inputDir, file); + if (claimed == null) { + continue; // another sweep/process grabbed it + } + work.add( + new ResolvedInput( + PolicyInputs.of(List.of(fileResource(claimed))), + success -> route(inputDir, claimed, success))); + } + } + return work; + } + + private Path claim(Path inputDir, Path file) { + try { + Path processingDir = workDir(inputDir, PROCESSING_SUBDIR); + Files.createDirectories(processingDir); + Path claimed = uniqueTarget(processingDir, file.getFileName().toString()); + Files.move(file, claimed, StandardCopyOption.ATOMIC_MOVE); + return claimed; + } catch (IOException e) { + log.debug("Could not claim {}: {}", file, e.getMessage()); + return null; + } + } + + private void route(Path inputDir, Path claimed, boolean success) { + String subdir = success ? DONE_SUBDIR : ERROR_SUBDIR; + try { + Path destDir = workDir(inputDir, subdir); + Files.createDirectories(destDir); + Files.move( + claimed, + uniqueTarget(destDir, claimed.getFileName().toString()), + StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + log.warn( + "Could not move processed input {} to {}: {}", claimed, subdir, e.getMessage()); + } + } + + /** A bookkeeping subdirectory under the watched folder's {@code .stirling} namespace. */ + private static Path workDir(Path inputDir, String subdir) { + return inputDir.resolve(WORK_SUBDIR).resolve(subdir); + } + + private static Resource fileResource(Path path) { + String name = path.getFileName().toString(); + return new FileSystemResource(path.toFile()) { + @Override + public String getFilename() { + return name; + } + }; + } + + private static Path uniqueTarget(Path dir, String filename) { + Path candidate = dir.resolve(filename); + if (!Files.exists(candidate)) { + return candidate; + } + int dot = filename.lastIndexOf('.'); + String base = dot < 0 ? filename : filename.substring(0, dot); + String ext = dot < 0 ? "" : filename.substring(dot); + for (int n = 1; ; n++) { + Path next = dir.resolve(base + " (" + n + ")" + ext); + if (!Files.exists(next)) { + return next; + } + } + } + + /** The typed, validated form of a folder source's options: the directory and dedup mode. */ + record FolderConfig(Path directory, boolean snapshot) { + + private static final String DIRECTORY_OPTION = "directory"; + private static final String MODE_OPTION = "mode"; + private static final String MODE_SNAPSHOT = "snapshot"; + + static FolderConfig from(Map options) { + Object directory = options.get(DIRECTORY_OPTION); + if (directory == null || directory.toString().isBlank()) { + throw new IllegalArgumentException("folder input requires a 'directory' option"); + } + Object mode = options.get(MODE_OPTION); + boolean snapshot = mode != null && MODE_SNAPSHOT.equals(mode.toString()); + return new FolderConfig(Path.of(directory.toString()), snapshot); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java new file mode 100644 index 000000000..a6f116d02 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/InputSource.java @@ -0,0 +1,49 @@ +package stirling.software.proprietary.policy.input; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +import stirling.software.proprietary.policy.model.InputSpec; + +/** + * Resolves one of a policy's {@link InputSpec sources} into the files to run on - answering + * where a run's files come from, independent of when it runs. The counterpart of + * {@code PolicyOutputSink}: implementations are beans selected by {@link #supports(InputSpec)}, so + * a new source kind (folder, S3) is just a new bean. + * + *

Driven by the {@code PolicyRunner}, which a trigger calls when a policy is due; a source is + * passive and knows nothing about what triggered the run. A manual run may instead supply files + * directly and bypass sources entirely. + */ +public interface InputSource { + + /** Stable identifier for this source, matching {@code InputSpec.type()} (e.g. "folder"). */ + String type(); + + /** Whether this source can handle the given spec. */ + boolean supports(InputSpec spec); + + /** + * Check that an input spec is usable, throwing {@link IllegalArgumentException} if not. Called + * when a policy is saved so misconfiguration fails fast rather than at run time. + */ + default void validate(InputSpec spec) {} + + /** + * Resolve the spec into zero or more units of work, each carrying the files for one run and a + * completion hook. Returning an empty list means there is nothing to run right now. + */ + List resolve(InputSpec spec) throws IOException; + + /** + * The local filesystem directories this source draws from, if any, for triggers that want to + * react to changes there (the folder-watch trigger) rather than poll. Advisory only: it merely + * tells a trigger where to watch; resolving the spec into files is still this source's + * job via {@link #resolve}. Non-filesystem sources (S3, ...) return an empty list and so are + * simply not watchable. Default: nothing to watch. + */ + default List watchTargets(InputSpec spec) { + return List.of(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolvedInput.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolvedInput.java new file mode 100644 index 000000000..285413436 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/input/ResolvedInput.java @@ -0,0 +1,23 @@ +package stirling.software.proprietary.policy.input; + +import java.util.function.Consumer; + +import stirling.software.proprietary.policy.model.PolicyInputs; + +/** + * One unit of work produced by an {@link InputSource}: the files to run plus a completion callback + * invoked with the run's success once it finishes (e.g. a folder source routes the input to {@code + * .stirling/done} or {@code .stirling/error}). A source may return several of these (e.g. one per + * file). + */ +public record ResolvedInput(PolicyInputs inputs, Consumer onComplete) { + + public ResolvedInput { + onComplete = onComplete == null ? success -> {} : onComplete; + } + + /** A unit of work with no completion side effect. */ + public static ResolvedInput of(PolicyInputs inputs) { + return new ResolvedInput(inputs, success -> {}); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/InputSpec.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/InputSpec.java new file mode 100644 index 000000000..d6816c60f --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/InputSpec.java @@ -0,0 +1,26 @@ +package stirling.software.proprietary.policy.model; + +import java.util.Map; + +/** + * One source a policy's input files come from. {@code type} selects an {@code InputSource} + * ("folder", and in future "s3", ...); {@code options} carries source-specific configuration (a + * directory, a bucket, a dedup mode, ...). + * + *

A policy holds a list of these; a run pulls from every one. An empty list means the policy + * runs with no input files (a generator pipeline, or files supplied directly to a manual run). + * + *

Data-driven and parallel to {@link OutputSpec}/{@link TriggerConfig}: a new source kind is a + * new {@code type} handled by a new {@code InputSource} bean. + */ +public record InputSpec(String type, Map options) { + + public InputSpec { + options = options == null ? Map.of() : options; + } + + /** Read input files from a directory on disk. */ + public static InputSpec folder(String directory) { + return new InputSpec("folder", Map.of("directory", directory)); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java index 4759810bc..3caab8afc 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/OutputSpec.java @@ -18,4 +18,9 @@ public record OutputSpec(String type, Map options) { public static OutputSpec inline() { return new OutputSpec("inline", Map.of()); } + + /** Write outputs to a directory on disk. */ + public static OutputSpec folder(String directory) { + return new OutputSpec("folder", Map.of("directory", directory)); + } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java index 3a2acaf0c..14fd35b63 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Policy.java @@ -3,18 +3,16 @@ package stirling.software.proprietary.policy.model; import java.util.List; /** - * A stored, owned automation: how it is triggered, the ordered tool steps to run, and where its - * output goes, plus identity and metadata. + * A stored automation: an ordered chain of tool steps, the sources its input files come from, and + * an output destination for the results. * - *

This is the central object of the feature. Everything that runs a chain of tools is a use of a - * Policy: a watched folder is a Policy with a folder {@link TriggerConfig} and a folder {@link - * OutputSpec}; a scheduled job is a Policy with a schedule trigger; manual/Automate/AI runs execute - * a Policy (or an ad-hoc {@link PipelineDefinition}) on demand. The engine itself only ever - * executes the {@link PipelineDefinition} this exposes via {@link #toDefinition()} - it is - * trigger-agnostic. + *

Every policy can always be run on demand (manually). It may additionally carry one automatic + * {@link TriggerConfig} - usually a schedule - that fires it without a person asking; a {@code + * null} trigger means manual-only. A trigger decides when a run happens and a {@link + * InputSpec source} decides where its files come from; the two are independent, and a run + * pulls from every configured source. * - *

{@code enabled} gates automatic triggering (a disabled policy is not picked up by its - * trigger); it does not block an explicit manual run. + *

This is the feature's central configuration object - what a user defines and the engine runs. */ public record Policy( String id, @@ -22,15 +20,28 @@ public record Policy( String owner, boolean enabled, TriggerConfig trigger, + List sources, List steps, OutputSpec output) { public Policy { - trigger = trigger == null ? TriggerConfig.manual() : trigger; + sources = sources == null ? List.of() : List.copyOf(sources); steps = steps == null ? List.of() : steps; output = output == null ? OutputSpec.inline() : output; } + /** A policy with no configured sources (a generator, or files supplied directly to a run). */ + public Policy( + String id, + String name, + String owner, + boolean enabled, + TriggerConfig trigger, + List steps, + OutputSpec output) { + this(id, name, owner, enabled, trigger, List.of(), steps, output); + } + /** The engine-level, trigger-agnostic view of this policy's pipeline. */ public PipelineDefinition toDefinition() { return new PipelineDefinition(name, steps, output); diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Schedule.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Schedule.java new file mode 100644 index 000000000..148534636 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/Schedule.java @@ -0,0 +1,141 @@ +package stirling.software.proprietary.policy.model; + +import java.time.DayOfWeek; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.EnumSet; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +/** + * When a scheduled policy should fire, expressed as intent ("every day at 02:00") rather than a + * cron string. A frontend builds one of these from a friendly picker; the schedule trigger asks it + * for the next firing after a given moment. + * + *

Sealed so every cadence is an explicit, self-validating shape and adding a new one forces a + * new subtype rather than another string convention to learn. The {@code type} discriminator stays + * on the wire so the frontend can switch on it without knowing the Java hierarchy. + * + *

Wall-clock kinds ({@link Daily}, {@link Weekly}, {@link Monthly}) are evaluated in the zone of + * the {@code after} argument; {@link Every} is a fixed offset and ignores wall-clock time. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = Schedule.Every.class, name = "every"), + @JsonSubTypes.Type(value = Schedule.Daily.class, name = "daily"), + @JsonSubTypes.Type(value = Schedule.Weekly.class, name = "weekly"), + @JsonSubTypes.Type(value = Schedule.Monthly.class, name = "monthly"), +}) +@JsonIgnoreProperties(ignoreUnknown = true) +public sealed interface Schedule { + + /** The next firing strictly after {@code after}, evaluated in {@code after}'s zone. */ + ZonedDateTime nextAfter(ZonedDateTime after); + + /** The granularities a fixed-interval schedule can repeat on. */ + enum Unit { + MINUTES, + HOURS, + DAYS + } + + /** + * A fixed cadence repeating from when the policy was last seen: "every 15 minutes", "every 6 + * hours". Time of day is irrelevant. + */ + record Every(long count, Unit unit) implements Schedule { + public Every { + if (count <= 0) { + throw new IllegalArgumentException("'every' schedule needs a positive count"); + } + if (unit == null) { + throw new IllegalArgumentException("'every' schedule needs a unit"); + } + } + + @Override + public ZonedDateTime nextAfter(ZonedDateTime after) { + return switch (unit) { + case MINUTES -> after.plusMinutes(count); + case HOURS -> after.plusHours(count); + case DAYS -> after.plusDays(count); + }; + } + } + + /** Once a day at a wall-clock time: "every day at 02:00". */ + record Daily(LocalTime at) implements Schedule { + public Daily { + requireTime(at); + } + + @Override + public ZonedDateTime nextAfter(ZonedDateTime after) { + ZonedDateTime today = after.with(at); + return today.isAfter(after) ? today : today.plusDays(1); + } + } + + /** On chosen weekdays at a wall-clock time: "every Monday and Thursday at 09:00". */ + record Weekly(Set days, LocalTime at) implements Schedule { + public Weekly { + if (days == null || days.isEmpty()) { + throw new IllegalArgumentException("'weekly' schedule needs at least one day"); + } + requireTime(at); + days = EnumSet.copyOf(days); + } + + @Override + public ZonedDateTime nextAfter(ZonedDateTime after) { + // The soonest of the next 7 days that lands on a chosen weekday, at the configured + // time. + for (int i = 0; i <= 7; i++) { + ZonedDateTime candidate = after.plusDays(i).with(at); + if (candidate.isAfter(after) && days.contains(candidate.getDayOfWeek())) { + return candidate; + } + } + throw new IllegalStateException("unreachable: a chosen weekday recurs within 8 days"); + } + } + + /** + * On a day of the month at a wall-clock time: "the 1st at 00:00". Months too short for the + * chosen day (e.g. the 31st in February) are skipped, not clamped. + */ + record Monthly(int dayOfMonth, LocalTime at) implements Schedule { + public Monthly { + if (dayOfMonth < 1 || dayOfMonth > 31) { + throw new IllegalArgumentException("'monthly' day-of-month must be 1-31"); + } + requireTime(at); + } + + @Override + public ZonedDateTime nextAfter(ZonedDateTime after) { + ZonedDateTime firstOfMonth = after.withDayOfMonth(1).with(at); + // Scan forward a few years' worth of months to skip ones without the chosen day. + for (int i = 0; i < 48; i++) { + ZonedDateTime month = firstOfMonth.plusMonths(i); + if (month.toLocalDate().lengthOfMonth() >= dayOfMonth) { + ZonedDateTime fire = month.withDayOfMonth(dayOfMonth); + if (fire.isAfter(after)) { + return fire; + } + } + } + throw new IllegalStateException( + "unreachable: a month with the chosen day recurs yearly"); + } + } + + private static void requireTime(LocalTime at) { + if (at == null) { + throw new IllegalArgumentException("schedule needs a time of day ('at')"); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java index 848c48e4f..e18347dc3 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/model/TriggerConfig.java @@ -3,23 +3,21 @@ package stirling.software.proprietary.policy.model; import java.util.Map; /** - * How a {@link Policy} is automatically triggered. {@code type} selects a trigger kind ("manual", - * "folder", "schedule", "s3"); {@code options} carries type-specific configuration (a folder path, - * a cron expression, a bucket, ...). + * A {@link Policy}'s optional automatic trigger - what fires it without a person asking. {@code + * type} selects a trigger kind ("schedule", and in future "webhook", "folder-watch", ...); {@code + * options} carries type-specific configuration (a {@link Schedule}, a webhook secret, ...). + * + *

Manual running is not a trigger kind: every policy can always be run on demand, so a + * policy with no automatic trigger simply has a {@code null} {@code TriggerConfig}. A trigger + * answers only "when"; where a run's files come from is a separate concern owned by the policy's + * {@link InputSpec sources}. * *

Data-driven and parallel to {@link OutputSpec}: new trigger kinds are new {@code type} values - * handled by a new trigger bean, with no change to the model. {@code "manual"} means there is no - * automatic trigger - the policy is only ever run on demand. + * handled by a new trigger bean, with no change to the model. */ public record TriggerConfig(String type, Map options) { public TriggerConfig { - type = type == null || type.isBlank() ? "manual" : type; options = options == null ? Map.of() : options; } - - /** No automatic trigger; the policy is run on demand only. */ - public static TriggerConfig manual() { - return new TriggerConfig("manual", Map.of()); - } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java new file mode 100644 index 000000000..6ecec3c93 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/FolderOutputSink.java @@ -0,0 +1,130 @@ +package stirling.software.proprietary.policy.output; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import org.apache.commons.io.FilenameUtils; +import org.springframework.core.io.Resource; +import org.springframework.http.MediaType; +import org.springframework.http.MediaTypeFactory; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.job.ResultFile; +import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.model.OutputSpec; + +/** + * Writes a run's output files to a directory on disk. The destination is the {@code directory} + * option of the {@link OutputSpec}. + * + *

Files are streamed to disk (so large outputs are not buffered) and given unique names to avoid + * clobbering existing files. The returned {@link ResultFile}s describe what was written (path + + * size); they carry a synthetic id because the deliverable is the file on disk, not a {@code + * FileStorage} entry, so folder outputs are not downloadable via {@code /files/{id}}. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class FolderOutputSink implements PolicyOutputSink { + + static final String TYPE = FolderAccessGuard.FOLDER_TYPE; + static final String DIRECTORY_OPTION = "directory"; + + private final FolderAccessGuard accessGuard; + + @Override + public String type() { + return TYPE; + } + + @Override + public boolean supports(OutputSpec spec) { + return spec != null && TYPE.equals(spec.type()); + } + + @Override + public void validate(OutputSpec spec) { + accessGuard.requirePermitted(directoryOf(spec)); + } + + @Override + public List deliver(String runId, List outputs, OutputSpec spec) + throws IOException { + Path targetDir = accessGuard.requirePermitted(directoryOf(spec)); + Files.createDirectories(targetDir); + + List results = new ArrayList<>(); + for (int i = 0; i < outputs.size(); i++) { + Resource resource = outputs.get(i); + String name = safeName(resource.getFilename(), i); + Path target = uniqueTarget(targetDir, name); + try (InputStream is = resource.getInputStream()) { + Files.copy(is, target); + } + long size = Files.size(target); + String contentType = + MediaTypeFactory.getMediaType(name) + .orElse(MediaType.APPLICATION_OCTET_STREAM) + .toString(); + results.add( + ResultFile.builder() + .fileId(UUID.randomUUID().toString()) + .fileName(target.toString()) + .contentType(contentType) + .fileSize(size) + .build()); + log.debug("Wrote policy run {} output to {}", runId, target); + } + return results; + } + + private static Path directoryOf(OutputSpec spec) { + Object directory = spec.options().get(DIRECTORY_OPTION); + if (directory == null || directory.toString().isBlank()) { + throw new IllegalArgumentException( + "folder output requires a '" + DIRECTORY_OPTION + "' option"); + } + return Path.of(directory.toString()); + } + + /** + * The resource's filename reduced to a bare, traversal-free name: any directory component or + * "../" is stripped so a crafted output name cannot escape {@code targetDir}. Falls back to a + * synthetic name when the filename is absent or reduces to nothing usable. + */ + private static String safeName(String filename, int index) { + if (filename == null || filename.isBlank()) { + return "output-" + index; + } + String name = FilenameUtils.getName(filename); + if (name.isBlank() || ".".equals(name) || "..".equals(name)) { + return "output-" + index; + } + return name; + } + + /** Resolve a non-colliding path in {@code dir}, appending " (n)" before the extension. */ + private static Path uniqueTarget(Path dir, String filename) { + Path candidate = dir.resolve(filename); + if (!Files.exists(candidate)) { + return candidate; + } + String base = FilenameUtils.getBaseName(filename); + String ext = FilenameUtils.getExtension(filename); + String suffix = ext.isEmpty() ? "" : "." + ext; + for (int n = 1; ; n++) { + Path next = dir.resolve(base + " (" + n + ")" + suffix); + if (!Files.exists(next)) { + return next; + } + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java index f6bf478e7..c98b55ad6 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/output/PolicyOutputSink.java @@ -23,6 +23,12 @@ public interface PolicyOutputSink { /** Whether this sink can handle the given output spec. */ boolean supports(OutputSpec spec); + /** + * Check that an output spec is usable, throwing {@link IllegalArgumentException} if not. Called + * when a policy is saved so misconfiguration fails fast rather than at run time. + */ + default void validate(OutputSpec spec) {} + /** * Persist/deliver the output files and return their descriptors. * diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java index df8fc9339..2ec94d1eb 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/InProcessPolicyStore.java @@ -30,6 +30,7 @@ public class InProcessPolicyStore implements PolicyStore { policy.owner(), policy.enabled(), policy.trigger(), + policy.sources(), policy.steps(), policy.output()); policies.put(id, stored); @@ -50,6 +51,7 @@ public class InProcessPolicyStore implements PolicyStore { public List findByTriggerType(String triggerType) { return policies.values().stream() .filter(Policy::enabled) + .filter(policy -> policy.trigger() != null) .filter(policy -> triggerType.equals(policy.trigger().type())) .toList(); } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java index 977e8c9c1..e4e05ea4c 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/store/JpaPolicyStore.java @@ -37,6 +37,7 @@ public class JpaPolicyStore implements PolicyStore { policy.owner(), policy.enabled(), policy.trigger(), + policy.sources(), policy.steps(), policy.output()); @@ -45,7 +46,7 @@ public class JpaPolicyStore implements PolicyStore { entity.setName(stored.name()); entity.setOwner(stored.owner()); entity.setEnabled(stored.enabled()); - entity.setTriggerType(stored.trigger().type()); + entity.setTriggerType(stored.trigger() == null ? null : stored.trigger().type()); entity.setPolicyJson(objectMapper.writeValueAsString(stored)); repository.save(entity); return stored; diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java new file mode 100644 index 000000000..dbff801b3 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java @@ -0,0 +1,302 @@ +package stirling.software.proprietary.policy.trigger; + +import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; + +import java.io.IOException; +import java.nio.file.ClosedWatchServiceException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.store.PolicyStore; + +/** + * Fires policies the moment a file lands in one of their folder sources, instead of polling on a + * timer. The trigger only reads that location; turning it into files is still the source's job. + * + *

The watcher is a latency optimisation, not a source of truth, so this pairs an event watch + * with a low-frequency reconcile sweep ({@code watchReconcileSeconds}). The reconcile both + * (a) re-syncs which directories are watched as policies are created/edited/deleted and folders + * appear on disk, and (b) runs every folder-watch policy once, catching files that pre-dated the + * watch, events lost to inotify-queue overflow, and changes on filesystems that do not deliver + * events at all (NFS, many container bind mounts). Both paths just call {@link PolicyRunner#run}; + * the {@link InputSource} does the claiming, so a redundant run finds nothing to claim and is + * harmless. + * + *

Like {@link ScheduleTrigger}, watch state is per-node and in memory; this assumes a single + * node and rebuilds its registrations on restart from the {@link PolicyStore}. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class FolderWatchTrigger implements PolicyTrigger { + + private static final String TYPE = "folder-watch"; + + private final PolicyStore policyStore; + private final PolicyRunner policyRunner; + private final List inputSources; + private final ApplicationProperties applicationProperties; + + private final Map keysByDir = new ConcurrentHashMap<>(); + private final Map dirByKey = new ConcurrentHashMap<>(); + + private volatile boolean running; + + // Package-visible (not private) so tests can drive syncRegistrations() against a real service. + volatile WatchService watchService; + + private volatile ScheduledExecutorService reconciler; + + @Override + public String type() { + return TYPE; + } + + @Override + public void validate(Policy policy) { + if (watchDirsOf(policy).isEmpty()) { + throw new IllegalArgumentException( + "folder-watch trigger requires at least one watchable (folder) input source"); + } + } + + @Override + public synchronized void start() { + if (watchService != null) { + return; + } + try { + watchService = FileSystems.getDefault().newWatchService(); + } catch (IOException e) { + log.error("Could not start folder-watch trigger: {}", e.getMessage(), e); + return; + } + running = true; + Thread.ofVirtual().name("policy-folder-watch").start(this::watchLoop); + long reconcileSeconds = applicationProperties.getPolicies().getWatchReconcileSeconds(); + reconciler = + Executors.newSingleThreadScheduledExecutor( + Thread.ofVirtual().name("policy-folder-reconcile-", 0).factory()); + // First reconcile runs immediately so pre-existing files are picked up at startup. + reconciler.scheduleAtFixedRate(this::safeReconcile, 0, reconcileSeconds, TimeUnit.SECONDS); + log.info("Folder-watch trigger started (reconcile every {}s)", reconcileSeconds); + } + + @Override + public synchronized void stop() { + running = false; + if (reconciler != null) { + reconciler.shutdownNow(); + reconciler = null; + } + if (watchService != null) { + try { + watchService.close(); // wakes the watch loop with ClosedWatchServiceException + } catch (IOException e) { + log.debug("Error closing folder watch service: {}", e.getMessage()); + } + watchService = null; + } + keysByDir.clear(); + dirByKey.clear(); + } + + private void watchLoop() { + // Capture the service once: stop() may null the field, and a local avoids racing that to an + // NPE (close() still wakes take()/poll() on this same instance). + WatchService watcher = watchService; + if (watcher == null) { + return; + } + while (running) { + WatchKey first; + try { + first = watcher.take(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } catch (ClosedWatchServiceException e) { + return; + } + runForChangedDirs(drainBurst(watcher, first)); + } + } + + /** + * Collect the directories touched by {@code first} and any further events that arrive within + * the quiet period, so a burst of file-system events becomes a single set of affected + * directories. The event kinds are irrelevant: any event on a watched dir just means "go look". + */ + private Set drainBurst(WatchService watcher, WatchKey first) { + long quietPeriodMs = applicationProperties.getPolicies().getWatchQuietPeriodMs(); + Set changed = new HashSet<>(); + WatchKey key = first; + while (key != null) { + key.pollEvents(); + Path dir = dirByKey.get(key); + if (dir != null) { + changed.add(dir); + } + key.reset(); + try { + key = watcher.poll(quietPeriodMs, TimeUnit.MILLISECONDS); + } catch (ClosedWatchServiceException | InterruptedException e) { + break; + } + } + return changed; + } + + /** Run every folder-watch policy that draws from one of the changed directories. */ + void runForChangedDirs(Set changedDirs) { + if (changedDirs.isEmpty()) { + return; + } + for (Policy policy : policyStore.findByTriggerType(TYPE)) { + List dirs; + try { + dirs = watchDirsOf(policy); + } catch (RuntimeException e) { + log.warn( + "Folder-watch policy {} is misconfigured: {}", policy.id(), e.getMessage()); + continue; + } + if (dirs.stream().anyMatch(changedDirs::contains)) { + log.debug("Folder-watch policy {} ({}) saw activity", policy.id(), policy.name()); + policyRunner.run(policy); + } + } + } + + private void safeReconcile() { + try { + syncRegistrations(); + runAll(); + } catch (RuntimeException e) { + log.error("Folder-watch reconcile failed: {}", e.getMessage(), e); + } + } + + /** The reconcile safety net: run every folder-watch policy regardless of watch events. */ + void runAll() { + for (Policy policy : policyStore.findByTriggerType(TYPE)) { + try { + policyRunner.run(policy); + } catch (RuntimeException e) { + log.warn( + "Folder-watch reconcile run failed for policy {}: {}", + policy.id(), + e.getMessage()); + } + } + } + + /** + * Bring the set of watched directories in line with the current folder-watch policies: register + * newly required directories that exist on disk, and cancel ones no longer wanted. + */ + synchronized void syncRegistrations() { + if (watchService == null) { + return; + } + Set desired = desiredDirs(); + + keysByDir + .entrySet() + .removeIf( + entry -> { + if (desired.contains(entry.getKey())) { + return false; + } + entry.getValue().cancel(); + dirByKey.remove(entry.getValue()); + return true; + }); + + for (Path dir : desired) { + if (keysByDir.containsKey(dir)) { + continue; + } + try { + WatchKey key = dir.register(watchService, ENTRY_CREATE, ENTRY_MODIFY); + keysByDir.put(dir, key); + dirByKey.put(key, dir); + log.info("Watching {} for folder-watch policies", dir); + } catch (IOException | RuntimeException e) { + log.warn("Could not watch {}: {}", dir, e.getMessage()); + } + } + } + + /** The directories currently registered with the watch service. Visible for tests. */ + Set watchedDirs() { + return Set.copyOf(keysByDir.keySet()); + } + + /** Every existing directory any current folder-watch policy wants watched. */ + private Set desiredDirs() { + Set dirs = new HashSet<>(); + for (Policy policy : policyStore.findByTriggerType(TYPE)) { + try { + for (Path dir : watchDirsOf(policy)) { + if (Files.isDirectory(dir)) { + dirs.add(dir); + } + } + } catch (RuntimeException e) { + log.warn( + "Folder-watch policy {} is misconfigured: {}", policy.id(), e.getMessage()); + } + } + return dirs; + } + + /** + * The normalised, absolute directories this policy's sources expose to watch. Normalisation + * makes registration keys and event-time matching comparable regardless of how the path was + * configured. + */ + private List watchDirsOf(Policy policy) { + List dirs = new ArrayList<>(); + for (InputSpec spec : policy.sources()) { + InputSource source = sourceFor(spec); + if (source == null) { + continue; + } + for (Path dir : source.watchTargets(spec)) { + dirs.add(dir.toAbsolutePath().normalize()); + } + } + return dirs; + } + + private InputSource sourceFor(InputSpec spec) { + return inputSources.stream() + .filter(source -> source.supports(spec)) + .findFirst() + .orElse(null); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java deleted file mode 100644 index 2ae24671a..000000000 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ManualTrigger.java +++ /dev/null @@ -1,41 +0,0 @@ -package stirling.software.proprietary.policy.trigger; - -import org.springframework.stereotype.Service; - -import lombok.RequiredArgsConstructor; - -import stirling.software.proprietary.policy.engine.PolicyEngine; -import stirling.software.proprietary.policy.engine.PolicyRunHandle; -import stirling.software.proprietary.policy.model.PipelineDefinition; -import stirling.software.proprietary.policy.model.Policy; -import stirling.software.proprietary.policy.model.PolicyInputs; -import stirling.software.proprietary.policy.progress.PolicyProgressListener; - -/** - * Runs policies on demand, in response to a request (the {@code PolicyController} endpoints, an AI, - * or another automation). It is the request-driven trigger: no background lifecycle, it just - * forwards to the engine. Any policy can be run manually regardless of its configured trigger type. - */ -@Service -@RequiredArgsConstructor -public class ManualTrigger implements PolicyTrigger { - - private final PolicyEngine policyEngine; - - @Override - public String type() { - return "manual"; - } - - /** Run a stored policy immediately and return its run handle. */ - public PolicyRunHandle run( - Policy policy, PolicyInputs inputs, PolicyProgressListener listener) { - return policyEngine.runPolicy(policy, inputs, listener); - } - - /** Run an ad-hoc pipeline (no stored policy), e.g. for AI or Automate one-offs. */ - public PolicyRunHandle fire( - PipelineDefinition definition, PolicyInputs inputs, PolicyProgressListener listener) { - return policyEngine.submit(definition, inputs, listener); - } -} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java index c63e5a1da..1e3718d7f 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java @@ -1,22 +1,36 @@ package stirling.software.proprietary.policy.trigger; +import stirling.software.proprietary.policy.model.Policy; + /** - * Activates policies of one trigger type. A trigger owns a {@link #type()} (matching {@code - * TriggerConfig.type()}); when its condition fires it runs the relevant {@code Policy} through the - * {@code PolicyEngine}. + * An automatic trigger: the thing that decides when a policy runs without a person asking. + * A trigger owns a {@link #type()} (matching {@code TriggerConfig.type()}); when its condition + * fires it hands the policy to the {@code PolicyRunner}, which pulls the policy's sources and + * starts the runs. A trigger never resolves sources itself. * - *

Background triggers (folder watcher, schedule) are driven by configuration: on {@link - * #start()} they begin watching/scheduling for the policies returned by {@code - * PolicyStore.findByTriggerType(type())}, and stop on {@link #stop()}. Request-driven triggers - * (manual) have no background lifecycle and run a policy directly in response to a call. New - * trigger kinds are new beans of this type; the engine and the {@code Policy} model do not change. + *

Triggers are background, configuration-driven beans (schedule, and in future webhook or + * folder-watch): on {@link #start()} they begin watching/scheduling for the policies returned by + * {@code PolicyStore.findByTriggerType(type())}, and stop on {@link #stop()}. New trigger kinds are + * new beans of this type; the runner and the {@code Policy} model do not change. + * + *

Manual running is not a trigger - every policy can always be run on demand via the {@code + * PolicyRunner} regardless of whether it has a trigger. */ public interface PolicyTrigger { /** Stable identifier for this trigger kind, matching {@code TriggerConfig.type()}. */ String type(); - /** Begin activating policies of this type (e.g. start a folder watcher). No-op for manual. */ + /** + * Check that this trigger is usable for the given policy, throwing {@link + * IllegalArgumentException} if not. Called when a policy is saved so misconfiguration fails + * fast rather than at fire time. Receives the whole {@link Policy} (not just its {@code + * TriggerConfig}) so a trigger whose firing depends on the policy's sources (folder-watch) can + * assert that relationship; most triggers only inspect {@code policy.trigger()}. + */ + default void validate(Policy policy) {} + + /** Begin activating policies of this type (e.g. start the schedule sweep). */ default void start() {} /** Stop activating and release any resources. */ diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManager.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManager.java new file mode 100644 index 000000000..1e87dcd4d --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManager.java @@ -0,0 +1,56 @@ +package stirling.software.proprietary.policy.trigger; + +import java.util.List; + +import org.springframework.context.SmartLifecycle; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * Starts and stops every {@link PolicyTrigger} with the application lifecycle. Background triggers + * (schedule, and future folder/S3) begin watching on {@link #start()} and release resources on + * {@link #stop()}; request-driven triggers (manual) are no-ops. + * + *

This is the single activation point for triggers - a new background trigger only has to be a + * {@link PolicyTrigger} bean. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class PolicyTriggerManager implements SmartLifecycle { + + private final List triggers; + + private volatile boolean running; + + @Override + public void start() { + for (PolicyTrigger trigger : triggers) { + try { + trigger.start(); + } catch (RuntimeException e) { + log.error("Failed to start trigger '{}': {}", trigger.type(), e.getMessage(), e); + } + } + running = true; + } + + @Override + public void stop() { + for (PolicyTrigger trigger : triggers) { + try { + trigger.stop(); + } catch (RuntimeException e) { + log.error("Failed to stop trigger '{}': {}", trigger.type(), e.getMessage(), e); + } + } + running = false; + } + + @Override + public boolean isRunning() { + return running; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java new file mode 100644 index 000000000..f138710d7 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/ScheduleTrigger.java @@ -0,0 +1,158 @@ +package stirling.software.proprietary.policy.trigger; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.Schedule; +import stirling.software.proprietary.policy.store.PolicyStore; + +import tools.jackson.databind.ObjectMapper; + +/** + * Fires policies on a {@link Schedule}. On {@link #start()} it sweeps on a fixed interval; each + * sweep finds the enabled "schedule" policies and runs any whose next firing has come due since it + * last fired. + * + *

The trigger only decides when: once a policy is due it hands it to the {@link + * PolicyRunner}, which pulls from the policy's configured sources and starts the runs. The trigger + * knows nothing about folders, buckets, or how many runs a sweep produces. + * + *

Caveat: last-fire times are tracked in memory, so this assumes a single node and resets + * on restart; cluster-wide coordination (leader election) is a follow-up. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class ScheduleTrigger implements PolicyTrigger { + + private static final String TYPE = "schedule"; + + private final PolicyStore policyStore; + private final PolicyRunner policyRunner; + private final ObjectMapper objectMapper; + private final ApplicationProperties applicationProperties; + + private final Map lastFiredByPolicy = new ConcurrentHashMap<>(); + private volatile ScheduledExecutorService scheduler; + + @Override + public String type() { + return TYPE; + } + + @Override + public void validate(Policy policy) { + ScheduleConfig.from(objectMapper, policy.trigger().options()); + } + + @Override + public synchronized void start() { + if (scheduler != null) { + return; + } + long sweepSeconds = applicationProperties.getPolicies().getScheduleSweepSeconds(); + scheduler = + Executors.newSingleThreadScheduledExecutor( + Thread.ofVirtual().name("policy-schedule-", 0).factory()); + scheduler.scheduleAtFixedRate( + this::safeSweep, sweepSeconds, sweepSeconds, TimeUnit.SECONDS); + log.info("Schedule trigger started (sweep every {}s)", sweepSeconds); + } + + @Override + public synchronized void stop() { + if (scheduler != null) { + scheduler.shutdownNow(); + scheduler = null; + } + } + + private void safeSweep() { + try { + sweep(Instant.now()); + } catch (RuntimeException e) { + log.error("Schedule sweep failed: {}", e.getMessage(), e); + } + } + + /** Fire every scheduled policy that is due as of {@code now}. Package-visible for testing. */ + void sweep(Instant now) { + for (Policy policy : policyStore.findByTriggerType(TYPE)) { + ScheduleConfig config; + try { + config = ScheduleConfig.from(objectMapper, policy.trigger().options()); + } catch (IllegalArgumentException e) { + log.warn("Scheduled policy {} is misconfigured: {}", policy.id(), e.getMessage()); + continue; + } + + // First time we see a policy, baseline its last-fire to now so it does not fire + // immediately; subsequent sweeps fire it once its next firing has passed. + Instant last = lastFiredByPolicy.computeIfAbsent(policy.id(), id -> now); + ZonedDateTime next = config.schedule().nextAfter(last.atZone(config.zone())); + if (!next.toInstant().isAfter(now)) { + lastFiredByPolicy.put(policy.id(), now); + log.info("Scheduled policy {} ({}) is due", policy.id(), policy.name()); + policyRunner.run(policy); + } + } + } + + /** + * The typed, validated form of a schedule trigger's options: the {@link Schedule} and the zone + * its wall-clock kinds are evaluated in (default UTC). Construction fails for a missing/invalid + * schedule or zone. + */ + record ScheduleConfig(Schedule schedule, ZoneId zone) { + + private static final String SCHEDULE_OPTION = "schedule"; + private static final String ZONE_OPTION = "zone"; + + static ScheduleConfig from(ObjectMapper mapper, Map options) { + Object scheduleNode = options.get(SCHEDULE_OPTION); + if (scheduleNode == null) { + throw new IllegalArgumentException("schedule trigger requires a 'schedule'"); + } + Schedule schedule; + try { + schedule = mapper.convertValue(scheduleNode, Schedule.class); + } catch (RuntimeException e) { + throw new IllegalArgumentException("invalid schedule: " + rootMessage(e), e); + } + + ZoneId zone = ZoneOffset.UTC; + Object zoneNode = options.get(ZONE_OPTION); + if (zoneNode != null && !zoneNode.toString().isBlank()) { + try { + zone = ZoneId.of(zoneNode.toString()); + } catch (RuntimeException e) { + throw new IllegalArgumentException("invalid zone '" + zoneNode + "'"); + } + } + return new ScheduleConfig(schedule, zone); + } + + private static String rootMessage(Throwable t) { + Throwable cause = t; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + return cause.getMessage(); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/config/FolderAccessGuardTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/config/FolderAccessGuardTest.java new file mode 100644 index 000000000..ea33319a8 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/config/FolderAccessGuardTest.java @@ -0,0 +1,99 @@ +package stirling.software.proprietary.policy.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.core.env.StandardEnvironment; + +import stirling.software.common.configuration.InstallationPathConfig; +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.Policy; + +/** + * Tests for {@link FolderAccessGuard}: folder access is fail-closed, confined to the configured + * allowed roots, never reaches Stirling's own config directory, and is off entirely under SaaS. + */ +class FolderAccessGuardTest { + + @TempDir Path tempDir; + + private FolderAccessGuard guard(List allowedRoots, String... activeProfiles) { + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowedFolderRoots(allowedRoots); + StandardEnvironment environment = new StandardEnvironment(); + environment.setActiveProfiles(activeProfiles); + return new FolderAccessGuard(properties, environment); + } + + @Test + void permitsAndNormalisesADirectoryWithinAnAllowedRoot() { + FolderAccessGuard guard = guard(List.of(tempDir.toString())); + Path within = tempDir.resolve("inbox"); + + assertEquals(within.toAbsolutePath().normalize(), guard.requirePermitted(within)); + } + + @Test + void rejectsADirectoryOutsideEveryAllowedRoot() { + FolderAccessGuard guard = guard(List.of(tempDir.toString())); + assertThrows( + IllegalArgumentException.class, + () -> guard.requirePermitted(tempDir.resolveSibling("elsewhere"))); + } + + @Test + void rejectsTraversalThatWalksOutOfAnAllowedRoot() { + FolderAccessGuard guard = guard(List.of(tempDir.toString())); + assertThrows( + IllegalArgumentException.class, + () -> guard.requirePermitted(tempDir.resolve("..").resolve("escaped"))); + } + + @Test + void rejectsEverythingWhenNoRootsAreConfigured() { + FolderAccessGuard guard = guard(List.of()); + assertThrows(IllegalArgumentException.class, () -> guard.requirePermitted(tempDir)); + } + + @Test + void rejectsTheStirlingConfigDirectoryEvenWhenItWouldBeInsideAnAllowedRoot() { + Path configDir = + Path.of(InstallationPathConfig.getConfigPath()).toAbsolutePath().normalize(); + // Allow the config dir's parent, so only the protected-path rule can reject it. + FolderAccessGuard guard = guard(List.of(configDir.getParent().toString())); + + assertThrows( + IllegalArgumentException.class, + () -> guard.requirePermitted(configDir.resolve("settings.yml"))); + } + + @Test + void refusesAllFolderAccessUnderTheSaasProfile() { + FolderAccessGuard guard = guard(List.of(tempDir.toString()), "saas"); + assertThrows(IllegalArgumentException.class, () -> guard.requirePermitted(tempDir)); + } + + @Test + void usesFolderAccessDetectsFolderSourcesAndOutputs() { + FolderAccessGuard guard = guard(List.of(tempDir.toString())); + + assertTrue( + guard.usesFolderAccess( + policy(List.of(InputSpec.folder("/in")), OutputSpec.inline()))); + assertTrue(guard.usesFolderAccess(policy(List.of(), OutputSpec.folder("/out")))); + assertFalse(guard.usesFolderAccess(policy(List.of(), OutputSpec.inline()))); + } + + private static Policy policy(List sources, OutputSpec output) { + return new Policy("p1", "p", "owner", true, null, sources, List.of(), output); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java index bf6442b42..e47916dbe 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyEngineTest.java @@ -50,7 +50,6 @@ import stirling.software.proprietary.policy.model.Policy; import stirling.software.proprietary.policy.model.PolicyInputs; import stirling.software.proprietary.policy.model.PolicyRun; import stirling.software.proprietary.policy.model.PolicyRunStatus; -import stirling.software.proprietary.policy.model.TriggerConfig; import stirling.software.proprietary.policy.output.InlineOutputSink; import stirling.software.proprietary.policy.progress.PolicyProgressListener; @@ -93,7 +92,7 @@ class PolicyEngineTest { toolMetadataService, tempFileManager, JsonMapper.builder().build()); - registry = new PolicyRunRegistry(30); + registry = new PolicyRunRegistry(new ApplicationProperties()); InlineOutputSink sink = new InlineOutputSink(fileStorage); engine = new PolicyEngine( @@ -190,7 +189,7 @@ class PolicyEngineTest { "rotate", "owner", true, - TriggerConfig.manual(), + null, List.of(new PipelineStep(ROTATE, Map.of())), OutputSpec.inline()); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java index 823d53edc..51645e71b 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunRegistryTest.java @@ -12,6 +12,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import stirling.software.common.model.ApplicationProperties; import stirling.software.proprietary.policy.model.PipelineDefinition; import stirling.software.proprietary.policy.model.PolicyRun; import stirling.software.proprietary.policy.model.WaitState; @@ -25,7 +26,7 @@ class PolicyRunRegistryTest { @BeforeEach void setUp() { - registry = new PolicyRunRegistry(30); + registry = new PolicyRunRegistry(new ApplicationProperties()); } @AfterEach diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java new file mode 100644 index 000000000..2cefb1b27 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyRunnerTest.java @@ -0,0 +1,159 @@ +package stirling.software.proprietary.policy.engine; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.PolicyInputs; +import stirling.software.proprietary.policy.model.PolicyRun; +import stirling.software.proprietary.policy.model.PolicyRunStatus; +import stirling.software.proprietary.policy.progress.PolicyProgressListener; + +/** + * Tests for {@link PolicyRunner}: the one place that turns a policy's sources into runs. Verifies + * it pulls every source, runs one job per unit of work, feeds each unit's completion hook the run + * outcome, and that a generator (no sources) still runs once. + */ +@ExtendWith(MockitoExtension.class) +class PolicyRunnerTest { + + @Mock private PolicyEngine policyEngine; + @Mock private InputSource folderSource; + + private PolicyRunner runner; + + @BeforeEach + void setUp() { + runner = new PolicyRunner(policyEngine, List.of(folderSource)); + } + + @Test + void runsOnceWithNoFilesWhenThePolicyHasNoSources() { + Policy policy = policy(List.of()); + when(policyEngine.runPolicy(eq(policy), any(), any())) + .thenReturn(new PolicyRunHandle("r", new CompletableFuture<>())); + + runner.run(policy); + + ArgumentCaptor inputs = ArgumentCaptor.forClass(PolicyInputs.class); + verify(policyEngine).runPolicy(eq(policy), inputs.capture(), any()); + assertTrue(inputs.getValue().primary().isEmpty()); + } + + @Test + void pullsEverySourceAndRunsOnePerUnitOfWork() throws Exception { + InputSpec spec = InputSpec.folder("/in"); + Policy policy = policy(List.of(spec)); + when(folderSource.supports(spec)).thenReturn(true); + when(folderSource.resolve(spec)) + .thenReturn( + List.of( + ResolvedInput.of(PolicyInputs.of(List.of())), + ResolvedInput.of(PolicyInputs.of(List.of())))); + when(policyEngine.runPolicy(any(), any(), any())) + .thenReturn(new PolicyRunHandle("r", new CompletableFuture<>())); + + runner.run(policy); + + verify(policyEngine, times(2)).runPolicy(eq(policy), any(), any()); + } + + @Test + void feedsEachUnitsCompletionHookTheRunOutcome() throws Exception { + InputSpec spec = InputSpec.folder("/in"); + Policy policy = policy(List.of(spec)); + AtomicBoolean outcome = new AtomicBoolean(false); + ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set); + when(folderSource.supports(spec)).thenReturn(true); + when(folderSource.resolve(spec)).thenReturn(List.of(unit)); + CompletableFuture completion = new CompletableFuture<>(); + when(policyEngine.runPolicy(any(), any(), any())) + .thenReturn(new PolicyRunHandle("r", completion)); + + runner.run(policy); + + PolicyRun run = mock(PolicyRun.class); + when(run.getStatus()).thenReturn(PolicyRunStatus.COMPLETED); + completion.complete(run); + + assertTrue(outcome.get()); + } + + @Test + void reportsFailureToTheCompletionHookWhenTheRunDoesNotComplete() throws Exception { + InputSpec spec = InputSpec.folder("/in"); + Policy policy = policy(List.of(spec)); + AtomicBoolean outcome = new AtomicBoolean(true); + ResolvedInput unit = new ResolvedInput(PolicyInputs.of(List.of()), outcome::set); + when(folderSource.supports(spec)).thenReturn(true); + when(folderSource.resolve(spec)).thenReturn(List.of(unit)); + CompletableFuture completion = new CompletableFuture<>(); + when(policyEngine.runPolicy(any(), any(), any())) + .thenReturn(new PolicyRunHandle("r", completion)); + + runner.run(policy); + completion.completeExceptionally(new RuntimeException("boom")); + + assertFalse(outcome.get()); + } + + @Test + void skipsSourcesWithNoMatchingBean() { + InputSpec spec = new InputSpec("s3", Map.of()); + Policy policy = policy(List.of(spec)); + when(folderSource.supports(spec)).thenReturn(false); + + runner.run(policy); + + verifyNoInteractions(policyEngine); + } + + @Test + void runWithSuppliedInputsBypassesSources() { + Policy policy = policy(List.of(InputSpec.folder("/in"))); + PolicyInputs inputs = PolicyInputs.of(List.of()); + PolicyRunHandle handle = new PolicyRunHandle("r", new CompletableFuture<>()); + when(policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP)) + .thenReturn(handle); + + assertSame(handle, runner.runWith(policy, inputs, PolicyProgressListener.NOOP)); + verifyNoInteractions(folderSource); + } + + private static Policy policy(List sources) { + return new Policy( + "p1", + "p", + "owner", + true, + null, + sources, + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyValidatorTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyValidatorTest.java new file mode 100644 index 000000000..edaa58651 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/engine/PolicyValidatorTest.java @@ -0,0 +1,114 @@ +package stirling.software.proprietary.policy.engine; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.TriggerConfig; +import stirling.software.proprietary.policy.output.PolicyOutputSink; +import stirling.software.proprietary.policy.trigger.PolicyTrigger; + +/** Tests for {@link PolicyValidator}: routes each facet to its handler and surfaces failures. */ +@ExtendWith(MockitoExtension.class) +class PolicyValidatorTest { + + @Mock private PolicyTrigger trigger; + @Mock private InputSource inputSource; + @Mock private PolicyOutputSink outputSink; + + private PolicyValidator validator; + + @BeforeEach + void setUp() { + validator = + new PolicyValidator(List.of(trigger), List.of(inputSource), List.of(outputSink)); + } + + @Test + void delegatesEachFacetToItsHandler() { + when(trigger.type()).thenReturn("schedule"); + when(inputSource.supports(any())).thenReturn(true); + when(outputSink.supports(any())).thenReturn(true); + Policy policy = policy("schedule"); + + validator.validate(policy); + + verify(trigger).validate(policy); + verify(inputSource).validate(policy.sources().get(0)); + verify(outputSink).validate(policy.output()); + } + + @Test + void skipsTriggerValidationForAManualOnlyPolicy() { + when(inputSource.supports(any())).thenReturn(true); + when(outputSink.supports(any())).thenReturn(true); + + validator.validate(manualOnly()); + + verify(trigger, never()).validate(any()); + } + + @Test + void surfacesAnInvalidConfigFromAHandler() { + when(trigger.type()).thenReturn("schedule"); + doThrow(new IllegalArgumentException("invalid schedule")).when(trigger).validate(any()); + + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> validator.validate(policy("schedule"))); + assertTrue(ex.getMessage().contains("schedule")); + } + + @Test + void rejectsAnUnknownTriggerType() { + when(trigger.type()).thenReturn("schedule"); + + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> validator.validate(policy("mystery"))); + assertTrue(ex.getMessage().contains("unknown trigger type")); + } + + private static Policy policy(String triggerType) { + return new Policy( + "p1", + "p", + "owner", + true, + new TriggerConfig(triggerType, Map.of()), + List.of(InputSpec.folder("/in")), + List.of(), + OutputSpec.inline()); + } + + private static Policy manualOnly() { + return new Policy( + "p1", + "p", + "owner", + true, + null, + List.of(InputSpec.folder("/in")), + List.of(), + OutputSpec.inline()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java new file mode 100644 index 000000000..3e9d34ccf --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/input/FolderInputSourceTest.java @@ -0,0 +1,137 @@ +package stirling.software.proprietary.policy.input; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.core.env.StandardEnvironment; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.util.FileReadinessChecker; +import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.model.InputSpec; + +/** Tests for {@link FolderInputSource}: consume (claim + route) and snapshot (read-only) modes. */ +@ExtendWith(MockitoExtension.class) +class FolderInputSourceTest { + + @Mock private FileReadinessChecker readinessChecker; + + @TempDir Path tempDir; + + private FolderInputSource source; + + @BeforeEach + void setUp() { + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString())); + FolderAccessGuard guard = new FolderAccessGuard(properties, new StandardEnvironment()); + source = new FolderInputSource(readinessChecker, guard); + // Lenient: the missing-dir / nonexistent-dir cases return before any readiness check. + lenient().when(readinessChecker.isReady(any())).thenReturn(true); + } + + @Test + void consumeClaimsFilesAndRoutesToDoneOnSuccess() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Files.writeString(inputDir.resolve("doc.pdf"), "data"); + + List work = source.resolve(InputSpec.folder(inputDir.toString())); + + assertEquals(1, work.size()); + assertEquals(1, work.get(0).inputs().primary().size()); + // Claimed out of the input dir. + assertFalse(Files.exists(inputDir.resolve("doc.pdf"))); + assertTrue( + Files.exists( + inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf"))); + + work.get(0).onComplete().accept(true); + assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("done").resolve("doc.pdf"))); + assertFalse( + Files.exists( + inputDir.resolve(".stirling").resolve("processing").resolve("doc.pdf"))); + } + + @Test + void consumeRoutesToErrorOnFailure() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Files.writeString(inputDir.resolve("doc.pdf"), "data"); + + List work = source.resolve(InputSpec.folder(inputDir.toString())); + work.get(0).onComplete().accept(false); + + assertTrue(Files.exists(inputDir.resolve(".stirling").resolve("error").resolve("doc.pdf"))); + } + + @Test + void snapshotReadsWithoutClaiming() throws IOException { + Path inputDir = Files.createDirectories(tempDir.resolve("in")); + Files.writeString(inputDir.resolve("doc.pdf"), "data"); + + List work = + source.resolve( + new InputSpec( + "folder", + Map.of("directory", inputDir.toString(), "mode", "snapshot"))); + + assertEquals(1, work.size()); + // Not moved, and completing the run is a no-op. + assertTrue(Files.exists(inputDir.resolve("doc.pdf"))); + work.get(0).onComplete().accept(true); + assertTrue(Files.exists(inputDir.resolve("doc.pdf"))); + } + + @Test + void missingDirectoryOptionFails() { + assertThrows( + IllegalArgumentException.class, + () -> source.resolve(new InputSpec("folder", Map.of()))); + } + + @Test + void nonexistentDirectoryYieldsNoWork() throws IOException { + List work = + source.resolve(InputSpec.folder(tempDir.resolve("nope").toString())); + assertTrue(work.isEmpty()); + } + + @Test + void validateRejectsMissingDirectory() { + assertThrows( + IllegalArgumentException.class, + () -> source.validate(new InputSpec("folder", Map.of()))); + } + + @Test + void rejectsADirectoryOutsideTheAllowedRoots() { + Path outside = tempDir.resolveSibling("not-allowed"); + assertThrows( + IllegalArgumentException.class, + () -> source.resolve(InputSpec.folder(outside.toString()))); + assertThrows( + IllegalArgumentException.class, + () -> source.validate(InputSpec.folder(outside.toString()))); + } + + @Test + void watchTargetsIsTheConfiguredDirectory() { + Path inputDir = tempDir.resolve("in"); + assertEquals(List.of(inputDir), source.watchTargets(InputSpec.folder(inputDir.toString()))); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java new file mode 100644 index 000000000..0d714162d --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/output/FolderOutputSinkTest.java @@ -0,0 +1,105 @@ +package stirling.software.proprietary.policy.output; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.model.job.ResultFile; +import stirling.software.proprietary.policy.config.FolderAccessGuard; +import stirling.software.proprietary.policy.model.OutputSpec; + +/** Tests for {@link FolderOutputSink}: outputs are written to the configured directory on disk. */ +class FolderOutputSinkTest { + + @TempDir Path tempDir; + + private FolderOutputSink sink; + + @BeforeEach + void setUp() { + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowedFolderRoots(List.of(tempDir.toString())); + sink = new FolderOutputSink(new FolderAccessGuard(properties, new StandardEnvironment())); + } + + @Test + void writesEachOutputToTheDirectory() throws IOException { + Path out = tempDir.resolve("out"); + List outputs = List.of(named("a.pdf", "aaa"), named("b.pdf", "bb")); + + List results = + sink.deliver("run-1", outputs, OutputSpec.folder(out.toString())); + + assertEquals(2, results.size()); + assertTrue(Files.exists(out.resolve("a.pdf"))); + assertEquals("aaa", Files.readString(out.resolve("a.pdf"))); + assertEquals("bb", Files.readString(out.resolve("b.pdf"))); + } + + @Test + void collidingNamesGetAUniqueSuffix() throws IOException { + Path out = tempDir.resolve("out"); + List outputs = List.of(named("a.pdf", "first"), named("a.pdf", "second")); + + sink.deliver("run-1", outputs, OutputSpec.folder(out.toString())); + + assertTrue(Files.exists(out.resolve("a.pdf"))); + assertTrue(Files.exists(out.resolve("a (1).pdf"))); + } + + @Test + void missingDirectoryOptionIsRejected() { + OutputSpec noDir = new OutputSpec("folder", Map.of()); + assertThrows(IllegalArgumentException.class, () -> sink.validate(noDir)); + assertThrows( + IllegalArgumentException.class, + () -> sink.deliver("run-1", List.of(named("a.pdf", "x")), noDir)); + } + + @Test + void aDirectoryOutsideTheAllowedRootsIsRejected() { + OutputSpec outside = OutputSpec.folder(tempDir.resolveSibling("not-allowed").toString()); + assertThrows(IllegalArgumentException.class, () -> sink.validate(outside)); + assertThrows( + IllegalArgumentException.class, + () -> sink.deliver("run-1", List.of(named("a.pdf", "x")), outside)); + } + + @Test + void filenamesWithPathTraversalAreConfinedToTheDirectory() throws IOException { + Path out = tempDir.resolve("out"); + List outputs = + List.of(named("../escape.pdf", "x"), named("nested/deep.pdf", "y")); + + sink.deliver("run-1", outputs, OutputSpec.folder(out.toString())); + + // Each name is reduced to its bare form inside the target dir; nothing escapes. + assertTrue(Files.exists(out.resolve("escape.pdf"))); + assertTrue(Files.exists(out.resolve("deep.pdf"))); + assertFalse(Files.exists(tempDir.resolve("escape.pdf"))); + } + + private static ByteArrayResource named(String filename, String content) { + return new ByteArrayResource(content.getBytes()) { + @Override + public String getFilename() { + return filename; + } + }; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java index 2e9701918..60f14be20 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/InProcessPolicyStoreTest.java @@ -28,7 +28,7 @@ class InProcessPolicyStoreTest { @Test void savedPolicyGetsAnIdAndIsRetrievable() { - Policy saved = store.save(policy(null, "compress", "manual", true)); + Policy saved = store.save(policy(null, "compress", null, true)); assertNotNull(saved.id()); assertFalse(saved.id().isBlank()); @@ -37,7 +37,7 @@ class InProcessPolicyStoreTest { @Test void savingWithAnExistingIdUpdatesInPlace() { - Policy created = store.save(policy(null, "before", "manual", true)); + Policy created = store.save(policy(null, "before", null, true)); store.save( new Policy( @@ -45,7 +45,7 @@ class InProcessPolicyStoreTest { "after", "owner", true, - TriggerConfig.manual(), + null, List.of(), OutputSpec.inline())); @@ -55,19 +55,20 @@ class InProcessPolicyStoreTest { @Test void findByTriggerTypeReturnsOnlyEnabledMatches() { - store.save(policy(null, "watch", "folder", true)); - store.save(policy(null, "watch-disabled", "folder", false)); store.save(policy(null, "nightly", "schedule", true)); + store.save(policy(null, "nightly-disabled", "schedule", false)); + store.save(policy(null, "hooked", "webhook", true)); + store.save(policy(null, "on-demand", null, true)); // manual-only: no trigger - List folder = store.findByTriggerType("folder"); + List scheduled = store.findByTriggerType("schedule"); - assertEquals(1, folder.size()); - assertEquals("watch", folder.get(0).name()); + assertEquals(1, scheduled.size()); + assertEquals("nightly", scheduled.get(0).name()); } @Test void deleteRemovesThePolicy() { - Policy saved = store.save(policy(null, "p", "manual", true)); + Policy saved = store.save(policy(null, "p", null, true)); assertTrue(store.delete(saved.id())); assertTrue(store.get(saved.id()).isEmpty()); @@ -75,12 +76,14 @@ class InProcessPolicyStoreTest { } private static Policy policy(String id, String name, String triggerType, boolean enabled) { + TriggerConfig trigger = + triggerType == null ? null : new TriggerConfig(triggerType, Map.of()); return new Policy( id, name, "owner", enabled, - new TriggerConfig(triggerType, Map.of()), + trigger, List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), OutputSpec.inline()); } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java index 2a8212e2d..2bdbc08a7 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/store/JpaPolicyStoreTest.java @@ -18,6 +18,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import stirling.software.proprietary.policy.model.InputSpec; import stirling.software.proprietary.policy.model.OutputSpec; import stirling.software.proprietary.policy.model.PipelineStep; import stirling.software.proprietary.policy.model.Policy; @@ -53,7 +54,8 @@ class JpaPolicyStoreTest { "compress incoming", "alice", true, - new TriggerConfig("folder", Map.of("path", "/in")), + new TriggerConfig("schedule", Map.of()), + List.of(InputSpec.folder("/in")), List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), OutputSpec.inline())); @@ -62,7 +64,7 @@ class JpaPolicyStoreTest { verify(repository).save(captor.capture()); PolicyEntity entity = captor.getValue(); assertEquals(saved.id(), entity.getId()); - assertEquals("folder", entity.getTriggerType()); + assertEquals("schedule", entity.getTriggerType()); assertTrue(entity.isEnabled()); // The stored JSON round-trips back to an equal policy. assertEquals(saved, objectMapper.readValue(entity.getPolicyJson(), Policy.class)); @@ -76,7 +78,7 @@ class JpaPolicyStoreTest { "rotate", "alice", true, - TriggerConfig.manual(), + null, // manual-only: no automatic trigger List.of( new PipelineStep( "/api/v1/general/rotate-pdf", Map.of("angle", 90))), @@ -94,16 +96,16 @@ class JpaPolicyStoreTest { "watch", "alice", true, - new TriggerConfig("folder", Map.of()), + new TriggerConfig("schedule", Map.of()), List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), OutputSpec.inline()); - when(repository.findByTriggerTypeAndEnabledTrue("folder")) + when(repository.findByTriggerTypeAndEnabledTrue("schedule")) .thenReturn(List.of(entityFor(policy))); - List folder = store.findByTriggerType("folder"); + List scheduled = store.findByTriggerType("schedule"); - assertEquals(1, folder.size()); - assertEquals("p1", folder.get(0).id()); + assertEquals(1, scheduled.size()); + assertEquals("p1", scheduled.get(0).id()); } @Test @@ -122,7 +124,7 @@ class JpaPolicyStoreTest { entity.setName(policy.name()); entity.setOwner(policy.owner()); entity.setEnabled(policy.enabled()); - entity.setTriggerType(policy.trigger().type()); + entity.setTriggerType(policy.trigger() == null ? null : policy.trigger().type()); entity.setPolicyJson(objectMapper.writeValueAsString(policy)); return entity; } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java new file mode 100644 index 000000000..9b0b9f1d2 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/FolderWatchTriggerTest.java @@ -0,0 +1,178 @@ +package stirling.software.proprietary.policy.trigger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.WatchService; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.TriggerConfig; +import stirling.software.proprietary.policy.store.PolicyStore; + +/** + * Tests for {@link FolderWatchTrigger}'s dispatch logic via the package-visible {@code + * runForChangedDirs}/{@code runAll}, plus its cross-facet validation. The OS watch loop and + * scheduled reconcile are thin glue around these and are not exercised here (a real {@code + * WatchService} is timing-dependent), mirroring how {@link ScheduleTriggerTest} drives {@code + * sweep} directly. The folder source is stubbed to mirror {@code FolderInputSource.watchTargets}. + */ +@ExtendWith(MockitoExtension.class) +class FolderWatchTriggerTest { + + @Mock private PolicyStore policyStore; + @Mock private PolicyRunner policyRunner; + @Mock private InputSource folderSource; + + @TempDir Path tempDir; + + private FolderWatchTrigger trigger; + + @BeforeEach + void setUp() { + trigger = + new FolderWatchTrigger( + policyStore, + policyRunner, + List.of(folderSource), + new ApplicationProperties()); + lenient().when(folderSource.supports(any())).thenReturn(true); + lenient() + .when(folderSource.watchTargets(any())) + .thenAnswer( + invocation -> { + InputSpec spec = invocation.getArgument(0); + Object dir = spec.options().get("directory"); + if (dir == null) { + throw new IllegalArgumentException( + "folder input requires a 'directory' option"); + } + return List.of(Path.of(dir.toString())); + }); + } + + @Test + void validateRejectsPolicyWithNoWatchableSource() { + assertThrows( + IllegalArgumentException.class, + () -> trigger.validate(folderWatch("p1", List.of()))); + } + + @Test + void validateAcceptsPolicyWithAFolderSource() { + trigger.validate(folderWatch("p1", List.of(InputSpec.folder("/in")))); + } + + @Test + void runsOnlyPoliciesDrawingFromTheChangedDirectory() { + Policy a = folderWatch("a", List.of(InputSpec.folder("/in/a"))); + Policy b = folderWatch("b", List.of(InputSpec.folder("/in/b"))); + when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b)); + + trigger.runForChangedDirs(Set.of(normalized("/in/a"))); + + verify(policyRunner).run(a); + verify(policyRunner, never()).run(b); + } + + @Test + void skipsAMisconfiguredPolicyButStillRunsTheOthers() { + Policy bad = folderWatch("bad", List.of(new InputSpec("folder", Map.of()))); + Policy good = folderWatch("good", List.of(InputSpec.folder("/in/a"))); + when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(bad, good)); + + trigger.runForChangedDirs(Set.of(normalized("/in/a"))); + + verify(policyRunner).run(good); + verify(policyRunner, never()).run(bad); + } + + @Test + void anEmptyChangeSetDoesNothing() { + trigger.runForChangedDirs(Set.of()); + + verifyNoInteractions(policyStore, policyRunner); + } + + @Test + void reconcileRunsEveryFolderWatchPolicyAsASafetyNet() { + Policy a = folderWatch("a", List.of(InputSpec.folder("/in/a"))); + Policy b = folderWatch("b", List.of(InputSpec.folder("/in/b"))); + when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b)); + + trigger.runAll(); + + verify(policyRunner).run(a); + verify(policyRunner).run(b); + } + + @Test + void syncRegistrationsWatchesExistingDirsAndCancelsRemovedOnes() throws Exception { + Path dirA = Files.createDirectories(tempDir.resolve("a")); + Path dirB = Files.createDirectories(tempDir.resolve("b")); + Path missing = tempDir.resolve("missing"); // never created on disk + + Policy a = folderWatch("a", List.of(InputSpec.folder(dirA.toString()))); + Policy b = folderWatch("b", List.of(InputSpec.folder(dirB.toString()))); + Policy m = folderWatch("m", List.of(InputSpec.folder(missing.toString()))); + + WatchService service = FileSystems.getDefault().newWatchService(); + try { + trigger.watchService = service; + + when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a, b, m)); + trigger.syncRegistrations(); + // Existing dirs are watched; the non-existent one is skipped. + assertEquals( + Set.of(normalized(dirA.toString()), normalized(dirB.toString())), + trigger.watchedDirs()); + + // b's policy is removed: its registration is cancelled, a remains. + when(policyStore.findByTriggerType("folder-watch")).thenReturn(List.of(a)); + trigger.syncRegistrations(); + assertEquals(Set.of(normalized(dirA.toString())), trigger.watchedDirs()); + } finally { + service.close(); + } + } + + private static Path normalized(String dir) { + return Path.of(dir).toAbsolutePath().normalize(); + } + + private static Policy folderWatch(String id, List sources) { + return new Policy( + id, + "watcher", + "owner", + true, + new TriggerConfig("folder-watch", Map.of()), + sources, + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManagerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManagerTest.java new file mode 100644 index 000000000..41504424a --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/PolicyTriggerManagerTest.java @@ -0,0 +1,51 @@ +package stirling.software.proprietary.policy.trigger; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Tests for {@link PolicyTriggerManager}: starts/stops every trigger, tolerating individual + * failures. + */ +@ExtendWith(MockitoExtension.class) +class PolicyTriggerManagerTest { + + @Mock private PolicyTrigger triggerA; + @Mock private PolicyTrigger triggerB; + + @Test + void startsAndStopsAllTriggers() { + PolicyTriggerManager manager = new PolicyTriggerManager(List.of(triggerA, triggerB)); + assertFalse(manager.isRunning()); + + manager.start(); + verify(triggerA).start(); + verify(triggerB).start(); + assertTrue(manager.isRunning()); + + manager.stop(); + verify(triggerA).stop(); + verify(triggerB).stop(); + assertFalse(manager.isRunning()); + } + + @Test + void oneTriggerFailingToStartDoesNotBlockTheOthers() { + doThrow(new RuntimeException("boom")).when(triggerA).start(); + PolicyTriggerManager manager = new PolicyTriggerManager(List.of(triggerA, triggerB)); + + manager.start(); + + verify(triggerB).start(); + assertTrue(manager.isRunning()); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java new file mode 100644 index 000000000..889c50ee4 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/trigger/ScheduleTriggerTest.java @@ -0,0 +1,147 @@ +package stirling.software.proprietary.policy.trigger; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.DayOfWeek; +import java.time.Instant; +import java.time.LocalTime; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.policy.engine.PolicyRunner; +import stirling.software.proprietary.policy.model.OutputSpec; +import stirling.software.proprietary.policy.model.PipelineStep; +import stirling.software.proprietary.policy.model.Policy; +import stirling.software.proprietary.policy.model.Schedule; +import stirling.software.proprietary.policy.model.TriggerConfig; +import stirling.software.proprietary.policy.store.PolicyStore; + +import tools.jackson.databind.json.JsonMapper; + +/** + * Tests for {@link ScheduleTrigger}'s due-firing logic via the package-visible {@code + * sweep(Instant)}. The trigger only decides when a policy is due; pulling sources and starting runs + * is the {@link PolicyRunner}'s job, so these assert it delegates to the runner. Schedules default + * to UTC, so explicit UTC instants make these deterministic. + */ +@ExtendWith(MockitoExtension.class) +class ScheduleTriggerTest { + + @Mock private PolicyStore policyStore; + @Mock private PolicyRunner policyRunner; + + private ScheduleTrigger trigger; + + @BeforeEach + void setUp() { + trigger = + new ScheduleTrigger( + policyStore, + policyRunner, + JsonMapper.builder().build(), + new ApplicationProperties()); + } + + @Test + void firesOncePerScheduleWhenItComesDue() { + Policy policy = scheduled("p1", new Schedule.Every(1, Schedule.Unit.MINUTES)); + when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy)); + + Instant t0 = Instant.parse("2026-06-05T10:00:30Z"); + trigger.sweep(t0); // first sight: baseline, must not fire immediately + verify(policyRunner, never()).run(any()); + + trigger.sweep(t0.plusSeconds(120)); // the one-minute mark has passed + verify(policyRunner, times(1)).run(eq(policy)); + } + + @Test + void doesNotFireBeforeTheNextScheduledTime() { + Policy policy = scheduled("p1", new Schedule.Daily(LocalTime.of(3, 0))); // 03:00 UTC daily + when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy)); + + Instant t0 = Instant.parse("2026-06-05T10:00:00Z"); + trigger.sweep(t0); + trigger.sweep(t0.plusSeconds(60)); // next 03:00 is far away + + verify(policyRunner, never()).run(any()); + } + + @Test + void firesWeeklyOnAChosenDay() { + // 2026-06-05 is a Friday; the next Monday 09:00 is the soonest firing. + Policy policy = + scheduled("p1", new Schedule.Weekly(Set.of(DayOfWeek.MONDAY), LocalTime.of(9, 0))); + when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy)); + + Instant friday = Instant.parse("2026-06-05T10:00:00Z"); + trigger.sweep(friday); // baseline + trigger.sweep(Instant.parse("2026-06-08T09:00:00Z")); // Monday 09:00 + + verify(policyRunner, times(1)).run(eq(policy)); + } + + @Test + void skipsPoliciesWithAnInvalidSchedule() { + Policy policy = scheduledWithRawOptions("p1", Map.of()); // no schedule + when(policyStore.findByTriggerType("schedule")).thenReturn(List.of(policy)); + + trigger.sweep(Instant.parse("2026-06-05T10:00:00Z")); + + verify(policyRunner, never()).run(any()); + } + + @Test + void validateRejectsMissingSchedule() { + assertThrows( + IllegalArgumentException.class, + () -> trigger.validate(scheduledWithRawOptions("p1", Map.of()))); + } + + @Test + void validateRejectsAnInvalidSchedule() { + Map options = + Map.of("schedule", Map.of("type", "every", "count", -5, "unit", "MINUTES")); + assertThrows( + IllegalArgumentException.class, + () -> trigger.validate(scheduledWithRawOptions("p1", options))); + } + + @Test + void validateAcceptsAValidScheduleAndZone() { + Map options = new LinkedHashMap<>(); + options.put("schedule", new Schedule.Daily(LocalTime.of(2, 0))); + options.put("zone", "Europe/London"); + trigger.validate(scheduledWithRawOptions("p1", options)); + } + + private static Policy scheduled(String id, Schedule schedule) { + return scheduledWithRawOptions(id, Map.of("schedule", schedule)); + } + + private static Policy scheduledWithRawOptions(String id, Map options) { + return new Policy( + id, + "nightly", + "owner", + true, + new TriggerConfig("schedule", options), + List.of(new PipelineStep("/api/v1/misc/compress-pdf", Map.of())), + OutputSpec.inline()); + } +} From f15e405759ceda24773d53f1c9862d2ee22a3c88 Mon Sep 17 00:00:00 2001 From: EthanHealy01 <80844253+EthanHealy01@users.noreply.github.com> Date: Wed, 10 Jun 2026 13:49:27 +0100 Subject: [PATCH 2/5] changes to the login and signup, similar to in the saas repo (#6577) Co-authored-by: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> --- .../public/locales/en-GB/translation.toml | 5 +- .../modern-logo/LoginDarkModeHeader.svg | 5 + .../modern-logo/LoginLightModeHeader.svg | 6 + .../tests/live/authentication-login.spec.ts | 7 +- .../editor/src/proprietary/routes/Login.tsx | 34 +- .../editor/src/proprietary/routes/Signup.tsx | 19 +- .../proprietary/routes/authShared/auth.css | 87 +++++ frontend/editor/src/saas/routes/Login.tsx | 300 ++++++++++++------ frontend/editor/src/saas/routes/Signup.tsx | 143 +++++---- .../src/saas/routes/authShared/saas-auth.css | 98 +++++- .../src/saas/routes/login/OAuthButtons.tsx | 20 +- .../editor/src/saas/styles/saas-theme.css | 4 +- 12 files changed, 534 insertions(+), 194 deletions(-) create mode 100644 frontend/editor/public/modern-logo/LoginDarkModeHeader.svg create mode 100644 frontend/editor/public/modern-logo/LoginLightModeHeader.svg diff --git a/frontend/editor/public/locales/en-GB/translation.toml b/frontend/editor/public/locales/en-GB/translation.toml index 39db57af9..71c77f866 100644 --- a/frontend/editor/public/locales/en-GB/translation.toml +++ b/frontend/editor/public/locales/en-GB/translation.toml @@ -4471,7 +4471,7 @@ alreadyHaveAccount = "Already have an account?" choosePassword = "Choose a password" confirmPassword = "Confirm password" confirmPasswordPlaceholder = "Re-enter your password" -createAccount = "Create Account" +createAccount = "Create account" creating = "Creating Account..." email = "Email address" emailPlaceholder = "Enter your email address" @@ -4554,6 +4554,7 @@ sending = "Sending…" sendMagicLink = "Send Magic Link" sendResetLink = "Send reset link" sessionExpired = "Your session has expired. Please sign in again." +createAccount = "Create an account" signin = "Sign in" signInAnonymously = "Sign Up as a Guest" signingIn = "Signing in..." @@ -6885,6 +6886,8 @@ passwordsDoNotMatch = "Passwords do not match" passwordTooShort = "Password must be at least 6 characters long" pleaseFillAllFields = "Please fill in all fields" signUp = "Sign Up" +signUpWith = "Sign up with" +skip = "Skip" subtitle = "Join Stirling PDF to get started" title = "Create an account" unexpectedError = "Unexpected error: {{message}}" diff --git a/frontend/editor/public/modern-logo/LoginDarkModeHeader.svg b/frontend/editor/public/modern-logo/LoginDarkModeHeader.svg new file mode 100644 index 000000000..59d84449f --- /dev/null +++ b/frontend/editor/public/modern-logo/LoginDarkModeHeader.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/frontend/editor/public/modern-logo/LoginLightModeHeader.svg b/frontend/editor/public/modern-logo/LoginLightModeHeader.svg new file mode 100644 index 000000000..6b7c6fa8c --- /dev/null +++ b/frontend/editor/public/modern-logo/LoginLightModeHeader.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/frontend/editor/src/core/tests/live/authentication-login.spec.ts b/frontend/editor/src/core/tests/live/authentication-login.spec.ts index 310dd5e6f..a8a6a75cf 100644 --- a/frontend/editor/src/core/tests/live/authentication-login.spec.ts +++ b/frontend/editor/src/core/tests/live/authentication-login.spec.ts @@ -19,12 +19,7 @@ test.describe("1. Authentication and Login", () => { .first(), ).toBeVisible(); - // Step 3: Confirm the heading for "Sign In" / "Login" is visible - await expect( - page.getByRole("heading", { name: /sign in|login|masuk/i }), - ).toBeVisible(); - - // Step 4: Confirm a "Username" text input field is present and empty + // Step 3: Confirm a "Username" text input field is present and empty const usernameInput = page.locator("#email"); await expect(usernameInput).toBeVisible(); await expect(usernameInput).toHaveValue(""); diff --git a/frontend/editor/src/proprietary/routes/Login.tsx b/frontend/editor/src/proprietary/routes/Login.tsx index 85c42fce8..c334bef8d 100644 --- a/frontend/editor/src/proprietary/routes/Login.tsx +++ b/frontend/editor/src/proprietary/routes/Login.tsx @@ -17,12 +17,11 @@ import { useDocumentMeta } from "@app/hooks/useDocumentMeta"; import AuthLayout from "@app/routes/authShared/AuthLayout"; import { useBackendProbe } from "@app/hooks/useBackendProbe"; import apiClient from "@app/services/apiClient"; -import { BASE_PATH } from "@app/constants/app"; +import { BASE_PATH, withBasePath } from "@app/constants/app"; import { type OAuthProvider } from "@app/auth/oauthTypes"; import { updateSupportedLanguages } from "@app/i18n"; // Import login components -import LoginHeader from "@app/routes/login/LoginHeader"; import ErrorMessage from "@app/routes/login/ErrorMessage"; import EmailPasswordForm from "@app/routes/login/EmailPasswordForm"; import OAuthButtons, { @@ -74,7 +73,6 @@ export default function Login() { const autoLoginErrorRecorded = useRef(false); const isUserPassAllowed = loginMethod === "all" || loginMethod === "normal"; const isSsoOnlyMode = loginMethod !== "all" && loginMethod !== "normal"; - const isSingleSsoOnly = !isUserPassAllowed && enabledProviders.length === 1; const AUTO_LOGIN_ATTEMPTS_KEY = "stirling_sso_auto_login_attempts"; const AUTO_LOGIN_ERRORS_KEY = "stirling_sso_auto_login_errors"; @@ -461,7 +459,6 @@ export default function Login() { // If backend isn't ready yet, show a lightweight status screen instead of the form if (backendProbe.status !== "up" && !loginDisabled) { - const backendTitle = t("backendStartup.notFoundTitle", "Backend not found"); const handleRetry = async () => { const result = await backendProbe.probe(); if (result.status === "up") { @@ -471,7 +468,18 @@ export default function Login() { }; return ( - +

+ Stirling PDF + Stirling PDF +
- +
+ Stirling PDF + Stirling PDF +
{/* Success message */} {successMessage && ( diff --git a/frontend/editor/src/proprietary/routes/Signup.tsx b/frontend/editor/src/proprietary/routes/Signup.tsx index 28676a085..9773106c4 100644 --- a/frontend/editor/src/proprietary/routes/Signup.tsx +++ b/frontend/editor/src/proprietary/routes/Signup.tsx @@ -5,10 +5,9 @@ import { useDocumentMeta } from "@app/hooks/useDocumentMeta"; import { useAuth } from "@app/auth/UseSession"; import AuthLayout from "@app/routes/authShared/AuthLayout"; import "@app/routes/authShared/auth.css"; -import { BASE_PATH } from "@app/constants/app"; +import { BASE_PATH, withBasePath } from "@app/constants/app"; // Import signup components -import LoginHeader from "@app/routes/login/LoginHeader"; import ErrorMessage from "@app/routes/login/ErrorMessage"; import DividerWithText from "@app/components/shared/DividerWithText"; import SignupForm from "@app/routes/signup/SignupForm"; @@ -92,10 +91,18 @@ export default function Signup() { return ( - +
+ Stirling PDF + Stirling PDF +
diff --git a/frontend/editor/src/proprietary/routes/authShared/auth.css b/frontend/editor/src/proprietary/routes/authShared/auth.css index e0abf4e7a..080c28092 100644 --- a/frontend/editor/src/proprietary/routes/authShared/auth.css +++ b/frontend/editor/src/proprietary/routes/authShared/auth.css @@ -615,3 +615,90 @@ background-color: #af3434 !important; opacity: 0.6 !important; } + +/* ── Logo block ─────────────────────────────────────────────────────── */ +.auth-logo-block { + display: flex; + flex-direction: column; + align-items: center; + margin-bottom: 2rem; + margin-top: 0.5rem; + animation: authFadeUp 0.4s ease both; +} + +.auth-logo-header { + height: 8rem; + width: auto; +} + +.auth-logo-header--dark { + display: none; +} + +/* ── Page entrance animation ────────────────────────────────────────── */ +@keyframes authFadeUp { + from { + opacity: 0; + transform: translateY(12px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +/* ── Expandable trigger button state ────────────────────────────────── */ +.auth-expandable-trigger { + transition: + background-color 180ms ease, + box-shadow 180ms ease, + border-color 180ms ease; +} + +.auth-expandable-trigger--active { + border-color: #af3434 !important; + box-shadow: 0 0 0 3px rgba(175, 52, 52, 0.12); +} + +/* ── Animated expand/collapse via grid-template-rows ───────────────── */ +.auth-expand-grid { + display: grid; + grid-template-rows: 0fr; + transition: + grid-template-rows 280ms cubic-bezier(0.4, 0, 0.2, 1), + opacity 220ms ease; + opacity: 0; +} + +.auth-expand-grid--open { + grid-template-rows: 1fr; + opacity: 1; +} + +.auth-expand-inner { + overflow: hidden; + min-height: 0; +} + +/* ── Icon+label group — keeps icons vertically aligned across buttons ── */ +.oauth-btn-group { + display: inline-flex; + align-items: center; + min-width: 12.5rem; +} + +.oauth-btn-label { + flex: 1; + text-align: center; +} + +/* ── Icon helpers inside oauth-button-fullwidth ─────────────────────── */ +.auth-at-icon { + font-size: 1.25rem; + line-height: 1; + margin-right: 0.5rem; + display: inline-block; + width: 1.75rem; + text-align: center; + flex-shrink: 0; +} diff --git a/frontend/editor/src/saas/routes/Login.tsx b/frontend/editor/src/saas/routes/Login.tsx index 848792147..4be5ed9b5 100644 --- a/frontend/editor/src/saas/routes/Login.tsx +++ b/frontend/editor/src/saas/routes/Login.tsx @@ -7,17 +7,18 @@ import { useDocumentMeta } from "@app/hooks/useDocumentMeta"; import AuthLayout from "@app/routes/authShared/AuthLayout"; import "@app/routes/authShared/auth.css"; import "@app/routes/authShared/saas-auth.css"; -import GuestSignInButton from "@app/routes/authShared/GuestSignInButton"; +import { + absoluteWithBasePath, + getBaseUrl, + withBasePath, +} from "@app/constants/app"; +import LinkRoundedIcon from "@mui/icons-material/LinkRounded"; // Import login components -import LoginHeader from "@app/routes/login/LoginHeader"; import ErrorMessage from "@app/routes/login/ErrorMessage"; import EmailPasswordForm from "@app/routes/login/EmailPasswordForm"; -import MagicLinkForm from "@app/routes/login/MagicLinkForm"; import OAuthButtons from "@app/routes/login/OAuthButtons"; -import DividerWithText from "@app/components/shared/DividerWithText"; import LoggedInState from "@app/routes/login/LoggedInState"; -import { absoluteWithBasePath, getBaseUrl } from "@app/constants/app"; export default function Login() { const navigate = useNavigate(); @@ -25,11 +26,13 @@ export default function Login() { const { t } = useTranslation(); const [isSigningIn, setIsSigningIn] = useState(false); const [error, setError] = useState(null); - const [showMagicLink, setShowMagicLink] = useState(false); + const [showMagicLinkForm, setShowMagicLinkForm] = useState(false); const [showEmailForm, setShowEmailForm] = useState(false); const [email, setEmail] = useState(""); const [password, setPassword] = useState(""); const [magicLinkEmail, setMagicLinkEmail] = useState(""); + const [magicLinkSent, setMagicLinkSent] = useState(false); + // Prefill email from query param (e.g. after password reset) useEffect(() => { try { @@ -37,6 +40,7 @@ export default function Login() { const emailFromQuery = url.searchParams.get("email"); if (emailFromQuery) { setEmail(emailFromQuery); + setShowEmailForm(true); } } catch (_) { // ignore @@ -174,12 +178,9 @@ export default function Login() { setError(error.message); } else { setError(null); - alert(t("login.magicLinkSent", { email: magicLinkEmail })); - setMagicLinkEmail(""); - setShowMagicLink(false); + setMagicLinkSent(true); } } catch (err) { - console.error("[Login] Unexpected error:", err); setError( t("login.unexpectedError", { message: err instanceof Error ? err.message : "Unknown error", @@ -190,10 +191,6 @@ export default function Login() { } }; - const handleForgotPassword = () => { - navigate("/auth/reset"); - }; - const handleAnonymousSignIn = async () => { try { setIsSigningIn(true); @@ -227,115 +224,210 @@ export default function Login() { } }; + const toggleEmailForm = () => { + setShowEmailForm((v) => !v); + setShowMagicLinkForm(false); + setMagicLinkSent(false); + }; + + const toggleMagicLink = () => { + setShowMagicLinkForm((v) => !v); + setShowEmailForm(false); + setMagicLinkSent(false); + }; + return ( - - + + {/* Centered logo */} +
+ Stirling PDF + Stirling PDF +
- {/* OAuth first */} - + {/* OAuth + magic link group — single flex column so gap is uniform */} +
+ - {/* Divider between OAuth and Email */} - + {/* Magic link button + its expandable form as one unit */} +
+ - {/* Sign in with email button (primary color to match signup CTA) */} -
+
+
+
+ {magicLinkSent ? ( +

+ {t("login.magicLinkSent", { email: magicLinkEmail })} +

+ ) : ( +
+ setMagicLinkEmail(e.target.value)} + onKeyDown={(e) => + e.key === "Enter" && + !isSigningIn && + signInWithMagicLink() + } + className="auth-input" + /> + +
+ )} +
+
+
+
+
+ + {/* Email & Password button */} + + + {/* Email form — animated expand */} +
+
+
+ + +
+
+
+ + {/* Skip */} +
- {showEmailForm && ( - - )} - - {showEmailForm && ( -
- -
- )} - - {/* Divider then Guest */} - - - - -
- - + {/* Bottom */} +
- - {/* Magic link form renders on demand */} - {showMagicLink && ( -
- -
- )} ); } diff --git a/frontend/editor/src/saas/routes/Signup.tsx b/frontend/editor/src/saas/routes/Signup.tsx index d772cbe37..724bf455a 100644 --- a/frontend/editor/src/saas/routes/Signup.tsx +++ b/frontend/editor/src/saas/routes/Signup.tsx @@ -4,18 +4,15 @@ import { signInAnonymously } from "@app/auth/supabase"; import { useAuth } from "@app/auth/UseSession"; import { useTranslation } from "@app/hooks/useTranslation"; import { useDocumentMeta } from "@app/hooks/useDocumentMeta"; -import { getBaseUrl } from "@app/constants/app"; +import { getBaseUrl, withBasePath } from "@app/constants/app"; import AuthLayout from "@app/routes/authShared/AuthLayout"; import "@app/routes/authShared/auth.css"; import "@app/routes/authShared/saas-auth.css"; -import GuestSignInButton from "@app/routes/authShared/GuestSignInButton"; import { alert } from "@app/components/toast"; // Import signup components -import LoginHeader from "@app/routes/login/LoginHeader"; import ErrorMessage from "@app/routes/login/ErrorMessage"; import OAuthButtons from "@app/routes/login/OAuthButtons"; -import DividerWithText from "@app/components/shared/DividerWithText"; import SignupForm from "@app/routes/signup/SignupForm"; import { useSignupFormValidation, @@ -184,84 +181,112 @@ export default function Signup() { return ( - + {/* Centered logo */} +
+ Stirling PDF + Stirling PDF +
- {/* OAuth first */} -
+ {/* OAuth providers */} +
- {/* Divider between OAuth and Email */} -
- + {/* Email & Password button */} + + + {/* Email form — animated expand */} +
+
+
+ +
+
- {/* Use Email Instead button (toggles email form) */} -
+ {/* Skip */} +
- {showEmailForm && ( - - )} - -
- -
- - - - {/* Bottom row */} -
+ {/* Bottom */} +
diff --git a/frontend/editor/src/saas/routes/authShared/saas-auth.css b/frontend/editor/src/saas/routes/authShared/saas-auth.css index 5073a24c9..9a8943fc2 100644 --- a/frontend/editor/src/saas/routes/authShared/saas-auth.css +++ b/frontend/editor/src/saas/routes/authShared/saas-auth.css @@ -14,7 +14,7 @@ justify-content: center; padding: 0.75rem 1rem; border: 1px solid #d1d5db; - border-radius: 0.625rem; + border-radius: 100px; background-color: #ffffff; font-size: 1rem; font-weight: 600; @@ -22,6 +22,10 @@ cursor: pointer; gap: 0.5rem; box-shadow: 0 0.125rem 0.375rem rgba(0, 0, 0, 0.04); + transition: + background-color 150ms ease, + box-shadow 150ms ease, + border-color 150ms ease; } .oauth-button-fullwidth:disabled { @@ -29,6 +33,11 @@ opacity: 0.6; } +.oauth-button-fullwidth:hover:not(:disabled) { + background-color: #fafafa; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.08); +} + .auth-dropdown-wrapper { position: relative; } @@ -72,3 +81,90 @@ color: #9c2f30; border: 2px solid currentColor; } + +/* ── Logo block ─────────────────────────────────────────────────────── */ +.auth-logo-block { + display: flex; + flex-direction: column; + align-items: center; + margin-bottom: 2rem; + margin-top: 0.5rem; + animation: authFadeUp 0.4s ease both; +} + +.auth-logo-header { + height: 8rem; + width: auto; +} + +.auth-logo-header--dark { + display: none; +} + +/* ── Page entrance animation ────────────────────────────────────────── */ +@keyframes authFadeUp { + from { + opacity: 0; + transform: translateY(12px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +/* ── Expandable trigger button state ────────────────────────────────── */ +.auth-expandable-trigger { + transition: + background-color 180ms ease, + box-shadow 180ms ease, + border-color 180ms ease; +} + +.auth-expandable-trigger--active { + border-color: #af3434 !important; + box-shadow: 0 0 0 3px rgba(175, 52, 52, 0.12); +} + +/* ── Animated expand/collapse via grid-template-rows ───────────────── */ +.auth-expand-grid { + display: grid; + grid-template-rows: 0fr; + transition: + grid-template-rows 280ms cubic-bezier(0.4, 0, 0.2, 1), + opacity 220ms ease; + opacity: 0; +} + +.auth-expand-grid--open { + grid-template-rows: 1fr; + opacity: 1; +} + +.auth-expand-inner { + overflow: hidden; + min-height: 0; +} + +/* ── Icon+label group — keeps icons vertically aligned across buttons ── */ +.oauth-btn-group { + display: inline-flex; + align-items: center; + min-width: 12.5rem; +} + +.oauth-btn-label { + flex: 1; + text-align: center; +} + +/* ── Icon helpers inside oauth-button-fullwidth ─────────────────────── */ +.auth-at-icon { + font-size: 1.25rem; + line-height: 1; + margin-right: 0.5rem; + display: inline-block; + width: 1.75rem; + text-align: center; + flex-shrink: 0; +} diff --git a/frontend/editor/src/saas/routes/login/OAuthButtons.tsx b/frontend/editor/src/saas/routes/login/OAuthButtons.tsx index e016c125e..717f96b9a 100644 --- a/frontend/editor/src/saas/routes/login/OAuthButtons.tsx +++ b/frontend/editor/src/saas/routes/login/OAuthButtons.tsx @@ -17,6 +17,7 @@ interface OAuthButtonsProps { isSubmitting: boolean; layout?: "vertical" | "grid" | "icons" | "fullwidth"; enabledProviders?: string[]; // List of enabled provider IDs from backend + labelPrefix?: string; } export default function OAuthButtons({ @@ -24,6 +25,7 @@ export default function OAuthButtons({ isSubmitting, layout = "vertical", enabledProviders: _enabledProviders = [], + labelPrefix = "", }: OAuthButtonsProps) { const { t } = useTranslation(); @@ -92,12 +94,18 @@ export default function OAuthButtons({ className="oauth-button-fullwidth" title={p.label} > - {p.label} - {p.label} + + {p.label} + + {labelPrefix} + {p.label} + + ))}
diff --git a/frontend/editor/src/saas/styles/saas-theme.css b/frontend/editor/src/saas/styles/saas-theme.css index 9d670c98d..2aad69c54 100644 --- a/frontend/editor/src/saas/styles/saas-theme.css +++ b/frontend/editor/src/saas/styles/saas-theme.css @@ -35,7 +35,7 @@ --auth-label-text: #2b3230; --auth-button-bg: #af3434; --auth-button-text: #ffffff; - --auth-magic-button-bg: #8b5cf6; + --auth-magic-button-bg: #af3434; --auth-magic-button-text: #ffffff; /* Light-only auth colors (no dark mode equivalents) used for login/signup */ @@ -45,7 +45,7 @@ --auth-label-text-light-only: #2b3230; --auth-button-bg-light-only: #af3434; --auth-button-text-light-only: #ffffff; - --auth-magic-button-bg-light-only: #8b5cf6; + --auth-magic-button-bg-light-only: #af3434; --auth-magic-button-text-light-only: #ffffff; --auth-bg-color-light-only: #ffffff; --auth-card-bg-light-only: #ffffff; From 2aa6768921f9b0cc1509169459f0dbe92f8b659e Mon Sep 17 00:00:00 2001 From: EthanHealy01 <80844253+EthanHealy01@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:47:57 +0100 Subject: [PATCH 3/5] show chat progress and other UX improvements (#6576) --- .taskfiles/engine.yml | 17 +- .../public/locales/en-GB/translation.toml | 7 + .../src/core/components/chat/ChatContext.tsx | 1 + .../components/chat/ChatContext.tsx | 78 ++++- .../proprietary/components/chat/ChatPanel.css | 184 ++++++++++- .../proprietary/components/chat/ChatPanel.tsx | 300 +++++++++++++++--- 6 files changed, 521 insertions(+), 66 deletions(-) diff --git a/.taskfiles/engine.yml b/.taskfiles/engine.yml index 5c9d6a8e6..1834495b1 100644 --- a/.taskfiles/engine.yml +++ b/.taskfiles/engine.yml @@ -1,5 +1,12 @@ version: '3' +vars: + # Engine-specific names to avoid overriding the root Taskfile's FIND_FREE_PORT_* + # vars (Task merges included-file vars into the global scope). + # Paths are relative to the engine/ include dir. + ENGINE_FIND_FREE_PORT_SH: "bash ../scripts/find-free-port.sh" + ENGINE_FIND_FREE_PORT_PS: "powershell -NoProfile -File ../scripts/find-free-port.ps1" + tasks: install: desc: "Install engine dependencies" @@ -29,7 +36,10 @@ tasks: ignore_error: true dir: src vars: - PORT: '{{.PORT | default "5001"}}' + # When PORT is provided (e.g. from dev:all), use it directly. + # When running standalone, probe for a free port starting at 5001. + PORT: + sh: '{{if .PORT}}echo {{.PORT}}{{else if eq OS "windows"}}{{.ENGINE_FIND_FREE_PORT_PS}} 5001{{else}}{{.ENGINE_FIND_FREE_PORT_SH}} 5001{{end}}' env: PYTHONUNBUFFERED: "1" cmds: @@ -41,7 +51,10 @@ tasks: ignore_error: true dir: src vars: - PORT: '{{.PORT | default "5001"}}' + # When PORT is provided (e.g. from dev:all), use it directly. + # When running standalone, probe for a free port starting at 5001. + PORT: + sh: '{{if .PORT}}echo {{.PORT}}{{else if eq OS "windows"}}{{.ENGINE_FIND_FREE_PORT_PS}} 5001{{else}}{{.ENGINE_FIND_FREE_PORT_SH}} 5001{{end}}' env: PYTHONUNBUFFERED: "1" cmds: diff --git a/frontend/editor/public/locales/en-GB/translation.toml b/frontend/editor/public/locales/en-GB/translation.toml index 71c77f866..e907cba19 100644 --- a/frontend/editor/public/locales/en-GB/translation.toml +++ b/frontend/editor/public/locales/en-GB/translation.toml @@ -2700,6 +2700,13 @@ whole_doc_compression_round = "Consolidating notes..." whole_doc_read_done = "Finished reading the document..." whole_doc_read_started = "Reading the document..." whole_doc_slice_done = "Reading the document... ({{percent}}% complete)" +ranForSeconds = "Ran for {{count}} seconds" +ranForSeconds_one = "Ran for 1 second" +ranForSeconds_other = "Ran for {{count}} seconds" +ranForMinutes = "Ran for {{count}} minutes" +ranForMinutes_one = "Ran for 1 minute" +ranForMinutes_other = "Ran for {{count}} minutes" +ranForMinutesSeconds = "Ran for {{minutes}} min, {{seconds}} sec" [chat.quickActions] browseYourFiles = "Browse your files" diff --git a/frontend/editor/src/core/components/chat/ChatContext.tsx b/frontend/editor/src/core/components/chat/ChatContext.tsx index 3ef7276fb..621fafd6e 100644 --- a/frontend/editor/src/core/components/chat/ChatContext.tsx +++ b/frontend/editor/src/core/components/chat/ChatContext.tsx @@ -10,6 +10,7 @@ export function useChat() { isOpen: false, isLoading: false, progress: null, + progressLog: [] as never[], toggleOpen: () => {}, setOpen: (_open: boolean) => {}, sendMessage: async (_content: string) => {}, diff --git a/frontend/editor/src/proprietary/components/chat/ChatContext.tsx b/frontend/editor/src/proprietary/components/chat/ChatContext.tsx index 6cd473eda..eb0534c1b 100644 --- a/frontend/editor/src/proprietary/components/chat/ChatContext.tsx +++ b/frontend/editor/src/proprietary/components/chat/ChatContext.tsx @@ -36,6 +36,14 @@ export interface ChatMessage { * turns that answered without running any tool. */ toolsUsed?: string[]; + /** + * Full ordered progress log captured during the AI turn that produced this message. + * Only set on assistant messages; used to render the "Ran for X seconds" collapsed + * history dropdown above the response. + */ + progressLog?: AiWorkflowProgress[]; + /** Wall-clock duration of the AI turn in milliseconds. Only set on assistant messages. */ + durationMs?: number; } export enum AiWorkflowPhase { @@ -177,12 +185,20 @@ interface ChatState { isOpen: boolean; isLoading: boolean; progress: AiWorkflowProgress | null; + /** Ordered log of every progress event in the current request. UI shows the last N entries. */ + progressLog: AiWorkflowProgress[]; } +/** + * Maximum number of progress steps retained in the live buffer. + */ +export const PROGRESS_LOG_MAX = 4; + type ChatAction = | { type: "ADD_MESSAGE"; message: ChatMessage } | { type: "SET_LOADING"; loading: boolean } | { type: "SET_PROGRESS"; progress: AiWorkflowProgress | null } + | { type: "APPEND_PROGRESS"; progress: AiWorkflowProgress } | { type: "TOGGLE_OPEN" } | { type: "SET_OPEN"; open: boolean } | { type: "CLEAR" }; @@ -192,15 +208,40 @@ function chatReducer(state: ChatState, action: ChatAction): ChatState { case "ADD_MESSAGE": return { ...state, messages: [...state.messages, action.message] }; case "SET_LOADING": - return { ...state, isLoading: action.loading }; + // Reset the log on both start (true) and end (false) of a request. + return { + ...state, + isLoading: action.loading, + progress: action.loading ? state.progress : null, + progressLog: [], + }; case "SET_PROGRESS": return { ...state, progress: action.progress }; + case "APPEND_PROGRESS": + // Cap the live buffer so each append copies at most PROGRESS_LOG_MAX elements + return { + ...state, + progress: action.progress, + progressLog: + state.progressLog.length < PROGRESS_LOG_MAX + ? [...state.progressLog, action.progress] + : [ + ...state.progressLog.slice(1 - PROGRESS_LOG_MAX), + action.progress, + ], + }; case "TOGGLE_OPEN": return { ...state, isOpen: !state.isOpen }; case "SET_OPEN": return { ...state, isOpen: action.open }; case "CLEAR": - return { ...state, messages: [], isLoading: false, progress: null }; + return { + ...state, + messages: [], + isLoading: false, + progress: null, + progressLog: [], + }; } } @@ -324,6 +365,8 @@ interface ChatContextValue { isOpen: boolean; isLoading: boolean; progress: AiWorkflowProgress | null; + /** Ordered log of every progress event for the current in-flight request. */ + progressLog: AiWorkflowProgress[]; toggleOpen: () => void; setOpen: (open: boolean) => void; sendMessage: (content: string) => Promise; @@ -338,6 +381,7 @@ const initialState: ChatState = { isOpen: false, isLoading: false, progress: null, + progressLog: [], }; export function ChatProvider({ children }: { children: ReactNode }) { @@ -439,6 +483,11 @@ export function ChatProvider({ children }: { children: ReactNode }) { abortRef.current = controller; const priorMessages = messagesRef.current; + const startTime = Date.now(); + // Mirror every progress event locally so we can attach the full log to + // the assistant message when the result arrives — without needing a ref + // into the reducer state. + const progressLogLocal: AiWorkflowProgress[] = []; const userMessage: ChatMessage = { id: generateId(), @@ -499,16 +548,15 @@ export function ChatProvider({ children }: { children: ReactNode }) { ) { toolsUsed.push(data.tool); } - dispatch({ - type: "SET_PROGRESS", - progress: { - phase: data.phase as AiWorkflowPhase, - tool: data.tool, - stepIndex: data.stepIndex, - stepCount: data.stepCount, - engineDetail: data.engineDetail, - }, - }); + const progressItem: AiWorkflowProgress = { + phase: data.phase as AiWorkflowPhase, + tool: data.tool, + stepIndex: data.stepIndex, + stepCount: data.stepCount, + engineDetail: data.engineDetail, + }; + progressLogLocal.push(progressItem); + dispatch({ type: "APPEND_PROGRESS", progress: progressItem }); }, onResult: (data) => { receivedResult = true; @@ -522,6 +570,11 @@ export function ChatProvider({ children }: { children: ReactNode }) { content: replyContent, timestamp: Date.now(), toolsUsed: toolsUsed.length > 0 ? toolsUsed : undefined, + progressLog: + progressLogLocal.length > 0 + ? [...progressLogLocal] + : undefined, + durationMs: Date.now() - startTime, }, }); if (data.fileId || data.resultFiles?.length) { @@ -595,6 +648,7 @@ export function ChatProvider({ children }: { children: ReactNode }) { isOpen: state.isOpen, isLoading: state.isLoading, progress: state.progress, + progressLog: state.progressLog, toggleOpen, setOpen, sendMessage, diff --git a/frontend/editor/src/proprietary/components/chat/ChatPanel.css b/frontend/editor/src/proprietary/components/chat/ChatPanel.css index 0176172ce..62402108f 100644 --- a/frontend/editor/src/proprietary/components/chat/ChatPanel.css +++ b/frontend/editor/src/proprietary/components/chat/ChatPanel.css @@ -396,7 +396,182 @@ margin-bottom: 0; } -/* Thinking indicator */ +/* ─── Progress step log ─────────────────────────────────────────────────── */ + +/* + * Shown while the AI is working. Each step slides in from below as events + * arrive; the active (last) step is full-opacity with a pulsing icon, and + * earlier completed steps are dimmed. Only the most-recent N steps are + * rendered — there is no scrollable list. + */ + +.chat-progress-log { + display: flex; + flex-direction: column; + padding: 0.15rem 0.5rem 0.35rem; + /* No gap — the connector line div provides the inter-step spacing. */ +} + +.chat-progress-step { + display: flex; + align-items: flex-start; + gap: 0.65rem; + /* + * Use color rather than opacity for past-step dimming so the connector + * line (a sibling element inside __left, not a text node) is NOT affected + * and stays clearly visible. currentColor flows into SVG icon fills; + * color inherits into the label span automatically. + */ + color: var(--text-muted); + transition: color 250ms ease-out; + animation: chat-step-enter 300ms cubic-bezier(0.22, 1, 0.36, 1) both; +} + +/* + * Active step: white text on dark mode so currentColor feeds directly into + * the StirlingLogoAnimated SVG paths and the label at the same time. + * Light-mode falls back to the default text colour so it's not invisible. + */ +/* Active step: icon and label are coloured independently so the logo can be + blue while the text uses the normal text colour. */ +.chat-progress-step--active .chat-progress-step__icon { + color: var(--mantine-color-blue-filled); +} + +.chat-progress-step--active .chat-progress-step__label { + color: var(--mantine-color-text); +} + +[data-mantine-color-scheme="dark"] + .chat-progress-step--active + .chat-progress-step__label { + color: #ffffff; +} + +.chat-progress-step__left { + display: flex; + flex-direction: column; + align-items: center; + flex-shrink: 0; + width: 20px; +} + +/* Icon cell — fixed height so the connector line aligns cleanly. */ +.chat-progress-step__icon { + width: 20px; + height: 24px; + display: flex; + align-items: center; + justify-content: center; + overflow: hidden; + flex-shrink: 0; + /* Inherits color from .chat-progress-step so currentColor flows into icons. */ +} + +/* + * Connector line between steps. Not a child of the label so it is unaffected + * by the step's color; it sits on its own and reads clearly against the + * toolbar background. + */ +.chat-progress-step__line { + width: 1px; + height: 18px; + background: var(--border-subtle); + flex-shrink: 0; + margin-top: 2px; +} + +/* Active step label — slightly larger, inherits the active color. */ +.chat-progress-step__label { + font-size: 0.875rem; + padding-top: 3px; + line-height: 1.35; +} + +/* Past step labels — a touch smaller so the active row reads as the primary. */ +.chat-progress-step:not(.chat-progress-step--active) + .chat-progress-step__label { + font-size: 0.8rem; + padding-top: 4px; +} + +/* + * Wrapper that scales a registry tool icon (LocalIcon, typically 1.5rem/24px) + * down to fit the 20px icon column. Transform does not affect layout, so the + * parent overflow:hidden clips the un-scaled footprint cleanly. + */ +.chat-step-icon-scaled { + display: inline-flex; + align-items: center; + justify-content: center; + transform: scale(0.75); + transform-origin: center; + flex-shrink: 0; +} + +/* Entry: steps slide up from a few pixels below their final position. */ +@keyframes chat-step-enter { + from { + transform: translateY(7px); + } + to { + transform: translateY(0); + } +} + +/* ─── Completed progress log dropdown ──────────────────────────────────── */ + +/* + * Shown above each completed assistant turn. The toggle displays a small + * "Ran for X seconds" label; expanding reveals the full ordered step list + * for that turn in a compact, read-only format. + */ + +.chat-completed-log { + margin-bottom: 0.55rem; +} + +.chat-completed-log__toggle { + display: inline-flex; + align-items: center; + padding: 0.1rem 0.25rem 0.1rem 0.1rem; + border-radius: 0.35rem; + color: var(--text-muted); + transition: background 120ms ease-out; +} + +.chat-completed-log__toggle:hover { + background: var(--mantine-color-default-hover); +} + +.chat-completed-log__steps { + display: flex; + flex-direction: column; + padding: 0.35rem 0.4rem 0.2rem 0.25rem; + margin-top: 0.2rem; +} + +/* In the completed log every step is a past step — no active highlighting, + no entry animation (the content is static once opened). */ +.chat-completed-log__steps .chat-progress-step { + animation: none; +} + +/* Tighten the icon and connector dimensions for the denser historical view. */ +.chat-completed-log__steps .chat-progress-step__icon { + height: 20px; +} + +.chat-completed-log__steps .chat-progress-step__line { + height: 14px; +} + +.chat-completed-log__steps .chat-progress-step__label { + font-size: 0.775rem; + padding-top: 2px; +} + +/* Legacy thinking indicator (kept in case any other code references it). */ .chat-thinking { display: flex; align-items: center; @@ -405,6 +580,13 @@ color: var(--mantine-color-blue-filled); } +@media (prefers-reduced-motion: reduce) { + .chat-progress-step { + animation: none; + transition: none; + } +} + /* ─── Stirling logo thinking animation ─────────────────────────────────── */ .stirling-thinking__path-right { diff --git a/frontend/editor/src/proprietary/components/chat/ChatPanel.tsx b/frontend/editor/src/proprietary/components/chat/ChatPanel.tsx index 7baf5d0a5..ff0fdc63b 100644 --- a/frontend/editor/src/proprietary/components/chat/ChatPanel.tsx +++ b/frontend/editor/src/proprietary/components/chat/ChatPanel.tsx @@ -4,6 +4,7 @@ import { useEffect, useState, type KeyboardEvent, + type ReactNode, } from "react"; import { renderMarkdown } from "@app/components/viewer/nonpdf/MarkdownRenderer"; import { TFunction } from "i18next"; @@ -13,7 +14,6 @@ import { Box, Collapse, Group, - List, Menu, Paper, ScrollArea, @@ -23,7 +23,10 @@ import { UnstyledButton, } from "@mantine/core"; import ArrowUpwardIcon from "@mui/icons-material/ArrowUpward"; +import ArticleOutlinedIcon from "@mui/icons-material/ArticleOutlined"; +import BuildOutlinedIcon from "@mui/icons-material/BuildOutlined"; import CloseIcon from "@mui/icons-material/Close"; +import CloudOutlinedIcon from "@mui/icons-material/CloudOutlined"; import ContentCopyIcon from "@mui/icons-material/ContentCopy"; import DeleteSweepIcon from "@mui/icons-material/DeleteSweep"; import ExpandLessIcon from "@mui/icons-material/ExpandLess"; @@ -33,14 +36,15 @@ import { useChat, AiWorkflowPhase, ChatRole, + PROGRESS_LOG_MAX, isKnownEngineProgressDetail, type AiWorkflowProgress, type AnyEngineProgressDetail, } from "@app/components/chat/ChatContext"; import { formatRelativeTime } from "@app/utils/timeUtils"; import { useTranslatedToolCatalog } from "@app/data/useTranslatedToolRegistry"; -import { StirlingLogoOutline } from "@app/components/agents/StirlingLogoOutline"; import { StirlingLogoAnimated } from "@app/components/agents/StirlingLogoAnimated"; +import { StirlingLogoOutline } from "@app/components/agents/StirlingLogoOutline"; import { ChatQuickActions } from "@app/components/chat/ChatQuickActions"; import "@app/components/chat/ChatPanel.css"; @@ -68,6 +72,27 @@ function useToolNameResolver(): ToolNameResolver { }, [allTools]); } +/** Resolver mapping a tool endpoint path to its registry icon ReactNode. */ +type ToolIconResolver = (endpoint: string) => ReactNode | null; + +/** + * Look up a tool's icon ReactNode from the tool catalog, keyed by API endpoint path. + * Returns null when the endpoint is not found (use a generic fallback icon in that case). + */ +function useToolIconResolver(): ToolIconResolver { + const { allTools } = useTranslatedToolCatalog(); + return useMemo(() => { + const iconByEndpoint = new Map(); + Object.values(allTools).forEach((tool) => { + const endpoint = tool.operationConfig?.endpoint; + if (typeof endpoint === "string") { + iconByEndpoint.set(endpoint, tool.icon); + } + }); + return (endpoint: string) => iconByEndpoint.get(endpoint) ?? null; + }, [allTools]); +} + function formatProgress( progress: AiWorkflowProgress, t: TranslateFn, @@ -125,31 +150,152 @@ function formatEngineProgress( } } -function ToolsUsedBlock({ - tools, - resolveToolName, +/** + * Choose an icon for a progress step. + * + * The active (current) step always shows the animated Stirling logo so it reads + * as the "live" indicator. Past steps get a phase-specific icon so the trail + * is scannable at a glance. + */ +function progressStepIcon( + progress: AiWorkflowProgress, + resolveToolIcon: ToolIconResolver, + isActive: boolean, +): ReactNode { + if (isActive) { + return ; + } + if (progress.phase === AiWorkflowPhase.EXECUTING_TOOL) { + const registryIcon = progress.tool ? resolveToolIcon(progress.tool) : null; + if (registryIcon) { + return {registryIcon}; + } + return ; + } + if ( + progress.phase === AiWorkflowPhase.EXTRACTING_CONTENT || + progress.phase === AiWorkflowPhase.ENGINE_PROGRESS + ) { + return ; + } + return ; +} + +/** + * Animated step-by-step progress log shown while the AI is working. + * Displays the last {@link PROGRESS_LOG_VISIBLE} steps from the live event stream, + * with the active (most recent) step highlighted and older steps dimmed. + */ +function ProgressLogDisplay({ + progressLog, t, + resolveToolName, + resolveToolIcon, }: { - tools: string[]; - resolveToolName: ToolNameResolver; + progressLog: AiWorkflowProgress[]; t: TranslateFn; + resolveToolName: ToolNameResolver; + resolveToolIcon: ToolIconResolver; +}) { + // Placeholder shown before the first SSE event arrives. + if (progressLog.length === 0) { + return ( +
+
+
+
+ +
+
+ + {t("chat.progress.thinking")} + +
+
+ ); + } + + // Chronological order: oldest at top, newest (active) at bottom. + // The reducer already caps progressLog at PROGRESS_LOG_MAX entries, so this + // slice is effectively a no-op but kept for defensive correctness. + const visibleSteps = progressLog.slice(-PROGRESS_LOG_MAX); + const startIndex = progressLog.length - visibleSteps.length; + + return ( +
+ {visibleSteps.map((step, i) => { + // Stable key based on absolute position in the full log — React reuses + // existing DOM elements and only mounts (and animates) new ones. + const globalIndex = startIndex + i; + const isActive = i === visibleSteps.length - 1; // last = newest = bottom + // Connector runs below every step except the active one at the bottom. + const showConnector = i < visibleSteps.length - 1; + const label = formatProgress(step, t, resolveToolName); + return ( +
+
+
+ {progressStepIcon(step, resolveToolIcon, isActive)} +
+ {showConnector &&
} +
+ {label} +
+ ); + })} +
+ ); +} + +function formatDuration(ms: number, t: TranslateFn): string { + const totalSeconds = Math.max(1, Math.round(ms / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + + if (minutes === 0) { + return t("chat.progress.ranForSeconds", { count: totalSeconds }); + } + if (seconds === 0) { + return t("chat.progress.ranForMinutes", { count: minutes }); + } + return t("chat.progress.ranForMinutesSeconds", { minutes, seconds }); +} + +/** + * Collapsed "Ran for X seconds" dropdown that appears above each completed + * assistant turn. Expands to show the full ordered progress log for that turn. + */ +function CompletedProgressLogDropdown({ + progressLog, + durationMs, + t, + resolveToolName, + resolveToolIcon, +}: { + progressLog: AiWorkflowProgress[]; + durationMs: number; + t: TranslateFn; + resolveToolName: ToolNameResolver; + resolveToolIcon: ToolIconResolver; }) { const [expanded, setExpanded] = useState(false); - const names = tools.map( - (endpoint) => resolveToolName(endpoint) ?? t("chat.toolsUsed.unknownTool"), - ); - const label = t("chat.toolsUsed.summary", { count: tools.length }); + const label = formatDuration(durationMs, t); + return ( - +
setExpanded((v) => !v)} aria-expanded={expanded} > - + {expanded ? ( - + ) : ( - + )} {label} @@ -157,19 +303,30 @@ function ToolsUsedBlock({ - - {names.map((name, i) => ( - {name} - ))} - +
+ {progressLog.map((step, i) => { + const showConnector = i < progressLog.length - 1; + const stepLabel = formatProgress(step, t, resolveToolName); + return ( +
+
+
+ {progressStepIcon(step, resolveToolIcon, false)} +
+ {showConnector && ( +
+ )} +
+ {stepLabel} +
+ ); + })} +
- +
); } @@ -177,15 +334,19 @@ function ChatMessageBubble({ role, content, timestamp, - toolsUsed, + progressLog, + durationMs, resolveToolName, + resolveToolIcon, t, }: { role: ChatRole; content: string; timestamp: number; - toolsUsed?: string[]; + progressLog?: AiWorkflowProgress[]; + durationMs?: number; resolveToolName: ToolNameResolver; + resolveToolIcon: ToolIconResolver; t: TranslateFn; }) { const [copied, setCopied] = useState(false); @@ -231,16 +392,18 @@ function ChatMessageBubble({ return (
+ {progressLog && progressLog.length > 0 && durationMs != null && ( + + )} {renderMarkdown(content)} - {toolsUsed && toolsUsed.length > 0 && ( - - )} {actions}
@@ -256,20 +419,55 @@ export interface ChatPanelProps { export function ChatPanel({ onBack, backLabel }: ChatPanelProps) { const { t } = useTranslation(); - const { messages, isLoading, progress, sendMessage, clearChat } = useChat(); + const { messages, isLoading, progressLog, sendMessage, clearChat } = + useChat(); const resolveToolName = useToolNameResolver(); + const resolveToolIcon = useToolIconResolver(); const [input, setInput] = useState(""); const scrollRef = useRef(null); const inputRef = useRef(null); + // Tracks whether the user manually scrolled away from the bottom. + // A ref (not state) so scroll events don't cause re-renders. + const userScrolledUp = useRef(false); + // Jump to the bottom on first render so existing conversations open at the + // most recent message rather than the top. useEffect(() => { - if (scrollRef.current) { - scrollRef.current.scrollTo({ - top: scrollRef.current.scrollHeight, - behavior: "smooth", + requestAnimationFrame(() => { + const el = scrollRef.current; + if (el) el.scrollTop = el.scrollHeight; + }); + }, []); + + // Attach a passive scroll listener to track whether the user has scrolled + // away from the bottom (breaks auto-scroll) or returned to it (re-latches). + useEffect(() => { + const el = scrollRef.current; + if (!el) return; + const onScroll = () => { + const distFromBottom = el.scrollHeight - el.scrollTop - el.clientHeight; + userScrolledUp.current = distFromBottom > 50; + }; + el.addEventListener("scroll", onScroll, { passive: true }); + return () => el.removeEventListener("scroll", onScroll); + }, []); + + // Scroll to the bottom when messages arrive or live progress steps update, + // unless the user has scrolled up (they're reading history). + // Scrolling back to the bottom resets the ref, so the next update re-latches. + // + // RAF defers the scroll until after the browser has laid out the new nodes, + // so scrollHeight is correct. Direct scrollTop assignment avoids the + // smooth-scroll interruption problem that occurs when SSE events arrive + // faster than a smooth animation can complete. + useEffect(() => { + if (!userScrolledUp.current) { + requestAnimationFrame(() => { + const el = scrollRef.current; + if (el) el.scrollTop = el.scrollHeight; }); } - }, [messages]); + }, [messages, progressLog]); useEffect(() => { inputRef.current?.focus(); @@ -343,21 +541,21 @@ export function ChatPanel({ onBack, backLabel }: ChatPanelProps) { role={msg.role} content={msg.content} timestamp={msg.timestamp} - toolsUsed={msg.toolsUsed} + progressLog={msg.progressLog} + durationMs={msg.durationMs} resolveToolName={resolveToolName} + resolveToolIcon={resolveToolIcon} t={t} /> ))} {isLoading && (
-
- - - {progress - ? formatProgress(progress, t, resolveToolName) - : t("chat.progress.thinking")} - -
+
)} From 5fca2f199ae5ba2ccae3ed93a09372d6948a9c59 Mon Sep 17 00:00:00 2001 From: EthanHealy01 <80844253+EthanHealy01@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:51:41 +0100 Subject: [PATCH 4/5] Feature/pdf ingestion jpdfium (#6525) --- .../SPDF/pdf/parser/CompositeTableParser.java | 73 -- .../pdf/parser/LineAlignmentTableParser.java | 528 --------- .../software/SPDF/pdf/parser/LineBuilder.java | 139 --- .../software/SPDF/pdf/parser/PdfIngester.java | 79 -- .../pdf/parser/WordExtractingStripper.java | 113 -- .../software/common/pdf/HeadingDetector.java | 191 +++ .../common/pdf/PdfMarkdownConverter.java | 1043 +++++++++++++++++ .../software/common/pdf/TableRenderer.java | 82 ++ .../parser/LineAlignmentTableParserTest.java | 153 --- .../common/pdf/PdfMarkdownConverterTest.java | 269 +++++ .../bordered-table-test_widget.md | 10 + .../bordered-table-test_widget.pdf | 74 ++ .../many-tables-test_stress.md | 222 ++++ .../many-tables-test_stress.pdf | 169 +++ .../multi-column-test_lorem.md | 25 + .../multi-column-test_lorem.pdf | 74 ++ .../wrapped-cell-test_expense-report.md | 62 + .../wrapped-cell-test_expense-report.pdf | Bin 0 -> 95841 bytes .../api/converters/ConvertPDFToMarkdown.java | 32 +- .../converters/ConvertPDFToMarkdownTest.java | 94 +- .../model/api/ai/AiWorkflowOutcome.java | 3 +- .../service/AiWorkflowService.java | 84 +- .../service/PdfContentExtractor.java | 78 -- .../service/AiWorkflowServiceTest.java | 29 + .../PageLayoutArtifactContractTest.java | 66 -- engine/src/stirling/agents/__init__.py | 2 - engine/src/stirling/agents/orchestrator.py | 35 +- .../agents/pdf_to_markdown/__init__.py | 3 - .../stirling/agents/pdf_to_markdown/agent.py | 435 ------- engine/src/stirling/contracts/__init__.py | 22 +- engine/src/stirling/contracts/common.py | 14 + engine/src/stirling/contracts/orchestrator.py | 5 +- .../src/stirling/contracts/pdf_to_markdown.py | 105 -- engine/tests/test_pdf_to_markdown.py | 138 --- testing/cucumber/features/external.feature | 3 +- testing/test.sh | 6 +- 36 files changed, 2439 insertions(+), 2021 deletions(-) delete mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java delete mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java delete mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java delete mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java delete mode 100644 app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java create mode 100644 app/common/src/main/java/stirling/software/common/pdf/HeadingDetector.java create mode 100644 app/common/src/main/java/stirling/software/common/pdf/PdfMarkdownConverter.java create mode 100644 app/common/src/main/java/stirling/software/common/pdf/TableRenderer.java delete mode 100644 app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java create mode 100644 app/common/src/test/java/stirling/software/common/pdf/PdfMarkdownConverterTest.java create mode 100644 app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.md create mode 100644 app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.pdf create mode 100644 app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.md create mode 100644 app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.pdf create mode 100644 app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.md create mode 100644 app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.pdf create mode 100644 app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.md create mode 100644 app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.pdf delete mode 100644 app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java delete mode 100644 engine/src/stirling/agents/pdf_to_markdown/__init__.py delete mode 100644 engine/src/stirling/agents/pdf_to_markdown/agent.py delete mode 100644 engine/src/stirling/contracts/pdf_to_markdown.py delete mode 100644 engine/tests/test_pdf_to_markdown.py diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java deleted file mode 100644 index 429f180f3..000000000 --- a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/CompositeTableParser.java +++ /dev/null @@ -1,73 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.io.IOException; -import java.util.List; - -import org.apache.pdfbox.pdmodel.PDDocument; -import org.springframework.context.annotation.Primary; -import org.springframework.stereotype.Service; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** - * Chains table parsers in priority order: Tabula lattice → Tabula stream → {@link - * LineAlignmentTableParser}. The first parser returning a result above {@link - * #TABULA_CONFIDENCE_THRESHOLD} wins; results from different parsers are never mixed on one page. - */ -@Service -@Primary -@RequiredArgsConstructor -@Slf4j -public class CompositeTableParser implements TableParser { - - /** Min Tabula confidence to accept results; below this LineAlignment is tried instead. */ - static final float TABULA_CONFIDENCE_THRESHOLD = 0.5f; - - private final TabulaTableParser tabulaParser; - private final LineAlignmentTableParser lineAlignmentParser; - - @Override - public List parse(PDDocument document, RawPage rawPage) throws IOException { - // Step 1: Tabula lattice mode (ruled/bordered tables). - List latticeResults = filterConfident(tabulaParser.parse(document, rawPage)); - if (!latticeResults.isEmpty()) { - log.debug( - "Page {}: using Tabula lattice ({} table(s))", - rawPage.pageNumber(), - latticeResults.size()); - return latticeResults; - } - - // Step 2: Tabula stream mode (borderless/whitespace-delimited tables). - // parseStream is not on the TableParser interface — this intentionally couples to the - // concrete TabulaTableParser since stream mode is a Tabula-specific concept. - List streamResults = - filterConfident(tabulaParser.parseStream(document, rawPage)); - if (!streamResults.isEmpty()) { - log.debug( - "Page {}: using Tabula stream ({} table(s))", - rawPage.pageNumber(), - streamResults.size()); - return streamResults; - } - - // Step 3: Geometry-based line-alignment fallback. - List lineResults = lineAlignmentParser.parse(document, rawPage); - if (!lineResults.isEmpty()) { - log.debug( - "Page {}: using LineAlignment ({} table(s))", - rawPage.pageNumber(), - lineResults.size()); - return lineResults; - } - - return List.of(); - } - - private List filterConfident(List tables) { - return tables.stream().filter(t -> t.confidence() >= TABULA_CONFIDENCE_THRESHOLD).toList(); - } -} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java deleted file mode 100644 index b2d8de516..000000000 --- a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParser.java +++ /dev/null @@ -1,528 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.regex.Pattern; - -import org.apache.pdfbox.pdmodel.PDDocument; -import org.springframework.stereotype.Service; - -import lombok.extern.slf4j.Slf4j; - -/** - * Fallback {@link TableParser} for borderless financial tables using text geometry. - * - *

Identifies "anchor lines" (≥2 numeric tokens), builds a column grid from their right-edge - * positions, groups vertically proximate anchor lines into table candidates, then scores each group - * on column consistency and anchor density (confidence ceiling 0.85). - */ -@Service -@Slf4j -public class LineAlignmentTableParser implements TableParser { - - /** Width in points of each column position bucket. */ - static final float COLUMN_BUCKET_PT = 5f; - - /** Tolerance in buckets when matching a token's right-edge to a confirmed column position. */ - private static final int COLUMN_MATCH_BUCKETS = 2; - - /** Maximum gap (as a multiple of modal line spacing) before splitting a group. */ - private static final float MAX_GAP_FACTOR = 2.5f; - - /** Minimum anchor rows (numeric-heavy) to form a valid table. */ - static final int MIN_TABLE_ROWS = 3; - - /** Minimum confirmed column positions to form a valid table. */ - static final int MIN_COLUMNS = 2; - - /** - * Min fraction of anchor lines a column must appear on to be confirmed (permissive for N/A - * rows). - */ - private static final double COLUMN_MIN_FREQUENCY = 0.40; - - /** - * Matches financial numeric tokens: integers, decimals, parenthetical negatives, currency, - * percent, nil dashes. - */ - private static final Pattern NUMERIC = - Pattern.compile("^[\\(\\-\\$£€¥]?\\d[\\d,\\.]*[\\)%]?$|^[-–—]$"); - - /** - * Lines within this y-distance are merged into one row (restores rows split by LineBuilder's - * column-gap logic). - */ - static final float ROW_MERGE_TOLERANCE_PT = 2f; - - // ── public API ─────────────────────────────────────────────────────────────────────────────── - - @Override - public List parse(PDDocument document, RawPage rawPage) throws IOException { - List lines = rawPage.lines(); - if (lines.size() < MIN_TABLE_ROWS) return List.of(); - - float modalSpacing = computeModalSpacing(lines); - List tokenized = - mergeCoincidentLines(lines.stream().map(this::tokenize).toList()); - - List anchors = tokenized.stream().filter(TokenizedLine::isAnchor).toList(); - - if (anchors.size() < MIN_TABLE_ROWS) return List.of(); - - List columnGrid = buildColumnGrid(anchors); - if (columnGrid.size() < MIN_COLUMNS) { - log.debug( - "Page {}: LineAlignment — fewer than {} confirmed columns, skipping", - rawPage.pageNumber(), - MIN_COLUMNS); - return List.of(); - } - - List> groups = groupRows(tokenized, columnGrid, modalSpacing); - - List results = new ArrayList<>(); - for (int i = 0; i < groups.size(); i++) { - buildFragment(groups.get(i), columnGrid, rawPage.pageNumber(), i) - .ifPresent(results::add); - } - - log.debug( - "Page {}: LineAlignment detected {} table(s) ({} anchor lines, {} columns)", - rawPage.pageNumber(), - results.size(), - anchors.size(), - columnGrid.size()); - return results; - } - - // ── coincident-line merging ────────────────────────────────────────────────────────────────── - - /** - * Merges tokenised lines sharing the same y-position into one row, rejoining label/value halves - * split by LineBuilder. - */ - List mergeCoincidentLines(List tokenized) { - if (tokenized.size() < 2) return tokenized; - - List result = new ArrayList<>(); - int i = 0; - - while (i < tokenized.size()) { - float baseY = tokenized.get(i).line().bounds().y(); - int j = i + 1; - while (j < tokenized.size() - && Math.abs(tokenized.get(j).line().bounds().y() - baseY) - <= ROW_MERGE_TOLERANCE_PT) { - j++; - } - - if (j == i + 1) { - result.add(tokenized.get(i)); - } else { - result.add(mergeGroup(tokenized.subList(i, j))); - } - i = j; - } - - return result; - } - - private TokenizedLine mergeGroup(List group) { - List mergedFragments = - group.stream() - .flatMap(tl -> tl.line().fragments().stream()) - .sorted(Comparator.comparingDouble(f -> f.bounds().x())) - .toList(); - - Bounds mergedBounds = - group.stream() - .map(tl -> tl.line().bounds()) - .reduce(Bounds::merge) - .orElse(group.get(0).line().bounds()); - - RawLine mergedLine = - new RawLine( - group.get(0).line().lineId(), - mergedFragments, - mergedBounds, - group.get(0).line().pageNumber()); - - return tokenize(mergedLine); - } - - // ── tokenisation ───────────────────────────────────────────────────────────────────────────── - - /** - * Splits fragments into word-level tokens; x-positions are estimated linearly within each - * fragment. - */ - TokenizedLine tokenize(RawLine line) { - List tokens = new ArrayList<>(); - for (TextFragment frag : line.fragments()) { - tokens.addAll(tokensFromFragment(frag)); - } - List numeric = tokens.stream().filter(LineToken::numeric).toList(); - return new TokenizedLine(line, tokens, numeric); - } - - private List tokensFromFragment(TextFragment frag) { - String raw = frag.text(); - if (raw == null || raw.isBlank()) return List.of(); - - float fragX = frag.bounds().x(); - float fragWidth = frag.bounds().width(); - int rawLen = raw.length(); - - List result = new ArrayList<>(); - int offset = 0; - for (String part : raw.split("\\s+")) { - if (part.isEmpty()) { - offset++; - continue; - } - int idx = raw.indexOf(part, offset); - if (idx < 0) idx = offset; - - float tokenX = rawLen > 0 ? fragX + ((float) idx / rawLen) * fragWidth : fragX; - float tokenRight = - rawLen > 0 - ? fragX + ((float) (idx + part.length()) / rawLen) * fragWidth - : fragX + fragWidth; - - result.add(new LineToken(part, tokenX, tokenRight, NUMERIC.matcher(part).matches())); - offset = idx + part.length(); - } - return result; - } - - // ── column grid ────────────────────────────────────────────────────────────────────────────── - - /** - * Returns confirmed column right-edge positions — those appearing on ≥ {@value - * #COLUMN_MIN_FREQUENCY} × N anchor lines. - */ - private List buildColumnGrid(List anchors) { - // bucket → set of line indices that contributed a numeric token to that bucket - Map> bucketLines = new HashMap<>(); - for (int i = 0; i < anchors.size(); i++) { - for (LineToken t : anchors.get(i).numeric()) { - int bucket = bucket(t.right()); - bucketLines.computeIfAbsent(bucket, k -> new ArrayList<>()).add(i); - } - } - - int minHits = - Math.max(MIN_TABLE_ROWS, (int) Math.ceil(anchors.size() * COLUMN_MIN_FREQUENCY)); - - // Confirmed buckets → average right-edge for that bucket - TreeMap confirmed = new TreeMap<>(); - for (Map.Entry> entry : bucketLines.entrySet()) { - // Count distinct lines - long distinctLines = entry.getValue().stream().distinct().count(); - if (distinctLines >= minHits) { - double avg = - entry.getValue().stream() - .distinct() // weight each line equally regardless of token count - .mapToDouble( - lineIdx -> - avgRightEdgeForBucket( - anchors, lineIdx, entry.getKey())) - .average() - .orElse(entry.getKey() * (double) COLUMN_BUCKET_PT); - confirmed.put(entry.getKey(), (float) avg); - } - } - - return new ArrayList<>(confirmed.values()); // already sorted by bucket (left to right) - } - - /** - * Returns the average right-edge position of tokens in {@code line} whose bucket matches {@code - * targetBucket}, falling back to the bucket's nominal centre when no tokens match. - */ - private double avgRightEdgeForBucket( - List anchors, int lineIdx, int targetBucket) { - return anchors.get(lineIdx).numeric().stream() - .filter(t -> bucket(t.right()) == targetBucket) - .mapToDouble(LineToken::right) - .average() - .orElse(targetBucket * (double) COLUMN_BUCKET_PT); - } - - // ── grouping ───────────────────────────────────────────────────────────────────────────────── - - /** - * Groups anchor lines into table candidates, including adjacent label rows; a gap > - * MAX_GAP_FACTOR × modal spacing splits groups. - */ - private List> groupRows( - List all, List columnGrid, float modalSpacing) { - float maxGap = modalSpacing > 0 ? modalSpacing * MAX_GAP_FACTOR : 30f; - - List> groups = new ArrayList<>(); - List current = new ArrayList<>(); - - for (int i = 0; i < all.size(); i++) { - TokenizedLine tl = all.get(i); - boolean fits = tl.isAnchor() && matchesGrid(tl, columnGrid); - - if (current.isEmpty()) { - if (fits) current.add(tl); - continue; - } - - float gap = - tl.line().bounds().y() - - current.get(current.size() - 1).line().bounds().bottom(); - - if (gap > maxGap) { - groups.add(current); - current = new ArrayList<>(); - if (fits) current.add(tl); - continue; - } - - if (fits) { - current.add(tl); - } else if (!tl.line().text().isBlank()) { - // Include non-anchor lines (labels) only if they have text and are within - // proximity. - current.add(tl); - } - } - - if (!current.isEmpty()) groups.add(current); - - return groups.stream().filter(g -> hasEnoughAnchorRows(g, columnGrid)).toList(); - } - - private boolean hasEnoughAnchorRows(List group, List columnGrid) { - return group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count() - >= MIN_TABLE_ROWS; - } - - /** A line "matches" the grid when ≥ 60 % of its numeric tokens land in confirmed columns. */ - private boolean matchesGrid(TokenizedLine tl, List columnGrid) { - if (tl.numeric().isEmpty()) return false; - long matches = - tl.numeric().stream() - .filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0) - .count(); - return (double) matches / tl.numeric().size() >= 0.60; - } - - private boolean hasInconsistentColumnMatch(TokenizedLine tl, List columnGrid) { - if (tl.numeric().isEmpty()) return false; - long hits = - tl.numeric().stream() - .filter(t -> nearestColumnIndex(t.right(), columnGrid) >= 0) - .count(); - return (double) hits / tl.numeric().size() < 0.60; - } - - // ── fragment assembly ──────────────────────────────────────────────────────────────────────── - - private Optional buildFragment( - List group, List columnGrid, int pageNumber, int tableIndex) { - - long anchorCount = - group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count(); - if (anchorCount < MIN_TABLE_ROWS) return Optional.empty(); - - List warnings = new ArrayList<>(); - List> rawRows = new ArrayList<>(); - List rows = new ArrayList<>(); - - for (int rowIdx = 0; rowIdx < group.size(); rowIdx++) { - TokenizedLine tl = group.get(rowIdx); - List rawRow = buildRawRow(tl, columnGrid); - rawRows.add(Collections.unmodifiableList(rawRow)); - rows.add(buildTableRow(rowIdx, tl, rawRow, columnGrid)); - } - - // Column count = 1 label column + confirmed numeric columns - int colCount = columnGrid.size() + 1; - Bounds bounds = computeGroupBounds(group); - float confidence = computeConfidence(group, columnGrid, warnings); - - return Optional.of( - new TableFragment( - "tbl-la-p" + pageNumber + "-" + tableIndex, - pageNumber, - bounds, - List.of(), - Collections.unmodifiableList(rows), - Collections.unmodifiableList(rawRows), - colCount, - confidence, - Collections.unmodifiableList(warnings), - null)); - } - - /** - * Builds a raw row as a list of strings: index 0 = label text, indices 1..N = column values. - */ - private List buildRawRow(TokenizedLine tl, List columnGrid) { - String[] cells = new String[columnGrid.size() + 1]; - Arrays.fill(cells, ""); - - // Separate label tokens (those not landing in any confirmed column) from column tokens. - List labelParts = new ArrayList<>(); - for (LineToken token : tl.all()) { - int col = nearestColumnIndex(token.right(), columnGrid); - if (col >= 0 && token.numeric()) { - int cellIdx = col + 1; - cells[cellIdx] = - cells[cellIdx].isEmpty() - ? token.text() - : cells[cellIdx] + " " + token.text(); - } else { - labelParts.add(token.text()); - } - } - cells[0] = String.join(" ", labelParts).trim(); - return Arrays.asList(cells); - } - - private TableRow buildTableRow( - int rowIdx, TokenizedLine tl, List rawRow, List columnGrid) { - List cells = new ArrayList<>(rawRow.size()); - - // Label cell: use the line's full bounds as an approximation. - cells.add(TableCell.of(0, rawRow.get(0), tl.line().bounds())); - - for (int col = 0; col < columnGrid.size(); col++) { - String text = col + 1 < rawRow.size() ? rawRow.get(col + 1) : ""; - float right = columnGrid.get(col); - float left = col > 0 ? columnGrid.get(col - 1) : right - 50f; - Bounds cellBounds = - new Bounds( - left, - tl.line().bounds().y(), - right - left, - tl.line().bounds().height()); - cells.add(TableCell.of(col + 1, text, cellBounds)); - } - return new TableRow(rowIdx, Collections.unmodifiableList(cells)); - } - - // ── confidence scoring ─────────────────────────────────────────────────────────────────────── - - /** - * Heuristic score in [0.0, 0.85] (ceiling keeps results below Tabula lattice which starts at - * 1.0). Base 0.70; +0.05/col beyond 2 (max +0.10); +0.05 at ≥5 anchors, +0.05 at ≥8; −0.15 if - * >30 % of anchors have inconsistent columns; −0.10 if non-anchors outnumber anchors. - */ - private float computeConfidence( - List group, List columnGrid, List warnings) { - float score = 0.70f; - - long anchorCount = - group.stream().filter(r -> r.isAnchor() && matchesGrid(r, columnGrid)).count(); - long totalRows = group.size(); - - // More columns - int extraCols = Math.min(columnGrid.size() - MIN_COLUMNS, 2); - score += extraCols * 0.05f; - - // More anchor rows - if (anchorCount >= 5) score += 0.05f; - if (anchorCount >= 8) score += 0.05f; - - // Inconsistent column matching - long inconsistent = - group.stream() - .filter(TokenizedLine::isAnchor) - .filter(tl -> hasInconsistentColumnMatch(tl, columnGrid)) - .count(); - if (inconsistent > anchorCount * 0.30) { - score -= 0.15f; - warnings.add( - "Column match inconsistent on " - + inconsistent - + "/" - + anchorCount - + " anchor rows"); - } - - // Label-heavy - long nonAnchor = totalRows - anchorCount; - if (nonAnchor > anchorCount) { - score -= 0.10f; - warnings.add( - "Non-anchor rows (" - + nonAnchor - + ") outnumber anchor rows (" - + anchorCount - + ")"); - } - - return Math.max(0f, Math.min(0.85f, score)); - } - - // ── utility ────────────────────────────────────────────────────────────────────────────────── - - /** - * Returns the grid index nearest to {@code rightEdge}, or -1 if none is within {@value - * #COLUMN_MATCH_BUCKETS} buckets. - */ - private int nearestColumnIndex(float rightEdge, List grid) { - int nearest = -1; - float minDist = COLUMN_MATCH_BUCKETS * COLUMN_BUCKET_PT + 1f; - for (int i = 0; i < grid.size(); i++) { - float dist = Math.abs(rightEdge - grid.get(i)); - if (dist < minDist) { - minDist = dist; - nearest = i; - } - } - return nearest; - } - - private Bounds computeGroupBounds(List group) { - return group.stream() - .map(tl -> tl.line().bounds()) - .reduce(Bounds::merge) - .orElse(new Bounds(0, 0, 0, 0)); - } - - /** Modal gap between consecutive line edges, used to calibrate the group-split threshold. */ - private float computeModalSpacing(List lines) { - if (lines.size() < 2) return 0f; - Map freq = new HashMap<>(); - for (int i = 1; i < lines.size(); i++) { - float gap = lines.get(i).bounds().y() - lines.get(i - 1).bounds().bottom(); - if (gap > 0) freq.merge(Math.round(gap / 2f) * 2f, 1L, Long::sum); - } - return freq.entrySet().stream() - .max(Map.Entry.comparingByValue()) - .map(Map.Entry::getKey) - .orElse(0f); - } - - private static int bucket(float x) { - return Math.round(x / COLUMN_BUCKET_PT); - } - - // ── private data types ─────────────────────────────────────────────────────────────────────── - - /** A word-level token with an approximate right-edge x-position. */ - record LineToken(String text, float x, float right, boolean numeric) {} - - /** A {@link RawLine} with tokens pre-computed; an "anchor" has ≥ 2 numeric tokens. */ - record TokenizedLine(RawLine line, List all, List numeric) { - boolean isAnchor() { - return numeric.size() >= 2; - } - } -} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java deleted file mode 100644 index 6831f6d73..000000000 --- a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/LineBuilder.java +++ /dev/null @@ -1,139 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - -import org.springframework.stereotype.Service; - -import lombok.extern.slf4j.Slf4j; - -/** - * Groups {@link TextFragment} objects into visual {@link RawLine}s using baseline proximity. - * - *

Fragments are on the same line when their baselines are within a font-size-derived tolerance. - * A new line starts whenever the horizontal gap exceeds an adaptive column-gap threshold ({@code - * max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT)}), splitting two-column text. - */ -@Service -@Slf4j -public class LineBuilder { - - /** Baseline tolerance as a fraction of font size; 0.5 keeps mixed-size text on one line. */ - private static final float BASELINE_TOLERANCE_FACTOR = 0.5f; - - /** Absolute minimum tolerance so tiny font sizes don't collapse multi-line content. */ - private static final float MIN_BASELINE_TOLERANCE = 2f; - - /** - * Column-gap threshold as a fraction of page width; 0.10 clears tab stops but stays below - * two-column gutters. - */ - static final float COLUMN_GAP_RATIO = 0.10f; - - /** Floor for the column-gap threshold so narrow pages don't over-split lines. */ - static final float COLUMN_GAP_MIN_PT = 40f; - - public List build(List fragments, int pageNumber) { - if (fragments.isEmpty()) return List.of(); - - float effectiveWidth = inferEffectiveWidth(fragments); - float columnGapThreshold = Math.max(effectiveWidth * COLUMN_GAP_RATIO, COLUMN_GAP_MIN_PT); - log.debug( - "LineBuilder page {}: effectiveWidth={:.1f}pt, columnGapThreshold={:.1f}pt", - pageNumber, - effectiveWidth, - columnGapThreshold); - - // Sort top-to-bottom first, then left-to-right within the same baseline band. - List sorted = - fragments.stream() - .sorted( - Comparator.comparingDouble(TextFragment::baseline) - .thenComparingDouble(f -> f.bounds().x())) - .toList(); - - List> groups = groupByBaseline(sorted, columnGapThreshold); - - List lines = new ArrayList<>(groups.size()); - for (int i = 0; i < groups.size(); i++) { - List group = - groups.get(i).stream() - .sorted(Comparator.comparingDouble(f -> f.bounds().x())) - .toList(); - - Bounds lineBounds = - group.stream() - .map(TextFragment::bounds) - .reduce(Bounds::merge) - .orElse(new Bounds(0, 0, 0, 0)); - - lines.add(new RawLine("ln-p" + pageNumber + "-" + i, group, lineBounds, pageNumber)); - } - return lines; - } - - private List> groupByBaseline( - List sorted, float columnGapThreshold) { - List> groups = new ArrayList<>(); - List current = new ArrayList<>(); - float currentBaseline = Float.NaN; - - for (TextFragment fragment : sorted) { - if (current.isEmpty()) { - current.add(fragment); - currentBaseline = fragment.baseline(); - continue; - } - - float maxFontSize = - Math.max( - fragment.fontSize(), - (float) - current.stream() - .mapToDouble(TextFragment::fontSize) - .max() - .orElse(0)); - float tolerance = - Math.max(maxFontSize * BASELINE_TOLERANCE_FACTOR, MIN_BASELINE_TOLERANCE); - - boolean sameBaseline = Math.abs(fragment.baseline() - currentBaseline) <= tolerance; - boolean columnGap = sameBaseline && hasColumnGap(fragment, current, columnGapThreshold); - - if (sameBaseline && !columnGap) { - current.add(fragment); - // Anchor to the weighted mean baseline so long lines stay stable. - currentBaseline = - (currentBaseline * (current.size() - 1) + fragment.baseline()) - / current.size(); - } else { - groups.add(current); - current = new ArrayList<>(); - current.add(fragment); - currentBaseline = fragment.baseline(); - } - } - - if (!current.isEmpty()) groups.add(current); - return groups; - } - - /** - * True when the gap from the rightmost fragment in {@code group} to {@code next} exceeds {@code - * threshold}. - */ - private static boolean hasColumnGap( - TextFragment next, List group, float threshold) { - float lastRight = group.get(group.size() - 1).bounds().right(); - return next.bounds().x() - lastRight > threshold; - } - - /** Infers effective page width from the rightmost fragment right-edge plus a 10 % margin. */ - private static float inferEffectiveWidth(List fragments) { - double maxRight = - fragments.stream().mapToDouble(f -> f.bounds().right()).max().orElse(500.0); - return (float) maxRight * 1.10f; - } -} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java deleted file mode 100644 index a7dc9c282..000000000 --- a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/PdfIngester.java +++ /dev/null @@ -1,79 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.common.PDRectangle; -import org.springframework.stereotype.Service; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** - * Runs the per-page ingestion pipeline: {@link WordExtractingStripper} → {@link LineBuilder} → - * {@link TableParser}, producing a {@link PdfModels.ParsedPage} per page. The caller owns the - * {@link PDDocument} lifecycle. - */ -@Service -@RequiredArgsConstructor -@Slf4j -public class PdfIngester { - - private final LineBuilder lineBuilder; - private final TableParser tableParser; - - public List parse(PDDocument document) throws IOException { - return parse(document, document.getNumberOfPages()); - } - - public List parse(PDDocument document, int maxPages) throws IOException { - int pageCount = Math.min(document.getNumberOfPages(), maxPages); - List pages = new ArrayList<>(pageCount); - long fragmentsMs = 0; - long tablesMs = 0; - long t0 = System.currentTimeMillis(); - - for (int p = 1; p <= pageCount; p++) { - long ft = System.currentTimeMillis(); - List fragments = extractFragments(document, p); - fragmentsMs += System.currentTimeMillis() - ft; - - PDPage page = document.getPage(p - 1); - PDRectangle mediaBox = page.getMediaBox(); - List lines = lineBuilder.build(fragments, p); - RawPage rawPage = new RawPage(p, mediaBox.getWidth(), mediaBox.getHeight(), lines); - - long tt = System.currentTimeMillis(); - List tables = tableParser.parse(document, rawPage); - tablesMs += System.currentTimeMillis() - tt; - - log.debug( - "Page {}: {} fragments → {} lines, {} table(s)", - p, - fragments.size(), - lines.size(), - tables.size()); - pages.add(new ParsedPage(p, mediaBox.getWidth(), mediaBox.getHeight(), tables, lines)); - } - - log.info( - "[timing] parse pages={} total={}ms fragments={}ms tables={}ms", - pageCount, - System.currentTimeMillis() - t0, - fragmentsMs, - tablesMs); - return pages; - } - - private List extractFragments(PDDocument document, int pageNumber) - throws IOException { - WordExtractingStripper stripper = new WordExtractingStripper(pageNumber); - stripper.getText(document); - return stripper.getFragments(); - } -} diff --git a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java b/app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java deleted file mode 100644 index 52ab9d9a1..000000000 --- a/app/common/src/main/java/stirling/software/SPDF/pdf/parser/WordExtractingStripper.java +++ /dev/null @@ -1,113 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.font.PDFont; -import org.apache.pdfbox.text.PDFTextStripper; -import org.apache.pdfbox.text.TextPosition; - -/** - * Extends {@link PDFTextStripper} to capture per-fragment geometry and font metadata. - * - *

Overrides {@link #writeString} to split each content-stream string into word-level {@link - * TextFragment}s with bounding boxes, baseline, font name, and bold flag. Coordinates are in - * PDFTextStripper space: (0,0) top-left, Y increases downward, {@code getY()} is the baseline. - */ -class WordExtractingStripper extends PDFTextStripper { - - private final int targetPage; - private final List fragments = new ArrayList<>(); - private int fragmentIndex = 0; - - WordExtractingStripper(int pageNumber) throws IOException { - this.targetPage = pageNumber; - setStartPage(pageNumber); - setEndPage(pageNumber); - setSortByPosition(true); - } - - @Override - protected void startPage(PDPage page) throws IOException { - super.startPage(page); - fragments.clear(); - fragmentIndex = 0; - } - - @Override - protected void writeString(String text, List textPositions) throws IOException { - if (text == null || text.isBlank()) return; - - // Fast path: no whitespace → emit one fragment (most financial PDFs have each - // number as its own string operation, so this is the common case). - if (text.indexOf(' ') < 0) { - emitFragment(text, textPositions); - return; - } - - // Per-word splitting requires 1:1 text-char to TextPosition correspondence. - // Fall back to one fragment when sizes differ (ligatures, encoding edge cases). - if (textPositions.size() != text.length()) { - emitFragment(text, textPositions); - return; - } - - // Emit one TextFragment per whitespace-delimited word with accurate per-word bounds. - int start = 0; - for (int i = 0; i <= text.length(); i++) { - if (i == text.length() || text.charAt(i) == ' ') { - if (start < i) { - emitFragment(text.substring(start, i), textPositions.subList(start, i)); - } - start = i + 1; - } - } - } - - private void emitFragment(String text, List positions) { - if (positions.isEmpty()) return; - - float minX = Float.MAX_VALUE; - float minY = Float.MAX_VALUE; - float maxRight = -Float.MAX_VALUE; - float maxBaseline = -Float.MAX_VALUE; - TextPosition first = null; - - for (TextPosition tp : positions) { - if (tp == null) continue; - if (first == null) first = tp; - - float x = tp.getX(); - // getY() is the baseline; top of character = getY() - getHeight(). - float top = tp.getY() - tp.getHeight(); - float right = x + tp.getWidth(); - float baseline = tp.getY(); - - minX = Math.min(minX, x); - minY = Math.min(minY, top); - maxRight = Math.max(maxRight, right); - maxBaseline = Math.max(maxBaseline, baseline); - } - - if (first == null) return; - - PDFont font = first.getFont(); - String fontName = font != null ? font.getName() : ""; - boolean bold = fontName != null && fontName.toLowerCase().contains("bold"); - // getHeight() gives the rendered glyph height, which is the most reliable visual size. - float fontSize = first.getHeight(); - - Bounds bounds = new Bounds(minX, minY, maxRight - minX, maxBaseline - minY); - String id = "tf-p" + targetPage + "-" + fragmentIndex++; - fragments.add(new TextFragment(id, text, bounds, maxBaseline, fontSize, fontName, bold)); - } - - List getFragments() { - return Collections.unmodifiableList(fragments); - } -} diff --git a/app/common/src/main/java/stirling/software/common/pdf/HeadingDetector.java b/app/common/src/main/java/stirling/software/common/pdf/HeadingDetector.java new file mode 100644 index 000000000..0937cef64 --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/pdf/HeadingDetector.java @@ -0,0 +1,191 @@ +package stirling.software.common.pdf; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import stirling.software.jpdfium.text.PageText; +import stirling.software.jpdfium.text.TextChar; +import stirling.software.jpdfium.text.TextLine; +import stirling.software.jpdfium.text.TextWord; + +final class HeadingDetector { + + private HeadingDetector() {} + + /** A heading is at most this many words; longer lines are treated as body text. */ + private static final int MAX_HEADING_WORDS = 12; + + /** + * Returns the Markdown heading prefix for a line. The decision combines several signals, never + * text matching, so a plain line that merely shares text with a heading is never promoted: + * + *

    + *
  • Size — dominant glyph font size vs. the document body median (primary signal). + * Some PDFs encode visual size in the text matrix, so every glyph reports ~1.0; for those + * the line height is used as the proxy instead. + *
  • Brevity — headings are short labels; a line over {@value #MAX_HEADING_WORDS} + * words is body text regardless of size. + *
  • Not a sentence — a line ending in {@code . ! ?} reads as prose, not a heading. + *
+ * + *

Boldness is deliberately not a heading signal — a bold-but-not-larger line is + * emphasis, not a heading (see {@link #isBoldLabel}); promoting it to {@code #}/{@code ##} is + * the main source of false-positive headings. + * + *

    + *
  • size > baseline * 1.4 → {@code "# "} + *
  • size > baseline * 1.2 → {@code "## "} + *
  • otherwise → {@code ""} + *
+ */ + static String headingPrefix(TextLine line, float medianBodySize, float medianBodyHeight) { + String text = line.text().strip(); + if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) { + return ""; + } + + float dominant = dominantFontSize(line); + float value; + float baseline; + if (dominant > 2f && medianBodySize > 2f) { + value = dominant; + baseline = medianBodySize; + } else { + value = line.height(); + baseline = medianBodyHeight; + } + if (baseline <= 0f) { + return ""; + } + + float ratio = value / baseline; + if (ratio > 1.4f) { + return "# "; + } + if (ratio > 1.2f) { + return "## "; + } + return ""; + } + + /** + * True when a line should be emphasised as bold (rendered {@code **like this**}) rather than + * promoted to a heading: it is bold, short, and not a full sentence. Used for bold labels that + * are not large enough to be headings. + */ + static boolean isBoldLabel(TextLine line) { + String text = line.text().strip(); + if (text.isEmpty() || wordCount(text) > MAX_HEADING_WORDS || endsLikeSentence(text)) { + return false; + } + return isBold(line); + } + + private static int wordCount(String text) { + return text.split("\\s+").length; + } + + private static boolean endsLikeSentence(String text) { + char last = text.charAt(text.length() - 1); + return last == '.' || last == '!' || last == '?'; + } + + /** True when the line's dominant font is bold, inferred from PostScript font names. */ + private static boolean isBold(TextLine line) { + Map counts = new HashMap<>(); + for (TextWord word : line.words()) { + for (TextChar ch : word.chars()) { + if (ch.isWhitespace() || ch.isNewline()) { + continue; + } + String name = ch.fontName(); + if (name != null && !name.isBlank()) { + counts.merge(name, 1, Integer::sum); + } + } + } + String dominantFont = ""; + int max = -1; + for (Map.Entry e : counts.entrySet()) { + if (e.getValue() > max) { + max = e.getValue(); + dominantFont = e.getKey(); + } + } + String lower = dominantFont.toLowerCase(java.util.Locale.ROOT); + return lower.contains("bold") + || lower.contains("black") + || lower.contains("heavy") + || lower.contains("semibold"); + } + + /** Computes the median glyph font size across all pages. */ + static float medianFontSize(List allPages) { + List sizes = new ArrayList<>(); + for (PageText page : allPages) { + for (TextChar ch : page.chars()) { + if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) { + sizes.add(ch.fontSize()); + } + } + } + return median(sizes, 12f); + } + + /** Computes the median TextLine height across all pages. Used when font size is degenerate. */ + static float medianLineHeight(List allPages) { + List heights = new ArrayList<>(); + for (PageText page : allPages) { + for (TextLine line : page.lines()) { + if (line.height() > 0f && !line.text().isBlank()) { + heights.add(line.height()); + } + } + } + return median(heights, 12f); + } + + private static float median(List values, float fallback) { + if (values.isEmpty()) { + return fallback; + } + Collections.sort(values); + int mid = values.size() / 2; + if (values.size() % 2 == 0) { + return (values.get(mid - 1) + values.get(mid)) / 2f; + } + return values.get(mid); + } + + /** + * Returns the font size that appears most often (by character count) in the given line. Ties + * are broken in favour of the larger size. + */ + private static float dominantFontSize(TextLine line) { + Map counts = new HashMap<>(); + for (TextWord word : line.words()) { + for (TextChar ch : word.chars()) { + if (!ch.isWhitespace() && !ch.isNewline() && ch.fontSize() > 0f) { + counts.merge(ch.fontSize(), 1, Integer::sum); + } + } + } + if (counts.isEmpty()) { + return 0f; + } + float dominant = 0f; + int maxCount = -1; + for (Map.Entry entry : counts.entrySet()) { + int count = entry.getValue(); + float size = entry.getKey(); + if (count > maxCount || (count == maxCount && size > dominant)) { + maxCount = count; + dominant = size; + } + } + return dominant; + } +} diff --git a/app/common/src/main/java/stirling/software/common/pdf/PdfMarkdownConverter.java b/app/common/src/main/java/stirling/software/common/pdf/PdfMarkdownConverter.java new file mode 100644 index 000000000..c19468b5e --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/pdf/PdfMarkdownConverter.java @@ -0,0 +1,1043 @@ +package stirling.software.common.pdf; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import stirling.software.jpdfium.PdfDocument; +import stirling.software.jpdfium.PdfPage; +import stirling.software.jpdfium.doc.ExtractedImage; +import stirling.software.jpdfium.doc.PdfImageExtractor; +import stirling.software.jpdfium.model.Rect; +import stirling.software.jpdfium.text.PageText; +import stirling.software.jpdfium.text.PdfTableExtractor; +import stirling.software.jpdfium.text.PdfTextExtractor; +import stirling.software.jpdfium.text.Table; +import stirling.software.jpdfium.text.TextLine; +import stirling.software.jpdfium.text.TextWord; + +/** + * Converts a PDF to Markdown using a TextLine-driven body pipeline. + * + *

Body text is rebuilt from {@link PdfTextExtractor} {@link TextLine}s. TextLines group words + * faithfully and keep paragraph order, so the only pre-processing needed is stitching narrow + * standalone glyph fragments (apostrophes, quotes, asterisks, superscript footnote markers, + * bullets) back into the line they belong to. Column layout and tables are derived from line/word + * geometry directly. + */ +public class PdfMarkdownConverter { + + private static final Pattern SOFT_HYPHEN = Pattern.compile("(\\w+)-\\n([a-z])"); + + /** Width below which a TextLine is treated as a stray glyph fragment to be stitched. */ + private static final float GLYPH_WIDTH = 7.5f; + + public String convert(PdfDocument doc) throws IOException { + List allPageText = PdfTextExtractor.extractAll(doc); + float medianSize = HeadingDetector.medianFontSize(allPageText); + float medianHeight = HeadingDetector.medianLineHeight(allPageText); + + int pageCount = doc.pageCount(); + // Elements are either rendered text (String) or a structured TableBlock. Tables stay + // structured until after the page loop so a table split across a page break can be stitched + // back together before rendering. + List output = new ArrayList<>(); + // Header text of a table that ended the previous page, used to spot a continuation whose + // header repeats at the top of the current page. Null when the previous page did not end in + // a table. + String prevPageTrailingTableHeader = null; + + for (int pageIndex = 0; pageIndex < pageCount; pageIndex++) { + List rawLines = + pageIndex < allPageText.size() ? allPageText.get(pageIndex).lines() : List.of(); + + // Stitch stray glyph fragments (apostrophes, asterisks, superscripts, bullets) into + // their host lines so paragraph assembly sees faithful, complete lines. + List lines = stitchGlyphs(rawLines); + if (lines.isEmpty()) { + emitImages(doc, pageIndex, output); + prevPageTrailingTableHeader = null; + continue; + } + + // Sort top-to-bottom (PDF y=0 is the bottom of the page). + lines.sort(Comparator.comparingDouble((Line l) -> l.y).reversed()); + + // Multi-column guard: only genuine two-column prose should be split. A table's column + // gutters must NOT be mistaken for a page-layout gutter, so this looks at whether row + // lines span the gutter (table) or stay within one side (two-column prose). + // A table that ran to the bottom of the previous page and repeats its header at the top + // of this page is a continuation, not a new two-column layout. Detecting the repeated + // header keeps this page out of the two-column path so the continuation is rebuilt as a + // table and stitched back onto the previous block. + final String continuationHeader = prevPageTrailingTableHeader; + boolean tableContinuation = + continuationHeader != null + && lines.stream() + .anyMatch( + l -> normaliseSpace(l.text).equals(continuationHeader)); + + boolean twoColumn = !tableContinuation && detectsTwoColumns(lines); + + // Tables are detected from text/word geometry (the word-grid detector), which handles + // both ruled and borderless tables and places cells by column alignment. The native + // ruled-line extractor is not used: it both mis-renders cells and double-emits rows. + Set tableRowTexts = new HashSet<>(); + List blocks = twoColumn ? List.of() : findTableBlocks(lines); + Set tableLines = new HashSet<>(); + for (TableBlock b : blocks) { + for (List row : b.rows()) { + for (Line l : row) { + tableLines.add(l); + tableRowTexts.add(repairHyphens(l.text).strip()); + } + } + } + + List pageItems = new ArrayList<>(); + if (twoColumn) { + for (List col : splitIntoColumns(lines)) { + List paras = new ArrayList<>(); + assembleParagraphs(col, medianSize, medianHeight, paras, tableRowTexts); + pageItems.addAll(paras); + } + } else { + // Interleave tables with surrounding text by vertical position. Each block sits in + // its own slot; non-table lines fall into the slot for their y (text above a block, + // between blocks, or below the last). This keeps multiple tables on one page + // separate and in reading order. + List> segments = new ArrayList<>(); + for (int s = 0; s <= blocks.size(); s++) { + segments.add(new ArrayList<>()); + } + for (Line l : lines) { + if (tableLines.contains(l)) { + continue; + } + int slot = 0; + for (TableBlock b : blocks) { + if (b.bottom() > l.y) { + slot++; + } + } + segments.get(slot).add(l); + } + for (int s = 0; s <= blocks.size(); s++) { + List paras = new ArrayList<>(); + assembleParagraphs( + segments.get(s), medianSize, medianHeight, paras, tableRowTexts); + pageItems.addAll(paras); + if (s < blocks.size()) { + pageItems.add(blocks.get(s)); + } + } + } + + emitImages(doc, pageIndex, pageItems); + + if (pageItems.isEmpty()) { + continue; + } + + mergeAcrossPageBoundary(output, pageItems); + output.addAll(pageItems); + prevPageTrailingTableHeader = trailingTableHeader(pageItems); + } + + // Stitch tables split across page breaks, then render every element to Markdown. + List stitched = stitchTables(output); + List rendered = new ArrayList<>(); + for (Object e : stitched) { + rendered.add(e instanceof TableBlock tb ? tb.render() : (String) e); + } + return String.join("\n\n", rendered); + } + + // --- Glyph stitching --------------------------------------------------- + + /** A mutable assembled line: text plus geometry used for ordering and heading detection. */ + private static final class Line { + String text; + float x; + float y; + float width; + float height; + final TextLine source; + + Line(TextLine src) { + this.source = src; + this.text = src.text(); + this.x = src.x(); + this.y = src.y(); + this.width = src.width(); + this.height = src.height(); + } + } + + /** + * Merges narrow glyph fragments (width < {@link #GLYPH_WIDTH}) into the line they belong to. + * + *
    + *
  • A glyph between a left fragment that ends near it and a right fragment that starts near + * it (both on the same baseline) is inserted inline: {@code aren} + {@code '} + {@code t} + * → {@code aren't}. + *
  • A glyph immediately right of a line's end is appended (e.g. superscript footnote marker + * after a number). + *
  • A glyph immediately left of a line's start is prepended (e.g. footnote marker before + * its text). + *
+ */ + private static List stitchGlyphs(List raw) { + List hosts = new ArrayList<>(); + List glyphs = new ArrayList<>(); + for (TextLine l : raw) { + String t = l.text().strip(); + if (t.isEmpty()) { + continue; + } + if (l.width() < GLYPH_WIDTH && t.length() <= 2) { + glyphs.add(l); + } else { + hosts.add(l); + } + } + + List lines = hosts.stream().map(Line::new).collect(Collectors.toList()); + + for (TextLine g : glyphs) { + String gt = g.text().strip(); + if (isBulletGlyph(gt)) { + attachBullet(g, gt, lines); + } else { + attachInlineGlyph(g, gt, lines); + } + } + return lines; + } + + private static boolean isBulletGlyph(String gt) { + return "•".equals(gt) || "▪".equals(gt) || "◦".equals(gt); + } + + /** + * Attaches a bullet glyph to the body line it introduces: the closest line that begins to the + * right of the bullet at roughly the same height or just below it. + */ + private static void attachBullet(TextLine g, String gt, List lines) { + Line best = null; + float bestScore = Float.MAX_VALUE; + for (Line h : lines) { + if (h.x < g.x() - 2f) { + continue; + } + float dy = g.y() - h.y; + if (dy < -4f || dy > 28f) { + continue; + } + float score = Math.abs(dy) + (h.x - g.x()) * 0.2f; + if (score < bestScore) { + bestScore = score; + best = h; + } + } + if (best != null && !best.text.startsWith("•")) { + best.text = "• " + best.text; + best.x = g.x(); + } else { + lines.add(new Line(g)); + } + } + + /** + * Stitches a narrow inline glyph (apostrophe, quote, asterisk, superscript marker) into the + * line it belongs to: inline between two same-baseline fragments, appended to the line that + * ends at it, or prepended to the line that starts at it. + */ + private static void attachInlineGlyph(TextLine g, String gt, List lines) { + Line left = null; + Line right = null; + float lb = 7f; + float rb = 7f; + for (Line h : lines) { + boolean sameBaseline = g.y() >= h.y - 4f && g.y() <= h.y + h.height + 5f; + if (!sameBaseline) { + continue; + } + float rightEdge = h.x + h.width; + float dxLeft = Math.abs(rightEdge - g.x()); + if (dxLeft < lb) { + lb = dxLeft; + left = h; + } + float dxRight = Math.abs(h.x - g.x()); + if (dxRight < rb) { + rb = dxRight; + right = h; + } + } + + if (left != null && right != null && left != right && Math.abs(left.y - right.y) < 6f) { + left.text = left.text + gt + right.text; + left.width = (right.x + right.width) - left.x; + lines.remove(right); + } else if (left != null) { + left.text = left.text + gt; + left.width = Math.max(left.width, g.x() + g.width() - left.x); + } else if (right != null) { + right.text = gt + right.text; + right.x = g.x(); + } else { + lines.add(new Line(g)); + } + } + + // --- Column detection (guard only) ------------------------------------- + + /** + * Returns true when the page is a genuine two-column layout. Uses line/word geometry: body + * blocks (ignoring narrow glyph blocks) and requires a wide horizontal gutter populated on both + * sides, so single apostrophe glyphs cannot create a false second column. + */ + private static boolean detectsTwoColumns(List lines) { + if (lines.size() < 8) { + return false; + } + float minX = Float.MAX_VALUE; + float maxX = -Float.MAX_VALUE; + for (Line l : lines) { + minX = Math.min(minX, l.x); + maxX = Math.max(maxX, l.x + l.width); + } + if (maxX - minX < 200f) { + return false; + } + + // Scan candidate gutter positions across the central band (35%-65% of width) and pick the + // one crossed by the fewest lines. Two-column prose has a gutter that only a handful of + // full-width lines (title, section headings) cross; a table's rows all span the full width, + // so every candidate gutter is crossed by most lines. + float centreLo = minX + (maxX - minX) * 0.35f; + float centreHi = minX + (maxX - minX) * 0.65f; + int bestCrossing = Integer.MAX_VALUE; + int bestLeft = 0; + int bestRight = 0; + for (float gutter = centreLo; gutter <= centreHi; gutter += 2f) { + int crossing = 0; + int leftOnly = 0; + int rightOnly = 0; + for (Line l : lines) { + float lx = l.x; + float rx = l.x + l.width; + if (lx < gutter - 5f && rx > gutter + 5f) { + crossing++; + } else if (rx <= gutter) { + leftOnly++; + } else { + rightOnly++; + } + } + if (crossing < bestCrossing) { + bestCrossing = crossing; + bestLeft = leftOnly; + bestRight = rightOnly; + } + } + + return bestLeft >= 4 && bestRight >= 4 && bestCrossing <= (int) (lines.size() * 0.25f); + } + + private static List> splitIntoColumns(List lines) { + List xs = + lines.stream() + .filter(l -> l.width >= 40f) + .map(l -> l.x) + .sorted() + .collect(Collectors.toList()); + if (xs.isEmpty()) { + return List.of(lines); + } + float minX = xs.get(0); + float maxX = xs.get(xs.size() - 1); + float splitAt = (minX + maxX) / 2f; + float biggestGap = 0; + for (int i = 1; i < xs.size(); i++) { + float gap = xs.get(i) - xs.get(i - 1); + if (gap > biggestGap) { + biggestGap = gap; + splitAt = (xs.get(i - 1) + xs.get(i)) / 2f; + } + } + List left = new ArrayList<>(); + List right = new ArrayList<>(); + for (Line l : lines) { + if (l.x < splitAt) { + left.add(l); + } else { + right.add(l); + } + } + if (left.isEmpty()) { + return List.of(right); + } + if (right.isEmpty()) { + return List.of(left); + } + return List.of(left, right); + } + + // --- Paragraph assembly ------------------------------------------------ + + private static void assembleParagraphs( + List lines, + float medianSize, + float medianHeight, + List out, + Set tableRowTexts) { + StringBuilder para = new StringBuilder(); + float prevBottomY = Float.MAX_VALUE; + float prevHeight = 0f; + + for (Line line : lines) { + String text = repairHyphens(line.text).strip(); + if (text.isEmpty()) { + continue; + } + if (tableRowTexts.contains(text)) { + continue; + } + + float blockTop = line.y + line.height; + float gap = prevBottomY - blockTop; + boolean paragraphBreak = prevHeight > 0f && gap > prevHeight * 0.8f; + + String prefix = HeadingDetector.headingPrefix(line.source, medianSize, medianHeight); + boolean isHeading = !prefix.isEmpty(); + boolean isBullet = startsWithBullet(text); + + if (isHeading) { + flushParagraph(para, out); + out.add(prefix + escapeMarkdown(text)); + } else if (isBullet) { + flushParagraph(para, out); + out.add(escapeMarkdown(text)); + } else if (HeadingDetector.isBoldLabel(line.source)) { + // Bold but not large enough to be a heading → emphasise as bold, don't promote. + flushParagraph(para, out); + out.add("**" + escapeMarkdown(text) + "**"); + } else if (paragraphBreak) { + flushParagraph(para, out); + para.append(text); + } else { + if (!para.isEmpty()) { + char fc = text.charAt(0); + boolean noSpace = fc == '\'' || fc == '’' || fc == '‘' || fc == '"'; + if (!noSpace) { + para.append(' '); + } + } + para.append(text); + } + + prevBottomY = line.y; + prevHeight = line.height; + } + flushParagraph(para, out); + } + + private static boolean startsWithBullet(String text) { + return text.startsWith("•") || text.startsWith("▪") || text.startsWith("◦"); + } + + // --- Word-grid table detection ----------------------------------------- + + /** + * A detected table. Each row is a list of source lines: usually one, but more when a cell wraps + * onto extra lines (those continuation lines are absorbed into the row they belong to). + */ + private record TableBlock(List> rows, float top, float bottom) { + String render() { + return buildTableFromRows(rows); + } + } + + /** + * Detects table blocks on a page. Anchor rows (lines with table-like column gaps) are grouped + * into vertically-contiguous runs separated by large vertical gaps, so multiple separate tables + * on one page stay separate. Non-anchor lines that fall within a run's vertical span are + * treated as wrapped-cell continuations and absorbed into the nearest anchor row above them. + */ + private static List findTableBlocks(List lines) { + List cands = + lines.stream() + .filter(l -> isTableCandidate(l.source)) + .sorted(Comparator.comparingDouble((Line l) -> l.y).reversed()) + .collect(Collectors.toList()); + if (cands.size() < 2) { + return List.of(); + } + + List gaps = new ArrayList<>(); + for (int i = 1; i < cands.size(); i++) { + gaps.add(cands.get(i - 1).y - cands.get(i).y); + } + List sorted = new ArrayList<>(gaps); + sorted.sort(Comparator.naturalOrder()); + float medianGap = sorted.get(sorted.size() / 2); + float splitThreshold = Math.max(medianGap * 2.5f, medianGap + 6f); + + List> anchorGroups = new ArrayList<>(); + List current = new ArrayList<>(); + current.add(cands.get(0)); + for (int i = 1; i < cands.size(); i++) { + float gap = cands.get(i - 1).y - cands.get(i).y; + if (gap > splitThreshold) { + anchorGroups.add(current); + current = new ArrayList<>(); + } + current.add(cands.get(i)); + } + anchorGroups.add(current); + + List nonCandidates = + lines.stream() + .filter(l -> !isTableCandidate(l.source)) + .collect(Collectors.toList()); + + List blocks = new ArrayList<>(); + for (List anchors : anchorGroups) { + if (anchors.size() < 2) { + continue; + } + float top = anchors.get(0).y; + float bottom = anchors.get(anchors.size() - 1).y; + + // Each anchor seeds a row; absorb wrapped continuation lines (non-anchors within the + // run's vertical span, with a little slack below the last row) into the anchor above. + List> rows = new ArrayList<>(); + for (Line a : anchors) { + List row = new ArrayList<>(); + row.add(a); + rows.add(row); + } + for (Line nc : nonCandidates) { + if (nc.y > top || nc.y < bottom - medianGap) { + continue; + } + int owner = 0; + float bestDelta = Float.MAX_VALUE; + for (int i = 0; i < anchors.size(); i++) { + float delta = anchors.get(i).y - nc.y; // positive when anchor is above nc + if (delta >= -1f && delta < bestDelta) { + bestDelta = delta; + owner = i; + } + } + rows.get(owner).add(nc); + } + + if (buildTableFromRows(rows).isBlank()) { + continue; + } + blocks.add(new TableBlock(rows, top, bottom)); + } + return blocks; + } + + private static String buildTableFromRows(List> rowGroups) { + // Detect columns by vertical-whitespace projection across all lines, rather than a 1-D gap + // threshold on pooled word x's. Pooled-gap detection is fragile when numbers are + // right-aligned (a 10-digit value starts well left of a 7-digit one) or when sparse cells + // sit in their own x-band. Projection asks "which x-bands are occupied across many rows", + // which is stable under those conditions. + List flat = rowGroups.stream().flatMap(List::stream).collect(Collectors.toList()); + List columns = findColumnRanges(flat); + if (columns.size() < 2 || columns.size() > 15) { + return ""; + } + + float[] centers = new float[columns.size()]; + for (int i = 0; i < columns.size(); i++) { + centers[i] = (columns.get(i)[0] + columns.get(i)[1]) / 2f; + } + + int cols = centers.length; + List rows = new ArrayList<>(); + for (List rowLines : rowGroups) { + String[] row = new String[cols]; + for (int i = 0; i < cols; i++) { + row[i] = ""; + } + // Top line first so a wrapped cell's words stay in reading order within the cell. + rowLines.sort(Comparator.comparingDouble((Line l) -> l.y).reversed()); + for (Line line : rowLines) { + for (TextWord word : line.source.words()) { + String wt = word.text().strip(); + if (wt.isEmpty()) { + continue; + } + int col = nearestColumn(word.x() + word.width() / 2f, centers); + row[col] = row[col].isEmpty() ? wt : row[col] + " " + wt; + } + } + rows.add(row); + } + + // Guard against false positives while tolerating uneven rows (sparse cells, merged/spanning + // headers). The columns already come from cross-row whitespace alignment, so a stable grid + // exists. Additionally require: at least one "anchor" row that nearly fills the grid (so + // the + // column count is real, not an artefact), and that most rows are genuinely multi-column. + int anchorWidth = Math.max(2, Math.round(cols * 0.6f)); + long anchorRows = rows.stream().filter(r -> filledCells(r) >= anchorWidth).count(); + long multiColumnRows = rows.stream().filter(r -> filledCells(r) >= 2).count(); + if (anchorRows < 1 || multiColumnRows < 2 || multiColumnRows < rows.size() * 0.5) { + return ""; + } + return renderGfm(rows, cols); + } + + /** + * Visible for testing: column detection depends only on word geometry, so tests can drive it + * from synthetic {@link TextLine}s to exercise degenerate-coordinate handling (the crash path + * an extreme text matrix can produce) without needing a binary PDF fixture. + */ + static List findColumnRangesFromLines(List rows) { + return findColumnRanges(rows.stream().map(Line::new).collect(Collectors.toList())); + } + + /** + * Finds column x-ranges by vertical-whitespace projection. Each row contributes coverage for + * the x-bands its words occupy; a column is a contiguous band covered by a sufficient fraction + * of rows, and the gaps between such bands are the gutters. + */ + private static List findColumnRanges(List rows) { + float minX = Float.MAX_VALUE; + float maxX = -Float.MAX_VALUE; + for (Line l : rows) { + for (TextWord w : l.source.words()) { + minX = Math.min(minX, w.x()); + maxX = Math.max(maxX, w.x() + w.width()); + } + } + // Real pages are under ~2000pt wide; anything larger is a malformed/crafted coordinate + // that would allocate a multi-GB array or produce a negative span on overflow. + if (maxX <= minX || (maxX - minX) > 2000f) { + return List.of(); + } + + int lo = (int) Math.floor(minX); + int span = Math.min((int) Math.ceil(maxX) - lo + 1, 2001); + int[] coverage = new int[span]; + for (Line l : rows) { + boolean[] covered = new boolean[span]; + for (TextWord w : l.source.words()) { + int a = Math.max(0, (int) Math.floor(w.x()) - lo); + int b = Math.min(span, (int) Math.ceil(w.x() + w.width()) - lo); + for (int x = a; x < b; x++) { + covered[x] = true; + } + } + for (int x = 0; x < span; x++) { + if (covered[x]) { + coverage[x]++; + } + } + } + + // A column band must be occupied by at least this many rows; below it is gutter. + int support = Math.max(2, Math.round(rows.size() * 0.35f)); + List columns = new ArrayList<>(); + int start = -1; + for (int x = 0; x < span; x++) { + boolean isColumn = coverage[x] >= support; + if (isColumn && start < 0) { + start = x; + } else if (!isColumn && start >= 0) { + columns.add(new float[] {lo + start, lo + x}); + start = -1; + } + } + if (start >= 0) { + columns.add(new float[] {(float) (lo + start), (float) (lo + span)}); + } + + // Merge bands separated by only a narrow gutter. A real column separator is several + // characters wide; the gaps *inside* a multi-word cell (ordinary word spacing) are about + // one character. Without this, a cell like "January 20th, 2026" — whose words align + // vertically across every row — would be split into three spurious columns. + float charWidth = averageCharWidth(rows); + float minGutter = Math.max(10f, charWidth * 2.5f); + List merged = new ArrayList<>(); + for (float[] band : columns) { + if (!merged.isEmpty() && band[0] - merged.get(merged.size() - 1)[1] < minGutter) { + merged.get(merged.size() - 1)[1] = band[1]; + } else { + merged.add(new float[] {band[0], band[1]}); + } + } + return merged; + } + + private static float averageCharWidth(List rows) { + double totalWidth = 0; + int totalChars = 0; + for (Line l : rows) { + for (TextWord w : l.source.words()) { + totalWidth += w.width(); + totalChars += Math.max(1, w.text().strip().length()); + } + } + return totalChars == 0 ? 6f : (float) (totalWidth / totalChars); + } + + private static int nearestColumn(float x, float[] centers) { + int best = 0; + float bestDist = Float.MAX_VALUE; + for (int i = 0; i < centers.length; i++) { + float d = Math.abs(x - centers[i]); + if (d < bestDist) { + bestDist = d; + best = i; + } + } + return best; + } + + private static int filledCells(String[] row) { + int count = 0; + for (String cell : row) { + if (!cell.isEmpty()) { + count++; + } + } + return count; + } + + private static String renderGfm(List rows, int cols) { + if (rows.isEmpty()) { + return ""; + } + int[] widths = new int[cols]; + for (int c = 0; c < cols; c++) { + widths[c] = 3; + } + for (String[] row : rows) { + for (int c = 0; c < cols; c++) { + if (c < row.length) { + widths[c] = Math.max(widths[c], escapeCell(row[c]).length()); + } + } + } + StringBuilder sb = new StringBuilder(); + sb.append(buildGfmRow(rows.get(0), widths, cols)).append('\n'); + sb.append('|'); + for (int c = 0; c < cols; c++) { + sb.append('-').append("-".repeat(widths[c])).append('-').append('|'); + } + for (int r = 1; r < rows.size(); r++) { + sb.append('\n').append(buildGfmRow(rows.get(r), widths, cols)); + } + return sb.toString(); + } + + /** + * A line looks like a table row if it has at least two words separated by a gap far wider than + * normal inter-word spacing. The threshold is derived from the line's own character width + * rather than a document font size, because some PDFs report a unit (matrix-scaled) font size + * that makes absolute thresholds meaningless. (Two-word rows are allowed so two-column tables + * are detected; spurious matches are filtered later by block contiguity and column + * consistency.) + */ + private static boolean isTableCandidate(TextLine line) { + List words = line.words(); + if (words.size() < 2) { + return false; + } + double totalWidth = 0; + int totalChars = 0; + for (TextWord w : words) { + totalWidth += w.width(); + totalChars += Math.max(1, w.text().strip().length()); + } + float charWidth = (float) (totalWidth / Math.max(1, totalChars)); + // A deliberate cell gap is several blank characters wide; ordinary word spaces are ~a third + // of a character. Floor at 8pt so tiny fonts still need a real gap. + float cellGap = Math.max(8f, charWidth * 3f); + for (int i = 1; i < words.size(); i++) { + TextWord prev = words.get(i - 1); + float gap = words.get(i).x() - (prev.x() + prev.width()); + if (gap >= cellGap) { + return true; + } + } + return false; + } + + private static String buildGfmRow(String[] row, int[] widths, int cols) { + StringBuilder sb = new StringBuilder().append('|'); + for (int c = 0; c < cols; c++) { + String cell = c < row.length ? escapeCell(row[c]) : ""; + sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|'); + } + return sb.toString(); + } + + private static String escapeCell(String cell) { + // Cell content is inline context: escape inline markdown (including the column delimiter) + // but not leading block markers, which have no meaning inside a table cell. + return escapeMarkdownInline(cell); + } + + /** + * Escapes Markdown control characters in body text extracted from the PDF so that literal + * characters (e.g. a line that reads {@code # Heading} or {@code [label](url)}, or an embedded + * {@code }) are emitted as text rather than being reinterpreted as structure or raw HTML. + * Applied to all body text — headings, paragraphs, bold labels, bullets — before emission. + * + *

The generated Markdown should still be treated as untrusted content by any downstream + * renderer: this hardens fidelity and is defence-in-depth, not a substitute for safe rendering. + */ + private static String escapeMarkdown(String text) { + if (text.isEmpty()) { + return text; + } + String inline = escapeMarkdownInline(text); + return escapeLeadingBlockMarker(inline, text); + } + + /** Escapes inline-significant Markdown characters anywhere in the string. */ + private static String escapeMarkdownInline(String text) { + StringBuilder sb = new StringBuilder(text.length() + 8); + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + switch (c) { + case '\\', '`', '*', '_', '[', ']', '<', '>', '|', '~' -> sb.append('\\').append(c); + default -> sb.append(c); + } + } + return sb.toString(); + } + + /** + * Escapes block-level markers that are only significant at the start of a line: ATX headings + * ({@code #}), unordered list / thematic break markers ({@code -}, {@code +}), and ordered list + * markers ({@code 1.} / {@code 1)}). {@code original} carries the unescaped leading characters, + * none of which are altered by inline escaping, so positions line up with {@code escaped}. + */ + private static String escapeLeadingBlockMarker(String escaped, String original) { + char c0 = original.charAt(0); + if (c0 == '#' || c0 == '-' || c0 == '+') { + return "\\" + escaped; + } + int i = 0; + while (i < original.length() && Character.isDigit(original.charAt(i))) { + i++; + } + if (i > 0 && i < original.length()) { + char delim = original.charAt(i); + if (delim == '.' || delim == ')') { + return escaped.substring(0, i) + "\\" + escaped.substring(i); + } + } + return escaped; + } + + private static String padRight(String s, int width) { + return s.length() >= width ? s : s + " ".repeat(width - s.length()); + } + + // --- Page-level emission helpers --------------------------------------- + + private static void emitImages(PdfDocument doc, int pageIndex, List pageItems) + throws IOException { + try (PdfPage page = doc.page(pageIndex)) { + List images = + PdfImageExtractor.extract(page.rawDocHandle(), page.rawHandle(), pageIndex); + for (ExtractedImage img : images) { + pageItems.add(describeImage(img)); + } + } + } + + /** + * Builds an image placeholder annotated with whatever metadata JPDFium exposes: pixel + * dimensions, on-page placement (points), effective DPI, encoded format, colour space and bit + * depth. Missing fields are simply omitted so the line stays valid for any image. + */ + private static String describeImage(ExtractedImage img) { + List parts = new ArrayList<>(); + if (img.width() > 0 && img.height() > 0) { + parts.add(img.width() + "x" + img.height() + "px"); + } + Rect b = img.bounds(); + if (b != null && b.width() > 0 && b.height() > 0) { + parts.add(String.format("%.0fx%.0fpt", b.width(), b.height())); + if (img.width() > 0) { + float dpiX = img.width() / (b.width() / 72f); + float dpiY = img.height() / (b.height() / 72f); + if (Float.isFinite(dpiX) && dpiX > 0) { + parts.add(String.format("~%.0fdpi", (dpiX + dpiY) / 2f)); + } + } + } + String ext = img.suggestedExtension(); + if (ext != null && !ext.isBlank()) { + parts.add(ext.replaceFirst("^\\.", "").toUpperCase(java.util.Locale.ROOT)); + } + if (img.colorSpace() != null) { + parts.add(img.colorSpace().toString()); + } + if (img.bitsPerPixel() > 0) { + parts.add(img.bitsPerPixel() + "bpp"); + } + + StringBuilder sb = new StringBuilder("'); + return sb.toString(); + } + + private static void mergeAcrossPageBoundary(List output, List pageItems) { + if (output.isEmpty() || pageItems.isEmpty()) { + return; + } + // Only merge a sentence continuation between two text paragraphs, never into/out of a + // table. + if (!(output.get(output.size() - 1) instanceof String last) + || !(pageItems.get(0) instanceof String first)) { + return; + } + if (!first.isEmpty() + && Character.isLowerCase(first.charAt(0)) + && !endsWithSentencePunctuation(last)) { + output.set(output.size() - 1, last + " " + first); + pageItems.remove(0); + } + } + + /** + * Joins tables split across a page break. Two consecutive {@link TableBlock}s (no text between + * them — i.e. one ended a page and the next began the following page) are merged when their + * column layouts match; a repeated header row on the continuation is dropped. + */ + private static List stitchTables(List elements) { + List out = new ArrayList<>(); + for (Object e : elements) { + if (e instanceof TableBlock tb + && !out.isEmpty() + && out.get(out.size() - 1) instanceof TableBlock prev + && columnsMatch(flatten(prev.rows()), flatten(tb.rows()))) { + List> merged = new ArrayList<>(prev.rows()); + List> tail = tb.rows(); + if (!tail.isEmpty() + && !prev.rows().isEmpty() + && rowText(tail.get(0)).equals(rowText(prev.rows().get(0)))) { + tail = tail.subList(1, tail.size()); + } + merged.addAll(tail); + out.set(out.size() - 1, new TableBlock(merged, prev.top(), tb.bottom())); + } else { + out.add(e); + } + } + return out; + } + + private static String normaliseSpace(String s) { + return s.strip().replaceAll("\\s+", " "); + } + + private static List flatten(List> rows) { + return rows.stream().flatMap(List::stream).collect(Collectors.toList()); + } + + /** Whitespace-normalised text of a row's lines (top to bottom), for header de-duplication. */ + /** + * Header text of a table at the very bottom of a page, or null if the page does not end in one. + * Trailing image placeholders are skipped; any other text after a table means it did not run to + * the page bottom and so is not a continuation candidate. + */ + private static String trailingTableHeader(List pageItems) { + for (int i = pageItems.size() - 1; i >= 0; i--) { + Object e = pageItems.get(i); + if (e instanceof String s && s.strip().startsWith(" row) { + List ordered = new ArrayList<>(row); + ordered.sort(Comparator.comparingDouble((Line l) -> l.y).reversed()); + StringBuilder sb = new StringBuilder(); + for (Line l : ordered) { + if (sb.length() > 0) { + sb.append(' '); + } + sb.append(l.text); + } + return normaliseSpace(sb.toString()); + } + + /** True when two table blocks have the same number of columns at near-identical x-centres. */ + private static boolean columnsMatch(List a, List b) { + List ca = findColumnRanges(a); + List cb = findColumnRanges(b); + if (ca.size() < 2 || ca.size() != cb.size()) { + return false; + } + for (int i = 0; i < ca.size(); i++) { + float centreA = (ca.get(i)[0] + ca.get(i)[1]) / 2f; + float centreB = (cb.get(i)[0] + cb.get(i)[1]) / 2f; + if (Math.abs(centreA - centreB) > 15f) { + return false; + } + } + return true; + } + + private static void flushParagraph(StringBuilder para, List out) { + if (!para.isEmpty()) { + out.add(escapeMarkdown(para.toString())); + para.setLength(0); + } + } + + private static String repairHyphens(String text) { + return SOFT_HYPHEN.matcher(text).replaceAll("$1$2"); + } + + private static boolean endsWithSentencePunctuation(String s) { + if (s.isEmpty()) { + return false; + } + char last = s.charAt(s.length() - 1); + return last == '.' || last == '?' || last == '!' || last == ':'; + } + + // --- Methods used by other components / tests -------------------------- + + List extractAllPageText(PdfDocument doc) throws IOException { + return PdfTextExtractor.extractAll(doc); + } + + List extractTables(PdfDocument doc, int pageIndex) throws IOException { + return PdfTableExtractor.extract(doc, pageIndex); + } + + List renderTables(List
tables) { + return tables.stream().map(TableRenderer::render).toList(); + } +} diff --git a/app/common/src/main/java/stirling/software/common/pdf/TableRenderer.java b/app/common/src/main/java/stirling/software/common/pdf/TableRenderer.java new file mode 100644 index 000000000..3f468699f --- /dev/null +++ b/app/common/src/main/java/stirling/software/common/pdf/TableRenderer.java @@ -0,0 +1,82 @@ +package stirling.software.common.pdf; + +import stirling.software.jpdfium.text.Table; + +final class TableRenderer { + private TableRenderer() {} + + /** Renders a Table as a GitHub-Flavoured Markdown table string. */ + static String render(Table table) { + if (table.rowCount() == 0) { + return ""; + } + + String[][] grid = table.asGrid(); + + if (table.rowCount() < 2) { + // No separator row possible — return plain lines + StringBuilder sb = new StringBuilder(); + for (int c = 0; c < grid[0].length; c++) { + if (c > 0) sb.append('\n'); + sb.append(escape(grid[0][c].trim())); + } + return sb.toString(); + } + + int cols = grid[0].length; + + // Compute column widths: max(3, max content length across all rows) + int[] widths = new int[cols]; + for (int c = 0; c < cols; c++) { + widths[c] = 3; + } + for (String[] row : grid) { + for (int c = 0; c < cols; c++) { + String cell = c < row.length ? row[c].trim() : ""; + widths[c] = Math.max(widths[c], escape(cell).length()); + } + } + + StringBuilder sb = new StringBuilder(); + + // Header row + sb.append(buildRow(grid[0], widths, cols)); + sb.append('\n'); + + // Separator row + sb.append('|'); + for (int c = 0; c < cols; c++) { + sb.append('-').append("-".repeat(widths[c])).append('-').append('|'); + } + sb.append('\n'); + + // Data rows + for (int r = 1; r < grid.length; r++) { + sb.append(buildRow(grid[r], widths, cols)); + if (r < grid.length - 1) { + sb.append('\n'); + } + } + + return sb.toString(); + } + + private static String buildRow(String[] row, int[] widths, int cols) { + StringBuilder sb = new StringBuilder(); + sb.append('|'); + for (int c = 0; c < cols; c++) { + String cell = c < row.length ? escape(row[c].trim()) : ""; + sb.append(' ').append(padRight(cell, widths[c])).append(' ').append('|'); + } + return sb.toString(); + } + + private static String escape(String cell) { + return cell.replace("|", "\\|"); + } + + private static String padRight(String s, int width) { + if (s.length() >= width) return s; + return s + " ".repeat(width - s.length()); + } +} diff --git a/app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java b/app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java deleted file mode 100644 index fbbf5af9c..000000000 --- a/app/common/src/test/java/stirling/software/SPDF/pdf/parser/LineAlignmentTableParserTest.java +++ /dev/null @@ -1,153 +0,0 @@ -package stirling.software.SPDF.pdf.parser; - -import static org.assertj.core.api.Assertions.assertThat; -import static stirling.software.SPDF.pdf.parser.PdfModels.*; - -import java.util.List; - -import org.junit.jupiter.api.Test; - -/** - * Unit tests for {@link LineAlignmentTableParser}, focused on the coincident-line merge logic and - * column-grid construction. - */ -class LineAlignmentTableParserTest { - - private final LineAlignmentTableParser parser = new LineAlignmentTableParser(); - - // ── mergeCoincidentLines ───────────────────────────────────────────────────────────────────── - - @Test - void mergeCoincidentLines_singleLine_unchanged() { - var lines = List.of(tokenized(rawLine(10f, 100f, "Revenue"))); - assertThat(parser.mergeCoincidentLines(lines)).hasSize(1); - } - - @Test - void mergeCoincidentLines_distinctYLines_unchanged() { - // Two lines at different y positions — must NOT be merged. - var lines = - List.of( - tokenized(rawLine(10f, 100f, "Revenue")), - tokenized(rawLine(10f, 115f, "Cost"))); - assertThat(parser.mergeCoincidentLines(lines)).hasSize(2); - } - - @Test - void mergeCoincidentLines_sameY_merged() { - // Simulates a financial-table row split by LineBuilder at the column gap: - // label fragment at x=72 → "Revenue" - // value fragment at x=350 → "1,234" - // Both have y=100. After merge they should form one TokenizedLine. - var label = rawLine(72f, 100f, "Revenue"); - var value = rawLine(350f, 100f, "1,234"); - - var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value))); - - assertThat(merged).hasSize(1); - // The merged line should contain tokens from both halves. - var tokens = merged.get(0).all(); - assertThat(tokens.stream().map(t -> t.text()).toList()) - .containsExactlyInAnyOrder("Revenue", "1,234"); - } - - @Test - void mergeCoincidentLines_sameY_mergedLineHasCorrectBounds() { - var label = rawLine(72f, 100f, "Revenue"); // 7 chars × 6pt = 42pt wide → right = 114 - var value = rawLine(350f, 100f, "1,234"); // 5 chars × 6pt = 30pt wide → right = 380 - - var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(value))); - - var bounds = merged.get(0).line().bounds(); - assertThat(bounds.x()).isEqualTo(72f); - assertThat(bounds.right()).isEqualTo(380f); - } - - @Test - void mergeCoincidentLines_withinTolerance_merged() { - // Lines 1.5pt apart (within ROW_MERGE_TOLERANCE_PT = 2pt) should merge. - var a = rawLine(10f, 100.0f, "Alpha"); - var b = rawLine(200f, 101.5f, "99"); - - var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b))); - assertThat(merged).hasSize(1); - } - - @Test - void mergeCoincidentLines_beyondTolerance_notMerged() { - // Lines 3pt apart (beyond ROW_MERGE_TOLERANCE_PT = 2pt) should NOT merge. - var a = rawLine(10f, 100.0f, "Alpha"); - var b = rawLine(200f, 103.0f, "99"); - - var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b))); - assertThat(merged).hasSize(2); - } - - @Test - void mergeCoincidentLines_threeCoincident_allMerged() { - // Three fragments at the same y (e.g. wide financial table with two value columns). - var a = rawLine(72f, 100f, "Revenue"); - var b = rawLine(300f, 100f, "1,234"); - var c = rawLine(400f, 100f, "5,678"); - - var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c))); - assertThat(merged).hasSize(1); - assertThat(merged.get(0).all()).hasSize(3); - } - - @Test - void mergeCoincidentLines_coincidentPairFollowedByDistinctLine_twoGroups() { - var a = rawLine(72f, 100f, "Revenue"); - var b = rawLine(350f, 100f, "1,234"); // same y as a → merges with a - var c = rawLine(10f, 115f, "Expenses"); // different y → stays separate - - var merged = parser.mergeCoincidentLines(List.of(tokenized(a), tokenized(b), tokenized(c))); - assertThat(merged).hasSize(2); - } - - @Test - void mergeCoincidentLines_numericAnchorStatus_correctAfterMerge() { - // After merging, the combined line should be an anchor (≥2 numeric tokens). - // "Revenue" alone → not an anchor. "1,234 567" alone → anchor. - // Merged → anchor with at least 2 numerics. - var label = rawLine(72f, 100f, "Revenue"); - var values = rawLineMultiWord(350f, 100f, "1,234", 30f, "567", 30f); - - var merged = parser.mergeCoincidentLines(List.of(tokenized(label), tokenized(values))); - - assertThat(merged).hasSize(1); - assertThat(merged.get(0).isAnchor()).isTrue(); - } - - // ── helpers ────────────────────────────────────────────────────────────────────────────────── - - /** Creates a RawLine with a single TextFragment of the given text at the given position. */ - private static RawLine rawLine(float x, float y, String text) { - float width = text.length() * 6f; // ~6pt per char — rough but consistent - float height = 12f; - Bounds bounds = new Bounds(x, y, width, height); - TextFragment fragment = - new TextFragment("tf-test", text, bounds, y + height, 11f, "Helvetica", false); - return new RawLine("ln-test", List.of(fragment), bounds, 1); - } - - /** - * Creates a RawLine with two TextFragments representing two words separated by a small gap. - * Used to simulate a values-only line with multiple numeric tokens. - */ - private static RawLine rawLineMultiWord( - float x, float y, String word1, float w1, String word2, float w2) { - float height = 12f; - Bounds b1 = new Bounds(x, y, w1, height); - Bounds b2 = new Bounds(x + w1 + 5f, y, w2, height); - TextFragment f1 = new TextFragment("tf-1", word1, b1, y + height, 11f, "Helvetica", false); - TextFragment f2 = new TextFragment("tf-2", word2, b2, y + height, 11f, "Helvetica", false); - Bounds lineBounds = new Bounds(x, y, x + w1 + 5f + w2 - x, height); - return new RawLine("ln-test", List.of(f1, f2), lineBounds, 1); - } - - /** Tokenises a RawLine via the parser's own tokenise logic (package-private access). */ - private LineAlignmentTableParser.TokenizedLine tokenized(RawLine line) { - return parser.tokenize(line); - } -} diff --git a/app/common/src/test/java/stirling/software/common/pdf/PdfMarkdownConverterTest.java b/app/common/src/test/java/stirling/software/common/pdf/PdfMarkdownConverterTest.java new file mode 100644 index 000000000..b3c104da8 --- /dev/null +++ b/app/common/src/test/java/stirling/software/common/pdf/PdfMarkdownConverterTest.java @@ -0,0 +1,269 @@ +package stirling.software.common.pdf; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import stirling.software.jpdfium.PdfDocument; +import stirling.software.jpdfium.text.TextLine; +import stirling.software.jpdfium.text.TextWord; + +/** + * Accuracy and robustness tests for {@link PdfMarkdownConverter}, comparing conversion output + * against hand-authored golden Markdown for a set of owned/synthetic fixtures. + * + *

The {@link #gatedFixtures()} set is enforced in CI: those fixtures currently convert within + * the accuracy threshold and guard against regressions. Fixtures still being iterated on live in + * {@link #wipFixtures()} under a {@link Disabled} test so the goldens stay in the tree without + * breaking the build. Enable the WIP test locally to see per-fixture scores while working on the + * converter. + */ +class PdfMarkdownConverterTest { + + /** Accuracy threshold: output must share at least this fraction of content with the golden. */ + private static final double THRESHOLD = 0.95; + + @TempDir Path tmp; + + /** Fixtures that meet the accuracy threshold today and therefore gate CI. */ + static Stream gatedFixtures() { + return Stream.of( + Arguments.of("multi-column-test_lorem.pdf", "multi-column-test_lorem.md"), + Arguments.of("bordered-table-test_widget.pdf", "bordered-table-test_widget.md"), + Arguments.of("many-tables-test_stress.pdf", "many-tables-test_stress.md")); + } + + /** Fixtures still below the threshold; tracked here, enable locally to iterate. */ + static Stream wipFixtures() { + return Stream.of( + Arguments.of( + "wrapped-cell-test_expense-report.pdf", + "wrapped-cell-test_expense-report.md")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("gatedFixtures") + void convertMatchesGoldenMarkdown(String pdfName, String mdName) throws IOException { + assertConversionMatchesGolden(pdfName, mdName); + } + + @Disabled("WIP fixtures below the accuracy threshold; enable locally to iterate") + @ParameterizedTest(name = "{0}") + @MethodSource("wipFixtures") + void convertMatchesGoldenMarkdownWip(String pdfName, String mdName) throws IOException { + assertConversionMatchesGolden(pdfName, mdName); + } + + /** + * Degenerate/extreme geometry must not crash the converter. A crafted or malformed PDF can + * position text anywhere via a text matrix, so a row's words can span from near the origin to a + * coordinate beyond {@link Integer#MAX_VALUE}. The old column-detection code sized an {@code + * int[]} straight from {@code (int) Math.ceil(maxX) - lo}, which either allocated a multi-GB + * array (OutOfMemoryError) or overflowed to a negative length (NegativeArraySizeException) — + * taking down the request thread. Detection must instead bail out and return no columns. + */ + @Test + void columnDetectionSurvivesDegenerateGeometry() { + // x ≈ 2.5e9 is past Integer.MAX_VALUE; combined with a near-origin word it yields an + // implausible span that the pre-fix code turned into a fatal array allocation. + List rows = new ArrayList<>(); + for (int r = 0; r < 4; r++) { + float y = 400f - r * 12f; + TextWord near = new TextWord(List.of(), 50f, y, 30f, 10f); + TextWord far = new TextWord(List.of(), 2_500_000_000f, y, 30f, 10f); + rows.add(new TextLine(List.of(near, far), 50f, y, 2_499_999_980f, 10f)); + } + + List columns = + assertDoesNotThrow(() -> PdfMarkdownConverter.findColumnRangesFromLines(rows)); + assertTrue( + columns.isEmpty(), + "implausible page span should disable column detection, not allocate from it"); + } + + private void assertConversionMatchesGolden(String pdfName, String mdName) throws IOException { + Path pdfPath = tmp.resolve(pdfName); + try (InputStream in = + getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + pdfName)) { + if (in == null) { + fail("Fixture not found on classpath: /pdf-ingestion-fixtures/" + pdfName); + } + Files.copy(in, pdfPath); + } + + String actual; + try (PdfDocument doc = PdfDocument.open(pdfPath)) { + actual = new PdfMarkdownConverter().convert(doc); + } + + String expected; + try (InputStream in = getClass().getResourceAsStream("/pdf-ingestion-fixtures/" + mdName)) { + if (in == null) { + fail("Golden file not found on classpath: /pdf-ingestion-fixtures/" + mdName); + } + expected = new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + + // Image placeholders are not scored: their body text is a TODO ("ideally, add the info + // available about the image...") rather than real content, so comparing it would penalise + // output for matching a placeholder we intend to replace. Drop those lines from both sides. + expected = stripImagePlaceholders(expected); + actual = stripImagePlaceholders(actual); + + double similarity = similarity(expected, actual); + if (similarity < THRESHOLD) { + fail( + String.format( + "Markdown output differs from golden file '%s' by %.1f%% (threshold %.0f%%):%n%s", + mdName, + (1.0 - similarity) * 100, + (1.0 - THRESHOLD) * 100, + unifiedDiff(expected, actual))); + } + } + + /** Substring identifying an image-placeholder line, which is excluded from scoring. */ + private static final String IMAGE_PLACEHOLDER_MARKER = "Image intentionally redacted"; + + /** + * Removes non-content lines from the comparison: image placeholders (TODO text we intend to + * replace) and GFM table separator rows (the {@code |---|---|} divider, whose exact dash count + * is cosmetic — any run of three or more dashes is valid Markdown). + */ + private static String stripImagePlaceholders(String md) { + StringBuilder sb = new StringBuilder(); + for (String line : md.split("\n", -1)) { + if (line.contains(IMAGE_PLACEHOLDER_MARKER) + || line.strip().startsWith(" 0) { + sb.append('\n'); + } + sb.append(line); + } + return sb.toString(); + } + + /** True for a GFM table separator row, e.g. {@code |---|:--:|---|} (only |, -, :, space). */ + private static boolean isTableSeparatorRow(String line) { + String t = line.strip(); + if (!t.contains("-")) { + return false; + } + return t.chars().allMatch(c -> c == '|' || c == '-' || c == ':' || c == ' '); + } + + /** + * Character-level similarity: proportion of expected characters that appear in the LCS. O(n*m) + * but golden files are small enough that this is fine. + */ + private static double similarity(String expected, String actual) { + if (expected.isEmpty() && actual.isEmpty()) return 1.0; + if (expected.isEmpty() || actual.isEmpty()) return 0.0; + // Strip all whitespace for a content-focused comparison + String e = expected.replaceAll("\\s+", " ").strip(); + String a = actual.replaceAll("\\s+", " ").strip(); + int lcs = lcsLength(e, a); + return (double) lcs / Math.max(e.length(), a.length()); + } + + private static int lcsLength(String a, String b) { + // Use two-row DP to keep memory reasonable + int m = a.length(), n = b.length(); + int[] prev = new int[n + 1]; + int[] curr = new int[n + 1]; + for (int i = 1; i <= m; i++) { + for (int j = 1; j <= n; j++) { + if (a.charAt(i - 1) == b.charAt(j - 1)) { + curr[j] = prev[j - 1] + 1; + } else { + curr[j] = Math.max(curr[j - 1], prev[j]); + } + } + int[] tmp = prev; + prev = curr; + curr = tmp; + java.util.Arrays.fill(curr, 0); + } + return prev[n]; + } + + private static String unifiedDiff(String expected, String actual) { + String[] expectedLines = expected.split("\n", -1); + String[] actualLines = actual.split("\n", -1); + + List diff = new ArrayList<>(); + diff.add("--- expected"); + diff.add("+++ actual"); + + int maxLines = Math.max(expectedLines.length, actualLines.length); + int context = 3; + boolean inHunk = false; + int hunkStart = -1; + List hunkLines = new ArrayList<>(); + + for (int i = 0; i < maxLines; i++) { + String exp = i < expectedLines.length ? expectedLines[i] : null; + String act = i < actualLines.length ? actualLines[i] : null; + + boolean changed = exp == null || act == null || !exp.equals(act); + if (changed) { + if (!inHunk) { + inHunk = true; + hunkStart = Math.max(0, i - context); + // add context lines before change + for (int c = hunkStart; c < i; c++) { + hunkLines.add(" " + (c < expectedLines.length ? expectedLines[c] : "")); + } + } + if (exp != null) hunkLines.add("-" + exp); + if (act != null) hunkLines.add("+" + act); + } else { + if (inHunk) { + hunkLines.add(" " + exp); + // check if we're far enough past the last change to close the hunk + boolean moreChanges = false; + for (int j = i + 1; j < Math.min(i + context, maxLines); j++) { + String e2 = j < expectedLines.length ? expectedLines[j] : null; + String a2 = j < actualLines.length ? actualLines[j] : null; + if (e2 == null || a2 == null || !e2.equals(a2)) { + moreChanges = true; + break; + } + } + if (!moreChanges && (i - hunkStart) >= context) { + diff.add("@@ -" + (hunkStart + 1) + " @@"); + diff.addAll(hunkLines); + hunkLines.clear(); + inHunk = false; + } + } + } + } + + if (inHunk && !hunkLines.isEmpty()) { + diff.add("@@ -" + (hunkStart + 1) + " @@"); + diff.addAll(hunkLines); + } + + return String.join("\n", diff); + } +} diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.md b/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.md new file mode 100644 index 000000000..4b590e4b6 --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.md @@ -0,0 +1,10 @@ +# Widget Inventory Report + +This report lists current stock levels for each warehouse. + +| Region | Units | Status | +|---|---|---| +| North | 1200 | OK | +| South | 950 | Low | +| East | 1430 | OK | +| West | 875 | Low | diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.pdf b/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.pdf new file mode 100644 index 000000000..8da041e28 --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/bordered-table-test_widget.pdf @@ -0,0 +1,74 @@ +%PDF-1.4 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20260603003133+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603003133+01'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 500 +>> +stream +Gas1[9i&Y\%))C:pc(6t3;pQ8%0T@tD+-Nf,MP8j(>R4tMCHL4NI*\1+UNiI'V9NC%VeJKn/YI0J];XQt&X83?=ihrg<*Mcn1n!1nWcDaQPe\P"9gnJuHl(jf]JQgZ[,&^uobI4QF',k"*^S)3c;)GMWC(T'=",ErnS#U=YCUN0&q4+*KmK1Zd*NI\GQDiZUG7;PTja8lulb"\PWWO#WcfI[ZB:6s*3g$be%?JH(n`oaEJ[XE'%QW=HE04M<,;ERm[MS=uYF=nN3jG'f@#?O48Ia,6Y-3m&tTWVq1?DeiBkp.Ug*;lVZX`Z=P.eklHhNV;!R_?QOuoeJ<0%7idG7GM8boU$^>N.N,2^;25]0Z8M<<]XMCct>noC'Qfb?`*[Mo+,F9#>t~>endstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000514 00000 n +0000000582 00000 n +0000000862 00000 n +0000000921 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1511 +%%EOF diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.md b/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.md new file mode 100644 index 000000000..4b456165d --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.md @@ -0,0 +1,222 @@ +Intro paragraph for section 1. + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | + +# Section 2 Heading + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | +| golf | 301 | india | + +## Section 3 Heading + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | +| golf | 301 | india | 23 | +| juliet | 401 | lima | 33 | + +Intro paragraph for section 4. + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | +| golf | 301 | india | 23 | kilo | +| juliet | 401 | lima | 33 | november | +| mike | 501 | oscar | 43 | alpha | + +# Section 5 Heading + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | +| golf | 301 | +| juliet | 401 | +| mike | 501 | +| papa | 601 | + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | + +# Section 7 Heading + +Intro paragraph for section 7. + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | +| golf | 301 | india | 23 | + +## Section 8 Heading + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | +| golf | 301 | india | 23 | kilo | +| juliet | 401 | lima | 33 | november | + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | +| golf | 301 | +| juliet | 401 | +| mike | 501 | + +# Section 10 Heading + +Intro paragraph for section 10. + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | +| golf | 301 | india | +| juliet | 401 | lima | +| mike | 501 | oscar | +| papa | 601 | bravo | + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | + +# Section 12 Heading + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | +| golf | 301 | india | 23 | kilo | + +## Section 13 Heading + +Intro paragraph for section 13. + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | +| golf | 301 | +| juliet | 401 | + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | +| golf | 301 | india | +| juliet | 401 | lima | +| mike | 501 | oscar | + +# Section 15 Heading + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | +| golf | 301 | india | 23 | +| juliet | 401 | lima | 33 | +| mike | 501 | oscar | 43 | +| papa | 601 | bravo | 53 | + +Intro paragraph for section 16. + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | + +# Section 17 Heading + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | +| golf | 301 | + +## Section 18 Heading + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | +| golf | 301 | india | +| juliet | 401 | lima | + +Intro paragraph for section 19. + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | +| golf | 301 | india | 23 | +| juliet | 401 | lima | 33 | +| mike | 501 | oscar | 43 | + +# Section 20 Heading + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | +| golf | 301 | india | 23 | kilo | +| juliet | 401 | lima | 33 | november | +| mike | 501 | oscar | 43 | alpha | +| papa | 601 | bravo | 53 | delta | + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | + +# Section 22 Heading + +Intro paragraph for section 22. + +| Name | Qty | Price | +|---|---|---| +| alpha | 101 | charlie | +| delta | 201 | foxtrot | +| golf | 301 | india | + +## Section 23 Heading + +| Name | Qty | Price | Region | +|---|---|---|---| +| alpha | 101 | charlie | 3 | +| delta | 201 | foxtrot | 13 | +| golf | 301 | india | 23 | +| juliet | 401 | lima | 33 | + +| Name | Qty | Price | Region | Status | +|---|---|---|---|---| +| alpha | 101 | charlie | 3 | echo | +| delta | 201 | foxtrot | 13 | hotel | +| golf | 301 | india | 23 | kilo | +| juliet | 401 | lima | 33 | november | +| mike | 501 | oscar | 43 | alpha | + +# Section 25 Heading + +Intro paragraph for section 25. + +| Name | Qty | +|---|---| +| alpha | 101 | +| delta | 201 | +| golf | 301 | +| juliet | 401 | +| mike | 501 | +| papa | 601 | diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.pdf b/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.pdf new file mode 100644 index 000000000..f12925cda --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/many-tables-test_stress.pdf @@ -0,0 +1,169 @@ +%PDF-1.4 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 13 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/Contents 14 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +6 0 obj +<< +/Contents 15 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +7 0 obj +<< +/Contents 16 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +8 0 obj +<< +/Contents 17 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +9 0 obj +<< +/Contents 18 0 R /MediaBox [ 0 0 612 792 ] /Parent 12 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +10 0 obj +<< +/PageMode /UseNone /Pages 12 0 R /Type /Catalog +>> +endobj +11 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20260603005358+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603005358+01'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +12 0 obj +<< +/Count 6 /Kids [ 4 0 R 5 0 R 6 0 R 7 0 R 8 0 R 9 0 R ] /Type /Pages +>> +endobj +13 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 1007 +>> +stream +GauHKgN)$k&:O:SnDZmeMUK]*(BdhV"2[X,j0;*\%_E*0HXJRc`q%4S#/VM`O[Rh4i5T.Phi,$]aT$05!q:PBuCQU/.p2]@1koN*Tb*K9k5G\ht+Dr\K=+8\NZ"alMaOEo**@OK:8-.O1X3-?Gg`@m3%,ti3'">T-&c=M&Wuu?cDbGDp.gOF0!r6&&CLC$tM?fIR"M37["/*k9@YkpKKSS@Np"1/4#R]`I^(g*1Sc,9L3Qt6N(T@F`A>oBGgL-!r#Y4)R\G`D,iVtFeJUE9u4iuUQ?D%C&SB4kkp.>D5tii>nDKJ"Y07jANhOb$R(_$=U7Stjs)-/KZ(6IBm`6u<3;i.Bh4+MFJ"H:.XWUQX6%LU(sg4Tt$_":5`p.2,gZkpUfdg,Hd(qR1)9\ltF2^8b5,,XbY%VfhD52O7A.c.u]dhTc5t%0<0L5E38`Bq+i;"%J7kcc#@i)@okdN-qiaA"33DdPgPrUs7;j%+47_cnGN@&ug9/dqGOAHA4f.N*&guHspf?E;GIc-Wt>:<(m1AmcS_Zc2VlEI'_S_>@#!MF^m1$$endstream +endobj +14 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 987 +>> +stream +GatU3gN&c;&:O:SkV;H,eI!J\f?X"LSPIZ'"4Y.F#oAAY?N.Y?8Iu:s)eV;,aNG1Lea>G$!MSc\rU6m7jF'AOI09V1,^TSTi$A+f4sZVQ%2^>uj$40\AI>WbZo9q!NL^CS9P3E9X.:XEMb$a/X"qH9hN6fY,]D!OEOVT,722%RJnVqK4f'[4d3(/hbJWJRs,>>28jk5A:nCl/%FlC#LntsAC5cE$q`QO_@Pk%Lp>7V23")NNrFkcXfuoI=SJJ-`g)_+K\@,SQ@8ORGg(_(B.[u[WXD%Q8@I";9d(]M62K_)Q+-<\kHY5,)o99%BB2bcQVkd:+MZVH=$Z`S@BhH]-XfPANk@qBN.:Jc'?.Kn\p7(aPIQOI(du7U]WDrNNPW%d"8P_j^Y8d%G,'JJLXrV.UD!ah19&9b#*fgR2o730<9)QX)X0"EK6u;mBJs3/fl\d&K$B3bXI5R>F*E,^\%+b?0o8s$o.CuX4uDAT_?G0(p4N3La,8qfi'j?e`e88KrZ8*NDgc:62'icCb],4B]Z"SO&e/BRC*C$2PmbSKAjc\$FLD8?&qHVH3G4p834/77n[%()-p/JDRBapcO2b\,3dW%#U9u!ilSkd%2'rr&g[u".1O]W%0J<@#(ptUD;%0:^.^ls:.#.V6NgR[`2\RUW!k$-*GZIf3DOBiB_Mu"=LVGlO\1Z]0pF(@UaiVQ=dh?h5kfQ_gj>$jHW$8TCt:C.jVU98ne.XoiMtoV;q8XhnlOE;3a]=dGJ8KL&+Khj"&P4Q=P-:nHsDprY"B4+#A3dF,;b&gf(sT\Ts*iH)f!KhfHV;\mMSHej.9.XrBG41_<:b)MG;-Y(:&nCKS0F]r&CGl>EfMc0td\0GHScl3Lr;?OUoo6619Qdr)S8S);]7L5rmWW&(5t4Dil"USDhhY/Y=!=!X:85**$:~>endstream +endobj +15 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 976 +>> +stream +GatU3gN&c;'R]XVkgAc"70atdYFXp#3h7VV#7/=-%N$"G?N.YO&iX#cVsS_l_5i8)eum:1-)(7`mXJ7LnhqY0hGZal,We=q^e"$M]Lu;_=4AoIN$QOjOF.@i*Sg!_rLL=-'?kk;m;M&R`$D>'HCL7]$A,-S`H1I/1T]4\B3=gQ9:I4j9f]+af*Z7*GH5H+;j73Z-T#DAcgpOW(/'b?XF9+X+'-Dqi"+1U7?6=ZRJNBSJ6ojDTa+]o4RF^26C+i0s(9G2o;$@RNbs\(f75],L(SVpI0M!'7JVhZ"$t/%kb1+)F\p54/@jpVJF2+)7^F=gVcp8(h*m%F#'p9c%9Ep9?b7g^F5>(>I3t^d_"f'aue%^E\X4kN:g\P[;sC=g3h,nD/n9*`94uoM?sBCL4;XilfMPT4]&FC.U%DI(c6uf3*XPJP2MRdq@Ag3uYF().nqLFAE2Nih'(^p\5FnSNa>n+cI?U65A61_N1<9ZXH4Xr)Fgq(6md3H9[MQ3Q^%aq>E?XM:O:RKGjc+QV2MB!.I.17QpSt]X[C*6ap.`#'o;8F(ebZ.VMM6k`BkZB\I,2MNoX]J"k]Qd";,@($aWX&29q"6k;>Let)@]QHm:i/lf[DB_&U@ROq-0MLQp;@j+]?$"E)brXVun52AVQo$"[a7@!?!LT*>$&:5X_.R;9)'$l-S.>WjLQIVTo.JU%(H\,-*pl_kP9~>endstream +endobj +16 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 1024 +>> +stream +GauHKflEQ9'Rf^WgrHc4<#\/Qm7c-rFIIq+TFSD%Z#L'6o(Nl&^_!k0ds=.%-s&pt#i0QBKE91*<^/MJJCcfoHA_e;aL?\&_BAj]Dt;HW$6(8Ql\%O7-5%uT80ZIL#\l)(+[ABOTNKmq!2F,S:.F93Z6QIu[nf\Q[qCE`Y-=.8k$67iCR!7=El+50a@f&b^6'Yg'mqlcRQ-5j:%[K@'5!A_m0L)&VlW1T50^X)"3Ma3,U1P8Di$>uPZ)Zj_igBc0lm]WE0#e>*M53(Yh6\9jHYh6B+3bM\b[Q9j'O_9:!:W0$Ijm,#(bKn_D?UYMGJ7LLqC0\[$l<8U*O5^GnN;!N6YB'?d[d8DAPt^>+E:r0<2$'VL/TtuNg;C@5iM#%KdS-GU->0]`/20eLXVW#qnjsU.P:Xj&=enVE[mpqDCn2!qDRuQhI/#cZ-#m6`U9'SI4"9\"r:4p.\W>IK_$#C,EkU#uO1g6*HoUK':XiJjtQfI'Sdc)B3J1eqgGJ+l?"$//&C$^f'TA!2:K!0pd;^Zif)c[:f!C/8hNKc7+'+1WbOFKr9T7c_G$s*O634O2Zs:"iN>j,pWU+:kEQ>5YbF=$27*59iJEA//7(KsL\pUG.G[+JB-r8t;Sih+(W.A'8Q*LZK*I9WjCLjEt>lT%2RFJZg0Ps:5c,HdK#tSNI:#Rp!]O$Ic=T%K0[._E_Fcjn>OcM,iSAf9iF7,,+Qe:@o'Y4o_:54t*l+TAgendstream +endobj +17 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 1075 +>> +stream +GauHKgN&c;&:O:SkV;H,6&QZ[5)3-pL]0bpl':D91fR,o"F49.1/bfmFt32ljt6eO[j7!K]\nZ:^QQASu[l@3no(6*HL'@3I!Qfi7%,.N\.>A91O)D\n5*o@JWSOI@ng5p/"_I`g*Y$j0+5PWaso'25ltcglFd,QIaO'dPO0Yr\h5/,=m\PiP:P+Gg0&N;_5iLN$]BD.#VL+h,tSe\fL&,;+2fg&%\pAJiV#eVhq@ro%%/`[(&8n29YiYd>bDF^Bg[AVf/n>[FE(au"ZAe&>%Q[7/n3-QhsPXuPbp#<$d"UR8u-nBnr!Q$Wjd1;9WN/,KG8])Ca5rrDrI'Ue'*I0C%pn+f`Hd"4sB_&2*)S$pX=F/6"DC"TU`bMtd/m7u]?]=J5L[SNBuE:6Ri,NmW7.*qUp)97Tt+a84b\os`Ti8/uRDlhrl*Xrn(a0\,%4%*k^rL\k3%6ako`&A3MoN^CXV77D!Qg%=j9Ge+0B@dZDHELb2fDU(,iiXXRl^*U,U#Fld!R72UIpKGWu#-DXi653a?U$fqCs18gt(5/#U%!u,gkMK;>OT_/='S:kn,iD@u]<8-rlTRu'+\7L$U7-#!5-5\WQn(n:q@-p24u!~>endstream +endobj +18 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 590 +>> +stream +GasbXgMWc?&;KY!MRgt!"n`^Bf\9$c*`Q-Sb6tfd8P(q-dT1eng=QV-!OG*\kiWo2AGBdKLqI^%h,XNB)-gDk+GFVBa?g*a%:!J\97Vc8-jH8>8P>0S@Wr)o*"Hu<6qPp`EF:M3'l4@S\c2]`&[J%dO@5p$<(([H6:SlB;FS8L`&pO%6Y\V"/O7F]E%L@d.%>L@"4bK_XXs&";n_.W^Nr847HH,X@$.pC"4bYC?9RH,dN?h8a+8/!?Q=D\F41Nc@'B`';4XgMeh!;U23C+>AbMQD-o--lJ/":m#(Yt,X5(`1KL,GTMBLlr6=-1mi#k.-Ou\\T4$Pun2dEU(\?$&;1@T4dm^t!KuOD-U@N_AMr"&uVpsGm,+8I7B*f!%9.o4cC1a[CZ12(hd>1*0bU`k2-MXo1[Gor#kmXGIM'#R49X#NSAOpdf'0ilUH4:M(^Snc3;m/+>endstream +endobj +xref +0 19 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000516 00000 n +0000000711 00000 n +0000000906 00000 n +0000001101 00000 n +0000001296 00000 n +0000001491 00000 n +0000001561 00000 n +0000001842 00000 n +0000001932 00000 n +0000003031 00000 n +0000004109 00000 n +0000005176 00000 n +0000006292 00000 n +0000007459 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 11 0 R +/Root 10 0 R +/Size 19 +>> +startxref +8140 +%%EOF diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.md b/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.md new file mode 100644 index 000000000..5c35de111 --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.md @@ -0,0 +1,25 @@ +# Lorem Ipsum in Two Columns + +## 1. Origins + +Lorem ipsum dolor sit amet consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. + +## 2. Structure + +Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. + +## 3. Usage + +Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. + +## 4. Variations + +Excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum. + +## 5. Typography + +Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam various turpis et commodo pharetra est. + +## 6. Conclusion + +Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.pdf b/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.pdf new file mode 100644 index 000000000..36dc3a1a6 --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/multi-column-test_lorem.pdf @@ -0,0 +1,74 @@ +%PDF-1.4 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20260603021636+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260603021636+01'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 861 +>> +stream +Gat=i>Ak00&;B$5/*8Q/Z)fmNp(^2^OK'MC[dT6#Oq.&b^4c4;1No6>+D%TC.n+fice+l9R0[L%'(osQ?dQP@AUa.A:T=#%ZIIl18hB7ZfI(hVU+?`]UK%;aabAH:q>9NUGQq^.=u])Wt-ETI))C'a[FE@elj!RSrf2Q'F>URAO.C,!DneTPqrj#4e2kb9%1"4qfZ)"#&^0j9nHQ?9nF!j7mVPP5\*Uq'_jMVS]9%`kQB\8*AF_bpr/hGj;HCUOSQU-%5:6S79Ud\b!*tPbr_'pCr$Ea#(FYP31NFhSX.-("1M:$cgH#hX8L(2]R3Q>'BYHCS%pI!;=WdJp,'ii[`QPZ_9mcd\baZ2U(_;c\-p,8EoIEpQ*lstL>]LE;C#\dLnT2R:)BM-fTc['3_He[U,k'!Bo".uERd>SkhRj^J+koSIrZ_dEf_5L'/1h.`+DTK(R:P,WH)h5\se=SZ"L/5b8b..,e/E\o+4YQ+*im^C>AERG/TieEK\)#>U@HXnJ,H0A9-MqhhkDp8%.6Lr,OrK*lih;B<-opZ8%EU?,$r^jmCDAQ`-0/-8/`[p]7Fm%0f:E&S*FV)DX2>#q\bRqA=^_`43#8EA$u%8r6F5`rc9>K@q>E4q^*~>endstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000514 00000 n +0000000582 00000 n +0000000862 00000 n +0000000921 00000 n +trailer +<< +/ID +[<21a9fbd0a0991a91b6e6e2db0856056e><21a9fbd0a0991a91b6e6e2db0856056e>] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1872 +%%EOF diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.md b/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.md new file mode 100644 index 000000000..8008a3b97 --- /dev/null +++ b/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.md @@ -0,0 +1,62 @@ +# Employee Expense Report + +Reimbursement Request + +EMP-1047 + +**Report Header** + +| Employee Name | Michael Tran | +|---|---| +| Employee ID | EMP-1047 | +| Department | Client Services | +| Report Date | January 20th, 2026 | +| Reporting Period | January 5th–16th, 2026 | +| Manager Approver | Laura Simmons | + +**Company Information** + +| Company | Summit Consulting Partners | +|---|---| +| Company Address | 88 Riverside Plaza, Suite 1400, New York, NY 10069 | +| Accounting Department Email | expenses@example.com | + +**Trip Purpose** + +The trip was undertaken for client onsite meetings with Atlantic Energy Solutions in Boston, MA. + +**Expense Details** + +| Description | Amount | Date | Category | +|---|---|---|---| +| Flight (NYC to Boston roundtrip) | $325.40 | January 5th, 2026 | Airline ticket | +| Hotel (3 nights at Harborview Hotel) | $822.75 | January 5th–8th, 2026 | Lodging | +| Taxi from airport to hotel | $48.00 | January 5th, 2026 | Ground transportation | +| Client dinner (3 attendees) | $186.20 | January 6th, 2026 | Meals | +| Parking at JFK Airport | $72.00 | January 5th–8th, 2026 | Parking | +| Breakfast (per diem not used) | $18.50 | January 7th, 2026 | Meals | + +| Description | Amount | Date | Category | +|---|---|---|---| +| Uber to client office | $22.10 | January 7th, 2026 | Ground transportation | +| Printing + presentation materials | $46.90 | January 8th, 2026 | Materials | +| Lunch with client | $39.75 | January 8th, 2026 | Meals | +| Office supplies (notebooks, pens) | $27.60 | January 10th, 2026 | Supplies | +| Mileage reimbursement (client visit in NJ, 42 miles @ $0.67/mile) | $28.14 | January 14th, 2026 | Mileage | +| Team lunch meeting (internal) | $64.30 | January 15th, 2026 | Meals | + +Total Expenses $1,701.64 + +Reimbursement Method + +Reimbursement method Direct deposit + +Notes + +All receipts are attached. Expenses are business-related and comply with company travel policy. + +**Approval** + +Michael Tran, Employee + +Laura Simmons, Manager diff --git a/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.pdf b/app/common/src/test/resources/pdf-ingestion-fixtures/wrapped-cell-test_expense-report.pdf new file mode 100644 index 0000000000000000000000000000000000000000..95a0b2e07a20b9c76d6f6901dc7ae55fdbf83b8a GIT binary patch literal 95841 zcmc$H2Rzp8_dm*Bl~ML>WR!K^wvd&PJqjW7w)d6@Wn~meM2KW0E1?KwWfP%n5|xn^ zng7qdjJ}WO39sMl@9C+J&;7a1ea^YgIoG+)^}eppX;o<%4lpM_A>Qf7FKW68@xUN% zkiD@LA)bf`my{FK$i>p$PTI%?$|@}k;fC;Y^K;^O1v2I;XX8JSqxx!5~foCC?*xj=0|zzvX!21pwO<~9KH83^dHfg~IpY@k|D zV+BhWE5xjyCysA3co4T4nome$oES+pD?aVm<`9W?FPB560 zn~h7;(!~bK>Wuo#!PJaHh*yXoEWpjp3lR_i^MC~e!F-$+F19voVq%1NP&-q&t{^*= zW)FyK=i&?k!|tksxRju#mPV5H9=gD<+#o(7K29O9kdPoqkQa=&(&th&asrxwzQHBUS0@vwGtdVYJX~ZoT!0P$u4Q>ZJn$#P#JFUDuK{k!fVlyMBAUs7!64ud{7Yqc zLA=N-J`j?u3_pk;c~1Z&fV=|uKwb%fgy2^IaX?Kb8c-KqE`d%Ufj;JvHwCn7>Eg*D$ED$F>;mHr#vs70{Utj~;2|i`3aL0a=P*(` zd4ehMe|VDo2cG_k9YjCummOrE{UbZDUi=?+r2di}M0f9(9fWBh`t?ukz)b0X*pdDR zcJ}HqK7^<2lP5leFZ?r4Fe(3srwjYx3E?XHH_a(C6gGoIQS|sNG2GcZeb%+2UPG1WcZb@hiyB>ZTwh?H6ePakT0Y-T z*FSPz={>aJ(`&8@QSTeZvJyeXEjREI`W9M&8*1H~>ow{9zlqOZcV3y$m@fZ4lex)& z=PsnSRHc=ktCEydexBnz(E!T>KKDtTN2WgvR(t7Q&*X4@4_2Sl!;wE%$7Z1-zV0Be z*yZ=o;b~18b*6J|!y_Jih@z_KB+IhKgxJP%YHQ-gTHo*Kdn&X^Ny_Dn^;IsTU021t zuDp>OyFgyLV9d&OMn55P{N%h%gzWUfjp-z?qbD=__>zZ@ zN-9xn)P30&cBk=z)~;9Vq*zW_K>-OBo1=pUWVc253!J~&T4Tc@^mCpZ1hfc(jn(0s4; z6>0OZ@0(=CODj{~OlGP&So(idC`aqbQ9h^hS~p+HUpiAIeoVGOXGG*~i-Gh>cI@^V z9Ad_Hoq}VRzWN1=g=~ntYY{vXE!)cV1&L=aicFN3M(q_B(JU9sAFJr8)rsrDBuhJ0{95sC z=7R*&E=8eSBTlogg{Gq&g5uv_SoO?ZOrS_(x*vRR^zMtAiw0Is;7b$u7o2kd6hh_PT50%U#q7+XAvyF@N3Vg!34OW{rDS_N=t7N3~V57eFEcCqs z?P;5^>Q@Cad};+3b(?WR0!~k+)4JTQKRW%9hh6DwnT9K=sMiZ?E2;q{UTxFd$9`B~ z-k*Xd_TN8^J;$g<|0So<&3JK|dWnG6(0=up^TV~Mk6p)!9}cGDP+d+ZeD)$#!auM< zi|OS3TW1WQ5z@vuJan?}2Pxi@j4KHhOo8Ny-i!|3zNJ6+Q2T35)u`O_v?tLYVix&c zT7)=y-Sk;)Yu-E+?X~y?qr$R&CTy7|HbGk9^_O$ufe-5aKes5$ad=$4MU1CcDCUPQ z^6`t-t!A6qno|sm5(!_LuIn>Iza5D__UfJ-t1fw5W|X_dnZjk7%y$MNgRLS`sh(I# zKJ{O9jOm6IsnwsSd{R&>O8$k(PIUTxhZws?lK<6yfj&$oap{d82KRr^?IoUcW75bp#?Y7~b|IW28%*`{6`; z3A){=nBPs>3qcGgd}0e2m5bPo#dVW@NqX0E)P8_CnK@2azj>B*q=1&XKV%Y}remm27`Km+y!kjmQ;tKk(bxEA=m>Q$(jr9_y_Xaz7*+G0tjh zdAznvYa}g3DMNa;GI-fe@-$c6gSRG!+Y+m&E<*)g(d3#V@RF^9D*}?6+=aoDaQ?f44^L zm;9w{e@Z>T>t?=6~IdryCkqn8P5olm>Ecu8?$z8HV7nweRY#nxE;d{8>@rwlYq_P(PT zSN3A6qzJ~3(a7cD+N12U{i;W#$ErtLCN2&*oGg`p?nY8o+mAK=!d&ItM#d%Cc5Lqc z^L?}DuLjoT;1DH4L~_y;g`?tl73MbvESlUpW6k4I8Q%&X^>gj3YHuREV5)hxJZ;s} z#Cnc1&gzGw{NfnUNqh-&&`Np%+7W-|v^OWAI!$Ls%3TEJKd!ek9 z0Y)jJ;EYT4Ik;CdYKGrtaueTpcoXvW9qltFmSZ*YR~_)^dkD+)jfHQ69#{uC@deyd z%W8PEc;c)p5jLKXc?!hh8p%y>-K#dlpH8XMXe)YDg)q?Z;b}SgJjZ!<2 zMb7HH>%U!CzZ@=6`rb#E$&@kI%Qos*YW|?ajrRANNwU{G15BN$;<6{T zd2~r0sRXywU)SJ&T#`JG?ydHnT0%XKhCc9=RG)(U3F{0yy)O25BsJ?H3zQ0J;b~FM z%^!liLd!Q&1S^}%E8QZ|QYvb(ZAWiDEU_v0DkafE>zi#ie)nGNi>{^Vvnwq887gJY z-_7TjX)$fClI4Zu#QQv%`bed8RVq8G?z2;ajEHVm&FTJv*(tO{s}rsUjY3L*y~#d?zfGsXs29d7?(qKir1GkP@h2R+2bxOYM1H-K| zvJ5xJV}Ir7*;M$heCXg?H$Aebf2+ED_=T!ZyCFWwaqw^Sj5#P?^8I0&F-MSYL0O4- zu>4K_o8(-W(Hz0|$E-hzL(lj)i&E$Zcv`=}1iLmA`j=wNPiZoQ3EPRrK372laXrh$ zHo(QZcCkMf!%MnyGVw0AQ#^O)%B64*dV)=yWeH*nGz?btF(>|9r&A8IF3`pi;%mLq z6<)nYg?d)~oYD{a>O|kTk1P$p37~xA4l*zN5IZX?WYrp|<~`d%RXzh=?C<`Ofe9t- z=PgtZyP_I5L*<}Yfjzc*6WxrXwEFVRBd1v7N10hY-7(osC+;Wug&2`bP2Qj=P8SM{ zrTnt`g6+r0QxcM?x^rJIqczJtZzB?yJ4&_`I59N#+3a@)aiTz&H#dZv;HCrG%FxM$ znwMvy-mB5AdDVKoXuZxwk!z9K%Imt`ik>4aa>;@tinB3Go3o?l#7EOiO#XZpPg?9K zrmGC|vuoLFs#h9fQl|^CKKJ*`u9=+|Eq#01i1?ZbdBmg5__ZcXs117_8)h*^JooRm z365OOrnT#zR$Mh6JW<9XIrG#9oK}I(W<>TS@e2Qkmn_2yzadvAuZ-zaHab20^!&}S zU!`?45SQFz_j8|?grANXVGLUf`g%3#_$7C@T=Kb9hum=2_`B6<8IdO$0zY<9;Wcpo z_Pyx$1~*HYHlpOt$?wS{>(rKuHGQt%epjb4RlWLsc~C9?6d27Ji+=MXwKCNY8_F^& zj7m%#+ya#{^!L&koy6VzZQ8%$<_kQrd(NMt^xZ(Gr*Naz@|}1WLp485u|9%5F6i_8 zrJBM_)p1LcSq4*QzZHBc{QGg7&kC_KxUWk;kBtAp=PKygJ;KO)vS)gX+etwhV`}}O z%%#>+lj2Dq#|^#!D|SLQwap53@ajv}&HxU9u8U83O$WH$c=CAN%NpFBDN|%yUw$Fx zB9w3{5m*^tiVCza!Dt4TazCMe5j?9ypsD+ce)%Siv+7tC&#*J7+oLl!?17-*=7*7? zE*=aZrb-OQkbJo%hct-S|v%!(+2~_^ZaZ^5093!-6G*6*}K+ ze*d{RJT)97B^vuirKMANHic80GiiN{xPQr^T}9B#fZSD@%`HO6a-1oj9OH9sNNjw1 zYpGVQomlB+htzm!ckT@BG;#ThY8tTXJ6XKO)-)x#+9J^K@*~AT%g%C+vHbcm>NMxC z9PTX`jw)&$%Ny}kqvlIN@#$QfD^psG7kJgB2Z9or96qIT_8zu>^fv_$~4VZOlPy-rr?8{*e48Vp*RJHs*H0(k28;sWxPt)4T1RV z!bd7k@$Y?mXZo5uu7&v?W`n68L?>CXh1I#9!Id-gBKR!P8*jewo8Ie@vZEvQB}~pU zQRzbC1`P|)V7G6L+}@esZi6C+wma+d=PNcxs#fU+y-%gLb&$HN1jP$z9-T=cKibf* z5%)AG%{Z6S%5QA1^jr&8v8f5+f!t%0CEccwb9*{soI+i38R zzszItj^Neu(OFzIiefdkGs>Mz_6+m1k)FNo(hu0NZ(^I19GB&m>GC!4x24QCF-lN&BGb2L~76JrxhzbUS0nE11-x>FBP-2 z4p$eq)r<0aj(rw|{EUx0&h<%Cqj9(kNI>bx~xccEdOs?VRGX-wbTT*-y2^>Nz2p| zpOD+Mv#LatNg1hfU07q0rFY7vm;Z<#gn8@pHR`uH$z(NH3B5#31iW8LBMWNv?_~E~ zpUvf}xUu;C>#zPGx{9@5-=1w=pYT~)?7p$-y|m)072PQgN@CJ4R2aDL5_+~TXfmf( zJ;m1TaSdmSND`fj%yDmiqeo|T62_t%MF@xZj-z{ukvf+2PPo%&;zzZt8|L#0 zVS8UU)CF1->IpOQZpY$eB66OY08hxB^nBn%nssxd-{K9vW>rm>6UIn*jM0dh#M{BC zTCSeQIC(sjkzJ?m^$&Bde}C5>pTyDkDfCupo>E=YwToj`r&)XndnD9;mYibD>yYg6 zIyw+`WZ15h;O2Z{DtSa3+L$V7bhs)ww5tKVET3A{Pio~O`j3W|xp8qlFWl;}@N+&fLmBNG_Hq3mc}i(yehelE z3qnaM=yQg{TWUmcSbvXvGQ>Vo;@d=G>N7sDKmjKFQ0Vrwn(CR-N4IYR>DS_>PS~?k zDyfM+mvs%U<(8?+AQI5F?mazF0!_j=wz*QIz@5eTQC~D`K6EnG!{oTHzk55~yzRhU z`%zJRr%&q?!rAU(M>2kOGt%EI$-(w#td`Vn>ovfyk=CC_Bl${E(L>UWO`4p1%}B1a zP(ikFMvTIjoW1V&6r0GciFI6(oN4;CPulBUuhV1XUv(Clo}oYcQ|)ouqZOZczXE*n$O3ZX?GyEVOzPBVtpt>Cg`Q0=?yR3o03sASK+cE*$CYa?IG=6!89zi;99 z+rV67gddq>Rg$y5@+gmV{{RMaaNeQ;PT>zu%xQ~#tb1y$kFiKy!LmX1+n~uer2JrCspE<_hj32aZlh?h3XwML?YA!)dLrV(5I1+b6n&xQqc2iM2jRUUOAO#j3wBQL{t%I#ty7`)|8sedH3&U%u~D(&h_m>4YM4QTW?*e9ho(h zQzv+8S!T#>zS!6oxlr%wNq&web}nYV$*|#-oS|(;RmxR7rsuDYf=?)i7xnxm<*?%W z=x{SZGPsHCJF#^vF?j6jN``R588s(ssV8x7*wKa(^c2Y`9|vd9pSob8u7kr-ZyoiW zJX*eXo}$eP zaV$DTc?*5d+l3)PY7z69$vL%+WzOhTHFXzHsLDUfEl)G(IO>tB~iw!z?XT_0U+bd{_wl^zDUKK!Iw95Wa^uJV*Jb`fiw5P4^y#Y zT-2ct$O0#sK%MZ;~_8K-|mvRAG3<{R1dpYO={JOxja=a|Lld#j+s@Rk;Tag`k3y19@^CE z`8uBV8tKbE8a+*vCPLEr59hqN4PU!Jt%Ug72AX(ZM+9H#r@#65$&(n@;bc~seCh`g zRV7U|wwQL0SrXqj2YmvHd#c zle6zJ&7a&u=MV18vaUZV99W|PX>{!;v-)BrJZ)0h7$N!_s!FOpF11;!XnUREH^B|E z0;Pe5lW#7umb$E+=D6Q28Sl>tp%rVj9dEvBBE^5&5IRXA^-2hu-#49DB!#dAqqtc_ry7sj7W6*Dxt5Z#!w_MyT$#K_n=1dxu zuFfsHczAjAcB~lWj}flB$%wE3ox5Pbfi4$?=g@4?oR zop&zDf_E;0%1diHL4ieF06+j={Z;^hkqdE%m1*Rs$hpV%J!N2NTLJ0`d^LQ*4zZXG zdsyAxe%AspoE&NyNI@3_Myxb}Ac$o-5Cq9I1o0Ih|CH!0P({ZL9olEYL4s{fTh2p1y{_L!axw@Lgc=Af;ZfzBLMJ()cFB;f@yl2 z76iF|ynmi{&TJJxf-s&Cop=DAc1{Ta!-60dM?nxo*8$Lw{q`c9!j_Z(MTAd)xM8%A z_PL8sxn0)(r*XlXZ;=BD!H6IiaQAN%Frl_+0jwdNfd|nOARgrJyvW~q5x?(b{BO+< z)^g`JA$$VFgRnz5#r-)4WVdthATc+9HV{nytz*C&ZgB=+cVG%ddF+0@3@*+#F&>l^ z?whIIo&g2}Od(aZ%T)f#_2Ip_#gs0H2N@HQ!TA8Xhf~<12;xE6AF%9>i1&YK74U}J zbifdpn5aQv-@4y904acYkmdwX+$U4JoB|m3U`!#$v3+|O+=I3x1ayzIdnCH)AiCde z_egLRjK}@iJ-p!-XCN>yjKo3h9xl!{G2Sh^-!D^t*gYS>6bcBoZ{5SA))rH`AYP>1 z^CCL)0J?`$*rEvH-LiXRfc)E-18=xZ2Min8k=*Uyz2V|)6XV^o`~5QYhuy=5cBHBf zz|BNdU?Dox`#L1rUQm$3rOzvFD6`^ZDL5=9E{@roDtq| zn+_N@sv}i(0H$E^Zi|>Mh!1J^eA{-v&qNeXVJAgoQUr$i5DNCU(?9<`h2euW-y#R( zk1#3+4f}AZwuk}JBJ3Z;kBC7ae&p{0h~IZ|{FkE?oZ`-JK=dtW3jzpmj~pNVmf|k& z03gVSQ4^qpbl$(EvwJLnz|sUhq~-1Gj=xG1;NonF2|_^?_stZ%;WixzY*0t4YX7zj z@5L=*x*&dJCW^!?!h3Unx`$KPq6k7lCSg1x2d%%fn84P|c%Mwc8*bBqzy@`sst&-^u0%)|B!IMg0Yv0IfZf9>Y_lhTinkzSVEo(h z6W(x}4g{7aAh|n$n7i#>V9W0J%hVrs4;$2xsyYBuf7rbM((VPf?S7x*C!E5T&Vb%T zWp#2Eog6Tf}rhf=II$+&24tsvb^ZCq-m3!~;`3vI@A1&wp3FAguWoIbco>qk>>^ z7nQ%0c9-=7(jv?sB!rL_s6`-|1tV*(z!oNCRs-yOg69mtybWQMzy;E%z{m^$C|V+- zDp0UO5(Xpn2$Up|lp)+m4f6qemz1RBr9ps1=BQdPqTiLI)WyX1QWvnl$Of@{2f6j= z&w3SHs4a~FLL!+GM6xS{6hsKg2rr5ez>xN21fif^8rM|+S1aA=(-OMCcc{RSApq+d zCicMgBPn|uS6e$Vu^k_%+q-Kx7}>!#LjiYz2N1uZ8p2c}FU@KPY&zhE|M2m%aRJ3g zlpmsc6Nsyta{tO2ti=um@jy@-0EV?aHGt6KE)D#h&FxGX|Bng24uJA~+O$Cv%ZBEaKNF#$Dzz&uX+ z-$D<(1&S%4WC}}Jky0PPj&{beJtY)iJB)-VXa9Q%cZNGyQ5Ys6Dn#$M!i+Egl)m;9 zQSdM#g1NU6Fk~9=pS_4^f?^905|;glXirk)33LQ>Ll#~8-PlDl@3W_a2 zN?6v5;1yVy1)>2m%>L!{9g$mYkpotcxwo<|WQGFfMwlD0ha1r+`|)=KITZapwa9-s zEh6J6OcBW7`d=;&=7zOF@udLbh2`rg*$<+||4FR=DXtyL>%qu1dN`whE#}T>1c(Vs z)WO`S2)Tc2*%?Rn?9Ib@KI+(j{q*LpJ;Q*Quwom^ZV%9#yLS8TDdyo^A9W1FzQo*> z5rKGN#Ws{z9z@KYDGv-i0tbS_FfK@-2XKKJ^pWU>efj<_T=|}AJe=>N&ZdAV0vSyI z)A!+RcKAN5JcyG0AZpwhO7;}fafX#YSQ@xoF#@PRQ$hziP<4*aVR|5{^$F<@os?cBXMQG1tNV0E>H;o5<|5w-`@p++tXSO=lk2ZI@Geye;0?~ zZBX3{s1cqe9vD-HP!@{Zo?;%(_qU*MK$jp_5&kp8{Q=G6gJ+2-1l$3vWoIbaQ_REp z{ua1yKVt4qZ}{L@+!hXT-;o?{=>OT9hx2_D6b=}${=PT=NN?a-+!nHN|6=Y6C4lcE z(SyJRBH;oqPzk^elpJ!k6y{_9TH8mE`%g6<&iA+AfPfm2(eyu^;tzNtT#H-4$?f%% zzb%>WLMiU)6o>QuEi~bN#N1WR0qS3{8N(J9^8j+fY~eq<^Ki}&$3Vgg*ZYxkXHWw6 z6~VLN9l*)$DfM411%r1%C->~n!+Adn(YP-;cPBXTZ2<^o5taUZ%egC>z}z3@){ z|D#ZtFc1D~IblH@<*$HWz!Tyv{OSSZ+!;^yl=E=@zl}THhnzd(5+El$Bi?~I-FJWP zj3;}_c{u;yhO+KM&fQ54e0v1SF%M+7yW$Dp|0pCYaE>%!Ik37v5^o4xAo75HrT)8M zt$XV6kp7Px^8vru#-;v8$?p%CEEt{>@4(6K*IsrZW%rcxaQ?pyzx{hT|3Kctw{)Po z_8|5GHwIKZ*;CHL`TrJVb|3xu2ZR^CO9W-P2aPSffPMKtoZNrf%i;Wg3*Wni=KSyRe>dzH*z*m` zHc_zN1C0N>aL0Sv%i;Wg3p%_HIsbq#!*`sZy7nOUvJ29@r<{lL|1Ie8e&pPp=fJm| zpe*-5a{g7Gb2$Hp1Evx492C0$Kc}sKfUF_#d~OHIdcWfqJdsEFC#>v`#2*3|h!hjR zL!%M_IQ9~D$_0Q@goPo1_5Ymy!^xrO?`bcG^M5348s@<}`1)6L9~j;S#TO7q;rSv8 zdwBr=hieN(Z%;W7>Hph#&fm+qbN&U$2`_PO0kQX=|L+1K?DTHA+x|8 zgxLYP#sQ#yyMV)c>hW;?zXi+P+Pm@J@)<{{ye6LJXq&l5=NV0+7D2p+bKJ7C`^7~)?pl|bOo@ts7G;9d}X%O~PH|77>ycOF9ifco;lvN^=} z{7LAqs{fE(SYKdt*v92z+q4zhsUup|$~>p|q+1z!csc^hiF#omAHWJWaE zk`l4Q1f{hDv9xO(1G)kQHH9Dni7>Z7>F#e$c^AGE=4+@U4te2DhC&eTrz`%1hQiiQ zQM?{V?p-4hj73Cr1};#xhy(-fV)?H%u-juGTd+rf{??TJ?_%ICcqf1sg;mH<+#X2s z-I*w43#xbk)^_^`d}jwre+Oah4=5!Bg~Nggyicvcn{N9Fd^ZMU3*Nc!d>gV2n*?kH z!3*j~$t)r;o43Is;2Yw)QApb?AOIxQbBH03xNW|wzTp{XsIAH%1;yw~u zg*wvC^)QlB1VMXm11W*r1$q_reV2fI)vrWOM8YlqUMM{mT_Af9eg4#Lob$rYJq3x)O{!G75}bWCxZi zkk$f5F4F>cP%NW57L4jyFmfpeh~y|8AdB@tX$MtSM4dMaM)fnII0w@%>L3I#vLpnU zG>Svyq#wA0ikqk{$f$W0pc3SlLXex=0No(J6qPUmcTn*c)yb&Q5zrqZsRAw#sW@7H7kNSk>ND6IePQP(1I7rCF36)ZPzN0Y z)`qwZKLlCI$i>LU-kj?)uvr{<_cIspa%is0mQZ)7ld2Qc3<_u%@nT|WOJ@fgBTwLf zV)$E=cYN*tJS_(N>=b0@sI~WNPohZ#ZUN%(qo|{wb4`%8( z*`lJBuSA_-o8?eT;n0bK>JtB$?94qUC0v)kSdf-rni+Lnj7+Fn+}lw_{MHZ#(;H^H z+e|o4HJF!t3w20pe*2QY{Y2;>*^cq;LNlEfsnVKC=u`C1%j}nUN~7gN=RW!2yWlvO ze~n)+A6$x`iI3~Y#I6kdG2}Q`9cS|+;Jnag9a=Hg(8%&Lg}Il-B1`Dgd}ym&%?63y z%e|#D81F)TbH2?r&7}q0Q5Hoj6RKQD%MAI{ExfX@d89`SO;JPRUMdN(&&IRkzCSQk zlWE@Ru3=UFM(0&*=6pl`5TxvDfL%hx<2;%=YcubT&VaGp`>DPAYrWPj0$)P3x%=&( z>YWX4)S$(dqfK0mP14o-yh45j{kr7XJpR3Tx=n|l{Nfwvl}A(uWoG$ex$$XmEO0T7 zzGnV8RQZC|#{1s49(_UIO^34)zIn4DKJyP9LM_?3QnQ1ODBrldwuB#%Gw9&D#+5mg z8?m0zZt_d1p4yg?Ik+AVzUMOi(1!K7z$`Xa&9qy<2B2E=WCdzPv?6#^j zq07(HWIbOty3?&!NnRf}p$+^G%FEZXWEMA{Np9e*w_a}@JijXH_5Hhq zAs2;gTYYZwy#z_WFh-eLHN$oLWbyCEhBze+9zP=j1wO~dyDR*((#2^xh`Lxf#X;Ae z)f=ZT5O*!0HI2-HO=mi3T6dK5DcRj<49?@qXNNTADk%pA9j*xqGvvNMD@Ag4&R2v_ zIRB}&X?I6ed;R^$?6N?fs*X9ztJFlfixJCiS@#$jA6a4wXUWzrWQV;!GB?ggm}E|Q zS*C!v1MgNR*fadjBWf|eGPgIa#a+DuBltmM8!wksh;rWyj^p16dqOthez(EuZe}bY z8|I4i6+g9)PG%lcukD|^n?f&GM{0Xat=}gbyyzl56%t5BbR`IbKhS!5lYTx0vQgdN1}Z zfoYft(TiAf!KdJm3#OpayUgBS1{ti?elwB(JkycjM`2bL%ogbAAD$*B^QoV7oGFJJ zpM)|>C(M{ygP`@vQ%6#gQ#Gf|v|AazI^Zgl)Lu^$eZWgMo?pqUUn5w$2{C22EHda6 zDXm^mffPpHbPP09dr^K%=)HC=bq=Iug0-RXB}6IfzUSAm6Z3u}krXB!O?N25z~7+1 zxKfGn2|CU;mDWHubGVtY8_izvs*LqyxQE&dD5Pd2LJ6tDv-4?k8ibO?+Dmh+Hqr`( zw5vtTVguD@*h|eHUXU4k@7QE$7ese|{4}{a>8I0Qhy9t~)V8UBf5kMMim$dNI0^NR z4A(Vee87t(1I``xxqqQ{-cU2VOr`bR`ngYCetJ5htqY2V%A>ysV)EY~YlYAS7MAE& zimY;tse9TbD0R9Sa|MlrJd`$V{Gke0;9Q_%vEI3X33jZT=)$7jPuqSNc{5Xy zSt)rLx9Nmo9XWc_tjdhfSF}n*i|TS}JI-6Vq(lNN!O2?lJR?qfZMR0g7s93Wrv8ba zxGCv3runl~3XJ%r>{B{SAd*h8m5De>S(@Ux#!^+u&ausAr!=XUOPJ<)dNNe42T5*% zbe>;6#qM&g$*e@5Tr)8vN%Ksn@ULo6Nj5)K7;l1MesY(sH~VCgX7yZPp>B@Ru|Bt; zBg|BUQt@Qv!d@S3vaTkKg+CMppZXy!?_VR4US}kmJTObZuQx`xWcTXp!~B77E>`@~ z*k6*h`Qo!4;JqB^cWPGA$b0$}>vCkEFB3g=w2v0m%hItNCo2AsKwk|>lb%IwE745N zjK!&tx1_&3-sE-nOk}^h8~o~-nZ}~}WBw8rEa&W;a@v#!GR56D72hcgeiVgfs!|Pb z-=Ont%eQ+q`$v$}yQq~9rA%uAPvGC)B31utoJA0hM$WQ|y?$uhH9{GdL-IvMiz&;W!*N@YuTjD&qw>%lwr2O9dTBfR(M{$|{8;{n>-U>QG4qgIXr&z&myyc{> z+Q)OX?fr8^i|RL~#oYx)Few8A_*bf#{l>+Z7nBmwhu*38oDMiW{q-x?tIy8yI!;C; zU;3)(=S)^MUebX^ztFP3Kluq#$3hk>jrrQOO(2?j*)-C2QWGs7hw=OZ9kZ7krCitFM8^D9Wg1ddFUh`&!x8^cn>dNMiOGa ztEES$lc@B@0gV&?HuU!u#<2+WCBix7?|U34HqaM}arZzBefyn34pO{;X1c-GuF1_A-Mw}?>=?trY5e39R>YI5 ziV>xr0=<{tJId0M3)=T?d>LnmFy&mxV<+%PBfv@Pk&imhvf{8&6g`B7Q{*Ri>KZN5QNA~R_l{6{qA~Kp7AJD z#<)*+vsUN&H#yynq*u>`H{a%0Vas3}hkKj!R~g`x+fB&3Ejm4l(5^1sJgcwO@|8r2 z?liXH?x;TwTqqXDE1dxkLBBwca@sw86HWn4i(_tO^b|tAgpBxYCE|!$+^Mg zNFUxEDrfDUR>L6OqpeRGij$o9@d(2sJj^wjCjQKmskrs=w@X!bYsRqdM=UgKFTr2MieLxJ^ah~z}U5+dwyX%U{^sLBQVc_A53v!>c6^xJD zP0)+ZrHhX)MB=yqdS|6;UT%5mB7y6Z))kMyPREo@!W zp7Z$eX~UOTS12i+(0^Fj7-pAxT?dgb#G86r%sQ%`bZ}N%YwFxKm*@ zSS;7BF=LO;zLh^lS`-l}R(lo0o%Bh;1&?7fP#G`X1ZDK4yC=xrQ{ho(4HESQrG61` zt@JY!Y(vKm9IC{jjtxB0=rFX4(P6{0)E@_=kBI6@^Yr~JSWA8r;))rj?Jt$*#) znauO|e|Th8&CF0ww>-*b8DIaFb64?Iqg#NazUBF)^`IjaAvGFYdN#cRo+2a))LnCa zt5PHWR2&!Ft|t%mottePDaR>|>356tDT2JadfSsMtFD(ewWFS4fR?k`*5`%k(#xj| zW-*T{Zo8%Z@N|+ocBJ{`$+zv7VpeaJwE2BvSY4A7C$_vH3ZtAgV zS9slHx|5)kepR?{vXvSK^IV5d`;X$kGIR@iD^aoNdroPCz*R8S zXZ5Tt!>xA)T0at9ZE5vBZ%mx{7AM}!L$m0pub)Bv1CLtAwQ(`0;}OsIh}ZK`MYDx3 z!qfE|iU$ouy^6%VL6gct&9v51Q>4esXT$O1>1v%9nco@EKPmF7!dxun&Sb=350>(y ze>UCg!qEv0o$M{~yO1aUZ0w5Ob)O`G)|pVsk#^%*T^^pn^7--Nt71#G%6)5hX08W0 zua7s!4;9hc&@e9OzNA8LOGOvO&$t%AzSxWzVM(H=Ce|RU@3JW{Vd5yz9Dvci@gjFa z?A6L*Md#|;v~)PjHwxi?g%yF#XsX@?Atw8g_0t%OmjgdJpVTW+Z0IyD!l(hHI z1#tkyH$e~wFHpk+_C*Q+H8ft}M0D8az$6+90jeR0vJLzetc1ZO0slEG2eom7x>%YR zDMMYM9Fq1nrd;yCB;V3R!p__V3gQOp0Z`k^08!+dU|=L+yf|4pxY#@Gn!E4X)+%Y_ z423=WFW&n0W6g2H+<)xL39Jo-8fRPZ`ZY=;8bemB?0f<@>K*HF?6L!p|yflnI zSY2a>Xk1!A83{PK9bVi5>PZ5E`oQZDfCHy_feStX;JjQ2L?5V}!VY)m)dxxsurjeA zP$K0A%CI~@2^I|hpBq?264D1sJ+OlUfd)db24Erm-Bp`i`*im%c0dGoz4BE*Qpv6g zbSz*g!8oR#!C&yPB~zoif@yu|Mur`cVO7tCEbu$i;a)A;KF#kKZ6;s6!)Wy`J!trZ zvnCq$x)$e=EAtTVfWM4IesRPU5BKG|OYJ#9wLCdl3Zg2F6=qrRN}h8NJW&eeWY7&VAT_q%7SMd8w3>*;=J(V%X8 z@;KzfDsMO~p>lcqDavlf#(-lfzGdu<1%?w%6)WIZpOtU$`MoB_{32(ACEPIedgpfSdWv;DJs?b>v_w~~4IxcRp$NGSo)Tqd zHYMgYY*6c)@fP>i-z}o%oIyiq+K$*FPp_6I)W0craYY;2AgrY7wzrorKf8?XONll( zvJ(EWu3}{&$k&Xf&1FFQ)74Igjw_YxNxu27c`H_a)eWvbsyM=)bVR1=!@J3~fyz!5 z^qix~C546D&o%>cFdWj+3b~%!y_p(hE=HdttF(Qdt)CPgbbU(UkjDKc9=S+Fo;Dkc!XEuhjqN}>Wx6^%2yda-X=!tqNUN;CGO~DUoiAyt1qFA zY*@yIkdx1$F=Vvzco565OB})Y{22{HiN(WV=o8Mb&0EIj(;U&bW?0&-mfN*RyxbSg z8wD@g28KISPTg1@3$Xt6RyN`N4P~DyIq|B7GNiU~MR z%bPT&miAHgL*y=EJ-~5q9MbskTfcmCEls_UeM;^e$^5wwo|cwwK3W}b%en5S;GY`{ zD-3N+gIb={x;WfF>@*~&$NE6Uy|GP|ny~5I?aaW`VKkax;BP089_>_&k)qO7 z&hT)ocScqrnIY<7U20Z2>77wgMrP4hl1S2uMpdoOlc&AwU?bCj9KD!1`Yl?f!mlmW zZ#bR%x~$Q1i^eH8vA20Y;wjv^c_!uS9e4uYw1yZBI-dGP7djl1&O-%AFEk*wL01@?w73o}OQeWAWKY1ByZAtDs#;*!K?4EZ@&ryGpny-w)bXd7=pP zn54TM6IW*;m00IV#239s3Ay)H>5a$*z_}3%&pw~b4~y5XA}`91z;rbwqOzH?l`HYR z)MP|`qAL#_x3T9Pvy^T%LrlIuw8Le|-D# zlaOH0jxT|9u(K>tq9#3A=DCEmuOVL|>0n=&ihhbAY4ivc{pWDQ3b({JI+QC2`a(cgYA{7rCOj>f{Ed#iU#rQbcN>#R z7FS!dMl@98-x}vJ4CV99d>Q34pTFkDYU(lRYgkA*g|jlrQsd6IaG~b(?HLlZc2fbo zyZ(~SPpNbHHvC-*PM`WDfjz{F&y4*}V)n`MKwvq(L+5a5Q%b#G#NAXOk~`gNgx+@~ zDNdjJf+a#8R1+@U)RDkeXZ`l;VwsiBZBO)yX4^%E{%^J|(W88skLgtPQcPWn(9lb} zyZrUu4Z*$C2OZig=Jms3LIDpNJ*h|q61q~upU{FG&ok@s z{KaEBbV)0n1_kkI3Kuy+cj3N-kUHycUl%8SU|g*IMwHw^;^=PZB#2d_ax2MxBAxVs zMFb&>Z*)b~gUT^{{pL8QvaZ3lR*VI6oaTIdOHl9Uk5RSu%23}JlHnxViKT+m+7U`) zegmq)v2|`M4mvr6J zj=N1%j#h-zNynYHxnVAL8)Ny;mQ-e&*bQu11<;WF)z1k5xl#`k5-JMDGUm zVE9B2*Fn~L0_kR`Ew<&tJD<&i=Tvxx@gaIYFIi!GH%OksRuWU$1@eiFy9n<$zgQ9# zLx6_miSUU9wMgera=cN@0UA=bfb#6ap4T5J#>G8BJ^ZS)>{ne$?8GhThEgH=!59N;O^dY0&%xlgysk0dOQr6^vaG|pC6=(yln!q7`!3v!SL?ZsjJT_ z1CH_3oz~QDe_WuO>`kF}Kk!Om+L@{-2~4-b7r1WZ&NTOQZ{Z1+#5UnVK7S+Rdz~8O zQACo$AMv__ic_7~wpxjX)9HNAs?cmTo>AtprK_)uY0okT3e!+8l~(t!e%5|0@$B1hySE$hD={2irypLp)4i~O- zBIm_FJ4sj$dCc;IUHWE1?8(8TL}q6N1)e)MlnNJUHn}THLxPWrW#@GWz41F~s%_0E zlJgeN&g@nl9WCC6$9auMDKPpC`1otmEplj(H~c7@&CVW;dFgV$B`d5&5p%(wCNUiA zC`IOHHRgt&=4XQ6RuY8bUes~AOs11{#xrA@PJk>_HmaccgC1|n9rD2RsbNXkyeg9f z+VfPmy$CusreDmixr938n=kUxf3B&6W2((|{DvaG|srFH9W<~K@X zl^0xMu4+<@mNPMwoqA@TjX~9<_>@A%+gS@*!b5i1InzE%C5E7xK+4;U^_N#fnAulh zll57f!NS;6&GzJ^tDjSkuo)P%4j4FSoiKUT)y?HMHHU7@$dDj6d?d~IwUO}W{=zFV zSrCT8tZo^MhT{)uUnGm1XgRV>h!%=wM@Q{8=WThipoU3p_z2!#?Rlx zS6a+XV~W4@Gs6a17*uJkG;8dl2-akvn8MCFOnJO($6oM)!zi$zqgbtSotY=CZ^9zn$aC= z=qiJ9&#_$A8&fpXi&+HLfi>wEsgzWyS6*g34v1Bz<4A~eORJqX(UM}PtY~_pY4heH zR%?#>WZMSgw7GP(<(g8!x@RrR*}I*$8F9%>kNa1ko#-{itTS4~9L+gtlFLs)iSVNlp!`@P!Va+0Y#;VyS zk)$I}K~uD%?juYSeNgkEmljjuN-o$hYXJ{OVAR>Dt(fS;Mbko%}r{ zPV$cFrDsRzJFY+34A3;q7I+xXuJrXf4V#t%|MY5OqO(4cmkn!2B!20_wP!pNj=!gb zQ_oJeCQRjpwwHV4a524RG%}EDiz{<v>N&Swy0YMcKlow3+8@`Ki~m zE`=x2#H2PnS94DaT+vTUKlU&toAPz)blHRz;}JbSQUB`e!*v~LQ5>^P($Czg{ap@FmZg@gDo;5}l*L36AR?X&Cs~Ht3;kHqYm(pI72zL|jOWSef7v z*3iNya46^{kW}Gkm$^{uo-KZwkGQYz!Q#i{&5ORrrmiZnVfFp+IqSvmg;h+jSPnf2gzk zV~ItO4{<*Jzg1XvVqUf?EdTfo|A)mDZtnjnuAol0`tRZjtY88ws=#U~{DOQyE#?2w z_SR8xwCkQW?j9OaM4&7kyvX{d+$4{3)IMF@-+rHh=y7D7^e7kN(*Q{E(#B{)&hB82>nqg@xrG zg_RFu@j8MLm7xazqF~YS|jRLNwpv z#>jMSq2r3fhQ95U!>PC$h&)1IX(!?}^JK4>cq_dQ~edFwF{G{)%J6!YM1X!gs zaX>V!4*Ta}a|j&MV!8agVY_1O)pu2ITMXjZtRmas_(l!-Ne5nHYHZ{3z#5z7dXkde zj%Sk6P(^PC%(4kUQq*oaoO8Pv&6Px|2r~C<#8(MOY3Jz56W_oXv>EGX7cBYR4 z+~0c6hv3KhQLOvB?&+TOQ_IR;d%)$*H6ZeSTd!3gf&m7MNC1Th1PtNArBbedxyL)Pma>T zK_0n5 zxAxCgO%Xt^kB)|~`r4)*D?v?A1wx3U))`;;1VTY>D)d&}-zGb7Yy9Pq;|SYpd<`FK z^%v1VxL`M8E9m+CqTA^TL06!O_-nmgc2@?}_~Fv(L0(^e-QwaR`=W~ayJNjB-6mUA z0(a2_^q?*pp)WS5#zEo;qB^A@7Y=kK42<>qX&H?Ho(ffHkW@E5lK}z)Ozsy>QTKcy zAY8kG{3PGkpK_bpFr)kiIbBmsV~9H?-p$)R#yRgW;+@||Vf2AFtbUug`o6Eq?<@`T zHZQ_H>DbaUU~oLIg`jY7`uvY*&S{;DPg95-{u2D{5MKufMA`Df2sV8oMRFbwv<$QY ziu6IBu!W`CMf5^JHp(7o1oBTONZLpG>8hrb4pe;zQ;nMWEfaBtF> z7Ba_+_bR!4lp5sZO8d8%?rpX5uMBF3oKIJzE|G5>WzD8~=f^?oWpKPgu@}^v%H7M| z%c1j#$Wa!0$T;sy=p3&BO|})8e!bInkDw#!bnnAITmn|aOL>2_`X#xz1_wL@n3GV*gPQO6=2$*c(rHtF|C@8@}E3k=sv-c9f*v~ta*JC zyg7U34A9HyBYT5lIZb_sVrhnyVdg0&93T#*h&9ey5$C`TNuL_@0cz6#yL?z8YP+!SL`D03D+tN?-5?(^cbZEt1CLtH!Dv z;n>Y|!n7L>+nwEw)P90^=c&fhZ_=C?(FQg(QrK<&HnO&=A+Qaiy?q+=2jkr?*!Q-b z+wHl!x4>(Ub!ya{I7gyFsS)4(tn27|`LHzyKePFB=+G00k!W7OmZrbrY0QOJHV=)C zwUjYz3H-r@TMLJf4hl(;JTlbx6SG@8@ivp=7lv$q{aT|3i#Lm&I@4vEP#!V*g`0B( zc8JWl)3gZ>hFQ2f9YI-`@$KOUWj>iojAWVF)QGY$@<^3rS~XO%Dwed-gjBOtdVV%h z8t`H&z2;3JX9{p{ z3VA(G3;{py>+ZTY*_?dCKr#mer-hK(z{`g=Oj>MWt{Ez{o>QD3F>s#D+mK1O_uGe!RcK(**;ts zmc|l8OPpJ9jUE<_0Vs9{L|U9Zo(iYM%s2-Kbq9jlxayeWxav^Em5mkpO|&_CNGpwg zC^g`@+ZApnotizQI?*=RoC9FKk-$o$3vq|YSgL>zZyj;X=mwxOP@Cn1grQ*%SEe%% z`J(%Mi1~c%MyHEKr(0Auu}q^MvfR-}jWuSZG2Aq301(p@;G)wFm9uSZnkK!3Rk_Kv ze`Bb#@ya0FA4>6@#9kwv^$s_mvbX^@zY$8|1Aa_*+{wi^YSX`Fkrc4gEim6(UaIQafc5ef!!hvDc+PLzH&uWE*z~>sQ!l2EDaCRJbWg^Wi^#P{R z>i2%*UD=8!E9V*)8r%+wsd7;uzRSFjEUt@s7tWyJyr7zn-AXR*4gH&?>}v>;GM zuu%4Cn6xOGFI6S7!se9Vm15wKC5r}(LUxl(6i^H0&qF961@YHR_3_`sn4?m}Gd|-t zA`~D^YDSBZ-iWouk5sW>Ke-J~`XOVRCH9a1U|%N=QdO1?jTqU$%W|FJV@B{FDm+>e z(*h~e{A9i_8`)HwDEX6uqQ{mwlQ<0`IpTovoT0~B&BaF~R1|V7MqH%cBu$g;7_@D* ze3;P*J7EUFm-&b;P%+u?EXfek`ZM;pcm__5aOjM6b*BNU3We=Za|c3M@^y86JbS0@ zOs}Nn%+lgeciHT)Nw``W0IDniaRsb$WbQD}k|_-&Gy1yvS3FXu#e8p*`;6i7iOKG) z*(Z=L)7r(4%8`xEi2uaeA&(_f253t*;r*#iA&i6^pw7!<=J4ReWPjGsoZ0)2IFvaw z{tE#bFMGTIBN1TF1$sO672aN&<^bX1yY$J)&W@^cP22^iD5_aYy2Yk|`eaiFg5pjy zpUE~9550|q;F@)W^$?0CAU(>TO0o(#Ml!E-hQ^&O{@Dz7!?CovK3=c$d@2-uQ?|0K zHnF#Cxv#roWY@8WMoUL$L?g8>_QSU8d2KUsE@fWH&emzIH)-UplEO&sHK8=Yj4c-b z+6U_1Mw83V3=;g30ZiurUmKSr&}-Afa9!1_L}pKow%8P-I6QV%__ZL?c@F(soYs&C zM@i5xkb^W}T#oNyQ@nGcoBj|PV)J6@RcAyew_`?d*2oR?8%iFZRs>j$D-ZJRr5Wq* zMe0E-U~hK?13FyKp<8fcN}n}~eXDS2526T(|I-&ysI65Rtf4Ej^0HcG7{lNxfT3Y( zgtI7dNUucPP{mG9wqPDcF=n_}0DVP;+Li$GcSP!m%+K+bhg)noriQ7pt|RuMgzWR@%?;cr_!xnBY#Ywd;qLrrD>Uo*e1Cw?%kXLl+n<9t!Skbomo+KQ? z{6)YN2!GyyKi0ufh`HDDFD_CC=-rT{8yIpVNZfVJZ_D0*@B+D#Pd@H~dyBselmvJ> zdksjE6HQwB)|il=c`@tLWY`-5e8iB9A4Zo5QIA+;V(p0I!_DoV=|OGHyr@wIeTM4zsBVNdRD508{E1- zqe;|~%0#QI=)a=DJkq>=Ixy&kYuQ&Z7^W4_3y@SXSWq(9r3d%Gkz`^Nd;5a%9v2uz zH?j|k-RkHFyMUx?xKI(u02@e93R`oUQ8(9I9Q#oxd#uhx5PDGe`lgwVg~E>M!j9pJ z)XW=(+F2V5sg3zx>n38FN(vYNGBvxD30A3?B^ue?WmKhKw7~BG=5CraIpN9lJ zx6w&~=Uyt=FxMwddG}#;^$ey`sq(cr&FX40>2)y{k9Pd-@W~4`itCIN{WOQ6U@K_1 zt-v&p_T@(7(QW{R+4FTj)8y25X+wt;rca1a&o^iJ$Z`2|pLm1`{#U!5(l&i^zh-r=%cC;!CjNjEC9}6PD7vK*I%}0Zp67aj z0aO({x;V#CT&IJmB3S&zRocYpqbp;N9!_Eow-mk1Ov2ad-1n9AMD5Bqbb-a6%s%OB zyLuTQ@QkW{{hyYigiKx_56IPRs!%A%8)A||jo;9kY9(`@SJv8gJrcO<>+c!JLatKTbOSqi(JKRXu zV$7mehw58xIO2Ezao*tPl4FJ@u10}OFMHDr`4qn@b-5Co^Y~(}`BF0_Gucv=|9c|> z)7TbcG3koIAgvCHAFhs-QE;OwqAwVow&95BD!a|a(fB-ZXW(#kb@;_^+|Rh3oLKZ+ z$iRv23mOFbz^K*jlKahP+i9k&R4)ccq#M%mcb>d!C$C)F3u6i|T}pO?S)C~wc6S^T z)p?fb92*zZdP^muCQF%T43l{xiUzXe6>>L88c7G*bbjIYyNC5@7XAlhzlq;@l&*e3 z@2jh?bh^QNp!=d z!Ambq(8-23fq&lInNXOTCg-eD&`7Z|@3V1s}Xe1puP>%su81=NFgUXwiIN8b*x zjuL?dvvD5sE`J*D5|RiSnvyNXAMR+XIzoVne7H}`*!(dxzBsDwENXqrao<|}GRO9P z`cK)R*dzoEwFQYQiAZVEY|TkvQ{2f6EX_%hh)Xge?1LkSWe~cI(n-2f--F$8Q7503 zM?**(`*5U{(PYAj{9&F@1Ocd{B!QwLRbOFp#38^Fp@y?J4z6T z8SXqpFI()`V{`I^WpT}6#`n;c4=8~LWl$iZFL6cJ{v1R(mrTJA<{qDfz|WaX4s&E( zGiL0kboRbX;%QfM0Pdo+e(uP}(5FfH%AmRh?YQ}6Q-IFbfN?9jh&Net73>t>iw8Hi z^bU(xZC}=*ggxOnp}Q?-RAXB7G(v`g_zt1CX0~y}fORP-Kq7V|t^$!jriE^GHlXc2 zTr8NsUNhQEUg*xXN#@Ocsn5IYzDs+jMMVfN6H&xqw*np2BLzJG8I=rG5E2zn3__kH zP&#}=Q{=)Z@l7i6%`)*VgyMB1ysoAYrhSMK4DsO?e(wY+-Q1nZ2^EScWKOntess#$ zBvd^YkIui zr7=HlC4LO$_ps*OtFD~2MXfB+J~DxJ(Q}gCn%{AIWXS@~OUP?vRc z5C6{-S02jTxyXDq`(-(&rg2B4{f(?m8*+0>AxD8p>GY5A;B6Sg2_VzBgkw-Yg1kCp zk-lWv>UjeHJ z=Izl2;J--;6lGa~I>Ik(`3GLefjcY^JLVcIJ=FtlcGI;%xnfK;?OnxZ7UI=WMmges zn~_leD2BYRhZ|YOzwVmbNnlm65I*u8!~2cJAWsi8-W5^rHrGqzoy@*U#|Q+6Rw@dK zbGN=Yf+^}gTi(fYBKnlv%+l31Kh#yNsqRKmb~fco#ch_-c)(b zOoKCz+$_RGaWvwy$6Whc@9gQtcU)7|4!6eX#ZNzPnMBANTUHq$Vf;4i!l7HR&YAp|Z~3Ycb9BnUQ#P)O zY?Q5|(xx`+FU5OmPyNDY#$6|0=-4w5>-i=I6U_%GQz%S594(AqP6}nd_*P)S0rsyR zn}n!T(njXTCyHjA_N{)wJEf1VKq}yQGD<<01cTJL>k87ZZ|M6mk%KKgp-kq-<3l-- zv@j_+wA(j&#)j2QbzglHFMqujm(a7{P0de!kQ0q}pu zs*bx;V*|XzYZoJuq@`kRiwBf#X-?F_`_@mHP4~u;k#qSoWN!)Aq{r2~jFL%*)_Qt) z9Yf4(wolxoe_0@xUNgLPht(vJyS4eXX{l?Y5Ffo^AF>h|-e>c9ol-fbS z=eAG2JoCN4cyyvMgvhnhMPVbdYH6dZLHLQwtWOH(VS5U8R$V9i%N0}*J#J4w34<{V z?1;($5z;!}KQ-ezRfbl~J-UpS3dwzX>>5XW@p z{WVa-%hGeXHPu(pCqQxx3s4Hs$L2|L z^h^8dt`}@IziUhWa+^#7^OgKTv;JmWH)NIisBW2O?(V$AN&La`+gPr$w4{c}(Xa3w zb)FN6E==}zaF0c{cuFa8$@G>i6&gQ7Lx-uUFmCbhzBd(W$oe(=n(~aTko# zL>c^D+Jm5m!G;olkmYQfDy#db^EtI>#*|>CIM=tCwKDB8YHrzvGk}qj&eok=3ij_| zslf)W`I)6s#-i@EnC7g6TCd!RQS=`0Aj&jXB>xvuz(ev#4>#y-Kd4T0m(VEwe4mR0 zhooQio8(jC{#su5n!{CQi(RdD1vDfd7onaBW!)UCq|rWM>2~mrDU2PoP)jRY)h9IW zbLoUKSKRu!p?)ZSYl*Hl%KeA&tV_i5C&S_c&waG=e~8O}3})7kjSu(v;~g>*aeQQzeF)AU?2h>( z4}s&)#s`4|{x#0V`41!dBe41-SK<%I;{g6GNq;~*mcNJ|@WZ73;CUb3^nVD{{{Vj< zNbfJ>nuY0u2Yw*FkF0?Y0sDV(rT^&Lf8XVQPo7x*6*cK5$VLV+qJ*A50yu4v`@Yl> z5(!z+x-THlog!B`ID=auF!4T|W182&dnvdlw2rc*LwI;lN8j;0*a!XSyjIKX?*j5< zf%%5X;bMrjP7N~z`_0a$KzewTexDN7&u}%c81}e;%_eBed>ARkKC3e9YO(6Hn7K^V z*0a%2;4Oa*Ac)!zG{v^dGZlQ(T5f2DCNQwpW;bZPo4HF-_oHm5faeiH8WKZ9jH*}L zu$%#v4nZN=wopVD?eAoLzl%+3Za8Xjb=NZE0g#|A%4}jw_KT?J^C~$Dd=(~Z5|^UF zoLP&x+xiqwHXMOgm@TkGIAwbYELWSL5SoTFt{E8$vkX;{Q93+mB@pChPIWvDN(7}( zNGTMkSjv~nS=s5VLn+x4IO6!xirP;ATzlVhOwd&*>F9x!*=?x;!k!w4%@`$h%HuÐlH z4Xf;U+T3_SV#LB|@L`?B661JpdR+wfjKwzB%o2sh*;)XY4YiUP&rDxj!-AjP`}_OZ z^85NKI1C6kL4bdo*IjBe()-~ze$YWMf3H|R#$uiahj%Lw517vH#>MBEx*pfVwf>h# zos+KX5!VBuYHufr0(l7T{ju_6e9j}U@b@5h=O8KD1I3iZYfn=QlpxFqdBZqSgqSIy zXpobR-mYig;uAv&!Mf1|=(B(>&R3%i1P{{w%^)wQCp*H$hui%m{-z+qLqm&)E&LuG zFdBRaShl=AM<**s$_HQk*zym}Z!1PA96ul90p*FfJRH37O2Z++JOyQjkK#p#cHX;b)A-7a^4-4h?p%=9!3CQq8RT+hAN38>=i^{yo!2;Q2A3&ffS zQFi7Y*S9CA-g&~*7s7%iW8WTOJR({}BI_hN1Um%ZZ#@Y&{V%LG=R^4!Uw&&asYS(N zy$AYLq`79iNtB1LV|Lipr@v3WC`+%^jMR)=TJJU=9vl;wjw|ee+dcf^rWP0j$fbL60+?PDCvR+C-DHdn%-=tZ-m1|{6bS$ zNK*||^MD0g$|Hi{D7W9&4+Q-jP7H#Lw&KwW5eVlfK>`AOE)Xvahw5%%jBm-<4Z>Lo zhA2B2^un>@^7K?Wbo4$il&kII)b{2SHND>R<=QQw?I|0udCxSw)byhKbbEe#JoRhv zcKL`fP$)M4$9m-?aqpl_^NFhpY~P-kz8WUIA&XV9c)Z1mIDu$#C<^I)cdw}#AEBTy z2ta*ZDIi9dhR$n1pUA{lh`cQJ2Wfa44193UJ-SL9k|=gIwkFX%hRP%ofGm}+1cfW9 z8uFWq^}4&KP*b7K5~qtKwsMTT+-EGUN0*PCPtzijnw@W#*ruRvQIwdWlqAQqhY=BD zkN_z5n1EO(3n6S}zP#8AP`>V>h-oRokZR-98X}Xsn03DPi$Mt!Lw6IB1R0tFc>Q(m z_KVwi2}h&+yzV;N()@UtTFny$!(o6*+AqbsSOy+6O{|(JB6-in4bkS*lQ!AJ%=X2Y=w!3J0fJ9OiBudhYP>_yq(DsWSFH3-2}hJQ%L+A zi(BGS(cmn+yN6L47!x>ZrIyg04^=r+*fff3u0*tW;s9x0WAsK0CY(a?-NAwJVkbIz z9fNt!K)rJ-xo`*uKS%BE$`k}TE5n3UM%xz(i>49d>597pYy8cE2|b|m+$V@**#yUh zpth{ZjjBLGG16|iLa;m8v$|nr47uetS;w~#%F+Xw_3 zq?p8l-T1(9o2)ILbn?U(TeE#uIY&h0X7bv!UPZk9JDForIU~44VBn|LT{^ivW+&<8(9J6P) z>w7%w=i;VRrxj?k*lyI!P2a3eRQk5qcXh@X9yFWf`%D^oW~xD%y|?0c)4 zZ!5TM3&4GsWvtqma^u-3(Cdg*7rFaNVLvEy2OP==nCLl8-w6cs{{9?s2e9I~%PISu zZ64zk{B44qsUl>@<5;Jgy$qDhe&H4R&?n8cbA)rwVp7J4ndVhwd$U%|aLdt_OS+Xh zzU3zc+aKx=KxJT&aSP$TLy9)fh|fal^&NuT^?P*FZyt_2n>5_#H8u_nSg^DjE!i(8 zR=!S5;AW_dwb6klp)hyzDl#M@iJ!BT1t2yGC6WY)Bpm_Cou*$HIEKmdP{>Hm9zCAhC?*2RvI;rGKOXjES+#z!&#; zCvuT>4!>!bhXK&aWo!}=F*NAJ&WgJXt2GRmaX*`-joEYBsf8tHBI9aHaoJRL_y1-; zj3d&WC6;8=l)+%>QFqnK`zc_jV(q^Qx6p+2+~OjEMmE%2%?c!!4E(|*Im+Q~a!BE_ z02+p9Uo~C>?h1TZJNS{yE=N>`$~erPiI}fi$JBP=QSN_e<_rYEd^YCdp$BfPG=8dD zN`J9!BFnFH93&s9;v!E4mxDGk+?PGN0OLd6 zFb+n1-UvWGGiy2t)lMN|!8dGJ9&>KYw&(oTtGh6GeGK47xd9cNEN~L zqP^1DfXh; zrajY5%YujqC0`Eti-5;Dmf7KaqTNf5Jh~c<-rqWJrU_eT5yX0 zf_@f9izY%9ULh1NUx}NFKKTRWXSvfifTXYg^ocoGL|PPZMlqZc?LuVbl99wvg`0E% zm|CH(6;RJQ*&hSWy*wF6moRoEf;JS4`xV^YnP!O0TQL?*lxEoEj7}2I|UwpXnv%y#SP25zH$ze5YtQk-Aq_&3&9HA@Akd%3Qu zb}rvuMXh%pZ-+4qOhB*okGd z>ET&iRX>Z0`zlV9yk?Ro^mdQZc06*c?_>62Nb0t<_<)4UDd-JR zHyDwar<_E|Xg|u8kIJDmyivtHRW*D>JNf}L+#QkR;M-5sb_94q5pIzi!CtxWH2-ky za3C^j?9ZWGokadx3O@%~Ur3UZuuf~&ie)^o-P^dbl|@?h=B?lRE|aYBiMqmd#Vryu zl{ssz#V3(&h{Z?mBF*>N9c@3a$Dzd;;_N8G&dTr3%N`CFq<-D6!JlU~GOmGkzj8`p z8j2S6CxYRwZ7CUQfN@Wdqg=7I?^rE#%B^Xvp3t~??wH7Lvq){<#>TZA$tPYrI}~|t zJ|gpafHsFtI3jXeTKDzY)i+x*9)hRqoV36-7j_>g8R`YXC8>my1A_V6aL8aE1G1Nu$BZ08q7vw5bk zj>T+0>ULx427hv6U+<&isMH?cs#JBo|7J*NoN8A4tMD7t;_5(^Vg>2XH0HjrIr}@1&{eRFj|%5r}{h-ULd%({lZxI0tZxj*Sa3kq(eTiM=Se} zM2AkrA6?_vCEaH~E+O;#yB&**iz8kEO*P~1$fC{60RrD?mEL!E8LT^s`OywL%()*Y zkw*h`Xp-U6hSrj41YOMK4)Pji7mc&-P>8L zE6V!w+fN2LqmtB&;qB-cD{JcH4o5|GCyZC-6rP)>`c+-Aan&mo7-Q1=3u*dfLzO~% zAdpV1;U;r8z){sUIesbg*DL(!n2Zx%pvn8{8z-DY9Ki3mc#K>aNp?(hs1kqYjGSIG zI+G-c0ECz#erRCmur|5b&u<5P;tJP+?^3l{fd+dXQ~Ow$_`pt{Yf%f*8B4) zCbbWUD_9x`BlyOP7KpF4?AuPpOFG+Qq}3Iz^}3q`E?ZS0iNcQn8*uc+)#6|EpG&{c zTHLNAe8r1ISVQxQ0Bk~G>$24yEf&r8*~Z$V%}pXvYH<$0WE$d(Cw0m1NXe7S&FyOs zlYKY63MAE2K@2qGD8(BQ(A!0KLcsb; z3%ZGCH68&6c-)tnR2$w8-`GvYChyw_w#gOSsVN&Wm z-6ciSHZe9DP3IjV4;%Y**n&4jPBOkgdh@hnNOJAw;V_l8?ZDT6q0Xiw$Gg+u%%<gQ0mMMe!`R`}X41@2JPffbGMq`xYLd2DB16QWJoQl{Jh&~id3 zTR1wd3-L4%gVWX+yr|f>>%QXfS{%YbS4Wgfb6-7N!rBm`3uj2>mduc4>_{uG8StE4 zVa@!4eRb(aIwwIqtNX~rX&!FNS@YA+k#2^|(Zv3x=Q6DC+e;_o;>l(4#70 zne&&2!BNV|nsC=k+?!g~iWuhRz-IK~LS1in2HZpffwT_Y{Jv_V4qN-OiA$yQF&UkO z%XWg8)p7!ycEqi!p}vwK1KXBdb8(B-A>0k)fh<$P9ex_WBN=*R@Cx=sHZ*cv@v&Qc zF*0RR_ys6QRBSyyl%XAtMewpa_cosM_^iRqW=#_-QcW^ki)T6~xzDd|#|o7nf^!Zj zx@->yfvq_<>7u@bWvmo(F6~}UMK*flrzch|q$C$R$15z&rj6|~p~GIDpH3l-!tQ); zxV=A<=Bs2(=IfoQ}j0X%^?7~*`5z;{~pX4ovpU3 zlUw5PbIGp5M!8c$hI4NW4fIv->LbYYdA;mInWKz&dq~mjtnTSEMz{LR<_wftGHRC9 zpRPNU##v8LlmA?;J@#e$vQ)V8u+cJc>^|lP6f>oN+Sww{xjb2s%97;1sJ<0Xjw|k7 zfv)lCEHCK#X?8NNqEs}kU3`oTwF#^VT)-2p2yBWL_?iln&O2&TnU#201Gnn9b6vDy zHZ8$}!*XevC%*wRe#SR7rGILqwjNnlKwB}1T#_O^)jz5mN z$6mz6Cw?>M&{r{-PAvZ*kQh*RkWTUWom`DX{xM&*T8ejed3GK(F#dgNX>IJjRF50e zCYf$4p71Lj3StP@)3`}!SZ-oNFf_9Tc|)SjYPQEjt1I)l9fpdb$_oVqi~^x!CYqmi z$kS0R(fdCJ=B-Hw+^njPXzMkaii%jwYWujfwT79Nz0FPrH#VlKWz0?vHa6m`5xzwP z7}2TWNsv8BPkkxnXJY+Dnz;C(v9q?3X09@gIo;E9bx)0KntpbJzFEPa3_{XY;c#qJ zzFv-)->`ld!?VA-S%ju^?T&y2GJ#W$_e2cL?Z{UQ;p)^IQwfX&~pdYhX7e4qf_n<%)bLpKR=^U7jSDymg}WzR82Q zu6M>gi5YE6KuC9vI3qmOEg)pLkPL+u5g?T0ZDD#6EmOZe=hfm>?1stGWb@=r)FHhn zX>J}(n8Ud3ef%lwpN$NGyS@}cJGkP(+%!H-N+7aOt*6wTw?cD9Bu2Sr_*n~=8s|D zH1Z^%STWcQHD>+n(JfW#CT&LV`{ejFfdLk5=1ALw@Rd$z;r;AoauEJaEavv$88$ZZ z6!Djb*(9bKd;6=>CY^Ub%n=f*@3x z&SEK%2<=4MH1*4=L2huf*A9(Fvd_16Dfc)1%Ww_;Rc^E@d1CFdHft>Nb^2Ep z+g=L(N!eyr*%Tp^Pr(h(eF!&JGImK^TXK$ z-==X(VA?F0GAm|txE!{e82lcublv=78js!|A16LdRMP7~X!=;WV48OD-4H|7^O`(& zl~EEyxjQdMHQQJtQ)Mt~5@xQePG9#>@<~kiTvf`%moV(#?!R*1BhE}cnGeQ$_EJML zgE37ysg%4CGKsb>{G00UpTO(CtG|CUod5O&o4;2J|0l!|`7a>O|GxyAe_2)hpD5?w zU+4cF&-vgi{~OQwd)f8>uRP~tQvOb``9pF3O0NE(I3KAq|0Fse+suEh!u01*&X3sU zKckEv$>D#f+SOFrC)S_fo4+pcKLc<+$j%1{`XEPt5ulIg zo4*cZ`)fgjKTPO<&%pUB0{idJ&i{ey{A(bKuF{C@2iIwRruCLVijNN9Aq)Nu6xx~$ zlez}&)hj77qT9xMc!pz;EF!34-OgUbpOcwOUCIzx3?ozp4{+_bM4oZKhdV&@G?pYO zVW%ml%CY={Z&=gLdW~*SV#AS;pO@q#WcPi;yxC)F8?ipP%?ojDA;-3&W)W>}CHpx5 z)H*LKiF|evl|o>&eR0o5UJrbMx4M+>t37jImk)3~&j zimGZdMERRrZo$!9>G|BiQC%VAeqbs*+CX>>S^f=*VvO4%Y`lQ8SW+K-+4|w;8=ja! zdfY*#VJc025~+ts55f@MkThAX3xRdpKy}S^0a?O9x@xQ@X+#cZ=m^t*9!b-lz$57P z4%K?Wi@lDrVazTE{ju+v#A+Kf?p9{FmnORr9ZzmF?rOePyFSELN#4JCn*Vc?;lG~| zHV&5m#B!LJ*_i)xk5``-tTo=e$J^R+Hm_M_dRx=}34N3FnJj%%E436!>CBudbs=E! zyNHxgK5er#q=QvwjVg7xL2n5fhhkQK^rr?`8PW)rHFk$M2(fY~80|WX3`XjWIhzwO zQQUa}OU;opFBa*)C?xQS-Yzk~aa-H3$LF z`K|nF!~E3_WQQ2UCrlon)A#sPy&VbW9;E+{bbuaD?tR(q9UUeK^gK3Z9MA7SYu$;s z^U}D}^XA~>W^_!yoewfj5k$3~$a}5|CJx4bu3%t-ug;{WfieGEWq*g`(nRxddfsSykaNbT(wW*~Js;nh zNRU@A5RNYE%zicv2Rt!_DpYW)s8dG&&QA5J8`1IYwjhwuTzXR!0LD7gEFpS@Qw;3@ zt3Gk(#&N;rv#zjRJVuWl)@Dd_$MSf^`_&akj`x5{Ec|@*zJS5(FMrH|ODw)12C%p) zn<3Z$E{CAuC}#}90a2Yk!YwWk4+~m675~IFe*`IC*>@Z;|J)5qkciLvEf?&yW8LL7 z-6e;&XAAASXZQ+4TY+YYo5c^s6wz=uP}fBB(fCu zL_Wu_e4*F{20Y|x!LQxEi4)F~HxTbHMV#h*c9)v%7LVU|`G1<1^P}S%1^6NTxY(0G z6qC5vAC!)Z7sMlOG=xSOYJ=do`l9`7OYr!I*9BS@RRJO#*a%0{Wc11p0PLb_ltK`u zSg;cf-ta?@Qx_9}M4LRXisdK5(vBBn9zLAbMkwOh%DsUIZ9a@ZD2yA@I=CRDCK-x< z;<`Y*D4L6u z)`80x?chr2w+H`0Z`9VWy@m(_gpj{UiN}2p%bP?Md3+)+Jf#1?H=Q-~=-{EWSO&*g zzA&ooZ&*L-AQi7bWU|aQFH0^t7*Al5%H_^6YvM3EGGh{xB@sgq8#N>hSzwVdQ3?76>F zG4v?j-+f(Rth^~Agq{2THa_%t9{b_1DnkOko5}(ZO)|#!TH%!I^SbjL7FjsoCER_T zkA?xSxU!Uc9e%q~4^oy=+LGidU_OG79Rri>6$w+>S*#gS!HYsP@lrA$XEKHa%)Qj} zCT<&ViUeAav=0TemOQkMd(^#(*#UKwAjdMGc9>?g=tgaO~>V@>`9JRMeE1z#q z?uJY)yFZ|g^~A39e}r=q+_iUv`Oxy7Oq5+XAVc$!Aw!?{_I!5R*_=G(#Tze!VL}#9 z;pM@}8IVHeuj&8xFsVzRyXlxhk7Ar1J_U!RZ8)`$jsb#ek2b&<6b+0}ZuFl?>p{iq z)1>+`Kg^47m2CO6!LmJ)nK%a^c>y2cvYrz_f>}eHs?b>SvYP z21B>*cEIa3qXI_ED16`>U{4c%zv1WyVEMo_X}hiSSF{E!W)27gc^fElTR=Gn-aW=h zmJfd8nf+AZWB$M)mfL5MwVxG$wbGm+8!y(oi~b&1AC451NL#su@@b1{q`BhHx9O| z>43ZJjhvM_qtwjjIwcOXRqXcju@+{l|)a`eyQ*VmdcEZ-LmZ8t5lpNQt)sP5Mm0R)+S!a59lw=6I1GcV({ z&CuT+q~@P_#QVcG6%joM@j`q?rsiU=zIudQr*}!NHh9^Cnm6_jG$Cp`ujKSozcpY> z(;6G!|5#Xf%~>!Yg)!(iWBK-|cm37_7TJezfeASU#xlW48#-jooN2(03tBK{&EE5c zIYZVT_HJD`bk=gpbfNs@*s?i2`a448B0`sSojo>Ei>B8Jco-mmA;}Qa&?oj|!03bL zm@It8fQN|eMBwPzty}qqfXbd(GGf29Fwe$S1zMIoWM5M{M8eqfIZnrorU$Aed&uBt z;mCMr|f&%dF$i#Z%CmK4cc2+P%pL;=k4l;T#Z@ulsa-0J^Rj01}++D zo36pySsJ*q4TJ>lZMM2%lkgm^R?czcnt^oHHn#C5*zBo;2yP@ej;r4g7`*wAV?Pn? z1VOjYIxl{1F>N%ZAl-3oNk?daw-=0%{OdHn5J7ypVyrNPU}cB4mCMJW^F4X!*Pd-v z((IK8gA%@^;XPF2?S7;+*qGs!h(3@{7uq)bY<@$ zy+H}Vd?`pbKM2hY*9$>?>|!0VRl(sM=~s82(QOEc0aQauC0cPtbOcCLOGDy#Ti6gH zNS)8Vd)r3rdAxT4!h+1$Go~hq;iT0W1Yd(mh()@QI^z+=N$eXWMlXn(;XZ5NRyPnz z*8QTMw56{j4wz$e?Ju~b+G3(8K#3gKgl#b4(^kY%yS z$ZxYZjzv1u1=B+t&26gElffI z`V-*36?SX=sGfs>a`mR16<9RPIXPo+3zh6M{-7};e`pLy+dack$Ey-ywd#BM8^$G? z@~h2N_s^pl`59ay35HippNKa!Y`jWhC|q;Q#JMm-Aq3yvI57m(Ud8~dJL1ru@J${g z_g2A?+REaaUds+IFwnV$as>joeZXujh0#z3MT1+PCI% zzR0r+`7&K~KJ9dFj&Ps1v(Buyzb^ETCE^=qhPWw)csR8A>vu+R7(nq>@gdNTehI+FXDlEzvkt&kW;8}J)A7en#l0!U zxiQDR!Nk2$HwX>V9KBL1JRJ7^v^)pLJkMzcrbBPu;pKA}2p9DqV;S2et6*Vv zj+enAG~qBf$l@;ykz{HYIu?t|-LUd+v#PxdJKDWsI|i&ZsQ06wn$b#wZy2H=A zM~)M)OV9+1mJpExsF>sc9xq&8arn>I#R#8npmZJvhl?$%s!1cV;4)RVJ=Q2%o?ZBG zzGCwO>Jz^6nz-@LG;ul|vB(wxI1D?g2U3-P#5arNd;$`Ca|lW!G=nXK}_FS28yqYJOFNjJN(A)&AtXW9B-@rBCG z(^D<4ibOrd;bF*LoeM?6E#$aT|9!as8LDTq>zO@4{%sU(kRox>KjfrUm3%PjY-ral z*39|2UCypE5TX5Be$cYHZhfV&QoN2y$>trbJGjscALwV4Nl?&NlqyKD?|*un=mk1= zpgdjkUx1QoY!G+2%Ijp9U{$_`%1p%_ViRX@x9fe7pxbku4!#8&rgXe{8}IgV$R>DC{wH{h&&#Ya-TLMN$bR;XJjh){RFx)jHpN+D&fQ1D0)sbH{LKY2L& z7?Fx4m{7w`fI(?&h-whDoFnh6U>tZ3D)>6pAHegKtTqc@7W=#Yv>?qKh4{_4Oh&?TUf3-|47YJoAgE`c)ITB^^zx~pi7 z+Ny);d?`78N{gE@L~XDAocgHzxEJc znxWn^q25EG-p#~*+;asjqs*S#3&+2M)b_%5e>5;?=>%6+$}+b3`w1MGGLn^fGb z-1Q|!PmLb}WD0Rw){BPr{Thm_Wi6OHFGgqj?!*pP5}o|3*ij;p+=g8juw8Ze%g8i- z{)(r^zO;B9z|d4mG~qE85l^~UCu`LTAtswc`4$=h8?ge78cw>)FUAkZH?W?``#xh} zIb#sE(xf$LjUs|Hp-85|$WpNfI)e7cU=&>ye%Jq$|0R^3& zc#m;>igKj##z{n)N`>))0+U|JFM**C-}hO|&wXJt4R{t>(Gsa9b_^A$=tL=_>qW#n ziT50r(mL0UPF`n?0)}x`F;&R`Na-q*lkra7dDzO6b7=jRd?}7L!(Hd}Cf?PSx}bhH z_My43t?Y8vx|-Eh_UH3nbfBBzyyh(`n@wVgaD$m!Akhw=kDLFwVrAekabt!W*P$di zu!ky4;k?qcbpiol(q`(MA7vtzS$#DVPx)AX~42a`;6O6bH3g?{MDu&!{s+oP_-Hv zk)NapiPHmJP)!WW0VHA}_Zjwki^Ae+%HQmV+q)U|Xc8GdA^B~RHbjhzg`j98gjH?Gs-aNc@xG8%2P>mz21%#a-KGjza zZ;dzM{QOm8->Wxg9K~p!M5G#ycbXG+M`pc;%!IXa#r=bpOwmGgUP>*?ZU zpv6<7dWWr1_oIXU1#^ekpj+lKm#_79NuHLua_it-MBun@(cOCnw68ANOQ~-F0ZB6~ zxcE^I4f!R3=nesm79qNdo}0vxxo%NeC9JWyo4M)i!nhN__=45s=@cknCBi4Es-Jh_ zw;0%D{*x!O;J2y5J%aB22)Z%rL7~QEPm3n~0jGYfi@b}fJA_XsFC8CQ3S6sQl26Bi zF=8=%;7e^J8htZl*O+nnhpdzRa%op9pO(VWIu&3j3fgbu{+DLKmkaN@?-y5*$+?)^ zH0SX6ZKc)@4O-)P_H#zL9IQ|H)e`P0?9f)4z@y2?TVpL*6@lX;$pEQC?lS^80{wrg z#pn}nF~4_02P~HXSS=<6t~vGM`tq71q`Qdv7#<^}g^*;tB!$FeEy9BsViWcL7C{l? zh%YLLFO7&V(TFc}T}=T88)*mGS#w(@$_SlwPLP$@ndVeW)dv9k)uMx?rK$2wq6-^a z2FVMJd35OTaAf)7dGzqGaAbvI*jQxDqS;*~enHJBF`3|-#7y~Bk)5e*oTG&FNRpksN99w)@SQ-Jq!hZLBP@^WwH901gGo{|QSIV~8) z12NgkF?wf0y)vSmzu&!I$rA&|cK%GE2%yj)3}GI2&H?vTH6o#Q^M?mr&X~P#ey6*e zH%rcD{s_2dO2ho=6rK=>$?xa1n`y^s0CV%Pn@m-<*0znYzRDu;v2^3beuO*aPh3f{ zl$@O!ALaeoXD_RA6xNyOAdCwbL6H7a9?x*EnP~;p{N98Slto4LCIS;3)9<0jUm}J= z?7tPIV5_FcuH^(rmlsIJj}gW7CdBocW0y^8U0f|zqB)zDT3TB57LLFVN}*Tz`$Zmd zUefJ9+8sw=K=0KDeiefgJ3kQ|_+m0L8fW8nWEqCqa^)J1&S*e+_^kctvBY38LRXyw zTf1rl9GfIbb}_!+XLCuFwoP*`$V0pfH%j;&PbZzj?T(5;_ecGonN?X4}9}sreKlGcUJUC-;CRxZSXbo8R+)dcPWX72a0Y{ zT!_vER%~P}eCUxXi;s>J{ep5|H(KR)BACwZkKT0L;^%H&(no)Jt$cx=gEB5YDld5P ze*!%k|2Oxp|4x7Vzo$g~b%z|79{!(7l=%O7CF<{>wf|J2{{43Te^#RYBP;TMKTF|1 zm8idzroU@ZK&k3~Rfz(25&d0~0x|^u>OT74SEPV4)_=)l_(uTi|AHRH@~@c;N&nEJ z#2-K4n3Ba!!!__=NyU{YuzbteiG|MtwP}Z0r-FKGGYTpeAf=9+EDpV5sIz|7Q##yS zQZ0Vn?yg@O;NR>9^9%l^=@!$*Nox~g&*2sq*-)`&hQ!R-lBb#L%qVEc?0gNsrFZ8N zEP!sWAI#;p-D$mgnQ3`7C^d#klO|?S zh^{O%^(2QuYPQdx<9N9A&T@vz&@oe&^fJ6aBNC;8Mu(Bi0K>!fJDLBJQurpsSO!{( zMo&Gdt)!A#y=kbB<8P2bZXWJ`fec>wz&hxTc>CjSN${w$^y;P4T6@4kF|{QP|X>miH+g4G3sX`Z{5e&zMB zu;qtO;*Yf5Zj#Oz*8O^;1#$+`MKzVf-*|I*F%7;J4|W=tT*j35B)9!oDExFQd^%UL zX94h+dAf)WkVlN@b+kGey_x*TvKYISvcHi@ekCONshkYvfGqs#gx-&4xbM42QGjrCjh;(4o0|Z3mSc~Pwq+>BI`EY0TC)Y2oIww1O(0%T9riU z25aNf!sa|21FYQYkqze5|Lm%i%V5DbCw@h{XHW;m;ZL+8A-@g2M|i}~UgJLNz3ChL zp4Z{Z&pY69>#hbG>7Y0mv`WEjqUs_dJ5vc;;c%kIB2X6&ATZ|FulXLv-t)8zO5_O@ zfgx1>KC67+f>~6+S#5-6{PMcM5CiPD@j+lYfW1cPlUf7jvITyq>lXFiUehWul}6$G z8fu821%clXy9byR?x%=PoMD-EmanZ%xp))-U#{kiUKkcR7JHRH@h2s@zdjQ35iAqb@&MeRax}8JKJ%V^GB-pkD#9+qt&ZY#Bk*5G z!)J$Tbbo3Uf5(FUj-`L#o?uQoOOM7L&GM&OQ8*B2LH`#l%oyiQaW{SE9w*IHLl`V|ja@3AdlU z{FYA9a&=gu0$nz~`Y>2+fcM}iGEe>=ZpH2<;|_~erCT;dd- z>gx3ZgPM=kRaX;=AJ5yLyPNa@l^qqyxGm+8XIJ36cYnSL`oO;)rVq5%`3pK(o085{ zF*4BKxi00K@0CAuGckpf^x-#Cf^{}$fy&{xv3SbPh__BR*enbDuJc%r}Zq3o~UEs z`+U$hJ{Pn>CT;~GBZ~M&q~_BgAz{wVP9_dkB@!i-7%FUPRM-e5>bBhTJQgWT(NE3t zfg;N?+pnxbCp)={CzoB4CON2l7T>SD9zUqOvqO*B$N>{m@(|vr`T~u!W`pAi6K;^g z^R_F>eZDembR;xkq(K+Hxr(QoJ&-FoXml0dZ?qLZX!JS2Lom9uZ9#y<$-|G<7OIZa z`vf9-GpYqx>UX-V4aq1gPs{J77*Q zCaNN>hwg!&#!qNdduYTw(Bv^I@haKFdvE#@W2k_v+4hj5XY@F_lB&*=+6MV3X`1 z_}}wPyY&5SF^wL9w=tVTPk(yS68Fop_ZquKkNS>bo#Pm8J+p_u#pUjQhOq?RW|uzx zS=F=5+`7$9=F=|TNKm1(wQ<94+rW3PcVwh%VwvP!+gGDQUdh(1#ZGxP!^G9?*9d3-xWmfm5H)?xBS+`uO?33s z8{53l!TDgPgYWHcX&dm`*=x;uIur5!L%c*?q;m0V+2VIs-uB7z`Lp}3n$V^4a3rtp zw0Bj9I&Up(I0g>OX5R8<+G_RMMJuB4t3e`LPJtM=SUBFCIA7Nns0**)0+G%s%K)O^ zs&_I}IdYu}kk!8jy((R;a+dl$t}`rKnb#JPL9Al63-q>%H;cJ&5V92rT_`0wf7Ru< z`PJ%|1;x$Q95tQA@NSwE{aR8GmqMu7tNZojt-tOP`1#%`(flhR_#WDP1%#OgU)DOw*(#kJXsDjL5t)!7TFpU6~g* z2C7J|Q^b}HCw2yp11ebjTa%<7&N@Ni$xc(29OVN?Cgz%rdF7%pltq(tQ$Ipd}i}mEz!y!?yL>1zcwg=HAYK;#=xO!1d44GOaPXrIE8HhkH0H zB5s7Reig*B6Q?tK(E7eN{rHu1r3 z!?RLZPbg~)tQSZUo&C!J;}^DbfWuIDqV%Fvx6b*?Ca&-9+(u}zZnIQTU__!u1hL^4 zI_t7UIMotux{PhQ3h62Z4Ey)cZ$u7X97Dmc`;QJ1G*0$i_%;>VbiKImE*R7C%{x~Q zV0a1KfIGXArqI?Z=Tejb$;MBF%uiH1a+bJW7}%mTI&Ed=u}caugSBr8ouXiuH*yrX zY!a}ZaP8q+*FjIDv2x)gh_r^g?nFrBM%2`;d5-n;?)PAZt-o9FAjJ(FU4Kak(vX9S zkQ9ZU0H$rxKKqd6m7Pau!&}XFqOmhi`gWD!*_tYQYwzzEl>6FINJ5!;_c15wb!VEt5hSFnPPv4eU-wvL8Os0H1)(~w7@%9>YnIYYjZ zHidpIP|Dg!OEn%73t0?;IY>@GNiQR#$yw{S-2pCO4l((xl53L8GK6S0w|SUg9gU;(qEj0fv7sv=FDvsQF5>n1jU6+&O)KmjM2 zC9si(*bGSwERLc*-^A*Rl5Yfl(7QL>ufyn$?bS&q^!uydDE2%IKymk3v(IDZXaJ~Z z5u~WCA`SOL=!6cEdN-ERQDHYYaC{-F96wfAXJdD*oJn&O$WzV5?eMA7Xw9$=$s+G8 zA;nXovd;_WXi_gm-AIzL5-}v_GU%9oJ&p%-NkR5s)l$&|_Xm-Gf% z)#7hn26r#|iFNdM>veW^!U0#|hoblmjC3pC9|qG!)7avGJ{Q!k|*z_kd7|_a?ZU5csqgD#l*q5spY*!7X=a<^qV?L`d z9=!6keniblr(%jj3~H>$udv>0u;;STV)9pj6?OQyAKey5C!rN@j54 z;dIKEWT>p{Yy-K-*GH8ong=Ju4?d|gU3sYedFlZ7hx66Oe5oz}aOD?Xe(IfgtK+F< zrK#l^hJ|gt%3tF!y2OarIMdbi>MaYUAOs)|V0|TC6YeE(29c~tx?R?)Htmtx z<(n*5pOUz(Fw8Vj%~0?QAAKYv%$%7UuQDZS-j48vlA46IgC=E~A4Sp@QhKL&%ITfX zRy0;>pC-mAo1`KP%ig)hu1=#u{QNVx2yRFS z$gg~HgakO;9`u;Q5239i8aXs#WsV_6G1( z@c0r>2M2&jW0tI-QPaV;)!d086(cmTI0SMs$=j`C$wXCr+&jJ>3yeUvY|QEf5@%9-chXWlO6#{l zgBBGOdWTBZ+5O;Cl{5$sA@Xs@6fskSfNY6S`Av41MhU|eBt;Sd7nJc81sY`#7Qzu$ zM7LsZ`SdB4_d}b|AN*HX3GA*s1g;8hxJp~JYVZKfQ^IZ@t#XD09K4hfE{(1P+&^l< z>z_3tb<=E!Vntoe)?rm%V}i-!){;>{O2s@BO&v9|BoVXR9aI=`20YvY7#%Q1VAGSy zMZ(X8gQaHX-VFhwgTL6pitG07@= zX#vMF@^1zGZv}~ux;dA=pErq9WA9_TAEY!o#T6D7j$AyGulZFNLV z%xJ_pOqj?BBjFOf%@VhO7uqIrL)R_ED_ zgIUWi^23BnyugKvpjt#++FEz}&18Iw`w&5mW*C zS4|kknBxv$S0IU%mD-ab>{TDJU$_BQGrklTJAEuy;-Mg1QN;ws!f^JG9HK3l^TD4e zy4I++!tI;1W&nApR7K^SVK;q(O#FyUv1L#by)$J3o%4$;!;aW{i0v^0;f}@3akRrMv5xeH1mid(W-$ld-*F8l5)tC2^` z&&IbyIOXGUblu)=?02=A_$P5Q=OeCgq|#g8@>a9!1RzzoE&qs42aDuB+-a`hBKb1c zlOlwsaOHP(JQzQD_>w00XMbX`L{Ra_vcsZ^+uhOUeAkDD^F!Bv1W@nNNpmd(VXy%B z7_=;NviU9i(A%L z(%$Y9c7EyfDHh&ADkdelg%mDs=>nf`>ji)q{}JZr(5us~Rk3Cn8GOu&7cr-ETsqgw zO7&a^+*rH@#{+x0z-)GA(4dL;)|1}+EfE3>byQ^vl%&UJlnoftKonXSBREn5VHhfu z{x2a8|GdUJeFB~e5kv5-`9ru+!OR?U6t#Q=d!8uFZX9c67CRC&_ZM`T?RXtEx4Mr! zt^j!%}zvrJ5Enb3ueUwSM<&b`Md(|;5`0t+P3NR7Y>c2|4xhr}b zJxmy0Rjl^gwb-+~zfg1!fX@-vlJ*Ti>9Fm>)je5uLv6kk%`y{Z!OOR=VR)`bVcI(ce|7jo>U!2rsSfP6cgPPvDI)~ilp%q^LcW* z@aQ7tvT2W}yPLq(!ZY}V&YSlS8jZqEffRvyf|E?E32Nyp;ej{JSXOdGZ>Z?XMYzue z#M2>x{q-yj?x8#Is!xRcKMv1CHj{~UC8hR^9B%tYNzUxW-@Eq*lcYW2p3E!km*{0; z1zIW62);;4(?=#nPSDQ@Wxca?JgNBK%Shdst*EYe%2S(8B< z1^ww58(Po6&f?{=e#~N57L+0;oidsnozmrCm1x&9NQ1NS#M|9?gUJh+c`Uj^x^vjJ zLagzz9xjB!jxVsD8GWhwrDuuvQWJ=bSDY_`i6E>f< z8k7KzkBLTp6h|zyo`JmX+dsOUzTc*+>_JQ8$P+!ZzNFzf*AlzFUCiG}aY?qCp@$|& zk@yu&je;eX0xMyVHK{FzqGsvl%=OKj(4br>9o7X+s9Yd@{)(5BItN~zY}{Z+A-s-#1rkekM%p8cZJQ`7o5iR zi>OWW*X-nF6|mRM32Gik>?tV8?CW1+S5^Iy-?{5SsM2-@&A1(#;|yx|xXzw{-*kvn z>F&!N{8pd5t!MREoq${B&Kld{p*i*ADm8lk=h|rY=DpS2-GUPvCzq=Yz^rMTH~U6? zCoRLF#TK6uKc|ZF8hDOx0Tohm??v9({i}?kT2v8D-T)vinyPbcV(QNF=y>N>LK(Xb zgW>_0!falY)N{Tk+J-|Q(j4CmX*)ywTAc}RsGaBcpGz28Y2-Dn$i2-syzhO(m@lBY zi6y_+mV&di7`R$#(-cyxqZyg3V|aUiH?D+OhXqPelA+d%>67f9_&-(?lTQs$NZ8r< z<(_ZpZs=Ng5p{kdaVVog#w1#30@M4BL*cMI*bF4_Xg#(k7q(HYW~x}6S!lmT>$lBD zLwJ1?VJe-820tExz1v0eDzJFH4xf$AlVoyF@P@=Ps<^CXmTw=C&=C=-SX9!jEXc56 z(L+R^%=XGInc17+az0JrP~s_@!oRB$na0b2EQ`41z`<76d^iM9vk{8urmOIf-NE!n zlklR5F$Jwek{qHA(m>q!K@bJroK`bJFW-6t35pUVpL&J>T#dvdMUN%vy7Wl@YxeSIeWDou3fK!du{KF&p=ukw%u@Rid(6DeB(iRzc1k zQrBohj45O9?=3G&iqedO+ZIy_XRz_XaogCKTD9_DRJA_Dm;K9{&w?LD06R=Uxq)!V zr#b{;#)0rk*SmjAT;Mf^?bFk53vyX&4k=GFcO)8o*$4|R-jrfu1A&9N2ZUk@K^(65 z>2KDw>F#$-cdel>*if|5j505wn*N-8DNWIlfpS1xi{Y@t+`Qd#Sy}wU9l>U*ym`cT zXXg4wF1(6!sligO!kym~Fgil}dbD5r&9Kn3QZk-qO-btE;&&H4+e)I@8K9O9m}lER znC)A>yg4TuWG+*@K zcqHChWtl!orpi(kgNC}sOdICpnqSe;=WWd8xT{&58P}7E(q|-JI@R8LNess4p-2#g zpcUfo^KI?dXc0zyrUE)`H%#D<+O;fuy=gR)VJ5!3hb$YtljGJww*ug~UbBtYb znu3vcY=@jaQdpm(7Ws^DZokkgdDSd@ zu*Wm^Wc_fQF490f*;R}xlH55?j6>@@2Ynv>i();SdGQ8rjP$#fy}i-xys5ue1D&Od zc$XN&f&bZybguX;>E}R$1t0j%(c#-f)jfUW0d6tpVT`|%T^x~s>*>rea(URKchdX4 z4S7F14WXP*?mAiOk|wYnU-{B8MO7e4Av-(kxqAL=i~m!AGu;1u@X()pzV{V-zwr+n zMu&jVe*75Kag1oPGk-^Wk(B5`A=%uMvUNNj{aFz$f%)uO!8c65ns2aNw5=ZEah_dg zZ&_A^hxXaT?fK&{@J`zQr7ZFvckL3_k; zv=ixDC-U4f$r!trXeDz=wZOP2dzwyR(TuF`+6`<>?K~9g2pydfD5M>}_2>3y3d(Tc z3z3+#Ou0zeD86zUd@%v%ySVs%by&Gi?5b*F%&2#qvDIDDOU9qEK@{3@S{teH=oJ|3Hv(lz3RP6k;O#UkmEl-1q_rLzmX1KOl>a&GlI;& zi#%w2EPz%LTqQDC*Q*DYh}4VB+6C{{vugi_l>lsKvvoGIx35gyKQndaRzYB&^j+g} zy1l-;0}0s?f<84jcvH0fQIe=wnoYAc*$7w%2+pYPS}>B&f4|dIG}>;fo!*^v=Y?3~x=MsxC(u&pL4oO)bYP!r%E7O~ldwyrEssj!{1a+R zKnKbCcJW6veeP->e%{X$HD0rxw1^0XGhmf)x3e6 zmwz{NZo@j7=(C5$V8y%^fvsKHbSVcndC!}}2Bl2*SPj5!7Bkjk9kO~hd7p+Rf%B#E znmPXbS59>INr^I`-ndubtl=D{6)Wn^CTiVeuk7(h^|FMf#{?q&)`9N18i7T`Lo%xd zPra(2t(_lWEiFEVGDooTkYds2RHc)%7h~KDu~Su|GdAXO7whhWZ{J%LoC(R-<&9wbOch~N6|Gm}=a_grS168Bw9;RAq)zGJiHnGZ-qe1sW`Fr~+Jh!J=kR%< zTbRZ-?t8aF-PmK%B z#?MR+%jRV`jFjPsVxAp>dllTC=*YZ>=YJnQg!u<1v@nd&gd)kzFNR-EmETuRu7#5Z zy(zS+l(WcyRcg!U68m@rn+kmsHA}luvgp=y`0`)Q$05&h1*_r-F z{3-C|Kip-k9DfH||A$ra|1|jgAB2X#I~67V8>iy`*EC|zzu*u5VU_z&kK%v3h5vi@ zL|_8&e+6ayHMGE^{$B-U{8#Y#KQoHifVM(zW*`m&2tfY}p8*77{7(ov`(K_zpndQ^ zXz2g=>pzzTJ_DEj>%abCQ{)8t6@kA1ZHj-fH2$Js0IiCD{|WRVa-{G${r{Df4qPE{2f2WsGY1Rs0esEQ&8$bl!u{8#fDHA2g{X7>9n|vgopo&NEPrP? z{0+8YgOo@tY)KK4l0fEoEL!pmVvH4~ioiM**ju|2s(K2l!v@IQG}AC0Dyhpk+8sbF zg_zEoT0$z_xkR|eZ^|)~x$R){6ZiPhgO@oPCCj5o^wQ4>E{irbMAfvpzd4boaPJUI zh<;TAwY-`pq}S*)BIZdvLVWzoi&5*xA3QwDhGi?CO0R7&v)4kXGd`iKTCHd78;+X9WBUD?5eKe4Yf>WrJtuh&@l_L7l3vj-P z*%wf_h`9(B^Uo?nW|-ZpI5qmK7L6K+r%;O4lbN^<(vkSP0%%28#gs>~;#6QYu|1*o zag!|Sek95kR~#NOn6@KDjiy&KP?%7V%TpZiWk>ScbsR+9uhDU|zHOMn5+$>_AN6@{ zU}P_`mQD4WWDXnYs*XRx8%I~A{`aQ@PS(FosDJz=U(tjmOJo?ssyErwFkVZ+$i~SU9_W*k zB0wOt1V8DE$q01lmWKt;L2xUGP>=V^xu&cb#r`FVPz*K9ReeIv05K zIr59&dY*dpJvt8pu^dQ7sqLOnNimk1>Ep!;vLd*SAODX%x=AgQ2l7xkGViZhB)kRanA#_>}G zS>D%d@lpXd@t~Uz=p%|b`tST+AYDVpR0>l$PY25_y5Pq`U>@ibEx7#fFkvF$a99tx#P#5U~0hX zYV9QPF^x*N457dD=GO{CT>JVElO#5KBl}I;urXK0%v5+)8}PMmcEYn{I4CqrKau*) z<$p^A7{WMR@Vmdid7p&?Z$BX)rupqFwDENfVnYG89)kdD0|Su60nZE3iP1Df znX7pnOUGQV4+aN6FQbIN6%?Q_xE&5|&fU=iNw|g@hV7QIecazFtR#Wu=wWmKPTe3y z0z*Y_BaBGqQs+)1EQ;14{DJ?#@p!+nSh>vj!`p)Q#!##i*?Mo2J^VucYFz&P#Qm_e z`*v}0>-H62YL<)n#tz_#AMdZURTO@jc=q{vNo{QIaC&> z__5&k@wout6XWMXJhLZ`mxT2WQ*}=TNIM~1OeeBWY{Iw-oR+T)uH+M53<}YUus-P6 zkTE*^UiAJ>7uXLK;f9+oFjDL+T?}5zM*=B@A;uv_+pKPktQf+~OuYry>(^ZHv36ON zp%tzU-Mn5f#~Hp*wC1Z!{T~V{!!kf8&e-zoIJc~ zI_5*Sa{*=+qL`6P%FhBVcjHplsZ+JX5a~U-vLkLK_)K@&Adl@6&`G;-I1a5!CQaL0!^9GB)y#ssGn#0bP86-|lTIeNKU__ga&~V4I73g_^MA!SJ0Mn?BV@>*SF`=_<*`Ll*nx<1 zLW%{LyyZY%;#QL*?@-6p?iB6%{CLHkx5?GJBq zw$+%ogsgIu=vo1S=S{;= zf*jEKC4_i{@FMXr!WOJX8+A>iJ=P8NEW}~X;8Zrmm+G_InZAAya2w9jims_)*z`*A zFl*n(Vb{9x1S5+H1i#h4;2=pn^A3G2j77f^cwoRK?7~JMTRX+W@EP1z3Eft9wpqcD z!h^>g97^jWVmO5CkB3Z|4GA1-8gKfJtP^*{qxF!|!I9c<*h~fwoPLpI>1BWCH#SJ0 zHT2NH1@M_YX`6BDDTEz$#Bg?sr_Q3f*SuD#bh2?-^tEN}+>qngB%+42Lh{z)+j+Fd zw2hpFE-oJK$TiYDb_j7iY}#4|FORd|S2x0v5*)h#oXBbBW z2TJ-;tq1Gk(llxaO)am%jc4BCg!?ST4reigYSi%U0;%TJrcGLcqJPG4Yi%)bY6=VY z`(PF^5c5$O!4_J*S?@t`nZP!TWX!yR4}}+SM!&oA>7RG$2t~l zffwspp2n44bIs;QFmxLd8U2CK;Sv%FHI(5XcMXJDgkaBK%&0ukgm)j>=80O9;puuQ zd!RL^Tnh1nS7Q7tZ$z~UDu=C!3SmVqhhGJ9w*F}^P6aF_6 zfJ{i6o;qzCeDNHgOrGcP9TioW8$bVcqH<@e&U>0zq%M~7K<6LP=hut*EYkj8DIW1hFv_$|coH~79lylv#an6TN&Id4xwpyVViEgd*TQItjUAjsQDPm;n zyi>*cD&J4KM==+hkK{-&`A)K?4K|s~X$o}p7xBW@m-!tBi+p)9UU!ydHHR}s3(7KY zx_KSeOosVXz{wawrejKFstrj(wd#1>Xpi3ed*_2VR&IN5_xpzh)}hd^%RCSxs^$H? zJIsuPsvBaH3_P(gFbvED<5+u)1v9PFxbTQZdeZ%f;lI`E`RS|92(w;ZjO~LvD;O6;MqKo zX-{gRip;vB+d^TKQ}j1Ai~M1?sqJ|QZ@?}3;dC%6IlGUofUxUsgDb(a-k>WAMP_ed zDi}>>_tj>i5JpH$&=`cQMFiNpO)osy{?fAZFGH6o8CqV|*HR_CV4PA3jP(tDWmTpw ziejbwHtd(e63;qS7*lq3XK{9ieRh|AcDJl#=b;Ns4Vw^PB&WkM|FHkcr?Ez{j3O*# z5512Uh1VvYDM07Iu^_3^3JFZ zPq8)pWOc=Q|9JoTb8&hkq-Y;N()>bvr}n~D>=+a3(TKJCOfR`D0zo6DBo2`rVJJkF zAqioD`fJ|xrP1{z+Vy4J^~E}Z-JB}ehQzA9=gacQgB_7XFYT;PS_<`D*9(Q6WbtVgCFXt{W75?&)Wc zTs_avipr=VN5|*xD3F*+Pw7~YKS^NinxYu65pE+O7#UG;*+Y0S+E{GAIeJL#NKnl9 zC@xc{WZFme+)k1Q^;^=t`KvGtn?dQ2B_iR5HGVEyxgKO`)FR#E-MzF_nm0s}<5+_?L2iMoP?gK-rD^=^DE1%l}UjV(NjU{-Hd4@CZcTg0?S!|38 z5w*+evHEm9AtlPK$uw$_o&K)(BdxvCRQAiq`_H%e5!HY~!Ba8p;h7;nGe%c?45c7Y zQ?H@`K??j>mW1y%7)nDZqfN=;IWVs4dPMdO!gJPuYQ5#?wyqFR}+c0L9g zUXP_!zwV@Sq|mOITw_v$W-k#COr4Zc?q0rot0lW7e!JPF1orcV(u0e(k(7ICr^29Rk+QQ%Ty2ZJ! zb(4nNwPJtu*XzKKSXHCy9J)f?Qf^Z+VYhKrtj{8Bk%FxMkKWz_s>-cv7!{C~P(bM} z0bv(A=xzZi>24_zNh#?@8Yux00hKNh5Rj4GKE z90GH%XU(yr$PckT5o1JaA(X(#Okij zEboIO0@kkKHklc9O((1zZ zW~MvU=`QMFbtjTxreg8e3tf4xp}@`ogQ_f{C-1}T3G6oE&pQUlMF?0z$cqWguCc7j z6KoUA@kx2URrRk?^#@FQy{dnVWB=#@O)DKi(dKYTyP7~;&Zb~<`&JgTYJyg(yE2Eo zB4eKlNl&l#qQeG{ET`5u>FDF8)Sx-xjca5HMxv8%ea9KX!tiZ4hvmsq@mSki)IDDc z6GfI;Cu0#hm_|(&+;YOTBosX3!uxzR@m`taz$CI>kn+$9*X)H=0+;DcW9^>}*Bvz% zG=D4gwmizKJ$c|NcVoiEWq7dHve@F}YgeFntO!etJ4xIVNdum_wciWB@{fDQ9V>&{ zDh-Ak3Vn_8`I-uKR&*WZJvC!Nj+|t?>z!NsAC=b90J4);?$?=B{r-kdb9bPvj|5IsSv8idN8xt<2IWPZ*Xo_lB?1evD zbJu(*V|2~Fit(L_^VFLPap-1k@bpN8NcPLk`Dnc5ij<#L^`Q{#cMPA4ADX4(CLz1% zT|XN*bd_Bn)yUPHq;vcDDqQocvBqQs$6hg3)}t9$edW;-sF4R^bU8oedH2V!R%!?* zC3eT1iiP!XnFP}5MfCy87jJuhHkCHKH!*?Q4tAG78}+W<}Uz3*2Tk>bikws(B@1YZeK-7C9S zGRFVqYbfL1L!V+k=hBS$S6Tx5q$$Dzla-TrTgO4zUq))~nYWF7gwfnu`w?ZmWiVRp z6&3rdI^9}W_xdzkmP+EGz^3uKsoo-mo{p);gJeRKnf(F}#R-H3hXT*E6-X3~OL2XS zY2LK8UFh^s=(IrSR6om)fwMZH((X>IY|ZnUYNcY~v1%%>T87mxx8^i_d_2a?2|a3S zQ(IGdin@c?*upq9i@K%R*zmbDgXO}A=UmLtvUl zhQ&o98l|LG#}pT6I=9@I@8WI`!o|eSbXI~J;c5E3N?4*$`f$!R+UzcUtX$YO%huv# zV)N~S2a`y#7tkU5&}PlH`vRIzYO3Uzp2Yk#SkU;!I?hAPRHL4DpNf%* zC+%meeoyfPE(o-~yAI?a49h-Noyx9i*fGp_?t4pe_-hu{gGF(@I`%c**_!}g zyDil|e$`a}HmKtF$=j=Vlh)(We z7u`;GUJhdM_`JBXMlai~#bl8PxxMF#6x|{NFttMV^!S|HJ7h(E5MM2%Miv z0tEwr`p|!rh5n*O2+p6t@sr;{Sb}#w{68xPBrRFr_Tw6 z#xgB#wOzsvB7P*)c4Dp^g0^x z7na)n7is=$2BZJJI8X?1R{ihOOgIb?o#?Q;R-$C^`Qd0vg#oXjyUqG^djhL=PJqa5)n5V;S;D^?fA^kj5Bz>Q zZbW{G^W7y}PNvDkHNP$Cy`Ps3l`jGH1N`|HW}?(^bg{mQxgQ;LPtG=<=>Z8>KE}>- z@zcW}E#2_D>>`eDDFzky*s3pDE&Yu$jbd0{OB46Dee+rTkSah0hr4fpN%Xmw^08gO z&L!I%aj*6Ixaj4xD`hI!Z(}J6W9c>G{C@L2E5-iU!$l_Qq30|0w%EhHsVoyol*H;4|D)ca;nZ8X*b>HgWo@KmNTgMg8$ce7x z-pC?o#d|*+A%402b3;kWv*2q@{X7b`+i`q#DDB*hSJ+waco)dJd(^QGU`RggCh#0eML};o7d> zxk@qyYX}%9E7jb!B|qg3E`{9w(xqvnJ?KWoPPJYv^@KFWNb^I7yPCX>lt75_(=?x( zi0e8lLsG3PY8#5x3u;5&-29d}-0-5bv8>y}(Q-IifS%N`Y45OmWs$C6`@gsZJPW z9Ude{tGJFRAXY7?hFlOL^TY!VKZ>PNXKHM`w>eGFz215LOv>d&AqAm2DYV1d)M(|~ zWDx~ROu+>yOyLEsr+PduO30~|U+>%4?ch*PyrQlr3p0sFpS1d-9U7wA_Q;mX@{`;8 z-Ka?W&Av!~D?7%YEBlWBt!)2{ey^b97mV~cgPw8INSZgyq%JJibD7Wio*Cu~6zhdi zvXI0$X{F%{2+mjxa0F-Lf*t09M|Yp7@IIDJSpaYxh6x@FWK&=b7UAJ$i*u`-n%3a2 zO5!D$BhT*0G}L@jK3N`62ox1|!79A3^#&6Sf44?VU`7n&wePsvdNWg!fEXQCnK=Qa zZ^KqK#R_vFmB+04tUNTM2MPoOH)*bBwkl>Wo$9{7lPQbG68&*(bTe!5?iGJZ8<%X& zehD(sJ@f4S`?0e<-mc-|eMerDYTMVB7P2!xu+X=NYz*8<|HjfgM@~9_xfBzni4J|> znG%;gN}t&2No#{HuyF|%_orvcOq`tIja#KI^iSGP+5Aed(V|zs`v4t6&i|D%drri{ z1KK1qDWzz?h+ta8h~a(CA^$#x&D2Rqzce;~;!Kd?61u+r-3VsE!I13!%8PPRt=+iaK)*=Ho zj~f+q{Z7dbMbEr5-rcr;E?4?VTzmB{DUW#Z*2|_hI`%SsF+Rq><&W{-visj&nfV;H zYjzraR=AThGz+7=;!JzI@a5I!{MJumk7gBrb1{cAkCfpQe}O7u6Dr{k)8CvnxjvZG zq`VrKoN#EiIA)lvUA?hp^56r5MeGgwoQG$=k7IYg9NjzJZ4vlF8Zaf5OT#I&K_~h~ zF2c@porssJ>sou(Mzu(D;cnK8CwX?Og?4pzw2lZ8@!M=l!{Nh-w|2A)h?p+?n!|G8 z*}_tDE?-R|GOuZJ_fH$SiA@n%C;B*Cr|j;Z5biv&770xe3_Jd8s^Xk;2HE*Lw;dWY z{MZtFC5{=ZhKu$t$dpT`n=k~l^TwQO}y3bf>p_> zW$#&Og*Y>ZWpo8H((g)!(TjAkM9`BJa>ch@an+R|y%FD5Aath0;1Argc{N|khCeJB z;dkGQkR#oR$}BhXjH)QEE!tp)Z@%x^wJnKjk9X3n^>~!0HClM}d86aCJ}&y?V|{a_ zjFgTvR|~aP99rSARLG-#m_|kF6Y%!KmHSjUL}Ak^RI$-{W18OHN zIBH{st7>aM)}}twx!!Q6#AiHyEO4WIGH+ZoTT6ZFAST3Sx50x~x^Z$loy0Cq>EMm0 zrMGJH{UFwnhS$8JBQ%K7rovgwY4+zOwW^Q`$v~&ZqH=HZ86|Oo3Xh|R2tSJJ^p-x% zcw*=R$d4deC3LO~y;9P%57bFx_#Cm45!>qD%o+UaWBm(afji9pI7(@xvBkf>tABH> zx#>o(ZCi(4+Hx3a&(?4rQ5h?17I|ysV*1GKyG6g{>`$BK-4n9ki|O+lnX|v9(lwbq znmh`b{5y`_!gsqzmMJnEDV$nI(i|y7AS{FwJP=2={+?^bZpju_ zn-6S%A2DKGmu(+EV+nv>I=)1YoMI{+ zAJ@almrG1*ynfBq%U~eBVvs?-g6e9FrEM#t`t`SAN$uIuFj^?D|KsHkE*`)u65Wxj z$M1A=7z=KNj<)l11IAWc|S% zZt0{&u`xYjTu|zZ8xjbgU}Z;aU|7|T}o7V5&3vi!MQDW za6}-?gg879Q*6B6_W`)nE?9VfTktD}Z`h=$`WJaJ^@?y6iOMdc=nVIWFS!OrMV@pGNh$D%<>xbY=c#kKrR`P@jAuW)|ok2|H%&q4Y?eSWY76mr+r7nEhalMhtGJWPQ8d){!sFX=Rs?Gw{Qs0a>Y)6 zIB(^rVq$m$e_!2IR5}v}@Pm>1N$^!FYA6<)wdn`!=(5fkrnF<4NTv}&Q#vqBWH{!y4559!4c4rzn3vX9<6eYnk{Td> z_BaY0T%&vO7LSSPis!qX9*3E49y0~BM?!ha?FP!fdT9FvAR(DaRIplFKlGeDyMl3-`Zm57z2((4W2G!Hn_{URksJiR7a$oE1 zzT$yzvWtpB$A1qupQPjXylLFp4!~vrNT_rq#Mfn>M6WNUsw0l!_tV!qId})tBE7$$ zE=6k5Wya)+4yEsB3yXSqvM6*0$Wyh@8Ob{9wpZVrg^$v;VYqrat|$>06jk$YMDly7=m zYd)tc44$1GLq{~D^1<)fh#=$|T^9esK1pY&+}p_Y*7c@^nx?LoyyJ9ND1{AUWuR4N zkxw5n_&sngLYVG(gt|7oiVbQjFybg}_NcIFn|D99sPD36S?+gub?R2uFui0uP>cK4 zB9^dbhOMHHBrl-Z^oM-K+na$H7|ZyRqE_BMGXm6j1bf6RizoRvgJi0vIqKFcv&^KM zuV??JeZmQe63Zwv6WpvodhP=7mtkkKMkkO#qB&4Yx>duI?&i9) zeVum4wTT7Kk{JA0z41k3DV6KLbbIt|%5`!ZMob$KECt4?=`w8hKUMUO2ODO7%Fa0~ zw*kbXf%rI-)iJrTAlnY5y*{NkrqsLVd6S;+13mESe8xC(Tx|P8;Fl74|3^L$7+QNd zoiXxy!?IeVE@#-|WYst>ZF^!e%hQ=RVuQk~GGjeEc$5%kyzW&WprShe~*AJI5!`mww&^SnZ+h zL8Rrr&w;n1HB*XM?HdT4V?gq-RkpolJyD6eY-pe7WjV2M0+X=k^`wEei5{vlUcha~ zTB?H4g4k=&sUM8(6f{6W%MVRQ&W{^!?xz@4*LJDgzgz0>Af1b1qI&Bc`Nn^9SE7En z!7AzYPYg<$y{A-sB{D5*wh$D69&chd*Ygj1-D@N$;NOm4Gea|BSE zjD>kXLgG^?7W%3LJysI=dur^luuHKl0ow#$|E6dHLwE3hA+h=FttoDHQPf|q7ICV6 z|2jQSyGn0oD9Mg%8d0(~j=4K?nyHlsoQeyJpYS8L=90&OS}T)F?DZAkOL+3htyMlSSoHO3Y{U9O@%=Q@o-x zGkQk|!lfk$u{)SenJ&`@1#FZ5jX|+;TW(`HOMiVEyp^dzC3c;dCf;62E4))yJGhQ1 zH{mqgP;7}O+>EiZmgmZs>34+>vI8el&)7(IHh9TI#3;A+ndsNXG-B^r?>08X?%zIL z*0QlaI_#c{>+OyI&bh7Gwl>}wo65^o=L*eazL0(4uVnH4t+jTe+c{o+Ewu}%EDuZ}@=%wjo+SCwO~1Pzg=D+E zDEUL%vlf01S7o%fA+j+K`7 z3^BT8R1UaEaXgq;z*+m7=4bd~N3WaBr=K-A|K|MdKfl-feE5~T@Fnjj4;&|_MfGpX zDYPW|rHAP5yJ$SP>%Zf%{}@6Tk-&h@?PYy-DcLfZbmok;@&54xw|q+YJL%cyv%w^U zDRw`)VE5Ez!_-53>xqt~`w$!!GS7wOse{-cXH?2tAkBE+6kef=>5?o}gh!5J>&OeXqF|NG|O+7EMc4{u1;BN$(xo`sO47Q0v)Y%6J_J>3jL zlxRzdP`UQecX7$I6r@6kw9*2I;Ob^rJca>8|FSjNi>sy5?Q6Vq_fgMv4Y-sWVhz+S z;EA>r9KQu6)5y`v?b)6?qXkBjHEYGEh2zed=!ZWpW8_EZmFCsuG7EJScoW zioML-h(&VE!-|JaUuszEt_n9*PemqGTdaq5^Mm7uM{|=mR>HW_Z;yF@XE=hM*Y3Z>K6N;mq}a5K1+oFRfXt>b3AO}OW`S-)MTtH*qHT%{vo z%6p$lM|8%1*fwwTk7{1UI54Y~iKflU(-5V7T|e=)cKG_Zp^_+7JtT5vef1@Q;76Cd zX!J+!+-PW0Ihhr`G|l%rp((E;b7W|k+r=CYEMMt*EpnCg-U1b#kq!RMa#+)X86`xUP5NcP}X-jp^=lQ{&+4 zesy#5_$S4Bi2J!YT@w2wsh7U61+<+-C2N_!)jCY-OeMQ4O zfsor*H=eXu?R}8E-k>C^C)G60l|ml1Nodpm&N3;ZTdA7!W8UGVhc0@{x6`!yOrw8f zMYynfmr*kGA7CHOxwxUfdju!$eX28$tRY)fkVmDm(KfVfB@Ak=^i)MOx%=2NI~Bap z<)NZrZH#ck*lXWIaK=qCl@})_PlZ0$X5#Z0ZKC^C(qxISb=%YJ8nPaJo2ST|7#;(7 zOGZ0*Qwo)tTAV!^6t*uo({Md%7oNkgdrFia#+#AG5> z?>vAdR__k%)Z!l5;3?tLC+!fgov!StxAJu03nc~-(!tp>j3i5tg7@oc_kY#6xx97cVd%hXky35a z)05@t(O|raJ~%S3wzGz(c~vkERi!#sG&)!`sO^s_Q*G0)TN}1)z8e%99Oofeu;Le> z@hN#Xvi>BlM%MX}uU+s$6;cIDr;^XPR-`4oL*YHkD%K>&iS& zw4T%MWJdn4kHSpGPKK+u=O_CaO}Cj+D8rSr4`y5x`&r7`)md}i_{Ip$w?lfG)~D}{ z87h%H)FnK;|6|$r(S!TeRatmpj7(3(>Q+LTVBanjT!xKqDnrN0F->$FJFm#-XX7|5bNEK-(xTnGA zGc)QZcIzi$I?q$Wbgkcq)xEp@fqqtMTS8JJRwY5h@>_V#i%01yGs(sR9QtlF(oZJX z@2~YcKgwHm`Z@XKi|G7wAQi-77yBd2NcSsF_kMTPZ#@5awUYX$vU^hgPLtV* zaXrsf^%v*{j?v^Ox7ZsKZhP^(qhI(%eyimtm)kmMszJ1-$nBU)WzuI6*zwN8biXQ2 zua$}6XoJswNVA&IpPUlhom%Yfxv8j=P#c=}tt;u4vEo9H zrn$XlOY(;O7fnX&8u{blb=Nn0C6>?J`TkAJ^r4K=xXv`Pj6>6)MshW zC~2VQqo8oK=wIM^@nU)I@Gz)$HlGK%-nYum6K6!kM7>!pr?mgUx8`vks`x5O?e^K{ zh-BcNIFhc7$?^568f+7PmNh43BK~;IJ4cOYtp!f%nEkE1kHiZ}Ny@_-GTY$8@l;2S zD$S~VqU8>*U(Fo?L8337l3U}`yeHRdgVP-;rD9qIjbFCbwqlbdzT587U(vO`Pf1}U z(Q!;GhyJyBxW_P$Oq&Gm~hhxpVTyBqK# z!_J*O?L(@$)};oEL7hWw7bE8MtA!%SDK+m6-pY3kx32lraPMv~-uHF&Xe;(EIw0fc z@tsLrCks0s&h9u49PCOAlE#;p%6$?f9i$c;Owgz5=F>P$F}^A@^RrLQk~fSip*x~2 z`QwuaE&Fc-`*)7*ZczP>J$lJL`6IRFsHCExp;P`=@tM0=@elWi}4(^knU zD7HuNDk--Uv>nS0LyCqhviPQj#uKf5Z+7VI;}x#1Y_=}pNb(Q(PdDZ-h)dvys=+2& zgf|jOHLm&|^#1Uwk6&FW8?j`R^cvt1Sp2$O*Db{sG3n(QFQ4G=z!yij01Xnk9^Z+( zaK9)P5DJsm^EZAT{Oo$*L}w|^R;>|YA^WKlr?(@;QmyF2zV5p|(Uj0mK0y7J7GQ7* z-5u6^6+bOuWk2BSpBPW~bU+WKFkHdKbCvYrHhaL%rYckru2u7bEyE{&dh$i>!v+hx z22_sAZ+$0=PxcXHRu^?WR|y+}9_w!?{YoNI zpgXm3fvRm9_q3nzhCEV=GHTFyUeETrDMi-ZhtFd%s3aiM^&#H_-tVUPeydm$YLSKu zAEY*`i&pF&7$8NSmihD+&ui^=t2vdN8v9JVo_sNAaLdw-?#nZe8IIW(Jay^#F24;+ zMuhD`$_GQXWP-IOckpQ^lB|-(Ec92{6Sy6rTS*FS+bpwAqlL+hVoKW8REk*%T=#Nr zJFf3NTy(FTNypmyqScOyT(9X_bpaBe5>DntzGSs$>mKMkxpG=RKA@@Xc;oPH?JQ0p z!C-9>(RE6uG>YV9g|LO^35UiLnPNRLpPrcHh{>zbXBG~K9DbzOb_*=Fhc67T{stJA z>z@1~runuugqh{L%Z|t5eFDTgsw~5gi;j^}rBMwO5z9G`<_>xUa!%%F#r7ja4H!GA~ zPzk@vPyHm|Y9qv)IC_kXEw(M(#>~M|D!Q(htf0(OJ}M&b7fIP2^kY`$lne`>g&2ACcvqkJg*ggd!Qq3+_3|pVXg4eXpi1iEK2`p&bf%Ze+M6 z+(W>z*vN2wZ7HQhp~tJFs%n^bTdnV)(8=s&qBh!q=i4jR8I30vRP`xDI3wn#7CP5M`ZThdM>JXiIkp|!C3 z%$`JU;=U+c482>|+}Gnb41Bvxx2_{_p-N&~@|wr{YSA=7azS6nqfdtH?>jj~_NqeT z=Q|nN@|6~K3R)RxoNqdn#0<4rXfuNolh>o((sfq>MExy_UlwN#wYd$`R94^o5aMgtqu+8Q!3Q z%7nI_d0uXkl_D=rBJ+w)BNmV?dhkazXOQH;x(yAqG5X7oDxudJCE5}tkn&C~{bZec zgJMtPEK2>mjyLG3aVyu)DxMFoxasc|vhaH6KN*){tHR3~w_T{Pe5ah^CX}UAs3}!~ zSYg-uWEbmx66?cl>il z+hf@>TihsG>*`@vg28mR>y^_EeiFm?g5;$iGJL+&kBqa@PDy)ecK=2qM=l}kleNdC zykV@U+rQA6tH)ik?t?v?^SAPtiG(E8+w$Cr^9UMcxpXtH_uxD=ru&l%TtX&-N1t_l zl=g9d9W-iB_4=|qtkW>+G>Wq9 zwhogVF_4Yp7W>a?IP+N=k(8~8G?JXX{z6LX9@fah>r_hSa^PF$MbDCx6ZJ%Gf;ENF zeIm;}s4X|-Hk}(f6QUnOnlWqb^B+1B599 zhBqQ{r3|C7ft3{&<&W!Gc3=5A(r)BL)=Qe$bJbPoKQoa)56>mIVqF=lm9C_oxlHlS zxU{oJQV#F2*?T&YHijMd;Cw76mI5YN0^^?7BgV>sk}!kWx!ohv`{|8uY&EYl{f}^k zKP(h2!E&clrbr*Fra8ky?5J{oBCTqloFh3NUfU#ZXwLfmKJoGlPX76Yt;Si+ zC7IH%`yU#!|HEkU+*ZCoF9C(MR%ig6$dVJf2@v#jY+Pf;CpMH(8Zw31Oz}42FPzVw)t*K za@9g|gSux^)M9Z%C^^&c*J|6%IPscfTpN!>cPG1ch!j??34dImatp2^Ok%JO&qkB+$m zKB7xfHw(@*{BNq?6dszNGI_JZHpRf>;k;4*Tg|$Z>d#TBoS6ig7Ulkz_&^j8(|Ix4 z_kTP85%N3;@vp(Y3jYx7`+uDO=&yC^{vGD~j~M%ok#P z|B?aek62tFq8A0UI?ssoM`SKD5{deIDkKz;9_l;=(*FeSoFhD%; z|4Ke2BW9p0AiVUCyhws*K_EaGgSnUz>91+H{(G1z9PziX)W4@hit%DsAh`Ljlt|db z)mqV3nK=F?;raWIA)6mC9{)++}jnt(?J!V-W)5Bi?B7_CDHPvf0p62ZZ_nI|3aHBKKeobgQwcs z+KZR9es7!x6#pDNt?G~u!}OtVi0r?Vp!A}2aOTLP9x9Q)y3egD%9D_3-Abs?A)6$G zk4x4V^jgQe^aP=QNQq^AP5zZ@-SPdq;WZzwb|*IHUFwD>q;V*T(BQpn;~OGWT;<^# zf+261^-~jbq`|K%>)n4;6`rnE)+9vdG3XHB$G<r!0o`eGI2gZShfN`MFU>un9l!WJ>f9q^0 z90CW;0}e0(s2vofub_as2jPIj&%KZfI1pgo1)71M6+HJKFFgB)S91Y|0@D<M1#!E%K9 z2R;B;7bKTxD9Dxpst&wvXgCryW`O-bV@3;NKx0Ouz;*$R2J06{y83P0B zdkhS$?=fKC42ls1`6^IAU4h1oL4oZh1`X1?Fks0pjv0V~bSw-41KB|s1nk2BrUG)2 z&M=7Jc~aiL$rcQPgkKyp41xmr5ikfEWRCz36QqY=fGWO7XTU^(WDuY{NCsh01V{#9 zP(hFk!k|cy48ovb{|bPC=K&DrqCS8D0|dihsEc|C2G~n53<2Uv7z_yk_lHD-VQ657 za$!AS7$^t^hk$GVa1#>b3jkUTBr^cFfMg#AM}X%BM}qV?42}ZXX&4+3%!^}&W57Iw zfPiU_fP!QJs9g!BJpwR<7jYmE;JG1?VA>| z5=?s}3d~nX49KPcyK<0igaKPb5baR_CtRc{3J#_{>K}L)ARqWy6liY%L!m(XHy8>7 zq};y12WSY$&w&A^5u~5dP;h@}7-;_nL&HI~0)|F_-aWtovjEbaXt19RLn8$*`b{u2 z3J!*W_Zl!X2IO}FF!V)V6o!F-WeWoZuP?*h&612yG z0ox<+m{I>QkAKit7_h#ALlB_%5pclfUE~co1Ot+f^XKm!mAko}8%k3a%1Z!X9T0+8Q} z>w!Rl_k+OJ0QBxg@E`V8f^am*R}ln6=HgfYmmlQoA%U>?i~0ZwL4x{2Lc#lKfU7`$ zEDC}I>0Q9%2l?$NV5@PFE`XK<*?7Rs2Dd{afVa_q*Fz{kRxio|;L(75Lf|bDh+ocK z;J^5PZmwo_4py!tL?R*ricZ$f%m9y^p8;q%JG(IhcK*-5H0`{tfcL0y0Yw=D5sVZB z7$)EeqA=2cvVuwjFAE?r87Y8fD6||z25=_N_p4ATBw8AQl$M3bArZ1NvNBSVz${>L zP%#lXlpID9BP)%9A!X1|7)%EE291QHrDV`ZB=8zZ?2qyT58TXL-Tr)ui-7>9nuLgh LLsm_Wgy{bP!KuOl literal 0 HcmV?d00001 diff --git a/app/core/src/main/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdown.java b/app/core/src/main/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdown.java index ce5a61078..42ebd51ab 100644 --- a/app/core/src/main/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdown.java +++ b/app/core/src/main/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdown.java @@ -1,11 +1,13 @@ package stirling.software.SPDF.model.api.converters; -import org.springframework.core.io.Resource; +import java.nio.charset.StandardCharsets; + import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.multipart.MultipartFile; +import io.github.pixee.security.Filenames; import io.swagger.v3.oas.annotations.Operation; import lombok.RequiredArgsConstructor; @@ -15,8 +17,11 @@ import stirling.software.common.annotations.AutoJobPostMapping; import stirling.software.common.annotations.api.ConvertApi; import stirling.software.common.enumeration.ResourceWeight; import stirling.software.common.model.api.PDFFile; -import stirling.software.common.util.PDFToFile; +import stirling.software.common.pdf.PdfMarkdownConverter; +import stirling.software.common.util.TempFile; import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.WebResponseUtils; +import stirling.software.jpdfium.PdfDocument; @ConvertApi @RequiredArgsConstructor @@ -33,10 +38,27 @@ public class ConvertPDFToMarkdown { summary = "Convert PDF to Markdown", description = "This endpoint converts a PDF file to Markdown format. Input:PDF Output:Markdown Type:SISO") - public ResponseEntity processPdfToMarkdown(@ModelAttribute PDFFile file) + public ResponseEntity processPdfToMarkdown(@ModelAttribute PDFFile file) throws Exception { MultipartFile inputFile = file.getFileInput(); - PDFToFile pdfToFile = new PDFToFile(tempFileManager); - return pdfToFile.processPdfToMarkdown(inputFile); + + String originalName = Filenames.toSimpleFileName(inputFile.getOriginalFilename()); + String baseName = + originalName.contains(".") + ? originalName.substring(0, originalName.lastIndexOf('.')) + : originalName; + + String markdown; + try (TempFile tempInput = new TempFile(tempFileManager, ".pdf")) { + inputFile.transferTo(tempInput.getFile()); + try (PdfDocument doc = PdfDocument.open(tempInput.getPath())) { + markdown = new PdfMarkdownConverter().convert(doc); + } + } + + return WebResponseUtils.bytesToWebResponse( + markdown.getBytes(StandardCharsets.UTF_8), + baseName + ".md", + MediaType.valueOf("text/markdown")); } } diff --git a/app/core/src/test/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdownTest.java b/app/core/src/test/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdownTest.java index b63e58b52..3bd6b7fad 100644 --- a/app/core/src/test/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdownTest.java +++ b/app/core/src/test/java/stirling/software/SPDF/model/api/converters/ConvertPDFToMarkdownTest.java @@ -1,16 +1,17 @@ package stirling.software.SPDF.model.api.converters; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; +import java.io.File; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.Resource; @@ -21,9 +22,10 @@ import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; -import org.springframework.web.multipart.MultipartFile; -import stirling.software.common.util.PDFToFile; +import stirling.software.common.pdf.PdfMarkdownConverter; +import stirling.software.common.util.TempFile; +import stirling.software.jpdfium.PdfDocument; class ConvertPDFToMarkdownTest { @@ -47,68 +49,68 @@ class ConvertPDFToMarkdownTest { @Test void pdfToMarkdownReturnsMarkdownBytes() throws Exception { byte[] md = "# heading\n\ncontent\n".getBytes(StandardCharsets.UTF_8); + String expectedMd = "# heading\n\ncontent\n"; - try (MockedConstruction construction = - Mockito.mockConstruction( - PDFToFile.class, - (mock, ctx) -> { - when(mock.processPdfToMarkdown(any(MultipartFile.class))) - .thenAnswer( - inv -> - ResponseEntity.ok() - .header("Content-Type", "text/markdown") - .body(new ByteArrayResource(md))); - })) { + File tmpFile = File.createTempFile("test", ".pdf"); + tmpFile.deleteOnExit(); - MockMvc mvc = mockMvc(); + try (MockedConstruction tempMock = + Mockito.mockConstruction( + TempFile.class, + (mock, ctx) -> { + when(mock.getFile()).thenReturn(tmpFile); + when(mock.getPath()).thenReturn(tmpFile.toPath()); + }); + MockedStatic docStatic = Mockito.mockStatic(PdfDocument.class); + MockedConstruction converterMock = + Mockito.mockConstruction( + PdfMarkdownConverter.class, + (mock, ctx) -> when(mock.convert(any())).thenReturn(expectedMd))) { + + PdfDocument mockDoc = Mockito.mock(PdfDocument.class); + docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc); MockMultipartFile file = new MockMultipartFile( - "fileInput", // must match the field name in PDFFile - "input.pdf", - "application/pdf", - new byte[] {1, 2, 3}); + "fileInput", "input.pdf", "application/pdf", new byte[] {1, 2, 3}); - // ResponseEntity is written synchronously on the request thread, - // so there is no async dispatch to wait for (unlike the old StreamingResponseBody - // path). - mvc.perform(multipart("/api/v1/convert/pdf/markdown").file(file)) + mockMvc() + .perform(multipart("/api/v1/convert/pdf/markdown").file(file)) .andExpect(status().isOk()) .andExpect(header().string("Content-Type", "text/markdown")) .andExpect(content().bytes(md)); - - // Verify that exactly one instance was created - assert construction.constructed().size() == 1; - - // And that the uploaded file was passed to processPdfToMarkdown() - PDFToFile created = construction.constructed().get(0); - ArgumentCaptor captor = ArgumentCaptor.forClass(MultipartFile.class); - verify(created, times(1)).processPdfToMarkdown(captor.capture()); - MultipartFile passed = captor.getValue(); - - // Minimal plausibility checks - assertEquals("input.pdf", passed.getOriginalFilename()); - assertEquals("application/pdf", passed.getContentType()); } } @Test void pdfToMarkdownWhenServiceThrowsReturns500() throws Exception { - try (MockedConstruction ignored = - Mockito.mockConstruction( - PDFToFile.class, - (mock, ctx) -> { - when(mock.processPdfToMarkdown(any(MultipartFile.class))) - .thenThrow(new RuntimeException("boom")); - })) { + File tmpFile = File.createTempFile("test", ".pdf"); + tmpFile.deleteOnExit(); - MockMvc mvc = mockMvc(); + try (MockedConstruction tempMock = + Mockito.mockConstruction( + TempFile.class, + (mock, ctx) -> { + when(mock.getFile()).thenReturn(tmpFile); + when(mock.getPath()).thenReturn(tmpFile.toPath()); + }); + MockedStatic docStatic = Mockito.mockStatic(PdfDocument.class); + MockedConstruction converterMock = + Mockito.mockConstruction( + PdfMarkdownConverter.class, + (mock, ctx) -> + when(mock.convert(any())) + .thenThrow(new RuntimeException("boom")))) { + + PdfDocument mockDoc = Mockito.mock(PdfDocument.class); + docStatic.when(() -> PdfDocument.open(any(Path.class))).thenReturn(mockDoc); MockMultipartFile file = new MockMultipartFile( "fileInput", "x.pdf", "application/pdf", new byte[] {0x01}); - mvc.perform(multipart("/api/v1/convert/pdf/markdown").file(file)) + mockMvc() + .perform(multipart("/api/v1/convert/pdf/markdown").file(file)) .andExpect(status().isInternalServerError()); } } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java index a7239e8f9..2bed56f0f 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/model/api/ai/AiWorkflowOutcome.java @@ -21,7 +21,8 @@ public enum AiWorkflowOutcome { COMPLETED("completed"), UNSUPPORTED_CAPABILITY("unsupported_capability"), CANNOT_CONTINUE("cannot_continue"), - GENERATE_FILE("generate_file"); + GENERATE_FILE("generate_file"), + CONVERT_MARKDOWN("convert_markdown"); private final String value; diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java index 5683d634c..8f82a3658 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/AiWorkflowService.java @@ -66,6 +66,7 @@ import tools.jackson.databind.ObjectMapper; public class AiWorkflowService { private static final String DOCUMENTS_ENDPOINT = "/api/v1/documents"; + private static final String PDF_TO_MARKDOWN_ENDPOINT = "/api/v1/convert/pdf/markdown"; private final CustomPDFDocumentFactory pdfDocumentFactory; private final AiEngineClient aiEngineClient; @@ -194,6 +195,7 @@ public class AiWorkflowService { return switch (response.getOutcome()) { case NEED_CONTENT -> onNeedContent(response, filesById, request, listener); case NEED_INGEST -> onNeedIngest(response, filesById, request, listener); + case CONVERT_MARKDOWN -> onConvertMarkdown(response, filesById, listener); case TOOL_CALL -> onToolCall(response, filesById, listener); case PLAN -> onPlan(response, filesById, request, listener); case ANSWER -> onAnswer(response, filesById, request, listener); @@ -330,6 +332,77 @@ public class AiWorkflowService { return new WorkflowState.Pending(nextRequest); } + /** + * Deterministically convert each requested PDF to Markdown via the {@code + * /convert/pdf/markdown} endpoint (backed by {@code PdfMarkdownConverter}) and return the + * {@code .md} file(s) as a completed result. No AI resume — the conversion output is the final + * answer. + */ + private WorkflowState onConvertMarkdown( + AiWorkflowResponse response, + Map filesById, + ProgressListener listener) { + List filesToConvert = response.getFilesToIngest(); + if (filesToConvert == null || filesToConvert.isEmpty()) { + return new WorkflowState.Terminal( + cannotContinue( + "AI engine requested markdown conversion without listing any files.")); + } + + try { + List resultFiles = new ArrayList<>(); + List inputNames = new ArrayList<>(); + for (int i = 0; i < filesToConvert.size(); i++) { + AiFile file = filesToConvert.get(i); + MultipartFile multipartFile = filesById.get(file.getId()); + if (multipartFile == null) { + return new WorkflowState.Terminal( + cannotContinue( + "AI engine requested markdown conversion for unknown file: " + + file.getName())); + } + listener.onProgress( + AiWorkflowProgressEvent.executingTool( + PDF_TO_MARKDOWN_ENDPOINT, i + 1, filesToConvert.size())); + Resource input = toResource(multipartFile); + PipelineDefinition definition = + new PipelineDefinition( + "convert-markdown", + List.of(new PipelineStep(PDF_TO_MARKDOWN_ENDPOINT, Map.of())), + null); + PolicyExecutionResult result = + policyExecutor.execute( + definition, + PolicyInputs.of(List.of(input)), + PolicyProgressListener.NOOP); + resultFiles.addAll(result.files()); + inputNames.add(multipartFile.getOriginalFilename()); + } + return new WorkflowState.Terminal( + buildCompletedResponse(null, resultFiles, inputNames, null)); + } catch (InternalApiTimeoutException e) { + log.error("PDF to Markdown conversion timed out: {}", e.getMessage()); + return new WorkflowState.Terminal( + cannotContinue(toolTimeoutMessage(PDF_TO_MARKDOWN_ENDPOINT, e))); + } catch (Exception e) { + log.error("Failed to convert PDF to Markdown: {}", e.getMessage(), e); + return new WorkflowState.Terminal( + cannotContinue(toolFailureMessage(PDF_TO_MARKDOWN_ENDPOINT, e))); + } + } + + private Resource toResource(MultipartFile file) throws IOException { + TempFile tempFile = tempFileManager.createManagedTempFile("ai-workflow"); + file.transferTo(tempFile.getPath()); + final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename()); + return new FileSystemResource(tempFile.getFile()) { + @Override + public String getFilename() { + return originalName; + } + }; + } + private void ingestFile(AiFile file, MultipartFile multipartFile) throws IOException { List pages = new ArrayList<>(); try (PDDocument document = pdfDocumentFactory.load(multipartFile, true)) { @@ -551,16 +624,7 @@ public class AiWorkflowService { private List toResources(Map filesById) throws IOException { List resources = new ArrayList<>(); for (MultipartFile file : filesById.values()) { - TempFile tempFile = tempFileManager.createManagedTempFile("ai-workflow"); - file.transferTo(tempFile.getPath()); - final String originalName = Filenames.toSimpleFileName(file.getOriginalFilename()); - resources.add( - new FileSystemResource(tempFile.getFile()) { - @Override - public String getFilename() { - return originalName; - } - }); + resources.add(toResource(file)); } return resources; } diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java b/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java index c06007f31..9dccb91f3 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/service/PdfContentExtractor.java @@ -30,11 +30,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import stirling.software.SPDF.pdf.parser.PageImageLocator; -import stirling.software.SPDF.pdf.parser.PdfIngester; -import stirling.software.SPDF.pdf.parser.PdfModels.ParsedPage; -import stirling.software.SPDF.pdf.parser.PdfModels.RawLine; import stirling.software.SPDF.pdf.parser.PdfModels.TableFragment; -import stirling.software.SPDF.pdf.parser.PdfModels.TextFragment; import stirling.software.SPDF.pdf.parser.TabulaTableParser; import stirling.software.common.util.ExceptionUtils; import stirling.software.common.util.PdfUtils; @@ -50,7 +46,6 @@ import stirling.software.proprietary.model.api.ai.FolioType; public class PdfContentExtractor { private final TabulaTableParser tabulaTableParser; - private final PdfIngester pdfIngester; private static final int MAX_CHARACTERS_PER_PAGE = 4_000; @@ -196,8 +191,6 @@ public class PdfContentExtractor { case PAGE_TEXT, FULL_TEXT -> Optional.ofNullable( extractText(lf, fileReq, remainingPages, remainingCharacters)); - case PAGE_LAYOUT -> - Optional.ofNullable(extractPageLayout(lf, remainingPages)); default -> { log.warn( "Content type {} not yet implemented, skipping for {}", @@ -222,35 +215,6 @@ public class PdfContentExtractor { return extracted.isEmpty() ? null : buildExtractedFileText(lf.fileName(), extracted); } - private PageLayoutFileResult extractPageLayout(LoadedFile lf, int maxPages) throws IOException { - List parsedPages = pdfIngester.parse(lf.document(), maxPages); - List pages = new ArrayList<>(); - for (ParsedPage pp : parsedPages) { - if (pp.layoutLines().isEmpty()) continue; - List lines = new ArrayList<>(); - for (RawLine rawLine : pp.layoutLines()) { - List fragments = new ArrayList<>(); - for (TextFragment tf : rawLine.fragments()) { - fragments.add( - new LayoutFragment( - tf.text(), - tf.bounds().x(), - tf.bounds().y(), - tf.bounds().width(), - tf.fontSize(), - tf.bold())); - } - lines.add(new LayoutLine(rawLine.bounds().y(), fragments)); - } - pages.add(new LayoutPage(pp.pageNumber(), lines)); - } - if (pages.isEmpty()) return null; - PageLayoutFileResult result = new PageLayoutFileResult(); - result.setFileName(lf.fileName()); - result.setPages(pages); - return result; - } - private WorkflowArtifact buildArtifact(ArtifactKind kind, List results) { return switch (kind) { case EXTRACTED_TEXT -> { @@ -258,11 +222,6 @@ public class PdfContentExtractor { artifact.setFiles(results.stream().map(ExtractedFileText.class::cast).toList()); yield artifact; } - case PAGE_LAYOUT -> { - PageLayoutArtifact artifact = new PageLayoutArtifact(); - artifact.setFiles(results.stream().map(PageLayoutFileResult.class::cast).toList()); - yield artifact; - } case TOOL_REPORT -> throw new IllegalArgumentException( "TOOL_REPORT artifacts are not produced by PdfContentExtractor"); @@ -569,7 +528,6 @@ public class PdfContentExtractor { */ enum ArtifactKind { EXTRACTED_TEXT("extracted_text"), - PAGE_LAYOUT("page_layout"), TOOL_REPORT("tool_report"); private final String value; @@ -633,40 +591,4 @@ public class PdfContentExtractor { this.report = report; } } - - // Serialization contract with the Python engine — see PageLayoutArtifactContractTest. - - /** One text fragment with its bounding-box geometry and font properties. */ - record LayoutFragment( - String text, float x, float y, float width, float fontSize, boolean bold) {} - - /** A visual line on the page: y-coordinate and all fragments on that line. */ - record LayoutLine(float y, List fragments) {} - - /** All layout lines for a single page. */ - record LayoutPage(int pageNumber, List lines) {} - - /** Page layout data for one file, as a PdfContentResult. */ - @Data - static final class PageLayoutFileResult implements PdfContentResult { - private String fileName; - private List pages = new ArrayList<>(); - - @Override - public ArtifactKind getArtifactKind() { - return ArtifactKind.PAGE_LAYOUT; - } - - @Override - public int pagesConsumed() { - return pages.size(); - } - } - - /** Artifact carrying full spatial page layout for all input files. */ - @Data - static final class PageLayoutArtifact implements WorkflowArtifact { - private final ArtifactKind kind = ArtifactKind.PAGE_LAYOUT; - private List files = new ArrayList<>(); - } } diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java index 81d1ccdfa..9e611cf32 100644 --- a/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java +++ b/app/proprietary/src/test/java/stirling/software/proprietary/service/AiWorkflowServiceTest.java @@ -439,6 +439,35 @@ class AiWorkflowServiceTest { verify(internalApiClient, never()).post(anyString(), any()); } + @Test + void convertMarkdownRunsDeterministicConversionAndReturnsMdFile() throws IOException { + MockMultipartFile input = pdf("multi-column-test_lorem.pdf", "pdf-bytes"); + when(fileIdStrategy.idFor(any())).thenReturn("doc-1"); + stubOrchestrator( + """ + { + "outcome":"convert_markdown", + "reason":"PDF to Markdown requested.", + "filesToIngest":[{"id":"doc-1","name":"multi-column-test_lorem.pdf"}] + } + """); + when(toolMetadataService.shouldUnpackZipResponse("/api/v1/convert/pdf/markdown")) + .thenReturn(false); + stubEndpoint( + "/api/v1/convert/pdf/markdown", + pdfResource("# Title", "multi-column-test_lorem.md")); + AtomicInteger ids = stubFileStorage(); + + AiWorkflowResponse result = service.orchestrate(requestFor(input, "convert to markdown")); + + assertEquals(AiWorkflowOutcome.COMPLETED, result.getOutcome()); + assertEquals(1, result.getResultFiles().size()); + // Extension changes (pdf -> md), so the converter's response filename wins. + assertEquals("multi-column-test_lorem.md", result.getResultFiles().get(0).getFileName()); + assertEquals(1, ids.get()); + verify(internalApiClient, times(1)).post(eq("/api/v1/convert/pdf/markdown"), any()); + } + @Test void toolCallWithoutEndpointFallsBackToCannotContinue() throws IOException { MockMultipartFile input = pdf("input.pdf", "bytes"); diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java deleted file mode 100644 index ae853b2e6..000000000 --- a/app/proprietary/src/test/java/stirling/software/proprietary/service/PageLayoutArtifactContractTest.java +++ /dev/null @@ -1,66 +0,0 @@ -package stirling.software.proprietary.service; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; - -import org.junit.jupiter.api.Test; - -import stirling.software.proprietary.service.PdfContentExtractor.LayoutFragment; -import stirling.software.proprietary.service.PdfContentExtractor.LayoutLine; -import stirling.software.proprietary.service.PdfContentExtractor.LayoutPage; -import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutArtifact; -import stirling.software.proprietary.service.PdfContentExtractor.PageLayoutFileResult; - -import tools.jackson.databind.JsonNode; -import tools.jackson.databind.json.JsonMapper; - -/** - * Contract test: verifies that {@link PageLayoutArtifact} serializes to the JSON field names that - * the Python engine expects in {@code engine/src/stirling/contracts/pdf_to_markdown.py}. - * - *

The companion Python test in {@code tests/test_pdf_to_markdown.py} deserializes the same JSON - * literal and asserts field values. If either side renames a field, one of these tests fails. - */ -class PageLayoutArtifactContractTest { - - static final String CONTRACT_JSON = - """ - {"kind":"page_layout","files":[{"fileName":"test.pdf","pages":[{"pageNumber":1,"lines":[{"y":10.0,"fragments":[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}"""; - - @Test - void pageLayoutArtifact_serialisesToExpectedJson() throws Exception { - LayoutFragment fragment = new LayoutFragment("Hello", 1.0f, 2.0f, 30.0f, 12.0f, true); - LayoutLine line = new LayoutLine(10.0f, List.of(fragment)); - LayoutPage page = new LayoutPage(1, List.of(line)); - - PageLayoutFileResult fileResult = new PageLayoutFileResult(); - fileResult.setFileName("test.pdf"); - fileResult.setPages(List.of(page)); - - PageLayoutArtifact artifact = new PageLayoutArtifact(); - artifact.setFiles(List.of(fileResult)); - - JsonNode json = new JsonMapper().valueToTree(artifact); - - assertEquals("page_layout", json.get("kind").asText()); - - JsonNode file = json.get("files").get(0); - assertEquals("test.pdf", file.get("fileName").asText()); - - JsonNode pg = file.get("pages").get(0); - assertEquals(1, pg.get("pageNumber").asInt()); - - JsonNode ln = pg.get("lines").get(0); - assertEquals(10.0, ln.get("y").asDouble(), 0.001); - - JsonNode frag = ln.get("fragments").get(0); - assertEquals("Hello", frag.get("text").asText()); - assertEquals(1.0, frag.get("x").asDouble(), 0.001); - assertEquals(2.0, frag.get("y").asDouble(), 0.001); - assertEquals(30.0, frag.get("width").asDouble(), 0.001); - assertEquals(12.0, frag.get("fontSize").asDouble(), 0.001); - assertTrue(frag.get("bold").asBoolean()); - } -} diff --git a/engine/src/stirling/agents/__init__.py b/engine/src/stirling/agents/__init__.py index cddd0275c..5410ac098 100644 --- a/engine/src/stirling/agents/__init__.py +++ b/engine/src/stirling/agents/__init__.py @@ -5,7 +5,6 @@ from .orchestrator import OrchestratorAgent from .pdf_edit import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection from .pdf_questions import PdfQuestionAgent from .pdf_review import PdfReviewAgent -from .pdf_to_markdown import PdfToMarkdownAgent from .user_spec import UserSpecAgent __all__ = [ @@ -16,6 +15,5 @@ __all__ = [ "PdfEditPlanSelection", "PdfQuestionAgent", "PdfReviewAgent", - "PdfToMarkdownAgent", "UserSpecAgent", ] diff --git a/engine/src/stirling/agents/orchestrator.py b/engine/src/stirling/agents/orchestrator.py index 4dbf0b65a..d2a0b4a19 100644 --- a/engine/src/stirling/agents/orchestrator.py +++ b/engine/src/stirling/agents/orchestrator.py @@ -11,14 +11,13 @@ from pydantic_ai.tools import RunContext from stirling.agents.pdf_edit import PdfEditAgent from stirling.agents.pdf_questions import PdfQuestionAgent from stirling.agents.pdf_review import PdfReviewAgent -from stirling.agents.pdf_to_markdown import PdfToMarkdownAgent from stirling.agents.user_spec import UserSpecAgent from stirling.contracts import ( AgentDraftWorkflowResponse, + ConvertMarkdownResponse, ExtractedTextArtifact, OrchestratorRequest, OrchestratorResponse, - PageLayoutArtifact, PdfEditResponse, PdfQuestionOrchestrateResponse, PdfReviewOrchestrateResponse, @@ -27,7 +26,6 @@ from stirling.contracts import ( format_conversation_history, format_file_names, ) -from stirling.contracts.pdf_to_markdown import PdfToMarkdownOrchestrateResponse from stirling.services import AppRuntime logger = logging.getLogger(__name__) @@ -72,9 +70,11 @@ class OrchestratorAgent: ), ), ToolOutput( - self.delegate_pdf_to_markdown, - name="delegate_pdf_to_markdown", - description=("Delegate requests to reconstruct a PDF as a Markdown document."), + self.delegate_pdf_ingest, + name="delegate_pdf_ingest", + description=( + "Delegate requests to convert a PDF to Markdown or extract its content as readable text." + ), ), ToolOutput( self.unsupported_capability, @@ -92,8 +92,8 @@ class OrchestratorAgent: "Use delegate_pdf_review when the user wants the PDF returned with review" " comments attached — anything like 'review this', 'annotate with comments'," " 'leave feedback on the PDF'. " - "Use delegate_pdf_to_markdown for any request to convert a PDF to Markdown " - "or reconstruct its content as readable text. " + "Use delegate_pdf_ingest for any request to convert a PDF to Markdown " + "or extract its content as readable text. " "Use unsupported_capability when the user asks about the assistant itself " "or when none of the other outputs fit; supply a helpful message." ), @@ -133,13 +133,12 @@ class OrchestratorAgent: return await self._run_pdf_edit(request) case SupportedCapability.AGENT_DRAFT: return await self._run_agent_draft(request) - case SupportedCapability.PDF_TO_MARKDOWN: - return await self._run_pdf_to_markdown(request) case ( SupportedCapability.ORCHESTRATE | SupportedCapability.AGENT_REVISE | SupportedCapability.AGENT_NEXT_ACTION | SupportedCapability.MATH_AUDITOR_AGENT + | SupportedCapability.PDF_TO_MARKDOWN ): raise ValueError(f"Cannot resume orchestrator with capability: {capability}") case _ as unreachable: @@ -163,11 +162,12 @@ class OrchestratorAgent: async def _run_agent_draft(self, request: OrchestratorRequest) -> AgentDraftWorkflowResponse: return await UserSpecAgent(self.runtime).orchestrate(request) - async def delegate_pdf_to_markdown(self, ctx: RunContext[OrchestratorDeps]) -> PdfToMarkdownOrchestrateResponse: - return await self._run_pdf_to_markdown(ctx.deps.request) - - async def _run_pdf_to_markdown(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse: - return await PdfToMarkdownAgent(self.runtime).orchestrate(request) + async def delegate_pdf_ingest(self, ctx: RunContext[OrchestratorDeps]) -> ConvertMarkdownResponse: + request = ctx.deps.request + return ConvertMarkdownResponse( + reason="PDF to Markdown requested — Java converts deterministically.", + files_to_ingest=request.files, + ) async def delegate_pdf_review(self, ctx: RunContext[OrchestratorDeps]) -> PdfReviewOrchestrateResponse: return await self._run_pdf_review(ctx.deps.request) @@ -204,10 +204,5 @@ class OrchestratorAgent: file_names = [f.file_name for f in artifact.files] descriptions.append(f"- extracted_text: {total_pages} pages from {file_names}") continue - if isinstance(artifact, PageLayoutArtifact): - total_pages = sum(len(f.pages) for f in artifact.files) - file_names = [f.file_name for f in artifact.files] - descriptions.append(f"- page_layout: {total_pages} pages from {file_names}") - continue descriptions.append("- unknown artifact") return "\n".join(descriptions) diff --git a/engine/src/stirling/agents/pdf_to_markdown/__init__.py b/engine/src/stirling/agents/pdf_to_markdown/__init__.py deleted file mode 100644 index d35ae05c7..000000000 --- a/engine/src/stirling/agents/pdf_to_markdown/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .agent import PdfToMarkdownAgent - -__all__ = ["PdfToMarkdownAgent"] diff --git a/engine/src/stirling/agents/pdf_to_markdown/agent.py b/engine/src/stirling/agents/pdf_to_markdown/agent.py deleted file mode 100644 index 8c0d7d8ee..000000000 --- a/engine/src/stirling/agents/pdf_to_markdown/agent.py +++ /dev/null @@ -1,435 +0,0 @@ -"""PDF to Markdown Agent. - -Converts a parsed PDF document into a single clean Markdown document, preserving -headings, paragraphs, and tables in reading order. -""" - -from __future__ import annotations - -import asyncio -import logging -import re -import time - -from pydantic import BaseModel, Field -from pydantic_ai import Agent -from pydantic_ai.output import NativeOutput - -from stirling.contracts import ( - EditCannotDoResponse, - GenerateFileResponse, - NeedContentFileRequest, - NeedContentResponse, - OrchestratorRequest, - PdfContentType, - SupportedCapability, - format_conversation_history, -) -from stirling.contracts.pdf_to_markdown import ( - PageLayout, - PageLayoutArtifact, - PdfToMarkdownCannotDoResponse, - PdfToMarkdownOrchestrateResponse, - PdfToMarkdownRequest, - PdfToMarkdownResponse, - PdfToMarkdownSuccessResponse, -) -from stirling.services import AppRuntime - -logger = logging.getLogger(__name__) - - -# Warn when output tokens are close to the typical model output limit (~8192 for most -# configurations). The actual limit is model-specific; this threshold catches likely truncation. -_OUTPUT_TOKEN_TRUNCATION_THRESHOLD = 7500 - -# Chunking limits — keep each LLM call to a manageable payload size. -# Fragment count is the primary driver of JSON payload size (each fragment carries x/y/width/ -# fontSize/bold metadata beyond its text). Page cap prevents low-text pages accumulating. -_MAX_CHUNK_FRAGMENTS = 1_000 -_MAX_CHUNK_PAGES = 10 - -# Max concurrent LLM calls — limits API rate pressure on large documents. -_MAX_PARALLEL_CHUNKS = 3 - -# ── LLM output model ──────────────────────────────────────────────────────────────────────────── - - -class _ReconstructionOutput(BaseModel): - markdown: str = Field(description="Full document reconstructed as clean Markdown.") - - -# ── Agent ──────────────────────────────────────────────────────────────────────────────────────── - - -class PdfToMarkdownAgent: - def __init__(self, runtime: AppRuntime) -> None: - self.runtime = runtime - self._sem = asyncio.Semaphore(_MAX_PARALLEL_CHUNKS) - self._reconstruct_agent = Agent( - model=runtime.smart_model, - output_type=NativeOutput(_ReconstructionOutput), - system_prompt=( - "You reconstruct PDF pages into clean Markdown from spatial fragment data.\n" - "Input: PAGE LAYOUT — per-fragment x/y/font data for structural analysis.\n\n" - "COLUMN DETECTION (for tables in page_layout):\n" - "- Look at the x-positions of fragments across 3+ consecutive lines.\n" - "- If fragments cluster at the same x-positions across multiple lines, those are table columns.\n" - "- Each distinct x-cluster is one column." - " Name them from the header row (the first line in the cluster).\n" - "- Do NOT merge values from different x-columns into one cell.\n\n" - "ROW DETECTION:\n" - "- Each unique y-coordinate (or group within 3pt) is one table row.\n" - "- Every line of layout data is its own row — do not merge rows.\n" - "- If a column has no fragment on a given y-row, that cell is empty.\n\n" - "TABLE RENDERING:\n" - "- Render as: | col1 | col2 | col3 |\n" - " | --- | --- | --- |\n" - " | val | val | val |\n" - "- One source row = one table row. Never collapse multiple rows into one.\n" - "- Preserve numeric values exactly (no rounding, no formatting changes).\n" - "- Bold cells: wrap with ** in the Markdown cell.\n" - "- CRITICAL: the separator row `| --- | --- |` appears EXACTLY ONCE per table, immediately\n" - " after the header row. NEVER put `| --- |` after a data row or between data rows.\n" - " NEVER put a blank line inside a table. All rows (header + data) must be consecutive.\n" - "- Do NOT produce a header-only table followed by a second table with the data rows.\n" - " One logical table = one markdown table block, with header, one separator, then all data.\n\n" - "GROUP HEADERS (label-only rows inside a table):\n" - "- A row is a group header when: the first column has text AND every numeric column is empty.\n" - "- Do NOT render group headers as table rows with empty cells.\n" - "- Break the table, emit the label as **bold text** on its own line," - " then start a new table for the rows that follow.\n" - "- Example labels: 'Policy functions', 'Non-current assets'.\n\n" - "TOTAL AND SUBTOTAL ROWS:\n" - "- Detect rows whose first cell contains (case-insensitive):" - " total, subtotal, surplus, balance, net, sum.\n" - "- These rows have numeric content — they are NOT group headers.\n" - "- Render the entire row in bold: | **Total income** | **1,234** | **5,678** |\n" - "- Keep total rows attached to the group they summarise.\n\n" - "MULTI-LEVEL TABLES (year or period as a row label):\n" - "- Detect when a row contains only a single label (a year like '2010' or period like 'Q1 2023')" - " with no numeric content, followed by repeated metric rows.\n" - "- Do NOT render the year as a table row.\n" - "- Normalise: add 'Year' as the first column, 'Metric' as the second," - " and repeat the year value on each metric row.\n\n" - "PROSE REGIONS:\n" - "- Lines where x-positions vary across lines (not repeating columns) are prose.\n" - "- Merge lines at the same x-level into paragraphs. Separate indented lines.\n\n" - "HEADINGS:\n" - "- A line is a heading when it is bold OR font_size ≥2pt above body.\n" - " CRITICAL EXCEPTION: a bold fragment is a TABLE HEADER CELL, not a document heading, when\n" - " the same y-row in page_layout contains other fragments at different x-positions.\n" - " Only classify a bold line as a document heading when it is the SOLE fragment on its y-row.\n" - " Example: 'Non-current assets' at y=120 with '2010'@x=350, '2009'@x=420, '2008'@x=490\n" - " → this is a table header row, NOT a heading. Render it as the first cell of the table.\n" - "- Use ## for section headings, ### for sub-headings. Use # only for the document title.\n\n" - "ORDERING:\n" - "- Process content top-to-bottom as it appears on the page.\n" - "- Interleave prose blocks and table blocks in page order.\n" - "- Do not move text that appears before a table to after it, or vice versa.\n\n" - "FIDELITY:\n" - "- Do NOT invent, summarise, or omit any content.\n" - "- Do NOT add commentary, metadata, or JSON — output Markdown only." - ), - model_settings={ - **runtime.smart_model_settings, - "temperature": 0.0, - "max_tokens": _OUTPUT_TOKEN_TRUNCATION_THRESHOLD, - }, - ) - - async def orchestrate(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse: - """Entry point for the orchestrator delegate. - - First turn: requests PAGE_LAYOUT extraction from Java via NeedContentResponse. - Resume turn: runs the LLM reconstruction and returns a write-file plan step. - """ - layout_artifact = next( - (a for a in request.artifacts if isinstance(a, PageLayoutArtifact)), - None, - ) - if layout_artifact is None: - return NeedContentResponse( - resume_with=SupportedCapability.PDF_TO_MARKDOWN, - reason="Page layout data is required to reconstruct the document.", - files=[ - NeedContentFileRequest(file=f, content_types=[PdfContentType.PAGE_LAYOUT]) for f in request.files - ], - max_pages=self.runtime.settings.max_pages, - max_characters=self.runtime.settings.max_characters, - ) - - page_layout = [page for entry in layout_artifact.files for page in entry.pages] - file_names = [f.name for f in request.files] - result = await self.handle( - PdfToMarkdownRequest( - user_message=request.user_message, - file_names=file_names, - conversation_history=request.conversation_history, - page_layout=page_layout, - ) - ) - if isinstance(result, PdfToMarkdownCannotDoResponse): - return EditCannotDoResponse(reason=result.reason) - - base = file_names[0].rsplit(".", 1)[0] if file_names else "document" - return GenerateFileResponse( - content=result.markdown, - filename=f"{base}-reconstruction.md", - summary="Reconstructed the document as a Markdown file.", - ) - - async def handle(self, request: PdfToMarkdownRequest) -> PdfToMarkdownResponse: - total_fragments = sum(len(line.fragments) for page in request.page_layout for line in page.lines) - logger.info( - "[pdf-to-markdown] received layout-pages=%d fragments=%d", - len(request.page_layout), - total_fragments, - ) - - if not request.page_layout: - logger.warning("[pdf-to-markdown] no content extracted from document; returning cannot_do") - return PdfToMarkdownCannotDoResponse( - reason=( - "No content was extracted from the document. " - "The file may be a scanned image PDF with no readable text. " - "Try running OCR on the document first." - ) - ) - - chunks = _build_page_chunks(request.page_layout) - logger.info("[pdf-to-markdown] chunks=%d (max %d in parallel)", len(chunks), _MAX_PARALLEL_CHUNKS) - - if len(chunks) == 1: - return await self._reconstruct_chunk(request, chunks[0], chunk_num=1, total_chunks=1) - - total = len(chunks) - results = await asyncio.gather( - *( - self._reconstruct_chunk(request, chunk, chunk_num=i + 1, total_chunks=total) - for i, chunk in enumerate(chunks) - ) - ) - - markdown_parts: list[str] = [] - for result in results: - if isinstance(result, PdfToMarkdownSuccessResponse) and result.markdown: - markdown_parts.append(result.markdown) - elif isinstance(result, PdfToMarkdownCannotDoResponse): - logger.warning("[pdf-to-markdown] chunk dropped: %s", result.reason) - - if not markdown_parts: - return PdfToMarkdownCannotDoResponse(reason="The document could not be reconstructed. All chunks failed.") - - logger.info("[pdf-to-markdown] assembly: %d/%d chunks produced output", len(markdown_parts), len(chunks)) - return PdfToMarkdownSuccessResponse(markdown="\n\n".join(markdown_parts)) - - async def _reconstruct_chunk( - self, - request: PdfToMarkdownRequest, - pages: list[PageLayout], - chunk_num: int, - total_chunks: int, - ) -> PdfToMarkdownResponse: - chunk_request = PdfToMarkdownRequest( - user_message=request.user_message, - file_names=request.file_names, - conversation_history=request.conversation_history, - page_layout=pages, - ) - try: - async with self._sem: - return await self._reconstruct_document(chunk_request, chunk_num, total_chunks) - except Exception as e: - logger.error("[pdf-to-markdown] chunk %d/%d failed: %s", chunk_num, total_chunks, e, exc_info=True) - return PdfToMarkdownCannotDoResponse( - reason="The document could not be reconstructed. The AI model failed to process it." - ) - - async def _reconstruct_document( - self, request: PdfToMarkdownRequest, chunk_num: int = 1, total_chunks: int = 1 - ) -> PdfToMarkdownSuccessResponse: - content = _build_reconstruction_prompt(request) - logger.info("[timing] chunk %d/%d llm-call prompt-chars=%d", chunk_num, total_chunks, len(content)) - t0 = time.monotonic() - result = await self._reconstruct_agent.run([content]) - llm_ms = int((time.monotonic() - t0) * 1000) - output: _ReconstructionOutput = result.output - usage = result.usage() - logger.info( - "[timing] chunk %d/%d llm-done ms=%d input-tokens=%s output-tokens=%s markdown-chars=%d", - chunk_num, - total_chunks, - llm_ms, - usage.input_tokens, - usage.output_tokens, - len(output.markdown), - ) - if usage.output_tokens and usage.output_tokens >= _OUTPUT_TOKEN_TRUNCATION_THRESHOLD: - logger.warning( - "[timing] chunk %d/%d output likely truncated (output-tokens=%d)", - chunk_num, - total_chunks, - usage.output_tokens, - ) - markdown = _remove_extra_separators(_fix_markdown_tables(_merge_orphaned_table_rows(output.markdown))) - return PdfToMarkdownSuccessResponse(markdown=markdown) - - -# ── Chunking ──────────────────────────────────────────────────────────────────────────────────── - - -def _build_page_chunks(pages: list[PageLayout]) -> list[list[PageLayout]]: - chunks: list[list[PageLayout]] = [] - current: list[PageLayout] = [] - current_fragments = 0 - for page in pages: - page_fragments = sum(len(line.fragments) for line in page.lines) - fragment_full = current and current_fragments + page_fragments > _MAX_CHUNK_FRAGMENTS - page_full = len(current) >= _MAX_CHUNK_PAGES - if fragment_full or page_full: - chunks.append(current) - current = [] - current_fragments = 0 - current.append(page) - current_fragments += page_fragments - if current: - chunks.append(current) - return chunks - - -# ── Prompt builders (module-level, no state) ──────────────────────────────────────────────────── - - -def _build_reconstruction_prompt(request: PdfToMarkdownRequest) -> str: - history = format_conversation_history(request.conversation_history) - file_names = ", ".join(request.file_names) if request.file_names else "Unknown files" - layout_section = _format_layout(request.page_layout) - - return ( - f"Files: {file_names}\n\n" - f"User request: {request.user_message}\n\n" - f"Conversation history:\n{history}\n\n" - "PAGE LAYOUT (structural source — x/y fragment positions):\n" - "Each line is: y=NNN | text@(x,y) fs=N text@(x,y) fs=N ...\n" - "- y=NNN is the vertical position (row). Lines close in y are the same visual row.\n" - "- x=NNN is the horizontal position (column). Consistent x across rows = a column.\n" - "- fs=N is font size. Larger = likely a heading.\n" - "- **bold** markers indicate bold text.\n\n" - f"{layout_section}" - ) - - -# ── LLM output post-processing ────────────────────────────────────────────────────────────────── - - -def _fix_markdown_tables(markdown: str) -> str: - """Remove blank lines between table rows produced by the LLM.""" - lines = markdown.split("\n") - result: list[str] = [] - i = 0 - while i < len(lines): - result.append(lines[i]) - if lines[i].strip().startswith("|"): - j = i + 1 - while j < len(lines) and lines[j].strip() == "": - j += 1 - if j < len(lines) and lines[j].strip().startswith("|"): - i = j - continue - i += 1 - return "\n".join(result) - - -_SEP_CELL = re.compile(r"^:?-+:?$") - - -def _is_sep_row(line: str) -> bool: - """Return True when a pipe row is a Markdown table separator (| --- | --- |).""" - stripped = line.strip() - if not stripped.startswith("|"): - return False - cells = [c.strip() for c in stripped.split("|") if c.strip()] - return bool(cells) and all(_SEP_CELL.match(c) for c in cells) - - -def _merge_orphaned_table_rows(markdown: str) -> str: - """Merge pipe-row blocks that lack a separator into the preceding table. - - When the LLM incorrectly breaks a table (e.g. on a false group-header), it emits - orphaned pipe rows with no header or separator. These are invalid markdown and get - merged back into the preceding table, discarding the intervening non-table content. - """ - lines = markdown.split("\n") - - segments: list[tuple[str, list[str]]] = [] - i = 0 - while i < len(lines): - if lines[i].strip().startswith("|"): - block: list[str] = [] - while i < len(lines) and lines[i].strip().startswith("|"): - block.append(lines[i]) - i += 1 - has_sep = any(_is_sep_row(row) for row in block) - segments.append(("table" if has_sep else "orphan", block)) - else: - block = [] - while i < len(lines) and not lines[i].strip().startswith("|"): - block.append(lines[i]) - i += 1 - segments.append(("prose", block)) - - result: list[tuple[str, list[str]]] = [] - last_table_idx: int | None = None - for seg_type, seg_lines in segments: - if seg_type == "orphan": - if last_table_idx is not None: - result = result[: last_table_idx + 1] - result[-1] = ("table", result[-1][1] + seg_lines) - else: - result.append((seg_type, seg_lines)) - else: - if seg_type == "table": - last_table_idx = len(result) - result.append((seg_type, seg_lines)) - - return "\n".join(line for _, seg_lines in result for line in seg_lines) - - -def _remove_extra_separators(markdown: str) -> str: - """Within each contiguous table block, keep only the first separator row.""" - lines = markdown.split("\n") - result: list[str] = [] - seen_sep = False - - for line in lines: - if not line.strip().startswith("|"): - seen_sep = False - result.append(line) - continue - if _is_sep_row(line): - if seen_sep: - continue - seen_sep = True - result.append(line) - - return "\n".join(result) - - -# ── Formatting helpers (module-level, no state) ────────────────────────────────────────────────── - - -def _format_layout(pages: list[PageLayout]) -> str: - if not pages: - return "None" - parts: list[str] = [] - for page in pages: - line_strs: list[str] = [] - for line in page.lines: - frags = " ".join( - f"{'**' if f.bold else ''}{f.text}{'**' if f.bold else ''}@({f.x:.0f},{f.y:.0f}) fs={f.font_size:.0f}" - for f in line.fragments - ) - line_strs.append(f"y={line.y:.0f} | {frags}") - parts.append(f"--- Page {page.page_number} ---\n" + "\n".join(line_strs)) - return "\n\n".join(parts) diff --git a/engine/src/stirling/contracts/__init__.py b/engine/src/stirling/contracts/__init__.py index 696749d7d..4bc4febcf 100644 --- a/engine/src/stirling/contracts/__init__.py +++ b/engine/src/stirling/contracts/__init__.py @@ -13,6 +13,7 @@ from .common import ( AiFile, ArtifactKind, ConversationMessage, + ConvertMarkdownResponse, ExtractedFileText, GenerateFileResponse, MathAuditorToolReportArtifact, @@ -96,17 +97,6 @@ from .pdf_questions import ( PdfQuestionTerminalResponse, ) from .pdf_review import PdfReviewOrchestrateResponse -from .pdf_to_markdown import ( - LayoutFragment, - LayoutLine, - PageLayout, - PageLayoutArtifact, - PageLayoutFileEntry, - PdfToMarkdownCannotDoResponse, - PdfToMarkdownOrchestrateResponse, - PdfToMarkdownRequest, - PdfToMarkdownResponse, -) from .progress import ( ProgressEvent, WholeDocCompressionRound, @@ -139,10 +129,6 @@ __all__ = [ "ConversationMessage", "DeleteDocumentResponse", "PurgeOwnerResponse", - "PdfToMarkdownCannotDoResponse", - "PdfToMarkdownOrchestrateResponse", - "PdfToMarkdownRequest", - "PdfToMarkdownResponse", "Discrepancy", "DiscrepancyKind", "EditCannotDoResponse", @@ -166,15 +152,11 @@ __all__ = [ "NeedContentFileRequest", "NeedContentResponse", "NeedIngestResponse", + "ConvertMarkdownResponse", "NextExecutionAction", "OrchestratorRequest", "OrchestratorResponse", - "LayoutFragment", - "LayoutLine", "Page", - "PageLayout", - "PageLayoutArtifact", - "PageLayoutFileEntry", "PageRange", "PageText", "PdfCommentInstruction", diff --git a/engine/src/stirling/contracts/common.py b/engine/src/stirling/contracts/common.py index 05103b1a4..b8030c58b 100644 --- a/engine/src/stirling/contracts/common.py +++ b/engine/src/stirling/contracts/common.py @@ -62,6 +62,7 @@ class WorkflowOutcome(StrEnum): CANNOT_CONTINUE = "cannot_continue" UNSUPPORTED_CAPABILITY = "unsupported_capability" GENERATE_FILE = "generate_file" + CONVERT_MARKDOWN = "convert_markdown" class ArtifactKind(StrEnum): @@ -183,6 +184,19 @@ class NeedIngestResponse(ApiModel): content_types: list[PdfContentType] = Field(default_factory=list) +class ConvertMarkdownResponse(ApiModel): + """Terminal signal: convert the listed files to Markdown deterministically. + + This is a deterministic, non-AI conversion. Java runs the PDF→Markdown converter + (``PdfMarkdownConverter``) on each file and returns the resulting ``.md`` file(s) as a + completed result. There is no resume turn — the conversion output is the final answer. + """ + + outcome: Literal[WorkflowOutcome.CONVERT_MARKDOWN] = WorkflowOutcome.CONVERT_MARKDOWN + reason: str + files_to_ingest: list[AiFile] + + class ToolOperationStep(ApiModel): kind: Literal[StepKind.TOOL] = StepKind.TOOL tool: AnyToolId diff --git a/engine/src/stirling/contracts/orchestrator.py b/engine/src/stirling/contracts/orchestrator.py index 1bf0f6eb3..8b916ccaf 100644 --- a/engine/src/stirling/contracts/orchestrator.py +++ b/engine/src/stirling/contracts/orchestrator.py @@ -11,6 +11,7 @@ from .common import ( AiFile, ArtifactKind, ConversationMessage, + ConvertMarkdownResponse, ExtractedFileText, GenerateFileResponse, NeedContentResponse, @@ -23,7 +24,6 @@ from .common import ( from .execution import NextExecutionAction from .pdf_edit import PdfEditTerminalResponse from .pdf_questions import PdfQuestionTerminalResponse -from .pdf_to_markdown import PageLayoutArtifact class ExtractedTextArtifact(ApiModel): @@ -32,7 +32,7 @@ class ExtractedTextArtifact(ApiModel): WorkflowArtifact = Annotated[ - ExtractedTextArtifact | PageLayoutArtifact | ToolReportArtifact, + ExtractedTextArtifact | ToolReportArtifact, Field(discriminator="kind"), ] @@ -61,6 +61,7 @@ type OrchestratorResponse = Annotated[ | GenerateFileResponse | NeedContentResponse | NeedIngestResponse + | ConvertMarkdownResponse | AgentDraftResponse | NextExecutionAction | UnsupportedCapabilityResponse, diff --git a/engine/src/stirling/contracts/pdf_to_markdown.py b/engine/src/stirling/contracts/pdf_to_markdown.py deleted file mode 100644 index 4d272e6e2..000000000 --- a/engine/src/stirling/contracts/pdf_to_markdown.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Contracts for the PDF to Markdown Agent. - -The agent accepts a parsed document and returns a single Markdown document that -faithfully reconstructs the PDF content — headings, paragraphs, and tables in -reading order, using page_layout as the primary source of truth for structure. - -Java extracts page layout via PdfIngester and returns it as a PageLayoutArtifact -through the orchestrator resume_with pattern. -""" - -from __future__ import annotations - -from typing import Annotated, Literal - -from pydantic import Field - -from stirling.models import ApiModel - -from .common import ArtifactKind, ConversationMessage, GenerateFileResponse, NeedContentResponse -from .pdf_edit import EditCannotDoResponse - -# ── Input: layout models (mirror Java's RawLine / TextFragment geometry) ──────────────────────── - - -class LayoutFragment(ApiModel): - """One text fragment with its bounding-box geometry and font properties.""" - - text: str - x: float - y: float - width: float - font_size: float - bold: bool - - -class LayoutLine(ApiModel): - """A visual line on the page: one y-coordinate and all fragments on that line.""" - - y: float - fragments: list[LayoutFragment] - - -class PageLayout(ApiModel): - """All layout lines for a single page, in top-to-bottom order.""" - - page_number: int - lines: list[LayoutLine] - - -# ── Artifact: page layout (produced by Java, consumed by orchestrate()) ────────────────────────── - - -class PageLayoutFileEntry(ApiModel): - """Page layout data for one file, as extracted by Java's PdfIngester.""" - - file_name: str - pages: list[PageLayout] = Field(default_factory=list) - - -class PageLayoutArtifact(ApiModel): - """Artifact carrying full spatial page layout for all input files.""" - - kind: Literal[ArtifactKind.PAGE_LAYOUT] = ArtifactKind.PAGE_LAYOUT - files: list[PageLayoutFileEntry] = Field(default_factory=list) - - -# ── Input: full request ────────────────────────────────────────────────────────────────────────── - - -class PdfToMarkdownRequest(ApiModel): - """Request sent by Java after PdfIngester has parsed the document. - - page_layout: per-fragment positional data from the original (y-sorted) line order. - Each fragment carries its x/y position, width, font size, and bold flag. - This is the primary source of truth for column detection and heading hierarchy. - """ - - user_message: str - file_names: list[str] = Field(default_factory=list) - conversation_history: list[ConversationMessage] = Field(default_factory=list) - page_layout: list[PageLayout] = Field(default_factory=list) - - -# ── Output: response variants ──────────────────────────────────────────────────────────────────── - - -class PdfToMarkdownSuccessResponse(ApiModel): - outcome: Literal["document_reconstructed"] = "document_reconstructed" - markdown: str - - -class PdfToMarkdownCannotDoResponse(ApiModel): - outcome: Literal["cannot_do"] = "cannot_do" - reason: str - - -type PdfToMarkdownResponse = Annotated[ - PdfToMarkdownSuccessResponse | PdfToMarkdownCannotDoResponse, - Field(discriminator="outcome"), -] - -type PdfToMarkdownOrchestrateResponse = Annotated[ - GenerateFileResponse | EditCannotDoResponse | NeedContentResponse, - Field(discriminator="outcome"), -] diff --git a/engine/tests/test_pdf_to_markdown.py b/engine/tests/test_pdf_to_markdown.py deleted file mode 100644 index 32870a945..000000000 --- a/engine/tests/test_pdf_to_markdown.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Tests for PDF to Markdown agent. - -Two cases: -1. Narrative-only page: request validates and routes to reconstruction. -2. Mixed text + table page: layout with table region validates correctly. -""" - -from __future__ import annotations - -from stirling.contracts.pdf_to_markdown import ( - LayoutFragment, - LayoutLine, - PageLayout, - PageLayoutArtifact, - PdfToMarkdownRequest, - PdfToMarkdownSuccessResponse, -) - - -def _frag(text: str, x: float, y: float, font_size: float = 10.0, bold: bool = False) -> LayoutFragment: - return LayoutFragment(text=text, x=x, y=y, width=float(len(text) * 6), font_size=font_size, bold=bold) - - -def _line(y: float, *frags: LayoutFragment) -> LayoutLine: - return LayoutLine(y=y, fragments=list(frags)) - - -# ── Test 1: Narrative-only reconstruction ──────────────────────────────────────────────────────── - - -# ── Contract test: Java serialization ↔ Python deserialization ────────────────────────────────── -# This JSON is also asserted field-by-field in PageLayoutArtifactContractTest.java. -# If either side renames a field, one of these tests fails. -_CONTRACT_JSON = ( - '{"kind":"page_layout","files":[{"fileName":"test.pdf","pages":' - '[{"pageNumber":1,"lines":[{"y":10.0,"fragments":' - '[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}' -) - - -def test_page_layout_artifact_deserialises_java_json() -> None: - artifact = PageLayoutArtifact.model_validate_json(_CONTRACT_JSON) - - assert artifact.kind == "page_layout" - assert artifact.files[0].file_name == "test.pdf" - page = artifact.files[0].pages[0] - assert page.page_number == 1 - line = page.lines[0] - assert line.y == 10.0 - frag = line.fragments[0] - assert frag.text == "Hello" - assert frag.x == 1.0 - assert frag.y == 2.0 - assert frag.width == 30.0 - assert frag.font_size == 12.0 - assert frag.bold is True - - -def test_narrative_reconstruction_request_validates() -> None: - """A prose-only page with no tables produces a valid PdfToMarkdownRequest.""" - layout = PageLayout( - page_number=1, - lines=[ - _line(72.0, _frag("Annual Report 2023", x=72.0, y=72.0, font_size=18.0, bold=True)), - _line(100.0, _frag("Our revenue grew significantly", x=72.0, y=100.0)), - _line(114.0, _frag("during the fiscal year ended", x=72.0, y=114.0)), - _line(128.0, _frag("December 31, 2023.", x=72.0, y=128.0)), - ], - ) - request = PdfToMarkdownRequest( - user_message="reconstruct this document", - page_layout=[layout], - ) - - assert len(request.page_layout) == 1 - assert len(request.page_layout[0].lines) == 4 - assert request.page_layout[0].lines[0].fragments[0].bold is True - assert request.page_layout[0].lines[0].fragments[0].font_size == 18.0 - - -def test_narrative_reconstruction_response_validates() -> None: - """PdfToMarkdownSuccessResponse accepts markdown and returns document_reconstructed outcome.""" - response = PdfToMarkdownSuccessResponse( - markdown="# Annual Report 2023\n\nOur revenue grew significantly during the fiscal year.", - ) - - assert response.outcome == "document_reconstructed" - assert response.markdown.startswith("#") - - -# ── Test 2: Mixed text + table reconstruction ───────────────────────────────────────────────────── - - -def test_mixed_page_layout_validates() -> None: - """A page with both prose lines and a table region produces a valid request.""" - layout = PageLayout( - page_number=1, - lines=[ - # Prose heading - _line(50.0, _frag("Projects in Development", x=72.0, y=50.0, font_size=14.0, bold=True)), - # Table header row - _line( - 80.0, - _frag("Project Name", x=72.0, y=80.0, bold=True), - _frag("Location", x=200.0, y=80.0, bold=True), - _frag("Size (MW)", x=290.0, y=80.0, bold=True), - ), - # Table data rows - _line( - 95.0, - _frag("Chaplin Wind 1", x=72.0, y=95.0), - _frag("Saskatchewan", x=200.0, y=95.0), - _frag("177", x=290.0, y=95.0), - ), - _line( - 110.0, - _frag("Amherst Island 2", x=72.0, y=110.0), - _frag("Ontario", x=200.0, y=110.0), - _frag("75", x=290.0, y=110.0), - ), - # Prose after table - _line(140.0, _frag("Notes:", x=72.0, y=140.0, bold=True)), - _line(154.0, _frag("1 PPA signed", x=85.0, y=154.0)), - ], - ) - request = PdfToMarkdownRequest( - user_message="markdown", - page_layout=[layout], - ) - - assert len(request.page_layout[0].lines) == 6 - # Header line has 3 fragments at distinct x-positions (column detection) - header_line = request.page_layout[0].lines[1] - xs = [f.x for f in header_line.fragments] - assert xs == [72.0, 200.0, 290.0] - # Data rows have matching x-positions - data_row = request.page_layout[0].lines[2] - assert [f.x for f in data_row.fragments] == [72.0, 200.0, 290.0] diff --git a/testing/cucumber/features/external.feature b/testing/cucumber/features/external.feature index bf392d720..e8e8d8527 100644 --- a/testing/cucumber/features/external.feature +++ b/testing/cucumber/features/external.feature @@ -233,8 +233,7 @@ Feature: API Validation When I send the API request to the endpoint "/api/v1/convert/pdf/markdown" Then the response status code should be 200 And the response file should have size greater than 100 - And the response file should have extension ".zip" - And the response ZIP should contain 4 files + And the response file should have extension ".md" @positive @pdftocsv diff --git a/testing/test.sh b/testing/test.sh index 0aa30294a..6c7c62c4a 100644 --- a/testing/test.sh +++ b/testing/test.sh @@ -911,7 +911,7 @@ main() { CUCUMBER_JUNIT_DIR="$PROJECT_ROOT/testing/cucumber/junit" mkdir -p "$CUCUMBER_JUNIT_DIR" cd "testing/cucumber" - start_test_timer "Stirling-PDF-Regression" + start_test_timer "Stirling-PDF-Regression $CONTAINER_NAME" # Snapshot docker log line count before behave so we can extract only behave-window logs DOCKER_LOG_BEFORE=$(docker logs "$CONTAINER_NAME" 2>&1 | wc -l) @@ -999,7 +999,7 @@ main() { # Save docker logs from the behave window to a dedicated file local cucumber_log="$REPORT_DIR/cucumber-docker-context.log" docker logs "$CONTAINER_NAME" 2>&1 | tail -n +"$((DOCKER_LOG_BEFORE + 1))" > "$cucumber_log" 2>/dev/null || true - test_failure_logs["Stirling-PDF-Regression"]="$cucumber_log" + test_failure_logs["Stirling-PDF-Regression $CONTAINER_NAME"]="$cucumber_log" gha_group "Docker logs during behave run: $CONTAINER_NAME" tail -100 "$cucumber_log" @@ -1012,7 +1012,7 @@ main() { capture_file_list "$CONTAINER_NAME" "$AFTER_FILE" compare_file_lists "$BEFORE_FILE" "$AFTER_FILE" "$DIFF_FILE" "$CONTAINER_NAME" fi - stop_test_timer "Stirling-PDF-Regression" + stop_test_timer "Stirling-PDF-Regression $CONTAINER_NAME" fi # `down` with the override removes the agent bind-mount cleanly. The # SIGTERM that `down` sends is what triggers dumponexit=true in the From 611468b97232d1332b528bfda0af7f6034101446 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:58:33 +0100 Subject: [PATCH 5/5] Add SaaS MCP usage tab (#6590) --- .../public/locales/en-GB/translation.toml | 54 ++-- .../config/configSections/McpSection.tsx | 262 ++++++++++++++++++ .../shared/config/saasConfigNavSections.tsx | 40 +++ .../saas/components/shared/config/types.ts | 9 +- .../src/saas/services/apiClient.test.ts | 9 + 5 files changed, 348 insertions(+), 26 deletions(-) create mode 100644 frontend/editor/src/saas/components/shared/config/configSections/McpSection.tsx diff --git a/frontend/editor/public/locales/en-GB/translation.toml b/frontend/editor/public/locales/en-GB/translation.toml index e907cba19..b3f5109ce 100644 --- a/frontend/editor/public/locales/en-GB/translation.toml +++ b/frontend/editor/public/locales/en-GB/translation.toml @@ -3049,6 +3049,37 @@ impact = "Any applications or services currently using these keys will stop work title = "Refresh API Keys" warning = "⚠️ Warning: This action will generate new API keys and make your previous keys invalid." +[config.mcp] +description = "Model Context Protocol (MCP) lets AI assistants like Claude use your Stirling PDF tools directly. Connect a client once and your assistant can convert, edit, secure and process documents on your behalf." +navLabel = "MCP Server" +tip = "Every action your assistant runs is performed as your account and counts towards your usage, just like using Stirling PDF directly." +title = "MCP Server" +viewApiKeys = "View API keys" + +[config.mcp.copy] +configLabel = "Config" +copied = "Copied" +copy = "Copy" +endpointLabel = "Endpoint URL" +tooltip = "Copy {{label}}" +tooltipCopied = "{{label}} copied" + +[config.mcp.endpoint] +label = "Your MCP endpoint" + +[config.mcp.setup] +addTo = "Add to" +hint = "Pick your client, paste the snippet into the file shown, then restart it. You'll sign in with your Stirling account on first use - no keys to copy." +title = "Connect your AI assistant" + +[config.mcp.tools] +ai = "AI" +convert = "Convert" +misc = "Misc" +pages = "Pages" +security = "Security" +title = "What your assistant can do" + [config.overview] description = "Current application settings and configuration details." error = "Error" @@ -3210,7 +3241,6 @@ creditsRemaining = "{{current}} of {{total}} remaining" creditsThisMonth = "Monthly credits" current = "Current Plan" customPricing = "Custom" -customWatchedFolders = "Custom Watched Folders" dedicatedSupportSlas = "Dedicated support & SLAs" enterpriseSubscription = "Enterprise" everythingInCredits = "Everything in Credits, plus:" @@ -3518,7 +3548,6 @@ addFiles = "Add Files" [fileManager] active = "Active" -addToWatchedFolder = "Add to Watched Folder" addToUpload = "Add to Upload" changesNotUploaded = "Changes not uploaded" clearAll = "Clear All" @@ -4563,10 +4592,8 @@ sendResetLink = "Send reset link" sessionExpired = "Your session has expired. Please sign in again." createAccount = "Create an account" signin = "Sign in" -signInAnonymously = "Sign Up as a Guest" signingIn = "Signing in..." signInWith = "Sign in with" -subtitle = "Sign back in to Stirling PDF" title = "Sign in" unexpectedError = "Unexpected error: {{message}}" updatePassword = "Update password" @@ -5782,7 +5809,6 @@ label = "SMTP Username" [quickAccess] access = "Access" -watchedFolders = "Watched Folders" accessAddPerson = "Add another person" accessBack = "Back" accessCopyLink = "Copy link" @@ -6873,6 +6899,7 @@ createFailed = "Failed to create signing request" [signup] accountCreatedSuccessfully = "Account created successfully! You can now sign in." +alreadyHaveAccount = "I already have an account" checkEmailConfirmation = "Check your email for a confirmation link to complete your registration." confirmPassword = "Confirm password" confirmPasswordPlaceholder = "Confirm password" @@ -6898,7 +6925,6 @@ skip = "Skip" subtitle = "Join Stirling PDF to get started" title = "Create an account" unexpectedError = "Unexpected error: {{message}}" -useEmailInstead = "Use Email Instead" [sizes] large = "Large" @@ -8172,20 +8198,14 @@ newFolder = "New folder" noFolders = "No watched folders yet" sidebarTitle = "Watched Folders" title = "Watched Folders" -sidebarFiles = "My Files" [watchedFolders.modal] automation = "Automation" -automationSaved = "Automation saved" createFolder = "Create Folder" -stepsSaved = "Steps saved — click Create Folder to finish" automationRequired = "Add at least one configured step before saving." color = "Accent color" createTitle = "New Watched Folder" -description = "Description" -descriptionPlaceholder = "What does this folder do?" editTitle = "Edit Watched Folder" -icon = "Icon" name = "Folder name" namePlaceholder = "My Watched Folder" nameRequired = "Folder name is required" @@ -8193,17 +8213,13 @@ nameTooLong = "Folder name must be 50 characters or less" saveChanges = "Save Changes" saveFailed = "Failed to save folder. Please try again." automationNameFallback = "Watched Folder automation" -retryLabel = "Auto-retry" maxRetries = "Max retries" -maxRetriesDesc = "0 to disable" retryDelay = "Delay (minutes)" -outputLabel = "Output" outputModeVersion = "Replace original?" outputModeVersionDesc = "Output becomes a new version of the input file rather than a separate file" outputModeNewDesc = "Output is saved as a new separate file" outputName = "Output filename prefix" outputNameSuffix = "Suffix" -outputNamePlaceholderVersion = "Same as original" sectionFolder = "Folder" sectionSourceOutput = "Source & Output" sectionSteps = "Steps" @@ -8230,14 +8246,10 @@ positionSuffix = "Suffix" [watchedFolders.home] create = "Create your first Watched Folder" -dropHere = "Drop to process" editFolder = "Edit folder" empty = "No watched folders yet" file = "file" files = "files" -openFolder = "Open folder" -title = "Watched Folders" -subtitle = "Folders that automatically process PDFs with your configured pipeline" emptyTitle = "Automate your PDF workflows" emptyDesc = "Set up a Watched Folder once. Drop PDFs in and they're automatically compressed, OCR'd, split, merged — whatever your pipeline does." addAnother = "Add another Watched Folder" @@ -8245,7 +8257,6 @@ addAnotherDesc = "Automatically process files with a new pipeline" resume = "Resume" pause = "Pause" deleteFolder = "Delete folder" -noSteps = "No automation steps configured" [watchedFolders.card] edit = "Edit folder" @@ -8328,7 +8339,6 @@ previewLoadFailed = "Could not load file preview." back = "Back" dismiss = "Dismiss" retry = "Retry" -view = "View" download = "Download" downloadInput = "Download input" downloadOutput = "Download output" diff --git a/frontend/editor/src/saas/components/shared/config/configSections/McpSection.tsx b/frontend/editor/src/saas/components/shared/config/configSections/McpSection.tsx new file mode 100644 index 000000000..8313fdd12 --- /dev/null +++ b/frontend/editor/src/saas/components/shared/config/configSections/McpSection.tsx @@ -0,0 +1,262 @@ +import { useMemo } from "react"; +import { useTranslation } from "react-i18next"; +import { + Stack, + Paper, + Text, + Group, + Alert, + Code, + Badge, + Button, + CopyButton, + Tabs, + Tooltip, + ThemeIcon, +} from "@mantine/core"; +import LocalIcon from "@app/components/shared/LocalIcon"; +import { useAppConfig } from "@app/contexts/AppConfigContext"; +import { openAppSettings } from "@app/utils/appSettings"; + +/** Strip a single trailing slash so we can safely append paths. */ +function trimTrailingSlash(url: string): string { + return url.endsWith("/") ? url.slice(0, -1) : url; +} + +/** A small copy-to-clipboard button that sits inline next to a URL/snippet. */ +function CopyInline({ value, label }: { value: string; label: string }) { + const { t } = useTranslation(); + return ( + + {({ copied, copy }) => ( + + + + )} + + ); +} + +// SaaS MCP guide: always shown, explains how to point an AI assistant at the +// OAuth-protected /mcp endpoint with per-client config. +export default function McpSection() { + const { t } = useTranslation(); + const { config } = useAppConfig(); + + const baseUrl = useMemo(() => { + const raw = + config?.baseUrl || + (typeof window !== "undefined" ? window.location.origin : ""); + return trimTrailingSlash(raw); + }, [config?.baseUrl]); + + const mcpUrl = `${baseUrl}/mcp`; + + // Per-client connection snippets pointing at this deployment's /mcp endpoint. + const clients = useMemo( + () => [ + { + value: "claude", + label: "Claude Desktop", + file: "claude_desktop_config.json", + config: JSON.stringify( + { mcpServers: { "stirling-pdf": { type: "http", url: mcpUrl } } }, + null, + 2, + ), + }, + { + value: "codex", + label: "Codex CLI", + file: "~/.codex/config.toml", + config: `[mcp_servers.stirling-pdf]\nurl = "${mcpUrl}"`, + }, + { + value: "vscode", + label: "VS Code", + file: ".vscode/mcp.json", + config: JSON.stringify( + { servers: { "stirling-pdf": { type: "http", url: mcpUrl } } }, + null, + 2, + ), + }, + ], + [mcpUrl], + ); + + const tools: { icon: string; key: string; fallback: string }[] = [ + { + icon: "sync-alt-rounded", + key: "config.mcp.tools.convert", + fallback: "Convert", + }, + { + icon: "description-rounded", + key: "config.mcp.tools.pages", + fallback: "Pages", + }, + { icon: "lock", key: "config.mcp.tools.security", fallback: "Security" }, + { + icon: "construction-rounded", + key: "config.mcp.tools.misc", + fallback: "Misc", + }, + { icon: "smart-toy-rounded", key: "config.mcp.tools.ai", fallback: "AI" }, + ]; + + return ( +

+ +
+ + + + + + {t("config.mcp.title", "MCP Server")} + + + + {t( + "config.mcp.description", + "Model Context Protocol (MCP) lets AI assistants like Claude use your Stirling PDF tools directly. Connect a client once and your assistant can convert, edit, secure and process documents on your behalf.", + )} + +
+ + {/* Endpoint */} + + + + + {t("config.mcp.endpoint.label", "Your MCP endpoint")} + + {mcpUrl} + + + + + + {/* Per-client setup */} + + + + {t("config.mcp.setup.title", "Connect your AI assistant")} + + + {t( + "config.mcp.setup.hint", + "Pick your client, paste the snippet into the file shown, then restart it. You'll sign in with your Stirling account on first use - no keys to copy.", + )} + + + + {clients.map((c) => ( + + {c.label} + + ))} + + {clients.map((c) => ( + + + + + {t("config.mcp.setup.addTo", "Add to")}{" "} + {c.file} + + + + {c.config} + + + ))} + + + + + {/* Available tools */} +
+ + {t("config.mcp.tools.title", "What your assistant can do")} + + + {tools.map((tool) => ( + + } + > + {t(tool.key, tool.fallback)} + + ))} + +
+ + {/* Tip / cross-link */} + } + > + + + {t( + "config.mcp.tip", + "Every action your assistant runs is performed as your account and counts towards your usage, just like using Stirling PDF directly.", + )} + + + + + +
+ ); +} diff --git a/frontend/editor/src/saas/components/shared/config/saasConfigNavSections.tsx b/frontend/editor/src/saas/components/shared/config/saasConfigNavSections.tsx index 5dd293451..27979ff5a 100644 --- a/frontend/editor/src/saas/components/shared/config/saasConfigNavSections.tsx +++ b/frontend/editor/src/saas/components/shared/config/saasConfigNavSections.tsx @@ -9,6 +9,7 @@ import GeneralSection from "@app/components/shared/config/configSections/General import PasswordSecurity from "@app/components/shared/config/configSections/PasswordSecurity"; import ApiKeys from "@app/components/shared/config/configSections/ApiKeys"; import Plan from "@app/components/shared/config/configSections/Plan"; +import McpSection from "@app/components/shared/config/configSections/McpSection"; type OverviewComponent = React.ComponentType<{ onLogoutClick: () => void }>; @@ -108,6 +109,44 @@ function appendBillingSection( ]; } +// Add an "MCP Server" tab in the Developer section. Always shown in SaaS; +// purely informational, so it appears for anonymous users too. +function appendMcpSection( + sections: ConfigNavSection[], + t: TFunction<"translation", undefined>, +): ConfigNavSection[] { + const hasMcp = sections.some((section) => + section.items.some((item) => item.key === "mcp"), + ); + + if (hasMcp) { + return sections; + } + + const mcpItem = { + key: "mcp" as const, + label: t("config.mcp.navLabel", "MCP Server"), + icon: "smart-toy-rounded", + component: , + }; + + const developerIndex = sections.findIndex((section) => + section.items.some( + (item) => item.key === "developer" || item.key === "api-keys", + ), + ); + + if (developerIndex === -1) { + return [...sections, { title: "Developer", items: [mcpItem] }]; + } + + return sections.map((section, index) => + index === developerIndex + ? { ...section, items: [...section.items, mcpItem] } + : section, + ); +} + export function createSaasConfigNavSections( Overview: OverviewComponent, onLogoutClick: () => void, @@ -151,6 +190,7 @@ export function createSaasConfigNavSections( sections = ensurePreferencesSection(sections); sections = appendDeveloperSection(sections); + sections = appendMcpSection(sections, t); if (!isAnonymous) { sections = appendBillingSection(sections, t); diff --git a/frontend/editor/src/saas/components/shared/config/types.ts b/frontend/editor/src/saas/components/shared/config/types.ts index e39944212..f80bd1ac1 100644 --- a/frontend/editor/src/saas/components/shared/config/types.ts +++ b/frontend/editor/src/saas/components/shared/config/types.ts @@ -1,8 +1,9 @@ import { VALID_NAV_KEYS as CORE_NAV_KEYS } from "@core/components/shared/config/types"; -// SaaS adds an "overview" account section. All other keys (including ones -// SaaS doesn't render today) come from core - subtracting them here would -// just break the type union without affecting runtime nav. -export const VALID_NAV_KEYS = [...CORE_NAV_KEYS, "overview"] as const; +// SaaS adds an "overview" account section and an "mcp" integrations tab. All +// other keys (including ones SaaS doesn't render today) come from core - +// subtracting them here would just break the type union without affecting +// runtime nav. +export const VALID_NAV_KEYS = [...CORE_NAV_KEYS, "overview", "mcp"] as const; export type NavKey = (typeof VALID_NAV_KEYS)[number]; diff --git a/frontend/editor/src/saas/services/apiClient.test.ts b/frontend/editor/src/saas/services/apiClient.test.ts index 56480bb7c..45646bf8d 100644 --- a/frontend/editor/src/saas/services/apiClient.test.ts +++ b/frontend/editor/src/saas/services/apiClient.test.ts @@ -13,6 +13,15 @@ vi.mock("@app/auth/supabase", () => ({ }, })); +// Stub apiClient's heavy UI/util deps so importing it stays cheap. None are +// exercised here (toast, plan settings and handleHttpError paths aren't hit), +// and re-importing the real toast graph per test made these tests time out. +vi.mock("@app/components/toast", () => ({ alert: vi.fn() })); +vi.mock("@app/utils/appSettings", () => ({ openPlanSettings: vi.fn() })); +vi.mock("@app/services/httpErrorHandler", () => ({ + handleHttpError: vi.fn().mockResolvedValue(undefined), +})); + describe("apiClient", () => { beforeEach(() => { vi.clearAllMocks();