Add Valkey cluster backplane and sticky-410 ownership (clusters) (#6472)

app/common/src/main/java/stirling/software/common/cluster/RateLimitStore.java

@@ -2,7 +2,13 @@ package stirling.software.common.cluster;
 
 import java.time.Duration;
 
-/** Token-bucket rate limiting backed by the cluster backplane. */
+/**
+ * Token-bucket rate limiting backed by the cluster backplane.
+ *
+ * <p>In-process implementations enforce a per-JVM limit; distributed implementations enforce a
+ * single global limit across every node. Both use a Bucket4j greedy-refill token bucket so the
+ * semantics match across single-node and cluster deployments.
+ */
 public interface RateLimitStore {
 
     /**

app/common/src/main/java/stirling/software/common/service/JobExecutorService.java

@@ -57,85 +57,38 @@ public class JobExecutorService {
         this.resourceMonitor = resourceMonitor;
         this.jobQueue = jobQueue;
 
-        // Parse session timeout and calculate effective timeout once during initialization
         long sessionTimeoutMs = parseSessionTimeout(sessionTimeout);
         this.effectiveTimeoutMs = Math.min(asyncRequestTimeoutMs, sessionTimeoutMs);
         log.debug(
                 "Job executor configured with effective timeout of {} ms", this.effectiveTimeoutMs);
     }
 
-    /**
-     * Run a job either asynchronously or synchronously
-     *
-     * @param async Whether to run the job asynchronously
-     * @param work The work to be done
-     * @return The response
-     */
     public ResponseEntity<?> runJobGeneric(boolean async, Supplier<Object> work) {
         return runJobGeneric(async, work, -1);
     }
 
-    /**
-     * Run a job either asynchronously or synchronously with a custom timeout
-     *
-     * @param async Whether to run the job asynchronously
-     * @param work The work to be done
-     * @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
-     * @return The response
-     */
     public ResponseEntity<?> runJobGeneric(
             boolean async, Supplier<Object> work, long customTimeoutMs) {
         return runJobGeneric(async, work, customTimeoutMs, false, 50);
     }
 
-    /**
-     * Run a job either asynchronously or synchronously with custom parameters
-     *
-     * @param async Whether to run the job asynchronously
-     * @param work The work to be done
-     * @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
-     * @param queueable Whether this job can be queued when system resources are limited
-     * @param resourceWeight The resource weight of this job (1-100)
-     * @return The response
-     */
     public ResponseEntity<?> runJobGeneric(
             boolean async,
             Supplier<Object> work,
             long customTimeoutMs,
             boolean queueable,
             int resourceWeight) {
-        // Generate base UUID
         String baseJobId = UUID.randomUUID().toString();
-
-        // Scope job to authenticated user if security is enabled
         String scopedJobKey = getScopedJobKey(baseJobId);
 
         log.debug("Generated jobId: {} (base: {})", scopedJobKey, baseJobId);
 
-        // Store the scoped job ID in the request for potential use by other components
         if (request != null) {
             request.setAttribute("jobId", scopedJobKey);
-
-            // Also track this job ID in the user's session for authorization purposes
-            // This ensures users can only cancel their own jobs
-            if (request.getSession() != null) {
-                @SuppressWarnings("unchecked")
-                java.util.Set<String> userJobIds =
-                        (java.util.Set<String>) request.getSession().getAttribute("userJobIds");
-
-                if (userJobIds == null) {
-                    userJobIds = new java.util.concurrent.ConcurrentSkipListSet<>();
-                    request.getSession().setAttribute("userJobIds", userJobIds);
-                }
-
-                userJobIds.add(scopedJobKey);
-                log.debug("Added scoped job ID {} to user session", scopedJobKey);
-            }
         }
 
         String jobId = scopedJobKey;
 
-        // Determine which timeout to use
         long timeoutToUse = customTimeoutMs > 0 ? customTimeoutMs : effectiveTimeoutMs;
 
         log.debug(
@@ -146,7 +99,6 @@ public class JobExecutorService {
                 queueable,
                 resourceWeight);
 
-        // Check if we need to queue this job based on resource availability
         boolean shouldQueue =
                 queueable
                         && async
@@ -154,7 +106,6 @@ public class JobExecutorService {
                         resourceMonitor.shouldQueueJob(resourceWeight);
 
         if (shouldQueue) {
-            // Queue the job instead of executing immediately
             log.debug(
                     "Queueing job {} due to resource constraints (weight: {})",
                     jobId,
@@ -162,18 +113,12 @@ public class JobExecutorService {
 
             taskManager.createTask(jobId);
 
-            // Create a specialized wrapper that updates the TaskManager
             final String capturedJobIdForQueue = jobId;
             Supplier<Object> wrappedWork =
                     () -> {
                         try {
-                            // Set jobId in ThreadLocal context for the queued job
                             stirling.software.common.util.JobContext.setJobId(
                                     capturedJobIdForQueue);
-                            log.debug(
-                                    "Set jobId {} in JobContext for queued job execution",
-                                    capturedJobIdForQueue);
-
                             Object result = work.get();
                             processJobResult(capturedJobIdForQueue, result);
                             return result;
@@ -186,21 +131,17 @@ public class JobExecutorService {
                             taskManager.setError(capturedJobIdForQueue, e.getMessage());
                             throw e;
                         } finally {
-                            // Clean up ThreadLocal to avoid memory leaks
                             stirling.software.common.util.JobContext.clear();
                         }
                     };
 
-            // Queue the job and get the future
             CompletableFuture<ResponseEntity<?>> future =
                     jobQueue.queueJob(jobId, resourceWeight, wrappedWork, timeoutToUse);
 
-            // Return immediately with job ID
             return ResponseEntity.ok().body(new JobResponse<>(true, jobId, null));
         } else if (async) {
             taskManager.createTask(jobId);
 
-            // Capture the jobId for the async thread
             final String capturedJobId = jobId;
 
             executor.execute(
@@ -211,13 +152,7 @@ public class JobExecutorService {
                                     capturedJobId,
                                     timeoutToUse);
 
-                            // Set jobId in ThreadLocal context for the async thread
                             stirling.software.common.util.JobContext.setJobId(capturedJobId);
-                            log.debug(
-                                    "Set jobId {} in JobContext for async execution",
-                                    capturedJobId);
-
-                            // Execute with timeout
                             Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
                             processJobResult(capturedJobId, result);
                         } catch (TimeoutException te) {
@@ -227,7 +162,6 @@ public class JobExecutorService {
                             log.error("Error executing job {}: {}", jobId, e.getMessage(), e);
                             taskManager.setError(jobId, e.getMessage());
                         } finally {
-                            // Clean up ThreadLocal to avoid memory leaks
                             stirling.software.common.util.JobContext.clear();
                         }
                     });
@@ -237,27 +171,19 @@ public class JobExecutorService {
             try {
                 log.debug("Running sync job with timeout {} ms", timeoutToUse);
 
-                // Make jobId available to downstream components on the worker thread
                 stirling.software.common.util.JobContext.setJobId(jobId);
-                log.debug("Set jobId {} in JobContext for sync execution", jobId);
-
-                // Execute with timeout
                 Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
 
-                // If the result is already a ResponseEntity, return it directly
                 if (result instanceof ResponseEntity) {
                     return (ResponseEntity<?>) result;
                 }
 
-                // Process different result types
                 return handleResultForSyncJob(result);
             } catch (TimeoutException te) {
                 log.error("Synchronous job timed out after {} ms", timeoutToUse);
                 return ResponseEntity.internalServerError()
                         .body(Map.of("error", "Job timed out after " + timeoutToUse + " ms"));
             } catch (RuntimeException e) {
-                // Check if this is a typed exception that should be handled by
-                // GlobalExceptionHandler (either directly or wrapped)
                 Throwable cause = e.getCause();
                 if (e instanceof IllegalArgumentException
                         || cause
@@ -267,16 +193,13 @@ public class JobExecutorService {
                                 instanceof
                                 stirling.software.common.util.ExceptionUtils
                                         .BaseValidationException) {
-                    // Rethrow so GlobalExceptionHandler can handle with proper HTTP status codes
                     throw e;
                 }
-                // Handle other RuntimeExceptions as generic errors
                 log.error("Error executing synchronous job: {}", e.getMessage(), e);
                 return ResponseEntity.internalServerError()
                         .body(Map.of("error", "Job failed: " + e.getMessage()));
             } catch (Exception e) {
                 log.error("Error executing synchronous job: {}", e.getMessage(), e);
-                // Construct a JSON error response
                 return ResponseEntity.internalServerError()
                         .body(Map.of("error", "Job failed: " + e.getMessage()));
             } finally {
@@ -285,23 +208,13 @@ public class JobExecutorService {
         }
     }
 
-    /**
-     * Process the result of an asynchronous job
-     *
-     * @param jobId The job ID
-     * @param result The result
-     */
     private void processJobResult(String jobId, Object result) {
         try {
             if (result instanceof byte[]) {
-                // Store byte array directly to disk to avoid double memory consumption
                 String fileId = fileStorage.storeBytes((byte[]) result, "result.pdf");
                 taskManager.setFileResult(
                         jobId, fileId, "result.pdf", MediaType.APPLICATION_PDF_VALUE);
                 log.debug("Stored byte[] result with fileId: {}", fileId);
-
-                // Let the byte array get collected naturally in the next GC cycle
-                // We don't need to force System.gc() which can be harmful
             } else if (result instanceof ResponseEntity) {
                 ResponseEntity<?> response = (ResponseEntity<?>) result;
                 Object body = response.getBody();
@@ -330,16 +243,13 @@ public class JobExecutorService {
                     taskManager.setFileResult(jobId, fileId, filename, contentType);
                     log.debug("Stored ResponseEntity<Resource> result with fileId: {}", fileId);
                 } else {
-                    // Check if the response body contains a fileId
                     if (body != null && body.toString().contains("fileId")) {
                         try {
-                            // Try to extract fileId using reflection
                             java.lang.reflect.Method getFileId =
                                     body.getClass().getMethod("getFileId");
                             String fileId = (String) getFileId.invoke(body);
 
                             if (fileId != null && !fileId.isEmpty()) {
-                                // Try to get filename and content type
                                 String filename = "result.pdf";
                                 String contentType = MediaType.APPLICATION_PDF_VALUE;
 
@@ -379,7 +289,6 @@ public class JobExecutorService {
                         }
                     }
 
-                    // Store generic result
                     taskManager.setResult(jobId, body);
                 }
             } else if (result instanceof MultipartFile file) {
@@ -388,16 +297,13 @@ public class JobExecutorService {
                         jobId, fileId, file.getOriginalFilename(), file.getContentType());
                 log.debug("Stored MultipartFile result with fileId: {}", fileId);
             } else {
-                // Check if result has a fileId field
                 if (result != null) {
                     try {
-                        // Try to extract fileId using reflection
                         java.lang.reflect.Method getFileId =
                                 result.getClass().getMethod("getFileId");
                         String fileId = (String) getFileId.invoke(result);
 
                         if (fileId != null && !fileId.isEmpty()) {
-                            // Try to get filename and content type
                             String filename = "result.pdf";
                             String contentType = MediaType.APPLICATION_PDF_VALUE;
 
@@ -435,7 +341,6 @@ public class JobExecutorService {
                     }
                 }
 
-                // Default case: store the result as is
                 taskManager.setResult(jobId, result);
             }
 
@@ -446,16 +351,8 @@ public class JobExecutorService {
         }
     }
 
-    /**
-     * Handle different result types for synchronous jobs
-     *
-     * @param result The result object
-     * @return The appropriate ResponseEntity
-     * @throws IOException If there is an error processing the result
-     */
     private ResponseEntity<?> handleResultForSyncJob(Object result) throws IOException {
         if (result instanceof byte[]) {
-            // Return byte array as PDF
             return ResponseEntity.ok()
                     .contentType(MediaType.APPLICATION_PDF)
                     .header(
@@ -463,7 +360,6 @@ public class JobExecutorService {
                             "form-data; name=\"attachment\"; filename=\"result.pdf\"")
                     .body(result);
         } else if (result instanceof MultipartFile file) {
-            // Return MultipartFile content
             return ResponseEntity.ok()
                     .contentType(MediaType.parseMediaType(file.getContentType()))
                     .header(
@@ -473,7 +369,6 @@ public class JobExecutorService {
                                     + "\"")
                     .body(file.getBytes());
         } else {
-            // Default case: return as JSON
             return ResponseEntity.ok(result);
         }
     }
@@ -493,15 +388,9 @@ public class JobExecutorService {
         return mediaType != null ? mediaType.toString() : MediaType.APPLICATION_PDF_VALUE;
     }
 
-    /**
-     * Parse session timeout string (e.g., "30m", "1h") to milliseconds
-     *
-     * @param timeout The timeout string
-     * @return The timeout in milliseconds
-     */
     private long parseSessionTimeout(String timeout) {
         if (timeout == null || timeout.isEmpty()) {
-            return 30 * 60 * 1000; // Default: 30 minutes
+            return 30 * 60 * 1000;
         }
 
         try {
@@ -523,27 +412,16 @@ public class JobExecutorService {
                 case "m" -> (long) (numericValue * 60 * 1000);
                 case "h" -> (long) (numericValue * 60 * 60 * 1000);
                 case "d" -> (long) (numericValue * 24 * 60 * 60 * 1000);
-                default -> (long) (numericValue * 60 * 1000); // Default to minutes
+                default -> (long) (numericValue * 60 * 1000);
             };
         } catch (Exception e) {
             log.warn("Could not parse session timeout '{}', using default", timeout);
-            return 30 * 60 * 1000; // Default: 30 minutes
+            return 30 * 60 * 1000;
         }
     }
 
-    /**
-     * Execute a supplier with a timeout
-     *
-     * @param supplier The supplier to execute
-     * @param timeoutMs The timeout in milliseconds
-     * @return The result from the supplier
-     * @throws TimeoutException If the execution times out
-     * @throws Exception If the supplier throws an exception
-     */
     private <T> T executeWithTimeout(Supplier<T> supplier, long timeoutMs)
             throws TimeoutException, Exception {
-        // Use the same executor as other async jobs for consistency
-        // This ensures all operations run on the same thread pool
         String currentJobId = stirling.software.common.util.JobContext.getJobId();
 
         java.util.concurrent.CompletableFuture<T> future =
@@ -577,17 +455,10 @@ public class JobExecutorService {
         }
     }
 
-    /**
-     * Get a scoped job key that includes user ownership when security is enabled.
-     *
-     * @param baseJobId the base job identifier
-     * @return scoped job key, or just baseJobId if no ownership service available
-     */
     private String getScopedJobKey(String baseJobId) {
         if (jobOwnershipService != null) {
             return jobOwnershipService.createScopedJobKey(baseJobId);
         }
-        // Security disabled, return unsecured job key
         return baseJobId;
     }
 }

app/common/src/test/java/stirling/software/common/cluster/inprocess/InProcessDistributedLockTest.java

@@ -24,7 +24,9 @@ class InProcessDistributedLockTest {
     }
 
     @Test
-    void reentryFromSameThreadFails() {
+    void reentryFromSameThreadFails_parityWithValkey() {
+        // The Valkey impl refuses reentry (SET NX semantics); the in-process impl must match,
+        // otherwise code working in single-instance silently breaks in cluster mode.
         DistributedLock lock = new InProcessDistributedLock();
         DistributedLock.LockHandle h1 = lock.tryAcquire("k", Duration.ofSeconds(30)).orElseThrow();
         Optional<DistributedLock.LockHandle> reentry = lock.tryAcquire("k", Duration.ofSeconds(30));

app/common/src/test/java/stirling/software/common/service/TaskManagerJobStoreDelegationTest.java

@@ -86,6 +86,8 @@ class TaskManagerJobStoreDelegationTest {
 
                     @Override
                     public boolean shouldRunLocalCleanup() {
+                        // Distributed backplanes own job TTL eviction themselves; this mock
+                        // mirrors the real ValkeyClusterBackplane override of the default true.
                         return false;
                     }
                 };

app/core/src/main/java/stirling/software/common/controller/JobController.java

@@ -4,6 +4,7 @@ import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.MediaType;
@@ -22,6 +23,10 @@ import jakarta.servlet.http.HttpServletRequest;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.StickyMissRecorder;
 import stirling.software.common.model.job.JobResult;
 import stirling.software.common.model.job.ResultFile;
 import stirling.software.common.service.FileStorage;
@@ -30,7 +35,6 @@ import stirling.software.common.service.JobQueue;
 import stirling.software.common.service.TaskManager;
 import stirling.software.common.util.RegexPatternUtils;
 
-/** REST controller for job-related endpoints */
 @RestController
 @RequiredArgsConstructor
 @Slf4j
@@ -42,20 +46,29 @@ public class JobController {
     private final FileStorage fileStorage;
     private final JobQueue jobQueue;
     private final HttpServletRequest request;
+    private final ClusterBackplane clusterBackplane;
+    private final JobStore jobStore;
+
+    // Short-TTL local cache fronting JobStore.get() on the sticky-410 path to avoid a Valkey
+    // HGETALL round-trip on every download retry for the same job.
+    private final JobOwnershipCache ownershipCache = new JobOwnershipCache();
 
     @Autowired(required = false)
     private JobOwnershipService jobOwnershipService;
 
-    /**
+    @Autowired(required = false)
-     * Get the status of a job
+    private StickyMissRecorder stickyMissRecorder;
-     *
+
-     * @param jobId The job ID
-     * @return The job result
-     */
     @GetMapping("/job/{jobId}")
     @Operation(summary = "Get job status")
     public ResponseEntity<?> getJobStatus(@PathVariable("jobId") String jobId) {
-        // Validate job ownership
+        // Sticky-410 must run before user-auth: a 403 here would leak job existence and defeat
+        // LB re-routing. The owner node is where the real auth check should happen.
+        Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
+        if (peerOwned.isPresent()) {
+            return peerOwned.get();
+        }
+
         if (!validateJobAccess(jobId)) {
             log.warn("Unauthorized attempt to access job status: {}", jobId);
             return ResponseEntity.status(403)
@@ -67,7 +80,6 @@ public class JobController {
             return ResponseEntity.notFound().build();
         }
 
-        // Check if the job is in the queue and add queue information
         if (!result.isComplete() && jobQueue.isJobQueued(jobId)) {
             int position = jobQueue.getJobPosition(jobId);
             Map<String, Object> resultWithQueueInfo =
@@ -82,16 +94,14 @@ public class JobController {
         return ResponseEntity.ok(result);
     }
 
-    /**
-     * Get the result of a job
-     *
-     * @param jobId The job ID
-     * @return The job result
-     */
     @GetMapping("/job/{jobId}/result")
     @Operation(summary = "Get job result")
     public ResponseEntity<?> getJobResult(@PathVariable("jobId") String jobId) {
-        // Validate job ownership
+        Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
+        if (peerOwned.isPresent()) {
+            return peerOwned.get();
+        }
+
         if (!validateJobAccess(jobId)) {
             log.warn("Unauthorized attempt to access job result: {}", jobId);
             return ResponseEntity.status(403)
@@ -111,7 +121,6 @@ public class JobController {
             return ResponseEntity.badRequest().body("Job failed: " + result.getError());
         }
 
-        // Handle multiple files - return metadata for client to download individually
         if (result.hasMultipleFiles()) {
             return ResponseEntity.ok()
                     .contentType(MediaType.APPLICATION_JSON)
@@ -125,11 +134,11 @@ public class JobController {
                                     result.getAllResultFiles()));
         }
 
-        // Handle single file (download directly)
         if (result.hasFiles() && !result.hasMultipleFiles()) {
             try {
                 List<ResultFile> files = result.getAllResultFiles();
                 ResultFile singleFile = files.get(0);
+
                 byte[] fileContent = fileStorage.retrieveBytes(singleFile.getFileId());
                 return ResponseEntity.ok()
                         .header("Content-Type", singleFile.getContentType())
@@ -147,30 +156,22 @@ public class JobController {
         return ResponseEntity.ok(result.getResult());
     }
 
-    // Admin-only endpoints have been moved to AdminJobController in the proprietary package
-
-    /**
-     * Cancel a job by its ID
-     *
-     * <p>This method should only allow cancellation of jobs that were created by the current user.
-     * The jobId should be part of the user's session or otherwise linked to their identity.
-     *
-     * @param jobId The job ID
-     * @return Response indicating whether the job was cancelled
-     */
     @DeleteMapping("/job/{jobId}")
     @Operation(summary = "Cancel a job")
     public ResponseEntity<?> cancelJob(@PathVariable("jobId") String jobId) {
         log.debug("Request to cancel job: {}", jobId);
 
-        // Validate job ownership
+        Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
+        if (peerOwned.isPresent()) {
+            return peerOwned.get();
+        }
+
         if (!validateJobAccess(jobId)) {
             log.warn("Unauthorized attempt to cancel job: {}", jobId);
             return ResponseEntity.status(403)
                     .body(Map.of("message", "You are not authorized to cancel this job"));
         }
 
-        // First check if the job is in the queue
         boolean cancelled = false;
         int queuePosition = -1;
 
@@ -180,11 +181,9 @@ public class JobController {
             log.info("Cancelled queued job: {} (was at position {})", jobId, queuePosition);
         }
 
-        // If not in queue or couldn't cancel, try to cancel in TaskManager
         if (!cancelled) {
             JobResult result = taskManager.getJobResult(jobId);
             if (result != null && !result.isComplete()) {
-                // Mark as error with cancellation message
                 taskManager.setError(jobId, "Job was cancelled by user");
                 cancelled = true;
                 log.info("Marked job as cancelled in TaskManager: {}", jobId);
@@ -201,7 +200,6 @@ public class JobController {
                             "queuePosition",
                             queuePosition >= 0 ? queuePosition : "n/a"));
         } else {
-            // Job not found or already complete
             JobResult result = taskManager.getJobResult(jobId);
             if (result == null) {
                 return ResponseEntity.notFound().build();
@@ -215,16 +213,14 @@ public class JobController {
         }
     }
 
-    /**
-     * Get the list of files for a job
-     *
-     * @param jobId The job ID
-     * @return List of files for the job
-     */
     @GetMapping("/job/{jobId}/result/files")
     @Operation(summary = "Get job result files")
     public ResponseEntity<?> getJobFiles(@PathVariable("jobId") String jobId) {
-        // Validate job ownership
+        Optional<ResponseEntity<?>> peerOwned = guardNonOwner(jobId);
+        if (peerOwned.isPresent()) {
+            return peerOwned.get();
+        }
+
         if (!validateJobAccess(jobId)) {
             log.warn("Unauthorized attempt to access job files: {}", jobId);
             return ResponseEntity.status(403)
@@ -252,28 +248,31 @@ public class JobController {
                         "files", files));
     }
 
-    /**
-     * Get metadata for an individual file by its file ID
-     *
-     * @param fileId The file ID
-     * @return The file metadata
-     */
     @GetMapping("/files/{fileId}/metadata")
     @Operation(summary = "Get file metadata")
     public ResponseEntity<?> getFileMetadata(@PathVariable("fileId") String fileId) {
         try {
-            String jobKey = taskManager.findJobKeyByFileId(fileId);
+            String jobKey;
+            try {
+                jobKey = taskManager.findJobKeyByFileId(fileId);
+            } catch (RuntimeException backplaneEx) {
+                return backplaneUnavailable(fileId, backplaneEx);
+            }
             if (jobKey == null) {
                 return ResponseEntity.notFound().build();
             }
 
+            Optional<ResponseEntity<?>> notOwner = guardNonOwner(jobKey);
+            if (notOwner.isPresent()) {
+                return notOwner.get();
+            }
+
             if (!validateJobAccess(jobKey)) {
                 log.warn("Unauthorized attempt to access file metadata: {}", fileId);
                 return ResponseEntity.status(403)
                         .body(Map.of("message", "You are not authorized to access this file"));
             }
 
-            // Find the file metadata from any job that contains this file
             ResultFile resultFile = taskManager.findResultFileByFileId(fileId);
 
             if (resultFile != null) {
@@ -281,12 +280,10 @@ public class JobController {
             }
 
             if (!isSecurityEnabled()) {
-                // Backwards compatibility when ownership service is unavailable
                 if (!fileStorage.fileExists(fileId)) {
                     return ResponseEntity.notFound().build();
                 }
 
-                // File exists but no metadata found, get basic info efficiently
                 long fileSize = fileStorage.getFileSize(fileId);
                 return ResponseEntity.ok(
                         Map.of(
@@ -308,32 +305,31 @@ public class JobController {
         }
     }
 
-    /**
-     * Download an individual file by its file ID
-     *
-     * @param fileId The file ID
-     * @return The file content
-     */
     @GetMapping("/files/{fileId}")
     @Operation(summary = "Download a file")
     public ResponseEntity<?> downloadFile(@PathVariable("fileId") String fileId) {
         try {
-            String jobKey = taskManager.findJobKeyByFileId(fileId);
+            String jobKey;
+            try {
+                jobKey = taskManager.findJobKeyByFileId(fileId);
+            } catch (RuntimeException backplaneEx) {
+                return backplaneUnavailable(fileId, backplaneEx);
+            }
             if (jobKey == null) {
                 return ResponseEntity.notFound().build();
             }
 
+            Optional<ResponseEntity<?>> notOwner = guardNonOwner(jobKey);
+            if (notOwner.isPresent()) {
+                return notOwner.get();
+            }
+
             if (!validateJobAccess(jobKey)) {
                 log.warn("Unauthorized attempt to download file: {}", fileId);
                 return ResponseEntity.status(403)
                         .body(Map.of("message", "You are not authorized to access this file"));
             }
 
-            // Retrieve file content
-            byte[] fileContent = fileStorage.retrieveBytes(fileId);
-
-            // Find the file metadata from any job that contains this file
-            // This is for getting the original filename and content type
             ResultFile resultFile = taskManager.findResultFileByFileId(fileId);
 
             String fileName = resultFile != null ? resultFile.getFileName() : "download";
@@ -342,6 +338,8 @@ public class JobController {
                             ? resultFile.getContentType()
                             : MediaType.APPLICATION_OCTET_STREAM_VALUE;
 
+            byte[] fileContent = fileStorage.retrieveBytes(fileId);
+
             return ResponseEntity.ok()
                     .header("Content-Type", contentType)
                     .header("Content-Disposition", createContentDispositionHeader(fileName))
@@ -357,11 +355,88 @@ public class JobController {
     }
 
     /**
-     * Create Content-Disposition header with UTF-8 filename support
+     * Returns 410 Gone when the job is owned by a peer node, empty otherwise. Uses a short-TTL
-     *
+     * local cache to avoid repeated Valkey lookups on the hot download path. When the backplane is
-     * @param fileName The filename to encode
+     * unreachable, a locally-held job is still served and anything else gets a retryable 503.
-     * @return Content-Disposition header value
      */
+    private Optional<ResponseEntity<?>> guardNonOwner(String jobId) {
+        if (clusterBackplane == null || jobStore == null) {
+            return Optional.empty();
+        }
+        Optional<JobStoreEntry> entry;
+        Optional<Optional<JobStoreEntry>> cached = ownershipCache.get(jobId);
+        if (cached.isPresent()) {
+            entry = cached.get();
+        } else {
+            try {
+                entry = jobStore.get(jobId);
+            } catch (RuntimeException ex) {
+                // Backplane unreachable: if we hold the job locally serve it, otherwise return a
+                // retryable 503 (same contract as the file endpoints) instead of a misleading 404.
+                if (taskManager.getJobResult(jobId) == null) {
+                    return Optional.of(backplaneUnavailable(jobId, ex));
+                }
+                log.warn(
+                        "JobStore lookup failed for jobId={}; serving locally-held job: {}",
+                        jobId,
+                        ex.getMessage());
+                return Optional.empty();
+            }
+            ownershipCache.put(jobId, entry);
+        }
+        if (entry.isEmpty()) {
+            return Optional.empty();
+        }
+        String owner = entry.get().owningNodeId();
+        if (owner == null || owner.isBlank()) {
+            return Optional.empty();
+        }
+        String localId = clusterBackplane.localNodeId();
+        if (owner.equals(localId)) {
+            return Optional.empty();
+        }
+        log.info(
+                "Sticky-session miss for jobId={} (owner={}, local={}); returning 410 so client"
+                        + " retries via LB affinity",
+                jobId,
+                owner,
+                localId);
+        if (stickyMissRecorder != null) {
+            stickyMissRecorder.recordStickyMiss();
+        }
+        return Optional.of(
+                ResponseEntity.status(410)
+                        .header("Retry-After", "0")
+                        .body(
+                                Map.of(
+                                        "message",
+                                        "Result lives on another node. Retry to be routed there"
+                                                + " by the load balancer's sticky-session"
+                                                + " affinity, or re-run the job.",
+                                        "ownedBy",
+                                        owner,
+                                        "currentNode",
+                                        localId == null ? "" : localId)));
+    }
+
+    /**
+     * When the backplane is unreachable we cannot resolve ownership or existence, and serving
+     * without that check would be unsafe - so return a retryable 503 (consistent with the
+     * sticky-410 retry model) rather than a misleading 404 or a generic 500.
+     */
+    private ResponseEntity<?> backplaneUnavailable(String id, RuntimeException ex) {
+        log.warn(
+                "Backplane lookup failed for {}; returning 503 (retryable): {}",
+                id,
+                ex.getMessage());
+        return ResponseEntity.status(503)
+                .header("Retry-After", "1")
+                .body(
+                        Map.of(
+                                "message",
+                                "Cluster backplane temporarily unavailable; retry shortly."));
+    }
+
     private String createContentDispositionHeader(String fileName) {
         try {
             String encodedFileName =
@@ -371,19 +446,11 @@ public class JobController {
                             .replaceAll("%20"); // URLEncoder uses + for spaces, but we want %20
             return "attachment; filename=\"" + fileName + "\"; filename*=UTF-8''" + encodedFileName;
         } catch (Exception e) {
-            // Fallback to basic filename if encoding fails
             return "attachment; filename=\"" + fileName + "\"";
         }
     }
 
-    /**
-     * Validate that the current user has access to the given job.
-     *
-     * @param jobId the job identifier to validate
-     * @return true if user has access, false otherwise
-     */
     private boolean validateJobAccess(String jobId) {
-        // If JobOwnershipService is available (security enabled), use it
         if (jobOwnershipService != null) {
             try {
                 return jobOwnershipService.validateJobAccess(jobId);
@@ -393,8 +460,6 @@ public class JobController {
             }
         }
 
-        // Security disabled - allow all access (backwards compatibility)
-        // When security is not enabled, any user can access any job by jobId
         return true;
     }
 }

app/core/src/main/java/stirling/software/common/controller/JobOwnershipCache.java

@@ -0,0 +1,47 @@
+package stirling.software.common.controller;
+
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import stirling.software.common.cluster.JobStoreEntry;
+
+/**
+ * Process-local TTL cache for {@link JobStoreEntry} lookups to suppress redundant Valkey HGETALL
+ * round-trips on the hot result-download path (sticky-410 ownership check).
+ *
+ * <p>5 second TTL is short enough that a job's lifecycle transitions (RUNNING -> COMPLETE -> TTL
+ * expiry) propagate to all nodes within the LB's sticky-session window, and short enough that a
+ * mistakenly-cached "not found" recovers quickly when an entry actually shows up. Cap the map at
+ * 2048 entries to bound memory; eviction is best-effort (clear-and-restart) since the cache is
+ * advisory.
+ */
+final class JobOwnershipCache {
+
+    private static final long TTL_NANOS = 5L * 1_000_000_000L; // 5 s
+    private static final int MAX_ENTRIES = 2048;
+
+    private final ConcurrentMap<String, Entry> entries = new ConcurrentHashMap<>();
+
+    Optional<Optional<JobStoreEntry>> get(String jobId) {
+        Entry e = entries.get(jobId);
+        if (e == null) {
+            return Optional.empty();
+        }
+        if (System.nanoTime() - e.storedAtNanos > TTL_NANOS) {
+            entries.remove(jobId, e);
+            return Optional.empty();
+        }
+        return Optional.of(e.value);
+    }
+
+    void put(String jobId, Optional<JobStoreEntry> value) {
+        if (entries.size() >= MAX_ENTRIES) {
+            // Best-effort eviction; under burst the cache simply rebuilds.
+            entries.clear();
+        }
+        entries.put(jobId, new Entry(value, System.nanoTime()));
+    }
+
+    private record Entry(Optional<JobStoreEntry> value, long storedAtNanos) {}
+}

app/core/src/test/java/stirling/software/common/controller/JobControllerOwnershipTest.java

@@ -0,0 +1,479 @@
+package stirling.software.common.controller;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import jakarta.servlet.http.HttpServletRequest;
+
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.StickyMissRecorder;
+import stirling.software.common.model.job.JobResult;
+import stirling.software.common.service.FileStorage;
+import stirling.software.common.service.JobOwnershipService;
+import stirling.software.common.service.JobQueue;
+import stirling.software.common.service.TaskManager;
+
+/**
+ * Sticky-410 ownership contract for {@link JobController}: peer-owned jobs return 410 Gone with
+ * ownedBy/currentNode fields; locally-owned and no-entry cases return 200. FileStorage is never
+ * touched on the 410 path. Manual mock construction so tests can vary backplane/jobstore combos.
+ */
+class JobControllerOwnershipTest {
+
+    private TaskManager taskManager;
+    private FileStorage fileStorage;
+    private JobQueue jobQueue;
+    private HttpServletRequest request;
+    private JobOwnershipService jobOwnershipService;
+    private ClusterBackplane clusterBackplane;
+    private JobStore jobStore;
+    private StickyMissRecorder stickyMissRecorder;
+
+    private static final String JOB_ID = "job-42";
+    private static final String FILE_ID = "file-abc";
+    private static final String LOCAL_NODE = "node-self";
+    private static final String PEER_NODE = "node-peer";
+
+    @BeforeEach
+    void setUp() {
+        taskManager = mock(TaskManager.class);
+        fileStorage = mock(FileStorage.class);
+        jobQueue = mock(JobQueue.class);
+        request = mock(HttpServletRequest.class);
+        jobOwnershipService = mock(JobOwnershipService.class);
+        clusterBackplane = mock(ClusterBackplane.class);
+        jobStore = mock(JobStore.class);
+        stickyMissRecorder = mock(StickyMissRecorder.class);
+    }
+
+    private JobController makeController(ClusterBackplane backplane, JobStore store) {
+        JobController c =
+                new JobController(taskManager, fileStorage, jobQueue, request, backplane, store);
+        ReflectionTestUtils.setField(c, "stickyMissRecorder", stickyMissRecorder);
+        return c;
+    }
+
+    private JobController makeController() {
+        return makeController(clusterBackplane, jobStore);
+    }
+
+    private JobStoreEntry entryOwnedBy(String ownerNodeId) {
+        return new JobStoreEntry(
+                JOB_ID,
+                JobStoreEntry.JobState.COMPLETE,
+                ownerNodeId,
+                Instant.now(),
+                Instant.now(),
+                null,
+                List.of(FILE_ID),
+                Map.of());
+    }
+
+    private JobResult completedJobWithFile() {
+        JobResult result = new JobResult();
+        result.setJobId(JOB_ID);
+        // completeWithSingleFile populates the resultFiles list, sets complete=true,
+        // and sets completedAt - all required for the getJobResult single-file branch.
+        result.completeWithSingleFile(FILE_ID, "out.pdf", "application/pdf", 7L);
+        return result;
+    }
+
+    @Test
+    @DisplayName(
+            "downloadFile peer-owned → full sticky-410 contract"
+                    + " (status + Retry-After + payload + metric + storage untouched)")
+    void downloadFile_peerOwned_fullStickyContract() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+
+        ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        assertEquals("0", response.getHeaders().getFirst("Retry-After"));
+
+        assertInstanceOf(Map.class, response.getBody());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals(3, body.size(), "exactly: message, ownedBy, currentNode");
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+        assertEquals(LOCAL_NODE, body.get("currentNode"));
+        assertNotNull(body.get("message"));
+        assertTrue(((String) body.get("message")).toLowerCase().contains("retry"));
+        assertNull(body.get("internalSecret"));
+        assertNull(body.get("filePath"));
+        verify(stickyMissRecorder).recordStickyMiss();
+        verify(fileStorage, never()).retrieveBytes(FILE_ID);
+    }
+
+    private static Stream<Arguments> downloadHappyPathScenarios() {
+        return Stream.of(
+                Arguments.of("locallyOwned", LOCAL_NODE, true),
+                Arguments.of("noJobStoreEntry", null, false),
+                Arguments.of("blankOwningNodeId", "", true));
+    }
+
+    @ParameterizedTest(name = "downloadFile {0} -> 200, no sticky-miss")
+    @MethodSource("downloadHappyPathScenarios")
+    void downloadFile_happyPath_returnsOkAndNoMetric(
+            String scenario, String ownerNodeId, boolean entryPresent) throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID))
+                .thenReturn(
+                        entryPresent ? Optional.of(entryOwnedBy(ownerNodeId)) : Optional.empty());
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode(), scenario);
+        verify(fileStorage).retrieveBytes(FILE_ID);
+        verify(stickyMissRecorder, never()).recordStickyMiss();
+    }
+
+    @Test
+    @DisplayName("getJobResult: locally-owned single-file result → reads from FileStorage, 200 OK")
+    void getJobResult_singleFile_locallyOwned_readsFromStorage() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        ResponseEntity<?> response = makeController().getJobResult(JOB_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+    }
+
+    private enum Endpoint {
+        DOWNLOAD_FILE,
+        GET_JOB_RESULT,
+        GET_JOB_STATUS,
+        GET_JOB_FILES,
+        GET_FILE_METADATA,
+        CANCEL_JOB
+    }
+
+    private static Stream<Arguments> peerOwned410Scenarios() {
+        return Stream.of(
+                Arguments.of(Endpoint.DOWNLOAD_FILE),
+                Arguments.of(Endpoint.GET_JOB_RESULT),
+                Arguments.of(Endpoint.GET_JOB_STATUS),
+                Arguments.of(Endpoint.GET_JOB_FILES),
+                Arguments.of(Endpoint.GET_FILE_METADATA),
+                Arguments.of(Endpoint.CANCEL_JOB));
+    }
+
+    @ParameterizedTest(name = "{0} peer-owned -> 410, ownedBy=peer, metric++")
+    @MethodSource("peerOwned410Scenarios")
+    void endpoint_peerOwned_returns410(Endpoint endpoint) throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+        switch (endpoint) {
+            case DOWNLOAD_FILE, GET_FILE_METADATA ->
+                    when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+            case GET_JOB_RESULT ->
+                    when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
+            case GET_JOB_STATUS, GET_JOB_FILES ->
+                    when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
+            case CANCEL_JOB -> {
+                when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
+                when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
+            }
+        }
+
+        ResponseEntity<?> response =
+                switch (endpoint) {
+                    case DOWNLOAD_FILE -> makeController().downloadFile(FILE_ID);
+                    case GET_JOB_RESULT -> makeController().getJobResult(JOB_ID);
+                    case GET_JOB_STATUS -> makeController().getJobStatus(JOB_ID);
+                    case GET_JOB_FILES -> makeController().getJobFiles(JOB_ID);
+                    case GET_FILE_METADATA -> makeController().getFileMetadata(FILE_ID);
+                    case CANCEL_JOB -> makeController().cancelJob(JOB_ID);
+                };
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+        assertEquals(LOCAL_NODE, body.get("currentNode"));
+        verify(stickyMissRecorder).recordStickyMiss();
+        verify(fileStorage, never()).retrieveBytes(FILE_ID);
+        if (endpoint == Endpoint.CANCEL_JOB) {
+            verify(taskManager, never()).setError(JOB_ID, "Job was cancelled by user");
+        }
+    }
+
+    private static Stream<Arguments> unknownJob404Scenarios() {
+        return Stream.of(Arguments.of(Endpoint.GET_JOB_STATUS), Arguments.of(Endpoint.CANCEL_JOB));
+    }
+
+    @ParameterizedTest(name = "{0} unknown jobId -> 404 (not 410), no metric")
+    @MethodSource("unknownJob404Scenarios")
+    void endpoint_unknownJob_returns404(Endpoint endpoint) {
+        when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.empty());
+        if (endpoint == Endpoint.CANCEL_JOB) {
+            when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
+        }
+
+        ResponseEntity<?> response =
+                switch (endpoint) {
+                    case GET_JOB_STATUS -> makeController().getJobStatus(JOB_ID);
+                    case CANCEL_JOB -> makeController().cancelJob(JOB_ID);
+                    default -> throw new IllegalArgumentException(endpoint.name());
+                };
+
+        assertEquals(HttpStatus.NOT_FOUND, response.getStatusCode());
+        verify(stickyMissRecorder, never()).recordStickyMiss();
+    }
+
+    @Test
+    @DisplayName("Single-instance install (no ClusterBackplane bean): no 410, no NPE")
+    void singleInstance_noClusterBackplane_noGoneResponse() throws Exception {
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        ResponseEntity<?> response = makeController(null, jobStore).downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+        verify(fileStorage).retrieveBytes(FILE_ID);
+    }
+
+    @Test
+    @DisplayName("Single-instance install (no JobStore bean): no 410, no NPE")
+    void singleInstance_noJobStore_noGoneResponse() throws Exception {
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        ResponseEntity<?> response = makeController(clusterBackplane, null).downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+        verify(fileStorage).retrieveBytes(FILE_ID);
+    }
+
+    @Test
+    @DisplayName("Single-instance (no StickyMissRecorder bean) → no NPE, still 200 OK")
+    void noStickyMissRecorder_works() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        JobController c = makeController();
+        ReflectionTestUtils.setField(c, "stickyMissRecorder", null);
+
+        ResponseEntity<?> response = c.downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+    }
+
+    @Test
+    @DisplayName(
+            "cluster-mode but localNodeId is null → no NPE; 410 because owner is set and"
+                    + " differs from blank")
+    void clusterBackplanePresent_butLocalNodeIdNull_falsBackGracefully() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(null);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+
+        // We still 410: owner is "node-peer", local is null → they don't match. Rather than
+        // silently 200-from-wrong-disk (which would serve garbage), we surface the mismatch.
+        ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals("", body.get("currentNode"), "blank when localNodeId is null");
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+    }
+
+    @Test
+    @DisplayName("Owner returns 410 even when JobOwnershipService allows access (orthogonal)")
+    void ownershipService_passes_butStickyStillReturns410() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+        lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(true);
+
+        JobController c = makeController();
+        ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
+        ResponseEntity<?> response = c.downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+    }
+
+    @Test
+    @DisplayName(
+            "downloadFile: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak"
+                    + " file existence")
+    void downloadFile_peerOwned_ownershipDenied_returns410NotForbidden() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+        lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
+
+        JobController c = makeController();
+        ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
+        ResponseEntity<?> response = c.downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+        verify(fileStorage, never()).retrieveBytes(FILE_ID);
+    }
+
+    @Test
+    @DisplayName(
+            "getJobStatus: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak"
+                    + " job existence")
+    void getJobStatus_peerOwned_ownershipDenied_returns410NotForbidden() {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+        lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
+
+        JobController c = makeController();
+        ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
+        ResponseEntity<?> response = c.getJobStatus(JOB_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+    }
+
+    @Test
+    @DisplayName(
+            "cancelJob: peer-owned + ownership-denied → 410 (NOT 403) so we don't leak job"
+                    + " existence")
+    void cancelJob_peerOwned_ownershipDenied_returns410NotForbidden() {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(jobQueue.isJobQueued(JOB_ID)).thenReturn(false);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(PEER_NODE)));
+        lenient().when(jobOwnershipService.validateJobAccess(JOB_ID)).thenReturn(false);
+
+        JobController c = makeController();
+        ReflectionTestUtils.setField(c, "jobOwnershipService", jobOwnershipService);
+        ResponseEntity<?> response = c.cancelJob(JOB_ID);
+
+        assertEquals(HttpStatus.GONE, response.getStatusCode());
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertEquals(PEER_NODE, body.get("ownedBy"));
+        verify(taskManager, never()).setError(JOB_ID, "Job was cancelled by user");
+    }
+
+    @Test
+    @DisplayName(
+            "guardNonOwner caches JobStore.get within TTL window: second call same jobId hits"
+                    + " cache, not Valkey")
+    void guardNonOwner_cachesJobStoreLookupWithinTtl() throws Exception {
+        when(clusterBackplane.localNodeId()).thenReturn(LOCAL_NODE);
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenReturn(Optional.of(entryOwnedBy(LOCAL_NODE)));
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        JobController c = makeController();
+        c.downloadFile(FILE_ID);
+        c.downloadFile(FILE_ID);
+        c.downloadFile(FILE_ID);
+
+        verify(jobStore, times(1)).get(JOB_ID);
+    }
+
+    @Test
+    @DisplayName(
+            "guardNonOwner: JobStore.get throws (Valkey timeout) → falls through to local-disk"
+                    + " path, no 500 leaks to caller")
+    void guardNonOwner_jobStoreException_fallsThroughToLocalPath() throws Exception {
+        when(taskManager.findJobKeyByFileId(FILE_ID)).thenReturn(JOB_ID);
+        when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout"));
+        when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
+        when(fileStorage.retrieveBytes(FILE_ID)).thenReturn("payload".getBytes());
+
+        ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+        verify(fileStorage).retrieveBytes(FILE_ID);
+        verify(stickyMissRecorder, never()).recordStickyMiss();
+    }
+
+    @Test
+    @DisplayName("backplane down + job NOT held locally → 503 retryable (not a misleading 404)")
+    void jobEndpoint_backplaneDown_notLocal_returns503() {
+        when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout"));
+        when(taskManager.getJobResult(JOB_ID)).thenReturn(null);
+
+        ResponseEntity<?> response = makeController().getJobStatus(JOB_ID);
+
+        assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
+        assertEquals("1", response.getHeaders().getFirst("Retry-After"));
+        verify(stickyMissRecorder, never()).recordStickyMiss();
+    }
+
+    @Test
+    @DisplayName("backplane down but job held locally → owner still serves (not 503)")
+    void jobEndpoint_backplaneDown_local_servesLocally() {
+        when(jobStore.get(JOB_ID)).thenThrow(new RuntimeException("Valkey command timeout"));
+        when(taskManager.getJobResult(JOB_ID)).thenReturn(completedJobWithFile());
+
+        ResponseEntity<?> response = makeController().getJobStatus(JOB_ID);
+
+        assertEquals(HttpStatus.OK, response.getStatusCode());
+    }
+
+    @Test
+    @DisplayName(
+            "downloadFile: findJobKeyByFileId throws (backplane down) → 503 + Retry-After,"
+                    + " not 404/500, storage untouched")
+    void downloadFile_findJobKeyThrows_returns503Retryable() throws Exception {
+        when(taskManager.findJobKeyByFileId(FILE_ID))
+                .thenThrow(new RuntimeException("Valkey command timeout"));
+
+        ResponseEntity<?> response = makeController().downloadFile(FILE_ID);
+
+        assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
+        assertEquals("1", response.getHeaders().getFirst("Retry-After"));
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertTrue(((String) body.get("message")).toLowerCase().contains("unavailable"));
+        verify(fileStorage, never()).retrieveBytes(FILE_ID);
+    }
+
+    @Test
+    @DisplayName(
+            "getFileMetadata: findJobKeyByFileId throws (backplane down) → 503 + Retry-After,"
+                    + " not 404/500")
+    void getFileMetadata_findJobKeyThrows_returns503Retryable() throws Exception {
+        when(taskManager.findJobKeyByFileId(FILE_ID))
+                .thenThrow(new RuntimeException("Valkey command timeout"));
+
+        ResponseEntity<?> response = makeController().getFileMetadata(FILE_ID);
+
+        assertEquals(HttpStatus.SERVICE_UNAVAILABLE, response.getStatusCode());
+        assertEquals("1", response.getHeaders().getFirst("Retry-After"));
+        Map<?, ?> body = (Map<?, ?>) response.getBody();
+        assertTrue(((String) body.get("message")).toLowerCase().contains("unavailable"));
+        verify(fileStorage, never()).retrieveBytes(FILE_ID);
+    }
+}

app/core/src/test/java/stirling/software/common/controller/JobControllerTest.java

@@ -19,6 +19,8 @@ import org.springframework.test.util.ReflectionTestUtils;
 
 import jakarta.servlet.http.HttpServletRequest;
 
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.JobStore;
 import stirling.software.common.model.job.JobResult;
 import stirling.software.common.service.FileStorage;
 import stirling.software.common.service.JobOwnershipService;
@@ -37,6 +39,10 @@ class JobControllerTest {
 
     @Mock private JobOwnershipService jobOwnershipService;
 
+    @Mock private ClusterBackplane clusterBackplane;
+
+    @Mock private JobStore jobStore;
+
     private MockHttpSession session;
 
     @InjectMocks private JobController controller;

app/proprietary/build.gradle

@@ -55,8 +55,13 @@ dependencies {
     api 'org.springframework.boot:spring-boot-starter-mail'
     api 'org.springframework.boot:spring-boot-starter-cache'
     api 'com.github.ben-manes.caffeine:caffeine'
+    implementation 'org.springframework.boot:spring-boot-starter-data-redis'
     api 'io.swagger.core.v3:swagger-core-jakarta:2.2.46'
-    implementation 'com.bucket4j:bucket4j_jdk17-core:8.18.0'
+    implementation 'com.bucket4j:bucket4j_jdk17-core:8.19.0'
+    // Lettuce-backed Bucket4j ProxyManager used by ValkeyRateLimitStore for cluster-wide
+    // token-bucket rate limiting (parity with in-process Bucket4j semantics; no fixed-window
+    // boundary doubling).
+    implementation 'com.bucket4j:bucket4j_jdk17-lettuce:8.19.0'
 
     // https://mvnrepository.com/artifact/com.bucket4j/bucket4j_jdk17
     implementation "org.bouncycastle:bcprov-jdk18on:$bouncycastleVersion"
@@ -77,9 +82,12 @@ dependencies {
     implementation "software.amazon.awssdk:s3:$awsSdkVersion"
     implementation "software.amazon.awssdk:url-connection-client:$awsSdkVersion"
 
+    // Testcontainers: real MinIO/LocalStack (S3) and Valkey for integration tests in CI without
+    // manually-started instances. Tests skip cleanly when Docker is unavailable.
+    testImplementation "org.testcontainers:testcontainers:$testcontainersMinioVersion"
     testImplementation "org.testcontainers:minio:$testcontainersMinioVersion"
-    testImplementation "org.testcontainers:junit-jupiter:$testcontainersMinioVersion"
     testImplementation "org.testcontainers:localstack:$testcontainersMinioVersion"
+    testImplementation "org.testcontainers:junit-jupiter:$testcontainersMinioVersion"
 }
 
 tasks.register('prepareKotlinBuildScriptModel') {}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterLicenseGate.java

@@ -0,0 +1,40 @@
+package stirling.software.proprietary.cluster;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Configuration;
+
+import jakarta.annotation.PostConstruct;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Runtime license gate for cluster mode. Cluster mode requires a SERVER or ENTERPRISE license; the
+ * SaaS flavor bypasses (no {@code runningProOrHigher} bean is published). The Valkey connection
+ * config {@code @DependsOn} this bean, so it runs before any Valkey bean is constructed.
+ */
+@Configuration
+@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
+@Slf4j
+public class ClusterLicenseGate {
+
+    @Autowired(required = false)
+    @Qualifier("runningProOrHigher")
+    private Boolean runningProOrHigher;
+
+    @PostConstruct
+    void verifyLicense() {
+        if (runningProOrHigher == null) {
+            return; // saas flavor - licensed via Stripe elsewhere
+        }
+        if (!runningProOrHigher) {
+            throw new IllegalStateException(
+                    "Cluster mode (cluster.enabled=true) requires a SERVER or"
+                            + " ENTERPRISE license. Configure stirling.premium.key with a valid"
+                            + " license key (contact [email protected] to obtain one), or set"
+                            + " cluster.enabled=false.");
+        }
+        log.info("Cluster license gate: SERVER/ENTERPRISE license verified, cluster mode allowed.");
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterMetrics.java

@@ -0,0 +1,113 @@
+package stirling.software.proprietary.cluster;
+
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Timer;
+
+import stirling.software.common.cluster.StickyMissRecorder;
+import stirling.software.common.model.ApplicationProperties;
+
+/**
+ * Cluster operation metrics exposed via {@code /actuator/prometheus}. Registered only when cluster
+ * mode is on.
+ */
+@Component
+@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
+public class ClusterMetrics implements StickyMissRecorder {
+
+    private final MeterRegistry registry;
+    private final ApplicationProperties applicationProperties;
+
+    private final Counter stickyMissTotal;
+    private final Counter rateLimitRejected;
+    private final Timer backplaneLatency;
+    private final Timer jobWaitSeconds;
+
+    // Per-lane queue depth gauges. Lanes are a fixed enum (FAST, SLOW, AI), so we register all
+    // three eagerly so dashboards never have a missing series.
+    private static final List<String> KNOWN_LANES = List.of("FAST", "SLOW", "AI");
+    private final ConcurrentHashMap<String, AtomicLong> queueDepth = new ConcurrentHashMap<>();
+
+    private final AtomicLong jobsInflight = new AtomicLong();
+
+    public ClusterMetrics(MeterRegistry registry, ApplicationProperties applicationProperties) {
+        this.registry = registry;
+        this.applicationProperties = applicationProperties;
+        this.stickyMissTotal =
+                Counter.builder("stirling_cluster_sticky_miss_total")
+                        .description(
+                                "Sticky-session misses: a download for a job whose result lives on"
+                                        + " a peer node landed on this node. High sustained value means"
+                                        + " LB affinity is broken.")
+                        .register(registry);
+        this.rateLimitRejected =
+                Counter.builder("stirling_cluster_ratelimit_rejected_total")
+                        .description("Cluster-wide rate limit rejections")
+                        .register(registry);
+        this.backplaneLatency =
+                Timer.builder("stirling_cluster_backplane_latency_seconds")
+                        .description("Backplane round-trip latency")
+                        .register(registry);
+        this.jobWaitSeconds =
+                Timer.builder("stirling_cluster_job_wait_seconds")
+                        .description("Time jobs spend queued before execution")
+                        .register(registry);
+        Gauge.builder("stirling_cluster_jobs_inflight", jobsInflight, AtomicLong::doubleValue)
+                .description("Jobs currently in flight on this node")
+                .tag("node", applicationProperties.getCluster().resolvedNodeId())
+                .register(registry);
+        for (String lane : KNOWN_LANES) {
+            ensureLaneGauge(lane);
+        }
+    }
+
+    @Override
+    public void recordStickyMiss() {
+        stickyMissTotal.increment();
+    }
+
+    public void recordRateLimitReject() {
+        rateLimitRejected.increment();
+    }
+
+    public Timer backplaneLatency() {
+        return backplaneLatency;
+    }
+
+    public Timer jobWaitSeconds() {
+        return jobWaitSeconds;
+    }
+
+    public void incrementInflight() {
+        jobsInflight.incrementAndGet();
+    }
+
+    public void decrementInflight() {
+        jobsInflight.decrementAndGet();
+    }
+
+    public void setQueueDepth(String lane, long depth) {
+        ensureLaneGauge(lane).set(depth);
+    }
+
+    private AtomicLong ensureLaneGauge(String lane) {
+        return queueDepth.computeIfAbsent(
+                lane,
+                l -> {
+                    AtomicLong holder = new AtomicLong();
+                    Gauge.builder("stirling_cluster_queue_depth", holder, AtomicLong::doubleValue)
+                            .description("Pending items in a job queue lane")
+                            .tag("lane", l)
+                            .register(registry);
+                    return holder;
+                });
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/ClusterNodeBootstrap.java

@@ -0,0 +1,201 @@
+package stirling.software.proprietary.cluster;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Locale;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.SmartLifecycle;
+import org.springframework.context.event.EventListener;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.cluster.ClusterNode;
+import stirling.software.common.cluster.InstanceRegistry;
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.model.ApplicationProperties.Cluster;
+
+/**
+ * Registers the local node with {@link InstanceRegistry} on startup, refreshes the entry at 1/3 of
+ * the TTL, and deregisters cleanly on shutdown.
+ *
+ * <p>Implements {@link SmartLifecycle} with {@code getPhase() == Integer.MAX_VALUE} so Spring tears
+ * this bean down before {@code LettuceConnectionFactory} - deregister therefore runs while the
+ * Valkey connection is still alive.
+ */
+@Component
+@Slf4j
+@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
+public class ClusterNodeBootstrap implements SmartLifecycle {
+
+    private final Duration heartbeatTtl;
+
+    private final ApplicationProperties applicationProperties;
+    private final InstanceRegistry instanceRegistry;
+
+    @Value("${server.port:8080}")
+    private int serverPort;
+
+    private volatile String nodeId;
+    private volatile String internalAddress;
+    private volatile boolean running = false;
+
+    public ClusterNodeBootstrap(
+            ApplicationProperties applicationProperties, InstanceRegistry instanceRegistry) {
+        this.applicationProperties = applicationProperties;
+        this.instanceRegistry = instanceRegistry;
+        Cluster cluster = applicationProperties.getCluster();
+        // Default must match the @Scheduled fallback below AND the model default
+        // (ApplicationProperties.Cluster.Node.heartbeatIntervalMs = 5000); otherwise the TTL is
+        // computed from a different interval than the scheduler runs at and the 3x margin breaks.
+        long heartbeatMs =
+                cluster.getNode() == null ? 5000L : cluster.getNode().getHeartbeatIntervalMs();
+        // TTL = 3x heartbeat: tolerate two missed ticks before the node drops out of the registry.
+        this.heartbeatTtl = Duration.ofMillis(heartbeatMs * 3);
+    }
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void registerOnStartup() {
+        nodeId = applicationProperties.getCluster().resolvedNodeId();
+        internalAddress = resolveInternalAddress();
+        registerSelf("register");
+    }
+
+    @Scheduled(fixedDelayString = "${cluster.node.heartbeat-interval-ms:5000}")
+    public void heartbeat() {
+        // Heartbeat-after-stop race: SmartLifecycle.stop() deregisters, but the @Scheduled
+        // tick keeps firing during a slow drain. Without this guard, the next tick re-registers
+        // the dead node and the entry resurfaces in the registry until TTL expiry.
+        if (!running) {
+            return;
+        }
+        if (nodeId == null) {
+            return; // not yet registered (startup race)
+        }
+        // Self-healing: register() is idempotent and re-populates every field, so a wiped
+        // Valkey (FLUSHALL, hash eviction) recovers on the next tick without operator action.
+        registerSelf("heartbeat");
+    }
+
+    private void registerSelf(String reason) {
+        try {
+            instanceRegistry.register(
+                    new ClusterNode(nodeId, internalAddress, Instant.now(), role()), heartbeatTtl);
+            if ("register".equals(reason)) {
+                log.info(
+                        "Cluster node registered: nodeId={}, internalAddress={}, role={}, ttl={}s",
+                        nodeId,
+                        internalAddress,
+                        role(),
+                        heartbeatTtl.toSeconds());
+            }
+        } catch (RuntimeException e) {
+            log.debug("Cluster {} failed for {}", reason, nodeId, e);
+        }
+    }
+
+    @Override
+    public void start() {
+        running = true;
+    }
+
+    @Override
+    public void stop() {
+        running = false;
+        if (nodeId == null) {
+            return;
+        }
+        try {
+            instanceRegistry.deregister(nodeId);
+            log.info("Cluster node deregistered: {}", nodeId);
+        } catch (RuntimeException e) {
+            // Registry entry will TTL-expire within heartbeatTtl anyway.
+            log.warn(
+                    "Cluster deregister failed for {} (will TTL-expire within {}s): {}",
+                    nodeId,
+                    heartbeatTtl.toSeconds(),
+                    e.getMessage());
+        }
+    }
+
+    @Override
+    public boolean isRunning() {
+        return running;
+    }
+
+    @Override
+    public int getPhase() {
+        return Integer.MAX_VALUE;
+    }
+
+    @Override
+    public boolean isAutoStartup() {
+        return true;
+    }
+
+    /**
+     * Resolve the address peers should hit. Order: explicit config -> {@code POD_IP} env (K8s
+     * downward API) -> JDK hostname -> fail loud (never silently fall back to a loopback).
+     *
+     * <p>Scheme is taken from {@code cluster.node.scheme} (default {@code http}). Set to {@code
+     * https} when nodes terminate TLS themselves; leave as {@code http} when an upstream LB
+     * terminates TLS and intra-cluster traffic is plain HTTP.
+     */
+    private String resolveInternalAddress() {
+        Cluster cluster = applicationProperties.getCluster();
+        String configured =
+                cluster.getNode() == null ? null : cluster.getNode().getInternalAddress();
+        if (configured != null && !configured.isBlank()) {
+            return ensurePort(configured);
+        }
+        String podIp = System.getenv("POD_IP");
+        if (podIp != null && !podIp.isBlank()) {
+            return scheme() + "://" + podIp + ":" + serverPort;
+        }
+        try {
+            return scheme()
+                    + "://"
+                    + InetAddress.getLocalHost().getHostAddress()
+                    + ":"
+                    + serverPort;
+        } catch (UnknownHostException e) {
+            throw new IllegalStateException(
+                    "Could not resolve this host's address for cluster registration; set"
+                            + " cluster.node.internal-address explicitly (or set POD_IP"
+                            + " in the Kubernetes downward API).",
+                    e);
+        }
+    }
+
+    private String ensurePort(String addr) {
+        if (addr.startsWith("http://") || addr.startsWith("https://")) {
+            return addr;
+        }
+        if (addr.contains(":")) {
+            return scheme() + "://" + addr;
+        }
+        return scheme() + "://" + addr + ":" + serverPort;
+    }
+
+    private String scheme() {
+        Cluster cluster = applicationProperties.getCluster();
+        if (cluster.getNode() == null
+                || cluster.getNode().getScheme() == null
+                || cluster.getNode().getScheme().isBlank()) {
+            return "http";
+        }
+        String s = cluster.getNode().getScheme().trim().toLowerCase(Locale.ROOT);
+        return "https".equals(s) ? "https" : "http";
+    }
+
+    private String role() {
+        Cluster.NodeRole r = applicationProperties.getCluster().resolvedRole();
+        return r == null ? "BOTH" : r.name();
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ConditionalOnValkeyBackplane.java

@@ -0,0 +1,19 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+
+/**
+ * Composite condition: matches only when cluster.enabled=true AND cluster.backplane=valkey. Both
+ * checks are required (enabled alone may select the in-process backplane, which must not load
+ * Valkey beans); a single {@code @ConditionalOnExpression} keeps the guard in one place.
+ */
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@ConditionalOnExpression(
+        "${cluster.enabled:false} and '${cluster.backplane:inprocess}'.equals('valkey')")
+public @interface ConditionalOnValkeyBackplane {}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplane.java

@@ -0,0 +1,55 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import org.springframework.data.redis.core.RedisCallback;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Component;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.model.ApplicationProperties;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+@ConditionalOnValkeyBackplane
+public class ValkeyClusterBackplane implements ClusterBackplane {
+
+    private final ApplicationProperties applicationProperties;
+    private final StringRedisTemplate template;
+
+    @Override
+    public boolean isHealthy() {
+        try {
+            // template.execute() borrows from the pool and returns the connection in a finally
+            // block - critical because isHealthy() is hit on every k8s liveness/readiness probe
+            // tick. Calling getConnectionFactory().getConnection() directly leaks the connection
+            // and exhausts the pool under monitoring load.
+            String pong = template.execute((RedisCallback<String>) connection -> connection.ping());
+            return "PONG".equalsIgnoreCase(pong);
+        } catch (RuntimeException ex) {
+            log.warn("Valkey backplane health check failed: {}", ex.getMessage());
+            return false;
+        }
+    }
+
+    @Override
+    public String backplaneType() {
+        return "valkey";
+    }
+
+    @Override
+    public String localNodeId() {
+        return applicationProperties.getCluster().resolvedNodeId();
+    }
+
+    /**
+     * Valkey TTL evicts job entries; local cleanup loop is redundant and would race with cluster
+     * state.
+     */
+    @Override
+    public boolean shouldRunLocalCleanup() {
+        return false;
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfiguration.java

@@ -0,0 +1,265 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.time.Duration;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.DependsOn;
+import org.springframework.data.redis.connection.RedisConnection;
+import org.springframework.data.redis.connection.RedisPassword;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.core.StringRedisTemplate;
+
+import io.lettuce.core.RedisCommandExecutionException;
+import io.lettuce.core.SslVerifyMode;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.model.ApplicationProperties;
+import stirling.software.common.model.ApplicationProperties.Cluster;
+
+@Slf4j
+@Configuration
+@RequiredArgsConstructor
+@ConditionalOnProperty(name = "cluster.enabled", havingValue = "true")
+@DependsOn("clusterLicenseGate")
+public class ValkeyConnectionConfiguration {
+
+    private final ApplicationProperties applicationProperties;
+
+    @Bean(destroyMethod = "destroy")
+    @ConditionalOnProperty(name = "cluster.backplane", havingValue = "valkey")
+    public LettuceConnectionFactory valkeyConnectionFactory() {
+        Cluster cluster = applicationProperties.getCluster();
+        Endpoint endpoint = parseUrl(cluster.getValkey().getUrl());
+        RedisStandaloneConfiguration cfg =
+                new RedisStandaloneConfiguration(endpoint.host(), endpoint.port());
+        if (endpoint.username() != null) {
+            cfg.setUsername(endpoint.username());
+        }
+        if (endpoint.password() != null) {
+            cfg.setPassword(RedisPassword.of(endpoint.password()));
+        }
+        boolean skipCertVerification =
+                cluster.getValkey().getTls() != null
+                        && cluster.getValkey().getTls().isSkipCertVerification();
+        LettuceClientConfiguration clientConfig =
+                buildClientConfiguration(endpoint.tls(), skipCertVerification);
+        LettuceConnectionFactory factory = new LettuceConnectionFactory(cfg, clientConfig);
+        factory.afterPropertiesSet();
+        // Eager handshake with retry tolerates docker-compose DNS races; fails boot loudly
+        // if Valkey is genuinely unreachable.
+        eagerHandshake(factory, endpoint.host(), endpoint.port(), endpoint.tls());
+        log.info(
+                "Valkey connection configured: {}:{} tls={} verifyPeer={}",
+                endpoint.host(),
+                endpoint.port(),
+                endpoint.tls(),
+                endpoint.tls() ? clientConfig.getVerifyMode() : "n/a");
+        return factory;
+    }
+
+    /** Parsed connection endpoint; username/password are null when absent. */
+    record Endpoint(String host, int port, boolean tls, String username, String password) {}
+
+    /**
+     * Parses {@code redis://[user:password@]host[:port]} (or {@code rediss://} for TLS) into an
+     * {@link Endpoint}. Package-private and side-effect-free so URL handling is unit-testable.
+     *
+     * <ul>
+     *   <li>Missing port defaults to 6379.
+     *   <li>{@code rediss} scheme selects TLS.
+     *   <li>Userinfo {@code :password@} (empty user) is treated as password-only auth against the
+     *       default user, not a login with an empty username.
+     *   <li>Reserved characters in the password ({@code @ : / # ?}) must be percent-encoded; {@link
+     *       URI} parses them structurally otherwise (e.g. {@code #} starts the fragment).
+     * </ul>
+     *
+     * @throws IllegalStateException if the URL is blank, syntactically invalid, or has no host
+     */
+    static Endpoint parseUrl(String url) {
+        if (url == null || url.isBlank()) {
+            throw new IllegalStateException("cluster.valkey.url must be set when backplane=valkey");
+        }
+        URI uri;
+        try {
+            uri = new URI(url);
+        } catch (URISyntaxException ex) {
+            throw new IllegalStateException(
+                    "cluster.valkey.url is not a valid URI: " + url + " (" + ex.getMessage() + ")",
+                    ex);
+        }
+        String host = uri.getHost();
+        if (host == null || host.isBlank()) {
+            throw new IllegalStateException(
+                    "cluster.valkey.url has no host: "
+                            + url
+                            + " (expected redis://[user:password@]host[:port])");
+        }
+        boolean tls = "rediss".equalsIgnoreCase(uri.getScheme());
+        int port = uri.getPort() <= 0 ? 6379 : uri.getPort();
+        String username = null;
+        String password = null;
+        String userInfo = uri.getUserInfo();
+        if (userInfo != null) {
+            String[] parts = userInfo.split(":", 2);
+            if (parts.length == 2) {
+                username = parts[0].isEmpty() ? null : parts[0];
+                password = parts[1];
+            } else if (!parts[0].isBlank()) {
+                password = parts[0];
+            }
+        }
+        return new Endpoint(host, port, tls, username, password);
+    }
+
+    /**
+     * Package-private for testing. verifyPeer(FULL) is pinned explicitly so a Spring Data Redis
+     * default change cannot silently weaken our TLS handshake. skipCertVerification is dev-only.
+     */
+    static LettuceClientConfiguration buildClientConfiguration(
+            boolean tls, boolean skipCertVerification) {
+        LettuceClientConfiguration.LettuceClientConfigurationBuilder clientBuilder =
+                LettuceClientConfiguration.builder();
+        // Bound every backplane command. Lettuce defaults to 60s; without this a partitioned or
+        // slow Valkey would stall hot-path calls (e.g. JobController.guardNonOwner -> jobStore.get
+        // on each request) for up to a minute, exhausting request threads. All backplane ops are
+        // non-blocking single commands, so a short timeout is safe.
+        clientBuilder.commandTimeout(Duration.ofSeconds(2));
+        if (tls) {
+            clientBuilder
+                    .useSsl()
+                    .verifyPeer(skipCertVerification ? SslVerifyMode.NONE : SslVerifyMode.FULL);
+            if (skipCertVerification) {
+                log.warn(
+                        "Valkey TLS hostname/chain verification DISABLED via"
+                                + " cluster.valkey.tls.skip-cert-verification=true"
+                                + " - insecure, dev-only");
+            }
+        }
+        return clientBuilder.build();
+    }
+
+    /**
+     * 10 x 3s = 30s boot-time retry. Auth failures (WRONGPASS/NOAUTH/NOPERM) short-circuit
+     * immediately; only transport errors get the loop. Package-private for testing.
+     */
+    static void eagerHandshake(
+            LettuceConnectionFactory factory, String host, int port, boolean tls) {
+        RuntimeException last = null;
+        for (int attempt = 1; attempt <= 10; attempt++) {
+            try {
+                String pong;
+                RedisConnection conn = factory.getConnection();
+                try {
+                    pong = conn.ping();
+                } finally {
+                    conn.close();
+                }
+                if (!"PONG".equalsIgnoreCase(pong)) {
+                    throw new IllegalStateException(
+                            "Valkey PING returned '" + pong + "' (expected PONG)");
+                }
+                if (attempt > 1) {
+                    log.info("Valkey reachable after {} attempts", attempt);
+                }
+                return;
+            } catch (RuntimeException ex) {
+                if (isAuthFailure(ex)) {
+                    factory.destroy();
+                    throw new IllegalStateException(
+                            "Valkey authentication failed for "
+                                    + host
+                                    + ":"
+                                    + port
+                                    + " (tls="
+                                    + tls
+                                    + "): "
+                                    + rootAuthMessage(ex)
+                                    + ". Check cluster.valkey.url credentials"
+                                    + " (user/password and ACL permissions).",
+                            ex);
+                }
+                last = ex;
+                log.warn(
+                        "Valkey PING attempt {}/10 failed ({}:{}, tls={}): {}",
+                        attempt,
+                        host,
+                        port,
+                        tls,
+                        ex.getMessage());
+                try {
+                    Thread.sleep(3000);
+                } catch (InterruptedException ie) {
+                    Thread.currentThread().interrupt();
+                    break;
+                }
+            }
+        }
+        factory.destroy();
+        throw new IllegalStateException(
+                "Valkey unreachable at boot after 10 attempts ("
+                        + host
+                        + ":"
+                        + port
+                        + ", tls="
+                        + tls
+                        + "): "
+                        + (last == null ? "no detail" : last.getMessage()),
+                last);
+    }
+
+    /**
+     * Walks the cause chain for WRONGPASS/NOAUTH/NOPERM replies. Spring Data Redis wraps Lettuce's
+     * RedisCommandExecutionException in RedisSystemException, so the auth signal may be one level
+     * down. No typed auth exception exists in spring-data-redis 4.0.5 / Lettuce 6.8.2.
+     */
+    static boolean isAuthFailure(Throwable t) {
+        for (Throwable cur = t; cur != null; cur = cur.getCause()) {
+            if (cur instanceof RedisCommandExecutionException && hasAuthPrefix(cur.getMessage())) {
+                return true;
+            }
+            if (hasAuthPrefix(cur.getMessage())) {
+                return true;
+            }
+            if (cur.getCause() == cur) {
+                break;
+            }
+        }
+        return false;
+    }
+
+    private static boolean hasAuthPrefix(String message) {
+        if (message == null) {
+            return false;
+        }
+        String upper = message.toUpperCase(java.util.Locale.ROOT).stripLeading();
+        return upper.startsWith("WRONGPASS")
+                || upper.startsWith("NOAUTH")
+                || upper.startsWith("NOPERM");
+    }
+
+    private static String rootAuthMessage(Throwable t) {
+        for (Throwable cur = t; cur != null; cur = cur.getCause()) {
+            if (cur instanceof RedisCommandExecutionException && cur.getMessage() != null) {
+                return cur.getMessage();
+            }
+            if (cur.getCause() == cur) {
+                break;
+            }
+        }
+        return t.getMessage();
+    }
+
+    @Bean
+    @ConditionalOnProperty(name = "cluster.backplane", havingValue = "valkey")
+    public StringRedisTemplate valkeyTemplate(LettuceConnectionFactory factory) {
+        return new StringRedisTemplate(factory);
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyDistributedLock.java

@@ -0,0 +1,102 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import java.time.Duration;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.data.redis.core.script.DefaultRedisScript;
+import org.springframework.data.redis.core.script.RedisScript;
+import org.springframework.stereotype.Component;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.cluster.DistributedLock;
+
+@Component
+@RequiredArgsConstructor
+@ConditionalOnValkeyBackplane
+@Slf4j
+public class ValkeyDistributedLock implements DistributedLock {
+
+    private static final String PREFIX = "stirling:lock:";
+
+    private static final RedisScript<Long> RELEASE_SCRIPT =
+            new DefaultRedisScript<>(
+                    "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end",
+                    Long.class);
+
+    private static final RedisScript<Long> RENEW_SCRIPT =
+            new DefaultRedisScript<>(
+                    "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end",
+                    Long.class);
+
+    private final StringRedisTemplate template;
+
+    @Override
+    public Optional<LockHandle> tryAcquire(String lockKey, Duration leaseTime) {
+        String key = PREFIX + lockKey;
+        String value = UUID.randomUUID().toString();
+        Boolean ok = template.opsForValue().setIfAbsent(key, value, leaseTime);
+        if (Boolean.TRUE.equals(ok)) {
+            return Optional.of(new ValkeyHandle(template, key, value));
+        }
+        return Optional.empty();
+    }
+
+    private static final class ValkeyHandle implements LockHandle {
+        private final StringRedisTemplate template;
+        private final String key;
+        private final String value;
+        private boolean released;
+
+        ValkeyHandle(StringRedisTemplate template, String key, String value) {
+            this.template = template;
+            this.key = key;
+            this.value = value;
+        }
+
+        @Override
+        public synchronized void release() {
+            if (released) {
+                return;
+            }
+            released = true;
+            // Swallow + log: LockHandle is AutoCloseable, so release() runs from close() inside
+            // try-with-resources. An uncaught Valkey error here would mask the body's exception.
+            // The lease TTL-expires anyway, so a failed explicit release is safe.
+            try {
+                template.execute(RELEASE_SCRIPT, Collections.singletonList(key), value);
+            } catch (RuntimeException ex) {
+                log.warn(
+                        "Lock release failed for {} (lease will TTL-expire): {}",
+                        key,
+                        ex.getMessage());
+            }
+        }
+
+        @Override
+        public synchronized boolean renew(Duration leaseTime) {
+            if (released) {
+                return false;
+            }
+            try {
+                Long result =
+                        template.execute(
+                                RENEW_SCRIPT,
+                                Collections.singletonList(key),
+                                value,
+                                Long.toString(leaseTime.toMillis()));
+                return result != null && result == 1L;
+            } catch (RuntimeException ex) {
+                log.warn(
+                        "Lock renew failed for {} (treated as lost lease): {}",
+                        key,
+                        ex.getMessage());
+                return false;
+            }
+        }
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyInstanceRegistry.java

@@ -0,0 +1,115 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.springframework.data.redis.core.Cursor;
+import org.springframework.data.redis.core.RedisCallback;
+import org.springframework.data.redis.core.ScanOptions;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Component;
+
+import lombok.RequiredArgsConstructor;
+
+import stirling.software.common.cluster.ClusterNode;
+import stirling.software.common.cluster.InstanceRegistry;
+
+/**
+ * Valkey-backed {@link InstanceRegistry}. Each node is stored as a hash with a TTL equal to the
+ * configured heartbeat TTL; the heartbeat re-arms the TTL.
+ */
+@Component
+@RequiredArgsConstructor
+@ConditionalOnValkeyBackplane
+public class ValkeyInstanceRegistry implements InstanceRegistry {
+
+    private static final String PREFIX = "stirling:nodes:";
+
+    private final StringRedisTemplate template;
+
+    @Override
+    public void register(ClusterNode node, Duration heartbeatTtl) {
+        String key = PREFIX + node.nodeId();
+        long ttlMs = heartbeatTtl.toMillis();
+        Map<String, String> fields = new LinkedHashMap<>();
+        fields.put("nodeId", node.nodeId());
+        fields.put("internalAddress", node.internalAddress());
+        fields.put("role", node.role());
+        fields.put("lastHeartbeat", node.lastHeartbeat().toString());
+
+        // MULTI/EXEC so the hash fields and the TTL commit together. Without this, a crash
+        // between HSET and EXPIRE leaves the hash with no TTL: it never expires, masks the
+        // dead node as alive, and only a subsequent successful register() would re-arm it.
+        template.execute(
+                (RedisCallback<Object>)
+                        connection -> {
+                            connection.multi();
+                            byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
+                            Map<byte[], byte[]> hashBytes = new LinkedHashMap<>();
+                            for (Map.Entry<String, String> f : fields.entrySet()) {
+                                hashBytes.put(
+                                        f.getKey().getBytes(StandardCharsets.UTF_8),
+                                        f.getValue().getBytes(StandardCharsets.UTF_8));
+                            }
+                            connection.hashCommands().hMSet(keyBytes, hashBytes);
+                            connection.keyCommands().pExpire(keyBytes, ttlMs);
+                            connection.exec();
+                            return null;
+                        });
+    }
+
+    @Override
+    public Optional<ClusterNode> lookup(String nodeId) {
+        return readNode(PREFIX + nodeId);
+    }
+
+    @Override
+    public Collection<ClusterNode> activeNodes() {
+        ScanOptions options = ScanOptions.scanOptions().match(PREFIX + "*").count(256).build();
+        List<ClusterNode> nodes = new ArrayList<>();
+        try (Cursor<String> cursor = template.scan(options)) {
+            while (cursor.hasNext()) {
+                readNode(cursor.next()).ifPresent(nodes::add);
+            }
+        }
+        return nodes;
+    }
+
+    @Override
+    public void deregister(String nodeId) {
+        template.delete(PREFIX + nodeId);
+    }
+
+    private Optional<ClusterNode> readNode(String key) {
+        Map<Object, Object> entries = template.opsForHash().entries(key);
+        if (entries == null || entries.isEmpty()) {
+            return Optional.empty();
+        }
+        Object nodeId = entries.get("nodeId");
+        if (nodeId == null) {
+            return Optional.empty();
+        }
+        Instant heartbeat = Instant.now();
+        Object hb = entries.get("lastHeartbeat");
+        if (hb != null) {
+            try {
+                heartbeat = Instant.parse(hb.toString());
+            } catch (RuntimeException ignored) {
+                // keep default
+            }
+        }
+        return Optional.of(
+                new ClusterNode(
+                        nodeId.toString(),
+                        String.valueOf(entries.getOrDefault("internalAddress", "")),
+                        heartbeat,
+                        String.valueOf(entries.getOrDefault("role", "BOTH"))));
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyJobStore.java

@@ -0,0 +1,297 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.springframework.data.redis.core.Cursor;
+import org.springframework.data.redis.core.RedisCallback;
+import org.springframework.data.redis.core.ScanOptions;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Component;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.JobStoreEntry;
+
+/**
+ * Valkey-backed {@link JobStore}. Each job is one hash; a reverse index maps fileId to jobId.
+ *
+ * <p><b>put() atomicity:</b> the hash fields, the per-job TTL, and the reverse-index entries are
+ * issued inside a single pipelined Redis transaction (MULTI/EXEC). A partial failure cannot leave
+ * the hash without a TTL or with half the file→job index entries written.
+ */
+@Component
+@RequiredArgsConstructor
+@ConditionalOnValkeyBackplane
+@Slf4j
+public class ValkeyJobStore implements JobStore {
+
+    private static final String JOB_PREFIX = "stirling:job:";
+    private static final String FILE_INDEX_PREFIX = "stirling:file2job:";
+
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+    private static final TypeReference<List<String>> LIST_STRING = new TypeReference<>() {};
+    private static final TypeReference<Map<String, String>> MAP_STRING = new TypeReference<>() {};
+
+    private final StringRedisTemplate template;
+
+    @Override
+    public void put(JobStoreEntry entry, Duration ttl) {
+        String key = JOB_PREFIX + entry.jobId();
+        long ttlMs = ttl.toMillis();
+        Map<String, String> fields = new LinkedHashMap<>();
+        fields.put("jobId", entry.jobId());
+        fields.put("state", entry.state().name());
+        fields.put("owningNodeId", entry.owningNodeId() == null ? "" : entry.owningNodeId());
+        if (entry.createdAt() != null) {
+            fields.put("createdAt", entry.createdAt().toString());
+        }
+        if (entry.completedAt() != null) {
+            fields.put("completedAt", entry.completedAt().toString());
+        }
+        if (entry.error() != null) {
+            fields.put("error", entry.error());
+        }
+        fields.put("fileIds", writeJson(entry.fileIds() == null ? List.of() : entry.fileIds()));
+        fields.put(
+                "resultMeta",
+                writeJson(entry.resultMeta() == null ? Map.of() : entry.resultMeta()));
+
+        // Build pipelined MULTI/EXEC so the hash, its TTL, and every reverse-index entry
+        // commit atomically.
+        template.execute(
+                (RedisCallback<Object>)
+                        connection -> {
+                            connection.multi();
+                            byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
+                            Map<byte[], byte[]> hashBytes = new LinkedHashMap<>();
+                            for (Map.Entry<String, String> f : fields.entrySet()) {
+                                hashBytes.put(
+                                        f.getKey().getBytes(StandardCharsets.UTF_8),
+                                        f.getValue().getBytes(StandardCharsets.UTF_8));
+                            }
+                            connection.hashCommands().hMSet(keyBytes, hashBytes);
+                            connection.keyCommands().pExpire(keyBytes, ttlMs);
+                            if (entry.fileIds() != null) {
+                                for (String fileId : entry.fileIds()) {
+                                    byte[] idxKey =
+                                            (FILE_INDEX_PREFIX + fileId)
+                                                    .getBytes(StandardCharsets.UTF_8);
+                                    connection
+                                            .stringCommands()
+                                            .set(
+                                                    idxKey,
+                                                    entry.jobId().getBytes(StandardCharsets.UTF_8));
+                                    connection.keyCommands().pExpire(idxKey, ttlMs);
+                                }
+                            }
+                            connection.exec();
+                            return null;
+                        });
+    }
+
+    @Override
+    public Optional<JobStoreEntry> get(String jobId) {
+        return readEntry(JOB_PREFIX + jobId);
+    }
+
+    @Override
+    public void delete(String jobId) {
+        // WATCH/MULTI/EXEC: read fileIds INSIDE the watched scope so a concurrent put() that
+        // adds new fileIds between our read and EXEC aborts the transaction. Without this guard,
+        // an interleaved put() that grows fileIds would leave orphaned reverse-index entries
+        // pointing at the deleted jobId until their TTL expires. One retry handles the common
+        // case; further contention falls through to lazy TTL cleanup (acceptable - this is an
+        // eviction path, not a correctness primitive).
+        String jobKey = JOB_PREFIX + jobId;
+        byte[] jobKeyBytes = jobKey.getBytes(StandardCharsets.UTF_8);
+        for (int attempt = 0; attempt < 2; attempt++) {
+            Boolean committed =
+                    template.execute(
+                            (RedisCallback<Boolean>)
+                                    connection -> {
+                                        connection.watch(jobKeyBytes);
+                                        // Read the single fileIds field with hGet rather than
+                                        // hGetAll + map.get: hGetAll returns a Map<byte[],byte[]>
+                                        // whose keys compare by identity, so a fresh
+                                        // "fileIds".getBytes() lookup never matches and the reverse
+                                        // index would be left orphaned. hGet resolves the field
+                                        // server-side.
+                                        byte[] fileIdsBytes =
+                                                connection
+                                                        .hashCommands()
+                                                        .hGet(
+                                                                jobKeyBytes,
+                                                                "fileIds"
+                                                                        .getBytes(
+                                                                                StandardCharsets
+                                                                                        .UTF_8));
+                                        List<byte[]> keysToDelete = new ArrayList<>();
+                                        keysToDelete.add(jobKeyBytes);
+                                        if (fileIdsBytes != null) {
+                                            List<String> fileIds =
+                                                    readJsonList(
+                                                            new String(
+                                                                    fileIdsBytes,
+                                                                    StandardCharsets.UTF_8),
+                                                            jobKey);
+                                            for (String fileId : fileIds) {
+                                                keysToDelete.add(
+                                                        (FILE_INDEX_PREFIX + fileId)
+                                                                .getBytes(StandardCharsets.UTF_8));
+                                            }
+                                        }
+                                        connection.multi();
+                                        for (byte[] key : keysToDelete) {
+                                            connection.keyCommands().del(key);
+                                        }
+                                        List<Object> results = connection.exec();
+                                        // exec() returns null when WATCH detected a concurrent
+                                        // write; spring-data-redis surfaces this as either null
+                                        // or empty depending on the driver path.
+                                        return results != null && !results.isEmpty();
+                                    });
+            if (Boolean.TRUE.equals(committed)) {
+                return;
+            }
+        }
+        log.warn(
+                "JobStore.delete({}) lost two WATCH races to concurrent put(); reverse-index"
+                        + " entries may linger until TTL expiry",
+                jobId);
+    }
+
+    @Override
+    public boolean exists(String jobId) {
+        Boolean exists = template.hasKey(JOB_PREFIX + jobId);
+        return Boolean.TRUE.equals(exists);
+    }
+
+    @Override
+    public Optional<String> findJobIdByFileId(String fileId) {
+        return Optional.ofNullable(template.opsForValue().get(FILE_INDEX_PREFIX + fileId));
+    }
+
+    @Override
+    public Collection<JobStoreEntry> all() {
+        // SCAN, not KEYS - KEYS blocks the Valkey server for the duration of the walk.
+        ScanOptions options = ScanOptions.scanOptions().match(JOB_PREFIX + "*").count(256).build();
+        List<JobStoreEntry> result = new ArrayList<>();
+        try (Cursor<String> cursor = template.scan(options)) {
+            while (cursor.hasNext()) {
+                readEntry(cursor.next()).ifPresent(result::add);
+            }
+        }
+        return result;
+    }
+
+    private Optional<JobStoreEntry> readEntry(String key) {
+        Map<Object, Object> entries = template.opsForHash().entries(key);
+        if (entries == null || entries.isEmpty()) {
+            return Optional.empty();
+        }
+        Object jobId = entries.get("jobId");
+        if (jobId == null) {
+            return Optional.empty();
+        }
+        Instant createdAt = parseInstant(entries.get("createdAt"), key, "createdAt");
+        Instant completedAt = parseInstant(entries.get("completedAt"), key, "completedAt");
+        List<String> fileIds = parseList(entries.get("fileIds"), key);
+        Map<String, String> resultMeta = parseMap(entries.get("resultMeta"), key);
+        String stateName =
+                String.valueOf(
+                        entries.getOrDefault("state", JobStoreEntry.JobState.PENDING.name()));
+        JobStoreEntry.JobState state;
+        try {
+            state = JobStoreEntry.JobState.valueOf(stateName);
+        } catch (IllegalArgumentException ex) {
+            log.warn("Unrecognised job state '{}' in {}, defaulting to PENDING", stateName, key);
+            state = JobStoreEntry.JobState.PENDING;
+        }
+        String owningNodeId = String.valueOf(entries.getOrDefault("owningNodeId", ""));
+        String error = entries.get("error") == null ? null : entries.get("error").toString();
+        return Optional.of(
+                new JobStoreEntry(
+                        jobId.toString(),
+                        state,
+                        owningNodeId,
+                        createdAt,
+                        completedAt,
+                        error,
+                        fileIds,
+                        resultMeta));
+    }
+
+    private Instant parseInstant(Object v, String key, String field) {
+        if (v == null) {
+            return null;
+        }
+        try {
+            return Instant.parse(v.toString());
+        } catch (RuntimeException e) {
+            log.warn(
+                    "JobStore {} field '{}' has malformed timestamp '{}' - treating as missing",
+                    key,
+                    field,
+                    v);
+            return null;
+        }
+    }
+
+    private List<String> parseList(Object v, String key) {
+        if (v == null) {
+            return new ArrayList<>();
+        }
+        return readJsonList(v.toString(), key);
+    }
+
+    private Map<String, String> parseMap(Object v, String key) {
+        if (v == null) {
+            return new HashMap<>();
+        }
+        try {
+            return MAPPER.readValue(v.toString(), MAP_STRING);
+        } catch (JsonProcessingException e) {
+            log.warn(
+                    "JobStore {} field 'resultMeta' is not valid JSON '{}' - treating as empty",
+                    key,
+                    v);
+            return new HashMap<>();
+        }
+    }
+
+    private static String writeJson(Object value) {
+        try {
+            return MAPPER.writeValueAsString(value);
+        } catch (JsonProcessingException e) {
+            throw new IllegalStateException("Failed to JSON-serialize JobStore field", e);
+        }
+    }
+
+    private List<String> readJsonList(String json, String key) {
+        try {
+            List<String> parsed = MAPPER.readValue(json, LIST_STRING);
+            return parsed == null ? new ArrayList<>() : parsed;
+        } catch (JsonProcessingException e) {
+            log.warn(
+                    "JobStore {} field 'fileIds' is not valid JSON '{}' - treating as empty",
+                    key,
+                    json);
+            return new ArrayList<>();
+        }
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyKeyValueCache.java

@@ -0,0 +1,61 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+import org.springframework.data.redis.core.Cursor;
+import org.springframework.data.redis.core.ScanOptions;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Component;
+
+import lombok.RequiredArgsConstructor;
+
+import stirling.software.common.cluster.KeyValueCache;
+
+@Component
+@RequiredArgsConstructor
+@ConditionalOnValkeyBackplane
+public class ValkeyKeyValueCache implements KeyValueCache {
+
+    private static final String PREFIX = "stirling:kv:";
+
+    private final StringRedisTemplate template;
+
+    @Override
+    public void put(String namespace, String key, String value, Duration ttl) {
+        template.opsForValue()
+                .set(buildKey(namespace, key), value, ttl.toMillis(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public Optional<String> get(String namespace, String key) {
+        return Optional.ofNullable(template.opsForValue().get(buildKey(namespace, key)));
+    }
+
+    @Override
+    public void evict(String namespace, String key) {
+        template.delete(buildKey(namespace, key));
+    }
+
+    @Override
+    public void evictNamespace(String namespace) {
+        ScanOptions options =
+                ScanOptions.scanOptions().match(PREFIX + namespace + ":*").count(256).build();
+        List<String> keys = new ArrayList<>();
+        try (Cursor<String> cursor = template.scan(options)) {
+            while (cursor.hasNext()) {
+                keys.add(cursor.next());
+            }
+        }
+        if (!keys.isEmpty()) {
+            template.delete(keys);
+        }
+    }
+
+    private String buildKey(String namespace, String key) {
+        return PREFIX + namespace + ":" + key;
+    }
+}

app/proprietary/src/main/java/stirling/software/proprietary/cluster/valkey/ValkeyRateLimitStore.java

@@ -0,0 +1,83 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.stereotype.Component;
+
+import io.github.bucket4j.BucketConfiguration;
+import io.github.bucket4j.ConsumptionProbe;
+import io.github.bucket4j.distributed.BucketProxy;
+import io.github.bucket4j.distributed.ExpirationAfterWriteStrategy;
+import io.github.bucket4j.distributed.proxy.ProxyManager;
+import io.github.bucket4j.redis.lettuce.Bucket4jLettuce;
+import io.lettuce.core.AbstractRedisClient;
+import io.lettuce.core.RedisClient;
+
+import jakarta.annotation.PostConstruct;
+import jakarta.annotation.PreDestroy;
+
+import stirling.software.common.cluster.RateLimitStore;
+
+/**
+ * Valkey-backed token-bucket rate limiting via Bucket4j's Lettuce ProxyManager. The token bucket
+ * refills continuously and enforces one global limit across nodes, with the same semantics as the
+ * in-process {@code InProcessRateLimitStore} (which also uses Bucket4j).
+ */
+@Component
+@ConditionalOnValkeyBackplane
+public class ValkeyRateLimitStore implements RateLimitStore {
+
+    private static final String PREFIX = "stirling:rl:";
+
+    private final LettuceConnectionFactory connectionFactory;
+    private ProxyManager<byte[]> proxyManager;
+
+    public ValkeyRateLimitStore(LettuceConnectionFactory connectionFactory) {
+        this.connectionFactory = connectionFactory;
+    }
+
+    @PostConstruct
+    void initProxyManager() {
+        AbstractRedisClient client = connectionFactory.getNativeClient();
+        if (!(client instanceof RedisClient redisClient)) {
+            throw new IllegalStateException(
+                    "ValkeyRateLimitStore requires a standalone Lettuce RedisClient; got "
+                            + (client == null ? "null" : client.getClass().getName())
+                            + " (cluster client not supported by this rate limit impl)");
+        }
+        // Expire idle bucket keys so they do not accumulate forever in Valkey (one key per
+        // user / API-key / IP). TTL tracks the time to refill the bucket from empty, capped at
+        // 25h to cover the longest (daily) rate-limit window; an idle bucket evicts after that.
+        this.proxyManager =
+                Bucket4jLettuce.casBasedBuilder(redisClient)
+                        .expirationAfterWrite(
+                                ExpirationAfterWriteStrategy.basedOnTimeForRefillingBucketUpToMax(
+                                        Duration.ofHours(25)))
+                        .build();
+    }
+
+    @PreDestroy
+    void shutdown() {
+        proxyManager = null;
+    }
+
+    @Override
+    public RateLimitDecision tryConsume(String bucketKey, long capacity, Duration refillPeriod) {
+        byte[] key = (PREFIX + bucketKey).getBytes(StandardCharsets.UTF_8);
+        BucketConfiguration cfg =
+                BucketConfiguration.builder()
+                        .addLimit(
+                                stage ->
+                                        stage.capacity(capacity)
+                                                .refillGreedy(capacity, refillPeriod))
+                        .build();
+        BucketProxy bucket = proxyManager.builder().build(key, () -> cfg);
+        ConsumptionProbe probe = bucket.tryConsumeAndReturnRemaining(1);
+        if (probe.isConsumed()) {
+            return new RateLimitDecision(true, probe.getRemainingTokens(), 0L);
+        }
+        return new RateLimitDecision(false, 0L, probe.getNanosToWaitForRefill());
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterLicenseGateTest.java

@@ -0,0 +1,62 @@
+package stirling.software.proprietary.cluster;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+import org.junit.jupiter.api.Test;
+
+class ClusterLicenseGateTest {
+
+    private void injectRunningProOrHigher(ClusterLicenseGate gate, Boolean value) throws Exception {
+        Field f = ClusterLicenseGate.class.getDeclaredField("runningProOrHigher");
+        f.setAccessible(true);
+        f.set(gate, value);
+    }
+
+    private void invokeVerify(ClusterLicenseGate gate) throws Throwable {
+        Method m = ClusterLicenseGate.class.getDeclaredMethod("verifyLicense");
+        m.setAccessible(true);
+        try {
+            m.invoke(gate);
+        } catch (InvocationTargetException e) {
+            throw e.getCause();
+        }
+    }
+
+    @Test
+    void serverOrEnterpriseLicense_allowsClusterMode() throws Throwable {
+        ClusterLicenseGate gate = new ClusterLicenseGate();
+        injectRunningProOrHigher(gate, Boolean.TRUE);
+        assertDoesNotThrow(() -> invokeVerify(gate));
+    }
+
+    @Test
+    void normalLicense_refusesClusterMode_withActionableMessage() throws Exception {
+        ClusterLicenseGate gate = new ClusterLicenseGate();
+        injectRunningProOrHigher(gate, Boolean.FALSE);
+        IllegalStateException ex =
+                assertThrows(IllegalStateException.class, () -> invokeVerify(gate));
+        String msg = ex.getMessage();
+        // The error message must tell the operator exactly what to do.
+        assertTrue(msg.contains("SERVER"), "message must mention SERVER license tier: " + msg);
+        assertTrue(msg.contains("ENTERPRISE"), "message must mention ENTERPRISE tier: " + msg);
+        assertTrue(
+                msg.contains("stirling.premium.key") || msg.contains("license key"),
+                "message must explain how to set the license: " + msg);
+        assertTrue(
+                msg.contains("cluster.enabled=false"),
+                "message must offer the opt-out (disable cluster): " + msg);
+    }
+
+    @Test
+    void saasFlavor_bypassesGate_whenRunningProOrHigherBeanAbsent() throws Throwable {
+        // In saas builds the runningProOrHigher bean is absent (@Autowired required=false -> null).
+        ClusterLicenseGate gate = new ClusterLicenseGate();
+        assertDoesNotThrow(() -> invokeVerify(gate));
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterMetricsTest.java

@@ -0,0 +1,122 @@
+package stirling.software.proprietary.cluster;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+
+import stirling.software.common.model.ApplicationProperties;
+
+/** Verifies every cluster metric is registered and recorder methods write to them. */
+class ClusterMetricsTest {
+
+    private SimpleMeterRegistry registry;
+    private ClusterMetrics metrics;
+    private static final String NODE = "test-node";
+
+    @BeforeEach
+    void setUp() {
+        registry = new SimpleMeterRegistry();
+        ApplicationProperties props = new ApplicationProperties();
+        props.getCluster().getNode().setId(NODE);
+        metrics = new ClusterMetrics(registry, props);
+    }
+
+    @Test
+    void registersAllRequiredMeters() {
+        assertNotNull(registry.find("stirling_cluster_sticky_miss_total").counter());
+        assertNotNull(registry.find("stirling_cluster_ratelimit_rejected_total").counter());
+        assertNotNull(registry.find("stirling_cluster_backplane_latency_seconds").timer());
+        assertNotNull(registry.find("stirling_cluster_job_wait_seconds").timer());
+        Gauge inflight = registry.find("stirling_cluster_jobs_inflight").tag("node", NODE).gauge();
+        assertNotNull(inflight, "jobs_inflight gauge with node tag must be registered eagerly");
+    }
+
+    @Test
+    void registersKnownLaneGaugesEagerly() {
+        for (String lane : new String[] {"FAST", "SLOW", "AI"}) {
+            Gauge g = registry.find("stirling_cluster_queue_depth").tag("lane", lane).gauge();
+            assertNotNull(g, "lane gauge must be eagerly registered for " + lane);
+            assertEquals(0.0, g.value(), "lane gauge default value must be 0 for " + lane);
+        }
+        assertEquals(
+                3,
+                registry.find("stirling_cluster_queue_depth").gauges().size(),
+                "exactly the three known lane gauges should be registered at boot");
+    }
+
+    @Test
+    void recordStickyMissIncrementsCounter() {
+        metrics.recordStickyMiss();
+        metrics.recordStickyMiss();
+        assertEquals(2.0, registry.find("stirling_cluster_sticky_miss_total").counter().count());
+    }
+
+    @Test
+    void recordRateLimitRejectIncrementsCounter() {
+        metrics.recordRateLimitReject();
+        assertEquals(
+                1.0, registry.find("stirling_cluster_ratelimit_rejected_total").counter().count());
+    }
+
+    @Test
+    void incrementAndDecrementInflightUpdatesGauge() {
+        metrics.incrementInflight();
+        metrics.incrementInflight();
+        metrics.incrementInflight();
+        metrics.decrementInflight();
+        Gauge gauge = registry.find("stirling_cluster_jobs_inflight").tag("node", NODE).gauge();
+        assertEquals(2.0, gauge.value(), "expected 2 inflight after 3 inc / 1 dec");
+    }
+
+    @Test
+    void setQueueDepthUpdatesEagerlyRegisteredLaneGauge() {
+        metrics.setQueueDepth("FAST", 4);
+        metrics.setQueueDepth("SLOW", 7);
+
+        Gauge fast = registry.find("stirling_cluster_queue_depth").tag("lane", "FAST").gauge();
+        Gauge slow = registry.find("stirling_cluster_queue_depth").tag("lane", "SLOW").gauge();
+        assertEquals(4.0, fast.value());
+        assertEquals(7.0, slow.value());
+    }
+
+    @Test
+    void setQueueDepthForUnknownLane_lazyRegistersFallbackGauge() {
+        metrics.setQueueDepth("custom-lane", 5);
+        Gauge g = registry.find("stirling_cluster_queue_depth").tag("lane", "custom-lane").gauge();
+        assertNotNull(g);
+        assertEquals(5.0, g.value());
+    }
+
+    @Test
+    void setQueueDepthIsIdempotentAcrossCalls() {
+        metrics.setQueueDepth("FAST", 1);
+        metrics.setQueueDepth("FAST", 2);
+        metrics.setQueueDepth("FAST", 9);
+
+        assertEquals(
+                1,
+                registry.find("stirling_cluster_queue_depth").tag("lane", "FAST").gauges().size());
+        assertEquals(
+                9.0,
+                registry.find("stirling_cluster_queue_depth").tag("lane", "FAST").gauge().value());
+    }
+
+    @Test
+    void backplaneLatencyTimerAcceptsRecordings() {
+        metrics.backplaneLatency().record(java.time.Duration.ofMillis(7));
+        metrics.backplaneLatency().record(java.time.Duration.ofMillis(11));
+        assertEquals(
+                2L, registry.find("stirling_cluster_backplane_latency_seconds").timer().count());
+    }
+
+    @Test
+    void jobWaitTimerAcceptsRecordings() {
+        metrics.jobWaitSeconds().record(java.time.Duration.ofMillis(50));
+        assertEquals(1L, registry.find("stirling_cluster_job_wait_seconds").timer().count());
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/ClusterNodeBootstrapTest.java

@@ -0,0 +1,115 @@
+package stirling.software.proprietary.cluster;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.time.Duration;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import stirling.software.common.cluster.ClusterNode;
+import stirling.software.common.cluster.InstanceRegistry;
+import stirling.software.common.model.ApplicationProperties;
+
+/** Verifies the bootstrap registers / heartbeats / deregisters as expected. */
+class ClusterNodeBootstrapTest {
+
+    private InstanceRegistry registry;
+    private ApplicationProperties props;
+    private ClusterNodeBootstrap bootstrap;
+
+    @BeforeEach
+    void setUp() {
+        registry = mock(InstanceRegistry.class);
+        props = new ApplicationProperties();
+        props.getCluster().setEnabled(true);
+        props.getCluster().getNode().setId("node-test-1");
+        props.getCluster().getNode().setRole("worker");
+        props.getCluster().getNode().setHeartbeatIntervalMs(10_000L);
+        bootstrap = new ClusterNodeBootstrap(props, registry);
+        ReflectionTestUtils.setField(bootstrap, "serverPort", 8080);
+    }
+
+    @Test
+    void registerOnStartupCallsRegistryWithResolvedNodeId() {
+        bootstrap.registerOnStartup();
+        ArgumentCaptor<ClusterNode> nodeCaptor = ArgumentCaptor.forClass(ClusterNode.class);
+        ArgumentCaptor<Duration> ttlCaptor = ArgumentCaptor.forClass(Duration.class);
+        verify(registry, times(1)).register(nodeCaptor.capture(), ttlCaptor.capture());
+        ClusterNode captured = nodeCaptor.getValue();
+        assertEquals("node-test-1", captured.nodeId());
+        assertTrue(captured.internalAddress().startsWith("http://"));
+        assertTrue(captured.internalAddress().endsWith(":8080"));
+        assertEquals("WORKER", captured.role());
+        assertEquals(30L, ttlCaptor.getValue().toSeconds());
+    }
+
+    @Test
+    void registerHonoursExplicitInternalAddress() {
+        props.getCluster().getNode().setInternalAddress("app-1:8080");
+        bootstrap.registerOnStartup();
+        ArgumentCaptor<ClusterNode> nodeCaptor = ArgumentCaptor.forClass(ClusterNode.class);
+        verify(registry).register(nodeCaptor.capture(), any());
+        assertEquals("http://app-1:8080", nodeCaptor.getValue().internalAddress());
+    }
+
+    @Test
+    void registerUsesHttpsSchemeWhenConfigured() {
+        props.getCluster().getNode().setInternalAddress("app-1:8443");
+        props.getCluster().getNode().setScheme("https");
+        ClusterNodeBootstrap httpsBootstrap = new ClusterNodeBootstrap(props, registry);
+        ReflectionTestUtils.setField(httpsBootstrap, "serverPort", 8443);
+        httpsBootstrap.registerOnStartup();
+        ArgumentCaptor<ClusterNode> nodeCaptor = ArgumentCaptor.forClass(ClusterNode.class);
+        verify(registry).register(nodeCaptor.capture(), any());
+        assertEquals("https://app-1:8443", nodeCaptor.getValue().internalAddress());
+    }
+
+    @Test
+    void heartbeatAfterStartup_callsRegister_forSelfHealing() {
+        bootstrap.start();
+        bootstrap.registerOnStartup();
+        bootstrap.heartbeat();
+        verify(registry, times(2))
+                .register(
+                        any(ClusterNode.class),
+                        org.mockito.ArgumentMatchers.eq(Duration.ofSeconds(30)));
+    }
+
+    @Test
+    void smartLifecycleStop_deregisters() {
+        bootstrap.start();
+        bootstrap.registerOnStartup();
+        bootstrap.stop();
+        verify(registry, times(1)).deregister("node-test-1");
+    }
+
+    @Test
+    void smartLifecycleStop_beforeStartup_isNoop() {
+        bootstrap.stop();
+        verify(registry, never()).deregister(any());
+    }
+
+    @Test
+    void heartbeatAfterStop_doesNotReRegister() {
+        // Heartbeat-after-stop race: the @Scheduled tick fires during a slow drain. Without
+        // a guard it would re-register the dead node until TTL expiry.
+        bootstrap.start();
+        bootstrap.registerOnStartup();
+        verify(registry, times(1)).register(any(ClusterNode.class), any(Duration.class));
+
+        bootstrap.stop();
+        verify(registry, times(1)).deregister("node-test-1");
+
+        bootstrap.heartbeat();
+        verify(registry, times(1)).register(any(ClusterNode.class), any(Duration.class));
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/MultiNodeClusterScenarioTest.java

@@ -0,0 +1,133 @@
+package stirling.software.proprietary.cluster;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import stirling.software.common.cluster.ClusterBackplane;
+import stirling.software.common.cluster.JobStore;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.KeyValueCache;
+import stirling.software.common.cluster.RateLimitStore;
+import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
+import stirling.software.common.cluster.inprocess.InProcessJobStore;
+import stirling.software.common.cluster.inprocess.InProcessKeyValueCache;
+import stirling.software.common.cluster.inprocess.InProcessRateLimitStore;
+
+/**
+ * Multi-node contract test using in-process impls shared across two "nodes". Verifies cross-node
+ * visibility, global rate-limit counters, and cache propagation without requiring Docker.
+ * Valkey-specific behavior (MULTI/EXEC atomicity, WATCH races, TTL) is in
+ * LiveValkeyIntegrationTest.
+ */
+class MultiNodeClusterScenarioTest {
+
+    private JobStore sharedJobStore;
+    private RateLimitStore sharedRateLimit;
+    private KeyValueCache sharedCache;
+    private ClusterBackplane backplaneA;
+    private ClusterBackplane backplaneB;
+
+    @BeforeEach
+    void setUp() {
+        sharedJobStore = new InProcessJobStore();
+        sharedRateLimit = new InProcessRateLimitStore();
+        sharedCache = new InProcessKeyValueCache();
+        backplaneA = constBackplane("node-A", "valkey");
+        backplaneB = constBackplane("node-B", "valkey");
+    }
+
+    @Test
+    @DisplayName("async job created on node-A is readable from node-B via shared JobStore")
+    void jobStatusVisibleCrossNode() {
+        JobStoreEntry entry =
+                new JobStoreEntry(
+                        "job-1",
+                        JobStoreEntry.JobState.RUNNING,
+                        "node-A",
+                        Instant.now(),
+                        null,
+                        null,
+                        List.of("file-1"),
+                        Map.of());
+        sharedJobStore.put(entry, Duration.ofMinutes(30));
+
+        Optional<JobStoreEntry> seenOnB = sharedJobStore.get("job-1");
+        assertTrue(seenOnB.isPresent(), "node-B must see node-A's job in shared JobStore");
+        assertEquals("node-A", seenOnB.get().owningNodeId());
+        assertEquals(JobStoreEntry.JobState.RUNNING, seenOnB.get().state());
+    }
+
+    @Test
+    @DisplayName("global rate limit - capacity counted once across both nodes")
+    void rateLimitGlobalAcrossNodes() {
+        long capacity = 4L;
+        RateLimitDecision a1 =
+                sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1));
+        RateLimitDecision b1 =
+                sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1));
+        RateLimitDecision a2 =
+                sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1));
+        RateLimitDecision b2 =
+                sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1));
+        RateLimitDecision a3 =
+                sharedRateLimit.tryConsume("user:bob", capacity, Duration.ofMinutes(1));
+
+        assertTrue(a1.allowed());
+        assertTrue(b1.allowed());
+        assertTrue(a2.allowed());
+        assertTrue(b2.allowed());
+        assertFalse(a3.allowed(), "5th request across both nodes must be rejected (limit=4)");
+    }
+
+    @Test
+    @DisplayName("KeyValueCache populated on A is observed on B; evict on A propagates")
+    void apiKeyCacheVisibleCrossNode() {
+        sharedCache.put("apikey", "hash-bob", "bob", Duration.ofSeconds(60));
+        assertEquals("bob", sharedCache.get("apikey", "hash-bob").orElse(null));
+        sharedCache.evict("apikey", "hash-bob");
+        assertFalse(sharedCache.get("apikey", "hash-bob").isPresent());
+    }
+
+    @Test
+    @DisplayName("backplaneType reports 'valkey' on every node; localNodeId is distinct")
+    void backplaneType() {
+        assertEquals("valkey", backplaneA.backplaneType());
+        assertEquals("valkey", backplaneB.backplaneType());
+        assertEquals("node-A", backplaneA.localNodeId());
+        assertEquals("node-B", backplaneB.localNodeId());
+        assertNotEquals(backplaneA.localNodeId(), backplaneB.localNodeId());
+        assertNotNull(backplaneA.localNodeId());
+    }
+
+    private ClusterBackplane constBackplane(String nodeId, String type) {
+        return new ClusterBackplane() {
+            @Override
+            public boolean isHealthy() {
+                return true;
+            }
+
+            @Override
+            public String backplaneType() {
+                return type;
+            }
+
+            @Override
+            public String localNodeId() {
+                return nodeId;
+            }
+        };
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveExternalClusterTest.java

@@ -0,0 +1,237 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.core.StringRedisTemplate;
+
+import stirling.software.common.cluster.ClusterNode;
+import stirling.software.common.cluster.DistributedLock;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
+import stirling.software.common.model.ApplicationProperties;
+
+/**
+ * Opt-in live cluster test against an EXTERNAL Valkey/Redis given by {@code
+ * STIRLING_TEST_VALKEY_URL} (e.g. a managed {@code rediss://} endpoint). Unlike {@link
+ * LiveValkeyIntegrationTest} (no-auth local container) this drives three independent node stacks
+ * through the production {@link ValkeyConnectionConfiguration#valkeyConnectionFactory()} bean - so
+ * a {@code rediss://} URL exercises the real TLS handshake (verifyPeer=FULL) and credential path
+ * end to end.
+ *
+ * <p>Skips unless the env var is set, so it never runs in normal CI. No secrets are committed.
+ */
+@EnabledIfEnvironmentVariable(named = "STIRLING_TEST_VALKEY_URL", matches = "rediss?://.+")
+class LiveExternalClusterTest {
+
+    private static final int NODES = 3;
+    private static final String RUN = UUID.randomUUID().toString().substring(0, 8);
+
+    private static final List<LettuceConnectionFactory> factories = new ArrayList<>();
+    private static final List<StringRedisTemplate> templates = new ArrayList<>();
+
+    @BeforeAll
+    static void connectAllNodes() {
+        String url = System.getenv("STIRLING_TEST_VALKEY_URL");
+        for (int i = 0; i < NODES; i++) {
+            ApplicationProperties p = new ApplicationProperties();
+            p.getCluster().setEnabled(true);
+            p.getCluster().setBackplane("valkey");
+            p.getCluster().getValkey().setUrl(url);
+            p.getCluster().getNode().setId(nodeId(i));
+            // Production bean: parse + credentials + TLS + eager PING handshake (proves reachable).
+            LettuceConnectionFactory f =
+                    new ValkeyConnectionConfiguration(p).valkeyConnectionFactory();
+            factories.add(f);
+            templates.add(new StringRedisTemplate(f));
+        }
+    }
+
+    @AfterAll
+    static void disconnect() {
+        for (LettuceConnectionFactory f : factories) {
+            try {
+                f.destroy();
+            } catch (RuntimeException ignored) {
+                // best effort
+            }
+        }
+    }
+
+    private static String nodeId(int i) {
+        return "ext-" + RUN + "-node-" + (i + 1);
+    }
+
+    private ApplicationProperties propsForNode(int i) {
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().setEnabled(true);
+        p.getCluster().setBackplane("valkey");
+        p.getCluster().getNode().setId(nodeId(i));
+        return p;
+    }
+
+    @Test
+    @DisplayName("TLS reachability: every node's backplane reports healthy over the external URL")
+    void allNodesHealthyOverTls() {
+        for (int i = 0; i < NODES; i++) {
+            ValkeyClusterBackplane bp =
+                    new ValkeyClusterBackplane(propsForNode(i), templates.get(i));
+            assertTrue(bp.isHealthy(), nodeId(i) + " must reach the external Valkey (PING)");
+            assertEquals(nodeId(i), bp.localNodeId());
+        }
+    }
+
+    @Test
+    @DisplayName("3-node registry: each node sees all peers; deregister drops one cluster-wide")
+    void threeNodeRegistryConverges() {
+        List<ValkeyInstanceRegistry> regs = new ArrayList<>();
+        for (int i = 0; i < NODES; i++) {
+            regs.add(new ValkeyInstanceRegistry(templates.get(i)));
+        }
+        for (int i = 0; i < NODES; i++) {
+            regs.get(i)
+                    .register(
+                            new ClusterNode(
+                                    nodeId(i), "10.0.0." + i + ":8080", Instant.now(), "BOTH"),
+                            Duration.ofSeconds(30));
+        }
+        try {
+            // Node 0's view must include all three registrations made on three connections.
+            for (int i = 0; i < NODES; i++) {
+                final String id = nodeId(i);
+                boolean seenByNode0 =
+                        regs.get(0).activeNodes().stream().anyMatch(n -> id.equals(n.nodeId()));
+                assertTrue(seenByNode0, "node-0 must see " + id + " registered by another node");
+            }
+            regs.get(1).deregister(nodeId(2));
+            assertFalse(
+                    regs.get(0).lookup(nodeId(2)).isPresent(),
+                    "deregister on node-1 must be visible from node-0");
+        } finally {
+            regs.get(0).deregister(nodeId(0));
+            regs.get(1).deregister(nodeId(1));
+        }
+    }
+
+    @Test
+    @DisplayName(
+            "Cross-node JobStore: put on node-0 visible on node-1/2; delete on node-2 clears it")
+    void jobStoreVisibleAcrossNodes() {
+        ValkeyJobStore s0 = new ValkeyJobStore(templates.get(0));
+        ValkeyJobStore s1 = new ValkeyJobStore(templates.get(1));
+        ValkeyJobStore s2 = new ValkeyJobStore(templates.get(2));
+        String jobId = "ext-job-" + RUN;
+        String fileId = "ext-file-" + RUN;
+
+        s0.put(
+                new JobStoreEntry(
+                        jobId,
+                        JobStoreEntry.JobState.RUNNING,
+                        nodeId(0),
+                        Instant.now(),
+                        null,
+                        null,
+                        List.of(fileId),
+                        Map.of("k", "v")),
+                Duration.ofSeconds(60));
+
+        Optional<JobStoreEntry> onNode1 = s1.get(jobId);
+        assertTrue(onNode1.isPresent(), "node-1 must see node-0's job write");
+        assertEquals(nodeId(0), onNode1.get().owningNodeId());
+        assertEquals(jobId, s2.findJobIdByFileId(fileId).orElse(null), "reverse index visible too");
+
+        s2.delete(jobId);
+        assertFalse(s0.exists(jobId), "delete on node-2 must clear the hash for node-0");
+        assertFalse(
+                s1.findJobIdByFileId(fileId).isPresent(),
+                "delete on node-2 must clear the reverse index for node-1");
+    }
+
+    @Test
+    @DisplayName("Global rate limit: ONE shared budget enforced across all three nodes")
+    void rateLimitGlobalAcrossThreeNodes() {
+        List<ValkeyRateLimitStore> stores = new ArrayList<>();
+        for (int i = 0; i < NODES; i++) {
+            ValkeyRateLimitStore s = new ValkeyRateLimitStore(factories.get(i));
+            s.initProxyManager();
+            stores.add(s);
+        }
+        String key = "ext-rl-" + RUN;
+        long capacity = 6;
+        AtomicInteger allowed = new AtomicInteger();
+        for (int i = 0; i < 12; i++) {
+            RateLimitDecision d =
+                    stores.get(i % NODES).tryConsume(key, capacity, Duration.ofSeconds(60));
+            if (d.allowed()) {
+                allowed.incrementAndGet();
+            }
+        }
+        assertEquals(
+                capacity,
+                allowed.get(),
+                "exactly the global capacity must be allowed across all three nodes");
+    }
+
+    @Test
+    @DisplayName("Distributed lock: held by node-0 excludes node-1/2; stale release cannot steal")
+    void distributedLockAcrossNodes() throws InterruptedException {
+        ValkeyDistributedLock l0 = new ValkeyDistributedLock(templates.get(0));
+        ValkeyDistributedLock l1 = new ValkeyDistributedLock(templates.get(1));
+        ValkeyDistributedLock l2 = new ValkeyDistributedLock(templates.get(2));
+        String key = "ext-lock-" + RUN;
+
+        Optional<DistributedLock.LockHandle> a = l0.tryAcquire(key, Duration.ofSeconds(30));
+        assertTrue(a.isPresent());
+        assertFalse(l1.tryAcquire(key, Duration.ofSeconds(30)).isPresent(), "node-1 excluded");
+        assertFalse(l2.tryAcquire(key, Duration.ofSeconds(30)).isPresent(), "node-2 excluded");
+        a.get().release();
+        Optional<DistributedLock.LockHandle> b = l1.tryAcquire(key, Duration.ofSeconds(30));
+        assertTrue(b.isPresent(), "node-1 acquires after node-0 releases");
+
+        // Short lease that expires, then node-2 takes it; node-1's stale release must not steal.
+        String key2 = "ext-lock2-" + RUN;
+        Optional<DistributedLock.LockHandle> shortHeld =
+                l1.tryAcquire(key2, Duration.ofMillis(500));
+        assertTrue(shortHeld.isPresent());
+        Thread.sleep(900);
+        Optional<DistributedLock.LockHandle> stolen = l2.tryAcquire(key2, Duration.ofSeconds(30));
+        assertTrue(stolen.isPresent(), "node-2 acquires after node-1's lease expires");
+        shortHeld.get().release(); // value-checked no-op
+        assertFalse(
+                l0.tryAcquire(key2, Duration.ofMillis(200)).isPresent(),
+                "node-2 still holds; node-1's stale release must not have stolen it");
+
+        b.get().release();
+        stolen.get().release();
+    }
+
+    @Test
+    @DisplayName("KeyValueCache: put on node-0 readable on node-1; evict visible on node-2")
+    void keyValueCacheAcrossNodes() {
+        ValkeyKeyValueCache c0 = new ValkeyKeyValueCache(templates.get(0));
+        ValkeyKeyValueCache c1 = new ValkeyKeyValueCache(templates.get(1));
+        ValkeyKeyValueCache c2 = new ValkeyKeyValueCache(templates.get(2));
+        String field = "ext-cache-" + RUN;
+
+        c0.put("apikey", field, "alice", Duration.ofSeconds(60));
+        assertEquals("alice", c1.get("apikey", field).orElse(null), "node-1 reads node-0's cache");
+        c2.evict("apikey", field);
+        assertFalse(c0.get("apikey", field).isPresent(), "evict on node-2 visible on node-0");
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyAuthIntegrationTest.java

@@ -0,0 +1,143 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIf;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.testcontainers.DockerClientFactory;
+import org.testcontainers.containers.Container;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.DockerImageName;
+
+import stirling.software.common.model.ApplicationProperties;
+
+/**
+ * Live AUTH coverage against a password-protected Valkey. The main {@link
+ * LiveValkeyIntegrationTest} runs against a no-auth instance, so the credential-bearing URL path
+ * (parse -> RedisStandaloneConfiguration -> real AUTH handshake) was otherwise unexercised. Drives
+ * the full production bean method {@code valkeyConnectionFactory()} so the parse, credential
+ * wiring, and eager-handshake all run exactly as at boot.
+ */
+@Testcontainers
+@EnabledIf("isDockerAvailable")
+class LiveValkeyAuthIntegrationTest {
+
+    private static final String PASSWORD = "s3cr3tpw";
+
+    // @Container is intentionally NOT used: lifecycle is managed manually so the container can be
+    // shared across the password-only and ACL-user cases without a per-method restart.
+    static final GenericContainer<?> VALKEY =
+            new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine"))
+                    .withExposedPorts(6379)
+                    .withCommand("valkey-server", "--requirepass", PASSWORD);
+
+    static boolean isDockerAvailable() {
+        return DockerClientFactory.instance().isDockerAvailable();
+    }
+
+    @org.junit.jupiter.api.BeforeAll
+    static void start() {
+        VALKEY.start();
+    }
+
+    @org.junit.jupiter.api.AfterAll
+    static void stop() {
+        VALKEY.stop();
+    }
+
+    private static String host() {
+        return VALKEY.getHost();
+    }
+
+    private static int port() {
+        return VALKEY.getMappedPort(6379);
+    }
+
+    private ApplicationProperties propsWithUrl(String url) {
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().setEnabled(true);
+        p.getCluster().setBackplane("valkey");
+        p.getCluster().getValkey().setUrl(url);
+        p.getCluster().getNode().setId("auth-test");
+        return p;
+    }
+
+    @Test
+    @DisplayName("password-only URL (redis://:pw@host) authenticates and round-trips a key")
+    void passwordOnlyAuthWorks() {
+        String url = "redis://:" + PASSWORD + "@" + host() + ":" + port();
+        LettuceConnectionFactory factory =
+                new ValkeyConnectionConfiguration(propsWithUrl(url)).valkeyConnectionFactory();
+        try {
+            StringRedisTemplate t = new StringRedisTemplate(factory);
+            t.opsForValue().set("auth:pwonly", "v1");
+            assertEquals(
+                    "v1",
+                    t.opsForValue().get("auth:pwonly"),
+                    "password-only AUTH must succeed and the key must round-trip");
+        } finally {
+            factory.destroy();
+        }
+    }
+
+    @Test
+    @DisplayName("user:password URL (ACL named user) authenticates and round-trips a key")
+    void namedUserAuthWorks() throws Exception {
+        // Default user is password-protected; create a named ACL user to exercise two-arg AUTH.
+        Container.ExecResult res =
+                VALKEY.execInContainer(
+                        "valkey-cli",
+                        "-a",
+                        PASSWORD,
+                        "ACL",
+                        "SETUSER",
+                        "alice",
+                        "on",
+                        ">alicepass",
+                        "~*",
+                        "+@all");
+        assertEquals(0, res.getExitCode(), "ACL SETUSER failed: " + res.getStderr());
+
+        String url = "redis://alice:alicepass@" + host() + ":" + port();
+        LettuceConnectionFactory factory =
+                new ValkeyConnectionConfiguration(propsWithUrl(url)).valkeyConnectionFactory();
+        try {
+            StringRedisTemplate t = new StringRedisTemplate(factory);
+            t.opsForValue().set("auth:named", "v2");
+            assertEquals(
+                    "v2",
+                    t.opsForValue().get("auth:named"),
+                    "named-user AUTH must succeed and the key must round-trip");
+        } finally {
+            factory.destroy();
+        }
+    }
+
+    @Test
+    @DisplayName(
+            "wrong password → fast IllegalStateException (real WRONGPASS short-circuits retries)")
+    void wrongPasswordFailsFast() {
+        String url = "redis://:wrong-" + PASSWORD + "@" + host() + ":" + port();
+        ValkeyConnectionConfiguration cfg = new ValkeyConnectionConfiguration(propsWithUrl(url));
+
+        long start = System.nanoTime();
+        IllegalStateException ex =
+                assertThrows(IllegalStateException.class, cfg::valkeyConnectionFactory);
+        long elapsedMs = (System.nanoTime() - start) / 1_000_000;
+
+        assertTrue(
+                ex.getMessage().toLowerCase().contains("authentication failed"),
+                "real Valkey WRONGPASS must be classified as an auth failure; got: "
+                        + ex.getMessage());
+        // The retry loop is 10 x 3s; a recognised auth failure must abort well before that.
+        assertTrue(
+                elapsedMs < 10_000,
+                "auth failure must short-circuit the 30s retry loop; elapsed=" + elapsedMs + " ms");
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyChaosTest.java

@@ -0,0 +1,118 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIf;
+import org.springframework.dao.DataAccessException;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.testcontainers.DockerClientFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.DockerImageName;
+
+import stirling.software.common.model.ApplicationProperties;
+
+/**
+ * Live failure-injection: a frozen (network-black-holed) Valkey must NOT stall hot-path commands
+ * for Lettuce's 60s default. {@link ValkeyConnectionConfiguration} pins a 2s command timeout, so a
+ * paused server must surface an error in seconds, and the connection must recover when it returns.
+ *
+ * <p>Uses {@code docker pause}/{@code unpause} (TCP stays ESTABLISHED but the server never replies)
+ * to reproduce a partition rather than {@code stop} (which would fail fast with
+ * connection-refused).
+ */
+@Testcontainers
+@EnabledIf("isDockerAvailable")
+class LiveValkeyChaosTest {
+
+    @Container
+    static final GenericContainer<?> VALKEY =
+            new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine"))
+                    .withExposedPorts(6379);
+
+    static boolean isDockerAvailable() {
+        return DockerClientFactory.instance().isDockerAvailable();
+    }
+
+    private boolean paused;
+
+    private LettuceConnectionFactory productionFactory() {
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().setEnabled(true);
+        p.getCluster().setBackplane("valkey");
+        p.getCluster()
+                .getValkey()
+                .setUrl("redis://" + VALKEY.getHost() + ":" + VALKEY.getMappedPort(6379));
+        p.getCluster().getNode().setId("chaos");
+        // Built via the production bean so the real 2s commandTimeout is in effect.
+        return new ValkeyConnectionConfiguration(p).valkeyConnectionFactory();
+    }
+
+    private void pause() {
+        DockerClientFactory.lazyClient().pauseContainerCmd(VALKEY.getContainerId()).exec();
+        paused = true;
+    }
+
+    private void unpause() {
+        DockerClientFactory.lazyClient().unpauseContainerCmd(VALKEY.getContainerId()).exec();
+        paused = false;
+    }
+
+    @AfterEach
+    void ensureUnpaused() {
+        if (paused) {
+            try {
+                unpause();
+            } catch (RuntimeException ignored) {
+                // container teardown will handle it
+            }
+        }
+    }
+
+    @Test
+    @DisplayName("frozen Valkey fails a command in seconds (2s timeout), not Lettuce's 60s default")
+    void commandTimeoutFiresUnderPartition() {
+        LettuceConnectionFactory factory = productionFactory();
+        try {
+            StringRedisTemplate t = new StringRedisTemplate(factory);
+            t.opsForValue().set("chaos:k", "before");
+            assertEquals("before", t.opsForValue().get("chaos:k"));
+
+            pause();
+            long start = System.nanoTime();
+            // DataAccessException is spring-data-redis's wrapper for the Lettuce timeout.
+            assertThrows(DataAccessException.class, () -> t.opsForValue().get("chaos:k"));
+            long elapsedMs = (System.nanoTime() - start) / 1_000_000;
+            assertTrue(
+                    elapsedMs < 15_000,
+                    "command must abort on the ~2s timeout, not hang on the 60s default; elapsed="
+                            + elapsedMs
+                            + " ms");
+
+            unpause();
+            // After the partition heals the client must recover (Lettuce reconnects lazily).
+            String recovered = null;
+            long deadline = System.currentTimeMillis() + 10_000;
+            while (System.currentTimeMillis() < deadline) {
+                try {
+                    recovered = t.opsForValue().get("chaos:k");
+                    break;
+                } catch (RuntimeException retry) {
+                    Thread.sleep(250);
+                }
+            }
+            assertEquals("before", recovered, "connection must recover after the partition heals");
+        } catch (InterruptedException ie) {
+            Thread.currentThread().interrupt();
+        } finally {
+            factory.destroy();
+        }
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/LiveValkeyIntegrationTest.java

@@ -0,0 +1,519 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIf;
+import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.testcontainers.DockerClientFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.DockerImageName;
+
+import stirling.software.common.cluster.ClusterNode;
+import stirling.software.common.cluster.DistributedLock;
+import stirling.software.common.cluster.JobStoreEntry;
+import stirling.software.common.cluster.RateLimitStore.RateLimitDecision;
+import stirling.software.common.model.ApplicationProperties;
+
+/**
+ * Live integration tests against a real Valkey instance, started by Testcontainers. The
+ * {@code @EnabledIf} guard probes the Docker daemon via {@link
+ * DockerClientFactory#isDockerAvailable()} (non-throwing) so the suite skips cleanly when Docker is
+ * unavailable - without that guard, {@code @Testcontainers} would throw {@code initializationError}
+ * (test FAILURE, not skip) on CI runners without Docker.
+ */
+@Testcontainers
+@EnabledIf("isDockerAvailable")
+class LiveValkeyIntegrationTest {
+
+    @Container
+    static final GenericContainer<?> VALKEY =
+            new GenericContainer<>(DockerImageName.parse("valkey/valkey:8.0-alpine"))
+                    .withExposedPorts(6379);
+
+    static boolean isDockerAvailable() {
+        return DockerClientFactory.instance().isDockerAvailable();
+    }
+
+    private static LettuceConnectionFactory factoryA;
+    private static LettuceConnectionFactory factoryB;
+    private static StringRedisTemplate templateA;
+    private static StringRedisTemplate templateB;
+
+    @BeforeAll
+    static void connect() {
+        String host = VALKEY.getHost();
+        int port = VALKEY.getMappedPort(6379);
+        factoryA = new LettuceConnectionFactory(new RedisStandaloneConfiguration(host, port));
+        factoryA.afterPropertiesSet();
+        factoryB = new LettuceConnectionFactory(new RedisStandaloneConfiguration(host, port));
+        factoryB.afterPropertiesSet();
+        templateA = new StringRedisTemplate(factoryA);
+        templateB = new StringRedisTemplate(factoryB);
+        templateA.getConnectionFactory().getConnection().serverCommands().flushAll();
+    }
+
+    @AfterAll
+    static void disconnect() {
+        if (factoryA != null) factoryA.destroy();
+        if (factoryB != null) factoryB.destroy();
+    }
+
+    @Test
+    @DisplayName("Valkey reachable and isHealthy() = true after PING round-trip")
+    void backplaneHealthy() {
+        ApplicationProperties propsA = newProps("node-A");
+        ValkeyClusterBackplane bp = new ValkeyClusterBackplane(propsA, templateA);
+        assertEquals("valkey", bp.backplaneType());
+        assertEquals("node-A", bp.localNodeId());
+        assertTrue(bp.isHealthy(), "Valkey must be reachable in the Testcontainers instance");
+    }
+
+    @Test
+    @DisplayName("JobStore put on connection A, get on connection B reads the same entry")
+    void jobStoreCrossConnectionVisibility() {
+        ValkeyJobStore storeA = new ValkeyJobStore(templateA);
+        ValkeyJobStore storeB = new ValkeyJobStore(templateB);
+
+        JobStoreEntry entry =
+                new JobStoreEntry(
+                        "live-job-1",
+                        JobStoreEntry.JobState.RUNNING,
+                        "node-A",
+                        Instant.now(),
+                        null,
+                        null,
+                        List.of("live-file-1"),
+                        Map.of("k", "v"));
+        storeA.put(entry, Duration.ofSeconds(30));
+
+        Optional<JobStoreEntry> seen = storeB.get("live-job-1");
+        assertTrue(seen.isPresent(), "storeB on different connection must see storeA's write");
+        assertEquals("node-A", seen.get().owningNodeId());
+        assertEquals(JobStoreEntry.JobState.RUNNING, seen.get().state());
+        assertEquals("live-job-1", storeB.findJobIdByFileId("live-file-1").orElse(null));
+    }
+
+    @Test
+    @DisplayName("JobStore entry expires after the configured duration")
+    void jobStoreTtlExpires() throws InterruptedException {
+        ValkeyJobStore store = new ValkeyJobStore(templateA);
+        store.put(
+                new JobStoreEntry(
+                        "ttl-job",
+                        JobStoreEntry.JobState.PENDING,
+                        "node-A",
+                        Instant.now(),
+                        null,
+                        null,
+                        List.of(),
+                        Map.of()),
+                Duration.ofSeconds(2));
+        assertTrue(store.exists("ttl-job"));
+        long deadline = System.currentTimeMillis() + 3000;
+        boolean expired = false;
+        while (System.currentTimeMillis() < deadline) {
+            if (!store.exists("ttl-job")) {
+                expired = true;
+                break;
+            }
+            Thread.sleep(100);
+        }
+        assertTrue(expired, "entry should TTL-expire within 3 s of a 2 s TTL");
+    }
+
+    @Test
+    @DisplayName("KeyValueCache propagates across connections; evict observed cross-connection")
+    void keyValueCacheCrossConnection() {
+        ValkeyKeyValueCache cacheA = new ValkeyKeyValueCache(templateA);
+        ValkeyKeyValueCache cacheB = new ValkeyKeyValueCache(templateB);
+
+        cacheA.put("apikey", "hash-bob", "bob", Duration.ofSeconds(30));
+        assertEquals("bob", cacheB.get("apikey", "hash-bob").orElse(null));
+
+        cacheA.evict("apikey", "hash-bob");
+        assertFalse(cacheB.get("apikey", "hash-bob").isPresent());
+    }
+
+    @Test
+    @DisplayName("RateLimitStore enforces ONE global budget across two instances")
+    void rateLimitGlobalAcrossInstances() {
+        ValkeyRateLimitStore storeA = newRateLimitStore(factoryA);
+        ValkeyRateLimitStore storeB = newRateLimitStore(factoryB);
+        String key = "live-user:alice";
+        long capacity = 4;
+
+        AtomicInteger allowed = new AtomicInteger();
+        for (int i = 0; i < 8; i++) {
+            // alternate consumers
+            var store = (i % 2 == 0) ? storeA : storeB;
+            RateLimitDecision d = store.tryConsume(key, capacity, Duration.ofSeconds(30));
+            if (d.allowed()) allowed.incrementAndGet();
+        }
+        assertEquals(
+                4,
+                allowed.get(),
+                "exactly 4 (the global capacity) must be allowed across both instances");
+    }
+
+    @Test
+    @DisplayName("DistributedLock excludes a second acquirer on a different connection")
+    void distributedLockMutualExclusion() {
+        ValkeyDistributedLock lockA = new ValkeyDistributedLock(templateA);
+        ValkeyDistributedLock lockB = new ValkeyDistributedLock(templateB);
+
+        Optional<DistributedLock.LockHandle> heldByA =
+                lockA.tryAcquire("election-X", Duration.ofSeconds(30));
+        assertTrue(heldByA.isPresent());
+
+        Optional<DistributedLock.LockHandle> heldByB =
+                lockB.tryAcquire("election-X", Duration.ofSeconds(30));
+        assertFalse(heldByB.isPresent(), "second acquirer must fail while A holds the lock");
+
+        heldByA.get().release();
+
+        // After release, B can acquire
+        Optional<DistributedLock.LockHandle> retry =
+                lockB.tryAcquire("election-X", Duration.ofSeconds(30));
+        assertTrue(retry.isPresent());
+        retry.get().release();
+    }
+
+    @Test
+    @DisplayName("renew() extends the lease TTL well beyond the original")
+    void distributedLockRenewExtendsLease() {
+        ValkeyDistributedLock lock = new ValkeyDistributedLock(templateA);
+        String key = "renew-" + java.util.UUID.randomUUID();
+        Optional<DistributedLock.LockHandle> held = lock.tryAcquire(key, Duration.ofSeconds(1));
+        assertTrue(held.isPresent());
+
+        assertTrue(held.get().renew(Duration.ofSeconds(30)), "renew on a held lock must succeed");
+        Long ttlMs =
+                templateA.getExpire(
+                        "stirling:lock:" + key, java.util.concurrent.TimeUnit.MILLISECONDS);
+        assertNotNull(ttlMs);
+        assertTrue(
+                ttlMs > 1500 && ttlMs <= 30_000,
+                "renew must reset TTL to the new 30s lease, got " + ttlMs + " ms");
+        held.get().release();
+    }
+
+    @Test
+    @DisplayName("value-check prevents a stale owner from releasing/renewing a re-acquired lock")
+    void distributedLockStealPrevention() throws InterruptedException {
+        ValkeyDistributedLock lockA = new ValkeyDistributedLock(templateA);
+        ValkeyDistributedLock lockB = new ValkeyDistributedLock(templateB);
+        String key = "steal-" + java.util.UUID.randomUUID();
+
+        Optional<DistributedLock.LockHandle> a = lockA.tryAcquire(key, Duration.ofMillis(500));
+        assertTrue(a.isPresent());
+
+        // Let A's 500ms lease TTL-expire so Valkey drops the key, then B takes a fresh lock.
+        Thread.sleep(800);
+        Optional<DistributedLock.LockHandle> b = lockB.tryAcquire(key, Duration.ofSeconds(30));
+        assertTrue(b.isPresent(), "B must acquire after A's lease expired");
+
+        // A's stale handle (different UUID value) must touch neither B's renew nor B's key.
+        assertFalse(
+                a.get().renew(Duration.ofSeconds(30)),
+                "stale owner must not be able to renew a lock now owned by B");
+        a.get().release(); // value-checked DEL: must be a no-op, must NOT delete B's key
+
+        assertFalse(
+                lockA.tryAcquire(key, Duration.ofMillis(100)).isPresent(),
+                "B must still hold the lock; A's stale release must not have stolen it");
+        b.get().release();
+    }
+
+    @Test
+    @DisplayName("register is atomic (hash + TTL committed together, no orphan keys on crash)")
+    void registryRegisterIsAtomic() {
+        ValkeyInstanceRegistry reg = new ValkeyInstanceRegistry(templateA);
+        ClusterNode node =
+                new ClusterNode(
+                        "atomic-node-" + java.util.UUID.randomUUID(),
+                        "10.0.0.99:8080",
+                        Instant.now(),
+                        "BOTH");
+        reg.register(node, Duration.ofSeconds(30));
+
+        // TTL must be positive; -1 would mean EXPIRE did not commit inside MULTI/EXEC.
+        Long ttlMs =
+                templateA.getExpire(
+                        "stirling:nodes:" + node.nodeId(),
+                        java.util.concurrent.TimeUnit.MILLISECONDS);
+        assertNotNull(ttlMs);
+        assertTrue(
+                ttlMs > 0 && ttlMs <= 30_000,
+                "register() must atomically arm TTL; expected (0, 30000] ms, got " + ttlMs);
+
+        Optional<ClusterNode> seen = reg.lookup(node.nodeId());
+        assertTrue(seen.isPresent(), "hash fields must be visible after atomic register()");
+        assertEquals("10.0.0.99:8080", seen.get().internalAddress());
+
+        reg.deregister(node.nodeId());
+    }
+
+    @Test
+    @DisplayName("register on connection A is visible from connection B")
+    void registryCrossConnection() {
+        ValkeyInstanceRegistry regA = new ValkeyInstanceRegistry(templateA);
+        ValkeyInstanceRegistry regB = new ValkeyInstanceRegistry(templateB);
+
+        ClusterNode node = new ClusterNode("live-node-7", "10.0.0.7:8080", Instant.now(), "BOTH");
+        regA.register(node, Duration.ofSeconds(30));
+
+        Optional<ClusterNode> seen = regB.lookup("live-node-7");
+        assertTrue(seen.isPresent());
+        assertEquals("10.0.0.7:8080", seen.get().internalAddress());
+
+        boolean inActive =
+                regB.activeNodes().stream().anyMatch(n -> "live-node-7".equals(n.nodeId()));
+        assertTrue(inActive);
+
+        regA.deregister("live-node-7");
+        assertFalse(regB.lookup("live-node-7").isPresent());
+    }
+
+    @Test
+    @DisplayName("Bucket4j: no fixed-window boundary doubling (parity with in-process semantics)")
+    void rateLimitNoBoundaryDoubling() throws InterruptedException {
+        ValkeyRateLimitStore store = newRateLimitStore(factoryA);
+        String key = "boundary-" + java.util.UUID.randomUUID();
+        long capacity = 5;
+        Duration window = Duration.ofMillis(500);
+
+        int firstAllowed = 0;
+        for (int i = 0; i < 10; i++) {
+            if (store.tryConsume(key, capacity, window).allowed()) firstAllowed++;
+        }
+        assertEquals(capacity, firstAllowed, "must allow exactly capacity tokens initially");
+
+        Thread.sleep(window.toMillis() + 50);
+        int secondAllowed = 0;
+        long start = System.nanoTime();
+        for (int i = 0; i < 20 && (System.nanoTime() - start) < 20_000_000L; i++) {
+            if (store.tryConsume(key, capacity, window).allowed()) secondAllowed++;
+        }
+        assertTrue(
+                secondAllowed <= capacity,
+                "token-bucket must not let a fresh full capacity be consumed instantly across"
+                        + " the boundary; got "
+                        + secondAllowed);
+    }
+
+    private ValkeyRateLimitStore newRateLimitStore(LettuceConnectionFactory factory) {
+        ValkeyRateLimitStore store = new ValkeyRateLimitStore(factory);
+        store.initProxyManager();
+        return store;
+    }
+
+    @Test
+    @DisplayName("JobStore put is atomic (hash + TTL + reverse index visible together)")
+    void jobStorePutIsAtomic() {
+        ValkeyJobStore store = new ValkeyJobStore(templateA);
+        String jobId = "atomic-job-" + java.util.UUID.randomUUID();
+        String fileId = "atomic-file-" + java.util.UUID.randomUUID();
+        store.put(
+                new JobStoreEntry(
+                        jobId,
+                        JobStoreEntry.JobState.PENDING,
+                        "node-A",
+                        Instant.now(),
+                        null,
+                        null,
+                        List.of(fileId),
+                        Map.of("k", "v")),
+                Duration.ofSeconds(30));
+
+        assertTrue(store.exists(jobId), "hash must be visible after put");
+        Long jobTtl =
+                templateA.getExpire(
+                        "stirling:job:" + jobId, java.util.concurrent.TimeUnit.MILLISECONDS);
+        assertNotNull(jobTtl);
+        assertTrue(jobTtl > 0, "hash must have TTL armed inside the same transaction");
+        assertEquals(jobId, store.findJobIdByFileId(fileId).orElse(null));
+        Long indexTtl =
+                templateA.getExpire(
+                        "stirling:file2job:" + fileId, java.util.concurrent.TimeUnit.MILLISECONDS);
+        assertNotNull(indexTtl);
+        assertTrue(indexTtl > 0, "reverse index must also have TTL armed");
+    }
+
+    @Test
+    @DisplayName(
+            "JobStore.delete(): WATCH aborts when put() races between read and EXEC, no orphaned"
+                    + " reverse-index entries")
+    void jobStoreDeleteWatchRaceRetriesAndCleansUp() {
+        ValkeyJobStore store = new ValkeyJobStore(templateA);
+        String jobId = "watch-race-job-" + java.util.UUID.randomUUID();
+        String originalFile = "orig-file-" + java.util.UUID.randomUUID();
+        String newFile = "new-file-" + java.util.UUID.randomUUID();
+
+        store.put(
+                new JobStoreEntry(
+                        jobId,
+                        JobStoreEntry.JobState.RUNNING,
+                        "node-A",
+                        Instant.now(),
+                        null,
+                        null,
+                        List.of(originalFile),
+                        Map.of()),
+                Duration.ofSeconds(30));
+
+        // Simulate the race: between delete()'s WATCH read and EXEC, add a new fileId.
+        // The first EXEC aborts; the retry catches the new fileId and deletes both entries.
+        Thread mutator =
+                new Thread(
+                        () -> {
+                            try {
+                                Thread.sleep(20);
+                            } catch (InterruptedException ignored) {
+                                Thread.currentThread().interrupt();
+                            }
+                            store.put(
+                                    new JobStoreEntry(
+                                            jobId,
+                                            JobStoreEntry.JobState.RUNNING,
+                                            "node-A",
+                                            Instant.now(),
+                                            null,
+                                            null,
+                                            List.of(originalFile, newFile),
+                                            Map.of()),
+                                    Duration.ofSeconds(30));
+                        });
+        mutator.start();
+
+        store.delete(jobId);
+        try {
+            mutator.join(2000);
+        } catch (InterruptedException ignored) {
+            Thread.currentThread().interrupt();
+        }
+
+        boolean hashGone = !store.exists(jobId);
+        boolean origIndexGone = !store.findJobIdByFileId(originalFile).isPresent();
+        boolean newIndexGone = !store.findJobIdByFileId(newFile).isPresent();
+        if (hashGone) {
+            assertTrue(
+                    origIndexGone,
+                    "if hash is deleted, original reverse-index entry must also be gone");
+            assertTrue(
+                    newIndexGone,
+                    "if hash is deleted after the racing put(), the WATCH retry must catch the"
+                            + " new fileId and delete its reverse-index entry too");
+        } else {
+            assertEquals(jobId, store.findJobIdByFileId(originalFile).orElse(null));
+            assertEquals(jobId, store.findJobIdByFileId(newFile).orElse(null));
+        }
+    }
+
+    @Test
+    @DisplayName("JobStore.delete() removes hash AND every reverse-index entry atomically")
+    void jobStoreDeleteRemovesReverseIndexEntries() {
+        ValkeyJobStore store = new ValkeyJobStore(templateA);
+        String jobId = "del-atomic-job-" + java.util.UUID.randomUUID();
+        String fileA = "del-atomic-fileA-" + java.util.UUID.randomUUID();
+        String fileB = "del-atomic-fileB-" + java.util.UUID.randomUUID();
+        store.put(
+                new JobStoreEntry(
+                        jobId,
+                        JobStoreEntry.JobState.COMPLETE,
+                        "node-A",
+                        Instant.now(),
+                        Instant.now(),
+                        null,
+                        List.of(fileA, fileB),
+                        Map.of()),
+                Duration.ofSeconds(30));
+        assertTrue(store.exists(jobId));
+        assertEquals(jobId, store.findJobIdByFileId(fileA).orElse(null));
+        assertEquals(jobId, store.findJobIdByFileId(fileB).orElse(null));
+
+        store.delete(jobId);
+
+        // Both the main hash AND every reverse-index entry must be gone; dangling reverse-index
+        // entries would cause findJobIdByFileId() to return a deleted jobId.
+        assertFalse(store.exists(jobId), "main hash must be deleted");
+        assertFalse(
+                store.findJobIdByFileId(fileA).isPresent(),
+                "reverse-index entry for fileA must not survive delete()");
+        assertFalse(
+                store.findJobIdByFileId(fileB).isPresent(),
+                "reverse-index entry for fileB must not survive delete()");
+        assertFalse(
+                Boolean.TRUE.equals(templateA.hasKey("stirling:file2job:" + fileA)),
+                "raw reverse-index key for fileA must not survive delete()");
+        assertFalse(
+                Boolean.TRUE.equals(templateA.hasKey("stirling:file2job:" + fileB)),
+                "raw reverse-index key for fileB must not survive delete()");
+    }
+
+    @Test
+    @DisplayName("JobStore.all() walks the keyspace via SCAN, not KEYS")
+    void jobStoreAllUsesScanNonBlocking() {
+        ValkeyJobStore store = new ValkeyJobStore(templateA);
+        for (int i = 0; i < 15; i++) {
+            store.put(
+                    new JobStoreEntry(
+                            "scan-job-" + i,
+                            JobStoreEntry.JobState.PENDING,
+                            "node-A",
+                            Instant.now(),
+                            null,
+                            null,
+                            List.of(),
+                            Map.of()),
+                    Duration.ofSeconds(30));
+        }
+        long observed = store.all().stream().filter(e -> e.jobId().startsWith("scan-job-")).count();
+        assertTrue(
+                observed >= 15,
+                "SCAN-based all() must surface every inserted job, saw " + observed);
+    }
+
+    @Test
+    @DisplayName("Valkey unreachable yields isHealthy() = false")
+    void unreachableBackplaneReportsUnhealthy() {
+        RedisStandaloneConfiguration cfg = new RedisStandaloneConfiguration("localhost", 16400);
+        LettuceConnectionFactory dead = new LettuceConnectionFactory(cfg);
+        dead.afterPropertiesSet();
+        try {
+            StringRedisTemplate t = new StringRedisTemplate(dead);
+            ValkeyClusterBackplane bp = new ValkeyClusterBackplane(newProps("orphan"), t);
+            assertFalse(bp.isHealthy(), "isHealthy must be false when Valkey is unreachable");
+        } finally {
+            dead.destroy();
+        }
+    }
+
+    private ApplicationProperties newProps(String nodeId) {
+        ApplicationProperties p = new ApplicationProperties();
+        p.getCluster().setEnabled(true);
+        p.getCluster().setBackplane("valkey");
+        p.getCluster()
+                .getValkey()
+                .setUrl("redis://" + VALKEY.getHost() + ":" + VALKEY.getMappedPort(6379));
+        p.getCluster().getNode().setId(nodeId);
+        return p;
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyClusterBackplaneTest.java

@@ -0,0 +1,61 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.data.redis.core.RedisCallback;
+import org.springframework.data.redis.core.StringRedisTemplate;
+
+import stirling.software.common.model.ApplicationProperties;
+
+/**
+ * S3 regression: {@link ValkeyClusterBackplane#isHealthy()} must route through {@code
+ * template.execute(...)} so the borrowed connection is always returned to the pool. Calling {@code
+ * getConnectionFactory().getConnection()} directly would leak the connection on every k8s liveness
+ * probe tick and exhaust the pool under monitoring load.
+ */
+class ValkeyClusterBackplaneTest {
+
+    @Test
+    void isHealthy_routesThroughTemplateExecute_andDoesNotTouchConnectionFactoryDirectly() {
+        StringRedisTemplate template = mock(StringRedisTemplate.class);
+        when(template.execute(any(RedisCallback.class))).thenReturn("PONG");
+
+        ApplicationProperties props = new ApplicationProperties();
+        props.getCluster().getNode().setId("n-1");
+        ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template);
+
+        assertTrue(bp.isHealthy());
+        verify(template, times(1)).execute(any(RedisCallback.class));
+        // Critical: never bypass the template's connection management.
+        verify(template, never()).getConnectionFactory();
+    }
+
+    @Test
+    void isHealthy_returnsFalseWhenExecuteThrows() {
+        StringRedisTemplate template = mock(StringRedisTemplate.class);
+        when(template.execute(any(RedisCallback.class))).thenThrow(new RuntimeException("boom"));
+
+        ApplicationProperties props = new ApplicationProperties();
+        props.getCluster().getNode().setId("n-1");
+        ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template);
+
+        assertFalse(bp.isHealthy());
+    }
+
+    @Test
+    void shouldRunLocalCleanup_returnsFalse_valkeyOwnsTtlEviction() {
+        StringRedisTemplate template = mock(StringRedisTemplate.class);
+        ApplicationProperties props = new ApplicationProperties();
+        props.getCluster().getNode().setId("n-1");
+        ValkeyClusterBackplane bp = new ValkeyClusterBackplane(props, template);
+        assertFalse(bp.shouldRunLocalCleanup());
+    }
+}

app/proprietary/src/test/java/stirling/software/proprietary/cluster/valkey/ValkeyConnectionConfigurationTest.java

@@ -0,0 +1,298 @@
+package stirling.software.proprietary.cluster.valkey;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.atMost;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.springframework.data.redis.RedisSystemException;
+import org.springframework.data.redis.connection.RedisConnection;
+import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
+import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
+
+import io.lettuce.core.RedisCommandExecutionException;
+import io.lettuce.core.SslVerifyMode;
+
+import stirling.software.proprietary.cluster.valkey.ValkeyConnectionConfiguration.Endpoint;
+
+/**
+ * Verifies auth-fast-fail on WRONGPASS/NOAUTH/NOPERM: these are unrecoverable so the handshake must
+ * short-circuit after one attempt, not burn the 30 s retry loop.
+ */
+class ValkeyConnectionConfigurationTest {
+
+    @Test
+    @DisplayName("WRONGPASS surfaces in one attempt (no 30s retry loop)")
+    void wrongpass_failsImmediately_withoutRetries() throws Exception {
+        LettuceConnectionFactory factory = mock(LettuceConnectionFactory.class);
+        RedisConnection conn = mock(RedisConnection.class);
+        when(factory.getConnection()).thenReturn(conn);
+        RedisCommandExecutionException auth =
+                new RedisCommandExecutionException("WRONGPASS invalid username-password pair");
+        when(conn.ping()).thenThrow(new RedisSystemException("Error in execution", auth));
+
+        long start = System.nanoTime();
+        IllegalStateException ex =
+                assertThrows(
+                        IllegalStateException.class,
+                        () ->
+                                ValkeyConnectionConfiguration.eagerHandshake(
+                                        factory, "valkey", 6379, false));
+        long elapsedMs = (System.nanoTime() - start) / 1_000_000;
+
+        verify(factory, times(1)).getConnection();
+        verify(conn, times(1)).ping();
+        assertTrue(
+                elapsedMs < 1500,
+                "Auth failure must short-circuit retries; elapsed=" + elapsedMs + " ms");
+        assertTrue(
+                ex.getMessage().contains("authentication failed"),
+                "Error message must explain the auth failure; got: " + ex.getMessage());
+        verify(factory, atMost(1)).destroy();
+    }
+
+    @Test
+    @DisplayName("NOAUTH surfaces in one attempt")
+    void noauth_failsImmediately() {
+        LettuceConnectionFactory factory = mock(LettuceConnectionFactory.class);
+        RedisConnection conn = mock(RedisConnection.class);
+        when(factory.getConnection()).thenReturn(conn);
+        when(conn.ping())
+                .thenThrow(
+                        new RedisSystemException(
+                                "Error in execution",
+                                new RedisCommandExecutionException(
+                                        "NOAUTH Authentication required.")));
+
+        assertThrows(
+                IllegalStateException.class,
+                () -> ValkeyConnectionConfiguration.eagerHandshake(factory, "v", 6379, false));
+        verify(conn, times(1)).ping();
+    }
+
+    @Test
+    @DisplayName("NOPERM surfaces in one attempt")
+    void noperm_failsImmediately() {
+        LettuceConnectionFactory factory = mock(LettuceConnectionFactory.class);
+        RedisConnection conn = mock(RedisConnection.class);
+        when(factory.getConnection()).thenReturn(conn);
+        when(conn.ping())
+                .thenThrow(
+                        new RedisSystemException(
+                                "Error in execution",
+                                new RedisCommandExecutionException(
+                                        "NOPERM this user has no permissions to run the 'ping'"
+                                                + " command")));
+
+        assertThrows(
+                IllegalStateException.class,
+                () -> ValkeyConnectionConfiguration.eagerHandshake(factory, "v", 6379, false));
+        verify(conn, times(1)).ping();
+    }
+
+    @Test
+    @DisplayName("isAuthFailure - direct RedisCommandExecutionException with auth prefix")
+    void isAuthFailure_directRedisCommandExecutionException() {
+        assertTrue(
+                ValkeyConnectionConfiguration.isAuthFailure(
+                        new RedisCommandExecutionException("WRONGPASS bad password")));
+        assertTrue(
+                ValkeyConnectionConfiguration.isAuthFailure(
+                        new RedisCommandExecutionException("NOAUTH required")));
+        assertTrue(
+                ValkeyConnectionConfiguration.isAuthFailure(
+                        new RedisCommandExecutionException("NOPERM denied")));
+    }
+
+    @Test
+    @DisplayName("isAuthFailure - wrapped inside RedisSystemException (production path)")
+    void isAuthFailure_wrappedBySpring() {
+        assertTrue(
+                ValkeyConnectionConfiguration.isAuthFailure(
+                        new RedisSystemException(
+                                "Error in execution",
+                                new RedisCommandExecutionException("WRONGPASS bad password"))));
+    }
+
+    @Test
+    @DisplayName("isAuthFailure - connection errors do NOT count as auth failures")
+    void isAuthFailure_connectionErrorReturnsFalse() {
+        assertFalse(
+                ValkeyConnectionConfiguration.isAuthFailure(
+                        new RedisSystemException(
+                                "Redis connection failed",
+                                new io.lettuce.core.RedisConnectionException(
+                                        "Connection refused"))));
+        assertFalse(
+                ValkeyConnectionConfiguration.isAuthFailure(
+                        new IllegalStateException("Valkey PING returned 'foo' (expected PONG)")));
+    }
+
+    @Test
+    @DisplayName("bad PONG (protocol error) is not an auth failure")
+    void unexpectedPong_isNotAuthFailure() {
+        assertFalse(
+                ValkeyConnectionConfiguration.isAuthFailure(
+                        new IllegalStateException("Valkey PING returned 'bar' (expected PONG)")));
+    }
+
+    @Test
+    @DisplayName("TLS on, skipCertVerification=false → useSsl + verifyPeer=FULL (default)")
+    void tls_defaultEnforcesFullPeerVerification() {
+        LettuceClientConfiguration cfg =
+                ValkeyConnectionConfiguration.buildClientConfiguration(true, false);
+        assertTrue(cfg.isUseSsl(), "TLS must be enabled");
+        assertSame(SslVerifyMode.FULL, cfg.getVerifyMode());
+        assertTrue(cfg.isVerifyPeer());
+    }
+
+    @Test
+    @DisplayName("TLS on, skipCertVerification=true → verifyPeer=NONE (dev override)")
+    void tls_skipCertVerificationOptOut() {
+        LettuceClientConfiguration cfg =
+                ValkeyConnectionConfiguration.buildClientConfiguration(true, true);
+        assertTrue(cfg.isUseSsl());
+        assertSame(SslVerifyMode.NONE, cfg.getVerifyMode());
+    }
+
+    @Test
+    @DisplayName("TLS off → no SSL, verify flag default (skipCertVerification ignored)")
+    void noTls_ignoresSkipFlag() {
+        LettuceClientConfiguration cfg =
+                ValkeyConnectionConfiguration.buildClientConfiguration(false, true);
+        assertFalse(cfg.isUseSsl());
+    }
+
+    @Nested
+    @DisplayName("parseUrl()")
+    class ParseUrl {
+
+        @Test
+        @DisplayName("host + explicit port, no auth, no TLS")
+        void hostAndPort() {
+            Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://valkey.internal:6380");
+            assertEquals("valkey.internal", e.host());
+            assertEquals(6380, e.port());
+            assertFalse(e.tls());
+            assertNull(e.username());
+            assertNull(e.password());
+        }
+
+        @Test
+        @DisplayName("missing port defaults to 6379")
+        void defaultPort() {
+            assertEquals(6379, ValkeyConnectionConfiguration.parseUrl("redis://host").port());
+        }
+
+        @Test
+        @DisplayName("rediss:// scheme selects TLS")
+        void redissSelectsTls() {
+            assertTrue(ValkeyConnectionConfiguration.parseUrl("rediss://host:6379").tls());
+        }
+
+        @Test
+        @DisplayName("user:password@ sets both credentials")
+        void userAndPassword() {
+            Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://alice:s3cret@host");
+            assertEquals("alice", e.username());
+            assertEquals("s3cret", e.password());
+        }
+
+        @Test
+        @DisplayName("empty user (:pw@) is password-only auth, username stays null")
+        void passwordOnlyEmptyUser() {
+            Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://:s3cret@host");
+            assertNull(e.username(), "empty user must NOT become an empty-string username");
+            assertEquals("s3cret", e.password());
+        }
+
+        @Test
+        @DisplayName("single userinfo token (no colon) is treated as the password")
+        void passwordOnlyNoColon() {
+            Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://s3cret@host");
+            assertNull(e.username());
+            assertEquals("s3cret", e.password());
+        }
+
+        @Test
+        @DisplayName("only the first colon splits user/password (colons allowed in password)")
+        void colonInPassword() {
+            Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://user:pa:ss:word@host");
+            assertEquals("user", e.username());
+            assertEquals("pa:ss:word", e.password());
+        }
+
+        @Test
+        @DisplayName("percent-encoded reserved chars are decoded in the password")
+        void percentEncodedPassword() {
+            // %40 -> '@', %23 -> '#': both must be encoded or URI parses them structurally.
+            Endpoint e = ValkeyConnectionConfiguration.parseUrl("redis://:p%40ss%23word@host");
+            assertNull(e.username());
+            assertEquals("p@ss#word", e.password());
+        }
+
+        @Test
+        @DisplayName("blank url throws with a backplane-config message")
+        void blankUrlThrows() {
+            IllegalStateException ex =
+                    assertThrows(
+                            IllegalStateException.class,
+                            () -> ValkeyConnectionConfiguration.parseUrl("   "));
+            assertTrue(ex.getMessage().contains("cluster.valkey.url must be set"));
+        }
+
+        @Test
+        @DisplayName("null url throws")
+        void nullUrlThrows() {
+            assertThrows(
+                    IllegalStateException.class,
+                    () -> ValkeyConnectionConfiguration.parseUrl(null));
+        }
+
+        @Test
+        @DisplayName("url with no host throws a clear error (scheme-less host:port pitfall)")
+        void noHostThrows() {
+            // "localhost:6379" parses 'localhost' as the scheme, leaving no authority/host.
+            IllegalStateException ex =
+                    assertThrows(
+                            IllegalStateException.class,
+                            () -> ValkeyConnectionConfiguration.parseUrl("localhost:6379"));
+            assertTrue(
+                    ex.getMessage().contains("has no host"),
+                    "message must name the missing host; got: " + ex.getMessage());
+        }
+
+        @Test
+        @DisplayName("non-numeric port yields no host (URI registry-authority fallback)")
+        void nonNumericPortHasNoHost() {
+            // java.net.URI does not throw on a bad port; it falls back to registry authority and
+            // reports host=null, so this must surface as the clear no-host error, not a NPE later.
+            IllegalStateException ex =
+                    assertThrows(
+                            IllegalStateException.class,
+                            () -> ValkeyConnectionConfiguration.parseUrl("redis://host:notaport"));
+            assertTrue(ex.getMessage().contains("has no host"));
+        }
+
+        @Test
+        @DisplayName("syntactically invalid uri throws with the offending url")
+        void invalidUriThrows() {
+            IllegalStateException ex =
+                    assertThrows(
+                            IllegalStateException.class,
+                            () -> ValkeyConnectionConfiguration.parseUrl("redis://ho st:6379"));
+            assertTrue(ex.getMessage().contains("not a valid URI"));
+            assertTrue(ex.getMessage().contains("redis://ho st:6379"));
+        }
+    }
+}